From 6b97b226d74af9d9af5c8555b06caca5839cbdb2 Mon Sep 17 00:00:00 2001 From: Hemant Kumar Date: Tue, 9 Jan 2024 10:20:02 -0500 Subject: [PATCH 1/2] Revert "Merge pull request #97 from jsafrane/rebase-v3.1.1" This reverts commit 15d2569a225b725113bf01e68888900677cabf4e, reversing changes made to ac7f4d3ce3507e820b825a7385fa7c85bc90da91. --- .github/ISSUE_TEMPLATE.md | 28 + .github/PULL_REQUEST_TEMPLATE.md | 22 + .gitlab-ci.yml | 59 +- README.md | 13 +- cmd/syncer/main.go | 126 +- cmd/vsphere-csi/main.go | 34 - docs/book/features/csi_driver_on_windows.md | 165 + docs/book/features/raw_block_volume.md | 63 + docs/book/features/volume_snapshot.md | 210 + docs/book/features/vsphere_csi_migration.md | 250 + .../example-raw-block-restore.yaml | 1 - ...le-sc-WaitForFirstConsumer-restricted.yaml | 4 +- .../example-sc-multiple-zones.yaml | 4 +- .../example-sc-single-zone.yaml | 4 +- hack/release.sh | 4 +- hack/run-e2e-test.sh | 4 - images/ci/Dockerfile | 2 +- images/ci/e2e/Dockerfile | 32 - images/driver/Dockerfile | 2 +- images/syncer/Dockerfile | 2 +- images/windows/driver/Dockerfile | 2 +- manifests/guestcluster/1.22/pvcsi.yaml | 2 +- manifests/guestcluster/1.23/pvcsi.yaml | 4 +- manifests/guestcluster/1.24/pvcsi.yaml | 32 +- manifests/guestcluster/1.25/pvcsi.yaml | 28 +- manifests/guestcluster/1.26/pvcsi.yaml | 28 +- manifests/guestcluster/1.27/pvcsi.yaml | 28 +- manifests/supervisorcluster/1.21/cns-csi.yaml | 10 +- manifests/supervisorcluster/1.22/cns-csi.yaml | 12 +- manifests/supervisorcluster/1.23/cns-csi.yaml | 43 +- .../supervisorcluster/1.23/kustomization.yaml | 5 - manifests/supervisorcluster/1.24/cns-csi.yaml | 43 +- .../supervisorcluster/1.24/kustomization.yaml | 5 - manifests/supervisorcluster/1.25/cns-csi.yaml | 43 +- .../supervisorcluster/1.25/kustomization.yaml | 5 - manifests/supervisorcluster/1.26/cns-csi.yaml | 678 --- .../supervisorcluster/1.26/kustomization.yaml | 5 - manifests/supervisorcluster/1.27/cns-csi.yaml | 678 --- .../supervisorcluster/1.27/kustomization.yaml | 5 - .../csi-snapshot-validatingwebhook.yaml | 2 +- .../vanilla/deploy-csi-snapshot-components.sh | 8 +- .../deploy-vsphere-csi-validation-webhook.sh | 2 +- manifests/vanilla/validatingwebhook.yaml | 6 +- manifests/vanilla/vsphere-csi-driver.yaml | 43 +- pipeline/deploy-staging.sh | 62 +- pipeline/deploy.sh | 99 +- pipeline/dev/patch.yaml | 35 +- pipeline/e2e-tests-staging.sh | 47 +- pipeline/e2e-tests.sh | 94 +- pipeline/patch-prod.sh | 72 - pipeline/release-testbed.sh | 44 - pipeline/send_slack_notification.sh | 54 - pkg/apis/migration/migration.go | 63 +- pkg/common/cns-lib/node/manager.go | 77 +- pkg/common/cns-lib/node/manager_test.go | 50 + pkg/common/cns-lib/node/nodes.go | 84 +- pkg/common/cns-lib/volume/listview.go | 53 +- pkg/common/cns-lib/volume/listview_if.go | 4 +- pkg/common/cns-lib/volume/listview_test.go | 9 +- pkg/common/cns-lib/volume/manager.go | 138 +- pkg/common/cns-lib/volume/util.go | 19 +- .../vsphere/cluster_compute_resource.go | 50 - pkg/common/cns-lib/vsphere/hostsystem.go | 41 +- pkg/common/cns-lib/vsphere/utils.go | 3 - pkg/common/cns-lib/vsphere/virtualcenter.go | 87 +- pkg/common/config/config.go | 30 +- pkg/common/fault/constants.go | 14 - pkg/common/fault/util.go | 48 - pkg/common/unittestcommon/utils.go | 15 - pkg/common/utils/utils.go | 61 +- pkg/common/utils/utils_test.go | 100 + pkg/csi/service/common/authmanager.go | 46 +- .../common/common_controller_helper.go | 17 +- pkg/csi/service/common/commonco/coagnostic.go | 3 - .../k8sorchestrator/k8sorchestrator.go | 9 +- .../commonco/k8sorchestrator/topology.go | 136 +- pkg/csi/service/common/constants.go | 9 +- .../common/placementengine/placement.go | 237 +- pkg/csi/service/common/topology.go | 354 +- pkg/csi/service/common/util_test.go | 15 - pkg/csi/service/common/vsphereutil.go | 36 +- pkg/csi/service/node.go | 17 +- pkg/csi/service/vanilla/controller.go | 163 +- pkg/csi/service/vanilla/controller_test.go | 13 +- pkg/csi/service/wcp/controller.go | 106 +- pkg/csi/service/wcp/controller_helper.go | 12 +- pkg/csi/service/wcp/controller_test.go | 19 +- pkg/csi/service/wcpguest/controller.go | 76 +- pkg/csi/service/wcpguest/controller_helper.go | 37 +- pkg/kubernetes/kubernetes.go | 19 +- .../admissionhandler/admissionhandler.go | 19 +- .../cnscsi_admissionhandler.go | 19 - .../pvcsi_admissionhandler.go | 98 - pkg/syncer/admissionhandler/validatepvc.go | 13 +- .../admissionhandler/validatepvc_test.go | 8 +- .../validatepvcannotationfortkgsha.go | 7 +- .../validatepvcannotationforvolumehealth.go | 6 - ...idatesnapshotoperationsupervisorrequest.go | 41 - .../admissionhandler/validatestorageclass.go | 13 +- .../validatestorageclass_test.go | 6 +- .../cnsnodevmattachment_controller.go | 3 - .../csinodetopology_controller.go | 13 +- .../csinodetopology_controller_test.go | 1 + pkg/syncer/metadatasyncer.go | 5 +- pkg/syncer/util.go | 1 - tests/e2e/OWNERS | 3 +- tests/e2e/config_change_test.go | 8 +- tests/e2e/config_secret.go | 199 +- tests/e2e/config_secret_utils.go | 60 +- tests/e2e/csi_cns_telemetry_statefulsets.go | 7 +- tests/e2e/csi_cns_telemetry_vc_reboot.go | 6 +- tests/e2e/csi_snapshot_basic.go | 4024 +++++------------ tests/e2e/csi_snapshot_file_volume.go | 31 +- tests/e2e/csi_snapshot_negative.go | 95 +- tests/e2e/csi_snapshot_utils.go | 725 --- tests/e2e/csi_static_provisioning_basic.go | 6 +- tests/e2e/data_persistence.go | 6 +- tests/e2e/docs/supervisor_cluster_setup.md | 4 - tests/e2e/e2e_common.go | 96 +- tests/e2e/file_volume_statefulsets.go | 124 +- tests/e2e/fullsync_test_for_block_volume.go | 13 +- tests/e2e/gc_block_resize_retain_policy.go | 1 - tests/e2e/gc_block_volume_expansion.go | 3 +- tests/e2e/gc_cns_nodevm_attachment.go | 1 - tests/e2e/gc_file_share_negative.go | 1 - tests/e2e/gc_full_sync.go | 1 - tests/e2e/gc_metadata_syncer.go | 1 - tests/e2e/gc_rwx_basic.go | 6 - tests/e2e/gc_rwx_deployments.go | 1 - tests/e2e/gc_rwx_destructive.go | 4 +- tests/e2e/gc_rwx_multi_gc.go | 1 - tests/e2e/gc_rwx_multi_ns_gc.go | 1 - tests/e2e/gc_rwx_non_vsan_datastore.go | 6 - tests/e2e/gc_rwx_operation_storm.go | 1 - tests/e2e/gc_rwx_parallel_claim.go | 1 - tests/e2e/gc_rwx_readonly.go | 5 - tests/e2e/gc_rwx_reclaim_policy.go | 1 - tests/e2e/gc_rwx_security_context.go | 1 - tests/e2e/gc_rwx_service_down.go | 1 - tests/e2e/gc_rwx_statefulsets.go | 7 +- tests/e2e/gc_rwx_static_provision.go | 5 - tests/e2e/gc_rwx_syncer.go | 1 - tests/e2e/gc_rwx_tkg_scale.go | 7 +- tests/e2e/gc_rwx_volume_health.go | 2 - tests/e2e/improved_csi_idempotency.go | 41 +- tests/e2e/invalid_topology_values.go | 6 +- tests/e2e/labelupdates.go | 16 +- tests/e2e/multi_master_k8s.go | 7 - tests/e2e/multi_vc.go | 1832 -------- tests/e2e/multi_vc_bootstrap.go | 54 - tests/e2e/multi_vc_config_secret.go | 1058 ----- tests/e2e/multi_vc_connection.go | 101 - tests/e2e/multi_vc_multi_replica.go | 259 -- tests/e2e/multi_vc_preferential_topology.go | 791 ---- tests/e2e/multi_vc_utils.go | 1016 ----- tests/e2e/multi_vc_vsphere.go | 380 -- tests/e2e/nodes_scaleup_scaledown.go | 7 +- tests/e2e/policy_driven_vol_allocation.go | 1947 +------- tests/e2e/preferential_topology.go | 364 +- tests/e2e/preferential_topology_disruptive.go | 200 +- tests/e2e/preferential_topology_snapshot.go | 117 +- tests/e2e/preferential_topology_utils.go | 227 +- tests/e2e/preupgrade_datasetup.go | 2 +- tests/e2e/prevent_duplicate_cluster_ids.go | 949 ---- tests/e2e/provision_with_multiple_zones.go | 14 +- tests/e2e/raw_block_volume.go | 1307 ------ tests/e2e/staging_env_basic.go | 8 +- tests/e2e/statefulset_with_topology.go | 6 - tests/e2e/statefulset_xfs.go | 2 +- tests/e2e/statefulsets.go | 92 +- tests/e2e/storage_policy_utils.go | 245 +- tests/e2e/storagepolicy.go | 2 - tests/e2e/svmotion_detached_volume.go | 444 ++ tests/e2e/svmotion_volumes.go | 1377 ------ .../statefulset/nginx/statefulset.yaml | 3 +- tests/e2e/testing-manifests/tkg/tkg.yaml | 29 +- tests/e2e/tkgs_ha.go | 668 +-- tests/e2e/tkgs_ha_site_down.go | 22 +- tests/e2e/tkgs_ha_utils.go | 169 +- tests/e2e/topology_aware_node_poweroff.go | 6 - tests/e2e/topology_multi_replica.go | 104 +- tests/e2e/topology_operation_strom_cases.go | 67 +- tests/e2e/topology_site_down_cases.go | 194 +- tests/e2e/topology_snapshot.go | 71 +- tests/e2e/util.go | 1107 ++--- tests/e2e/vc_reboot_volume_lifecycle.go | 14 +- tests/e2e/vcp_to_csi_syncer.go | 442 +- tests/e2e/vmc_create_gc.go | 17 +- tests/e2e/vmc_csi_deployments.go | 2 +- tests/e2e/volume_health_test.go | 2 - ...olume_provisioning_with_level5_topology.go | 130 +- tests/e2e/vsan_stretched_cluster.go | 34 +- tests/e2e/vsan_stretched_cluster_utils.go | 45 +- tests/e2e/vsphere.go | 167 +- tests/e2e/vsphere_shared_datastore.go | 9 - tests/e2e/vsphere_volume_disksize.go | 2 - tests/e2e/vsphere_volume_expansion.go | 208 +- tests/e2e/vsphere_volume_fsgroup.go | 9 - .../e2e/vsphere_volume_with_alpha_feature.go | 207 - .../github.com/MakeNowJust/heredoc/README.md | 104 +- .../github.com/PuerkitoBio/purell/.gitignore | 5 + .../github.com/PuerkitoBio/purell/.travis.yml | 12 + vendor/github.com/PuerkitoBio/purell/LICENSE | 12 + .../github.com/PuerkitoBio/purell/README.md | 188 + .../github.com/PuerkitoBio/purell/purell.go | 379 ++ .../github.com/PuerkitoBio/urlesc/.travis.yml | 15 + vendor/github.com/PuerkitoBio/urlesc/LICENSE | 27 + .../github.com/PuerkitoBio/urlesc/README.md | 16 + .../github.com/PuerkitoBio/urlesc/urlesc.go | 180 + .../jsonreference/internal/normalize_url.go | 63 - .../go-openapi/jsonreference/reference.go | 6 +- .../github.com/go-openapi/swag/.gitattributes | 2 - .../github.com/go-openapi/swag/.golangci.yml | 15 - vendor/github.com/go-openapi/swag/.travis.yml | 37 + vendor/github.com/go-openapi/swag/doc.go | 15 +- vendor/github.com/go-openapi/swag/file.go | 33 - vendor/github.com/go-openapi/swag/loading.go | 11 +- .../github.com/go-openapi/swag/post_go18.go | 1 - .../github.com/go-openapi/swag/post_go19.go | 1 - vendor/github.com/go-openapi/swag/pre_go18.go | 1 - vendor/github.com/go-openapi/swag/pre_go19.go | 1 - vendor/github.com/go-openapi/swag/util.go | 17 +- vendor/github.com/go-openapi/swag/yaml.go | 252 +- .../google/gnostic/jsonschema/display.go | 17 +- .../google/gnostic/jsonschema/models.go | 8 +- .../google/gnostic/jsonschema/reader.go | 1 + .../google/gnostic/jsonschema/writer.go | 30 +- .../google/gnostic/openapiv2/OpenAPIv2.go | 7 +- .../google/gnostic/openapiv3/OpenAPIv3.go | 7 +- .../google/gnostic/openapiv3/OpenAPIv3.pb.go | 13 +- .../google/gnostic/openapiv3/OpenAPIv3.proto | 2 +- .../google/gnostic/openapiv3/README.md | 4 - .../gnostic/openapiv3/annotations.pb.go | 183 - .../gnostic/openapiv3/annotations.proto | 60 - .../client/{v6 => v4}/LICENSE | 0 .../{v6 => v4}/apis/volumesnapshot/v1/doc.go | 0 .../apis/volumesnapshot/v1/register.go | 0 .../apis/volumesnapshot/v1/types.go | 46 +- .../v1/zz_generated.deepcopy.go | 9 +- .../v4/apis/volumesnapshot/v1beta1/doc.go | 20 + .../apis/volumesnapshot/v1beta1/register.go | 58 + .../v4/apis/volumesnapshot/v1beta1/types.go | 395 ++ .../v1beta1/zz_generated.deepcopy.go | 424 ++ .../clientset/versioned/clientset.go | 58 +- .../{v6 => v4}/clientset/versioned/doc.go | 2 +- .../versioned/fake/clientset_generated.go | 20 +- .../clientset/versioned/fake/doc.go | 2 +- .../clientset/versioned/fake/register.go | 20 +- .../clientset/versioned/scheme/doc.go | 2 +- .../clientset/versioned/scheme/register.go | 20 +- .../versioned/typed/volumesnapshot/v1/doc.go | 2 +- .../typed/volumesnapshot/v1/fake/doc.go | 2 +- .../v1/fake/fake_volumesnapshot.go | 6 +- .../v1/fake/fake_volumesnapshot_client.go | 4 +- .../v1/fake/fake_volumesnapshotclass.go | 6 +- .../v1/fake/fake_volumesnapshotcontent.go | 6 +- .../volumesnapshot/v1/generated_expansion.go | 2 +- .../typed/volumesnapshot/v1/volumesnapshot.go | 6 +- .../v1/volumesnapshot_client.go | 26 +- .../volumesnapshot/v1/volumesnapshotclass.go | 6 +- .../v1/volumesnapshotcontent.go | 6 +- .../typed/volumesnapshot/v1beta1/doc.go | 20 + .../typed/volumesnapshot/v1beta1/fake/doc.go | 20 + .../v1beta1/fake/fake_volumesnapshot.go | 142 + .../fake/fake_volumesnapshot_client.go | 48 + .../v1beta1/fake/fake_volumesnapshotclass.go | 122 + .../fake/fake_volumesnapshotcontent.go | 133 + .../v1beta1/generated_expansion.go | 25 + .../volumesnapshot/v1beta1/volumesnapshot.go | 195 + .../v1beta1/volumesnapshot_client.go | 99 + .../v1beta1/volumesnapshotclass.go | 168 + .../v1beta1/volumesnapshotcontent.go | 184 + .../mailru/easyjson/jlexer/lexer.go | 14 - .../github.com/vmware/govmomi/find/finder.go | 14 +- .../github.com/vmware/govmomi/list/lister.go | 4 +- .../vmware/govmomi/lookup/client.go | 4 +- .../vmware/govmomi/simulator/container.go | 14 - vendor/golang.org/x/text/width/kind_string.go | 28 + .../golang.org/x/text/width/tables10.0.0.go | 1329 ++++++ .../golang.org/x/text/width/tables11.0.0.go | 1341 ++++++ .../golang.org/x/text/width/tables12.0.0.go | 1361 ++++++ .../golang.org/x/text/width/tables13.0.0.go | 1362 ++++++ .../golang.org/x/text/width/tables15.0.0.go | 1368 ++++++ vendor/golang.org/x/text/width/tables9.0.0.go | 1297 ++++++ vendor/golang.org/x/text/width/transform.go | 239 + vendor/golang.org/x/text/width/trieval.go | 30 + vendor/golang.org/x/text/width/width.go | 206 + 287 files changed, 16764 insertions(+), 23717 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 docs/book/features/csi_driver_on_windows.md create mode 100644 docs/book/features/raw_block_volume.md create mode 100644 docs/book/features/volume_snapshot.md create mode 100644 docs/book/features/vsphere_csi_migration.md delete mode 100644 images/ci/e2e/Dockerfile delete mode 100644 manifests/supervisorcluster/1.23/kustomization.yaml delete mode 100644 manifests/supervisorcluster/1.24/kustomization.yaml delete mode 100644 manifests/supervisorcluster/1.25/kustomization.yaml delete mode 100644 manifests/supervisorcluster/1.26/cns-csi.yaml delete mode 100644 manifests/supervisorcluster/1.26/kustomization.yaml delete mode 100644 manifests/supervisorcluster/1.27/cns-csi.yaml delete mode 100644 manifests/supervisorcluster/1.27/kustomization.yaml delete mode 100755 pipeline/patch-prod.sh delete mode 100755 pipeline/release-testbed.sh delete mode 100755 pipeline/send_slack_notification.sh create mode 100644 pkg/common/cns-lib/node/manager_test.go delete mode 100644 pkg/common/cns-lib/vsphere/cluster_compute_resource.go delete mode 100644 pkg/common/fault/util.go delete mode 100644 pkg/syncer/admissionhandler/pvcsi_admissionhandler.go delete mode 100644 pkg/syncer/admissionhandler/validatesnapshotoperationsupervisorrequest.go delete mode 100644 tests/e2e/csi_snapshot_utils.go delete mode 100644 tests/e2e/multi_vc.go delete mode 100644 tests/e2e/multi_vc_bootstrap.go delete mode 100644 tests/e2e/multi_vc_config_secret.go delete mode 100644 tests/e2e/multi_vc_connection.go delete mode 100644 tests/e2e/multi_vc_multi_replica.go delete mode 100644 tests/e2e/multi_vc_preferential_topology.go delete mode 100644 tests/e2e/multi_vc_utils.go delete mode 100644 tests/e2e/multi_vc_vsphere.go delete mode 100644 tests/e2e/prevent_duplicate_cluster_ids.go delete mode 100644 tests/e2e/raw_block_volume.go create mode 100644 tests/e2e/svmotion_detached_volume.go delete mode 100644 tests/e2e/svmotion_volumes.go delete mode 100644 tests/e2e/vsphere_volume_with_alpha_feature.go create mode 100644 vendor/github.com/PuerkitoBio/purell/.gitignore create mode 100644 vendor/github.com/PuerkitoBio/purell/.travis.yml create mode 100644 vendor/github.com/PuerkitoBio/purell/LICENSE create mode 100644 vendor/github.com/PuerkitoBio/purell/README.md create mode 100644 vendor/github.com/PuerkitoBio/purell/purell.go create mode 100644 vendor/github.com/PuerkitoBio/urlesc/.travis.yml create mode 100644 vendor/github.com/PuerkitoBio/urlesc/LICENSE create mode 100644 vendor/github.com/PuerkitoBio/urlesc/README.md create mode 100644 vendor/github.com/PuerkitoBio/urlesc/urlesc.go delete mode 100644 vendor/github.com/go-openapi/jsonreference/internal/normalize_url.go delete mode 100644 vendor/github.com/go-openapi/swag/.gitattributes create mode 100644 vendor/github.com/go-openapi/swag/.travis.yml delete mode 100644 vendor/github.com/go-openapi/swag/file.go delete mode 100644 vendor/github.com/google/gnostic/openapiv3/annotations.pb.go delete mode 100644 vendor/github.com/google/gnostic/openapiv3/annotations.proto rename vendor/github.com/kubernetes-csi/external-snapshotter/client/{v6 => v4}/LICENSE (100%) rename vendor/github.com/kubernetes-csi/external-snapshotter/client/{v6 => v4}/apis/volumesnapshot/v1/doc.go (100%) rename vendor/github.com/kubernetes-csi/external-snapshotter/client/{v6 => v4}/apis/volumesnapshot/v1/register.go (100%) rename vendor/github.com/kubernetes-csi/external-snapshotter/client/{v6 => v4}/apis/volumesnapshot/v1/types.go (93%) rename vendor/github.com/kubernetes-csi/external-snapshotter/client/{v6 => v4}/apis/volumesnapshot/v1/zz_generated.deepcopy.go (98%) create mode 100644 vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1beta1/doc.go create mode 100644 vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1beta1/register.go create mode 100644 vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1beta1/types.go create mode 100644 vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1beta1/zz_generated.deepcopy.go rename vendor/github.com/kubernetes-csi/external-snapshotter/client/{v6 => v4}/clientset/versioned/clientset.go (69%) rename vendor/github.com/kubernetes-csi/external-snapshotter/client/{v6 => v4}/clientset/versioned/doc.go (94%) rename vendor/github.com/kubernetes-csi/external-snapshotter/client/{v6 => v4}/clientset/versioned/fake/clientset_generated.go (79%) rename vendor/github.com/kubernetes-csi/external-snapshotter/client/{v6 => v4}/clientset/versioned/fake/doc.go (94%) rename vendor/github.com/kubernetes-csi/external-snapshotter/client/{v6 => v4}/clientset/versioned/fake/register.go (72%) rename vendor/github.com/kubernetes-csi/external-snapshotter/client/{v6 => v4}/clientset/versioned/scheme/doc.go (94%) rename vendor/github.com/kubernetes-csi/external-snapshotter/client/{v6 => v4}/clientset/versioned/scheme/register.go (73%) rename vendor/github.com/kubernetes-csi/external-snapshotter/client/{v6 => v4}/clientset/versioned/typed/volumesnapshot/v1/doc.go (94%) rename vendor/github.com/kubernetes-csi/external-snapshotter/client/{v6 => v4}/clientset/versioned/typed/volumesnapshot/v1/fake/doc.go (94%) rename vendor/github.com/kubernetes-csi/external-snapshotter/client/{v6 => v4}/clientset/versioned/typed/volumesnapshot/v1/fake/fake_volumesnapshot.go (96%) rename vendor/github.com/kubernetes-csi/external-snapshotter/client/{v6 => v4}/clientset/versioned/typed/volumesnapshot/v1/fake/fake_volumesnapshot_client.go (92%) rename vendor/github.com/kubernetes-csi/external-snapshotter/client/{v6 => v4}/clientset/versioned/typed/volumesnapshot/v1/fake/fake_volumesnapshotclass.go (96%) rename vendor/github.com/kubernetes-csi/external-snapshotter/client/{v6 => v4}/clientset/versioned/typed/volumesnapshot/v1/fake/fake_volumesnapshotcontent.go (96%) rename vendor/github.com/kubernetes-csi/external-snapshotter/client/{v6 => v4}/clientset/versioned/typed/volumesnapshot/v1/generated_expansion.go (94%) rename vendor/github.com/kubernetes-csi/external-snapshotter/client/{v6 => v4}/clientset/versioned/typed/volumesnapshot/v1/volumesnapshot.go (98%) rename vendor/github.com/kubernetes-csi/external-snapshotter/client/{v6 => v4}/clientset/versioned/typed/volumesnapshot/v1/volumesnapshot_client.go (74%) rename vendor/github.com/kubernetes-csi/external-snapshotter/client/{v6 => v4}/clientset/versioned/typed/volumesnapshot/v1/volumesnapshotclass.go (97%) rename vendor/github.com/kubernetes-csi/external-snapshotter/client/{v6 => v4}/clientset/versioned/typed/volumesnapshot/v1/volumesnapshotcontent.go (98%) create mode 100644 vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/doc.go create mode 100644 vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/fake/doc.go create mode 100644 vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshot.go create mode 100644 vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshot_client.go create mode 100644 vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshotclass.go create mode 100644 vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshotcontent.go create mode 100644 vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/generated_expansion.go create mode 100644 vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshot.go create mode 100644 vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshot_client.go create mode 100644 vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshotclass.go create mode 100644 vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshotcontent.go create mode 100644 vendor/golang.org/x/text/width/kind_string.go create mode 100644 vendor/golang.org/x/text/width/tables10.0.0.go create mode 100644 vendor/golang.org/x/text/width/tables11.0.0.go create mode 100644 vendor/golang.org/x/text/width/tables12.0.0.go create mode 100644 vendor/golang.org/x/text/width/tables13.0.0.go create mode 100644 vendor/golang.org/x/text/width/tables15.0.0.go create mode 100644 vendor/golang.org/x/text/width/tables9.0.0.go create mode 100644 vendor/golang.org/x/text/width/transform.go create mode 100644 vendor/golang.org/x/text/width/trieval.go create mode 100644 vendor/golang.org/x/text/width/width.go diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000000..a2f6e4eda8 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,28 @@ + + +**Is this a BUG REPORT or FEATURE REQUEST?**: + +> Uncomment only one, leave it on its own line: +> +> /kind bug +> /kind feature + + +**What happened**: + +**What you expected to happen**: + +**How to reproduce it (as minimally and precisely as possible)**: + + +**Anything else we need to know?**: + +**Environment**: +- csi-vsphere version: +- vsphere-cloud-controller-manager version: +- Kubernetes version: +- vSphere version: +- OS (e.g. from /etc/os-release): +- Kernel (e.g. `uname -a`): +- Install tools: +- Others: diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000000..ebb972b613 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,22 @@ + + +**What this PR does / why we need it**: + +**Which issue this PR fixes** *(optional, in `fixes #(, fixes #, ...)` format, will close that issue when PR gets merged)*: fixes # + +**Testing done**: +A PR must be marked "[WIP]", if no test result is provided. A WIP PR won't be reviewed, nor merged. +The requester can determine a sufficient test, e.g. build for a cosmetic change, E2E test in a predeployed setup, etc. +For new features, new tests should be done, in addition to regression tests. +If jtest is used to trigger precheckin tests, paste the result after jtest completes and remove [WIP] in the PR subject. +The review cycle will start, only after "[WIP]" is removed from the PR subject. + +**Special notes for your reviewer**: + +**Release note**: + +```release-note +``` diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index af27b77b49..22d12927aa 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,8 +5,6 @@ stages: - e2e-tests-dev - deploy-staging - tests-staging - - prod-rollout - - cleanup-dev run-unit-tests: stage: unit-test @@ -53,16 +51,13 @@ deploy-images-dev: stage: deploy-dev # This resource group is configured with process_mode=oldest_first to make sure the pipelines are run serially. resource_group: production - # Image built from cd-infra/images/ci-deploy/Dockerfile. + # Image built from cd-infra/images/ci-deploy/Dockerfile from Calatrava project. image: $CNS_IMAGE_CI_DEPLOY_STAGE script: - ./pipeline/deploy.sh dependencies: - build-images artifacts: - paths: - - ./env.json - - ./sv_kubeconfig_content.yaml reports: dotenv: build.env @@ -70,26 +65,15 @@ e2e-tests-dev: stage: e2e-tests-dev # This resource group is configured with process_mode=oldest_first to make sure the pipelines are run serially. resource_group: production - # Image built by docker file images/ci/e2e/Dockerfile. - # Command to build new image is - # docker build -t --platform=Linux/x86_64 -f Dockerfile . - # docker tag / - # docker push / - image: $CSI_FVT_GOLANG + image: $CNS_IMAGE_GOLANG dependencies: - deploy-images-dev script: - ./pipeline/e2e-tests.sh - artifacts: - paths: - - ./env.json - - ./sv_kubeconfig_content.yaml - reports: - dotenv: build.env deploy-images-staging: stage: deploy-staging - # Image built from cd-infra/images/ci-deploy/Dockerfile. + # Image built from cd-infra/images/ci-deploy/Dockerfile from Calatrava project. image: $CNS_IMAGE_CI_DEPLOY_STAGE script: - ./pipeline/deploy-staging.sh @@ -97,15 +81,14 @@ deploy-images-staging: - build-images only: - master + artifacts: + reports: + dotenv: build.env e2e-tests-staging: stage: tests-staging - # Image built by docker file images/ci/e2e/Dockerfile. - # Command to build new image is - # docker build -t --platform=Linux/x86_64 -f Dockerfile . - # docker tag / - # docker push / - image: $CSI_FVT_GOLANG + # Image built from cd-infra//images/ci-e2e/Dockerfile from Calatrava project. + image: $CNS_IMAGE_E2E dependencies: - deploy-images-staging script: @@ -115,7 +98,7 @@ e2e-tests-staging: system-tests-staging: stage: tests-staging - # Image built from cd-infra//images/ci-e2e/Dockerfile. + # Image built from cd-infra//images/ci-e2e/Dockerfile from Calatrava project. image: $CNS_IMAGE_E2E dependencies: - deploy-images-staging @@ -126,7 +109,7 @@ system-tests-staging: perf-tests-staging: stage: tests-staging - # Image built from cd-infra//images/ci-e2e/Dockerfile. + # Image built from cd-infra//images/ci-e2e/Dockerfile from Calatrava project. image: $CNS_IMAGE_E2E dependencies: - deploy-images-staging @@ -134,25 +117,3 @@ perf-tests-staging: - echo "TODO - Add perf tests." only: - master - -patch-prod-images: - stage: prod-rollout - # Image built from cd-infra/images/ci-deploy/Dockerfile - image: $CNS_IMAGE_CI_DEPLOY_STAGE - script: - - ./pipeline/patch-prod.sh - dependencies: - - build-images - only: - - master - -cleanup-dev: - stage: cleanup-dev - image: $CNS_IMAGE_CI_DEPLOY_STAGE - when: always - artifacts: - reports: - dotenv: build.env - script: - - chmod 777 pipeline/release-testbed.sh - - ./pipeline/release-testbed.sh diff --git a/README.md b/README.md index c5fcdf3c80..e0f91997ad 100644 --- a/README.md +++ b/README.md @@ -5,16 +5,20 @@ The vSphere CSI Driver is a Kubernetes plugin that allows persistent storage for This driver is in a stable `GA` state and is suitable for production use. +The vSphere CSI Driver is supported on vSphere 6.7U3 and later versions. +If you are using an earlier version of vSphere, you may need to upgrade to a supported version to use the vSphere CSI Driver. +It's also important to note that the vSphere CSI Driver has different versions, and each version may have different requirements or limitations, so it's essential to check the [documentation](https://docs.vmware.com/en/VMware-vSphere-Container-Storage-Plug-in/2.0/vmware-vsphere-csp-getting-started/GUID-D4AAD99E-9128-40CE-B89C-AD451DA8379D.html) for your specific version. + It is recommended to install an out-of-tree Cloud Provider Interface like [vSphere Cloud Provider Interface](https://github.com/kubernetes/cloud-provider-vsphere) in the Kubernetes cluster to keep the Kubernetes cluster fully operational. ## Documentation Documentation for vSphere CSI Driver is available here: -* [vSphere CSI Driver Concepts](https://docs.vmware.com/en/VMware-vSphere-Container-Storage-Plug-in/3.0/vmware-vsphere-csp-getting-started/GUID-74AF02D7-1562-48BD-A9FE-C81A53342AC3.html) -* [vSphere CSI Driver Features](https://docs.vmware.com/en/VMware-vSphere-Container-Storage-Plug-in/3.0/vmware-vsphere-csp-getting-started/GUID-D4AAD99E-9128-40CE-B89C-AD451DA8379D.html#GUID-E59B13F5-6F49-4619-9877-DF710C365A1E) -* [vSphere CSI Driver Deployment Guide](https://docs.vmware.com/en/VMware-vSphere-Container-Storage-Plug-in/3.0/vmware-vsphere-csp-getting-started/GUID-6DBD2645-FFCF-4076-80BE-AD44D7141521.html) -* [vSphere CSI Driver User Guide](https://docs.vmware.com/en/VMware-vSphere-Container-Storage-Plug-in/3.0/vmware-vsphere-csp-getting-started/GUID-6DBD2645-FFCF-4076-80BE-AD44D7141521.html) +* [vSphere CSI Driver Concepts](https://docs.vmware.com/en/VMware-vSphere-Container-Storage-Plug-in/2.0/vmware-vsphere-csp-getting-started/GUID-74AF02D7-1562-48BD-A9FE-C81A53342AC3.html) +* [vSphere CSI Driver Features](https://docs.vmware.com/en/VMware-vSphere-Container-Storage-Plug-in/2.0/vmware-vsphere-csp-getting-started/GUID-D4AAD99E-9128-40CE-B89C-AD451DA8379D.html#GUID-E59B13F5-6F49-4619-9877-DF710C365A1E) +* [vSphere CSI Driver Deployment Guide](https://docs.vmware.com/en/VMware-vSphere-Container-Storage-Plug-in/2.0/vmware-vsphere-csp-getting-started/GUID-6DBD2645-FFCF-4076-80BE-AD44D7141521.html) +* [vSphere CSI Driver User Guide](https://docs.vmware.com/en/VMware-vSphere-Container-Storage-Plug-in/2.0/vmware-vsphere-csp-getting-started/GUID-6DBD2645-FFCF-4076-80BE-AD44D7141521.html) ## vSphere CSI Driver Releases @@ -24,6 +28,7 @@ Documentation for vSphere CSI Driver is available here: * [Release 2.5](https://docs.vmware.com/en/VMware-vSphere-Container-Storage-Plug-in/2.5/rn/vmware-vsphere-container-storage-plugin-25-release-notes/index.html) * [Release 2.4](https://docs.vmware.com/en/VMware-vSphere-Container-Storage-Plug-in/2.4/rn/vmware-vsphere-container-storage-plugin-24-release-notes/index.html) * [Release 2.3](https://docs.vmware.com/en/VMware-vSphere-Container-Storage-Plug-in/2.3/rn/vmware-vsphere-container-storage-plugin-23-release-notes/index.html) +* [Release 2.2](https://docs.vmware.com/en/VMware-vSphere-Container-Storage-Plug-in/2.2/rn/vmware-vsphere-container-storage-plugin-22-release-notes/index.html) ## Contributing diff --git a/cmd/syncer/main.go b/cmd/syncer/main.go index fa9292cf30..39dadf18f9 100644 --- a/cmd/syncer/main.go +++ b/cmd/syncer/main.go @@ -22,21 +22,15 @@ import ( "fmt" "net/http" "os" - "os/signal" - "strings" - "syscall" "time" "github.com/kubernetes-csi/csi-lib-utils/leaderelection" "github.com/prometheus/client_golang/prometheus/promhttp" cnstypes "github.com/vmware/govmomi/cns/types" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" "sigs.k8s.io/vsphere-csi-driver/v3/pkg/common/cns-lib/node" "sigs.k8s.io/vsphere-csi-driver/v3/pkg/common/config" "sigs.k8s.io/vsphere-csi-driver/v3/pkg/common/prometheus" - "sigs.k8s.io/vsphere-csi-driver/v3/pkg/common/utils" "sigs.k8s.io/vsphere-csi-driver/v3/pkg/csi/service/common" "sigs.k8s.io/vsphere-csi-driver/v3/pkg/csi/service/common/commonco" "sigs.k8s.io/vsphere-csi-driver/v3/pkg/csi/service/logger" @@ -100,27 +94,6 @@ func main() { *internalFSSName, *internalFSSNamespace, "", *operationMode) admissionhandler.COInitParams = &syncer.COInitParams - // Disconnect VC session on restart - defer func() { - log.Info("Cleaning up vc sessions") - if r := recover(); r != nil { - cleanupSessions(ctx, r) - } - }() - - ch := make(chan os.Signal, 1) - signal.Notify(ch, syscall.SIGTERM) - go func() { - for { - sig := <-ch - if sig == syscall.SIGTERM { - log.Info("SIGTERM signal received") - utils.LogoutAllvCenterSessions(ctx) - os.Exit(0) - } - } - }() - if *operationMode == operationModeWebHookServer { log.Infof("Starting container with operation mode: %v", operationModeWebHookServer) if webHookStartError := admissionhandler.StartWebhookServer(ctx); webHookStartError != nil { @@ -139,12 +112,6 @@ func main() { // K8sCloudOperator should run on every node where csi controller can run. if clusterFlavor == cnstypes.CnsClusterFlavorWorkload { go func() { - defer func() { - log.Info("Cleaning up vc sessions cloud operator service") - if r := recover(); r != nil { - cleanupSessions(ctx, r) - } - }() if err := k8scloudoperator.InitK8sCloudOperatorService(ctx); err != nil { log.Fatalf("Error initializing K8s Cloud Operator gRPC sever. Error: %+v", err) } @@ -153,12 +120,6 @@ func main() { // Go module to keep the metrics http server running all the time. go func() { - defer func() { - log.Info("Cleaning up vc sessions prometheus metrics") - if r := recover(); r != nil { - cleanupSessions(ctx, r) - } - }() prometheus.SyncerInfo.WithLabelValues(syncer.Version).Set(1) for { log.Info("Starting the http server to expose Prometheus metrics..") @@ -211,13 +172,7 @@ func initSyncerComponents(ctx context.Context, clusterFlavor cnstypes.CnsCluster coInitParams *interface{}) func(ctx context.Context) { return func(ctx context.Context) { log := logger.GetLogger(ctx) - // Disconnect vCenter sessions on restart - defer func() { - log.Info("Cleaning up vc sessions syncer components") - if r := recover(); r != nil { - cleanupSessions(ctx, r) - } - }() + if err := manager.InitCommonModules(ctx, clusterFlavor, coInitParams); err != nil { log.Errorf("Error initializing common modules for all flavors. Error: %+v", err) os.Exit(1) @@ -242,98 +197,35 @@ func initSyncerComponents(ctx context.Context, clusterFlavor cnstypes.CnsCluster // Initialize CNS Operator for Supervisor clusters. if clusterFlavor == cnstypes.CnsClusterFlavorWorkload { go func() { - defer func() { - log.Info("Cleaning up vc sessions storage pool service") - if r := recover(); r != nil { - cleanupSessions(ctx, r) - } - }() if err := storagepool.InitStoragePoolService(ctx, configInfo, coInitParams); err != nil { log.Errorf("Error initializing StoragePool Service. Error: %+v", err) - utils.LogoutAllvCenterSessions(ctx) - os.Exit(0) + os.Exit(1) } }() } if clusterFlavor == cnstypes.CnsClusterFlavorVanilla { // Initialize node manager so that syncer components can // retrieve NodeVM using the NodeID. + useNodeUuid := false + if commonco.ContainerOrchestratorUtility.IsFSSEnabled(ctx, common.UseCSINodeId) { + useNodeUuid = true + } nodeMgr := &node.Nodes{} - err = nodeMgr.Initialize(ctx) + err = nodeMgr.Initialize(ctx, useNodeUuid) if err != nil { log.Errorf("failed to initialize nodeManager. Error: %+v", err) os.Exit(1) } - if configInfo.Cfg.Global.ClusterDistribution == "" { - config, err := rest.InClusterConfig() - if err != nil { - log.Errorf("failed to get InClusterConfig: %v", err) - os.Exit(1) - } - clientset, err := kubernetes.NewForConfig(config) - if err != nil { - log.Errorf("failed to create kubernetes client with err: %v", err) - os.Exit(1) - } - - // Get the version info for the Kubernetes API server - versionInfo, err := clientset.Discovery().ServerVersion() - if err != nil { - log.Errorf("failed to fetch versionInfo with err: %v", err) - os.Exit(1) - } - - // Extract the version string from the version info - version := versionInfo.GitVersion - var ClusterDistNameToServerVersion = map[string]string{ - "gke": "Anthos", - "racher": "Rancher", - "rke": "Rancher", - "docker": "DockerEE", - "dockeree": "DockerEE", - "openshift": "Openshift", - "wcp": "Supervisor", - "vmware": "TanzuKubernetesCluster", - "nativek8s": "VanillaK8S", - } - distributionUnknown := true - for distServerVersion, distName := range ClusterDistNameToServerVersion { - if strings.Contains(version, distServerVersion) { - configInfo.Cfg.Global.ClusterDistribution = distName - distributionUnknown = false - break - } - } - if distributionUnknown { - configInfo.Cfg.Global.ClusterDistribution = ClusterDistNameToServerVersion["nativek8s"] - } - } } go func() { - defer func() { - log.Info("Cleaning up vc sessions cns operator") - if r := recover(); r != nil { - cleanupSessions(ctx, r) - } - }() if err := manager.InitCnsOperator(ctx, clusterFlavor, configInfo, coInitParams); err != nil { log.Errorf("Error initializing Cns Operator. Error: %+v", err) - utils.LogoutAllvCenterSessions(ctx) - os.Exit(0) + os.Exit(1) } }() if err := syncer.InitMetadataSyncer(ctx, clusterFlavor, configInfo); err != nil { log.Errorf("Error initializing Metadata Syncer. Error: %+v", err) - utils.LogoutAllvCenterSessions(ctx) - os.Exit(0) + os.Exit(1) } } } - -func cleanupSessions(ctx context.Context, r interface{}) { - log := logger.GetLogger(ctx) - log.Errorf("Observed a panic and a restart was invoked, panic: %+v", r) - log.Info("Recovered from panic. Disconnecting the existing vc sessions.") - utils.LogoutAllvCenterSessions(ctx) - os.Exit(0) -} diff --git a/cmd/vsphere-csi/main.go b/cmd/vsphere-csi/main.go index c652ac505c..a6ff940069 100644 --- a/cmd/vsphere-csi/main.go +++ b/cmd/vsphere-csi/main.go @@ -17,15 +17,11 @@ limitations under the License. package main import ( - "context" "flag" "fmt" "os" - "os/signal" - "syscall" csiconfig "sigs.k8s.io/vsphere-csi-driver/v3/pkg/common/config" - "sigs.k8s.io/vsphere-csi-driver/v3/pkg/common/utils" "sigs.k8s.io/vsphere-csi-driver/v3/pkg/csi/service" "sigs.k8s.io/vsphere-csi-driver/v3/pkg/csi/service/common/commonco" "sigs.k8s.io/vsphere-csi-driver/v3/pkg/csi/service/logger" @@ -70,37 +66,7 @@ func main() { log.Error("CSI endpoint cannot be empty. Please set the env variable.") os.Exit(1) } - log.Info("Enable logging off for vCenter sessions on exit") - // Disconnect VC session on restart - defer func() { - if r := recover(); r != nil { - log.Info("Cleaning up vc sessions") - cleanupSessions(ctx, r) - } - }() - - ch := make(chan os.Signal, 1) - signal.Notify(ch, syscall.SIGTERM) - go func() { - for { - sig := <-ch - if sig == syscall.SIGTERM { - log.Info("SIGTERM signal received") - utils.LogoutAllvCenterSessions(ctx) - os.Exit(0) - } - } - }() vSphereCSIDriver := service.NewDriver() vSphereCSIDriver.Run(ctx, CSIEndpoint) - -} - -func cleanupSessions(ctx context.Context, r interface{}) { - log := logger.GetLogger(ctx) - log.Errorf("Observed a panic and a restart was invoked, panic: %+v", r) - log.Info("Recovered from panic. Disconnecting the existing vc sessions.") - utils.LogoutAllvCenterSessions(ctx) - os.Exit(0) } diff --git a/docs/book/features/csi_driver_on_windows.md b/docs/book/features/csi_driver_on_windows.md new file mode 100644 index 0000000000..3e9248c93d --- /dev/null +++ b/docs/book/features/csi_driver_on_windows.md @@ -0,0 +1,165 @@ + + +# vSphere CSI Driver - Windows Support + +- [Introduction](#introduction) +- [Prerequisite](#prereq) +- [How to enable vSphere CSI with windows nodes](#how-to-enable-vsphere-csi-win) +- [Examples to Deploy a Windows pod with PVC mount](#examples) + +## Introduction + +Windows node support is added in vSphere CSI driver v2.4.0 as an Alpha feature. + +Following features are not supported for Windows Node: + +1. ReadWriteMany volumes based on vSAN file service are not supported on Windows Node. +2. [Raw Block Volumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#raw-block-volume-support) are not supported. +3. Windows Nodes will be used as Worker nodes only. vSphere CSI will not support a mixture of Linux worker nodes and Windows Worker Nodes. + +## Prerequisite + +In addition to prerequisites mentioned [here](https://docs.vmware.com/en/VMware-vSphere-Container-Storage-Plug-in/2.0/vmware-vsphere-csp-getting-started/GUID-0AB6E692-AA47-4B6A-8CEA-38B754E16567.html), following needs to be fullfilled to support windows in vSphere CSI: + +1. Minimum kubernetes version required is 1.20. +2. Minimum vSphere CSI driver version required is 2.4. +3. Master nodes should be running Linux. +4. Worker nodes should have Windows server 2019. Other windows server version are not supported. Please refer [this](https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/adding-windows-nodes/) +5. if containerd is used in nodes, containerd version should be greater than or equal to 1.5, refer: https://github.com/containerd/containerd/issues/5405 +6. `CSI Proxy` should be installed in each of the Windows nodes. To install csi proxy follow steps from https://github.com/kubernetes-csi/csi-proxy#installation + +## How to enable vSphere CSI with Windows nodes + +- Install vSphere CSI driver 2.4 by following https://docs.vmware.com/en/VMware-vSphere-Container-Storage-Plug-in/2.0/vmware-vsphere-csp-getting-started/GUID-A1982536-F741-4614-A6F2-ADEE21AA4588.html +- To enable windows support, patch the configmap to enable csi-windows-support feature switch by running following command: + + ```bash + kubectl patch configmap/internal-feature-states.csi.vsphere.vmware.com \ + -n vmware-system-csi \ + --type merge \ + -p '{"data":{"csi-windows-support":"true"}}' + ``` + +- vSphere CSI driver v2.4.0 introduces a new node daemonset which will be running on all windows nodes. To verify this run: + + ```bash + $ kubectl get daemonsets vsphere-csi-node-windows --namespace=vmware-system-csi + NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE + vsphere-csi-node-windows 1 1 1 1 1 kubernetes.io/os=windows 7m10s + ``` + +## Examples + +- Define a storage class example-windows-sc.yaml as defined [here](https://raw.githubusercontent.com/kubernetes-sigs/vsphere-csi-driver/master/example/vanilla-k8s-RWO-filesystem-volumes/example-windows-sc.yaml) + + ```bash + kind: StorageClass + apiVersion: storage.k8s.io/v1 + metadata: + name: example-windows-sc + provisioner: csi.vsphere.vmware.com + allowVolumeExpansion: true # Optional: only applicable to vSphere 7.0U1 and above + parameters: + storagepolicyname: "vSAN Default Storage Policy" # Optional Parameter + #datastoreurl: "ds:///vmfs/volumes/vsan:52cdfa80721ff516-ea1e993113acfc77/" # Optional Parameter + #csi.storage.k8s.io/fstype: "ntfs" # Optional Parameter + ``` + + 'csi.storage.k8s.io/fstype' is an optional parameter. From the Windows file systems, only ntfs can be set to its value, as vSphere CSI Driver can only support the NTFS filesystem on Windows Nodes. + +- Import this `StorageClass` into `Vanilla Kubernetes` cluster: + + ```bash + kubectl create -f example-sc.yaml + ``` + +- Define a `PersistentVolumeClaim` request example-windows-pvc.yaml as shown in the spec [here](https://raw.githubusercontent.com/kubernetes-sigs/vsphere-csi-driver/master/example/vanilla-k8s-RWO-filesystem-volumes/example-windows-pvc.yaml) + + ```bash + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: example-windows-pvc + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi + storageClassName: example-windows-sc + volumeMode: Filesystem + ``` + + The pvc definition is same as Linux and only 'ReadWriteOnce' can be specified as accessModes. + +- Import this `PersistentVolumeClaim` into `Vanilla Kubernetes` cluster: + + ```bash + kubectl create -f example-windows-pvc.yaml + ``` + +- Verify a `PersistentVolume` was created successfully + + The `Status` should say `Bound`. + + ```bash + $ kubectl describe pvc example-windows-pvc + Name: example-windows-pvc + Namespace: default + StorageClass: example-windows-sc + Status: Bound + Volume: pvc-e64e0716-ff63-47b8-8ee4-d1eb4f86dcd7 + Labels: + Annotations: pv.kubernetes.io/bind-completed: yes + pv.kubernetes.io/bound-by-controller: yes + volume.beta.kubernetes.io/storage-provisioner: csi.vsphere.vmware.com + Finalizers: [kubernetes.io/pvc-protection] + Capacity: 5Gi + Access Modes: RWO + VolumeMode: Filesystem + Used By: example-windows-pod + Events: + ``` + +- To use the above pvc in a Windows pod, you can create a pod spec example-windows-pod.yaml like [this](https://raw.githubusercontent.com/kubernetes-sigs/vsphere-csi-driver/master/example/vanilla-k8s-RWO-filesystem-volumes/example-windows-pod.yaml) + + ```bash + apiVersion: v1 + kind: Pod + metadata: + name: example-windows-pod + spec: + nodeSelector: + kubernetes.io/os: windows + containers: + - name: test-container + image: mcr.microsoft.com/windows/servercore:ltsc2019 + command: + - "powershell.exe" + - "-Command" + - "while (1) { Add-Content -Encoding Ascii C:\\test\\data.txt $(Get-Date -Format u); sleep 1 }" + volumeMounts: + - name: test-volume + mountPath: "/test/" + readOnly: false + volumes: + - name: test-volume + persistentVolumeClaim: + claimName: example-windows-pvc + ``` + +- Create the pod + + ```bash + kubectl create -f example-windows-pod.yaml + ``` + +- Verify pod was created succussfully + + ```bash + $ kubectl get pod example-windows-pod + NAME READY STATUS RESTARTS AGE + example-windows-pod 1/1 Running 0 4m13s + ``` + + In this example, example-windows-pvc is formatted as NTFS file system and is mounted to "C:\\test" directory. diff --git a/docs/book/features/raw_block_volume.md b/docs/book/features/raw_block_volume.md new file mode 100644 index 0000000000..aec3bd9e68 --- /dev/null +++ b/docs/book/features/raw_block_volume.md @@ -0,0 +1,63 @@ +# vSphere CSI Driver - Single-Access, Block Based Volume (Raw Block Volume) + +[Raw Block Volume](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#raw-block-volume-support) feature in Kubernetes was promoted to stable in Kubernetes 1.18. +vSphere CSI Driver release `v2.3.0` onwards has Raw Block Volume feature released as `Alpha`. We do not recommend `Alpha` features for production use. Raw Block Volume feature is only supported for linux based nodes. + +This feature allows persistent volumes to be exposed inside containers as a block device instead of as a mounted file system. + +There are some specialized applications that require direct access to a block device because the file system layer introduces unneeded overhead. +The ability to use a raw block device without a filesystem will allow Kubernetes better support for high-performance applications that are capable of consuming and manipulating block storage for their needs. The most common case is databases (MongoDB, Cassandra) that require consistent I/O performance and low latency, which prefer to organize their data directly on the underlying storage. + +## Creating a new raw block PVC + +Create a storage class. + +```yaml +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: example-raw-block-sc +provisioner: csi.vsphere.vmware.com +``` + +To request a raw block PersistentVolumeClaim, volumeMode = "Block" must be specified in the PersistentVolumeClaimSpec. +Raw Block Volume should be created using accessModes `ReadWriteOnce`. vSphere CSI Driver does not support creating raw block volume using `ReadWriteMany` accessModes. + +```yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: example-raw-block-pvc +spec: + volumeMode: Block + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi + storageClassName: example-raw-block-sc +``` + +## Using a raw block PVC + +When you use the PVC in a pod definition, you get to choose the device path for the block device rather than the mount path for the file system. + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: example-raw-block-pod +spec: + containers: + - name: test-container + image: gcr.io/google_containers/busybox:1.24 + command: ["/bin/sh", "-c", "while true ; do sleep 2 ; done"] + volumeDevices: + - devicePath: /dev/xvda + name: data + restartPolicy: Never + volumes: + - name: data + persistentVolumeClaim: + claimName: example-raw-block-pvc +``` diff --git a/docs/book/features/volume_snapshot.md b/docs/book/features/volume_snapshot.md new file mode 100644 index 0000000000..d84cea0cfa --- /dev/null +++ b/docs/book/features/volume_snapshot.md @@ -0,0 +1,210 @@ + + +# vSphere CSI Driver - Volume Snapshot & Restore + +- [Introduction](#introduction) +- [Prerequisite](#prereq) +- [How to enable Volume Snapshot & Restore feature in vSphere CSI](#how-to-deploy) +- [How to use Volume Snapshot & Restore feature](#how-to-use) +- [Configuration - Maximum Number of Snapshots per Volume](#config-param) + +## Introduction + +CSI Volume Snapshot & Restore feature was introduced as an alpha feature in Kubernetes 1.12, promoted to beta in Kubernetes 1.17 and moved to GA in Kubernetes 1.20. +Volume Snapshot & Restore feature will be added in vSphere CSI driver 2.4 as an Alpha feature. + +Known limitations for the Alpha feature in vSphere CSI driver 2.4 are listed below. + +1. It is only supported in ReadWriteOnce volumes based on First Class Disk, i.e., FCD or CNS block volume, while not yet supported in ReadWriteMany volumes based on vSAN file service. +2. It is only supported in [Vanilla Kubernetes](https://github.com/kubernetes/kubernetes) cluster now, while not yet supported in either [vSphere with Kubernetes](https://blogs.vmware.com/vsphere/2019/08/introducing-project-pacific.html) cluster aka Supervisor Cluster or [Tanzu Kubernetes Grid Service](https://blogs.vmware.com/vsphere/2020/03/vsphere-7-tanzu-kubernetes-clusters.html) cluster aka Guest Cluster. +3. Volume restore can only create a PVC with the same storage capacity as the source VolumeSnapshot. +4. vSphere CSI introduces a constraint on the maximum number of snapshots per ReadWriteOnce volume. The maximum is configurable but set to 3 by default. Please refer to the section of [Configuration - Maximum Number of Snapshots per Volume](#config-param) for more detail. +5. It is not supported to expand/delete volumes with snapshots. +6. It is not supported to snapshot CSI migrated volumes. + +## Prerequisite + +In addition to prerequisites mentioned [here](https://docs.vmware.com/en/VMware-vSphere-Container-Storage-Plug-in/2.0/vmware-vsphere-csp-getting-started/GUID-0AB6E692-AA47-4B6A-8CEA-38B754E16567.html), following prerequisites to be fulfilled to support Volume Snapshot & Restore feature in vSphere CSI: + +1. Minimum kubernetes version required is 1.20. +2. Minimum CSI upstream external-snapshotter/snapshot-controller version required is 4.1. +3. Minimum vSphere CSI driver version required is 2.4. +4. Minimum vSphere version required is 7.0U3. (The minimum version applies to both vCenter version and ESXi version) + +## How to enable Volume Snapshot & Restore feature in vSphere CSI + +- Install vSphere CSI driver 2.4 by following https://vsphere-csi-driver.sigs.k8s.io/driver-deployment/installation.html +- To enable Volume Snapshot feature, patch the configmap to enable `block-volume-snapshot` feature switch by running following command: + + ```bash + $ kubectl patch configmap/internal-feature-states.csi.vsphere.vmware.com \ + -n vmware-system-csi \ + --type merge \ + -p '{"data":{"block-volume-snapshot":"true"}}' + ``` + +- To deploy required components for CSI volume snapshot feature, the following script is available for an easy deployment. +To get to know the step-by-step workflow of the script, please check out using the command `bash deploy-csi-snapshot-components.sh -h`. + + ```bash + $ wget https://raw.githubusercontent.com/kubernetes-sigs/vsphere-csi-driver/v2.4.0/manifests/vanilla/deploy-csi-snapshot-components.sh + $ bash deploy-csi-snapshot-components.sh + ✅ Verified that block-volume-snapshot feature is enabled + ... + ✅ Successfully deployed all components for CSI Snapshot feature. + ``` + + Below is the expected view in `vmware-system-csi` namespace when the deployment is completed in a single-master cluster: + + ```bash + $ kubectl -n vmware-system-csi get pod,deploy + NAME READY STATUS RESTARTS AGE + pod/vsphere-csi-controller-6c46964474-bcx5t 7/7 Running 0 164m + pod/vsphere-csi-node-8pspp 3/3 Running 0 3h55m + pod/vsphere-csi-node-lgthd 3/3 Running 0 3h55m + pod/vsphere-csi-node-nzvx8 3/3 Running 0 3h55m + pod/vsphere-csi-node-x4zch 3/3 Running 0 3h55m + + NAME READY UP-TO-DATE AVAILABLE AGE + deployment.apps/vsphere-csi-controller 1/1 1 1 3h55m + ``` + + At this point, you are good to try out CSI Volume Snapshot & Restore feature in vSphere CSI driver. + +## How to use Volume Snapshot & Restore feature + +To use volume snapshot and restore feature in vSphere CSI, please refer to example yaml files for FileSystem volumes, [vanilla-k8s-RWO-filesystem-volumes](https://github.com/kubernetes-sigs/vsphere-csi-driver/tree/master/example/vanilla-k8s-RWO-filesystem-volumes), and example yaml files for Raw Block volumes, [vanilla-k8s-RWO-Block-Volumes](https://github.com/kubernetes-sigs/vsphere-csi-driver/tree/master/example/vanilla-k8s-RWO-Block-Volumes). Below is an example for FileSystem volumes. + +### Volume Snapshot + +#### Dynamic-provisioned Snapshot + +Below is an example StorageClass yaml from [vanilla-k8s-RWO-filesystem-volumes/example-sc.yaml](https://github.com/kubernetes-sigs/vsphere-csi-driver/blob/master/example/vanilla-k8s-RWO-filesystem-volumes/example-sc.yaml), with optional parameters being commented out. + +```yaml +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: example-vanilla-rwo-filesystem-sc + annotations: + storageclass.kubernetes.io/is-default-class: "true" # Optional +provisioner: csi.vsphere.vmware.com +allowVolumeExpansion: true # Optional: only applicable to vSphere 7.0U1 and above +#parameters: +# datastoreurl: "ds:///vmfs/volumes/vsan:52cdfa80721ff516-ea1e993113acfc77/" # Optional Parameter +# storagepolicyname: "vSAN Default Storage Policy" # Optional Parameter +# csi.storage.k8s.io/fstype: "ext4" # Optional Parameter +``` + +Create a StorageClass. + +```bash +$ kubectl apply -f example-sc.yaml +$ kubectl get sc +NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE +example-vanilla-rwo-filesystem-sc (default) csi.vsphere.vmware.com Delete Immediate true 2s +``` + +Create a PVC: + +```bash +$ kubectl apply -f example-pvc.yaml +$ kubectl get pvc +NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +example-vanilla-rwo-pvc Bound pvc-2dc37ea0-dee0-4ad3-96ca-82f0159d7532 5Gi RWO example-vanilla-rwo-filesystem-sc 7s +``` + +Create a VolumeSnapshotClass: + +```bash +$ kubectl apply -f example-snapshotclass.yaml +$ kubectl get volumesnapshotclass +NAME DRIVER DELETIONPOLICY AGE +example-vanilla-rwo-filesystem-snapshotclass csi.vsphere.vmware.com Delete 4s +``` + +Create a VolumeSnapshot: + +```bash +$ kubectl apply -f example-snapshot.yaml +$ kubectl get volumesnapshot +NAME READYTOUSE SOURCEPVC SOURCESNAPSHOTCONTENT RESTORESIZE SNAPSHOTCLASS SNAPSHOTCONTENT CREATIONTIME AGE +example-vanilla-rwo-filesystem-snapshot true example-vanilla-rwo-pvc 5Gi example-vanilla-rwo-filesystem-snapshotclass snapcontent-a7c00b7f-f727-4010-9b1a-d546df9a8bab 57s 58s +``` + +#### Static-provisioned Snapshot + +Below are prerequisites for creating static-provisioned snapshot: + +1. Make sure a FCD snapshot on which the static-provisioned snapshot is created is available in your vSphere. +2. Construct the snapshotHandle based on the combination of FCD Volume ID and FCD Snapshot ID of the snapshot. For example, FCD Volume ID and FCD Snapshot ID of a FCD snapshot are `4ef058e4-d941-447d-a427-438440b7d306` and `766f7158-b394-4cc1-891b-4667df0822fa`. Then, the constructed snapshotHandle is `4ef058e4-d941-447d-a427-438440b7d306+766f7158-b394-4cc1-891b-4667df0822fa`. +3. Update the `spec.source.snapshotHandle` field in the VolumeSnapshotContent object of example-static-snapshot.yaml with the constructed snapshotHandle in step 2. + +Create a static-provisioned VolumeSnapshot: + +```bash +$ kubectl apply -f example-static-snapshot.yaml +$ kubectl get volumesnapshot static-vanilla-rwo-filesystem-snapshot +NAME READYTOUSE SOURCEPVC SOURCESNAPSHOTCONTENT RESTORESIZE SNAPSHOTCLASS SNAPSHOTCONTENT CREATIONTIME AGE +static-vanilla-rwo-filesystem-snapshot true static-vanilla-rwo-filesystem-snapshotcontent 5Gi static-vanilla-rwo-filesystem-snapshotcontent 76m 22m +``` + +### Volume Restore + +Make sure the VolumeSnapshot to be restored is available in the current Kubernetes cluster. + +```bash +$ kubectl get volumesnapshot +NAME READYTOUSE SOURCEPVC SOURCESNAPSHOTCONTENT RESTORESIZE SNAPSHOTCLASS SNAPSHOTCONTENT CREATIONTIME AGE +example-vanilla-rwo-filesystem-snapshot true example-vanilla-rwo-pvc 5Gi example-vanilla-rwo-filesystem-snapshotclass snapcontent-a7c00b7f-f727-4010-9b1a-d546df9a8bab 22m 22m +``` + +Create a PVC from a VolumeSnapshot: + +```bash +$ kubectl create -f example-restore.yaml +$ kubectl get pvc +NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +example-vanilla-rwo-filesystem-restore Bound pvc-202c1dfc-78be-4835-89d5-110f739a87dd 5Gi RWO example-vanilla-rwo-filesystem-sc 78s +``` + +## Configuration - Maximum Number of Snapshots per Volume + +Per the [best practices for using VMware snapshots](https://kb.vmware.com/s/article/1025279), it is recommended to use only 2 to 3 snapshots per virtual disk for a better performance. +So, we make the global constraint, i.e., maximum number of snapshots per volume, configurable and meanwhile set the default to 3. +Additionally, the best-practice guideline only applies to virtual disks on VMFS and NFS datastores while not to those on VVOL and VSAN. +Therefore, we also introduces granular configuration parameters on the constraint, apart from the global configuration parameter. + +Below are configuration parameters available: + +- `global-max-snapshots-per-block-volume`: Global configuration parameter that applies to volumes on all kinds of datastores. By default, it is set to 3. +- `granular-max-snapshots-per-block-volume-vsan`: Granular configuration parameter on VSAN datastore only. It overrides the global constraint if set, while it falls back to the global constraint if unset. +- `granular-max-snapshots-per-block-volume-vvol`: Granular configuration parameter on VVOL datastore only. It overrides the global constraint if set, while it falls back to the global constraint if unset. + +**Note**: Users only need to configure it when the default constraint does not work for their user cases. For others, just skip the configuration below. + +Here is an example of vSphere CSI about how to configure the constraints. Firstly, delete the Secret that stores vSphere config. (Kubernetes doesn't allow to update Secret resources in place) + +```bash +kubectl delete secret vsphere-config-secret --namespace=vmware-system-csi +``` + +Secondly, update the config file of vSphere CSI and add configuration parameters for snapshot feature under the `[Snapshot]` section. + +```bash +$ cat /etc/kubernetes/csi-vsphere.conf +[Global] +... + +[Snapshot] +global-max-snapshots-per-block-volume = 5 # optional, set to 3 if unset +granular-max-snapshots-per-block-volume-vsan = 7 # optional, fall back to the global constraint if unset +granular-max-snapshots-per-block-volume-vvol = 8 # optional, fall back to the global constraint if unset +... +``` + +Finally, create a new Secret with the updated config file. + +```bash +kubectl create secret generic vsphere-config-secret --from-file=csi-vsphere.conf --namespace=vmware-system-csi +``` diff --git a/docs/book/features/vsphere_csi_migration.md b/docs/book/features/vsphere_csi_migration.md new file mode 100644 index 0000000000..1033ce289a --- /dev/null +++ b/docs/book/features/vsphere_csi_migration.md @@ -0,0 +1,250 @@ + + +# vSphere CSI Driver - vSphere CSI Migration + +- [Introduction](#introduction) +- [Things to consider before turning on Migration](#consider-followings-before-turning-on-migration) +- [How to enable vSphere CSI Migration](#how-to-enable-vsphere-csi-migration) + +## Introduction + +**Note:** Feature to migrate in-tree vSphere volumes to CSI is released as **beta** with [v2.1.0](https://github.com/kubernetes-sigs/vsphere-csi-driver/releases/tag/v2.1.0). + +vSphere CSI driver and CNS bring in a lot of features that are not available in the in-tree vSphere volume plugin. + +Refer to the following feature comparisons table to know what is added in the vSphere CSI driver. + +| Feature | In-tree vSphere Volume Plugin | vSphere CSI Driver | +|---------|-------------------------------|--------------------| +| Block volume (`ReadWriteOnce` access mode) | Supported. Block volumes are backed by vmdks. | Supported. Block volumes are backed by vSphere Improved Virtual Disk(management layer on top of vmdk). | +| File volume (`ReadWriteMany`/`ReadOnlyMany` access modes) | Not Supported | Supported. File volumes are backed by vSAN file shares. | +| Dynamic provisioning(with and without SPBM) | Supported for block volumes only | Supported for block and file volumes | +| Static Provisioning | Supported for block volumes only | Supported for block and file volumes | +| Expand volume (Offline) | Not supported | Supported for block volumes | +| Storage vMotion of block volumes | Not supported | Not supported | +| vSphere UI integration(CNS dashboard, vSAN virtual objects, vSphere space) | Not supported | Supported | +| [Tanzu™ Kubernetes Grid™ Service](https://docs.vmware.com/en/VMware-vSphere/7.0/vmware-vsphere-with-kubernetes/GUID-152BE7D2-E227-4DAA-B527-557B564D9718.html) | Not supported | Supported | +| Tolerate Datacenter name, Datastore name and Storage policy name changes | Not Supported | supported | +| Volume Encryption | Not supported | Supported | +| Kubernetes Cluster spread across multiple vCenter Servers | Supported | Not supported | +| Kubernetes Cluster spread across multiple datacenters within a vCenter Server | Supported | Supported | +| Volume Topology(with `waitForFirstConsumer`) | Supported | Supported | +| Thick disk provisioning | Supported on all datastore types (vSAN, VVOL, VMFS and NFS) | Supported only on vSAN Datastore using Storage Policy Capability - `Object space reservation` | +| Raw Block Volume | Supported | Not supported | +| Inline volumes in Pod spec | Supported | Not supported | + +In addition to the above feature comparison, one of the most important things customers need to consider is that Kubernetes will deprecate In-tree vSphere volume plugin and it will be removed in the future Kubernetes releases. +Volumes provisioned using vSphere in-tree plugin do not get the additional new features supported by the vSphere CSI driver. + +Kubernetes has provided a seamless procedure to help migrate in-tree vSphere volumes to a vSphere CSI driver. After in-tree vSphere volumes migrated to vSphere CSI driver, all subsequent operations on migrated volumes are performed by the vSphere CSI driver. +Migrated vSphere volume will not get additional capabilities vSphere CSI driver supports. + +## Things to consider before turning on Migration + +- vSphere CSI Migration is released with `beta` feature-gate in Kubernetes 1.19. Refer [release note announcement](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.19.md#csi-migration---azuredisk-and-vsphere-beta). +- Kubernetes 1.19 release has deprecated vSAN raw policy parameters for the in-tree vSphere Volume plugin and these parameters will be removed in a future release. Refer [deprecation announcement](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.19.md#deprecation) +- Following vSphere in-tree StorageClass parameters will not be supported after enabling migration. + - `hostfailurestotolerate` + - `forceprovisioning` + - `cachereservation` + - `diskstripes` + - `objectspacereservation` + - `iopslimit` + - `diskformat` +- Storage Policy consumed by in-tree vSphere volume should not be renamed or deleted. Volume migration requires original storage policy used for provisioning the volume to be present on vCenter for registration of volume as Container Volume in vSphere. +- Datastore consumed by in-tree vSphere volume should not be renamed. Volume migration relies on the original datastore name present on the volume source for registration of volume as Container Volume in vSphere. +- For statically created vSphere in-tree Persistent Volume Claims and Persistent Volumes, make sure to add the following annotations before enabling migration. Statically provisioned in-tree vSphere volumes can't be migrated to CSI without adding these annotations. This also applies to new static in-tree PVs and PVCs created after the migration is enabled. + + Annotation on PV + + annotations: + pv.kubernetes.io/provisioned-by: kubernetes.io/vsphere-volume + + Annotation on PVC + + annotations: + volume.beta.kubernetes.io/storage-provisioner: kubernetes.io/vsphere-volume + +- vSphere CSI Driver does not support Provisioning `eagerzeroedthick` and `zeroedthick` volume. After the migration is enabled, when a new volume is requested using the in-tree provisioner and `diskformat` parameter set to `eagerzeroedthick` or `zeroedthick`, volume creation will be failed by vSphere CSI Driver. Post migration only supported value for `diskformat` parameter will be `thin`. Existing volumes created before the migration using disk format `eagerzeroedthick` or `zeroedthick` will be migrated to CSI. +- vSphere CSI Driver does not support raw vSAN policy parameters. After the migration is enabled, when a new volume is requested using in-tree provisioner and vSAN raw policy parameters, Volume Creation will be failed by vSphere CSI Driver. +- vSphere CSI Migration requires vSphere 7.0u1. Customers who have in-tree vSphere volumes must upgrade vSphere to 7.0u1. Customers who do not need to migrate in-tree vSphere volumes can use vSphere 67u3 and above. +- vSphere CSI driver does not support volumes formatted with the Windows file system. Migrated in-tree vSphere volumes using the windows file system can't be used with vSphere CSI driver. +- In-tree vSphere volume plugin is heavily relying on the name of the datastore set on the PV’s Source. After migration is enabled, Storage DRS or vmotion should not be enabled. If storage DRS moves disk from one datastore to another further volume operations may break. + +## How to enable vSphere CSI Migration + +In Kubernetes 1.19 release, vSphere CSI Migration is available with `beta` feature-gates. + +To try out vSphere CSI migration in beta for vSphere plugin, perform the following steps. + +1. Upgrade vSphere to 7.0u1. +2. Upgrade kubernetes to 1.19 release. +3. Ensure that your version of kubectl is also at 1.19 or later. +4. Install vSphere Cloud Provider Interface (CPI). Please follow guideline mentioned at https://vsphere-csi-driver.sigs.k8s.io/driver-deployment/prerequisites.html#vsphere_cpi +5. Install vSphere CSI Driver [v2.3.0](https://github.com/kubernetes-sigs/vsphere-csi-driver/releases/tag/v2.3.0) + - Make sure to enable csi-migration feature gate in the deployment yaml file. + + apiVersion: v1 + data: + "csi-migration": "true" + kind: ConfigMap + metadata: + name: internal-feature-states.csi.vsphere.vmware.com + namespace: vmware-system-csi + +6. Install admission webhook. + - vSphere CSI driver does not support provisioning of volume by specifying migration specific parameters in the StorageClass. + These parameters were added by vSphere CSI translation library, and should not be used in the storage class directly. + + Validating admission controller helps prevent user from creating or updating StorageClass using `csi.vsphere.vmware.com` as provisioner with these parameters. + + - `csimigration` + - `datastore-migrationparam` + - `diskformat-migrationparam` + - `hostfailurestotolerate-migrationparam` + - `forceprovisioning-migrationparam` + - `cachereservation-migrationparam` + - `diskstripes-migrationparam` + - `objectspacereservation-migrationparam` + - `iopslimit-migrationparam` + + This Validating admission controller also helps prevent user from creating or updating StorageClass using `kubernetes.io/vsphere-volume` as provisioner with `AllowVolumeExpansion` to `true`. + + - Pre-requisite: `kubectl`, `openssl` and `base64` commands should be pre-installed on the system from where we can invoke admission webhook installation scripts. + - Installation steps: + - Script is available to deploy the admission webhook and it is located at https://github.com/kubernetes-sigs/vsphere-csi-driver/tree/v2.3.0/manifests/vanilla on the repository. + - Download the scripts + + $ curl -O https://raw.githubusercontent.com/kubernetes-sigs/vsphere-csi-driver/v2.3.0/manifests/vanilla/generate-signed-webhook-certs.sh + + $ curl -O https://raw.githubusercontent.com/kubernetes-sigs/vsphere-csi-driver/v2.3.0/manifests/vanilla/create-validation-webhook.sh + + $ curl -O https://raw.githubusercontent.com/kubernetes-sigs/vsphere-csi-driver/v2.3.0/manifests/vanilla/validatingwebhook.yaml + + - Generate the self-signed certificate + + $ bash generate-signed-webhook-certs.sh + creating certs in tmpdir /tmp/tmp.jmZqh2bAwJ + Generating RSA private key, 2048 bit long modulus (2 primes) + ........................................+++++ + ............+++++ + e is 65537 (0x010001) + certificatesigningrequest.certificates.k8s.io "vsphere-webhook-svc.vmware-system-csi" deleted + Warning: certificates.k8s.io/v1beta1 CertificateSigningRequest is deprecated in v1.19+, unavailable in v1.22+; use certificates.k8s.io/v1 CertificateSigningRequest + certificatesigningrequest.certificates.k8s.io/vsphere-webhook-svc.vmware-system-csi created + NAME AGE SIGNERNAME REQUESTOR CONDITION + vsphere-webhook-svc.vmware-system-csi 0s kubernetes.io/legacy-unknown kubernetes-admin Pending + certificatesigningrequest.certificates.k8s.io/vsphere-webhook-svc.vmware-system-csi approved + secret/vsphere-webhook-certs configured + - Create the validation webhook + + $ bash create-validation-webhook.sh + service/vsphere-webhook-svc created + validatingwebhookconfiguration.admissionregistration.k8s.io/validation.csi.vsphere.vmware.com created + serviceaccount/vsphere-csi-webhook created + role.rbac.authorization.k8s.io/vsphere-csi-webhook-role created + rolebinding.rbac.authorization.k8s.io/vsphere-csi-webhook-role-binding created + deployment.apps/vsphere-csi-webhook created + +7. Enable feature flags `CSIMigration` and `CSIMigrationvSphere` + + - `CSIMigrationvSphere` flag enables shims and translation logic to route volume operations from the vSphere in-tree plugin to vSphere CSI plugin. Supports falling back to in-tree vSphere plugin if a node does not have a vSphere CSI plugin installed and configured. + - `CSIMigrationvSphere` requires `CSIMigration` feature flag to be enabled. This flag is enabling CSI migration on the Kubernetes Cluster. + + 7.1 Steps for the control plane node(s) + + - Enable feature flags `CSIMigration` and `CSIMigrationvSphere` on `kube-controller` and `kubelet` on all control plane nodes. + - update kube-controller-manager manifest file and following arguments. This file is generally available at `/etc/kubernetes/manifests/kube-controller-manager.yaml` + + `- --feature-gates=CSIMigration=true,CSIMigrationvSphere=true` + + - update kubelet configuration file and add following flags. This file is generally available at `/var/lib/kubelet/config.yaml` + + featureGates: + CSIMigration: true + CSIMigrationvSphere: true + + - Restart the kubelet on the control plane nodes using the command: + + systemctl restart kubelet + + - Verify that the kubelet is functioning correctly using the following command: + + systemctl status kubelet + + - If there are any issues with the kubelet, check the logs on the control plane node using the following command: + + journalctl -xe + + 7.2 Steps for the worker node(s) + + - Enable feature flags `CSIMigration` and `CSIMigrationvSphere` on `kubelet` on all workload nodes. Please note that before changing the configuration on the Kubelet on each node we **must drain** the node (remove running application workloads). + - Node drain example. + + $ kubectl drain k8s-node1 --force --ignore-daemonsets + node/k8s-node1 cordoned + WARNING: deleting Pods not managed by ReplicationController, ReplicaSet, Job, DaemonSet or StatefulSet: default/vcppod; ignoring DaemonSet-managed Pods: kube-system/kube-flannel-ds-amd64-gs7fr, kube-system/kube-proxy-rbjx4, vmware-system-csi/vsphere-csi-node-fh9f6 + evicting pod default/vcppod + pod/vcppod evicted + node/k8s-node1 evicted + + - After migration is enabled, make sure `csinodes` instance for the node is updated with `storage.alpha.kubernetes.io/migrated-plugins` annotation. + + $ kubectl describe csinodes k8s-node1 + Name: k8s-node1 + Labels: + Annotations: storage.alpha.kubernetes.io/migrated-plugins: kubernetes.io/vsphere-volume + CreationTimestamp: Wed, 29 Apr 2020 17:51:35 -0700 + Spec: + Drivers: + csi.vsphere.vmware.com: + Node ID: k8s-node1 + Events: + + - Restart the kubelet on the workload nodes using the command: + + systemctl restart kubelet + + - Verify that the kubelet is functioning correctly using the following command: + + systemctl status kubelet + + - If there are any issues with the kubelet, check the logs on the workload node using the following command: + + journalctl -xe + + - Once the kubelet is restarted, `uncordon` the node so that it can be used for scheduling workloads: + + kubectl uncordon k8s-node1 + + - Repeat these steps for all workload nodes in the Kubernetes Cluster. + +8. There is also an optional `CSIMigrationvSphereComplete` flag that can be enabled if all the nodes have CSI migration enabled. `CSIMigrationvSphereComplete` helps stop registering the vSphere in-tree plugin in kubelet and volume controllers and enables shims and translation logic to route volume operations from the vSphere in-tree plugin to vSphere CSI plugin. `CSIMigrationvSphereComplete` flag requires `CSIMigration` and `CSIMigrationvSphere` feature flags enabled and vSphere CSI plugin installed and configured on all nodes in the cluster. + +9. Verify vSphere in-tree PVCs and PVs are migrated to vSphere CSI driver, verify `pv.kubernetes.io/migrated-to: csi.vsphere.vmware.com` annotations are present on PVCs and PVs. + + Annotations on PVCs + + Annotations: pv.kubernetes.io/bind-completed: yes + pv.kubernetes.io/bound-by-controller: yes + pv.kubernetes.io/migrated-to: csi.vsphere.vmware.com + volume.beta.kubernetes.io/storage-provisioner: kubernetes.io/vsphere-volume + + Annotations on PVs + + Annotations: kubernetes.io/createdby: vsphere-volume-dynamic-provisioner + pv.kubernetes.io/bound-by-controller: yes + pv.kubernetes.io/migrated-to: csi.vsphere.vmware.com + pv.kubernetes.io/provisioned-by: kubernetes.io/vsphere-volume + + New in-tree vSphere volumes created by vSphere CSI driver after migration is enabled, can be identified by following annotations. PV spec will still hold vSphere Volume Path, so in case when migration needs to be disabled, provisioned volume can be used by the in-tree vSphere plugin. + + Annotations on PVCs + + Annotations: pv.kubernetes.io/bind-completed: yes + pv.kubernetes.io/bound-by-controller: yes + volume.beta.kubernetes.io/storage-provisioner: csi.vsphere.vmware.com + + Annotations on PVs + + Annotations: pv.kubernetes.io/provisioned-by: csi.vsphere.vmware.com diff --git a/example/vanilla-k8s-RWO-Block-Volumes/example-raw-block-restore.yaml b/example/vanilla-k8s-RWO-Block-Volumes/example-raw-block-restore.yaml index 39392540de..48121b5375 100644 --- a/example/vanilla-k8s-RWO-Block-Volumes/example-raw-block-restore.yaml +++ b/example/vanilla-k8s-RWO-Block-Volumes/example-raw-block-restore.yaml @@ -8,7 +8,6 @@ spec: name: example-raw-block-snapshot kind: VolumeSnapshot apiGroup: snapshot.storage.k8s.io - volumeMode: Block accessModes: - ReadWriteOnce resources: diff --git a/example/vanilla-k8s-RWO-filesystem-volumes/example-sc-WaitForFirstConsumer-restricted.yaml b/example/vanilla-k8s-RWO-filesystem-volumes/example-sc-WaitForFirstConsumer-restricted.yaml index 7688874526..3e94186961 100644 --- a/example/vanilla-k8s-RWO-filesystem-volumes/example-sc-WaitForFirstConsumer-restricted.yaml +++ b/example/vanilla-k8s-RWO-filesystem-volumes/example-sc-WaitForFirstConsumer-restricted.yaml @@ -9,10 +9,10 @@ parameters: storagepolicyname: "vSAN Default Storage Policy" # Optional Parameter allowedTopologies: - matchLabelExpressions: - - key: topology.csi.vmware.com/k8s-zone + - key: topology.kubernetes.io/zone values: - us-west-CA - us-west-WA - - key: topology.csi.vmware.com/k8s-region + - key: topology.kubernetes.io/region values: - us-west diff --git a/example/vanilla-k8s-RWO-filesystem-volumes/example-sc-multiple-zones.yaml b/example/vanilla-k8s-RWO-filesystem-volumes/example-sc-multiple-zones.yaml index 301f605d4b..5ce7e4d172 100644 --- a/example/vanilla-k8s-RWO-filesystem-volumes/example-sc-multiple-zones.yaml +++ b/example/vanilla-k8s-RWO-filesystem-volumes/example-sc-multiple-zones.yaml @@ -8,10 +8,10 @@ parameters: storagepolicyname: "vSAN Default Storage Policy" # Optional Parameter allowedTopologies: - matchLabelExpressions: - - key: topology.csi.vmware.com/k8s-zone + - key: topology.kubernetes.io/zone values: - us-west-WA - us-west-CA - - key: topology.csi.vmware.com/k8s-region + - key: topology.kubernetes.io/region values: - us-west diff --git a/example/vanilla-k8s-RWO-filesystem-volumes/example-sc-single-zone.yaml b/example/vanilla-k8s-RWO-filesystem-volumes/example-sc-single-zone.yaml index ad739a27d2..5a94e738ab 100644 --- a/example/vanilla-k8s-RWO-filesystem-volumes/example-sc-single-zone.yaml +++ b/example/vanilla-k8s-RWO-filesystem-volumes/example-sc-single-zone.yaml @@ -8,9 +8,9 @@ parameters: storagepolicyname: "vSAN Default Storage Policy" # Optional Parameter allowedTopologies: - matchLabelExpressions: - - key: topology.csi.vmware.com/k8s-zone + - key: topology.kubernetes.io/zone values: - us-west-CA - - key: topology.csi.vmware.com/k8s-region + - key: topology.kubernetes.io/region values: - us-west diff --git a/hack/release.sh b/hack/release.sh index d45498eeb5..4a57241c53 100755 --- a/hack/release.sh +++ b/hack/release.sh @@ -56,8 +56,8 @@ BUILD_RELEASE_TYPE="${BUILD_RELEASE_TYPE:-}" # CUSTOM_REPO_FOR_GOLANG can be used to pass custom repository for golang builder image. # Please ensure it ends with a '/'. -# Example: CUSTOM_REPO_FOR_GOLANG=/dockerhub-proxy-cache/library/ -GOLANG_IMAGE=${CUSTOM_REPO_FOR_GOLANG:-}golang:1.20 +# Example: CUSTOM_REPO_FOR_GOLANG=harbor-repo.vmware.com/dockerhub-proxy-cache/library/ +GOLANG_IMAGE=${CUSTOM_REPO_FOR_GOLANG:-}golang:1.19 ARCH=amd64 OSVERSION=1809 diff --git a/hack/run-e2e-test.sh b/hack/run-e2e-test.sh index caaaebe855..74d3fe11a9 100755 --- a/hack/run-e2e-test.sh +++ b/hack/run-e2e-test.sh @@ -68,10 +68,6 @@ then fi OPTS+=(-p) ginkgo -mod=mod "${OPTS[@]}" --focus="csi-block-vanilla-parallelized" tests/e2e -elif [ "$FOCUS" == "csi-block-vanilla-parallelized" ] -then - OPTS+=(-p) - ginkgo -mod=mod "${OPTS[@]}" --focus="csi-block-vanilla-parallelized" tests/e2e else ginkgo -mod=mod "${OPTS[@]}" --focus="$FOCUS" tests/e2e fi diff --git a/images/ci/Dockerfile b/images/ci/Dockerfile index dff464355d..2a44b5e964 100644 --- a/images/ci/Dockerfile +++ b/images/ci/Dockerfile @@ -17,7 +17,7 @@ ################################################################################ # The golang image is used to create the project's module and build caches # and is also the image on which this image is based. -ARG GOLANG_IMAGE=golang:1.20 +ARG GOLANG_IMAGE=golang:1.19 ################################################################################ ## GO MOD CACHE STAGE ## diff --git a/images/ci/e2e/Dockerfile b/images/ci/e2e/Dockerfile deleted file mode 100644 index 6b45b97921..0000000000 --- a/images/ci/e2e/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2023 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# To build an image from this docker file follow the below steps -# cd to images/ci/e2e folder -# docker build -t --platform=Linux/x86_64 -f Dockerfile . -# docker tag / -# docker push / - -FROM --platform=linux/amd64 golang:1.19 - -RUN apt-get update && curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && \ - install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl - -RUN echo "Downloading and installing govc..." && \ - wget -qO- https://github.com/vmware/govmomi/releases/download/v0.30.2/govc_Linux_x86_64.tar.gz | tar -C /usr/local/bin -xvzf - govc && \ - chmod +x /usr/local/bin/govc - -RUN echo "Updating and installing sshpass..." && \ - apt-get update -y && \ - apt-get install -y sshpass diff --git a/images/driver/Dockerfile b/images/driver/Dockerfile index 5779ac1c77..201d7957fa 100644 --- a/images/driver/Dockerfile +++ b/images/driver/Dockerfile @@ -16,7 +16,7 @@ ## BUILD ARGS ## ################################################################################ # This build arg allows the specification of a custom Golang image. -ARG GOLANG_IMAGE=golang:1.20 +ARG GOLANG_IMAGE=golang:1.19 # This build arg allows the specification of a custom base image. ARG BASE_IMAGE=gcr.io/cloud-provider-vsphere/extra/csi-driver-base:latest diff --git a/images/syncer/Dockerfile b/images/syncer/Dockerfile index 6b05236070..7e64be826f 100644 --- a/images/syncer/Dockerfile +++ b/images/syncer/Dockerfile @@ -14,7 +14,7 @@ ## BUILD ARGS ## ################################################################################ # This build arg allows the specification of a custom Golang image. -ARG GOLANG_IMAGE=golang:1.20 +ARG GOLANG_IMAGE=golang:1.19 # This build arg allows the specification of a custom base image. ARG BASE_IMAGE=gcr.io/cloud-provider-vsphere/extra/csi-driver-base:latest diff --git a/images/windows/driver/Dockerfile b/images/windows/driver/Dockerfile index 430b8bbf41..5edd043928 100644 --- a/images/windows/driver/Dockerfile +++ b/images/windows/driver/Dockerfile @@ -16,7 +16,7 @@ ## BUILD ARGS ## ################################################################################ # This build arg allows the specification of a custom Golang image. -ARG GOLANG_IMAGE=golang:1.20 +ARG GOLANG_IMAGE=golang:1.19 ARG OSVERSION ARG ARCH=amd64 diff --git a/manifests/guestcluster/1.22/pvcsi.yaml b/manifests/guestcluster/1.22/pvcsi.yaml index 9c5f1bd43c..acb5c867d6 100644 --- a/manifests/guestcluster/1.22/pvcsi.yaml +++ b/manifests/guestcluster/1.22/pvcsi.yaml @@ -62,7 +62,7 @@ rules: verbs: ["get", "update", "watch", "list"] - apiGroups: [ "snapshot.storage.k8s.io" ] resources: [ "volumesnapshots" ] - verbs: [ "get", "list", "patch" ] + verbs: [ "get", "list" ] - apiGroups: [ "snapshot.storage.k8s.io" ] resources: [ "volumesnapshotclasses" ] verbs: [ "watch", "get", "list" ] diff --git a/manifests/guestcluster/1.23/pvcsi.yaml b/manifests/guestcluster/1.23/pvcsi.yaml index 4b8fcbd2f1..309a2b6b0b 100644 --- a/manifests/guestcluster/1.23/pvcsi.yaml +++ b/manifests/guestcluster/1.23/pvcsi.yaml @@ -62,7 +62,7 @@ rules: verbs: ["get", "update", "watch", "list"] - apiGroups: [ "snapshot.storage.k8s.io" ] resources: [ "volumesnapshots" ] - verbs: [ "get", "list", "patch" ] + verbs: [ "get", "list" ] - apiGroups: [ "snapshot.storage.k8s.io" ] resources: [ "volumesnapshotclasses" ] verbs: [ "watch", "get", "list" ] @@ -208,8 +208,6 @@ spec: value: /etc/cloud/pvcsi-config/cns-csi.conf - name: PROVISION_TIMEOUT_MINUTES value: "4" - - name: SNAPSHOT_TIMEOUT_MINUTES - value: "4" - name: ATTACHER_TIMEOUT_MINUTES value: "4" - name: RESIZE_TIMEOUT_MINUTES diff --git a/manifests/guestcluster/1.24/pvcsi.yaml b/manifests/guestcluster/1.24/pvcsi.yaml index 6516505eb2..68f2d449d8 100644 --- a/manifests/guestcluster/1.24/pvcsi.yaml +++ b/manifests/guestcluster/1.24/pvcsi.yaml @@ -62,7 +62,7 @@ rules: verbs: ["get", "update", "watch", "list"] - apiGroups: [ "snapshot.storage.k8s.io" ] resources: [ "volumesnapshots" ] - verbs: [ "get", "list", "patch" ] + verbs: [ "get", "list" ] - apiGroups: [ "snapshot.storage.k8s.io" ] resources: [ "volumesnapshotclasses" ] verbs: [ "watch", "get", "list" ] @@ -211,8 +211,6 @@ spec: value: /etc/cloud/pvcsi-config/cns-csi.conf - name: PROVISION_TIMEOUT_MINUTES value: "4" - - name: SNAPSHOT_TIMEOUT_MINUTES - value: "4" - name: ATTACHER_TIMEOUT_MINUTES value: "4" - name: RESIZE_TIMEOUT_MINUTES @@ -231,10 +229,6 @@ spec: value: {{ .PVCSINamespace }} - name: X_CSI_SERIAL_VOL_ACCESS_TIMEOUT value: 3m - securityContext: - runAsNonRoot: true - runAsUser: 65532 - runAsGroup: 65532 volumeMounts: - mountPath: /etc/cloud/pvcsi-provider name: pvcsi-provider-volume @@ -271,10 +265,6 @@ spec: value: "PRODUCTION" # Options: DEVELOPMENT, PRODUCTION - name: CSI_NAMESPACE value: {{ .PVCSINamespace }} - securityContext: - runAsNonRoot: true - runAsUser: 65532 - runAsGroup: 65532 volumeMounts: - mountPath: /etc/cloud/pvcsi-provider name: pvcsi-provider-volume @@ -332,24 +322,6 @@ spec: volumeMounts: - mountPath: /csi name: socket-dir - - name: csi-snapshotter - image: vmware.io/csi-snapshotter/csi-snapshotter: - args: - - "--v=4" - - "--timeout=300s" - - "--csi-address=$(ADDRESS)" - - "--leader-election" - - "--kube-api-qps=100" - - "--kube-api-burst=100" - - "--leader-election-lease-duration=120s" - - "--leader-election-renew-deadline=60s" - - "--leader-election-retry-period=30s" - env: - - name: ADDRESS - value: /csi/csi.sock - volumeMounts: - - mountPath: /csi - name: socket-dir volumes: - name: pvcsi-provider-volume secret: @@ -521,7 +493,7 @@ data: "online-volume-extend": "true" "file-volume": "true" "csi-sv-feature-states-replication": "false" - "block-volume-snapshot": "true" + "block-volume-snapshot": "false" "tkgs-ha": "true" "cnsmgr-suspend-create-volume": "true" kind: ConfigMap diff --git a/manifests/guestcluster/1.25/pvcsi.yaml b/manifests/guestcluster/1.25/pvcsi.yaml index e591f5c456..e0dd975276 100644 --- a/manifests/guestcluster/1.25/pvcsi.yaml +++ b/manifests/guestcluster/1.25/pvcsi.yaml @@ -243,10 +243,6 @@ spec: value: {{ .PVCSINamespace }} - name: X_CSI_SERIAL_VOL_ACCESS_TIMEOUT value: 3m - securityContext: - runAsNonRoot: true - runAsUser: 65532 - runAsGroup: 65532 volumeMounts: - mountPath: /etc/cloud/pvcsi-provider name: pvcsi-provider-volume @@ -283,10 +279,6 @@ spec: value: "PRODUCTION" # Options: DEVELOPMENT, PRODUCTION - name: CSI_NAMESPACE value: {{ .PVCSINamespace }} - securityContext: - runAsNonRoot: true - runAsUser: 65532 - runAsGroup: 65532 volumeMounts: - mountPath: /etc/cloud/pvcsi-provider name: pvcsi-provider-volume @@ -344,24 +336,6 @@ spec: volumeMounts: - mountPath: /csi name: socket-dir - - name: csi-snapshotter - image: vmware.io/csi-snapshotter/csi-snapshotter: - args: - - "--v=4" - - "--timeout=300s" - - "--csi-address=$(ADDRESS)" - - "--leader-election" - - "--kube-api-qps=100" - - "--kube-api-burst=100" - - "--leader-election-lease-duration=120s" - - "--leader-election-renew-deadline=60s" - - "--leader-election-retry-period=30s" - env: - - name: ADDRESS - value: /csi/csi.sock - volumeMounts: - - mountPath: /csi - name: socket-dir volumes: - name: pvcsi-provider-volume secret: @@ -531,7 +505,7 @@ data: "online-volume-extend": "true" "file-volume": "true" "csi-sv-feature-states-replication": "false" # Do not enable for guest cluster, Refer PR#2386 for details - "block-volume-snapshot": "true" + "block-volume-snapshot": "false" "tkgs-ha": "true" "cnsmgr-suspend-create-volume": "true" kind: ConfigMap diff --git a/manifests/guestcluster/1.26/pvcsi.yaml b/manifests/guestcluster/1.26/pvcsi.yaml index e591f5c456..e0dd975276 100644 --- a/manifests/guestcluster/1.26/pvcsi.yaml +++ b/manifests/guestcluster/1.26/pvcsi.yaml @@ -243,10 +243,6 @@ spec: value: {{ .PVCSINamespace }} - name: X_CSI_SERIAL_VOL_ACCESS_TIMEOUT value: 3m - securityContext: - runAsNonRoot: true - runAsUser: 65532 - runAsGroup: 65532 volumeMounts: - mountPath: /etc/cloud/pvcsi-provider name: pvcsi-provider-volume @@ -283,10 +279,6 @@ spec: value: "PRODUCTION" # Options: DEVELOPMENT, PRODUCTION - name: CSI_NAMESPACE value: {{ .PVCSINamespace }} - securityContext: - runAsNonRoot: true - runAsUser: 65532 - runAsGroup: 65532 volumeMounts: - mountPath: /etc/cloud/pvcsi-provider name: pvcsi-provider-volume @@ -344,24 +336,6 @@ spec: volumeMounts: - mountPath: /csi name: socket-dir - - name: csi-snapshotter - image: vmware.io/csi-snapshotter/csi-snapshotter: - args: - - "--v=4" - - "--timeout=300s" - - "--csi-address=$(ADDRESS)" - - "--leader-election" - - "--kube-api-qps=100" - - "--kube-api-burst=100" - - "--leader-election-lease-duration=120s" - - "--leader-election-renew-deadline=60s" - - "--leader-election-retry-period=30s" - env: - - name: ADDRESS - value: /csi/csi.sock - volumeMounts: - - mountPath: /csi - name: socket-dir volumes: - name: pvcsi-provider-volume secret: @@ -531,7 +505,7 @@ data: "online-volume-extend": "true" "file-volume": "true" "csi-sv-feature-states-replication": "false" # Do not enable for guest cluster, Refer PR#2386 for details - "block-volume-snapshot": "true" + "block-volume-snapshot": "false" "tkgs-ha": "true" "cnsmgr-suspend-create-volume": "true" kind: ConfigMap diff --git a/manifests/guestcluster/1.27/pvcsi.yaml b/manifests/guestcluster/1.27/pvcsi.yaml index e591f5c456..e0dd975276 100644 --- a/manifests/guestcluster/1.27/pvcsi.yaml +++ b/manifests/guestcluster/1.27/pvcsi.yaml @@ -243,10 +243,6 @@ spec: value: {{ .PVCSINamespace }} - name: X_CSI_SERIAL_VOL_ACCESS_TIMEOUT value: 3m - securityContext: - runAsNonRoot: true - runAsUser: 65532 - runAsGroup: 65532 volumeMounts: - mountPath: /etc/cloud/pvcsi-provider name: pvcsi-provider-volume @@ -283,10 +279,6 @@ spec: value: "PRODUCTION" # Options: DEVELOPMENT, PRODUCTION - name: CSI_NAMESPACE value: {{ .PVCSINamespace }} - securityContext: - runAsNonRoot: true - runAsUser: 65532 - runAsGroup: 65532 volumeMounts: - mountPath: /etc/cloud/pvcsi-provider name: pvcsi-provider-volume @@ -344,24 +336,6 @@ spec: volumeMounts: - mountPath: /csi name: socket-dir - - name: csi-snapshotter - image: vmware.io/csi-snapshotter/csi-snapshotter: - args: - - "--v=4" - - "--timeout=300s" - - "--csi-address=$(ADDRESS)" - - "--leader-election" - - "--kube-api-qps=100" - - "--kube-api-burst=100" - - "--leader-election-lease-duration=120s" - - "--leader-election-renew-deadline=60s" - - "--leader-election-retry-period=30s" - env: - - name: ADDRESS - value: /csi/csi.sock - volumeMounts: - - mountPath: /csi - name: socket-dir volumes: - name: pvcsi-provider-volume secret: @@ -531,7 +505,7 @@ data: "online-volume-extend": "true" "file-volume": "true" "csi-sv-feature-states-replication": "false" # Do not enable for guest cluster, Refer PR#2386 for details - "block-volume-snapshot": "true" + "block-volume-snapshot": "false" "tkgs-ha": "true" "cnsmgr-suspend-create-volume": "true" kind: ConfigMap diff --git a/manifests/supervisorcluster/1.21/cns-csi.yaml b/manifests/supervisorcluster/1.21/cns-csi.yaml index c64e35fdfb..f682aa4319 100644 --- a/manifests/supervisorcluster/1.21/cns-csi.yaml +++ b/manifests/supervisorcluster/1.21/cns-csi.yaml @@ -503,14 +503,9 @@ webhooks: rules: - apiGroups: [""] apiVersions: ["v1", "v1beta1"] - operations: ["CREATE", "UPDATE", "DELETE"] + operations: ["CREATE", "UPDATE"] resources: ["persistentvolumeclaims"] scope: "Namespaced" - - apiGroups: ["snapshot.storage.k8s.io"] - apiVersions: ["v1"] - operations: ["CREATE"] - resources: ["volumesnapshots"] - scope: "Namespaced" sideEffects: None admissionReviewVersions: ["v1"] failurePolicy: Fail @@ -523,9 +518,6 @@ rules: - apiGroups: [""] resources: ["persistentvolumes", "persistentvolumeclaims"] verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/manifests/supervisorcluster/1.22/cns-csi.yaml b/manifests/supervisorcluster/1.22/cns-csi.yaml index 4ea49ee18b..1958cf6249 100644 --- a/manifests/supervisorcluster/1.22/cns-csi.yaml +++ b/manifests/supervisorcluster/1.22/cns-csi.yaml @@ -80,7 +80,7 @@ rules: verbs: ["get", "list", "watch"] - apiGroups: [ "snapshot.storage.k8s.io" ] resources: [ "volumesnapshots" ] - verbs: [ "get", "list", "patch" ] + verbs: [ "get", "list" ] - apiGroups: [ "snapshot.storage.k8s.io" ] resources: [ "volumesnapshotclasses" ] verbs: [ "watch", "get", "list" ] @@ -512,14 +512,9 @@ webhooks: rules: - apiGroups: [""] apiVersions: ["v1", "v1beta1"] - operations: ["CREATE", "UPDATE", "DELETE"] + operations: ["CREATE", "UPDATE"] resources: ["persistentvolumeclaims"] scope: "Namespaced" - - apiGroups: ["snapshot.storage.k8s.io"] - apiVersions: ["v1"] - operations: ["CREATE"] - resources: ["volumesnapshots"] - scope: "Namespaced" sideEffects: None admissionReviewVersions: ["v1"] failurePolicy: Fail @@ -532,9 +527,6 @@ rules: - apiGroups: [""] resources: ["persistentvolumes", "persistentvolumeclaims"] verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/manifests/supervisorcluster/1.23/cns-csi.yaml b/manifests/supervisorcluster/1.23/cns-csi.yaml index 3689c52c50..1ea3ce9d0c 100644 --- a/manifests/supervisorcluster/1.23/cns-csi.yaml +++ b/manifests/supervisorcluster/1.23/cns-csi.yaml @@ -80,7 +80,7 @@ rules: verbs: ["get", "list", "watch"] - apiGroups: [ "snapshot.storage.k8s.io" ] resources: [ "volumesnapshots" ] - verbs: [ "get", "list", "patch" ] + verbs: [ "get", "list" ] - apiGroups: [ "snapshot.storage.k8s.io" ] resources: [ "volumesnapshotclasses" ] verbs: [ "watch", "get", "list" ] @@ -213,7 +213,7 @@ spec: priorityClassName: system-node-critical containers: - name: csi-provisioner - image: localhost:5000/vmware/csi-provisioner/csi-provisioner:v3.4.0_vmware.1 + image: localhost:5000/vmware/csi-provisioner/csi-provisioner:v3.3.0_vmware.1 args: - "--v=4" - "--timeout=300s" @@ -248,7 +248,7 @@ spec: - name: socket-dir mountPath: /csi - name: csi-attacher - image: localhost:5000/vmware.io/csi-attacher:v4.3.0_vmware.1 + image: localhost:5000/vmware.io/csi-attacher:v4.0.0_vmware.1 args: - "--v=4" - "--timeout=300s" @@ -272,7 +272,7 @@ spec: - name: socket-dir mountPath: /csi - name: csi-resizer - image: localhost:5000/vmware/kubernetes-csi_external-resizer/kubernetes-csi_external-resizer:v1.8.0_vmware.1 + image: localhost:5000/vmware/kubernetes-csi_external-resizer/kubernetes-csi_external-resizer:v1.6.0_vmware.1 imagePullPolicy: IfNotPresent args: - --v=4 @@ -346,7 +346,7 @@ spec: - mountPath: /etc/vmware/wcp/tls/ name: host-vmca - name: liveness-probe - image: localhost:5000/vmware.io/csi-livenessprobe:v2.10.0_vmware.1 + image: localhost:5000/vmware.io/csi-livenessprobe:v2.7.0_vmware.1 args: - "--csi-address=/csi/csi.sock" volumeMounts: @@ -395,23 +395,6 @@ spec: readOnly: true - mountPath: /etc/vmware/wcp/tls/ name: host-vmca - - name: csi-snapshotter - image: localhost:5000/vmware.io/csi-snapshotter:v6.1.0_vmware.2 - args: - - "--v=4" - - "--timeout=300s" - - "--csi-address=$(ADDRESS)" - - "--leader-election" - - "--kube-api-qps=100" - - "--kube-api-burst=100" - - "--extra-create-metadata" - env: - - name: ADDRESS - value: /csi/csi.sock - imagePullPolicy: "IfNotPresent" - volumeMounts: - - mountPath: /csi - name: socket-dir volumes: - name: vsphere-config-volume secret: @@ -443,12 +426,12 @@ data: "fake-attach": "true" "async-query-volume": "true" "improved-csi-idempotency": "true" - "block-volume-snapshot": "true" + "block-volume-snapshot": "false" "sibling-replica-bound-pvc-check": "true" "tkgs-ha": "true" - "list-volumes": "true" + "list-volumes": "false" "cnsmgr-suspend-create-volume": "true" - "listview-tasks": "true" + "listview-tasks": "false" kind: ConfigMap metadata: name: csi-feature-states @@ -533,14 +516,9 @@ webhooks: rules: - apiGroups: [""] apiVersions: ["v1", "v1beta1"] - operations: ["CREATE", "UPDATE", "DELETE"] + operations: ["CREATE", "UPDATE"] resources: ["persistentvolumeclaims"] scope: "Namespaced" - - apiGroups: ["snapshot.storage.k8s.io"] - apiVersions: ["v1"] - operations: ["CREATE"] - resources: ["volumesnapshots"] - scope: "Namespaced" sideEffects: None admissionReviewVersions: ["v1"] failurePolicy: Fail @@ -553,9 +531,6 @@ rules: - apiGroups: [""] resources: ["persistentvolumes", "persistentvolumeclaims"] verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/manifests/supervisorcluster/1.23/kustomization.yaml b/manifests/supervisorcluster/1.23/kustomization.yaml deleted file mode 100644 index fb9558639f..0000000000 --- a/manifests/supervisorcluster/1.23/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: - - cns-csi.yaml diff --git a/manifests/supervisorcluster/1.24/cns-csi.yaml b/manifests/supervisorcluster/1.24/cns-csi.yaml index 4d0f8c6821..4dc581f98b 100644 --- a/manifests/supervisorcluster/1.24/cns-csi.yaml +++ b/manifests/supervisorcluster/1.24/cns-csi.yaml @@ -80,7 +80,7 @@ rules: verbs: ["get", "list", "watch"] - apiGroups: [ "snapshot.storage.k8s.io" ] resources: [ "volumesnapshots" ] - verbs: [ "get", "list", "patch" ] + verbs: [ "get", "list" ] - apiGroups: [ "snapshot.storage.k8s.io" ] resources: [ "volumesnapshotclasses" ] verbs: [ "watch", "get", "list" ] @@ -216,7 +216,7 @@ spec: priorityClassName: system-node-critical containers: - name: csi-provisioner - image: localhost:5000/vmware/csi-provisioner/csi-provisioner:v3.4.0_vmware.1 + image: localhost:5000/vmware/csi-provisioner/csi-provisioner:v3.3.0_vmware.1 args: - "--v=4" - "--timeout=300s" @@ -251,7 +251,7 @@ spec: - name: socket-dir mountPath: /csi - name: csi-attacher - image: localhost:5000/vmware.io/csi-attacher:v4.3.0_vmware.1 + image: localhost:5000/vmware.io/csi-attacher:v4.0.0_vmware.1 args: - "--v=4" - "--timeout=300s" @@ -275,7 +275,7 @@ spec: - name: socket-dir mountPath: /csi - name: csi-resizer - image: localhost:5000/vmware/kubernetes-csi_external-resizer/kubernetes-csi_external-resizer:v1.8.0_vmware.1 + image: localhost:5000/vmware/kubernetes-csi_external-resizer/kubernetes-csi_external-resizer:v1.6.0_vmware.1 imagePullPolicy: IfNotPresent args: - --v=4 @@ -349,7 +349,7 @@ spec: - mountPath: /etc/vmware/wcp/tls/ name: host-vmca - name: liveness-probe - image: localhost:5000/vmware.io/csi-livenessprobe:v2.10.0_vmware.1 + image: localhost:5000/vmware.io/csi-livenessprobe:v2.7.0_vmware.1 args: - "--csi-address=/csi/csi.sock" volumeMounts: @@ -398,23 +398,6 @@ spec: readOnly: true - mountPath: /etc/vmware/wcp/tls/ name: host-vmca - - name: csi-snapshotter - image: localhost:5000/vmware.io/csi-snapshotter:v6.1.0_vmware.2 - args: - - "--v=4" - - "--timeout=300s" - - "--csi-address=$(ADDRESS)" - - "--leader-election" - - "--kube-api-qps=100" - - "--kube-api-burst=100" - - "--extra-create-metadata" - env: - - name: ADDRESS - value: /csi/csi.sock - imagePullPolicy: "IfNotPresent" - volumeMounts: - - mountPath: /csi - name: socket-dir volumes: - name: vsphere-config-volume secret: @@ -446,12 +429,12 @@ data: "fake-attach": "true" "async-query-volume": "true" "improved-csi-idempotency": "true" - "block-volume-snapshot": "true" + "block-volume-snapshot": "false" "sibling-replica-bound-pvc-check": "true" "tkgs-ha": "true" - "list-volumes": "true" + "list-volumes": "false" "cnsmgr-suspend-create-volume": "true" - "listview-tasks": "true" + "listview-tasks": "false" kind: ConfigMap metadata: name: csi-feature-states @@ -536,14 +519,9 @@ webhooks: rules: - apiGroups: [""] apiVersions: ["v1", "v1beta1"] - operations: ["CREATE", "UPDATE", "DELETE"] + operations: ["CREATE", "UPDATE"] resources: ["persistentvolumeclaims"] scope: "Namespaced" - - apiGroups: ["snapshot.storage.k8s.io"] - apiVersions: ["v1"] - operations: ["CREATE"] - resources: ["volumesnapshots"] - scope: "Namespaced" sideEffects: None admissionReviewVersions: ["v1"] failurePolicy: Fail @@ -556,9 +534,6 @@ rules: - apiGroups: [""] resources: ["persistentvolumes", "persistentvolumeclaims"] verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/manifests/supervisorcluster/1.24/kustomization.yaml b/manifests/supervisorcluster/1.24/kustomization.yaml deleted file mode 100644 index fb9558639f..0000000000 --- a/manifests/supervisorcluster/1.24/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: - - cns-csi.yaml diff --git a/manifests/supervisorcluster/1.25/cns-csi.yaml b/manifests/supervisorcluster/1.25/cns-csi.yaml index e38a5cc0d4..fe0250dd63 100644 --- a/manifests/supervisorcluster/1.25/cns-csi.yaml +++ b/manifests/supervisorcluster/1.25/cns-csi.yaml @@ -80,7 +80,7 @@ rules: verbs: ["get", "list", "watch"] - apiGroups: [ "snapshot.storage.k8s.io" ] resources: [ "volumesnapshots" ] - verbs: [ "get", "list", "patch" ] + verbs: [ "get", "list" ] - apiGroups: [ "snapshot.storage.k8s.io" ] resources: [ "volumesnapshotclasses" ] verbs: [ "watch", "get", "list" ] @@ -213,7 +213,7 @@ spec: priorityClassName: system-node-critical containers: - name: csi-provisioner - image: localhost:5000/vmware/csi-provisioner/csi-provisioner:v3.4.0_vmware.1 + image: localhost:5000/vmware/csi-provisioner/csi-provisioner:v3.3.0_vmware.1 args: - "--v=4" - "--timeout=300s" @@ -248,7 +248,7 @@ spec: - name: socket-dir mountPath: /csi - name: csi-attacher - image: localhost:5000/vmware.io/csi-attacher:v4.3.0_vmware.1 + image: localhost:5000/vmware.io/csi-attacher:v4.0.0_vmware.1 args: - "--v=4" - "--timeout=300s" @@ -272,7 +272,7 @@ spec: - name: socket-dir mountPath: /csi - name: csi-resizer - image: localhost:5000/vmware/kubernetes-csi_external-resizer/kubernetes-csi_external-resizer:v1.8.0_vmware.1 + image: localhost:5000/vmware/kubernetes-csi_external-resizer/kubernetes-csi_external-resizer:v1.6.0_vmware.1 imagePullPolicy: IfNotPresent args: - --v=4 @@ -346,7 +346,7 @@ spec: - mountPath: /etc/vmware/wcp/tls/ name: host-vmca - name: liveness-probe - image: localhost:5000/vmware.io/csi-livenessprobe:v2.10.0_vmware.1 + image: localhost:5000/vmware.io/csi-livenessprobe:v2.7.0_vmware.1 args: - "--csi-address=/csi/csi.sock" volumeMounts: @@ -395,23 +395,6 @@ spec: readOnly: true - mountPath: /etc/vmware/wcp/tls/ name: host-vmca - - name: csi-snapshotter - image: localhost:5000/vmware.io/csi-snapshotter:v6.1.0_vmware.2 - args: - - "--v=4" - - "--timeout=300s" - - "--csi-address=$(ADDRESS)" - - "--leader-election" - - "--kube-api-qps=100" - - "--kube-api-burst=100" - - "--extra-create-metadata" - env: - - name: ADDRESS - value: /csi/csi.sock - imagePullPolicy: "IfNotPresent" - volumeMounts: - - mountPath: /csi - name: socket-dir volumes: - name: vsphere-config-volume secret: @@ -443,12 +426,12 @@ data: "fake-attach": "true" "async-query-volume": "true" "improved-csi-idempotency": "true" - "block-volume-snapshot": "true" + "block-volume-snapshot": "false" "sibling-replica-bound-pvc-check": "true" "tkgs-ha": "true" - "list-volumes": "true" + "list-volumes": "false" "cnsmgr-suspend-create-volume": "true" - "listview-tasks": "true" + "listview-tasks": "false" kind: ConfigMap metadata: name: csi-feature-states @@ -533,14 +516,9 @@ webhooks: rules: - apiGroups: [""] apiVersions: ["v1", "v1beta1"] - operations: ["CREATE", "UPDATE", "DELETE"] + operations: ["CREATE", "UPDATE"] resources: ["persistentvolumeclaims"] scope: "Namespaced" - - apiGroups: ["snapshot.storage.k8s.io"] - apiVersions: ["v1"] - operations: ["CREATE"] - resources: ["volumesnapshots"] - scope: "Namespaced" sideEffects: None admissionReviewVersions: ["v1"] failurePolicy: Fail @@ -553,9 +531,6 @@ rules: - apiGroups: [""] resources: ["persistentvolumes", "persistentvolumeclaims"] verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/manifests/supervisorcluster/1.25/kustomization.yaml b/manifests/supervisorcluster/1.25/kustomization.yaml deleted file mode 100644 index fb9558639f..0000000000 --- a/manifests/supervisorcluster/1.25/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: - - cns-csi.yaml diff --git a/manifests/supervisorcluster/1.26/cns-csi.yaml b/manifests/supervisorcluster/1.26/cns-csi.yaml deleted file mode 100644 index e38a5cc0d4..0000000000 --- a/manifests/supervisorcluster/1.26/cns-csi.yaml +++ /dev/null @@ -1,678 +0,0 @@ -kind: ServiceAccount -apiVersion: v1 -metadata: - name: vsphere-csi-controller - namespace: vmware-system-csi ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: vsphere-csi-controller-role -rules: - - apiGroups: [""] - resources: ["nodes", "pods", "configmaps", "resourcequotas", "namespaces", "services"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "create", "update", "patch"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update", "create", "delete", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["csinodes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments/status"] - verbs: ["patch"] - - apiGroups: ["cns.vmware.com"] - resources: ["cnsnodevmattachments", "cnsvolumemetadatas", "cnsfileaccessconfigs"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["cns.vmware.com"] - resources: ["cnscsisvfeaturestates"] - verbs: ["create", "get", "list", "update", "watch"] - - apiGroups: ["cns.vmware.com"] - resources: ["cnsfilevolumeclients"] - verbs: ["get", "update", "create", "delete"] - - apiGroups: ["cns.vmware.com"] - resources: ["cnsregistervolumes"] - verbs: ["get", "list", "watch", "update", "delete"] - - apiGroups: ["cns.vmware.com"] - resources: ["triggercsifullsyncs"] - verbs: ["create", "get", "update", "watch", "list"] - - apiGroups: ["cns.vmware.com"] - resources: ["storagepools"] - verbs: ["get", "watch", "list", "delete", "update", "create", "patch"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["get", "create", "update"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] - - apiGroups: ["vmoperator.vmware.com"] - resources: ["virtualmachines"] - verbs: ["get", "list"] - - apiGroups: ["vmware.com"] - resources: ["virtualnetworks"] - verbs: ["get"] - - apiGroups: ["netoperator.vmware.com"] - resources: ["networkinterfaces"] - verbs: ["get"] - - apiGroups: ["cns.vmware.com"] - resources: ["cnsvolumeoperationrequests"] - verbs: ["create", "get", "list", "update", "delete"] - - apiGroups: ["apps"] - resources: ["statefulsets"] - verbs: ["list"] - - apiGroups: ["topology.tanzu.vmware.com"] - resources: ["availabilityzones"] - verbs: ["get", "list", "watch"] - - apiGroups: [ "snapshot.storage.k8s.io" ] - resources: [ "volumesnapshots" ] - verbs: [ "get", "list", "patch" ] - - apiGroups: [ "snapshot.storage.k8s.io" ] - resources: [ "volumesnapshotclasses" ] - verbs: [ "watch", "get", "list" ] - - apiGroups: [ "snapshot.storage.k8s.io" ] - resources: [ "volumesnapshotcontents" ] - verbs: [ "create", "get", "list", "watch", "update", "delete", "patch"] - - apiGroups: [ "snapshot.storage.k8s.io" ] - resources: [ "volumesnapshotcontents/status" ] - verbs: [ "update", "patch" ] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: vsphere-csi-controller-binding -subjects: - - kind: ServiceAccount - name: vsphere-csi-controller - namespace: vmware-system-csi -roleRef: - kind: ClusterRole - name: vsphere-csi-controller-role - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: csiRole - namespace: vmware-system-csi -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: wcp-privileged-psp -subjects: - # For the vmware-system-csi nodes. - - apiGroup: rbac.authorization.k8s.io - kind: Group - name: system:serviceaccounts:vmware-system-csi ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: vsphere-admin-csi-role -rules: - - apiGroups: ["cns.vmware.com"] - resources: ["cnsregistervolumes"] - verbs: ["get", "list", "create", "delete", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "update", "delete"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "update", "delete"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: wcp:administrators:cluster-edit-csirole -subjects: - - kind: Group - name: sso:Administrators@ - apiGroup: rbac.authorization.k8s.io -roleRef: - kind: ClusterRole - name: vsphere-admin-csi-role - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - namespace: vmware-system-csi - name: vsphere-csi-secret-reader -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "watch", "list"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: vsphere-csi-provisioner-secret-binding - namespace: vmware-system-csi -subjects: - - kind: ServiceAccount - name: vsphere-csi-controller - namespace: vmware-system-csi -roleRef: - kind: Role - name: vsphere-csi-secret-reader - apiGroup: rbac.authorization.k8s.io ---- -kind: Deployment -apiVersion: apps/v1 -metadata: - name: vsphere-csi-controller - namespace: vmware-system-csi -spec: - replicas: 3 - strategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - maxSurge: 0 - selector: - matchLabels: - app: vsphere-csi-controller - template: - metadata: - labels: - app: vsphere-csi-controller - role: vsphere-csi - spec: - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: "app" - operator: In - values: - - vsphere-csi-controller - topologyKey: "kubernetes.io/hostname" - serviceAccount: vsphere-csi-controller - nodeSelector: - node-role.kubernetes.io/control-plane: '' - tolerations: - - operator: "Exists" - key: "node-role.kubernetes.io/control-plane" - effect: "NoSchedule" - hostNetwork: true - priorityClassName: system-node-critical - containers: - - name: csi-provisioner - image: localhost:5000/vmware/csi-provisioner/csi-provisioner:v3.4.0_vmware.1 - args: - - "--v=4" - - "--timeout=300s" - - "--csi-address=$(ADDRESS)" - - "--feature-gates=Topology=true" - - "--strict-topology" - - "--leader-election" - - "--enable-hostlocal-placement=true" - - "--kube-api-qps=100" - - "--kube-api-burst=100" - - "--default-fstype=ext4" - - "--use-service-for-placement-engine=false" - - "--tkgs-ha=true" - - "--leader-election-lease-duration=120s" - - "--leader-election-renew-deadline=60s" - - "--leader-election-retry-period=30s" - env: - - name: ADDRESS - value: /csi/csi.sock - - name: KUBERNETES_SERVICE_HOST - value: "127.0.0.1" - - name: KUBERNETES_SERVICE_PORT - value: "6443" - - name: VSPHERE_CLOUD_OPERATOR_SERVICE_PORT - value: "29000" - - name: VSPHERE_CLOUD_OPERATOR_SERVICE_NAME # service name to be used by csi-provisioner to connect to placement engine - value: vmware-system-psp-operator-k8s-cloud-operator-service - - name: VSPHERE_CLOUD_OPERATOR_SERVICE_NAMESPACE # namespace for service name to be used by csi-provisioner to connect to placement engine - value: vmware-system-appplatform-operator-system - imagePullPolicy: "IfNotPresent" - volumeMounts: - - name: socket-dir - mountPath: /csi - - name: csi-attacher - image: localhost:5000/vmware.io/csi-attacher:v4.3.0_vmware.1 - args: - - "--v=4" - - "--timeout=300s" - - "--csi-address=$(ADDRESS)" - - "--leader-election" - - "--kube-api-qps=100" - - "--kube-api-burst=100" - - "--leader-election-lease-duration=120s" - - "--leader-election-renew-deadline=60s" - - "--leader-election-retry-period=30s" - - "--worker-threads=25" - env: - - name: ADDRESS - value: /csi/csi.sock - - name: KUBERNETES_SERVICE_HOST - value: "127.0.0.1" - - name: KUBERNETES_SERVICE_PORT - value: "6443" - imagePullPolicy: "IfNotPresent" - volumeMounts: - - name: socket-dir - mountPath: /csi - - name: csi-resizer - image: localhost:5000/vmware/kubernetes-csi_external-resizer/kubernetes-csi_external-resizer:v1.8.0_vmware.1 - imagePullPolicy: IfNotPresent - args: - - --v=4 - - --timeout=300s - - --handle-volume-inuse-error=false # Set this to true if used in vSphere 7.0U1 - - --csi-address=$(ADDRESS) - - --leader-election - - --kube-api-qps=100 - - --kube-api-burst=100 - - "--leader-election-lease-duration=120s" - - "--leader-election-renew-deadline=60s" - - "--leader-election-retry-period=30s" - env: - - name: ADDRESS - value: /csi/csi.sock - resources: {} - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - volumeMounts: - - mountPath: /csi - name: socket-dir - - name: vsphere-csi-controller - image: localhost:5000/vmware/vsphere-csi: - ports: - - containerPort: 2112 - name: prometheus - protocol: TCP - - name: healthz - containerPort: 9808 - protocol: TCP - livenessProbe: - httpGet: - path: /healthz - port: healthz - initialDelaySeconds: 30 - timeoutSeconds: 10 - periodSeconds: 180 - failureThreshold: 3 - env: - - name: CSI_ENDPOINT - value: unix:///csi/csi.sock - - name: CLUSTER_FLAVOR - value: "WORKLOAD" - - name: X_CSI_MODE - value: "controller" - - name: X_CSI_SERIAL_VOL_ACCESS_TIMEOUT - value: 3m - - name: KUBERNETES_SERVICE_HOST - value: "127.0.0.1" - - name: KUBERNETES_SERVICE_PORT - value: "6443" - - name: POD_LISTENER_SERVICE_PORT - value: "29000" - - name: VSPHERE_CSI_CONFIG - value: "/etc/vmware/wcp/vsphere-cloud-provider.conf" # here vsphere-cloud-provider.conf is the name of the file used for creating secret using "--from-file" flag - - name: LOGGER_LEVEL - value: "PRODUCTION" # Options: DEVELOPMENT, PRODUCTION - - name: INCLUSTER_CLIENT_QPS - value: "50" - - name: INCLUSTER_CLIENT_BURST - value: "50" - - name: GODEBUG - value: x509sha1=1 - imagePullPolicy: "IfNotPresent" - volumeMounts: - - mountPath: /etc/vmware/wcp - name: vsphere-config-volume - readOnly: true - - mountPath: /csi - name: socket-dir - - mountPath: /etc/vmware/wcp/tls/ - name: host-vmca - - name: liveness-probe - image: localhost:5000/vmware.io/csi-livenessprobe:v2.10.0_vmware.1 - args: - - "--csi-address=/csi/csi.sock" - volumeMounts: - - mountPath: /csi - name: socket-dir - - name: vsphere-syncer - image: localhost:5000/vmware/syncer: - args: - - "--leader-election" - - "--leader-election-lease-duration=120s" - - "--leader-election-renew-deadline=60s" - - "--leader-election-retry-period=30s" - env: - - name: CLUSTER_FLAVOR - value: "WORKLOAD" - - name: KUBERNETES_SERVICE_HOST - value: "127.0.0.1" - - name: KUBERNETES_SERVICE_PORT - value: "6443" - - name: FULL_SYNC_INTERVAL_MINUTES - value: "30" - - name: VOLUME_HEALTH_INTERVAL_MINUTES - value: "5" - - name: POD_POLL_INTERVAL_SECONDS - value: "2" - - name: POD_LISTENER_SERVICE_PORT - value: "29000" - - name: VSPHERE_CSI_CONFIG - value: "/etc/vmware/wcp/vsphere-cloud-provider.conf" # here vsphere-cloud-provider.conf is the name of the file used for creating secret using "--from-file" flag - - name: LOGGER_LEVEL - value: "PRODUCTION" # Options: DEVELOPMENT, PRODUCTION - - name: INCLUSTER_CLIENT_QPS - value: "50" - - name: INCLUSTER_CLIENT_BURST - value: "50" - - name: GODEBUG - value: x509sha1=1 - imagePullPolicy: "IfNotPresent" - ports: - - containerPort: 2113 - name: prometheus - protocol: TCP - volumeMounts: - - mountPath: /etc/vmware/wcp - name: vsphere-config-volume - readOnly: true - - mountPath: /etc/vmware/wcp/tls/ - name: host-vmca - - name: csi-snapshotter - image: localhost:5000/vmware.io/csi-snapshotter:v6.1.0_vmware.2 - args: - - "--v=4" - - "--timeout=300s" - - "--csi-address=$(ADDRESS)" - - "--leader-election" - - "--kube-api-qps=100" - - "--kube-api-burst=100" - - "--extra-create-metadata" - env: - - name: ADDRESS - value: /csi/csi.sock - imagePullPolicy: "IfNotPresent" - volumeMounts: - - mountPath: /csi - name: socket-dir - volumes: - - name: vsphere-config-volume - secret: - secretName: vsphere-config-secret - - name: socket-dir - emptyDir: {} - - name: host-vmca - hostPath: - path: /etc/vmware/wcp/tls/ - type: Directory ---- -apiVersion: storage.k8s.io/v1 -kind: CSIDriver -metadata: - name: csi.vsphere.vmware.com -spec: - attachRequired: true - podInfoOnMount: false ---- -apiVersion: v1 -data: - "volume-extend": "true" - "volume-health": "true" - "online-volume-extend": "true" - "file-volume": "true" - "csi-auth-check": "true" - "trigger-csi-fullsync": "false" - "csi-sv-feature-states-replication": "true" - "fake-attach": "true" - "async-query-volume": "true" - "improved-csi-idempotency": "true" - "block-volume-snapshot": "true" - "sibling-replica-bound-pvc-check": "true" - "tkgs-ha": "true" - "list-volumes": "true" - "cnsmgr-suspend-create-volume": "true" - "listview-tasks": "true" -kind: ConfigMap -metadata: - name: csi-feature-states - namespace: vmware-system-csi ---- -apiVersion: v1 -kind: Service -metadata: - name: vsphere-csi-controller - namespace: vmware-system-csi - labels: - app: vsphere-csi-controller -spec: - ports: - - name: ctlr - port: 2112 - targetPort: 2112 - protocol: TCP - - name: syncer - port: 2113 - targetPort: 2113 - protocol: TCP - selector: - app: vsphere-csi-controller - type: LoadBalancer ---- -apiVersion: v1 -kind: Service -metadata: - name: vmware-system-csi-webhook-service - namespace: vmware-system-csi - labels: - app: vsphere-csi-webhook -spec: - ports: - - port: 443 - targetPort: 9883 - selector: - app: vsphere-csi-webhook ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app: vsphere-csi-webhook - name: vmware-system-csi-serving-cert - namespace: vmware-system-csi -spec: - dnsNames: - - vmware-system-csi-webhook-service.vmware-system-csi.svc - - vmware-system-csi-webhook-service.vmware-system-csi.svc.cluster.local - issuerRef: - kind: Issuer - name: vmware-system-csi-selfsigned-issuer - secretName: vmware-system-csi-webhook-service-cert ---- -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - labels: - app: vsphere-csi-webhook - name: vmware-system-csi-selfsigned-issuer - namespace: vmware-system-csi -spec: - selfSigned: {} ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: vmware-system-csi-validating-webhook-configuration - labels: - app: vsphere-csi-webhook - annotations: - cert-manager.io/inject-ca-from: vmware-system-csi/vmware-system-csi-serving-cert -webhooks: - - name: validation.csi.vsphere.vmware.com - clientConfig: - service: - name: vmware-system-csi-webhook-service - namespace: vmware-system-csi - path: "/validate" - rules: - - apiGroups: [""] - apiVersions: ["v1", "v1beta1"] - operations: ["CREATE", "UPDATE", "DELETE"] - resources: ["persistentvolumeclaims"] - scope: "Namespaced" - - apiGroups: ["snapshot.storage.k8s.io"] - apiVersions: ["v1"] - operations: ["CREATE"] - resources: ["volumesnapshots"] - scope: "Namespaced" - sideEffects: None - admissionReviewVersions: ["v1"] - failurePolicy: Fail ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: vsphere-csi-webhook-cluster-role -rules: - - apiGroups: [""] - resources: ["persistentvolumes", "persistentvolumeclaims"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: vsphere-csi-webhook-cluster-role-binding -subjects: - - kind: ServiceAccount - name: default - namespace: vmware-system-csi -roleRef: - kind: ClusterRole - name: vsphere-csi-webhook-cluster-role - apiGroup: rbac.authorization.k8s.io ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: vsphere-csi-webhook-role - namespace: vmware-system-csi -rules: - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "watch"] ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: vsphere-csi-webhook-role-binding - namespace: vmware-system-csi -subjects: - - kind: ServiceAccount - name: default - namespace: vmware-system-csi -roleRef: - kind: Role - name: vsphere-csi-webhook-role - apiGroup: rbac.authorization.k8s.io ---- -kind: Deployment -apiVersion: apps/v1 -metadata: - name: vsphere-csi-webhook - namespace: vmware-system-csi - labels: - app: vsphere-csi-webhook -spec: - replicas: 3 - strategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - maxSurge: 0 - selector: - matchLabels: - app: vsphere-csi-webhook - template: - metadata: - labels: - app: vsphere-csi-webhook - spec: - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - vsphere-csi-webhook - topologyKey: kubernetes.io/hostname - hostNetwork: true - nodeSelector: - node-role.kubernetes.io/control-plane: "" - terminationGracePeriodSeconds: 10 - tolerations: - - key: node-role.kubernetes.io/control-plane - operator: Exists - effect: NoSchedule - - effect: NoExecute - key: node.alpha.kubernetes.io/notReady - operator: Exists - - effect: NoExecute - key: node.alpha.kubernetes.io/unreachable - operator: Exists - containers: - - name: vsphere-webhook - image: localhost:5000/vmware/syncer: - args: - - "--operation-mode=WEBHOOK_SERVER" - - "--fss-name=internal-feature-states.csi.vsphere.vmware.com" - - "--fss-namespace=$(CSI_NAMESPACE)" - imagePullPolicy: "IfNotPresent" - ports: - - containerPort: 9883 - name: webhook-server - protocol: TCP - env: - - name: CNSCSI_WEBHOOK_SERVICE_CONTAINER_PORT - value: "9883" - - name: CLUSTER_FLAVOR - value: "WORKLOAD" - - name: LOGGER_LEVEL - value: "PRODUCTION" # Options: DEVELOPMENT, PRODUCTION - - name: INCLUSTER_CLIENT_QPS - value: "50" - - name: INCLUSTER_CLIENT_BURST - value: "50" - - name: CSI_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - volumeMounts: - - mountPath: /tmp/k8s-webhook-server/serving-certs - name: webhook-certs - readOnly: true - volumes: - - name: webhook-certs - secret: - defaultMode: 420 - secretName: vmware-system-csi-webhook-service-cert diff --git a/manifests/supervisorcluster/1.26/kustomization.yaml b/manifests/supervisorcluster/1.26/kustomization.yaml deleted file mode 100644 index fb9558639f..0000000000 --- a/manifests/supervisorcluster/1.26/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: - - cns-csi.yaml diff --git a/manifests/supervisorcluster/1.27/cns-csi.yaml b/manifests/supervisorcluster/1.27/cns-csi.yaml deleted file mode 100644 index e38a5cc0d4..0000000000 --- a/manifests/supervisorcluster/1.27/cns-csi.yaml +++ /dev/null @@ -1,678 +0,0 @@ -kind: ServiceAccount -apiVersion: v1 -metadata: - name: vsphere-csi-controller - namespace: vmware-system-csi ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: vsphere-csi-controller-role -rules: - - apiGroups: [""] - resources: ["nodes", "pods", "configmaps", "resourcequotas", "namespaces", "services"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "create", "update", "patch"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update", "create", "delete", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["csinodes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments/status"] - verbs: ["patch"] - - apiGroups: ["cns.vmware.com"] - resources: ["cnsnodevmattachments", "cnsvolumemetadatas", "cnsfileaccessconfigs"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["cns.vmware.com"] - resources: ["cnscsisvfeaturestates"] - verbs: ["create", "get", "list", "update", "watch"] - - apiGroups: ["cns.vmware.com"] - resources: ["cnsfilevolumeclients"] - verbs: ["get", "update", "create", "delete"] - - apiGroups: ["cns.vmware.com"] - resources: ["cnsregistervolumes"] - verbs: ["get", "list", "watch", "update", "delete"] - - apiGroups: ["cns.vmware.com"] - resources: ["triggercsifullsyncs"] - verbs: ["create", "get", "update", "watch", "list"] - - apiGroups: ["cns.vmware.com"] - resources: ["storagepools"] - verbs: ["get", "watch", "list", "delete", "update", "create", "patch"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["get", "create", "update"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] - - apiGroups: ["vmoperator.vmware.com"] - resources: ["virtualmachines"] - verbs: ["get", "list"] - - apiGroups: ["vmware.com"] - resources: ["virtualnetworks"] - verbs: ["get"] - - apiGroups: ["netoperator.vmware.com"] - resources: ["networkinterfaces"] - verbs: ["get"] - - apiGroups: ["cns.vmware.com"] - resources: ["cnsvolumeoperationrequests"] - verbs: ["create", "get", "list", "update", "delete"] - - apiGroups: ["apps"] - resources: ["statefulsets"] - verbs: ["list"] - - apiGroups: ["topology.tanzu.vmware.com"] - resources: ["availabilityzones"] - verbs: ["get", "list", "watch"] - - apiGroups: [ "snapshot.storage.k8s.io" ] - resources: [ "volumesnapshots" ] - verbs: [ "get", "list", "patch" ] - - apiGroups: [ "snapshot.storage.k8s.io" ] - resources: [ "volumesnapshotclasses" ] - verbs: [ "watch", "get", "list" ] - - apiGroups: [ "snapshot.storage.k8s.io" ] - resources: [ "volumesnapshotcontents" ] - verbs: [ "create", "get", "list", "watch", "update", "delete", "patch"] - - apiGroups: [ "snapshot.storage.k8s.io" ] - resources: [ "volumesnapshotcontents/status" ] - verbs: [ "update", "patch" ] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: vsphere-csi-controller-binding -subjects: - - kind: ServiceAccount - name: vsphere-csi-controller - namespace: vmware-system-csi -roleRef: - kind: ClusterRole - name: vsphere-csi-controller-role - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: csiRole - namespace: vmware-system-csi -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: wcp-privileged-psp -subjects: - # For the vmware-system-csi nodes. - - apiGroup: rbac.authorization.k8s.io - kind: Group - name: system:serviceaccounts:vmware-system-csi ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: vsphere-admin-csi-role -rules: - - apiGroups: ["cns.vmware.com"] - resources: ["cnsregistervolumes"] - verbs: ["get", "list", "create", "delete", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "update", "delete"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "update", "delete"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: wcp:administrators:cluster-edit-csirole -subjects: - - kind: Group - name: sso:Administrators@ - apiGroup: rbac.authorization.k8s.io -roleRef: - kind: ClusterRole - name: vsphere-admin-csi-role - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - namespace: vmware-system-csi - name: vsphere-csi-secret-reader -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "watch", "list"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: vsphere-csi-provisioner-secret-binding - namespace: vmware-system-csi -subjects: - - kind: ServiceAccount - name: vsphere-csi-controller - namespace: vmware-system-csi -roleRef: - kind: Role - name: vsphere-csi-secret-reader - apiGroup: rbac.authorization.k8s.io ---- -kind: Deployment -apiVersion: apps/v1 -metadata: - name: vsphere-csi-controller - namespace: vmware-system-csi -spec: - replicas: 3 - strategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - maxSurge: 0 - selector: - matchLabels: - app: vsphere-csi-controller - template: - metadata: - labels: - app: vsphere-csi-controller - role: vsphere-csi - spec: - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: "app" - operator: In - values: - - vsphere-csi-controller - topologyKey: "kubernetes.io/hostname" - serviceAccount: vsphere-csi-controller - nodeSelector: - node-role.kubernetes.io/control-plane: '' - tolerations: - - operator: "Exists" - key: "node-role.kubernetes.io/control-plane" - effect: "NoSchedule" - hostNetwork: true - priorityClassName: system-node-critical - containers: - - name: csi-provisioner - image: localhost:5000/vmware/csi-provisioner/csi-provisioner:v3.4.0_vmware.1 - args: - - "--v=4" - - "--timeout=300s" - - "--csi-address=$(ADDRESS)" - - "--feature-gates=Topology=true" - - "--strict-topology" - - "--leader-election" - - "--enable-hostlocal-placement=true" - - "--kube-api-qps=100" - - "--kube-api-burst=100" - - "--default-fstype=ext4" - - "--use-service-for-placement-engine=false" - - "--tkgs-ha=true" - - "--leader-election-lease-duration=120s" - - "--leader-election-renew-deadline=60s" - - "--leader-election-retry-period=30s" - env: - - name: ADDRESS - value: /csi/csi.sock - - name: KUBERNETES_SERVICE_HOST - value: "127.0.0.1" - - name: KUBERNETES_SERVICE_PORT - value: "6443" - - name: VSPHERE_CLOUD_OPERATOR_SERVICE_PORT - value: "29000" - - name: VSPHERE_CLOUD_OPERATOR_SERVICE_NAME # service name to be used by csi-provisioner to connect to placement engine - value: vmware-system-psp-operator-k8s-cloud-operator-service - - name: VSPHERE_CLOUD_OPERATOR_SERVICE_NAMESPACE # namespace for service name to be used by csi-provisioner to connect to placement engine - value: vmware-system-appplatform-operator-system - imagePullPolicy: "IfNotPresent" - volumeMounts: - - name: socket-dir - mountPath: /csi - - name: csi-attacher - image: localhost:5000/vmware.io/csi-attacher:v4.3.0_vmware.1 - args: - - "--v=4" - - "--timeout=300s" - - "--csi-address=$(ADDRESS)" - - "--leader-election" - - "--kube-api-qps=100" - - "--kube-api-burst=100" - - "--leader-election-lease-duration=120s" - - "--leader-election-renew-deadline=60s" - - "--leader-election-retry-period=30s" - - "--worker-threads=25" - env: - - name: ADDRESS - value: /csi/csi.sock - - name: KUBERNETES_SERVICE_HOST - value: "127.0.0.1" - - name: KUBERNETES_SERVICE_PORT - value: "6443" - imagePullPolicy: "IfNotPresent" - volumeMounts: - - name: socket-dir - mountPath: /csi - - name: csi-resizer - image: localhost:5000/vmware/kubernetes-csi_external-resizer/kubernetes-csi_external-resizer:v1.8.0_vmware.1 - imagePullPolicy: IfNotPresent - args: - - --v=4 - - --timeout=300s - - --handle-volume-inuse-error=false # Set this to true if used in vSphere 7.0U1 - - --csi-address=$(ADDRESS) - - --leader-election - - --kube-api-qps=100 - - --kube-api-burst=100 - - "--leader-election-lease-duration=120s" - - "--leader-election-renew-deadline=60s" - - "--leader-election-retry-period=30s" - env: - - name: ADDRESS - value: /csi/csi.sock - resources: {} - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - volumeMounts: - - mountPath: /csi - name: socket-dir - - name: vsphere-csi-controller - image: localhost:5000/vmware/vsphere-csi: - ports: - - containerPort: 2112 - name: prometheus - protocol: TCP - - name: healthz - containerPort: 9808 - protocol: TCP - livenessProbe: - httpGet: - path: /healthz - port: healthz - initialDelaySeconds: 30 - timeoutSeconds: 10 - periodSeconds: 180 - failureThreshold: 3 - env: - - name: CSI_ENDPOINT - value: unix:///csi/csi.sock - - name: CLUSTER_FLAVOR - value: "WORKLOAD" - - name: X_CSI_MODE - value: "controller" - - name: X_CSI_SERIAL_VOL_ACCESS_TIMEOUT - value: 3m - - name: KUBERNETES_SERVICE_HOST - value: "127.0.0.1" - - name: KUBERNETES_SERVICE_PORT - value: "6443" - - name: POD_LISTENER_SERVICE_PORT - value: "29000" - - name: VSPHERE_CSI_CONFIG - value: "/etc/vmware/wcp/vsphere-cloud-provider.conf" # here vsphere-cloud-provider.conf is the name of the file used for creating secret using "--from-file" flag - - name: LOGGER_LEVEL - value: "PRODUCTION" # Options: DEVELOPMENT, PRODUCTION - - name: INCLUSTER_CLIENT_QPS - value: "50" - - name: INCLUSTER_CLIENT_BURST - value: "50" - - name: GODEBUG - value: x509sha1=1 - imagePullPolicy: "IfNotPresent" - volumeMounts: - - mountPath: /etc/vmware/wcp - name: vsphere-config-volume - readOnly: true - - mountPath: /csi - name: socket-dir - - mountPath: /etc/vmware/wcp/tls/ - name: host-vmca - - name: liveness-probe - image: localhost:5000/vmware.io/csi-livenessprobe:v2.10.0_vmware.1 - args: - - "--csi-address=/csi/csi.sock" - volumeMounts: - - mountPath: /csi - name: socket-dir - - name: vsphere-syncer - image: localhost:5000/vmware/syncer: - args: - - "--leader-election" - - "--leader-election-lease-duration=120s" - - "--leader-election-renew-deadline=60s" - - "--leader-election-retry-period=30s" - env: - - name: CLUSTER_FLAVOR - value: "WORKLOAD" - - name: KUBERNETES_SERVICE_HOST - value: "127.0.0.1" - - name: KUBERNETES_SERVICE_PORT - value: "6443" - - name: FULL_SYNC_INTERVAL_MINUTES - value: "30" - - name: VOLUME_HEALTH_INTERVAL_MINUTES - value: "5" - - name: POD_POLL_INTERVAL_SECONDS - value: "2" - - name: POD_LISTENER_SERVICE_PORT - value: "29000" - - name: VSPHERE_CSI_CONFIG - value: "/etc/vmware/wcp/vsphere-cloud-provider.conf" # here vsphere-cloud-provider.conf is the name of the file used for creating secret using "--from-file" flag - - name: LOGGER_LEVEL - value: "PRODUCTION" # Options: DEVELOPMENT, PRODUCTION - - name: INCLUSTER_CLIENT_QPS - value: "50" - - name: INCLUSTER_CLIENT_BURST - value: "50" - - name: GODEBUG - value: x509sha1=1 - imagePullPolicy: "IfNotPresent" - ports: - - containerPort: 2113 - name: prometheus - protocol: TCP - volumeMounts: - - mountPath: /etc/vmware/wcp - name: vsphere-config-volume - readOnly: true - - mountPath: /etc/vmware/wcp/tls/ - name: host-vmca - - name: csi-snapshotter - image: localhost:5000/vmware.io/csi-snapshotter:v6.1.0_vmware.2 - args: - - "--v=4" - - "--timeout=300s" - - "--csi-address=$(ADDRESS)" - - "--leader-election" - - "--kube-api-qps=100" - - "--kube-api-burst=100" - - "--extra-create-metadata" - env: - - name: ADDRESS - value: /csi/csi.sock - imagePullPolicy: "IfNotPresent" - volumeMounts: - - mountPath: /csi - name: socket-dir - volumes: - - name: vsphere-config-volume - secret: - secretName: vsphere-config-secret - - name: socket-dir - emptyDir: {} - - name: host-vmca - hostPath: - path: /etc/vmware/wcp/tls/ - type: Directory ---- -apiVersion: storage.k8s.io/v1 -kind: CSIDriver -metadata: - name: csi.vsphere.vmware.com -spec: - attachRequired: true - podInfoOnMount: false ---- -apiVersion: v1 -data: - "volume-extend": "true" - "volume-health": "true" - "online-volume-extend": "true" - "file-volume": "true" - "csi-auth-check": "true" - "trigger-csi-fullsync": "false" - "csi-sv-feature-states-replication": "true" - "fake-attach": "true" - "async-query-volume": "true" - "improved-csi-idempotency": "true" - "block-volume-snapshot": "true" - "sibling-replica-bound-pvc-check": "true" - "tkgs-ha": "true" - "list-volumes": "true" - "cnsmgr-suspend-create-volume": "true" - "listview-tasks": "true" -kind: ConfigMap -metadata: - name: csi-feature-states - namespace: vmware-system-csi ---- -apiVersion: v1 -kind: Service -metadata: - name: vsphere-csi-controller - namespace: vmware-system-csi - labels: - app: vsphere-csi-controller -spec: - ports: - - name: ctlr - port: 2112 - targetPort: 2112 - protocol: TCP - - name: syncer - port: 2113 - targetPort: 2113 - protocol: TCP - selector: - app: vsphere-csi-controller - type: LoadBalancer ---- -apiVersion: v1 -kind: Service -metadata: - name: vmware-system-csi-webhook-service - namespace: vmware-system-csi - labels: - app: vsphere-csi-webhook -spec: - ports: - - port: 443 - targetPort: 9883 - selector: - app: vsphere-csi-webhook ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - labels: - app: vsphere-csi-webhook - name: vmware-system-csi-serving-cert - namespace: vmware-system-csi -spec: - dnsNames: - - vmware-system-csi-webhook-service.vmware-system-csi.svc - - vmware-system-csi-webhook-service.vmware-system-csi.svc.cluster.local - issuerRef: - kind: Issuer - name: vmware-system-csi-selfsigned-issuer - secretName: vmware-system-csi-webhook-service-cert ---- -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - labels: - app: vsphere-csi-webhook - name: vmware-system-csi-selfsigned-issuer - namespace: vmware-system-csi -spec: - selfSigned: {} ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: vmware-system-csi-validating-webhook-configuration - labels: - app: vsphere-csi-webhook - annotations: - cert-manager.io/inject-ca-from: vmware-system-csi/vmware-system-csi-serving-cert -webhooks: - - name: validation.csi.vsphere.vmware.com - clientConfig: - service: - name: vmware-system-csi-webhook-service - namespace: vmware-system-csi - path: "/validate" - rules: - - apiGroups: [""] - apiVersions: ["v1", "v1beta1"] - operations: ["CREATE", "UPDATE", "DELETE"] - resources: ["persistentvolumeclaims"] - scope: "Namespaced" - - apiGroups: ["snapshot.storage.k8s.io"] - apiVersions: ["v1"] - operations: ["CREATE"] - resources: ["volumesnapshots"] - scope: "Namespaced" - sideEffects: None - admissionReviewVersions: ["v1"] - failurePolicy: Fail ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: vsphere-csi-webhook-cluster-role -rules: - - apiGroups: [""] - resources: ["persistentvolumes", "persistentvolumeclaims"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: vsphere-csi-webhook-cluster-role-binding -subjects: - - kind: ServiceAccount - name: default - namespace: vmware-system-csi -roleRef: - kind: ClusterRole - name: vsphere-csi-webhook-cluster-role - apiGroup: rbac.authorization.k8s.io ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: vsphere-csi-webhook-role - namespace: vmware-system-csi -rules: - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "watch"] ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: vsphere-csi-webhook-role-binding - namespace: vmware-system-csi -subjects: - - kind: ServiceAccount - name: default - namespace: vmware-system-csi -roleRef: - kind: Role - name: vsphere-csi-webhook-role - apiGroup: rbac.authorization.k8s.io ---- -kind: Deployment -apiVersion: apps/v1 -metadata: - name: vsphere-csi-webhook - namespace: vmware-system-csi - labels: - app: vsphere-csi-webhook -spec: - replicas: 3 - strategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - maxSurge: 0 - selector: - matchLabels: - app: vsphere-csi-webhook - template: - metadata: - labels: - app: vsphere-csi-webhook - spec: - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - vsphere-csi-webhook - topologyKey: kubernetes.io/hostname - hostNetwork: true - nodeSelector: - node-role.kubernetes.io/control-plane: "" - terminationGracePeriodSeconds: 10 - tolerations: - - key: node-role.kubernetes.io/control-plane - operator: Exists - effect: NoSchedule - - effect: NoExecute - key: node.alpha.kubernetes.io/notReady - operator: Exists - - effect: NoExecute - key: node.alpha.kubernetes.io/unreachable - operator: Exists - containers: - - name: vsphere-webhook - image: localhost:5000/vmware/syncer: - args: - - "--operation-mode=WEBHOOK_SERVER" - - "--fss-name=internal-feature-states.csi.vsphere.vmware.com" - - "--fss-namespace=$(CSI_NAMESPACE)" - imagePullPolicy: "IfNotPresent" - ports: - - containerPort: 9883 - name: webhook-server - protocol: TCP - env: - - name: CNSCSI_WEBHOOK_SERVICE_CONTAINER_PORT - value: "9883" - - name: CLUSTER_FLAVOR - value: "WORKLOAD" - - name: LOGGER_LEVEL - value: "PRODUCTION" # Options: DEVELOPMENT, PRODUCTION - - name: INCLUSTER_CLIENT_QPS - value: "50" - - name: INCLUSTER_CLIENT_BURST - value: "50" - - name: CSI_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - volumeMounts: - - mountPath: /tmp/k8s-webhook-server/serving-certs - name: webhook-certs - readOnly: true - volumes: - - name: webhook-certs - secret: - defaultMode: 420 - secretName: vmware-system-csi-webhook-service-cert diff --git a/manifests/supervisorcluster/1.27/kustomization.yaml b/manifests/supervisorcluster/1.27/kustomization.yaml deleted file mode 100644 index fb9558639f..0000000000 --- a/manifests/supervisorcluster/1.27/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: - - cns-csi.yaml diff --git a/manifests/vanilla/csi-snapshot-validatingwebhook.yaml b/manifests/vanilla/csi-snapshot-validatingwebhook.yaml index c9949a84e4..3ec1dd68ab 100644 --- a/manifests/vanilla/csi-snapshot-validatingwebhook.yaml +++ b/manifests/vanilla/csi-snapshot-validatingwebhook.yaml @@ -84,7 +84,7 @@ spec: serviceAccountName: snapshot-webhook containers: - name: snapshot-validation - image: registry.k8s.io/sig-storage/snapshot-validation-webhook:v6.2.2 # change the image if you wish to use your own custom validation server image + image: k8s.gcr.io/sig-storage/snapshot-validation-webhook:v6.2.1 # change the image if you wish to use your own custom validation server image imagePullPolicy: IfNotPresent args: ['--tls-cert-file=/run/secrets/tls/tls.crt', '--tls-private-key-file=/run/secrets/tls/tls.key'] ports: diff --git a/manifests/vanilla/deploy-csi-snapshot-components.sh b/manifests/vanilla/deploy-csi-snapshot-components.sh index 4c7ff70667..7a9e8ae24b 100755 --- a/manifests/vanilla/deploy-csi-snapshot-components.sh +++ b/manifests/vanilla/deploy-csi-snapshot-components.sh @@ -59,7 +59,7 @@ else exit 1 fi -qualified_version="v6.2.2" +qualified_version="v6.2.1" volumesnapshotclasses_crd="volumesnapshotclasses.snapshot.storage.k8s.io" volumesnapshotcontents_crd="volumesnapshotcontents.snapshot.storage.k8s.io" volumesnapshots_crd="volumesnapshots.snapshot.storage.k8s.io" @@ -200,7 +200,7 @@ EOF openssl req -nodes -new -x509 -keyout "${tmpdir}"/ca.key -out "${tmpdir}"/ca.crt -subj "/CN=vSphere CSI Admission Controller Webhook CA" openssl genrsa -out "${tmpdir}"/webhook-server-tls.key 2048 openssl req -new -key "${tmpdir}"/webhook-server-tls.key -subj "/CN=${service}.${namespace}.svc" -config "${tmpdir}"/server.conf \ - | openssl x509 -req -CA "${tmpdir}"/ca.crt -CAkey "${tmpdir}"/ca.key -days 180 -CAcreateserial -out "${tmpdir}"/webhook-server-tls.crt -extensions v3_req -extfile "${tmpdir}"/server.conf + | openssl x509 -req -CA "${tmpdir}"/ca.crt -CAkey "${tmpdir}"/ca.key -CAcreateserial -out "${tmpdir}"/webhook-server-tls.crt -extensions v3_req -extfile "${tmpdir}"/server.conf cat <"${tmpdir}"/webhook.config [WebHookConfig] port = "8443" @@ -237,7 +237,7 @@ spec: spec: containers: - name: csi-snapshotter - image: 'registry.k8s.io/sig-storage/csi-snapshotter:${qualified_version}' + image: 'k8s.gcr.io/sig-storage/csi-snapshotter:${qualified_version}' args: - '--v=4' - '--kube-api-qps=100' @@ -265,7 +265,7 @@ EOF check_snapshotter_sidecar(){ local found="false" local container_images - local csi_snapshotter_image="registry.k8s.io/sig-storage/csi-snapshotter" + local csi_snapshotter_image="k8s.gcr.io/sig-storage/csi-snapshotter" container_images=$(kubectl -n vmware-system-csi get deployment vsphere-csi-controller -o jsonpath='{.spec.template.spec.containers[*].image}') IFS=' ' read -r -a container_images_arr <<< "$container_images" diff --git a/manifests/vanilla/deploy-vsphere-csi-validation-webhook.sh b/manifests/vanilla/deploy-vsphere-csi-validation-webhook.sh index 88ad82ccad..5831654ab5 100755 --- a/manifests/vanilla/deploy-vsphere-csi-validation-webhook.sh +++ b/manifests/vanilla/deploy-vsphere-csi-validation-webhook.sh @@ -81,7 +81,7 @@ EOF openssl req -nodes -new -x509 -keyout "${tmpdir}"/ca.key -out "${tmpdir}"/ca.crt -subj "/CN=vSphere CSI Admission Controller Webhook CA" openssl genrsa -out "${tmpdir}"/webhook-server-tls.key 2048 openssl req -new -key "${tmpdir}"/webhook-server-tls.key -subj "/CN=${service}.${namespace}.svc" -config "${tmpdir}"/server.conf \ - | openssl x509 -req -CA "${tmpdir}"/ca.crt -CAkey "${tmpdir}"/ca.key -days 180 -CAcreateserial -out "${tmpdir}"/webhook-server-tls.crt -extensions v3_req -extfile "${tmpdir}"/server.conf + | openssl x509 -req -CA "${tmpdir}"/ca.crt -CAkey "${tmpdir}"/ca.key -CAcreateserial -out "${tmpdir}"/webhook-server-tls.crt -extensions v3_req -extfile "${tmpdir}"/server.conf cat <"${tmpdir}"/webhook.config [WebHookConfig] diff --git a/manifests/vanilla/validatingwebhook.yaml b/manifests/vanilla/validatingwebhook.yaml index 4e2e648563..a597750e6f 100644 --- a/manifests/vanilla/validatingwebhook.yaml +++ b/manifests/vanilla/validatingwebhook.yaml @@ -134,7 +134,7 @@ spec: dnsPolicy: "Default" containers: - name: vsphere-webhook - image: gcr.io/cloud-provider-vsphere/csi/release/syncer:v3.1.1 + image: gcr.io/cloud-provider-vsphere/csi/release/syncer:v3.0.2 args: - "--operation-mode=WEBHOOK_SERVER" - "--fss-name=internal-feature-states.csi.vsphere.vmware.com" @@ -149,10 +149,6 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - securityContext: - runAsNonRoot: true - runAsUser: 65532 - runAsGroup: 65532 volumeMounts: - mountPath: /run/secrets/tls name: webhook-certs diff --git a/manifests/vanilla/vsphere-csi-driver.yaml b/manifests/vanilla/vsphere-csi-driver.yaml index cb73947f45..291b05d27d 100644 --- a/manifests/vanilla/vsphere-csi-driver.yaml +++ b/manifests/vanilla/vsphere-csi-driver.yaml @@ -155,6 +155,7 @@ data: "async-query-volume": "true" "block-volume-snapshot": "true" "csi-windows-support": "true" + "use-csinode-id": "true" "list-volumes": "true" "pv-to-backingdiskobjectid-mapping": "false" "cnsmgr-suspend-create-volume": "true" @@ -162,7 +163,7 @@ data: "max-pvscsi-targets-per-vm": "true" "multi-vcenter-csi-topology": "true" "csi-internal-generated-cluster-id": "true" - "listview-tasks": "true" + "listview-tasks": "false" kind: ConfigMap metadata: name: internal-feature-states.csi.vsphere.vmware.com @@ -243,7 +244,7 @@ spec: dnsPolicy: "Default" containers: - name: csi-attacher - image: registry.k8s.io/sig-storage/csi-attacher:v4.3.0 + image: k8s.gcr.io/sig-storage/csi-attacher:v4.2.0 args: - "--v=4" - "--timeout=300s" @@ -261,7 +262,7 @@ spec: - mountPath: /csi name: socket-dir - name: csi-resizer - image: registry.k8s.io/sig-storage/csi-resizer:v1.8.0 + image: k8s.gcr.io/sig-storage/csi-resizer:v1.7.0 args: - "--v=4" - "--timeout=300s" @@ -280,7 +281,7 @@ spec: - mountPath: /csi name: socket-dir - name: vsphere-csi-controller - image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.1.1 + image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.0.2 args: - "--fss-name=internal-feature-states.csi.vsphere.vmware.com" - "--fss-namespace=$(CSI_NAMESPACE)" @@ -306,10 +307,6 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - securityContext: - runAsNonRoot: true - runAsUser: 65532 - runAsGroup: 65532 volumeMounts: - mountPath: /etc/cloud name: vsphere-config-volume @@ -332,7 +329,7 @@ spec: periodSeconds: 180 failureThreshold: 3 - name: liveness-probe - image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 + image: k8s.gcr.io/sig-storage/livenessprobe:v2.9.0 args: - "--v=4" - "--csi-address=/csi/csi.sock" @@ -340,12 +337,12 @@ spec: - name: socket-dir mountPath: /csi - name: vsphere-syncer - image: gcr.io/cloud-provider-vsphere/csi/release/syncer:v3.1.1 + image: gcr.io/cloud-provider-vsphere/csi/release/syncer:v3.0.2 args: - "--leader-election" - - "--leader-election-lease-duration=30s" - - "--leader-election-renew-deadline=20s" - - "--leader-election-retry-period=10s" + - "--leader-election-lease-duration=120s" + - "--leader-election-renew-deadline=60s" + - "--leader-election-retry-period=30s" - "--fss-name=internal-feature-states.csi.vsphere.vmware.com" - "--fss-namespace=$(CSI_NAMESPACE)" imagePullPolicy: "Always" @@ -370,16 +367,12 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - securityContext: - runAsNonRoot: true - runAsUser: 65532 - runAsGroup: 65532 volumeMounts: - mountPath: /etc/cloud name: vsphere-config-volume readOnly: true - name: csi-provisioner - image: registry.k8s.io/sig-storage/csi-provisioner:v3.5.0 + image: k8s.gcr.io/sig-storage/csi-provisioner:v3.4.0 args: - "--v=4" - "--timeout=300s" @@ -401,7 +394,7 @@ spec: - mountPath: /csi name: socket-dir - name: csi-snapshotter - image: registry.k8s.io/sig-storage/csi-snapshotter:v6.2.2 + image: k8s.gcr.io/sig-storage/csi-snapshotter:v6.2.1 args: - "--v=4" - "--kube-api-qps=100" @@ -452,7 +445,7 @@ spec: dnsPolicy: "ClusterFirstWithHostNet" containers: - name: node-driver-registrar - image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0 + image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.7.0 args: - "--v=5" - "--csi-address=$(ADDRESS)" @@ -475,7 +468,7 @@ spec: - --mode=kubelet-registration-probe initialDelaySeconds: 3 - name: vsphere-csi-node - image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.1.1 + image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.0.2 args: - "--fss-name=internal-feature-states.csi.vsphere.vmware.com" - "--fss-namespace=$(CSI_NAMESPACE)" @@ -537,7 +530,7 @@ spec: periodSeconds: 5 failureThreshold: 3 - name: liveness-probe - image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 + image: k8s.gcr.io/sig-storage/livenessprobe:v2.9.0 args: - "--v=4" - "--csi-address=/csi/csi.sock" @@ -599,7 +592,7 @@ spec: serviceAccountName: vsphere-csi-node containers: - name: node-driver-registrar - image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0 + image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.7.0 args: - "--v=5" - "--csi-address=$(ADDRESS)" @@ -622,7 +615,7 @@ spec: - --mode=kubelet-registration-probe initialDelaySeconds: 3 - name: vsphere-csi-node - image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.1.1 + image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.0.2 args: - "--fss-name=internal-feature-states.csi.vsphere.vmware.com" - "--fss-namespace=$(CSI_NAMESPACE)" @@ -679,7 +672,7 @@ spec: periodSeconds: 5 failureThreshold: 3 - name: liveness-probe - image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 + image: k8s.gcr.io/sig-storage/livenessprobe:v2.9.0 args: - "--v=4" - "--csi-address=/csi/csi.sock" diff --git a/pipeline/deploy-staging.sh b/pipeline/deploy-staging.sh index 851d8431f0..d4c6404e9b 100755 --- a/pipeline/deploy-staging.sh +++ b/pipeline/deploy-staging.sh @@ -1,19 +1,5 @@ #!/bin/bash -# Copyright 2023 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - set +x if [[ -z "${VSPHERE_CSI_CONTROLLER_IMAGE}" ]] @@ -35,30 +21,10 @@ then fi git clone "$CNS_CSI_STAGING_REPO" || exit 1 - -# Update the CNS-CSI manifest files to capture any changes. -cp -R manifests/supervisorcluster/* staging-cd/ || exit 1 cd staging-cd || exit 1 -# Patch the CSI controller patch yaml file with the driver and syncer images from build job. -yq -i '(.spec.template.spec.containers[0].image = env(VSPHERE_CSI_CONTROLLER_IMAGE)) | (.spec.template.spec.containers[1].image = env(VSPHERE_SYNCER_IMAGE))' staging/csi-controller-patch.yaml || exit 1 - -# Patch the CSI webhook patch yaml file with the syncer images from build job. -yq -i '(.spec.template.spec.containers[0].image = env(VSPHERE_SYNCER_IMAGE))' staging/csi-webhook-patch.yaml || exit 1 - -# The kubeconfig has restricted read-only access to only vmware-system-csi namespace in the Supervisor. -# It can only monitor the CSI deployments. -export KUBECONFIG=$CNS_STAGING_SV_KUBECONFIG -export K8S_MAJOR_VERSION -export K8S_MINOR_VERSION - -K8S_MAJOR_VERSION=$(kubectl version -o json | jq .serverVersion.major | tr -d '"') -K8S_MINOR_VERSION=$(kubectl version -o json | jq .serverVersion.minor | tr -d '"') - -echo "Supervisor version: $K8S_MAJOR_VERSION.$K8S_MINOR_VERSION" - -# Replace the kubernetes version in kustomization.yaml -yq -i '.bases = ["../" + env(K8S_MAJOR_VERSION) + "." + env(K8S_MINOR_VERSION)]' staging/kustomization.yaml || exit 1 +# Patch the yaml file with the driver and syncer images from build job. +yq -i '(.spec.template.spec.containers[0].image = env(VSPHERE_CSI_CONTROLLER_IMAGE)) | (.spec.template.spec.containers[1].image = env(VSPHERE_SYNCER_IMAGE))' staging/patch.yaml || exit 1 # If there are any changes, then commit the code changes and push it to the repo. if git diff | grep diff; @@ -68,32 +34,12 @@ then git add . || exit 1 git config user.email "svc.bot-cns@vmware.com" || exit 1 git config user.name "svc.bot-cns" || exit 1 - git commit -m "Pipeline updated manifest files with images $VSPHERE_CSI_CONTROLLER_IMAGE and $VSPHERE_SYNCER_IMAGE" || exit 1 + git commit -m "Pipeline updated staging/patch.yaml with images $VSPHERE_CSI_CONTROLLER_IMAGE and $VSPHERE_SYNCER_IMAGE" || exit 1 git push origin main || exit 1 else echo "No code changes pushed to the staging repo." fi -while true -do - tmp_file=/tmp/$$ - kubectl -n vmware-system-csi get deployment vsphere-csi-controller -o=jsonpath="{'\n'}{range .spec.template.spec.containers[*]}{.image}{'\n'}{end}" | tee $tmp_file - if grep "$VSPHERE_CSI_CONTROLLER_IMAGE" $tmp_file; - then - echo "CSI deployment is patched." - break - fi - echo "CSI deployment not yet patched. Sleeping for 1 second..." - sleep 1 -done - -echo "Sleeping for 1 min for WCP to reconcile the CSI patch.." -sleep 60 +# TODO: Add code to wait for the CD infra to update the CSI in the staging environment. -echo "Wait for 2 mins for the CSI deployment to be ready." -if ! kubectl wait deployment -n vmware-system-csi vsphere-csi-controller --for=jsonpath="{.status.readyReplicas}"=3 --timeout=120s; -then - echo "CSI deployment is not ready within 120s." - exit 1 -fi echo "Completed deploying CSI images to the staging environment." diff --git a/pipeline/deploy.sh b/pipeline/deploy.sh index e124409fba..2c90418a00 100755 --- a/pipeline/deploy.sh +++ b/pipeline/deploy.sh @@ -1,123 +1,60 @@ #!/bin/bash -# Copyright 2023 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - set +x if [[ -z "${VSPHERE_CSI_CONTROLLER_IMAGE}" ]] then - echo "Env variable unset: VSPHERE_CSI_CONTROLLER_IMAGE." + echo "Env variable unset: VSPHERE_CSI_CONTROLLER_IMAGE" exit 1 fi if [[ -z "${VSPHERE_SYNCER_IMAGE}" ]] then - echo "Env variable unset: VSPHERE_SYNCER_IMAGE." + echo "Env variable unset: VSPHERE_SYNCER_IMAGE" exit 1 fi # Borrow a testbed from CSI Testbed Pool Svc. -if ! testbed=$(curl -X 'PUT' "$TESTBED_POOL_SERVICE_ENDPOINT/v1/pool/$TESTBED_POOL_ID/borrowTestbed" -H 'accept: application/json' -u "$CNS_MANAGER_USERNAME:$CNS_MANAGER_PASSWORD"); then - echo "Unable to borrow a testbed" - exit 1 -else - # borrowTestbed API succeed - echo "Got a testbed from testbed pool service" -fi - -# Get TESTBED_ID -testbedId=$(echo "$testbed" | jq '.id' | tr -d '"') -if [ -z "${testbedId}" ] +if ! testbed=$(curl -X 'GET' "${CNS_TESTBEDPOOL_SVC_URL}"); then - echo "testbedId is empty" + echo "Unable to borrow a testbed!" exit 1 fi +echo "TestbedInfo: $testbed" -# Store the testbed ID. -echo "TESTBED_ID=$testbedId" >> build.env - -# Extract VC IP from borrow testbed API Response. if ! vcIp=$(echo "$testbed" | jq '.vcIp'|tr -d '"'); then - echo "Error getting the vcIp." - exit 1 -fi - -# Extract External Gateway VM IP from borrow testbed API Response. -if ! externalVMGatewayIp=$(echo "$testbed" | jq '.externalVMGatewayIp'|tr -d '"'); -then - echo "Error getting the externalVMGatewayIp." + echo "Error getting the vcIp!" exit 1 fi +echo "VC_IP=$vcIp" >> build.env -# Extract VC root password from borrow testbed API Response. -if ! vcRootPassword=$(echo "$testbed" | jq '.vcRootPassword'|tr -d '"'); +if ! svAdminCreds=$(echo "$testbed" | jq '.svAdminCreds'|tr -d '"'|base64 -d); then - echo "Error getting the vcRootPassword." - exit 1 -fi - -# Extract VC username from borrow testbed API Response. -if ! vimUsername=$(echo "$testbed" | jq '.vimUsername'|tr -d '"'); -then - echo "Error getting the vimUsername." + echo "Error getting the svAdminCreds!" exit 1 fi +echo "svAdminCreds = $svAdminCreds" +SV_KUBECONFIG=/tmp/$$ +echo "$testbed" | jq '.svAdminCreds'|tr -d '"'|base64 -d > $SV_KUBECONFIG +export KUBECONFIG=$SV_KUBECONFIG -# Extract VC Admin password from borrow testbed API Response. -if ! vimPassword=$(echo "$testbed" | jq '.vimPassword'|tr -d '"'); +if ! kustomize build pipeline/dev | envsubst | kubectl apply -f -; then - echo "Error getting the vimPassword." + echo "Error patching the CSI images in the testbed!" exit 1 fi -# Print all the values into Console. -echo "VSPHERE_CSI_CONTROLLER_IMAGE = $VSPHERE_CSI_CONTROLLER_IMAGE" -echo "VSPHERE_SYNCER_IMAGE = $VSPHERE_SYNCER_IMAGE" - -# Store all the values into Artifacts. -{ echo "TESTBED_ID=$testbedId"; echo "vcIp=$vcIp"; echo "vcRootPassword=$vcRootPassword"; echo "vimPassword=$vimPassword"; echo "vimUsername=$vimUsername"; echo "externalVMGatewayIp=$externalVMGatewayIp";} >> ./env.json - -SV_KUBECONFIG=/tmp/$$ - -echo "$testbed" | jq '.kubeConfig'|tr -d '"'|base64 -d > $SV_KUBECONFIG - -export KUBECONFIG=$SV_KUBECONFIG - -echo "sv_kubeconfig_content=$(cat $SV_KUBECONFIG)" > ./sv_kubeconfig_content.yaml - -# Pod status on testbed before patching the CSI Images -kubectl get pods -n vmware-system-csi - -kubectl set env deployment/vsphere-csi-controller -n vmware-system-csi --containers=vsphere-syncer FULL_SYNC_INTERVAL_MINUTES=2 VOLUME_HEALTH_INTERVAL_MINUTES=2 -kubectl set image deployment/vsphere-csi-controller -n vmware-system-csi vsphere-syncer="$VSPHERE_SYNCER_IMAGE" vsphere-csi-controller="$VSPHERE_CSI_CONTROLLER_IMAGE" -kubectl set image deployment/vsphere-csi-webhook -n vmware-system-csi vsphere-webhook="$VSPHERE_SYNCER_IMAGE" - # Sleep for 60 seconds so that k8s can act on the applied changes. echo "Sleeping for 60 seconds..." sleep 60 # Wait for 2 mins for the CSI deployment to be ready. -if ! kubectl rollout status deployment/vsphere-csi-controller -n vmware-system-csi --timeout=120s; +if ! kubectl wait deployment -n vmware-system-csi vsphere-csi-controller --for=jsonpath="{.status.readyReplicas}"=3 --timeout=120s; then - echo "CSI deployment is not ready within 120s." - exit 1 + echo "CSI deployment is not ready within 120s." + exit 1 fi -# Pod status on testbed after patching the CSI Images -kubectl get pods -n vmware-system-csi - echo "Successfully patched the CSI images." diff --git a/pipeline/dev/patch.yaml b/pipeline/dev/patch.yaml index 55c18fbe87..0b4cfe0ec6 100644 --- a/pipeline/dev/patch.yaml +++ b/pipeline/dev/patch.yaml @@ -11,24 +11,35 @@ spec: image: ${VSPHERE_CSI_CONTROLLER_IMAGE} - name: vsphere-syncer image: ${VSPHERE_SYNCER_IMAGE} + - name: csi-provisioner + image: localhost:5000/vmware/csi-provisioner/csi-provisioner:v3.1.0_vmware.2 + - name: csi-attacher + image: localhost:5000/vmware.io/csi-attacher:v3.4.0_vmware.1 + - name: csi-resizer + image: localhost:5000/vmware/kubernetes-csi_external-resizer/kubernetes-csi_external-resizer:v1.4.0_vmware.1 + - name: liveness-probe + image: localhost:5000/vmware.io/csi-livenessprobe:v2.6.0_vmware.1 --- - -kind: Deployment -apiVersion: apps/v1 -metadata: - name: vsphere-csi-webhook - namespace: vmware-system-csi -spec: - template: - spec: - containers: - - name: vsphere-webhook - image: ${VSPHERE_SYNCER_IMAGE} --- apiVersion: v1 data: + "volume-extend": "true" + "volume-health": "true" + "online-volume-extend": "true" + "file-volume": "true" + "csi-auth-check": "true" + "trigger-csi-fullsync": "false" + "csi-sv-feature-states-replication": "true" + "fake-attach": "true" + "async-query-volume": "true" + "improved-csi-idempotency": "true" + "block-volume-snapshot": "false" + "sibling-replica-bound-pvc-check": "true" # Set tkgs-ha to false, else csi-controller and csi-syncer containers do not start. "tkgs-ha": "false" + "list-volumes": "false" + "cnsmgr-suspend-create-volume": "true" + "listview-tasks": "false" kind: ConfigMap metadata: name: csi-feature-states diff --git a/pipeline/e2e-tests-staging.sh b/pipeline/e2e-tests-staging.sh index daa6e61e83..ce75ecd789 100755 --- a/pipeline/e2e-tests-staging.sh +++ b/pipeline/e2e-tests-staging.sh @@ -1,47 +1,8 @@ #!/bin/bash -# Copyright 2023 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +set -x -set +x +echo "Running e2e tests in the staging environment..." +# TODO: Implement the script to run the e2e tests. -export E2E_TEST_CONF_FILE=$CI_BUILDS_DIR/e2eTest.conf - -tee "$E2E_TEST_CONF_FILE" >/dev/null < sv_kubeconfig_content.yaml -kubeconfigPath="$(pwd)/sv_kubeconfig_content.yaml" - -echo "$vcIp" -echo "$vcRootPassword" > vc_pwd - -export GOVC_INSECURE=1 -export GOVC_URL="https://$vimUsername:$vimPassword@$vcIp" - -DATACENTER=$(govc datacenter.info | grep -i path | awk '{print $2}') -export DATACENTER=$DATACENTER - -COMPUTE_CLUSTER_NAME=$(govc namespace.cluster.ls | awk -F'/' '{print $5}') -export COMPUTE_CLUSTER_NAME=$COMPUTE_CLUSTER_NAME - -export E2E_TEST_CONF_FILE=$CI_BUILDS_DIR/e2eTest.conf - -echo "$GOVC_URL" -echo "$DATACENTER" -echo "$COMPUTE_CLUSTER_NAME" -echo "$E2E_TEST_CONF_FILE" - -tee "$E2E_TEST_CONF_FILE" >/dev/null < /dev/null diff --git a/pkg/apis/migration/migration.go b/pkg/apis/migration/migration.go index 56ca341f74..f5b48629b1 100644 --- a/pkg/apis/migration/migration.go +++ b/pkg/apis/migration/migration.go @@ -35,7 +35,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/cache" "sigs.k8s.io/controller-runtime/pkg/client" - migrationconfig "sigs.k8s.io/vsphere-csi-driver/v3/pkg/apis/migration/config" migrationv1alpha1 "sigs.k8s.io/vsphere-csi-driver/v3/pkg/apis/migration/v1alpha1" @@ -68,13 +67,9 @@ type VolumeMigrationService interface { GetVolumeID(ctx context.Context, volumeSpec *VolumeSpec, registerIfNotFound bool) (string, error) // GetVolumePath returns VolumePath for a given VolumeID. - // It will also create a CnsVSphereVolumeMigration CR if it's not present. + // Returns an error if not able to retrieve VolumePath. GetVolumePath(ctx context.Context, volumeID string) (string, error) - // GetVolumePathFromMigrationServiceCache checks the in-memory cache for a volumeID - // a cache hit means that the volume is a migrated in-tree volume - GetVolumePathFromMigrationServiceCache(ctx context.Context, volumeID string) (string, error) - // DeleteVolumeInfo helps delete mapping of volumePath to VolumeID for // specified volumeID. DeleteVolumeInfo(ctx context.Context, volumeID string) error @@ -367,26 +362,6 @@ func (volumeMigration *volumeMigration) GetVolumePath(ctx context.Context, volum return fileBackingInfo.FilePath, nil } -// GetVolumePathFromMigrationServiceCache checks the in-memory cache for a volumeID -// a cache hit means that the volume is a migrated in-tree volume -func (volumeMigration *volumeMigration) GetVolumePathFromMigrationServiceCache(ctx context.Context, - volumeID string) (string, error) { - log := logger.GetLogger(ctx) - var volumePath string - volumeMigration.volumePathToVolumeID.Range(func(key, value interface{}) bool { - if value.(string) == volumeID { - volumePath = key.(string) - log.Infof("Found VolumePath %v for VolumeID: %q in the cache", volumePath, volumeID) - return false - } - return true - }) - if volumePath != "" { - return volumePath, nil - } - return "", common.ErrNotFound -} - // saveVolumeInfo helps create CR for given cnsVSphereVolumeMigration. This func // also update local cache with supplied cnsVSphereVolumeMigration, after // successful creation of CR @@ -610,41 +585,13 @@ func (volumeMigration *volumeMigration) cleanupStaleCRDInstances() { cnsVolumesMap[vol.VolumeId.Id] = true } log.Debugf("cnsVolumesMap: %v:", cnsVolumesMap) - k8sclient, err := k8s.NewClient(ctx) - if err != nil { - log.Errorf("failed to get k8sclient with error: %v", err) - continue - } - // The runCleanupRoutine is only triggered from Syncer container, - // hence we are checking for PV objects in the k8s in the below code snippet. - pvList, err := k8sclient.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{}) - if err != nil { - log.Errorf("failed to list PersistentVolumes with error %v.", err) - continue - } for _, volumeMigrationResource := range volumeMigrationResourceList.Items { if _, existsInCNSVolumesMap := cnsVolumesMap[volumeMigrationResource.Name]; !existsInCNSVolumesMap { log.Debugf("Volume with id %s is not found in CNS", volumeMigrationResource.Name) - // Check if a PV exists for the given volumePath in CnsVSphereVolumeMigration CR - pvFound := false - for _, pv := range pvList.Items { - if pv.Spec.VsphereVolume != nil && - pv.Spec.VsphereVolume.VolumePath == volumeMigrationResource.Spec.VolumePath { - pvFound = true - log.Infof("PV %s with VolumePath %s is found in k8s, but CNS does not have an entry."+ - "Skipping the %s CR deletion.", - pv.Name, pv.Spec.VsphereVolume.VolumePath, volumeMigrationResource.Name) - break - } - } - // Delete the CnsVSphereVolumeMigration CR only when there is no corresponding PV in k8s - if !pvFound { - log.Debugf("Deleting CnsVSphereVolumeMigration CR: %s", volumeMigrationResource.Name) - err = volumeMigrationInstance.DeleteVolumeInfo(ctx, volumeMigrationResource.Name) - if err != nil { - log.Warnf("failed to delete volume mapping CR for %s with error %+v", volumeMigrationResource.Name, err) - continue - } + err = volumeMigrationInstance.DeleteVolumeInfo(ctx, volumeMigrationResource.Name) + if err != nil { + log.Warnf("failed to delete volume mapping CR for %s with error %+v", volumeMigrationResource.Name, err) + continue } } } diff --git a/pkg/common/cns-lib/node/manager.go b/pkg/common/cns-lib/node/manager.go index fe5af29492..71e1673087 100644 --- a/pkg/common/cns-lib/node/manager.go +++ b/pkg/common/cns-lib/node/manager.go @@ -40,6 +40,10 @@ var ( type Manager interface { // SetKubernetesClient sets kubernetes client for node manager. SetKubernetesClient(client clientset.Interface) + // SetUseNodeUuid sets whether the node manager should use + // K8s CSINode API object or the K8s Node API object to retrieve + // the node UUID. + SetUseNodeUuid(useNodeUuid bool) // RegisterNode registers a node given its UUID, name. RegisterNode(ctx context.Context, nodeUUID string, nodeName string) error // DiscoverNode discovers a registered node given its UUID. This method @@ -52,16 +56,13 @@ type Manager interface { // given its UUID. If datacenter is present, GetNode will search within this // datacenter given its UUID. If not, it will search in all registered // datacenters. - GetNodeVMAndUpdateCache(ctx context.Context, nodeUUID string, dc *vsphere.Datacenter) (*vsphere.VirtualMachine, error) - // GetNodeVMByUuid returns the VirtualMachine for a registered node - // given its UUID. - GetNodeVMByUuid(ctx context.Context, nodeUUID string) (*vsphere.VirtualMachine, error) - // GetNodeVMByNameAndUpdateCache refreshes and returns the VirtualMachine for a registered + GetNode(ctx context.Context, nodeUUID string, dc *vsphere.Datacenter) (*vsphere.VirtualMachine, error) + // GetNodeByName refreshes and returns the VirtualMachine for a registered // node given its name. - GetNodeVMByNameAndUpdateCache(ctx context.Context, nodeName string) (*vsphere.VirtualMachine, error) - // GetNodeVMByNameOrUUID refreshes and returns VirtualMachine for a registered node + GetNodeByName(ctx context.Context, nodeName string) (*vsphere.VirtualMachine, error) + // GetNodeByNameOrUUID refreshes and returns VirtualMachine for a registered node // using either its name or UUID. - GetNodeVMByNameOrUUID(ctx context.Context, nodeNameOrUuid string) (*vsphere.VirtualMachine, error) + GetNodeByNameOrUUID(ctx context.Context, nodeName string) (*vsphere.VirtualMachine, error) // GetNodeNameByUUID fetches the name of the node given the VM UUID. GetNodeNameByUUID(ctx context.Context, nodeUUID string) (string, error) // GetAllNodes refreshes and returns VirtualMachine for all registered @@ -107,6 +108,9 @@ type defaultManager struct { nodeNameToUUID sync.Map // k8s client. k8sClient clientset.Interface + // useNodeUuid uses K8s CSINode API instead of + // K8s Node to retrieve the node UUID. + useNodeUuid bool } // SetKubernetesClient sets specified kubernetes client to defaultManager.k8sClient @@ -114,6 +118,13 @@ func (m *defaultManager) SetKubernetesClient(client clientset.Interface) { m.k8sClient = client } +// SetUseNodeUuid sets whether the node manager should use +// K8s CSINode API object or the K8s Node API object to retrieve +// node UUID. +func (m *defaultManager) SetUseNodeUuid(useNodeUuid bool) { + m.useNodeUuid = useNodeUuid +} + // RegisterNode registers a node with node manager using its UUID, name. func (m *defaultManager) RegisterNode(ctx context.Context, nodeUUID string, nodeName string) error { log := logger.GetLogger(ctx) @@ -144,27 +155,27 @@ func (m *defaultManager) DiscoverNode(ctx context.Context, nodeUUID string) erro return nil } -// GetNodeVMByNameAndUpdateCache refreshes and returns the VirtualMachine for a registered node +// GetNodeByName refreshes and returns the VirtualMachine for a registered node // given its name. -func (m *defaultManager) GetNodeVMByNameAndUpdateCache(ctx context.Context, - nodeName string) (*vsphere.VirtualMachine, error) { +func (m *defaultManager) GetNodeByName(ctx context.Context, nodeName string) (*vsphere.VirtualMachine, error) { log := logger.GetLogger(ctx) nodeUUID, found := m.nodeNameToUUID.Load(nodeName) if found && nodeUUID != nil && nodeUUID.(string) != "" { - return m.GetNodeVMAndUpdateCache(ctx, nodeUUID.(string), nil) + return m.GetNode(ctx, nodeUUID.(string), nil) } log.Infof("Empty nodeUUID observed in cache for the node: %q", nodeName) - k8snodeUUID, err := k8s.GetNodeUUID(ctx, m.k8sClient, nodeName) + k8snodeUUID, err := k8s.GetNodeUUID(ctx, m.k8sClient, nodeName, + m.useNodeUuid) if err != nil { log.Errorf("failed to get node UUID from node: %q. Err: %v", nodeName, err) return nil, err } m.nodeNameToUUID.Store(nodeName, k8snodeUUID) - return m.GetNodeVMAndUpdateCache(ctx, k8snodeUUID, nil) + return m.GetNode(ctx, k8snodeUUID, nil) } -func (m *defaultManager) GetNodeVMByNameOrUUID( +func (m *defaultManager) GetNodeByNameOrUUID( ctx context.Context, nodeNameOrUUID string) (*vsphere.VirtualMachine, error) { log := logger.GetLogger(ctx) nodeUUID, found := m.nodeNameToUUID.Load(nodeNameOrUUID) @@ -173,15 +184,16 @@ func (m *defaultManager) GetNodeVMByNameOrUUID( return nil, ErrNodeNotFound } if nodeUUID != nil && nodeUUID.(string) != "" { - return m.GetNodeVMAndUpdateCache(ctx, nodeUUID.(string), nil) + return m.GetNode(ctx, nodeUUID.(string), nil) } log.Infof("Empty nodeUUID observed in cache for the node: %q", nodeNameOrUUID) - k8snodeUUID, err := k8s.GetNodeUUID(ctx, m.k8sClient, nodeNameOrUUID) + k8snodeUUID, err := k8s.GetNodeUUID(ctx, m.k8sClient, nodeNameOrUUID, m.useNodeUuid) if err != nil { log.Errorf("failed to get node UUID from node: %q. Err: %v", nodeNameOrUUID, err) return nil, err } - return m.GetNodeVMAndUpdateCache(ctx, k8snodeUUID, nil) + m.nodeNameToUUID.Store(nodeNameOrUUID, k8snodeUUID) + return m.GetNode(ctx, k8snodeUUID, nil) } // GetNodeNameByUUID fetches the name of the node given the VM UUID. @@ -214,7 +226,7 @@ func (m *defaultManager) GetK8sNode(ctx context.Context, nodename string) (*v1.N // GetNode refreshes and returns the VirtualMachine for a registered node // given its UUID. -func (m *defaultManager) GetNodeVMAndUpdateCache(ctx context.Context, +func (m *defaultManager) GetNode(ctx context.Context, nodeUUID string, dc *vsphere.Datacenter) (*vsphere.VirtualMachine, error) { log := logger.GetLogger(ctx) vmInf, discovered := m.nodeVMs.Load(nodeUUID) @@ -254,27 +266,6 @@ func (m *defaultManager) GetNodeVMAndUpdateCache(ctx context.Context, return vm, nil } -// GetNodeVMByUuid returns the VirtualMachine for a registered node -// given its UUID. This is called by ControllerPublishVolume and -// ControllerUnpublishVolume to perform attach and detach operations. -func (m *defaultManager) GetNodeVMByUuid(ctx context.Context, - nodeUUID string) (*vsphere.VirtualMachine, error) { - log := logger.GetLogger(ctx) - vmInf, discovered := m.nodeVMs.Load(nodeUUID) - if !discovered { - log.Infof("Node VM not found with nodeUUID %s", nodeUUID) - vm, err := vsphere.GetVirtualMachineByUUID(ctx, nodeUUID, false) - if err != nil { - log.Errorf("Couldn't find VM instance with nodeUUID %s, failed to discover with err: %v", nodeUUID, err) - return nil, err - } - log.Infof("Node was successfully found with nodeUUID %s in vm %v", nodeUUID, vm) - return vm, nil - } - vm := vmInf.(*vsphere.VirtualMachine) - return vm, nil -} - // GetAllNodes refreshes and returns VirtualMachine for all registered nodes. func (m *defaultManager) GetAllNodes(ctx context.Context) ([]*vsphere.VirtualMachine, error) { log := logger.GetLogger(ctx) @@ -285,7 +276,8 @@ func (m *defaultManager) GetAllNodes(ctx context.Context) ([]*vsphere.VirtualMac m.nodeNameToUUID.Range(func(nodeName, nodeUUID interface{}) bool { if nodeName != nil && nodeUUID != nil && nodeUUID.(string) == "" { log.Infof("Empty node UUID observed for the node: %q", nodeName) - k8snodeUUID, err := k8s.GetNodeUUID(ctx, m.k8sClient, nodeName.(string)) + k8snodeUUID, err := k8s.GetNodeUUID(ctx, m.k8sClient, + nodeName.(string), m.useNodeUuid) if err != nil { log.Errorf("failed to get node UUID from node: %q. Err: %v", nodeName, err) return true @@ -351,7 +343,8 @@ func (m *defaultManager) GetAllNodesByVC(ctx context.Context, vcHost string) ([] m.nodeNameToUUID.Range(func(nodeName, nodeUUID interface{}) bool { if nodeName != nil && nodeUUID != nil && nodeUUID.(string) == "" { log.Infof("Empty node UUID observed for the node: %q", nodeName) - k8snodeUUID, err := k8s.GetNodeUUID(ctx, m.k8sClient, nodeName.(string)) + k8snodeUUID, err := k8s.GetNodeUUID(ctx, m.k8sClient, + nodeName.(string), m.useNodeUuid) if err != nil { log.Errorf("failed to get node UUID from node: %q. Err: %v", nodeName, err) return true diff --git a/pkg/common/cns-lib/node/manager_test.go b/pkg/common/cns-lib/node/manager_test.go new file mode 100644 index 0000000000..e7692f62ae --- /dev/null +++ b/pkg/common/cns-lib/node/manager_test.go @@ -0,0 +1,50 @@ +package node + +import ( + "context" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" +) + +func TestDefaultManager_GetNodeByName(t *testing.T) { + nodeName := "foobar.dev.lan" + m := defaultManager{ + nodeVMs: sync.Map{}, + nodeNameToUUID: sync.Map{}, + k8sClient: nil, + useNodeUuid: false, + } + + k8sClient := k8sClientWithNodes(nodeName) + m.SetKubernetesClient(k8sClient) + + vm, _ := m.GetNodeByName(context.TODO(), nodeName) + if vm != nil { + t.Errorf("Unexpected vm found:%v", vm) + } + + nodeUUID, ok := m.nodeNameToUUID.Load(nodeName) + if !ok { + t.Errorf("node name should be loaded into nodeUUID map") + } + assert.Equal(t, "foobar", nodeUUID) +} + +func k8sClientWithNodes(nodeName string) clientset.Interface { + node := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + }, + Spec: v1.NodeSpec{ + ProviderID: "vsphere://foobar", + }, + } + client := fake.NewSimpleClientset(node) + return client +} diff --git a/pkg/common/cns-lib/node/nodes.go b/pkg/common/cns-lib/node/nodes.go index 031e17c561..06989e6401 100644 --- a/pkg/common/cns-lib/node/nodes.go +++ b/pkg/common/cns-lib/node/nodes.go @@ -20,6 +20,7 @@ import ( "context" "fmt" + v1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" cnsvsphere "sigs.k8s.io/vsphere-csi-driver/v3/pkg/common/cns-lib/vsphere" @@ -34,8 +35,11 @@ type Nodes struct { } // Initialize helps initialize node manager and node informer manager. -func (nodes *Nodes) Initialize(ctx context.Context) error { +// If useNodeUuid is set, an informer on K8s CSINode is created. +// if not, an informer on K8s Node API object is created. +func (nodes *Nodes) Initialize(ctx context.Context, useNodeUuid bool) error { nodes.cnsNodeManager = GetManager(ctx) + nodes.cnsNodeManager.SetUseNodeUuid(useNodeUuid) k8sclient, err := k8s.NewClient(ctx) if err != nil { log := logger.GetLogger(ctx) @@ -44,12 +48,68 @@ func (nodes *Nodes) Initialize(ctx context.Context) error { } nodes.cnsNodeManager.SetKubernetesClient(k8sclient) nodes.informMgr = k8s.NewInformer(ctx, k8sclient, true) - nodes.informMgr.AddCSINodeListener(nodes.csiNodeAdd, - nodes.csiNodeUpdate, nodes.csiNodeDelete) + if useNodeUuid { + nodes.informMgr.AddCSINodeListener(nodes.csiNodeAdd, + nodes.csiNodeUpdate, nodes.csiNodeDelete) + } else { + nodes.informMgr.AddNodeListener(nodes.nodeAdd, + nodes.nodeUpdate, nodes.nodeDelete) + } nodes.informMgr.Listen() return nil } +func (nodes *Nodes) nodeAdd(obj interface{}) { + ctx, log := logger.GetNewContextWithLogger() + node, ok := obj.(*v1.Node) + if node == nil || !ok { + log.Warnf("nodeAdd: unrecognized object %+v", obj) + return + } + err := nodes.cnsNodeManager.RegisterNode(ctx, + cnsvsphere.GetUUIDFromProviderID(node.Spec.ProviderID), node.Name) + if err != nil { + log.Warnf("failed to register node:%q. err=%v", node.Name, err) + } +} + +func (nodes *Nodes) nodeUpdate(oldObj interface{}, newObj interface{}) { + ctx, log := logger.GetNewContextWithLogger() + newNode, ok := newObj.(*v1.Node) + if !ok { + log.Warnf("nodeUpdate: unrecognized object newObj %[1]T%+[1]v", newObj) + return + } + oldNode, ok := oldObj.(*v1.Node) + if !ok { + log.Warnf("nodeUpdate: unrecognized object oldObj %[1]T%+[1]v", oldObj) + return + } + if oldNode.Spec.ProviderID != newNode.Spec.ProviderID { + log.Infof("nodeUpdate: Observed ProviderID change from %q to %q for the node: %q", + oldNode.Spec.ProviderID, newNode.Spec.ProviderID, newNode.Name) + + err := nodes.cnsNodeManager.RegisterNode(ctx, + cnsvsphere.GetUUIDFromProviderID(newNode.Spec.ProviderID), newNode.Name) + if err != nil { + log.Warnf("nodeUpdate: Failed to register node:%q. err=%v", newNode.Name, err) + } + } +} + +func (nodes *Nodes) nodeDelete(obj interface{}) { + ctx, log := logger.GetNewContextWithLogger() + node, ok := obj.(*v1.Node) + if node == nil || !ok { + log.Warnf("nodeDelete: unrecognized object %+v", obj) + return + } + err := nodes.cnsNodeManager.UnregisterNode(ctx, node.Name) + if err != nil { + log.Warnf("failed to unregister node:%q. err=%v", node.Name, err) + } +} + func (nodes *Nodes) csiNodeAdd(obj interface{}) { ctx, log := logger.GetNewContextWithLogger() csiNode, ok := obj.(*storagev1.CSINode) @@ -122,19 +182,19 @@ func (nodes *Nodes) csiNodeDelete(obj interface{}) { } } -// GetNodeVMByNameAndUpdateCache returns VirtualMachine object for given nodeName. +// GetNodeByName returns VirtualMachine object for given nodeName. // This is called by ControllerPublishVolume and ControllerUnpublishVolume // to perform attach and detach operations. -func (nodes *Nodes) GetNodeVMByNameAndUpdateCache(ctx context.Context, nodeName string) ( +func (nodes *Nodes) GetNodeByName(ctx context.Context, nodeName string) ( *cnsvsphere.VirtualMachine, error) { - return nodes.cnsNodeManager.GetNodeVMByNameAndUpdateCache(ctx, nodeName) + return nodes.cnsNodeManager.GetNodeByName(ctx, nodeName) } -// GetNodeVMByNameOrUUID returns VirtualMachine object for given nodeName +// GetNodeByNameOrUUID returns VirtualMachine object for given nodeName // This function can be called either using nodeName or nodeUID. -func (nodes *Nodes) GetNodeVMByNameOrUUID( +func (nodes *Nodes) GetNodeByNameOrUUID( ctx context.Context, nodeNameOrUUID string) (*cnsvsphere.VirtualMachine, error) { - return nodes.cnsNodeManager.GetNodeVMByNameOrUUID(ctx, nodeNameOrUUID) + return nodes.cnsNodeManager.GetNodeByNameOrUUID(ctx, nodeNameOrUUID) } // GetNodeNameByUUID fetches the name of the node given the VM UUID. @@ -143,11 +203,11 @@ func (nodes *Nodes) GetNodeNameByUUID(ctx context.Context, nodeUUID string) ( return nodes.cnsNodeManager.GetNodeNameByUUID(ctx, nodeUUID) } -// GetNodeVMByUuid returns VirtualMachine object for given nodeUuid. +// GetNodeByUuid returns VirtualMachine object for given nodeUuid. // This is called by ControllerPublishVolume and ControllerUnpublishVolume // to perform attach and detach operations. -func (nodes *Nodes) GetNodeVMByUuid(ctx context.Context, nodeUuid string) (*cnsvsphere.VirtualMachine, error) { - return nodes.cnsNodeManager.GetNodeVMByUuid(ctx, nodeUuid) +func (nodes *Nodes) GetNodeByUuid(ctx context.Context, nodeUuid string) (*cnsvsphere.VirtualMachine, error) { + return nodes.cnsNodeManager.GetNode(ctx, nodeUuid, nil) } // GetAllNodes returns VirtualMachine objects for all registered nodes in cluster. diff --git a/pkg/common/cns-lib/volume/listview.go b/pkg/common/cns-lib/volume/listview.go index 1da95db8a1..3e1e3e1fce 100644 --- a/pkg/common/cns-lib/volume/listview.go +++ b/pkg/common/cns-lib/volume/listview.go @@ -12,7 +12,6 @@ import ( "github.com/vmware/govmomi/vim25/types" cnsvsphere "sigs.k8s.io/vsphere-csi-driver/v3/pkg/common/cns-lib/vsphere" - "sigs.k8s.io/vsphere-csi-driver/v3/pkg/common/config" "sigs.k8s.io/vsphere-csi-driver/v3/pkg/csi/service/logger" ) @@ -88,16 +87,21 @@ func (l *ListViewImpl) createListView(ctx context.Context, tasks []types.Managed return err } l.listView = listView - log.Infof("created listView object %+v for virtualCenter: %+v", - l.listView.Reference(), l.virtualCenter.Config.Host) + log.Infof("created listView object %+v for virtualCenter: %+v", l.listView.Reference(), l.virtualCenter) return nil } // SetVirtualCenter is a setter method for vc. use case: ReloadConfiguration -func (l *ListViewImpl) SetVirtualCenter(ctx context.Context, virtualCenter *cnsvsphere.VirtualCenter) { +func (l *ListViewImpl) SetVirtualCenter(ctx context.Context, virtualCenter *cnsvsphere.VirtualCenter) error { log := logger.GetLogger(ctx) l.virtualCenter = virtualCenter - log.Debugf("New virtualCenter object stored for use by ListView") + client, err := virtualCenter.NewClient(ctx) + if err != nil { + return logger.LogNewErrorf(log, "failed to create a govmomiClient for listView. error: %+v", err) + } + client.Timeout = noTimeout + l.govmomiClient = client + return nil } func getListViewWaitFilter(listView *view.ListView) *property.WaitFilter { @@ -118,6 +122,11 @@ func getListViewWaitFilter(listView *view.ListView) *property.WaitFilter { func (l *ListViewImpl) AddTask(ctx context.Context, taskMoRef types.ManagedObjectReference, ch chan TaskResult) error { log := logger.GetLogger(ctx) log.Infof("AddTask called for %+v", taskMoRef) + err := l.listView.Add(l.ctx, []types.ManagedObjectReference{taskMoRef}) + if err != nil { + return logger.LogNewErrorf(log, "failed to add task to ListView. error: %+v", err) + } + log.Infof("task %+v added to listView", taskMoRef) l.taskMap.Upsert(taskMoRef, TaskDetails{ Reference: taskMoRef, @@ -125,13 +134,6 @@ func (l *ListViewImpl) AddTask(ctx context.Context, taskMoRef types.ManagedObjec ResultCh: ch, }) log.Debugf("task %+v added to map", taskMoRef) - - err := l.listView.Add(l.ctx, []types.ManagedObjectReference{taskMoRef}) - if err != nil { - l.taskMap.Delete(taskMoRef) - return logger.LogNewErrorf(log, "failed to add task to ListView. error: %+v", err) - } - log.Infof("task %+v added to listView", taskMoRef) return nil } @@ -164,18 +166,8 @@ func (l *ListViewImpl) isClientValid() error { } else if userSession != nil { return nil } - - err := cnsvsphere.ReadVCConfigs(l.ctx, l.virtualCenter) - if err != nil { - return logger.LogNewErrorf(log, "failed to read VC config. err: %v", err) - } // If session has expired, create a new instance. - useragent, err := config.GetSessionUserAgent(l.ctx) - if err != nil { - return logger.LogNewErrorf(log, "failed to get useragent for vCenter session. error: %+v", err) - } - useragent = useragent + "-listview" - client, err := l.virtualCenter.NewClient(l.ctx, useragent) + client, err := l.virtualCenter.NewClient(l.ctx) if err != nil { return logger.LogNewErrorf(log, "failed to create a govmomi client for listView. error: %+v", err) } @@ -237,8 +229,7 @@ func (l *ListViewImpl) listenToTaskUpdates() { // we want to immediately return a fault for all the pending tasks in the map // note: this is not a task error but an error from the vc if err != nil { - log.Errorf("WaitForUpdates returned err: %v for vc: %+v", err, - l.virtualCenter.Config.Host) + log.Errorf("WaitForUpdates returned err: %v for vc: %+v", err, l.virtualCenter) recreateView = true l.reportErrorOnAllPendingTasks(err) } @@ -328,15 +319,3 @@ func (l *ListViewImpl) MarkTaskForDeletion(ctx context.Context, taskMoRef types. log.Infof("%v marked for deletion", taskMoRef) return nil } - -// LogoutSession is a setter method to logout vcenter session created -func (l *ListViewImpl) LogoutSession(ctx context.Context) error { - log := logger.GetLogger(ctx) - err := l.govmomiClient.Logout(l.ctx) - if err != nil { - log.Errorf("Error while logout vCenter session (list-view) for host %s, Error: %+v", l.virtualCenter.Config.Host, err) - return err - } - log.Infof("Logged out list-view vCenter session for host %s", l.virtualCenter.Config.Host) - return nil -} diff --git a/pkg/common/cns-lib/volume/listview_if.go b/pkg/common/cns-lib/volume/listview_if.go index 9f0bbfaab0..036d95ef48 100644 --- a/pkg/common/cns-lib/volume/listview_if.go +++ b/pkg/common/cns-lib/volume/listview_if.go @@ -17,9 +17,7 @@ type ListViewIf interface { RemoveTask(ctx context.Context, taskMoRef types.ManagedObjectReference) error // SetVirtualCenter is a setter method for the reference to the global vcenter object. // use case: ReloadConfiguration - SetVirtualCenter(ctx context.Context, virtualCenter *cnsvsphere.VirtualCenter) - // LogoutSession logout the vCenter Session - LogoutSession(ctx context.Context) error + SetVirtualCenter(ctx context.Context, virtualCenter *cnsvsphere.VirtualCenter) error // MarkTaskForDeletion marks a given task MoRef for deletion by a cleanup goroutine // use case: failure to remove task due to a vc issue MarkTaskForDeletion(ctx context.Context, taskMoRef types.ManagedObjectReference) error diff --git a/pkg/common/cns-lib/volume/listview_test.go b/pkg/common/cns-lib/volume/listview_test.go index bfce9c2f39..0d4b9c45fc 100644 --- a/pkg/common/cns-lib/volume/listview_test.go +++ b/pkg/common/cns-lib/volume/listview_test.go @@ -18,10 +18,9 @@ import ( "sigs.k8s.io/vsphere-csi-driver/v3/pkg/csi/service/logger" ) -const testVCHost = "testVCHost" - func TestAddRemoveListView(t *testing.T) { ctx := logger.NewContextWithLogger(context.Background()) + model := simulator.VPX() defer model.Remove() if err := model.Create(); err != nil { @@ -38,9 +37,6 @@ func TestAddRemoveListView(t *testing.T) { t.Fatal(err) } - config := vsphere.VirtualCenterConfig{Host: testVCHost} - virtualCenter.Config = &config - listViewImpl, err := NewListViewImpl(ctx, virtualCenter, virtualCenter.Client) assert.NoError(t, err) listViewImpl.shouldStopListening = true @@ -87,9 +83,6 @@ func TestMarkForDeletion(t *testing.T) { t.Fatal(err) } - config := vsphere.VirtualCenterConfig{Host: testVCHost} - virtualCenter.Config = &config - listViewImpl, err := NewListViewImpl(ctx, virtualCenter, virtualCenter.Client) assert.NoError(t, err) listViewImpl.shouldStopListening = true diff --git a/pkg/common/cns-lib/volume/manager.go b/pkg/common/cns-lib/volume/manager.go index c26cdfa1aa..4efcd17105 100644 --- a/pkg/common/cns-lib/volume/manager.go +++ b/pkg/common/cns-lib/volume/manager.go @@ -35,7 +35,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" cnsvsphere "sigs.k8s.io/vsphere-csi-driver/v3/pkg/common/cns-lib/vsphere" - "sigs.k8s.io/vsphere-csi-driver/v3/pkg/common/config" csifault "sigs.k8s.io/vsphere-csi-driver/v3/pkg/common/fault" "sigs.k8s.io/vsphere-csi-driver/v3/pkg/common/prometheus" "sigs.k8s.io/vsphere-csi-driver/v3/pkg/csi/service/logger" @@ -55,11 +54,6 @@ const ( // used only for listView noTimeout = 0 * time.Minute - // VolumeOperationTimeoutInSeconds specifies the default CSI operation timeout in seconds - VolumeOperationTimeoutInSeconds = 300 - - listviewAdditionError = "failed to add task to list view" - // defaultOpsExpirationTimeInHours is expiration time for create volume operations. // TODO: This timeout will be configurable in future releases defaultOpsExpirationTimeInHours = 1 @@ -138,8 +132,6 @@ type Manager interface { task *object.Task, volNameFromInputSpec string, clusterID string) (*CnsVolumeInfo, string, error) // GetOperationStore returns the VolumeOperationRequest interface GetOperationStore() cnsvolumeoperationrequest.VolumeOperationRequest - // LogoutListViewVCSession logout current vCenter session for list-view - LogoutListViewVCSession(ctx context.Context) error } // CnsVolumeInfo hold information related to volume created by CNS. @@ -313,7 +305,10 @@ func (m *defaultManager) ResetManager(ctx context.Context, vcenter *cnsvsphere.V log.Infof("Re-initializing defaultManager.virtualCenter") managerInstance.virtualCenter = vcenter if m.tasksListViewEnabled { - m.listViewIf.SetVirtualCenter(ctx, managerInstance.virtualCenter) + err := m.listViewIf.SetVirtualCenter(ctx, managerInstance.virtualCenter) + if err != nil { + return logger.LogNewErrorf(log, "failed to set virtual center to listView instance. err: %v", err) + } } if m.virtualCenter.Client != nil { m.virtualCenter.Client.Timeout = time.Duration(vcenter.Config.VCClientTimeout) * time.Minute @@ -391,7 +386,6 @@ func (m *defaultManager) MonitorCreateVolumeTask(ctx context.Context, return nil, ExtractFaultTypeFromErr(ctx, err), err } - // WaitForResult can fail for many reasons, including: // - CNS restarted and marked "InProgress" tasks as "Failed". // - Any failures from CNS. @@ -500,9 +494,9 @@ func (m *defaultManager) createVolumeWithImprovedIdempotency(ctx context.Context }, }, "", nil } - // Validate if previous operation is pending. - if IsTaskPending(volumeOperationDetails) { + if volumeOperationDetails.OperationDetails.TaskStatus == taskInvocationStatusInProgress && + volumeOperationDetails.OperationDetails.TaskID != "" { log.Infof("Volume with name %s has CreateVolume task %s pending on CNS.", volNameFromInputSpec, volumeOperationDetails.OperationDetails.TaskID) @@ -569,21 +563,6 @@ func (m *defaultManager) createVolumeWithImprovedIdempotency(ctx context.Context return resp, faultType, err } -// IsTaskPending returns true in two cases - -// 1. if the task status was in progress -// 2. if the status was an error but the error was for adding the task to the listview -// (as we don't know the status of the task on CNS) -func IsTaskPending(volumeOperationDetails *cnsvolumeoperationrequest.VolumeOperationRequestDetails) bool { - if volumeOperationDetails.OperationDetails.TaskStatus == taskInvocationStatusInProgress && - volumeOperationDetails.OperationDetails.TaskID != "" { - return true - } else if volumeOperationDetails.OperationDetails.TaskStatus == taskInvocationStatusError && - strings.Contains(volumeOperationDetails.OperationDetails.Error, listviewAdditionError) { - return true - } - return false -} - func (m *defaultManager) waitOnTask(csiOpContext context.Context, taskMoRef vim25types.ManagedObjectReference) (*vim25types.TaskInfo, error) { log := logger.GetLogger(csiOpContext) @@ -596,7 +575,7 @@ func (m *defaultManager) waitOnTask(csiOpContext context.Context, ch := make(chan TaskResult) err := m.listViewIf.AddTask(csiOpContext, taskMoRef, ch) if err != nil { - return nil, logger.LogNewErrorf(log, "%s. err: %v", listviewAdditionError, err) + return nil, logger.LogNewErrorf(log, "failed to add task to list view. err: %v", err) } // deferring removal of task after response from CNS @@ -611,6 +590,7 @@ func (m *defaultManager) waitOnTask(csiOpContext context.Context, } } }() + return waitForResultOrTimeout(csiOpContext, taskMoRef, ch) } @@ -634,7 +614,6 @@ func waitForResultOrTimeout(csiOpContext context.Context, taskMoRef vim25types.M func (m *defaultManager) initListView() error { ctx := logger.NewContextWithLogger(context.Background()) - log := logger.GetLogger(ctx) log.Debugf("Initializing new listView object for vc: %+v", m.virtualCenter) if m.virtualCenter.Client == nil { @@ -645,18 +624,7 @@ func (m *defaultManager) initListView() error { } } - err := cnsvsphere.ReadVCConfigs(ctx, m.virtualCenter) - if err != nil { - return logger.LogNewErrorf(log, "failed to read VC config. err: %v", err) - } - - useragent, err := config.GetSessionUserAgent(ctx) - if err != nil { - return logger.LogNewErrorf(log, "failed to get useragent for vCenter session. error: %+v", err) - } - useragent = useragent + "-listview" - - govmomiClient, err := m.virtualCenter.NewClient(ctx, useragent) + govmomiClient, err := m.virtualCenter.NewClient(ctx) if err != nil { return logger.LogNewErrorf(log, "failed to create a separate govmomi client for listView. error: %+v", err) } @@ -772,8 +740,6 @@ func (m *defaultManager) createVolume(ctx context.Context, spec *cnstypes.CnsVol // CreateVolume creates a new volume given its spec. func (m *defaultManager) CreateVolume(ctx context.Context, spec *cnstypes.CnsVolumeCreateSpec) (*CnsVolumeInfo, string, error) { - ctx, cancelFunc := ensureOperationContextHasATimeout(ctx) - defer cancelFunc() internalCreateVolume := func() (*CnsVolumeInfo, string, error) { log := logger.GetLogger(ctx) var faultType string @@ -810,23 +776,9 @@ func (m *defaultManager) CreateVolume(ctx context.Context, spec *cnstypes.CnsVol return resp, faultType, err } -// ensureOperationContextHasATimeout checks if the passed context has a timeout associated with it. -// If there is no timeout set, we set it to 300 seconds. This is the same as set by sidecars. -// If a timeout is already set, we don't change it. -func ensureOperationContextHasATimeout(ctx context.Context) (context.Context, context.CancelFunc) { - _, ok := ctx.Deadline() - if !ok { - // no timeout is set, so we need to set it - return context.WithTimeout(ctx, VolumeOperationTimeoutInSeconds*time.Second) - } - return context.WithCancel(ctx) -} - // AttachVolume attaches a volume to a virtual machine given the spec. func (m *defaultManager) AttachVolume(ctx context.Context, vm *cnsvsphere.VirtualMachine, volumeID string, checkNVMeController bool) (string, string, error) { - ctx, cancelFunc := ensureOperationContextHasATimeout(ctx) - defer cancelFunc() internalAttachVolume := func() (string, string, error) { log := logger.GetLogger(ctx) var faultType string @@ -933,8 +885,6 @@ func (m *defaultManager) AttachVolume(ctx context.Context, // DetachVolume detaches a volume from the virtual machine given the spec. func (m *defaultManager) DetachVolume(ctx context.Context, vm *cnsvsphere.VirtualMachine, volumeID string) (string, error) { - ctx, cancelFunc := ensureOperationContextHasATimeout(ctx) - defer cancelFunc() internalDetachVolume := func() (string, error) { log := logger.GetLogger(ctx) var faultType string @@ -1075,8 +1025,6 @@ func (m *defaultManager) DetachVolume(ctx context.Context, vm *cnsvsphere.Virtua // DeleteVolume deletes a volume given its spec. func (m *defaultManager) DeleteVolume(ctx context.Context, volumeID string, deleteDisk bool) (string, error) { - ctx, cancelFunc := ensureOperationContextHasATimeout(ctx) - defer cancelFunc() internalDeleteVolume := func() (string, error) { log := logger.GetLogger(ctx) var faultType string @@ -1167,12 +1115,6 @@ func (m *defaultManager) deleteVolume(ctx context.Context, volumeID string, dele volumeOperationRes := taskResult.GetCnsVolumeOperationResult() if volumeOperationRes.Fault != nil { faultType = ExtractFaultTypeFromVolumeResponseResult(ctx, volumeOperationRes) - // If volume is not found on host, but is present in CNS DB, we will get vim.fault.NotFound fault. - // Send back success as the volume is already deleted. - if IsNotFoundFault(ctx, faultType) { - log.Infof("DeleteVolume: VolumeID %q, not found, thus returning success", volumeID) - return "", nil - } return faultType, logger.LogNewErrorf(log, "failed to delete volume: %q, fault: %q, opID: %q", volumeID, spew.Sdump(volumeOperationRes.Fault), taskInfo.ActivationId) } @@ -1215,7 +1157,8 @@ func (m *defaultManager) deleteVolumeWithImprovedIdempotency(ctx context.Context return "", nil } // Validate if previous operation is pending. - if IsTaskPending(volumeOperationDetails) { + if volumeOperationDetails.OperationDetails.TaskStatus == taskInvocationStatusInProgress && + volumeOperationDetails.OperationDetails.TaskID != "" { taskMoRef := vim25types.ManagedObjectReference{ Type: "Task", Value: volumeOperationDetails.OperationDetails.TaskID, @@ -1332,14 +1275,6 @@ func (m *defaultManager) deleteVolumeWithImprovedIdempotency(ctx context.Context volumeOperationRes := taskResult.GetCnsVolumeOperationResult() if volumeOperationRes.Fault != nil { faultType = ExtractFaultTypeFromVolumeResponseResult(ctx, volumeOperationRes) - - // If volume is not found on host, but is present in CNS DB, we will get vim.fault.NotFound fault. - // In such a case, send back success as the volume is already deleted. - if IsNotFoundFault(ctx, faultType) { - log.Infof("DeleteVolume: VolumeID %q, not found, thus returning success", volumeID) - return "", nil - } - msg := fmt.Sprintf("failed to delete volume: %q, fault: %q, opID: %q", volumeID, spew.Sdump(volumeOperationRes.Fault), taskInfo.ActivationId) volumeOperationDetails = createRequestDetails(instanceName, "", "", 0, @@ -1358,8 +1293,6 @@ func (m *defaultManager) deleteVolumeWithImprovedIdempotency(ctx context.Context // UpdateVolumeMetadata updates a volume given its spec. func (m *defaultManager) UpdateVolumeMetadata(ctx context.Context, spec *cnstypes.CnsVolumeMetadataUpdateSpec) error { - ctx, cancelFunc := ensureOperationContextHasATimeout(ctx) - defer cancelFunc() internalUpdateVolumeMetadata := func() error { log := logger.GetLogger(ctx) err := validateManager(ctx, m) @@ -1449,8 +1382,6 @@ func (m *defaultManager) UpdateVolumeMetadata(ctx context.Context, spec *cnstype // ExpandVolume expands a volume given its spec. func (m *defaultManager) ExpandVolume(ctx context.Context, volumeID string, size int64) (string, error) { - ctx, cancelFunc := ensureOperationContextHasATimeout(ctx) - defer cancelFunc() internalExpandVolume := func() (string, error) { log := logger.GetLogger(ctx) var faultType string @@ -1582,7 +1513,8 @@ func (m *defaultManager) expandVolumeWithImprovedIdempotency(ctx context.Context log.Infof("Volume with ID %s already expanded to size %v", volumeID, size) return "", nil } - if IsTaskPending(volumeOperationDetails) { + if volumeOperationDetails.OperationDetails.TaskStatus == taskInvocationStatusInProgress && + volumeOperationDetails.OperationDetails.TaskID != "" { log.Infof("Volume with ID %s has ExtendVolume task %s pending on CNS.", volumeID, volumeOperationDetails.OperationDetails.TaskID) @@ -1739,8 +1671,6 @@ func (m *defaultManager) expandVolumeWithImprovedIdempotency(ctx context.Context // QueryVolume returns volumes matching the given filter. func (m *defaultManager) QueryVolume(ctx context.Context, queryFilter cnstypes.CnsQueryFilter) (*cnstypes.CnsQueryResult, error) { - ctx, cancelFunc := ensureOperationContextHasATimeout(ctx) - defer cancelFunc() internalQueryVolume := func() (*cnstypes.CnsQueryResult, error) { log := logger.GetLogger(ctx) err := validateManager(ctx, m) @@ -1777,8 +1707,6 @@ func (m *defaultManager) QueryVolume(ctx context.Context, // QueryAllVolume returns all volumes matching the given filter and selection. func (m *defaultManager) QueryAllVolume(ctx context.Context, queryFilter cnstypes.CnsQueryFilter, querySelection cnstypes.CnsQuerySelection) (*cnstypes.CnsQueryResult, error) { - ctx, cancelFunc := ensureOperationContextHasATimeout(ctx) - defer cancelFunc() internalQueryAllVolume := func() (*cnstypes.CnsQueryResult, error) { log := logger.GetLogger(ctx) err := validateManager(ctx, m) @@ -1816,8 +1744,6 @@ func (m *defaultManager) QueryAllVolume(ctx context.Context, queryFilter cnstype // which CnsQueryVolumeInfoResult is extracted. func (m *defaultManager) QueryVolumeInfo(ctx context.Context, volumeIDList []cnstypes.CnsVolumeId) (*cnstypes.CnsQueryVolumeInfoResult, error) { - ctx, cancelFunc := ensureOperationContextHasATimeout(ctx) - defer cancelFunc() internalQueryVolumeInfo := func() (*cnstypes.CnsQueryVolumeInfoResult, error) { log := logger.GetLogger(ctx) err := validateManager(ctx, m) @@ -1885,8 +1811,6 @@ func (m *defaultManager) QueryVolumeInfo(ctx context.Context, func (m *defaultManager) RelocateVolume(ctx context.Context, relocateSpecList ...cnstypes.BaseCnsVolumeRelocateSpec) (*object.Task, error) { - ctx, cancelFunc := ensureOperationContextHasATimeout(ctx) - defer cancelFunc() internalRelocateVolume := func() (*object.Task, error) { log := logger.GetLogger(ctx) err := validateManager(ctx, m) @@ -1922,8 +1846,6 @@ func (m *defaultManager) RelocateVolume(ctx context.Context, // ConfigureVolumeACLs configures net permissions for a given CnsVolumeACLConfigureSpec. func (m *defaultManager) ConfigureVolumeACLs(ctx context.Context, spec cnstypes.CnsVolumeACLConfigureSpec) error { - ctx, cancelFunc := ensureOperationContextHasATimeout(ctx) - defer cancelFunc() internalConfigureVolumeACLs := func() error { log := logger.GetLogger(ctx) err := validateManager(ctx, m) @@ -2060,8 +1982,6 @@ func (m *defaultManager) RetrieveVStorageObject(ctx context.Context, // parameters are not specified. func (m *defaultManager) QueryVolumeAsync(ctx context.Context, queryFilter cnstypes.CnsQueryFilter, querySelection *cnstypes.CnsQuerySelection) (*cnstypes.CnsQueryResult, error) { - ctx, cancelFunc := ensureOperationContextHasATimeout(ctx) - defer cancelFunc() log := logger.GetLogger(ctx) err := validateManager(ctx, m) if err != nil { @@ -2126,8 +2046,6 @@ func (m *defaultManager) QueryVolumeAsync(ctx context.Context, queryFilter cnsty func (m *defaultManager) QuerySnapshots(ctx context.Context, snapshotQueryFilter cnstypes.CnsSnapshotQueryFilter) ( *cnstypes.CnsSnapshotQueryResult, error) { - ctx, cancelFunc := ensureOperationContextHasATimeout(ctx) - defer cancelFunc() internalQuerySnapshots := func() (*cnstypes.CnsSnapshotQueryResult, error) { log := logger.GetLogger(ctx) err := validateManager(ctx, m) @@ -2216,7 +2134,8 @@ func (m *defaultManager) createSnapshotWithImprovedIdempotencyCheck(ctx context. }, nil } // Validate if previous operation is pending. - if IsTaskPending(volumeOperationDetails) { + if volumeOperationDetails.OperationDetails.TaskStatus == taskInvocationStatusInProgress && + volumeOperationDetails.OperationDetails.TaskID != "" { log.Infof("Snapshot with name %s has CreateSnapshot task %s pending on CNS.", instanceName, volumeOperationDetails.OperationDetails.TaskID) @@ -2401,8 +2320,6 @@ func (m *defaultManager) createSnapshotWithImprovedIdempotencyCheck(ctx context. // which is generated by the CSI snapshotter sidecar. func (m *defaultManager) CreateSnapshot( ctx context.Context, volumeID string, snapshotName string) (*CnsSnapshotInfo, error) { - ctx, cancelFunc := ensureOperationContextHasATimeout(ctx) - defer cancelFunc() internalCreateSnapshot := func() (*CnsSnapshotInfo, error) { log := logger.GetLogger(ctx) err := validateManager(ctx, m) @@ -2462,7 +2379,8 @@ func (m *defaultManager) deleteSnapshotWithImprovedIdempotencyCheck( return nil } // Validate if previous operation is pending. - if IsTaskPending(volumeOperationDetails) { + if volumeOperationDetails.OperationDetails.TaskStatus == taskInvocationStatusInProgress && + volumeOperationDetails.OperationDetails.TaskID != "" { taskMoRef := vim25types.ManagedObjectReference{ Type: "Task", Value: volumeOperationDetails.OperationDetails.TaskID, @@ -2626,8 +2544,6 @@ func (m *defaultManager) deleteSnapshotWithImprovedIdempotencyCheck( } func (m *defaultManager) DeleteSnapshot(ctx context.Context, volumeID string, snapshotID string) error { - ctx, cancelFunc := ensureOperationContextHasATimeout(ctx) - defer cancelFunc() internalDeleteSnapshot := func() error { log := logger.GetLogger(ctx) err := validateManager(ctx, m) @@ -2680,23 +2596,3 @@ func (m *defaultManager) ProtectVolumeFromVMDeletion(ctx context.Context, volume log.Infof("Successfully set keepAfterDeleteVm control flag for volumeID: %q", volumeID) return nil } - -func (m *defaultManager) LogoutListViewVCSession(ctx context.Context) error { - log := logger.GetLogger(ctx) - if m.listViewIf != nil { - log.Info("Logging out list view vCenter session") - return m.listViewIf.LogoutSession(ctx) - } - return nil -} - -// GetAllManagerInstances returns all Manager instances -func GetAllManagerInstances(ctx context.Context) map[string]*defaultManager { - newManagerInstanceMap := make(map[string]*defaultManager) - if len(managerInstanceMap) != 0 { - newManagerInstanceMap = managerInstanceMap - } else if managerInstance != nil { - newManagerInstanceMap[managerInstance.virtualCenter.Config.Host] = managerInstance - } - return newManagerInstanceMap -} diff --git a/pkg/common/cns-lib/volume/util.go b/pkg/common/cns-lib/volume/util.go index 8fc0bdf7bc..db0e3bd121 100644 --- a/pkg/common/cns-lib/volume/util.go +++ b/pkg/common/cns-lib/volume/util.go @@ -37,6 +37,10 @@ import ( "sigs.k8s.io/vsphere-csi-driver/v3/pkg/csi/service/logger" ) +const ( + vimFaultPrefix = "vim.fault." +) + func validateManager(ctx context.Context, m *defaultManager) error { log := logger.GetLogger(ctx) if m.virtualCenter == nil { @@ -57,9 +61,6 @@ func IsDiskAttached(ctx context.Context, vm *cnsvsphere.VirtualMachine, volumeID log.Errorf("failed to get devices from vm: %s", vm.InventoryPath) return "", err } - if len(vmDevices) == 0 { - return "", logger.LogNewErrorf(log, "virtual devices list is empty for the vm: %s", vm.InventoryPath) - } // Build a map of NVME Controller key : NVME controller name. // This is needed to check if disk in contention is attached to a NVME // controller. The virtual disk devices do not contain the controller type @@ -349,7 +350,7 @@ func ExtractFaultTypeFromErr(ctx context.Context, err error) string { faultType = reflect.TypeOf(soapFault.VimFault()).String() log.Infof("Extract vimfault type: +%v. SoapFault Info: +%v from err +%v", faultType, soapFault, err) slice := strings.Split(faultType, ".") - vimFaultType := csifault.VimFaultPrefix + slice[1] + vimFaultType := vimFaultPrefix + slice[1] return vimFaultType } log.Infof("err %+v is not a SoapFault\n", err) @@ -373,7 +374,7 @@ func ExtractFaultTypeFromVolumeResponseResult(ctx context.Context, log.Infof("Extract vimfault type: %+v vimFault: %+v Fault: %+v from resp: %+v", faultType, fault.Fault, fault, resp) slice := strings.Split(faultType, ".") - vimFaultType := csifault.VimFaultPrefix + slice[1] + vimFaultType := vimFaultPrefix + slice[1] return vimFaultType } else { faultType = reflect.TypeOf(fault).String() @@ -549,11 +550,3 @@ func queryCreatedSnapshotByName(ctx context.Context, m *defaultManager, volumeID } return nil, false } - -// IsNotFoundFault returns true if a given faultType value is vim.fault.NotFound -func IsNotFoundFault(ctx context.Context, faultType string) bool { - log := logger.GetLogger(ctx) - log.Infof("Checking fault type: %q is vim.fault.NotFound", faultType) - return faultType == "vim.fault.NotFound" - -} diff --git a/pkg/common/cns-lib/vsphere/cluster_compute_resource.go b/pkg/common/cns-lib/vsphere/cluster_compute_resource.go deleted file mode 100644 index c2d8244af5..0000000000 --- a/pkg/common/cns-lib/vsphere/cluster_compute_resource.go +++ /dev/null @@ -1,50 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package vsphere - -import ( - "context" - - "github.com/vmware/govmomi/object" - "github.com/vmware/govmomi/vim25/mo" - - "sigs.k8s.io/vsphere-csi-driver/v3/pkg/csi/service/logger" -) - -// ClusterComputeResource holds details of a cluster instance. -type ClusterComputeResource struct { - // ClusterComputeResource represents a vSphere cluster. - *object.ClusterComputeResource - // VirtualCenterHost denotes the virtual center host address. - VirtualCenterHost string -} - -// GetHosts fetches the hosts under the ClusterComputeResource. -func (ccr *ClusterComputeResource) GetHosts(ctx context.Context) ([]*HostSystem, error) { - log := logger.GetLogger(ctx) - cluster := mo.ClusterComputeResource{} - err := ccr.Properties(ctx, ccr.Reference(), []string{"host"}, &cluster) - if err != nil { - return nil, logger.LogNewErrorf(log, - "failed to retrieve host property for cluster %+v", ccr.Reference()) - } - var hostList []*HostSystem - for _, host := range cluster.Host { - hostList = append(hostList, &HostSystem{HostSystem: object.NewHostSystem(ccr.Client(), host)}) - } - return hostList, nil -} diff --git a/pkg/common/cns-lib/vsphere/hostsystem.go b/pkg/common/cns-lib/vsphere/hostsystem.go index f8670050e4..1547e02af2 100644 --- a/pkg/common/cns-lib/vsphere/hostsystem.go +++ b/pkg/common/cns-lib/vsphere/hostsystem.go @@ -19,7 +19,6 @@ package vsphere import ( "context" "encoding/json" - "errors" "sigs.k8s.io/vsphere-csi-driver/v3/pkg/csi/service/logger" @@ -30,8 +29,6 @@ import ( "github.com/vmware/govmomi/vim25/types" ) -var ErrNoSharedDSFound = errors.New("no shared datastores found among given hosts") - // HostSystem holds details of a host instance. type HostSystem struct { // HostSystem represents the host system. @@ -43,7 +40,8 @@ type HostSystem struct { func (host *HostSystem) GetAllAccessibleDatastores(ctx context.Context) ([]*DatastoreInfo, error) { log := logger.GetLogger(ctx) var hostSystemMo mo.HostSystem - err := host.Properties(ctx, host.Reference(), []string{"datastore"}, &hostSystemMo) + s := object.NewSearchIndex(host.Client()) + err := s.Properties(ctx, host.Reference(), []string{"datastore"}, &hostSystemMo) if err != nil { log.Errorf("failed to retrieve datastores for host %v with err: %v", host, err) return nil, err @@ -184,38 +182,3 @@ func (host *HostSystem) GetHostVsanCapacity(ctx context.Context) (*VsanHostCapac } return &out, nil } - -// GetSharedDatastoresForHosts returns shared datastores accessible to hosts mentioned in the input parameter. -func GetSharedDatastoresForHosts(ctx context.Context, hosts []*HostSystem) ([]*DatastoreInfo, error) { - log := logger.GetLogger(ctx) - var sharedDatastores []*DatastoreInfo - - for _, host := range hosts { - accessibleDatastores, err := host.GetAllAccessibleDatastores(ctx) - if err != nil { - return nil, logger.LogNewErrorf(log, "failed to fetch datastores from host %+v. Error: %+v", - host, err) - } - if len(sharedDatastores) == 0 { - sharedDatastores = accessibleDatastores - } else { - var sharedAccessibleDatastores []*DatastoreInfo - for _, sharedDs := range sharedDatastores { - // Check if sharedDatastores is found in accessibleDatastores. - for _, accessibleDs := range accessibleDatastores { - // Intersection is performed based on the datastoreUrl as this - // uniquely identifies the datastore. - if accessibleDs.Info.Url == sharedDs.Info.Url { - sharedAccessibleDatastores = append(sharedAccessibleDatastores, sharedDs) - break - } - } - } - sharedDatastores = sharedAccessibleDatastores - } - if len(sharedDatastores) == 0 { - return nil, ErrNoSharedDSFound - } - } - return sharedDatastores, nil -} diff --git a/pkg/common/cns-lib/vsphere/utils.go b/pkg/common/cns-lib/vsphere/utils.go index f59461b5d0..fd39b38d6b 100644 --- a/pkg/common/cns-lib/vsphere/utils.go +++ b/pkg/common/cns-lib/vsphere/utils.go @@ -382,12 +382,10 @@ func signer(ctx context.Context, client *vim25.Client, username string, password // GetTagManager returns tagManager connected to given VirtualCenter. func GetTagManager(ctx context.Context, vc *VirtualCenter) (*tags.Manager, error) { - log := logger.GetLogger(ctx) // Validate input. if vc == nil || vc.Client == nil || vc.Client.Client == nil { return nil, fmt.Errorf("vCenter not initialized") } - restClient := rest.NewClient(vc.Client.Client) signer, err := signer(ctx, vc.Client.Client, vc.Config.Username, vc.Config.Password) if err != nil { @@ -406,7 +404,6 @@ func GetTagManager(ctx context.Context, vc *VirtualCenter) (*tags.Manager, error if tagManager == nil { return nil, fmt.Errorf("failed to create a tagManager") } - log.Infof("New tag manager with useragent '%s'", tagManager.UserAgent) return tagManager, nil } diff --git a/pkg/common/cns-lib/vsphere/virtualcenter.go b/pkg/common/cns-lib/vsphere/virtualcenter.go index be0fd71445..6f5d5ae801 100644 --- a/pkg/common/cns-lib/vsphere/virtualcenter.go +++ b/pkg/common/cns-lib/vsphere/virtualcenter.go @@ -28,23 +28,23 @@ import ( "sync" "time" - "github.com/vmware/govmomi" "github.com/vmware/govmomi/cns" + "github.com/vmware/govmomi/property" + "github.com/vmware/govmomi/vsan" + "github.com/vmware/govmomi/vslm" + "sigs.k8s.io/vsphere-csi-driver/v3/pkg/common/config" + "sigs.k8s.io/vsphere-csi-driver/v3/pkg/csi/service/logger" + + "github.com/vmware/govmomi" "github.com/vmware/govmomi/find" "github.com/vmware/govmomi/object" "github.com/vmware/govmomi/pbm" - "github.com/vmware/govmomi/property" "github.com/vmware/govmomi/session" "github.com/vmware/govmomi/sts" "github.com/vmware/govmomi/vim25" "github.com/vmware/govmomi/vim25/mo" "github.com/vmware/govmomi/vim25/soap" "github.com/vmware/govmomi/vim25/types" - "github.com/vmware/govmomi/vsan" - "github.com/vmware/govmomi/vslm" - - "sigs.k8s.io/vsphere-csi-driver/v3/pkg/common/config" - "sigs.k8s.io/vsphere-csi-driver/v3/pkg/csi/service/logger" ) const ( @@ -141,7 +141,7 @@ type VirtualCenterConfig struct { } // NewClient creates a new govmomi Client instance. -func (vc *VirtualCenter) NewClient(ctx context.Context, useragent string) (*govmomi.Client, error) { +func (vc *VirtualCenter) NewClient(ctx context.Context) (*govmomi.Client, error) { log := logger.GetLogger(ctx) if vc.Config.Scheme == "" { vc.Config.Scheme = DefaultScheme @@ -177,7 +177,7 @@ func (vc *VirtualCenter) NewClient(ctx context.Context, useragent string) (*govm log.Errorf("Failed to set vimClient service version to vsan. err: %v", err) return nil, err } - vimClient.UserAgent = useragent + vimClient.UserAgent = "k8s-csi-useragent" client := &govmomi.Client{ Client: vimClient, SessionManager: session.NewManager(vimClient), @@ -185,7 +185,6 @@ func (vc *VirtualCenter) NewClient(ctx context.Context, useragent string) (*govm err = vc.login(ctx, client) if err != nil { - log.Errorf("failed to login to vc. err: %v", err) return nil, err } @@ -280,20 +279,9 @@ func (vc *VirtualCenter) connect(ctx context.Context, requestNewSession bool) er // If client was never initialized, initialize one. var err error - useragent, err := config.GetSessionUserAgent(ctx) - if err != nil { - log.Errorf("failed to get useragent for vCenter session. error: %+v", err) - return err - } if vc.Client == nil { - if vc.Config.ReloadVCConfigForNewClient { - err = ReadVCConfigs(ctx, vc) - if err != nil { - return err - } - } log.Infof("VirtualCenter.connect() creating new client") - if vc.Client, err = vc.NewClient(ctx, useragent); err != nil { + if vc.Client, err = vc.NewClient(ctx); err != nil { log.Errorf("failed to create govmomi client with err: %v", err) if !vc.Config.Insecure { log.Errorf("failed to connect to vCenter using CA file: %q", vc.Config.CAFile) @@ -319,12 +307,30 @@ func (vc *VirtualCenter) connect(ctx context.Context, requestNewSession bool) er // If session has expired, create a new instance. log.Infof("Creating a new client session as the existing one isn't valid or not authenticated") if vc.Config.ReloadVCConfigForNewClient { - err = ReadVCConfigs(ctx, vc) + log.Info("Reloading latest VC config from vSphere Config Secret") + cfg, err := config.GetConfig(ctx) if err != nil { - return err + return logger.LogNewErrorf(log, "failed to read config. Error: %+v", err) + } + var foundVCConfig bool + newVcenterConfigs, err := GetVirtualCenterConfigs(ctx, cfg) + if err != nil { + return logger.LogNewErrorf(log, "failed to get VirtualCenterConfigs. err=%v", err) + } + for _, newvcconfig := range newVcenterConfigs { + if newvcconfig.Host == vc.Config.Host { + newvcconfig.ReloadVCConfigForNewClient = true + vc.Config = newvcconfig + log.Infof("Successfully set latest VC config for vcenter: %q", vc.Config.Host) + foundVCConfig = true + break + } + } + if !foundVCConfig { + return logger.LogNewErrorf(log, "failed to get vCenter config for Host: %q", vc.Config.Host) } } - if vc.Client, err = vc.NewClient(ctx, useragent); err != nil { + if vc.Client, err = vc.NewClient(ctx); err != nil { log.Errorf("failed to create govmomi client with err: %v", err) if !vc.Config.Insecure { log.Errorf("failed to connect to vCenter using CA file: %q", vc.Config.CAFile) @@ -364,37 +370,6 @@ func (vc *VirtualCenter) connect(ctx context.Context, requestNewSession bool) er return nil } -// ReadVCConfigs will ensure we are always reading the latest config -// before attempting to create a new govmomi client. -// It works in case of both vanilla (including multi-vc) and wcp -func ReadVCConfigs(ctx context.Context, vc *VirtualCenter) error { - log := logger.GetLogger(ctx) - log.Infof("Reloading latest VC config from vSphere Config Secret for vcenter: %q", vc.Config.Host) - cfg, err := config.GetConfig(ctx) - if err != nil { - return logger.LogNewErrorf(log, "failed to read config. Error: %+v", err) - } - var foundVCConfig bool - newVcenterConfigs, err := GetVirtualCenterConfigs(ctx, cfg) - if err != nil { - return logger.LogNewErrorf(log, "failed to get VirtualCenterConfigs. err=%v", err) - } - for _, newvcconfig := range newVcenterConfigs { - if newvcconfig.Host == vc.Config.Host { - newvcconfig.ReloadVCConfigForNewClient = true - vc.Config = newvcconfig - log.Infof("Successfully set latest VC config for vcenter: %q", vc.Config.Host) - foundVCConfig = true - break - } - } - if !foundVCConfig { - return logger.LogNewErrorf(log, "failed to get vCenter config for Host: %q", vc.Config.Host) - } - - return nil -} - // ListDatacenters returns all Datacenters. func (vc *VirtualCenter) ListDatacenters(ctx context.Context) ( []*Datacenter, error) { diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index 4d9c73fc89..dd008cc000 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -401,7 +401,7 @@ func validateConfig(ctx context.Context, cfg *Config) error { } if cfg.NetPermissions == nil { // If no net permissions are given, assume default. - log.Debug("No Net Permissions given in Config. Using default permissions.") + log.Info("No Net Permissions given in Config. Using default permissions.") if clusterFlavor == cnstypes.CnsClusterFlavorVanilla { cfg.NetPermissions = map[string]*NetPermissionConfig{"#": GetDefaultNetPermission()} } @@ -735,31 +735,3 @@ func GetConfigPath(ctx context.Context) string { } return cfgPath } - -// GetSessionUserAgent returns clusterwise unique useragent -func GetSessionUserAgent(ctx context.Context) (string, error) { - log := logger.GetLogger(ctx) - clusterFlavor, err := GetClusterFlavor(ctx) - if err != nil { - log.Errorf("failed retrieving cluster flavor. Error: %+v", err) - return "", err - } - cfg, err := GetConfig(ctx) - if err != nil { - log.Errorf("failed to read config. Error: %+v", err) - return "", err - } - useragent := "k8s-csi-useragent" - if clusterFlavor == cnstypes.CnsClusterFlavorVanilla { - if cfg.Global.ClusterID != "" { - useragent = useragent + "-" + cfg.Global.ClusterID - } else { - useragent = useragent + "-" + GeneratedVanillaClusterID - } - } else if clusterFlavor == cnstypes.CnsClusterFlavorWorkload { - if cfg.Global.SupervisorID != "" { - useragent = useragent + "-" + cfg.Global.SupervisorID - } - } - return useragent, nil -} diff --git a/pkg/common/fault/constants.go b/pkg/common/fault/constants.go index 1c5612927c..b3ce20b408 100644 --- a/pkg/common/fault/constants.go +++ b/pkg/common/fault/constants.go @@ -19,11 +19,6 @@ const ( // CSITaskInfoEmptyFault is the fault type when taskInfo is empty. CSITaskInfoEmptyFault = "csi.fault.TaskInfoEmpty" - // CSINonStorageFaultPrefix is the prefix used for faults originating due to components other than - // downstream vSphere storage stack. - CSINonStorageFaultPrefix = "csi.fault.nonstorage." - // VimFaultPrefix is the prefix used for vim faults from downstream components. - VimFaultPrefix = "vim.fault." // CSIVmUuidNotFoundFault is the fault type when Pod VMs do not have the vmware-system-vm-uuid annotation. CSIVmUuidNotFoundFault = "csi.fault.nonstorage.VmUuidNotFound" // CSIVmNotFoundFault is the fault type when VM object is not found in the VC @@ -68,13 +63,4 @@ const ( CSIUnimplementedFault = "csi.fault.Unimplemented" // CSIInvalidStoragePolicyConfigurationFault is the fault type returned when the user provides invalid storage policy. CSIInvalidStoragePolicyConfigurationFault = "csi.fault.invalidconfig.InvalidStoragePolicyConfiguration" - - // Below is the list of faults coming from downstream vCenter components that we want to classify - // as non-storage faults. - - // VimFaultInvalidHostState is the fault returned from CNS when host is not in a state to perform the volume - // operation e.g. maintenance mode. - VimFaultInvalidHostState = VimFaultPrefix + "InvalidHostState" - // VimFaultHostNotConnected is the fault returned from CNS when host is not connected. - VimFaultHostNotConnected = VimFaultPrefix + "HostNotConnected" ) diff --git a/pkg/common/fault/util.go b/pkg/common/fault/util.go deleted file mode 100644 index 0284adbfcf..0000000000 --- a/pkg/common/fault/util.go +++ /dev/null @@ -1,48 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package fault - -import ( - "context" - - "sigs.k8s.io/vsphere-csi-driver/v3/pkg/csi/service/logger" -) - -var ( - // VimNonStorageFaultsList contains the list of faults that needs to classified as non-storage faults. - VimNonStorageFaultsList = []string{VimFaultInvalidHostState, VimFaultHostNotConnected} -) - -// IsNonStorageFault checks if the fault type is in a pre-defined list of non-storage faults -// and returns a bool value accordingly. -func IsNonStorageFault(fault string) bool { - for _, nonStorageFault := range VimNonStorageFaultsList { - if nonStorageFault == fault { - return true - } - } - return false -} - -// AddCsiNonStoragePrefix adds "csi.fault.nonstorage." prefix to the faults. -func AddCsiNonStoragePrefix(ctx context.Context, fault string) string { - log := logger.GetLogger(ctx) - if fault != "" { - log.Infof("Adding %q prefix to fault %q", CSINonStorageFaultPrefix, fault) - return CSINonStorageFaultPrefix + fault - } - return fault -} diff --git a/pkg/common/unittestcommon/utils.go b/pkg/common/unittestcommon/utils.go index 6c14ee8bf5..d0625b9dee 100644 --- a/pkg/common/unittestcommon/utils.go +++ b/pkg/common/unittestcommon/utils.go @@ -54,16 +54,6 @@ func GetFakeContainerOrchestratorInterface(orchestratorType int) (commonco.COCom "tkgs-ha": "true", "list-volumes": "true", "csi-internal-generated-cluster-id": "true", - "online-volume-extend": "true", - "async-query-volume": "true", - "csi-windows-support": "true", - "use-csinode-id": "true", - "pv-to-backingdiskobjectid-mapping": "false", - "cnsmgr-suspend-create-volume": "true", - "topology-preferential-datastores": "true", - "max-pvscsi-targets-per-vm": "true", - "multi-vcenter-csi-topology": "true", - "listview-tasks": "true", }, } return fakeCO, nil @@ -304,8 +294,3 @@ func (c *FakeK8SOrchestrator) GetCSINodeTopologyInstanceByName(nodeName string) item interface{}, exists bool, err error) { return nil, false, nil } - -// GetPVCNamespaceFromVolumeID retrieves the pv name from volumeID. -func (c *FakeK8SOrchestrator) GetPVNameFromCSIVolumeID(volumeID string) (string, bool) { - return "", false -} diff --git a/pkg/common/utils/utils.go b/pkg/common/utils/utils.go index 03671f8299..1a3e386aed 100644 --- a/pkg/common/utils/utils.go +++ b/pkg/common/utils/utils.go @@ -22,6 +22,7 @@ import ( "strconv" cnstypes "github.com/vmware/govmomi/cns/types" + "github.com/vmware/govmomi/vim25/types" "google.golang.org/grpc/codes" cnsvolume "sigs.k8s.io/vsphere-csi-driver/v3/pkg/common/cns-lib/volume" cnsvsphere "sigs.k8s.io/vsphere-csi-driver/v3/pkg/common/cns-lib/vsphere" @@ -224,34 +225,44 @@ func QueryVolumeDetailsUtil(ctx context.Context, m cnsvolume.Manager, volumeIds return volumeDetailsMap, nil } -// LogoutAllSessions will logout all vCenter sessions and disconnect vCenter client -func LogoutAllvCenterSessions(ctx context.Context) { +// GetDatastoreRefByURLFromGivenDatastoreList fetches the datastore reference by datastore URL +// from a list of datastore references. +// If the datastore with dsURL can be found in the same datacenter as the given VC +// and it is also found in the given datastoreList, return the reference of the datastore. +// Otherwise, return error. +func GetDatastoreRefByURLFromGivenDatastoreList(ctx context.Context, vc *cnsvsphere.VirtualCenter, + datastoreList []types.ManagedObjectReference, dsURL string) (*types.ManagedObjectReference, error) { log := logger.GetLogger(ctx) - log.Info("Logging out all vCenter sessions") - virtualcentermanager := cnsvsphere.GetVirtualCenterManager(ctx) - vCenters := virtualcentermanager.GetAllVirtualCenters() - managerInstanceMap := cnsvolume.GetAllManagerInstances(ctx) - for _, vc := range vCenters { - if vc.Client == nil { - continue - } - log.Info("Closing idle vCenter session") - vc.Client.CloseIdleConnections() - // logout vCenter session for list-view - mgr, ok := managerInstanceMap[vc.Config.Host] - if ok && mgr != nil { - err := mgr.LogoutListViewVCSession(ctx) - if err != nil { - continue - } - } - log.Infof("Disconnecting vCenter client for host %s", vc.Config.Host) - err := vc.Disconnect(ctx) + // Get all datacenters in the virtualcenter + datacenters, err := vc.GetDatacenters(ctx) + if err != nil { + log.Errorf("failed to find datacenters from VC: %q, Error: %+v", vc.Config.Host, err) + return nil, err + } + var candidateDsObj *cnsvsphere.Datastore + // traverse each datacenter and find the datastore with the specified dsURL + for _, datacenter := range datacenters { + candidateDsInfoObj, err := datacenter.GetDatastoreInfoByURL(ctx, dsURL) if err != nil { - log.Errorf("Error while disconnect vCenter client for host %s. Error: %+v", vc.Config.Host, err) + log.Errorf("failed to find datastore with URL %q in datacenter %q from VC %q, Error: %+v", + dsURL, datacenter.InventoryPath, vc.Config.Host, err) continue } - log.Infof("Disconnected vCenter client for host %s", vc.Config.Host) + candidateDsObj = candidateDsInfoObj.Datastore + break + } + if candidateDsObj == nil { + // fail if the candidate datastore is not found in the virtualcenter + return nil, logger.LogNewErrorf(log, + "failed to find datastore with URL %q in VC %q", dsURL, vc.Config.Host) + } + + for _, datastoreRef := range datastoreList { + if datastoreRef == candidateDsObj.Reference() { + log.Infof("compatible datastore found, dsURL = %q, dsRef = %v", dsURL, datastoreRef) + return &datastoreRef, nil + } } - log.Info("Successfully logged out vCenter sessions") + return nil, logger.LogNewErrorf(log, + "failed to find datastore with URL %q from the input datastore list, %v", dsURL, datastoreList) } diff --git a/pkg/common/utils/utils_test.go b/pkg/common/utils/utils_test.go index ca1ce41145..539c7efea4 100644 --- a/pkg/common/utils/utils_test.go +++ b/pkg/common/utils/utils_test.go @@ -9,9 +9,13 @@ import ( "sync" "testing" + "github.com/stretchr/testify/assert" + "github.com/vmware/govmomi/vim25/mo" + cnssim "github.com/vmware/govmomi/cns/simulator" "github.com/vmware/govmomi/cns/types" "github.com/vmware/govmomi/simulator" + vim25types "github.com/vmware/govmomi/vim25/types" cnsvolumes "sigs.k8s.io/vsphere-csi-driver/v3/pkg/common/cns-lib/volume" cnsvsphere "sigs.k8s.io/vsphere-csi-driver/v3/pkg/common/cns-lib/vsphere" @@ -163,3 +167,99 @@ func TestQuerySnapshotsUtil(t *testing.T) { t.Log(entry) } } + +func TestGetDatastoreRefByURLFromGivenDatastoreList(t *testing.T) { + type funcArgs struct { + ctx context.Context + vc *cnsvsphere.VirtualCenter + dsMoRefList []vim25types.ManagedObjectReference + dsURL string + } + + // Create context + commonUtilsTestInstance := getCommonUtilsTest(t) + + dsReferenceList := simulator.Map.AllReference("Datastore") + var dsEntityList []mo.Entity + var dsMoRefList []vim25types.ManagedObjectReference + for _, dsReference := range dsReferenceList { + dsMoRefList = append(dsMoRefList, dsReference.Reference()) + dsEntityList = append(dsEntityList, dsReference.(mo.Entity)) + } + + // case 2: a list of all datastore MoRef except the last one + dsMoRefListButLastOne := dsMoRefList[:len(dsMoRefList)-1] + + // the datastore url for the last one in the list + dsReferenceFortheLast := dsReferenceList[len(dsReferenceList)-1].Reference() + dsUrl := dsEntityList[len(dsEntityList)-1].(*simulator.Datastore).Info.GetDatastoreInfo().Url + + // an invalid datastore url + invalidDsUrl := "an-invalid-datastore-url" + + tests := []struct { + name string + args funcArgs + expectedDsRef *vim25types.ManagedObjectReference + expectedErr error + }{ + { + name: "CompatibleDatastoreFound", + args: funcArgs{ + ctx: context.TODO(), + vc: commonUtilsTestInstance.vcenter, + dsMoRefList: dsMoRefList, + dsURL: dsUrl, + }, + expectedDsRef: &dsReferenceFortheLast, + expectedErr: nil, + }, + { + name: "FailToFindGivenDatastoreInCompatibleList", + args: funcArgs{ + ctx: context.TODO(), + vc: commonUtilsTestInstance.vcenter, + dsMoRefList: dsMoRefListButLastOne, + dsURL: dsUrl, + }, + expectedDsRef: nil, + expectedErr: fmt.Errorf("failed to find datastore with URL %q from "+ + "the input datastore list, %v", dsUrl, dsMoRefListButLastOne), + }, + { + name: "FailToFindGivenDatastoreInVC", + args: funcArgs{ + ctx: context.TODO(), + vc: commonUtilsTestInstance.vcenter, + dsMoRefList: dsMoRefList, + dsURL: invalidDsUrl, + }, + expectedDsRef: nil, + expectedErr: fmt.Errorf("failed to find datastore with URL %q in VC %q", + invalidDsUrl, commonUtilsTestInstance.vcenter.Config.Host), + }, + { + name: "EmptyDatastoreURLFromInput", + args: funcArgs{ + ctx: context.TODO(), + vc: commonUtilsTestInstance.vcenter, + dsMoRefList: dsMoRefList, + dsURL: "", + }, + expectedDsRef: nil, + expectedErr: fmt.Errorf("failed to find datastore with URL %q in VC %q", + "", commonUtilsTestInstance.vcenter.Config.Host), + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + actualDsRef, actualErr := GetDatastoreRefByURLFromGivenDatastoreList( + test.args.ctx, test.args.vc, test.args.dsMoRefList, test.args.dsURL) + assert.Equal(t, test.expectedErr == nil, actualErr == nil) + if test.expectedErr != nil && actualErr != nil { + assert.Equal(t, test.expectedErr.Error(), actualErr.Error()) + } + assert.Equal(t, test.expectedDsRef, actualDsRef) + }) + } +} diff --git a/pkg/csi/service/common/authmanager.go b/pkg/csi/service/common/authmanager.go index 5fdc028f3a..ef04ad0406 100644 --- a/pkg/csi/service/common/authmanager.go +++ b/pkg/csi/service/common/authmanager.go @@ -382,14 +382,9 @@ func getDatastoresWithBlockVolumePrivs(ctx context.Context, vc *cnsvsphere.Virtu privIds, entities, userName, vc.Config.Host) return nil, err } - if len(result) == 0 { - log.Infof("auth manager: HasUserPrivilegeOnEntities returned empty result when checking privileges %v "+ - "on entities %v for user %s and for vCenter %q", privIds, entities, userName, vc.Config.Host) - } else { - log.Debugf("auth manager: HasUserPrivilegeOnEntities returned %v when checking privileges %v on entities %v "+ - "for user %s and for vCenter %q", result, privIds, entities, userName, vc.Config.Host) - } - + log.Debugf( + "auth manager: HasUserPrivilegeOnEntities returns %v, when checking privileges %v on entities %v for user %s "+ + "and for vCenter %q", result, privIds, entities, userName, vc.Config.Host) for index, entityPriv := range result { hasPriv := true privAvails := entityPriv.PrivAvailability @@ -406,11 +401,6 @@ func getDatastoresWithBlockVolumePrivs(ctx context.Context, vc *cnsvsphere.Virtu "for vCenter %q", dsInfos[index].Info.Name, dsURLs[index], vc.Config.Host) } } - if len(result) != 0 && len(dsURLToInfoMap) == 0 { - log.Infof("auth manager: user %s on vCenter %q doesn't have privileges for any datastore. "+ - "HasUserPrivilegeOnEntities returns %v, when checking privileges %v on entities %v."+ - userName, vc.Config.Host, result, privIds, entities) - } return dsURLToInfoMap, nil } @@ -531,14 +521,9 @@ func getFSEnabledClustersWithPriv(ctx context.Context, vc *cnsvsphere.VirtualCen privIds, entities, userName, vc.Config.Host) return nil, err } - if len(result) == 0 { - log.Infof("auth manager: HasUserPrivilegeOnEntities returned empty result when checking privileges %v "+ - "on entities %v for user %s and for vCenter %q", privIds, entities, userName, vc.Config.Host) - } else { - log.Debugf("auth manager: HasUserPrivilegeOnEntities returned %v when checking privileges %v on entities %v "+ - "for user %s and for vCenter %q", result, privIds, entities, userName, vc.Config.Host) - } - + log.Debugf( + "auth manager: HasUserPrivilegeOnEntities returns %v when checking privileges %v on entities %v for user %s "+ + "and for vCenter %q", result, privIds, entities, userName, vc.Config.Host) clusterComputeResourceWithPriv := []*object.ClusterComputeResource{} for _, entityPriv := range result { hasPriv := true @@ -555,14 +540,8 @@ func getFSEnabledClustersWithPriv(ctx context.Context, vc *cnsvsphere.VirtualCen clusterComputeResourcesMap[entityPriv.Entity.Value]) } } - if len(result) != 0 && len(clusterComputeResourceWithPriv) == 0 { - log.Infof("auth manager: user %s on vCenter %q doesn't have privileges for any ClusterComputeResource. "+ - "HasUserPrivilegeOnEntities returns %v, when checking privileges %v on entities %v."+ - userName, vc.Config.Host, result, privIds, entities) - } else { - log.Debugf("Clusters with priv: %s and vCenter: %q are : %+v", HostConfigStoragePriv, - vc.Config.Host, clusterComputeResourceWithPriv) - } + log.Debugf("Clusters with priv: %s and vCenter: %q are : %+v", HostConfigStoragePriv, + vc.Config.Host, clusterComputeResourceWithPriv) // Get clusters which are vSAN and have vSAN FS enabled. clusterComputeResourceWithPrivAndFS := []*object.ClusterComputeResource{} @@ -579,18 +558,15 @@ func getFSEnabledClustersWithPriv(ctx context.Context, vc *cnsvsphere.VirtualCen cluster, vc.Config.Host) continue } else if config.FileServiceConfig == nil { - log.Infof("VsanClusterGetConfig.FileServiceConfig is empty. Skipping this cluster: %+v with "+ + log.Debugf("VsanClusterGetConfig.FileServiceConfig is empty. Skipping this cluster: %+v with "+ "vCenter: %q and with config: %+v", cluster, vc.Config.Host, config) continue } + log.Debugf("cluster: %+v and vCenter: %q has vSAN file services enabled: %t", cluster, vc.Config.Host, + config.FileServiceConfig.Enabled) if config.FileServiceConfig.Enabled { clusterComputeResourceWithPrivAndFS = append(clusterComputeResourceWithPrivAndFS, cluster) - log.Debugf("vSAN file service is enabled for cluster: %+v and vCenter: %q.", - cluster, vc.Config.Host) - } else { - log.Infof("vSAN file service is disabled for cluster: %+v and vCenter: %q.", - cluster, vc.Config.Host) } } diff --git a/pkg/csi/service/common/common_controller_helper.go b/pkg/csi/service/common/common_controller_helper.go index 2408715c47..7109f6d526 100644 --- a/pkg/csi/service/common/common_controller_helper.go +++ b/pkg/csi/service/common/common_controller_helper.go @@ -23,8 +23,8 @@ import ( "strings" "time" - snap "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" - snapshotterClientSet "github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned" + snap "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" + snapshotterClientSet "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" @@ -318,7 +318,7 @@ func IsVolumeSnapshotReady(ctx context.Context, client snapshotterClientSet.Inte var svs *snap.VolumeSnapshot waitErr := wait.PollImmediate(5*time.Second, timeout, func() (done bool, err error) { - svs, err = client.SnapshotV1().VolumeSnapshots(namespace). + svs, err := client.SnapshotV1().VolumeSnapshots(namespace). Get(ctx, supervisorVolumeSnapshotName, metav1.GetOptions{}) if err != nil { msg := fmt.Sprintf("unable to fetch volumesnapshot %q/%q "+ @@ -327,21 +327,14 @@ func IsVolumeSnapshotReady(ctx context.Context, client snapshotterClientSet.Inte log.Warnf(msg) return false, logger.LogNewErrorf(log, msg) } - if svs == nil || svs.Status == nil || svs.Status.ReadyToUse == nil { - log.Infof("Waiting up to %d seconds for VolumeSnapshot %v in namespace %s to be ReadyToUse, %+vs "+ - "since the start time", timeoutSeconds, supervisorVolumeSnapshotName, namespace, - time.Since(startTime).Seconds()) - return false, nil - } isSnapshotReadyToUse := *svs.Status.ReadyToUse if isSnapshotReadyToUse { log.Infof("VolumeSnapshot %s/%s is in ReadyToUse state", namespace, supervisorVolumeSnapshotName) isReadyToUse = true return true, nil } else { - log.Infof("Waiting up to %d seconds for VolumeSnapshot %v in namespace %s to be ReadyToUse, %+vs "+ - "since the start time", timeoutSeconds, supervisorVolumeSnapshotName, namespace, - time.Since(startTime).Seconds()) + log.Warnf("Waiting for VolumeSnapshot %s/%s to be ready since %+vs", namespace, + supervisorVolumeSnapshotName, time.Since(startTime).Seconds()) } return false, nil }) diff --git a/pkg/csi/service/common/commonco/coagnostic.go b/pkg/csi/service/common/commonco/coagnostic.go index 6355493b76..1882325063 100644 --- a/pkg/csi/service/common/commonco/coagnostic.go +++ b/pkg/csi/service/common/commonco/coagnostic.go @@ -81,9 +81,6 @@ type COCommonInterface interface { GetCSINodeTopologyInstancesList() []interface{} // GetCSINodeTopologyInstanceByName fetches the CSINodeTopology instance for a given node name in the cluster. GetCSINodeTopologyInstanceByName(nodeName string) (item interface{}, exists bool, err error) - // GetPVNameFromCSIVolumeID retrieves the pv name from the volumeID. - // This method will not return pv name in case of in-tree migrated volumes - GetPVNameFromCSIVolumeID(volumeID string) (string, bool) } // GetContainerOrchestratorInterface returns orchestrator object for a given diff --git a/pkg/csi/service/common/commonco/k8sorchestrator/k8sorchestrator.go b/pkg/csi/service/common/commonco/k8sorchestrator/k8sorchestrator.go index b9a64c5c17..71c340ee02 100644 --- a/pkg/csi/service/common/commonco/k8sorchestrator/k8sorchestrator.go +++ b/pkg/csi/service/common/commonco/k8sorchestrator/k8sorchestrator.go @@ -27,7 +27,7 @@ import ( "sync/atomic" "time" - snapshotterClientSet "github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned" + snapshotterClientSet "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned" cnstypes "github.com/vmware/govmomi/cns/types" pbmtypes "github.com/vmware/govmomi/pbm/types" v1 "k8s.io/api/core/v1" @@ -1067,7 +1067,7 @@ func (c *K8sOrchestrator) IsFSSEnabled(ctx context.Context, featureName string) } if !supervisorFeatureState { // If FSS set to false, return. - log.Infof("%s feature state is set to false in %s ConfigMap", featureName, c.supervisorFSS.configMapName) + log.Infof("%s feature state set to false in %s ConfigMap", featureName, c.supervisorFSS.configMapName) return supervisorFeatureState } } else { @@ -1544,8 +1544,3 @@ func (c *K8sOrchestrator) CreateConfigMap(ctx context.Context, name string, name return nil } - -// GetPVNameFromCSIVolumeID retrieves the pv name from volumeID using volumeIDToNameMap. -func (c *K8sOrchestrator) GetPVNameFromCSIVolumeID(volumeID string) (string, bool) { - return c.volumeIDToNameMap.get(volumeID) -} diff --git a/pkg/csi/service/common/commonco/k8sorchestrator/topology.go b/pkg/csi/service/common/commonco/k8sorchestrator/topology.go index 04cf3fed8d..21749602d3 100644 --- a/pkg/csi/service/common/commonco/k8sorchestrator/topology.go +++ b/pkg/csi/service/common/commonco/k8sorchestrator/topology.go @@ -109,7 +109,7 @@ var ( // isMultiVCSupportEnabled is set to true only when the MultiVCenterCSITopology FSS // is enabled. isMultivCenterCluster is set to true only when the MultiVCenterCSITopology FSS // is enabled and the K8s cluster involves multiple VCs. - isMultiVCSupportEnabled bool + isMultiVCSupportEnabled, isMultivCenterCluster bool // csiNodeTopologyInformer refers to a shared K8s informer listening on CSINodeTopology instances // in the cluster. csiNodeTopologyInformer *cache.SharedIndexInformer @@ -128,6 +128,9 @@ type nodeVolumeTopology struct { k8sConfig *restclient.Config // clusterFlavor is the cluster flavor. clusterFlavor cnstypes.CnsClusterFlavor + // isCSINodeIdFeatureEnabled indicates whether the + // use-csinode-id feature is enabled or not. + isCSINodeIdFeatureEnabled bool } // controllerVolumeTopology implements the commoncotypes.ControllerTopologyService interface @@ -142,6 +145,9 @@ type controllerVolumeTopology struct { nodeMgr node.Manager // clusterFlavor is the cluster flavor. clusterFlavor cnstypes.CnsClusterFlavor + // isCSINodeIdFeatureEnabled indicates whether the + // use-csinode-id feature is enabled or not. + isCSINodeIdFeatureEnabled bool // isAcceptPreferredDatastoresFSSEnabled indicates whether the // accept-preferred-datastores feature is enabled or not. isTopologyPreferentialDatastoresFSSEnabled bool @@ -197,19 +203,22 @@ func (c *K8sOrchestrator) InitTopologyServiceInController(ctx context.Context) ( // Set isMultivCenterCluster if the K8s cluster is a multi-VC cluster. isMultiVCSupportEnabled = c.IsFSSEnabled(ctx, common.MultiVCenterCSITopology) - - // Create a cache of topology tags -> VC -> associated MoRefs in that VC to ease volume provisioning. - err = common.DiscoverTagEntities(ctx) - if err != nil { - return nil, logger.LogNewErrorf(log, - "failed to update cache with topology information. Error: %+v", err) + if isMultiVCSupportEnabled { + cfg, err := cnsconfig.GetConfig(ctx) + if err != nil { + return nil, logger.LogNewErrorf(log, "failed to read config. Error: %+v", err) + } + if len(cfg.VirtualCenter) > 1 { + isMultivCenterCluster = true + } } controllerVolumeTopologyInstance = &controllerVolumeTopology{ - k8sConfig: config, - nodeMgr: nodeManager, - csiNodeTopologyInformer: *csiNodeTopologyInformer, - clusterFlavor: clusterFlavor, + k8sConfig: config, + nodeMgr: nodeManager, + csiNodeTopologyInformer: *csiNodeTopologyInformer, + clusterFlavor: clusterFlavor, + isCSINodeIdFeatureEnabled: c.IsFSSEnabled(ctx, common.UseCSINodeId), isTopologyPreferentialDatastoresFSSEnabled: c.IsFSSEnabled(ctx, common.TopologyPreferentialDatastores), } @@ -536,6 +545,9 @@ func topoCRAdded(obj interface{}) { } if isMultiVCSupportEnabled { common.AddNodeToDomainNodeMapNew(ctx, nodeTopoObj) + if isMultivCenterCluster { + common.AddLabelsToTopologyVCMap(ctx, nodeTopoObj) + } } else { addNodeToDomainNodeMap(ctx, nodeTopoObj) } @@ -587,6 +599,9 @@ func topoCRUpdated(oldObj interface{}, newObj interface{}) { oldNodeTopoObj, newNodeTopoObj) if isMultiVCSupportEnabled { common.RemoveNodeFromDomainNodeMapNew(ctx, oldNodeTopoObj) + if isMultivCenterCluster { + common.RemoveLabelsFromTopologyVCMap(ctx, oldNodeTopoObj) + } } else { removeNodeFromDomainNodeMap(ctx, oldNodeTopoObj) } @@ -595,6 +610,9 @@ func topoCRUpdated(oldObj interface{}, newObj interface{}) { if newNodeTopoObj.Status.Status == csinodetopologyv1alpha1.CSINodeTopologySuccess { if isMultiVCSupportEnabled { common.AddNodeToDomainNodeMapNew(ctx, newNodeTopoObj) + if isMultivCenterCluster { + common.AddLabelsToTopologyVCMap(ctx, newNodeTopoObj) + } } else { addNodeToDomainNodeMap(ctx, newNodeTopoObj) } @@ -616,6 +634,9 @@ func topoCRDeleted(obj interface{}) { if nodeTopoObj.Status.Status == csinodetopologyv1alpha1.CSINodeTopologySuccess { if isMultiVCSupportEnabled { common.RemoveNodeFromDomainNodeMapNew(ctx, nodeTopoObj) + if isMultivCenterCluster { + common.RemoveLabelsFromTopologyVCMap(ctx, nodeTopoObj) + } } else { removeNodeFromDomainNodeMap(ctx, nodeTopoObj) } @@ -698,11 +719,12 @@ func (c *K8sOrchestrator) InitTopologyServiceInNode(ctx context.Context) ( } nodeVolumeTopologyInstance = &nodeVolumeTopology{ - csiNodeTopologyK8sClient: crClient, - csiNodeTopologyWatcher: crWatcher, - k8sClient: k8sClient, - k8sConfig: config, - clusterFlavor: clusterFlavor, + csiNodeTopologyK8sClient: crClient, + csiNodeTopologyWatcher: crWatcher, + k8sClient: k8sClient, + k8sConfig: config, + clusterFlavor: clusterFlavor, + isCSINodeIdFeatureEnabled: c.IsFSSEnabled(ctx, common.UseCSINodeId), } log.Infof("Topology service initiated successfully") } @@ -716,49 +738,45 @@ func (c *K8sOrchestrator) InitTopologyServiceInNode(ctx context.Context) ( func (volTopology *nodeVolumeTopology) GetNodeTopologyLabels(ctx context.Context, nodeInfo *commoncotypes.NodeInfo) ( map[string]string, error) { log := logger.GetLogger(ctx) - var err error - if volTopology.clusterFlavor == cnstypes.CnsClusterFlavorGuest { + var err error + csiNodeTopology := &csinodetopologyv1alpha1.CSINodeTopology{} + csiNodeTopologyKey := types.NamespacedName{ + Name: nodeInfo.NodeName, + } + err = volTopology.csiNodeTopologyK8sClient.Get(ctx, csiNodeTopologyKey, csiNodeTopology) + csiNodeTopologyFound := true + if err != nil { + if !apierrors.IsNotFound(err) { + msg := fmt.Sprintf("failed to get CsiNodeTopology for the node: %q. Error: %+v", nodeInfo.NodeName, err) + return nil, logger.LogNewErrorCodef(log, codes.Internal, msg) + } + csiNodeTopologyFound = false err = createCSINodeTopologyInstance(ctx, volTopology, nodeInfo) if err != nil { return nil, logger.LogNewErrorCodef(log, codes.Internal, err.Error()) } - } else { - csiNodeTopology := &csinodetopologyv1alpha1.CSINodeTopology{} - csiNodeTopologyKey := types.NamespacedName{ - Name: nodeInfo.NodeName, + } + + // there is an already existing topology + if csiNodeTopologyFound && volTopology.clusterFlavor == cnstypes.CnsClusterFlavorVanilla { + newCSINodeTopology := csiNodeTopology.DeepCopy() + + if volTopology.isCSINodeIdFeatureEnabled { + newCSINodeTopology = volTopology.updateNodeIDForTopology(ctx, nodeInfo, newCSINodeTopology) } - err = volTopology.csiNodeTopologyK8sClient.Get(ctx, csiNodeTopologyKey, csiNodeTopology) - csiNodeTopologyFound := true + // reset the status so as syncer can sync the object again + newCSINodeTopology.Status.Status = "" + _, err = volTopology.patchCSINodeTopology(ctx, csiNodeTopology, newCSINodeTopology) if err != nil { - if !apierrors.IsNotFound(err) { - msg := fmt.Sprintf("failed to get CsiNodeTopology for the node: %q. Error: %+v", nodeInfo.NodeName, err) - return nil, logger.LogNewErrorCodef(log, codes.Internal, msg) - } - csiNodeTopologyFound = false - err = createCSINodeTopologyInstance(ctx, volTopology, nodeInfo) - if err != nil { - return nil, logger.LogNewErrorCodef(log, codes.Internal, err.Error()) - } - } - // There is an already existing topology. - if csiNodeTopologyFound { - newCSINodeTopology := csiNodeTopology.DeepCopy() - newCSINodeTopology = volTopology.updateNodeIDForTopology(ctx, nodeInfo, newCSINodeTopology) - // reset the status so as syncer can sync the object again - newCSINodeTopology.Status.Status = "" - _, err = volTopology.patchCSINodeTopology(ctx, csiNodeTopology, newCSINodeTopology) - if err != nil { - msg := fmt.Sprintf("Fail to patch CsiNodeTopology for the node: %q "+ - "with nodeUUID: %s. Error: %+v", - nodeInfo.NodeName, nodeInfo.NodeID, err) - return nil, logger.LogNewErrorCodef(log, codes.Internal, msg) - } - log.Infof("Successfully patched CSINodeTopology instance: %q with Uuid: %q", - nodeInfo.NodeName, nodeInfo.NodeID) + msg := fmt.Sprintf("Fail to patch CsiNodeTopology for the node: %q "+ + "with nodeUUID: %s. Error: %+v", + nodeInfo.NodeName, nodeInfo.NodeID, err) + return nil, logger.LogNewErrorCodef(log, codes.Internal, msg) } + log.Infof("Successfully patched CSINodeTopology instance: %q with Uuid: %q", + nodeInfo.NodeName, nodeInfo.NodeID) } - // Create a watcher for CSINodeTopology CRs. timeoutSeconds := int64((time.Duration(getCSINodeTopologyWatchTimeoutInMin(ctx)) * time.Minute).Seconds()) watchCSINodeTopology, err := volTopology.csiNodeTopologyWatcher.Watch(metav1.ListOptions{ @@ -894,7 +912,7 @@ func getPatchData(oldObj, newObj interface{}) ([]byte, error) { // Create new CSINodeTopology instance if it doesn't exist // Create CSINodeTopology instance with spec.nodeID and spec.nodeUUID -// if cluster flavor is Vanilla +// if cluster flavor is Vanilla and UseCSINodeId feature is enabled // else create with spec.nodeID only. func createCSINodeTopologyInstance(ctx context.Context, volTopology *nodeVolumeTopology, @@ -924,7 +942,7 @@ func createCSINodeTopologyInstance(ctx context.Context, // If both useCnsNodeId feature is enabled and clusterFlavor is Vanilla, // create the CsiNodeTopology instance with nodeID set to node name and // nodeUUID set to node uuid. - if volTopology.clusterFlavor == cnstypes.CnsClusterFlavorVanilla { + if volTopology.isCSINodeIdFeatureEnabled && volTopology.clusterFlavor == cnstypes.CnsClusterFlavorVanilla { csiNodeTopologySpec.Spec = csinodetopologyv1alpha1.CSINodeTopologySpec{ NodeID: nodeInfo.NodeName, NodeUUID: nodeInfo.NodeID, @@ -1232,11 +1250,12 @@ func (volTopology *controllerVolumeTopology) getTopologySegmentsWithMatchingNode // If there is a match, fetch the nodeVM object and add it to matchingNodeVMs. if isMatch { var nodeVM *cnsvsphere.VirtualMachine - if volTopology.clusterFlavor == cnstypes.CnsClusterFlavorVanilla { - nodeVM, err = volTopology.nodeMgr.GetNodeVMAndUpdateCache(ctx, + if volTopology.isCSINodeIdFeatureEnabled && + volTopology.clusterFlavor == cnstypes.CnsClusterFlavorVanilla { + nodeVM, err = volTopology.nodeMgr.GetNode(ctx, nodeTopologyInstance.Spec.NodeUUID, nil) } else { - nodeVM, err = volTopology.nodeMgr.GetNodeVMByNameAndUpdateCache(ctx, + nodeVM, err = volTopology.nodeMgr.GetNodeByName(ctx, nodeTopologyInstance.Spec.NodeID) } if err != nil { @@ -1302,11 +1321,12 @@ func (volTopology *controllerVolumeTopology) getNodesMatchingTopologySegment(ctx } if isMatch { var nodeVM *cnsvsphere.VirtualMachine - if volTopology.clusterFlavor == cnstypes.CnsClusterFlavorVanilla { - nodeVM, err = volTopology.nodeMgr.GetNodeVMAndUpdateCache(ctx, + if volTopology.isCSINodeIdFeatureEnabled && + volTopology.clusterFlavor == cnstypes.CnsClusterFlavorVanilla { + nodeVM, err = volTopology.nodeMgr.GetNode(ctx, nodeTopologyInstance.Spec.NodeUUID, nil) } else { - nodeVM, err = volTopology.nodeMgr.GetNodeVMByNameAndUpdateCache(ctx, + nodeVM, err = volTopology.nodeMgr.GetNodeByName(ctx, nodeTopologyInstance.Spec.NodeID) } if err != nil { diff --git a/pkg/csi/service/common/constants.go b/pkg/csi/service/common/constants.go index be98380bdd..d5bf1d7841 100644 --- a/pkg/csi/service/common/constants.go +++ b/pkg/csi/service/common/constants.go @@ -312,11 +312,6 @@ const ( // on the VolumeSnapshot CR VolumeSnapshotInfoKey = "csi.vsphere.volume/snapshot" - // SupervisorVolumeSnapshotAnnotationKey represents the annotation key on VolumeSnapshot CR - // in Supervisor cluster which is used to indicate that snapshot operation is initiated from - // Guest cluster. - SupervisorVolumeSnapshotAnnotationKey = "csi.vsphere.guest-initiated-csi-snapshot" - // AttributeSupervisorVolumeSnapshotClass represents name of VolumeSnapshotClass AttributeSupervisorVolumeSnapshotClass = "svvolumesnapshotclass" @@ -371,6 +366,10 @@ const ( // CSIWindowsSupport is the feature to support csi block volumes for windows // node. CSIWindowsSupport = "csi-windows-support" + // UseCSINodeId is the feature to make sure CSI will no longer use + // ProviderID on K8s Node API object set by CPI. If not set, CSI + // will continue to use the Provider ID from K8s Node API object. + UseCSINodeId = "use-csinode-id" // TKGsHA is the feature gate to check whether TKGS HA feature // is enabled. TKGsHA = "tkgs-ha" diff --git a/pkg/csi/service/common/placementengine/placement.go b/pkg/csi/service/common/placementengine/placement.go index 91973097e2..9efd251fbe 100644 --- a/pkg/csi/service/common/placementengine/placement.go +++ b/pkg/csi/service/common/placementengine/placement.go @@ -16,169 +16,137 @@ import ( csinodetopologyv1alpha1 "sigs.k8s.io/vsphere-csi-driver/v3/pkg/internalapis/csinodetopology/v1alpha1" ) -// GetSharedDatastores retrieves the shared accessible datastores for hosts associated -// with the topology segments requested by user. func GetSharedDatastores(ctx context.Context, reqParams interface{}) ( []*cnsvsphere.DatastoreInfo, error) { log := logger.GetLogger(ctx) params := reqParams.(VanillaSharedDatastoresParams) - nodeMgr := node.GetManager(ctx) - - log.Infof("GetSharedDatastores called for VC %q with policyID: %q , Topology Segment List: %v", - params.Vcenter.Config.Host, params.StoragePolicyID, params.TopologySegmentsList) var sharedDatastores []*cnsvsphere.DatastoreInfo + nodeMgr := node.GetManager(ctx) + log.Infof("GetSharedDatastores called with policyID: %q , Topology Segment List: %v", + params.StoragePolicyID, params.TopologySegmentsList) // Iterate through each set of topology segments and find shared datastores for that segment. - /* For example, if the topology environment is as follows: - VC - |-> DC (Category: region, Tag: region1) - |-> Cluster1 (Category: zone, Tag: zone1) - Node1 - Node2 - |-> Cluster2 (Category: zone, Tag: zone2) - Node3 - Node4 - |-> Cluster3 (Category: zone, Tag: zone3) - (No nodeVMs in Cluster3 yet) - - If the user chooses to provision a volume in region1, according to the code below: - `params.TopologySegmentsList` will look like - [ - map[string]string{region:region1} - ] - `reqSegment` will look like map[string]string{region:region1} - `completeTopologySegments` will look like - [ - map[string]string{region:region1, zone:zone1}, - map[string]string{region:region1, zone:zone2} - ] - */ - for _, reqSegment := range params.TopologySegmentsList { - // Fetch the complete hierarchy of topology segments. - completeTopologySegments, err := getExpandedTopologySegments(ctx, reqSegment, nodeMgr) + for _, segments := range params.TopologySegmentsList { + // Fetch nodes compatible with the requested topology segments. + matchingNodeVMs, completeTopologySegments, err := getTopologySegmentsWithMatchingNodes(ctx, + segments, nodeMgr) if err != nil { return nil, logger.LogNewErrorf(log, "failed to find nodes in topology segment %+v. Error: %+v", - reqSegment, err) + segments, err) } - if len(completeTopologySegments) == 0 { + if len(matchingNodeVMs) == 0 { log.Warnf("No nodes in the cluster matched the topology requirement: %+v", - reqSegment) + segments) continue } - log.Infof("TopologySegment %+v expanded as: %+v", reqSegment, completeTopologySegments) - // For each segment in the complete topology segments hierarchy, get the matching hosts. - for _, segment := range completeTopologySegments { - hostMoRefs, err := common.GetHostsForSegment(ctx, segment, params.Vcenter) - if err != nil { - return nil, logger.LogNewErrorf(log, - "failed to fetch hosts belonging to topology segment %+v. Error: %+v", segment, err) + log.Infof("Obtained list of nodeVMs %+v", matchingNodeVMs) + log.Debugf("completeTopologySegments map: %+v", completeTopologySegments) + // Fetch shared datastores for the matching nodeVMs. + sharedDatastoresInTopology, err := cnsvsphere.GetSharedDatastoresForVMs(ctx, matchingNodeVMs) + if err != nil { + if err == cnsvsphere.ErrNoSharedDatastoresFound { + log.Warnf("no shared datastores found for topology segment: %+v", segments) + continue } - // 1. Fetch shared datastores accessible to all the hosts in this segment. - sharedDatastoresInTopologySegment, err := cnsvsphere.GetSharedDatastoresForHosts(ctx, hostMoRefs) + return nil, logger.LogNewErrorf(log, "failed to get shared datastores for nodes: %+v "+ + "in topology segment %+v. Error: %+v", matchingNodeVMs, segments, err) + } + log.Infof("Obtained list of shared datastores as %+v", sharedDatastoresInTopology) + + // Check storage policy compatibility, if given. + // Datastore comparison by moref. + if params.StoragePolicyID != "" { + var sharedDSMoRef []vimtypes.ManagedObjectReference + for _, ds := range sharedDatastoresInTopology { + sharedDSMoRef = append(sharedDSMoRef, ds.Reference()) + } + compat, err := params.Vcenter.PbmCheckCompatibility(ctx, sharedDSMoRef, params.StoragePolicyID) if err != nil { - if err == cnsvsphere.ErrNoSharedDSFound { - log.Warnf("no shared datastores found for hosts %+v belonging to topology segment: %+v", - hostMoRefs, segment) - continue - } - return nil, logger.LogNewErrorf(log, "failed to get shared datastores for hosts: %+v "+ - "in topology segment %+v. Error: %+v", hostMoRefs, segment, err) + return nil, logger.LogNewErrorf(log, "failed to find datastore compatibility "+ + "with storage policy ID %q. vCenter: %q Error: %+v", params.StoragePolicyID, params.Vcenter.Config.Host, err) } - log.Infof("Obtained list of shared datastores %+v for hosts %+v", sharedDatastoresInTopologySegment, - hostMoRefs) + compatibleDsMoids := make(map[string]struct{}) + for _, ds := range compat.CompatibleDatastores() { + compatibleDsMoids[ds.HubId] = struct{}{} + } + log.Infof("Datastores compatible with storage policy %q are %+v for vCenter: %q", params.StoragePolicyID, + compatibleDsMoids, params.Vcenter.Config.Host) - // 2. Check storage policy compatibility, if given. - // Storage policy compatibility is given a higher preference than - // preferential datastore in that topology segment. - if params.StoragePolicyID != "" { - var sharedDSMoRef []vimtypes.ManagedObjectReference - for _, ds := range sharedDatastoresInTopologySegment { - sharedDSMoRef = append(sharedDSMoRef, ds.Reference()) - } - compat, err := params.Vcenter.PbmCheckCompatibility(ctx, sharedDSMoRef, params.StoragePolicyID) - if err != nil { - return nil, logger.LogNewErrorf(log, "failed to find datastore compatibility "+ - "with storage policy ID %q. vCenter: %q Error: %+v", params.StoragePolicyID, - params.Vcenter.Config.Host, err) + // Filter compatible datastores from shared datastores list. + var compatibleDatastores []*cnsvsphere.DatastoreInfo + for _, ds := range sharedDatastoresInTopology { + if _, exists := compatibleDsMoids[ds.Reference().Value]; exists { + compatibleDatastores = append(compatibleDatastores, ds) } - compatibleDsMoids := make(map[string]struct{}) - for _, ds := range compat.CompatibleDatastores() { - compatibleDsMoids[ds.HubId] = struct{}{} + } + if len(compatibleDatastores) == 0 { + log.Errorf("No compatible shared datastores found for storage policy %q on vCenter: %q", + params.StoragePolicyID, params.Vcenter.Config.Host) + continue + } + sharedDatastoresInTopology = compatibleDatastores + } + // Further, filter the compatible datastores with preferential datastores, if any. + // Datastore comparison by URL. + if common.PreferredDatastoresExist { + // Fetch all preferred datastore URLs for the matching topology segments. + allPreferredDSURLs := make(map[string]struct{}) + for _, topoSegs := range completeTopologySegments { + prefDS := common.GetPreferredDatastoresInSegments(ctx, topoSegs, params.Vcenter.Config.Host) + log.Infof("Preferential datastores: %v for topology segment: %v on vCenter: %q", prefDS, + topoSegs, params.Vcenter.Config.Host) + for key, val := range prefDS { + allPreferredDSURLs[key] = val } - log.Infof("Datastores compatible with storage policy %q are %+v for vCenter: %q", - params.StoragePolicyID, compatibleDsMoids, params.Vcenter.Config.Host) - - // Filter compatible datastores from shared datastores list. - var compatibleDatastores []*cnsvsphere.DatastoreInfo - for _, ds := range sharedDatastoresInTopologySegment { - // Datastore comparison by moref. - if _, exists := compatibleDsMoids[ds.Reference().Value]; exists { - compatibleDatastores = append(compatibleDatastores, ds) + } + if len(allPreferredDSURLs) != 0 { + // If there are preferred datastores among the compatible + // datastores, choose the preferred datastores, otherwise + // choose the compatible datastores. + log.Debugf("Filtering preferential datastores from compatible datastores") + var preferredDS []*cnsvsphere.DatastoreInfo + for _, dsInfo := range sharedDatastoresInTopology { + if _, ok := allPreferredDSURLs[dsInfo.Info.Url]; ok { + preferredDS = append(preferredDS, dsInfo) } } - if len(compatibleDatastores) == 0 { - log.Infof("No compatible shared datastores found for storage policy %q on vCenter: %q", - params.StoragePolicyID, params.Vcenter.Config.Host) + if len(preferredDS) != 0 { + sharedDatastoresInTopology = preferredDS + log.Infof("Using preferred datastores: %+v", preferredDS) } else { - log.Infof("Shared datastores compatible with storage policy %q are %+v for vCenter: %q", - params.StoragePolicyID, compatibleDatastores, params.Vcenter.Config.Host) - sharedDatastoresInTopologySegment = compatibleDatastores + log.Infof("No preferential datastore selected for volume provisoning") } } + } - // 3. Filter the shared datastores with preferential datastores, if any. - // Datastore comparison by URL. - if common.PreferredDatastoresExist { - // Fetch all preferred datastore URLs for the topology segment. - prefDS := common.GetPreferredDatastoresInSegments(ctx, segment, params.Vcenter.Config.Host) - log.Infof("Preferential datastores %v found for topology segment: %v on vCenter: %q", prefDS, - segment, params.Vcenter.Config.Host) - if len(prefDS) != 0 { - // If there are preferred datastores among the shared compatible - // datastores, choose the preferred datastores. - var preferredDS []*cnsvsphere.DatastoreInfo - for _, dsInfo := range sharedDatastoresInTopologySegment { - if _, ok := prefDS[dsInfo.Info.Url]; ok { - preferredDS = append(preferredDS, dsInfo) - } - } - if len(preferredDS) != 0 { - sharedDatastoresInTopologySegment = preferredDS - log.Infof("Using preferred datastores: %+v", preferredDS) - } else { - log.Infof("No preferential datastore selected for volume provisioning") - } + // Update sharedDatastores with the list of datastores received. + // Duplicates will not be added. + for _, ds := range sharedDatastoresInTopology { + var found bool + for _, sharedDS := range sharedDatastores { + if sharedDS.Info.Url == ds.Info.Url { + found = true + break } } - // Add the datastore list to sharedDatastores without duplicates. - for _, ds := range sharedDatastoresInTopologySegment { - var found bool - for _, sharedDS := range sharedDatastores { - if ds.Info.Url == sharedDS.Info.Url { - found = true - break - } - } - if !found { - sharedDatastores = append(sharedDatastores, ds) - } + if !found { + sharedDatastores = append(sharedDatastores, ds) } } } - if len(sharedDatastores) != 0 { log.Infof("Shared compatible datastores being considered for volume provisioning on vCenter: %q are: %+v", - params.Vcenter.Config.Host, sharedDatastores) + sharedDatastores, params.Vcenter.Config.Host) } return sharedDatastores, nil } -// getExpandedTopologySegments expands the user given topology requirement to depict the complete hierarchy. -// NOTE: If there is no nodeVM in an AZ, that AZ will be skipped in complete topology hierarchy. -func getExpandedTopologySegments(ctx context.Context, requestedSegments map[string]string, - nodeMgr node.Manager) ([]map[string]string, error) { +func getTopologySegmentsWithMatchingNodes(ctx context.Context, requestedSegments map[string]string, + nodeMgr node.Manager) ([]*cnsvsphere.VirtualMachine, []map[string]string, error) { log := logger.GetLogger(ctx) + var ( vcHost string + matchingNodeVMs []*cnsvsphere.VirtualMachine completeTopologySegments []map[string]string ) // Fetch node topology information from informer cache. @@ -188,13 +156,13 @@ func getExpandedTopologySegments(ctx context.Context, requestedSegments map[stri err := runtime.DefaultUnstructuredConverter.FromUnstructured(val.(*unstructured.Unstructured).Object, &nodeTopologyInstance) if err != nil { - return nil, logger.LogNewErrorf(log, "failed to convert unstructured object %+v to "+ + return nil, nil, logger.LogNewErrorf(log, "failed to convert unstructured object %+v to "+ "CSINodeTopology instance. Error: %+v", val, err) } // Check CSINodeTopology instance `Status` field for success. if nodeTopologyInstance.Status.Status != csinodetopologyv1alpha1.CSINodeTopologySuccess { - return nil, logger.LogNewErrorf(log, "node %q not yet ready. Found CSINodeTopology instance "+ + return nil, nil, logger.LogNewErrorf(log, "node %q not yet ready. Found CSINodeTopology instance "+ "status: %q with error message: %q", nodeTopologyInstance.Name, nodeTopologyInstance.Status.Status, nodeTopologyInstance.Status.ErrorMessage) } @@ -213,21 +181,23 @@ func getExpandedTopologySegments(ctx context.Context, requestedSegments map[stri break } } - // If there is a match, check if each compatible NodeVM belongs to the same VC. If not, - // error out as we do not support cross-zonal volume provisioning. + // If there is a match, fetch the nodeVM object and add it to matchingNodeVMs. if isMatch { - nodeVM, err := nodeMgr.GetNodeVMAndUpdateCache(ctx, nodeTopologyInstance.Spec.NodeUUID, nil) + nodeVM, err := nodeMgr.GetNode(ctx, nodeTopologyInstance.Spec.NodeUUID, nil) if err != nil { - return nil, logger.LogNewErrorf(log, + return nil, nil, logger.LogNewErrorf(log, "failed to retrieve NodeVM %q. Error - %+v", nodeTopologyInstance.Spec.NodeID, err) } + // Check if each compatible NodeVM belongs to the same VC. If not, + // error out as we do not support cross-zonal volume provisioning. if vcHost == "" { vcHost = nodeVM.VirtualCenterHost } else if vcHost != nodeVM.VirtualCenterHost { - return nil, logger.LogNewErrorf(log, + return nil, nil, logger.LogNewErrorf(log, "found NodeVM %q belonging to different vCenter: %q. Expected vCenter: %q", nodeVM.Name(), nodeVM.VirtualCenterHost, vcHost) } + matchingNodeVMs = append(matchingNodeVMs, nodeVM) // Store the complete hierarchy of topology requestedSegments for future use. var exists bool @@ -242,8 +212,7 @@ func getExpandedTopologySegments(ctx context.Context, requestedSegments map[stri } } } - - return completeTopologySegments, nil + return matchingNodeVMs, completeTopologySegments, nil } // GetTopologyInfoFromNodes retrieves the topology information of the given diff --git a/pkg/csi/service/common/topology.go b/pkg/csi/service/common/topology.go index bdb620b1a0..026426f1d3 100644 --- a/pkg/csi/service/common/topology.go +++ b/pkg/csi/service/common/topology.go @@ -2,15 +2,15 @@ package common import ( "context" - "strings" "sync" "github.com/container-storage-interface/spec/lib/go/csi" "github.com/vmware/govmomi/object" "github.com/vmware/govmomi/vim25/mo" + "sigs.k8s.io/vsphere-csi-driver/v3/pkg/common/config" + "sigs.k8s.io/vsphere-csi-driver/v3/pkg/common/cns-lib/node" cnsvsphere "sigs.k8s.io/vsphere-csi-driver/v3/pkg/common/cns-lib/vsphere" - "sigs.k8s.io/vsphere-csi-driver/v3/pkg/common/config" "sigs.k8s.io/vsphere-csi-driver/v3/pkg/csi/service/logger" csinodetopologyv1alpha1 "sigs.k8s.io/vsphere-csi-driver/v3/pkg/internalapis/csinodetopology/v1alpha1" ) @@ -35,274 +35,17 @@ var ( preferredDatastoresMap = make(map[string]map[string][]string) // preferredDatastoresMapInstanceLock guards the preferredDatastoresMap from read-write overlaps. preferredDatastoresMapInstanceLock = &sync.RWMutex{} - // tagVCEntityMoRefMap maintains a cache of topology tags to the vCenter IP/FQDN & the MoRef of the - // entity which holds the tag. - // Example - { - // "region-1": {"vc1": [{Type:Datacenter Value:datacenter-3}], "vc2": [{Type:Datacenter Value:datacenter-5}] }, - // "zone-1": {"vc1": [{Type:ClusterComputeResource Value:domain-c12}] }, - // "zone-2": {"vc2": [{Type:ClusterComputeResource Value:domain-c8] },} - tagVCEntityMoRefMap = make(map[string]map[string][]mo.Reference) + // topologyVCMapInstanceLock guards the topologyVCMap instance from concurrent writes. + topologyVCMapInstanceLock = &sync.RWMutex{} + // topologyVCMap maintains a cache of topology tags to the vCenter IP/FQDN which holds the tag. + // Example - {region1: {VC1: struct{}{}, VC2: struct{}{}}, + // zone1: {VC1: struct{}{}}, + // zone2: {VC2: struct{}{}}} + // The vCenter IP/FQDN under each tag are maintained as a map of string with nil values to improve + // retrieval and deletion performance. + topologyVCMap = make(map[string]map[string]struct{}) ) -// DiscoverTagEntities populates tagVCEntityMoRefMap with tagName -> VC -> associated MoRefs mapping. -// NOTE: Any edits to existing topology labels will require a restart of the controller. -func DiscoverTagEntities(ctx context.Context) error { - log := logger.GetLogger(ctx) - // Get CNS config. - cnsCfg, err := config.GetConfig(ctx) - if err != nil { - return logger.LogNewErrorf(log, "failed to fetch CNS config. Error: %+v", err) - } - - var categories []string - zoneCat := strings.TrimSpace(cnsCfg.Labels.Zone) - regionCat := strings.TrimSpace(cnsCfg.Labels.Region) - if zoneCat != "" && regionCat != "" { - categories = []string{zoneCat, regionCat} - } else if strings.TrimSpace(cnsCfg.Labels.TopologyCategories) != "" { - categories = strings.Split(cnsCfg.Labels.TopologyCategories, ",") - for index := range categories { - categories[index] = strings.TrimSpace(categories[index]) - } - } else { - log.Infof("DiscoverTagEntities: No topology information found in CNS config.") - return nil - } - log.Infof("Topology categories being considered for tag to VC mapping are %+v", categories) - - vcenterConfigs, err := cnsvsphere.GetVirtualCenterConfigs(ctx, cnsCfg) - if err != nil { - return logger.LogNewErrorf(log, "failed to get VirtualCenterConfigs. Error: %v", err) - } - for _, vcenterCfg := range vcenterConfigs { - // Get VC instance - vcenter, err := cnsvsphere.GetVirtualCenterInstanceForVCenterConfig(ctx, vcenterCfg, false) - if err != nil { - return logger.LogNewErrorf(log, "failed to get vCenterInstance for vCenter Host: %q. Error: %v", - vcenterCfg.Host, err) - } - // Get tag manager instance. - tagManager, err := cnsvsphere.GetTagManager(ctx, vcenter) - if err != nil { - return logger.LogNewErrorf(log, "failed to create tagManager. Error: %v", err) - } - defer func() { - err := tagManager.Logout(ctx) - if err != nil { - log.Errorf("failed to logout tagManager. Error: %v", err) - } - }() - for _, cat := range categories { - topoTags, err := tagManager.GetTagsForCategory(ctx, cat) - if err != nil { - return logger.LogNewErrorf(log, "failed to fetch tags for category %q", cat) - } - log.Infof("Tags associated with category %q are %+v", cat, topoTags) - for _, tag := range topoTags { - objMORs, err := tagManager.ListAttachedObjects(ctx, tag.ID) - if err != nil { - return logger.LogNewErrorf(log, "failed to fetch objects associated with tag %q", tag.Name) - } - log.Infof("Entities associated with tag %q are %+v", tag.Name, objMORs) - if len(objMORs) == 0 { - continue - } - if _, exists := tagVCEntityMoRefMap[tag.Name]; !exists { - tagVCEntityMoRefMap[tag.Name] = map[string][]mo.Reference{vcenterCfg.Host: objMORs} - } else { - tagVCEntityMoRefMap[tag.Name][vcenterCfg.Host] = objMORs - } - } - } - } - log.Debugf("tagVCEntityMoRefMap: %+v", tagVCEntityMoRefMap) - return nil -} - -// GetHostsForSegment retrieves the list of hosts for a topology segment by first -// finding the entities associated with the tag lower in hierarchy. -func GetHostsForSegment(ctx context.Context, topoSegment map[string]string, vCenter *cnsvsphere.VirtualCenter) ( - []*cnsvsphere.HostSystem, error) { - log := logger.GetLogger(ctx) - var ( - allhostSlices [][]*cnsvsphere.HostSystem - ) - - // Get the entity MoRefs for each tag. - for key, tag := range topoSegment { - var hostList []*cnsvsphere.HostSystem - entityMorefs, exists := areEntityMorefsPresentForTag(tag, vCenter.Config.Host) - if !exists { - // Refresh cache to see if the tag has been added recently. - log.Infof("Refresh cache to see if the tag has been added recently") - err := DiscoverTagEntities(ctx) - if err != nil { - return nil, logger.LogNewErrorf(log, - "failed to update cache with topology information. Error: %+v", err) - } - entityMorefs, exists = areEntityMorefsPresentForTag(tag, vCenter.Config.Host) - if !exists { - return nil, logger.LogNewErrorf(log, "failed to find tag %q in VC %q.", tag, vCenter.Config.Host) - } - } - log.Infof("Tag %q is applied on entities %+v", tag, entityMorefs) - log.Debugf("Fetching hosts for entities %+v", entityMorefs) - for _, entity := range entityMorefs { - hosts, err := fetchHosts(ctx, entity, vCenter) - if err != nil { - return nil, logger.LogNewErrorf(log, "failed to fetch hosts from entity %+v. Error: %+v", - entity.Reference(), err) - } - hostList = append(hostList, hosts...) - } - log.Infof("Hosts returned for topology category: %q and tag: %q are %v", key, tag, hostList) - allhostSlices = append(allhostSlices, hostList) - } - commonHosts := findCommonHostsforAllTopologyKeys(ctx, allhostSlices) - log.Infof("common hosts: %v for all segments: %v", commonHosts, topoSegment) - return commonHosts, nil -} - -// findCommonHostsforAllTopologyKeys helps find common hosts across all slices in hostLists -func findCommonHostsforAllTopologyKeys(ctx context.Context, - hostLists [][]*cnsvsphere.HostSystem) []*cnsvsphere.HostSystem { - log := logger.GetLogger(ctx) - log.Infof("finding common hosts for hostlists: %v", hostLists) - if len(hostLists) == 0 { - return []*cnsvsphere.HostSystem{} - } - // Create a map to store hosts and their occurrence count - hostCount := make(map[string]int) - // Count occurrences of elements in the first slice - for _, host := range hostLists[0] { - hostCount[host.String()]++ - } - log.Debugf("hostCount after setting count in the first slice : %v", hostCount) - // Iterate through the remaining slices and update the hostCount map - for i := 1; i < len(hostLists); i++ { - for _, host := range hostLists[i] { - // If the host exists in the map, increment its count - if count, exists := hostCount[host.String()]; exists { - hostCount[host.String()] = count + 1 - } - } - } - log.Debugf("hostCount after iterate through the remaining slices and updated hostCount map : %v", hostCount) - // Create a slice to store the intersection - var commonHosts []string - - // Check if each hosts occurred in all slices - for hostMoRefName, count := range hostCount { - if count == len(hostLists) { - commonHosts = append(commonHosts, hostMoRefName) - } - } - log.Debugf("common hosts: %v", commonHosts) - - // Iterate through the all slices and get common hosts - var commonHostSystem []*cnsvsphere.HostSystem - for _, host := range commonHosts { - out: - for i := 0; i < len(hostLists); i++ { - for _, hostSystem := range hostLists[i] { - if hostSystem.String() == host { - commonHostSystem = append(commonHostSystem, hostSystem) - break out - } - } - } - } - log.Debugf("commonHostSystem: %v", commonHostSystem) - return commonHostSystem -} - -// fetchHosts gives a list of hosts under the entity given as input. -func fetchHosts(ctx context.Context, entity mo.Reference, vCenter *cnsvsphere.VirtualCenter) ( - []*cnsvsphere.HostSystem, error) { - log := logger.GetLogger(ctx) - var hosts []*cnsvsphere.HostSystem - log.Infof("fetching hosts for entity: %v on vCenter: %q", entity, vCenter.Config.Host) - switch entity.Reference().Type { - case "rootFolder": - folder := object.NewFolder(vCenter.Client.Client, entity.Reference()) - children, err := folder.Children(ctx) - if err != nil { - return nil, logger.LogNewErrorf(log, - "failed to retrieve child entities of the rootFolder %+v. Error: %+v", entity.Reference(), err) - } - for _, child := range children { - hostList, err := fetchHosts(ctx, child.Reference(), vCenter) - if err != nil { - return nil, logger.LogNewErrorf(log, "failed to fetch hosts from entity %+v. Error: %+v", - child.Reference(), err) - } - hosts = append(hosts, hostList...) - } - case "Datacenter": - dc := cnsvsphere.Datacenter{ - Datacenter: object.NewDatacenter(vCenter.Client.Client, entity.Reference()), - VirtualCenterHost: vCenter.Config.Host} - var dcMo mo.Datacenter - err := dc.Properties(ctx, dc.Reference(), []string{"hostFolder"}, &dcMo) - if err != nil { - return nil, logger.LogNewErrorf(log, ""+ - "failed to retrieve hostFolder property for datacenter %+v", dc.Reference()) - } - hostList, err := fetchHosts(ctx, dcMo.HostFolder, vCenter) - if err != nil { - return nil, logger.LogNewErrorf(log, "failed to fetch hosts from entity %+v. Error: %+v", - dcMo, err) - } - hosts = append(hosts, hostList...) - case "Folder": - folder := object.NewFolder(vCenter.Client.Client, entity.Reference()) - children, err := folder.Children(ctx) - if err != nil { - return nil, logger.LogNewErrorf(log, "failed to fetch child entities of Folder %+v. Error: %+v", - entity.Reference(), err) - } - for _, child := range children { - hostList, err := fetchHosts(ctx, child.Reference(), vCenter) - if err != nil { - return nil, logger.LogNewErrorf(log, "failed to fetch hosts from entity %+v. Error: %+v", - child.Reference(), err) - } - hosts = append(hosts, hostList...) - } - case "ClusterComputeResource": - ccr := cnsvsphere.ClusterComputeResource{ - ClusterComputeResource: object.NewClusterComputeResource(vCenter.Client.Client, entity.Reference()), - VirtualCenterHost: vCenter.Config.Host} - hostList, err := ccr.GetHosts(ctx) - if err != nil { - return nil, logger.LogNewErrorf(log, "failed to retrieve hosts from cluster %+v. Error: %+v", - entity.Reference(), err) - } - hosts = append(hosts, hostList...) - case "HostSystem": - host := cnsvsphere.HostSystem{HostSystem: object.NewHostSystem(vCenter.Client.Client, entity.Reference())} - hosts = append(hosts, &host) - default: - return nil, logger.LogNewErrorf(log, "unrecognised entity type found %+v.", entity.Reference()) - } - - return hosts, nil -} - -// areEntityMorefsPresentForTag retrieves the entities in given VC which have the -// input tag associated with them. -func areEntityMorefsPresentForTag(tag, vcHost string) ([]mo.Reference, bool) { - vcEntityMap, exists := tagVCEntityMoRefMap[tag] - if !exists { - return nil, false - } - entityMorefs, exists := vcEntityMap[vcHost] - if !exists { - return nil, false - } - return entityMorefs, true -} - // GetAccessibilityRequirementsByVC clubs the accessibility requirements by the VC they belong to. func GetAccessibilityRequirementsByVC(ctx context.Context, topoReq *csi.TopologyRequirement) ( map[string][]map[string]string, error) { @@ -325,7 +68,7 @@ func GetAccessibilityRequirementsByVC(ctx context.Context, topoReq *csi.Topology return vcTopoSegmentsMap, nil } -// getVCForTopologySegments uses the tagVCEntityMoRefMap to retrieve the +// getVCForTopologySegments uses the topologyVCMap to retrieve the // VC instance for the given topology segments map in a multi-VC environment. func getVCForTopologySegments(ctx context.Context, topologySegments map[string]string) (string, error) { log := logger.GetLogger(ctx) @@ -334,10 +77,10 @@ func getVCForTopologySegments(ctx context.Context, topologySegments map[string]s vcCountMap := make(map[string]int) // Find the VC which contains all the labels given in the topologySegments. - // For example, if tagVCEntityMoRefMap looks like - // {"region-1": {"vc1": [{Type:Datacenter Value:datacenter-3}], "vc2": [{Type:Datacenter Value:datacenter-5}] }, - // "zone-1": {"vc1": [{Type:ClusterComputeResource Value:domain-c12}] }, - // "zone-2": {"vc2": [{Type:ClusterComputeResource Value:domain-c8] },} + // For example, if topologyVCMap looks like + // {"region-1": {"vc1": struct{}{}, "vc2": struct{}{} }, + // "zone-1": {"vc1": struct{}{} }, + // "zone-2": {"vc2": struct{}{} },} // For a given topologySegment // {"topology.csi.vmware.com/k8s-region": "region-1", // "topology.csi.vmware.com/k8s-zone": "zone-2"} @@ -345,22 +88,12 @@ func getVCForTopologySegments(ctx context.Context, topologySegments map[string]s // We go over the vcCountMap to check which VC has a count equal to // the len(topologySegment), in this case 2 and return that VC. for topologyKey, label := range topologySegments { - vcMap, exists := tagVCEntityMoRefMap[label] - if !exists { - // Refresh cache to see if the tag has been added recently. - err := DiscoverTagEntities(ctx) - if err != nil { - return "", logger.LogNewErrorf(log, - "failed to update cache with tag to VC to MoRef mapping. Error: %+v", err) - } - vcMap, exists = tagVCEntityMoRefMap[label] - } - if exists { - for vc := range vcMap { + if vcList, exists := topologyVCMap[label]; exists { + for vc := range vcList { vcCountMap[vc] = vcCountMap[vc] + 1 } } else { - return "", logger.LogNewErrorf(log, "Topology label %q not found in tag to vCenter mapping.", + return "", logger.LogNewErrorf(log, "Topology label %q not found in topology to vCenter mapping.", topologyKey+":"+label) } } @@ -472,7 +205,7 @@ func RefreshPreferentialDatastoresForMultiVCenter(ctx context.Context) error { preferredDatastoresMap = prefDatastoresMap PreferredDatastoresExist = true log.Debugf("preferredDatastoresMap :%v", preferredDatastoresMap) - log.Debugf("PreferredDatastoresExist: %t", PreferredDatastoresExist) + log.Debugf("PreferredDatastoresExist: %v", PreferredDatastoresExist) } return nil } @@ -502,6 +235,53 @@ func GetPreferredDatastoresInSegments(ctx context.Context, segments map[string]s return allPreferredDSURLs } +// AddLabelsToTopologyVCMap adds topology label to VC mapping for given CSINodeTopology instance +// in the topologyVCMap variable. +func AddLabelsToTopologyVCMap(ctx context.Context, nodeTopoObj csinodetopologyv1alpha1.CSINodeTopology) { + log := logger.GetLogger(ctx) + // Get node manager instance. + nodeManager := node.GetManager(ctx) + nodeVM, err := nodeManager.GetNode(ctx, nodeTopoObj.Spec.NodeUUID, nil) + if err != nil { + log.Errorf("Node %q is not yet registered in the node manager. Error: %+v", + nodeTopoObj.Spec.NodeUUID, err) + return + } + log.Infof("Topology labels %+v belong to %q VC", nodeTopoObj.Status.TopologyLabels, + nodeVM.VirtualCenterHost) + // Update topologyVCMap with topology label and associated VC host. + topologyVCMapInstanceLock.Lock() + defer topologyVCMapInstanceLock.Unlock() + for _, label := range nodeTopoObj.Status.TopologyLabels { + if _, exists := topologyVCMap[label.Value]; !exists { + topologyVCMap[label.Value] = map[string]struct{}{nodeVM.VirtualCenterHost: {}} + } else { + topologyVCMap[label.Value][nodeVM.VirtualCenterHost] = struct{}{} + } + } +} + +// RemoveLabelsFromTopologyVCMap removes the topology label to VC mapping for given CSINodeTopology +// instance in the topologyVCMap variable. +func RemoveLabelsFromTopologyVCMap(ctx context.Context, nodeTopoObj csinodetopologyv1alpha1.CSINodeTopology) { + log := logger.GetLogger(ctx) + // Get node manager instance. + nodeManager := node.GetManager(ctx) + nodeVM, err := nodeManager.GetNode(ctx, nodeTopoObj.Spec.NodeUUID, nil) + if err != nil { + log.Errorf("Node %q is not yet registered in the node manager. Error: %+v", + nodeTopoObj.Spec.NodeUUID, err) + return + } + log.Infof("Removing VC %q mapping for TopologyLabels %+v.", nodeVM.VirtualCenterHost, + nodeTopoObj.Status.TopologyLabels) + topologyVCMapInstanceLock.Lock() + defer topologyVCMapInstanceLock.Unlock() + for _, label := range nodeTopoObj.Status.TopologyLabels { + delete(topologyVCMap[label.Value], nodeVM.VirtualCenterHost) + } +} + // AddNodeToDomainNodeMapNew adds the CR instance name in the domainNodeMap wherever appropriate. func AddNodeToDomainNodeMapNew(ctx context.Context, nodeTopoObj csinodetopologyv1alpha1.CSINodeTopology) { log := logger.GetLogger(ctx) diff --git a/pkg/csi/service/common/util_test.go b/pkg/csi/service/common/util_test.go index 77f6079b0f..ee1e2f2fab 100644 --- a/pkg/csi/service/common/util_test.go +++ b/pkg/csi/service/common/util_test.go @@ -340,21 +340,6 @@ func TestInvalidVolumeCapabilitiesForFile(t *testing.T) { if err := IsValidVolumeCapabilities(ctx, volCap); err == nil { t.Errorf("Invalid file VolCap = %+v passed validation!", volCap) } - - // Invalid case: volumeMode=block and accessMode=MULTI_NODE_READER_ONLY - volCap = []*csi.VolumeCapability{ - { - AccessType: &csi.VolumeCapability_Block{ - Block: &csi.VolumeCapability_BlockVolume{}, - }, - AccessMode: &csi.VolumeCapability_AccessMode{ - Mode: csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY, - }, - }, - } - if err := IsValidVolumeCapabilities(ctx, volCap); err == nil { - t.Errorf("Invalid file VolCap = %+v passed validation!", volCap) - } } func isStorageClassParamsEqual(expected *StorageClassParams, actual *StorageClassParams) bool { diff --git a/pkg/csi/service/common/vsphereutil.go b/pkg/csi/service/common/vsphereutil.go index bb6b71dc4f..d6b44105d7 100644 --- a/pkg/csi/service/common/vsphereutil.go +++ b/pkg/csi/service/common/vsphereutil.go @@ -17,6 +17,7 @@ limitations under the License. package common import ( + "fmt" "strconv" "strings" "time" @@ -89,7 +90,6 @@ func CreateBlockVolumeUtil(ctx context.Context, clusterFlavor cnstypes.CnsCluste } var datastoreObj *vsphere.Datastore var datastores []vim25types.ManagedObjectReference - var datastoreInfoList []*vsphere.DatastoreInfo if spec.ScParams.DatastoreURL == "" { // Check if datastore URL is specified by the storage pool parameter. if spec.VsanDirectDatastoreURL != "" { @@ -109,9 +109,8 @@ func CreateBlockVolumeUtil(ctx context.Context, clusterFlavor cnstypes.CnsCluste } // Search the datastore from the URL in the datacenter list. var datastoreObj *vsphere.Datastore - var datastoreInfoObj *vsphere.DatastoreInfo for _, datacenter := range dcList { - datastoreInfoObj, err = datacenter.GetDatastoreInfoByURL(ctx, spec.VsanDirectDatastoreURL) + datastoreInfoObj, err := datacenter.GetDatastoreInfoByURL(ctx, spec.VsanDirectDatastoreURL) if err != nil { log.Warnf("Failed to find datastore with URL %q in datacenter %q from VC %q, Error: %+v", spec.VsanDirectDatastoreURL, datacenter.InventoryPath, vc.Config.Host, err) @@ -125,7 +124,6 @@ func CreateBlockVolumeUtil(ctx context.Context, clusterFlavor cnstypes.CnsCluste log.Debugf("Successfully fetched the datastore %v from the URL: %v", datastoreObj.Reference(), spec.VsanDirectDatastoreURL) datastores = append(datastores, datastoreObj.Reference()) - datastoreInfoList = append(datastoreInfoList, datastoreInfoObj) break } if datastores == nil { @@ -139,7 +137,6 @@ func CreateBlockVolumeUtil(ctx context.Context, clusterFlavor cnstypes.CnsCluste // If DatastoreURL is not specified in StorageClass, get all shared // datastores. datastores = getDatastoreMoRefs(sharedDatastores) - datastoreInfoList = sharedDatastores } } else { // vc.GetDatacenters returns datacenters found on the VirtualCenter. @@ -155,9 +152,8 @@ func CreateBlockVolumeUtil(ctx context.Context, clusterFlavor cnstypes.CnsCluste return nil, csifault.CSIInternalFault, err } // Check if DatastoreURL specified in the StorageClass is present in any one of the datacenters. - var datastoreInfoObj *vsphere.DatastoreInfo for _, datacenter := range datacenters { - datastoreInfoObj, err = datacenter.GetDatastoreInfoByURL(ctx, spec.ScParams.DatastoreURL) + datastoreInfoObj, err := datacenter.GetDatastoreInfoByURL(ctx, spec.ScParams.DatastoreURL) if err != nil { log.Warnf("failed to find datastore with URL %q in datacenter %q from VC %q, Error: %+v", spec.ScParams.DatastoreURL, datacenter.InventoryPath, vc.Config.Host, err) @@ -188,7 +184,6 @@ func CreateBlockVolumeUtil(ctx context.Context, clusterFlavor cnstypes.CnsCluste } if isSharedDatastoreURL { datastores = append(datastores, datastoreObj.Reference()) - datastoreInfoList = append(datastoreInfoList, datastoreInfoObj) } else { // TODO: Need to figure out which fault need to return when datastore is not accessible to all nodes. // Currently, just return csi.fault.Internal. @@ -281,28 +276,18 @@ func CreateBlockVolumeUtil(ctx context.Context, clusterFlavor cnstypes.CnsCluste } // step 2: validate if the snapshot datastore is compatible with datastore candidates in create spec - var compatibleDatastore vim25types.ManagedObjectReference - var foundCompatibleDatastore bool = false - for _, dsInfo := range datastoreInfoList { - if dsInfo.Info.Url == cnsVolume.DatastoreUrl { - log.Infof("compatible datastore found, dsURL = %q, dsRef = %v", dsInfo.Info.Url, - dsInfo.Datastore.Reference()) - compatibleDatastore = dsInfo.Datastore.Reference() - foundCompatibleDatastore = true - break - } - } - if !foundCompatibleDatastore { + compatibleDatastore, err := utils.GetDatastoreRefByURLFromGivenDatastoreList( + ctx, vc, createSpec.Datastores, cnsVolume.DatastoreUrl) + if err != nil { return nil, csifault.CSIInternalFault, logger.LogNewErrorf(log, "failed to get the compatible datastore for create volume from snapshot %s with error: %+v", spec.ContentSourceSnapshotID, err) } - // overwrite the datatstores field in create spec with the compatible datastore log.Infof("Overwrite the datatstores field in create spec %v with the compatible datastore %v "+ - "when create volume from snapshot %s", createSpec.Datastores, compatibleDatastore, + "when create volume from snapshot %s", createSpec.Datastores, *compatibleDatastore, spec.ContentSourceSnapshotID) - createSpec.Datastores = []vim25types.ManagedObjectReference{compatibleDatastore} + createSpec.Datastores = []vim25types.ManagedObjectReference{*compatibleDatastore} } log.Debugf("vSphere CSI driver creating volume %s with create spec %+v", spec.Name, spew.Sdump(createSpec)) @@ -1019,8 +1004,9 @@ func isExpansionRequired(ctx context.Context, volumeID string, requestedSize int if len(queryResult.Volumes) > 0 { currentSize = queryResult.Volumes[0].BackingObjectDetails.GetCnsBackingObjectDetails().CapacityInMb } else { - // Error out as volume is not found during a resize operation. - return false, logger.LogNewErrorf(log, "failed to find volume by querying volumeID: %q", volumeID) + msg := fmt.Sprintf("failed to find volume by querying volumeID: %q", volumeID) + log.Error(msg) + return false, err } log.Infof("isExpansionRequired: Found current size of volumeID %q to be %d Mb. "+ diff --git a/pkg/csi/service/node.go b/pkg/csi/service/node.go index bec4d0a7a6..92988b77df 100644 --- a/pkg/csi/service/node.go +++ b/pkg/csi/service/node.go @@ -349,21 +349,15 @@ func (driver *vsphereCSIDriver) NodeGetInfo( return nil, logger.LogNewErrorCode(log, codes.Internal, "ENV NODE_NAME is not set") } - - clusterFlavor, err = cnsconfig.GetClusterFlavor(ctx) - if err != nil { - return nil, err - } - - if clusterFlavor == cnstypes.CnsClusterFlavorGuest { - nodeID = nodeName - } else { + if commonco.ContainerOrchestratorUtility.IsFSSEnabled(ctx, common.UseCSINodeId) { // Get VM UUID nodeID, err = driver.osUtils.GetSystemUUID(ctx) if err != nil { return nil, logger.LogNewErrorCodef(log, codes.Internal, "failed to get system uuid for node VM with error: %v", err) } + } else { + nodeID = nodeName } var maxVolumesPerNode int64 @@ -396,6 +390,11 @@ func (driver *vsphereCSIDriver) NodeGetInfo( accessibleTopology map[string]string ) + clusterFlavor, err = cnsconfig.GetClusterFlavor(ctx) + if err != nil { + return nil, err + } + if clusterFlavor == cnstypes.CnsClusterFlavorGuest { if !commonco.ContainerOrchestratorUtility.IsFSSEnabled(ctx, common.TKGsHA) { nodeInfoResponse = &csi.NodeGetInfoResponse{ diff --git a/pkg/csi/service/vanilla/controller.go b/pkg/csi/service/vanilla/controller.go index e94f74bc85..bf9ef57c0c 100644 --- a/pkg/csi/service/vanilla/controller.go +++ b/pkg/csi/service/vanilla/controller.go @@ -58,12 +58,12 @@ import ( // NodeManagerInterface provides functionality to manage (VM) nodes. type NodeManagerInterface interface { - Initialize(ctx context.Context) error + Initialize(ctx context.Context, useNodeUuid bool) error GetSharedDatastoresInK8SCluster(ctx context.Context) ([]*cnsvsphere.DatastoreInfo, error) - GetNodeVMByNameAndUpdateCache(ctx context.Context, nodeName string) (*cnsvsphere.VirtualMachine, error) - GetNodeVMByNameOrUUID(ctx context.Context, nodeName string) (*cnsvsphere.VirtualMachine, error) + GetNodeByName(ctx context.Context, nodeName string) (*cnsvsphere.VirtualMachine, error) + GetNodeByNameOrUUID(ctx context.Context, nodeName string) (*cnsvsphere.VirtualMachine, error) GetNodeNameByUUID(ctx context.Context, nodeUUID string) (string, error) - GetNodeVMByUuid(ctx context.Context, nodeUuid string) (*cnsvsphere.VirtualMachine, error) + GetNodeByUuid(ctx context.Context, nodeUuid string) (*cnsvsphere.VirtualMachine, error) GetAllNodes(ctx context.Context) ([]*cnsvsphere.VirtualMachine, error) GetAllNodesByVC(ctx context.Context, vcHost string) ([]*cnsvsphere.VirtualMachine, error) } @@ -105,7 +105,6 @@ var ( // variable for list snapshots CNSSnapshotsForListSnapshots = make([]cnstypes.CnsSnapshotQueryResultEntry, 0) CNSVolumeDetailsMap = make([]map[string]*utils.CnsVolumeDetails, 0) - volumeIDToNodeUUIDMap = make(map[string]string) ) // New creates a CNS controller. @@ -296,9 +295,12 @@ func (c *controller) Init(config *cnsconfig.Config, version string) error { } } } - + useNodeUuid := false + if commonco.ContainerOrchestratorUtility.IsFSSEnabled(ctx, common.UseCSINodeId) { + useNodeUuid = true + } c.nodeMgr = &node.Nodes{} - err = c.nodeMgr.Initialize(ctx) + err = c.nodeMgr.Initialize(ctx, useNodeUuid) if err != nil { log.Errorf("failed to initialize nodeMgr. err=%v", err) return err @@ -442,7 +444,7 @@ func (c *controller) ReloadConfiguration() error { // Re-Initialize Node Manager to cache latest vCenter config. log.Debug("Re-Initializing node manager") c.nodeMgr = &node.Nodes{} - err = c.nodeMgr.Initialize(ctx) + err = c.nodeMgr.Initialize(ctx, true) if err != nil { log.Errorf("failed to re-initialize nodeMgr. err=%v", err) return err @@ -485,8 +487,12 @@ func (c *controller) ReloadConfiguration() error { } c.manager.VcenterConfig = newVCConfig // Re-Initialize Node Manager to cache latest vCenter config. + useNodeUuid := false + if commonco.ContainerOrchestratorUtility.IsFSSEnabled(ctx, common.UseCSINodeId) { + useNodeUuid = true + } c.nodeMgr = &node.Nodes{} - err = c.nodeMgr.Initialize(ctx) + err = c.nodeMgr.Initialize(ctx, useNodeUuid) if err != nil { log.Errorf("failed to re-initialize nodeMgr. err=%v", err) return err @@ -705,7 +711,9 @@ func (c *controller) createBlockVolume(ctx context.Context, req *csi.CreateVolum }, } volTaskAlreadyRegistered = true - } else if cnsvolume.IsTaskPending(volumeOperationDetails) { + } else if volumeOperationDetails.OperationDetails.TaskStatus == + cnsvolumeoperationrequest.TaskInvocationStatusInProgress && + volumeOperationDetails.OperationDetails.TaskID != "" { // If task is created in CNS for this volume but task is in progress, then // we need to monitor the task to check if volume creation is completed or not. log.Infof("Volume with name %s has CreateVolume task %s pending on CNS.", @@ -1163,7 +1171,9 @@ func (c *controller) createBlockVolumeWithPlacementEngineForMultiVC(ctx context. }, } volTaskAlreadyRegistered = true - } else if cnsvolume.IsTaskPending(volumeOperationDetails) { + } else if volumeOperationDetails.OperationDetails.TaskStatus == + cnsvolumeoperationrequest.TaskInvocationStatusInProgress && + volumeOperationDetails.OperationDetails.TaskID != "" { // If task is already created in CNS for this volume but task is in progress, // we need to monitor the task to check if volume creation is complete or not. log.Infof("Volume with name %s has CreateVolume task %s pending on VC %q.", @@ -1552,7 +1562,7 @@ func (c *controller) calculateAccessibleTopologiesForDatastore(ctx context.Conte log := logger.GetLogger(ctx) var datastoreAccessibleTopology []map[string]string - // Find out all nodeVMs which have access to the chosen datastore among all the nodes in k8s cluster. + // Find out all nodes which have access to the chosen datastore. accessibleNodes, err := common.GetNodeVMsWithAccessToDatastore(ctx, vcenter, datastoreURL, allNodeVMs) if err != nil || len(accessibleNodes) == 0 { return nil, logger.LogNewErrorCodef(log, codes.Internal, @@ -1653,7 +1663,9 @@ func (c *controller) createFileVolume(ctx context.Context, req *csi.CreateVolume volumeID = volumeOperationDetails.VolumeID volTaskAlreadyRegistered = true - } else if cnsvolume.IsTaskPending(volumeOperationDetails) { + } else if volumeOperationDetails.OperationDetails.TaskStatus == + cnsvolumeoperationrequest.TaskInvocationStatusInProgress && + volumeOperationDetails.OperationDetails.TaskID != "" { var ( volumeInfo *cnsvolume.CnsVolumeInfo vcenter *cnsvsphere.VirtualCenter @@ -1792,7 +1804,7 @@ func (c *controller) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequ createVolumeInternal := func() ( *csi.CreateVolumeResponse, string, error) { log.Infof("CreateVolume: called with args %+v", *req) - // TODO: If the err is returned by invoking CNS API, then faultType should be + //TODO: If the err is returned by invoking CNS API, then faultType should be // populated by the underlying layer. // If the request failed due to validate the request, "csi.fault.InvalidArgument" will be return. // If thr reqeust failed due to object not found, "csi.fault.NotFound" will be return. @@ -1847,9 +1859,6 @@ func (c *controller) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequ resp, faultType, err := createVolumeInternal() log.Debugf("createVolumeInternal: returns fault %q", faultType) if err != nil { - if csifault.IsNonStorageFault(faultType) { - faultType = csifault.AddCsiNonStoragePrefix(ctx, faultType) - } log.Errorf("Operation failed, reporting failure status to Prometheus."+ " Operation Type: %q, Volume Type: %q, Fault Type: %q", prometheus.PrometheusCreateVolumeOpType, volumeType, faultType) @@ -1875,7 +1884,7 @@ func (c *controller) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequ deleteVolumeInternal := func() ( *csi.DeleteVolumeResponse, string, error) { log.Infof("DeleteVolume: called with args: %+v", *req) - // TODO: If the err is returned by invoking CNS API, then faultType should be + //TODO: If the err is returned by invoking CNS API, then faultType should be // populated by the underlying layer. // If the request failed due to validate the request, "csi.fault.InvalidArgument" will be return. // If thr reqeust failed due to object not found, "csi.fault.NotFound" will be return. @@ -1990,9 +1999,6 @@ func (c *controller) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequ resp, faultType, err := deleteVolumeInternal() log.Debugf("deleteVolumeInternal: returns fault %q for volume %q", faultType, req.VolumeId) if err != nil { - if csifault.IsNonStorageFault(faultType) { - faultType = csifault.AddCsiNonStoragePrefix(ctx, faultType) - } log.Errorf("Operation failed, reporting failure status to Prometheus."+ " Operation Type: %q, Volume Type: %q, Fault Type: %q", prometheus.PrometheusDeleteVolumeOpType, volumeType, faultType) @@ -2018,7 +2024,7 @@ func (c *controller) ControllerPublishVolume(ctx context.Context, req *csi.Contr controllerPublishVolumeInternal := func() ( *csi.ControllerPublishVolumeResponse, string, error) { log.Infof("ControllerPublishVolume: called with args %+v", *req) - // TODO: If the err is returned by invoking CNS API, then faultType should be + //TODO: If the err is returned by invoking CNS API, then faultType should be // populated by the underlying layer. // If the request failed due to validate the request, "csi.fault.InvalidArgument" will be return. // If thr reqeust failed due to object not found, "csi.fault.NotFound" will be return. @@ -2107,12 +2113,17 @@ func (c *controller) ControllerPublishVolume(ctx context.Context, req *csi.Contr } } var nodevm *cnsvsphere.VirtualMachine - // if node is not yet updated to run the release of the driver publishing Node VM UUID as Node ID - // look up Node by name - nodevm, err = c.nodeMgr.GetNodeVMByNameOrUUID(ctx, req.NodeId) - if err == node.ErrNodeNotFound { - log.Infof("Performing node VM lookup using node VM UUID: %q", req.NodeId) - nodevm, err = c.nodeMgr.GetNodeVMByUuid(ctx, req.NodeId) + if commonco.ContainerOrchestratorUtility.IsFSSEnabled(ctx, common.UseCSINodeId) { + // if node is not yet updated to run the release of the driver publishing Node VM UUID as Node ID + // look up Node by name + nodevm, err = c.nodeMgr.GetNodeByNameOrUUID(ctx, req.NodeId) + if err == node.ErrNodeNotFound { + log.Infof("Performing node VM lookup using node VM UUID: %q", req.NodeId) + nodevm, err = c.nodeMgr.GetNodeByUuid(ctx, req.NodeId) + } + + } else { + nodevm, err = c.nodeMgr.GetNodeByName(ctx, req.NodeId) } if err != nil { return nil, csifault.CSIInternalFault, logger.LogNewErrorCodef(log, codes.Internal, @@ -2137,9 +2148,6 @@ func (c *controller) ControllerPublishVolume(ctx context.Context, req *csi.Contr resp, faultType, err := controllerPublishVolumeInternal() log.Debugf("controllerPublishVolumeInternal: returns fault %q for volume %q", faultType, req.VolumeId) if err != nil { - if csifault.IsNonStorageFault(faultType) { - faultType = csifault.AddCsiNonStoragePrefix(ctx, faultType) - } log.Errorf("Operation failed, reporting failure status to Prometheus."+ " Operation Type: %q, Volume Type: %q, Fault Type: %q", prometheus.PrometheusAttachVolumeOpType, volumeType, faultType) @@ -2166,7 +2174,7 @@ func (c *controller) ControllerUnpublishVolume(ctx context.Context, req *csi.Con *csi.ControllerUnpublishVolumeResponse, string, error) { var faultType string log.Infof("ControllerUnpublishVolume: called with args %+v", *req) - // TODO: If the err is returned by invoking CNS API, then faultType should be + //TODO: If the err is returned by invoking CNS API, then faultType should be // populated by the underlying layer. // If the request failed due to validate the request, "csi.fault.InvalidArgument" will be return. // If thr reqeust failed due to object not found, "csi.fault.NotFound" will be return. @@ -2246,12 +2254,16 @@ func (c *controller) ControllerUnpublishVolume(ctx context.Context, req *csi.Con // Block Volume. volumeType = prometheus.PrometheusBlockVolumeType var nodevm *cnsvsphere.VirtualMachine - // if node is not yet updated to run the release of the driver publishing Node VM UUID as Node ID - // look up Node by name - nodevm, err = c.nodeMgr.GetNodeVMByNameOrUUID(ctx, req.NodeId) - if err == node.ErrNodeNotFound { - log.Infof("Performing node VM lookup using node VM UUID: %q", req.NodeId) - nodevm, err = c.nodeMgr.GetNodeVMByUuid(ctx, req.NodeId) + if commonco.ContainerOrchestratorUtility.IsFSSEnabled(ctx, common.UseCSINodeId) { + // if node is not yet updated to run the release of the driver publishing Node VM UUID as Node ID + // look up Node by name + nodevm, err = c.nodeMgr.GetNodeByNameOrUUID(ctx, req.NodeId) + if err == node.ErrNodeNotFound { + log.Infof("Performing node VM lookup using node VM UUID: %q", req.NodeId) + nodevm, err = c.nodeMgr.GetNodeByUuid(ctx, req.NodeId) + } + } else { + nodevm, err = c.nodeMgr.GetNodeByName(ctx, req.NodeId) } if err != nil { if err == cnsvsphere.ErrVMNotFound { @@ -2274,9 +2286,6 @@ func (c *controller) ControllerUnpublishVolume(ctx context.Context, req *csi.Con resp, faultType, err := controllerUnpublishVolumeInternal() log.Debugf("controllerUnpublishVolumeInternal: returns fault %q for volume %q", faultType, req.VolumeId) if err != nil { - if csifault.IsNonStorageFault(faultType) { - faultType = csifault.AddCsiNonStoragePrefix(ctx, faultType) - } log.Errorf("Operation failed, reporting failure status to Prometheus."+ " Operation Type: %q, Volume Type: %q, Fault Type: %q", prometheus.PrometheusDetachVolumeOpType, volumeType, faultType) @@ -2318,15 +2327,8 @@ func (c *controller) ControllerExpandVolume(ctx context.Context, req *csi.Contro // csifault.CSIInternalFault csifault.CSIUnimplementedFault csifault.CSIInvalidArgumentFault if strings.Contains(req.VolumeId, ".vmdk") { - if err := initVolumeMigrationService(ctx, c); err != nil { - // Error is already wrapped in CSI error code. - return nil, csifault.CSIInternalFault, err - } - req.VolumeId, err = volumeMigrationService.GetVolumeID(ctx, &migration.VolumeSpec{VolumePath: req.VolumeId}, false) - if err != nil { - return nil, csifault.CSIInternalFault, logger.LogNewErrorCodef(log, codes.Internal, - "failed to get VolumeID from volumeMigrationService for volumePath: %q", req.VolumeId) - } + return nil, csifault.CSIUnimplementedFault, logger.LogNewErrorCodef(log, codes.Unimplemented, + "cannot expand migrated vSphere volume. :%q", req.VolumeId) } // Fetch vCenterHost, vCenterManager & volumeManager for given volume, based on VC configuration @@ -2383,7 +2385,7 @@ func (c *controller) ControllerExpandVolume(ctx context.Context, req *csi.Contro volSizeMB, commonco.ContainerOrchestratorUtility.IsFSSEnabled(ctx, common.AsyncQueryVolume)) if err != nil { return nil, faultType, logger.LogNewErrorCodef(log, codes.Internal, - "failed to expand volume: %q to size: %d with error: %+v", "df", volSizeMB, err) + "failed to expand volume: %q to size: %d with error: %+v", volumeID, volSizeMB, err) } // Always set nodeExpansionRequired to true, even if requested size is equal @@ -2409,9 +2411,6 @@ func (c *controller) ControllerExpandVolume(ctx context.Context, req *csi.Contro resp, faultType, err := controllerExpandVolumeInternal() if err != nil { log.Debugf("controllerExpandVolumeInternal: returns fault %q for volume %q", faultType, req.VolumeId) - if csifault.IsNonStorageFault(faultType) { - faultType = csifault.AddCsiNonStoragePrefix(ctx, faultType) - } log.Errorf("Operation failed, reporting failure status to Prometheus."+ " Operation Type: %q, Volume Type: %q, Fault Type: %q", prometheus.PrometheusExpandVolumeOpType, volumeType, faultType) @@ -2489,7 +2488,6 @@ func (c *controller) ListVolumes(ctx context.Context, req *csi.ListVolumesReques querySelection := cnstypes.CnsQuerySelection{ Names: []string{ string(cnstypes.QuerySelectionNameTypeVolumeType), - string(cnstypes.QuerySelectionNameTypeVolumeName), }, } // For multi-VC configuration, query volumes from all vCenters @@ -2513,22 +2511,7 @@ func (c *controller) ListVolumes(ctx context.Context, req *csi.ListVolumesReques } CNSVolumesforListVolume = cnsQueryResult.Volumes } - - // Get all nodes from the vanilla K8s cluster from the node manager - allNodeVMs, err := c.nodeMgr.GetAllNodes(ctx) - if err != nil { - return nil, csifault.CSIInternalFault, logger.LogNewErrorCodef(log, codes.Internal, - "failed to get nodes(node vms) in the vanilla cluster. Error: %v", err) - } - - // Fetching below map once per resync cycle to be used later while processing the volumes - volumeIDToNodeUUIDMap, err = getBlockVolumeIDToNodeUUIDMap(ctx, c, allNodeVMs) - if err != nil { - return nil, csifault.CSIInternalFault, logger.LogNewErrorCodef(log, codes.Internal, - "get block volumeIDToNodeUUIDMap failed with err = %+v ", err) - } } - // Step 3: If the difference between number of K8s volumes and CNS volumes is greater than threshold, // fail the operation, as it can result in too many attach calls. if len(volIDsInK8s)-len(CNSVolumesforListVolume) > cfg.Global.ListVolumeThreshold { @@ -2543,13 +2526,21 @@ func (c *controller) ListVolumes(ctx context.Context, req *csi.ListVolumesReques maxEntries = len(CNSVolumesforListVolume) } // Step 4: process queryLimit number of items starting from ListVolumeRequest.start_token + var allNodeVMs []*cnsvsphere.VirtualMachine var entries []*csi.ListVolumesResponse_Entry + // Get all nodes from the vanilla K8s cluster from the node manager + allNodeVMs, err = c.nodeMgr.GetAllNodes(ctx) + if err != nil { + return nil, csifault.CSIInternalFault, logger.LogNewErrorCodef(log, codes.Internal, + "failed to get nodes(node vms) in the vanilla cluster. Error: %v", err) + } + nextToken := "" log.Debugf("Starting token: %d, Length of Query volume result: %d, Max entries: %d ", startingToken, len(CNSVolumesforListVolume), maxEntries) entries, nextToken, volumeType, err = c.processQueryResultsListVolumes(ctx, startingToken, maxEntries, - CNSVolumesforListVolume) + CNSVolumesforListVolume, allNodeVMs) if err != nil { return nil, csifault.CSIInternalFault, fmt.Errorf("error while processing query results for list "+ " volumes, err: %v", err) @@ -2565,9 +2556,6 @@ func (c *controller) ListVolumes(ctx context.Context, req *csi.ListVolumesReques listVolResponse, faultType, err := listVolumesInternal() log.Debugf("List volume response: %+v", listVolResponse) if err != nil { - if csifault.IsNonStorageFault(faultType) { - faultType = csifault.AddCsiNonStoragePrefix(ctx, faultType) - } log.Errorf("Operation failed, reporting failure status to Prometheus."+ " Operation Type: %q, Volume Type: %q, Fault Type: %q", prometheus.PrometheusListVolumeOpType, volumeType, faultType) @@ -2581,7 +2569,7 @@ func (c *controller) ListVolumes(ctx context.Context, req *csi.ListVolumesReques } func (c *controller) processQueryResultsListVolumes(ctx context.Context, startingToken int, maxEntries int, - cnsVolumes []cnstypes.CnsVolume) ([]*csi.ListVolumesResponse_Entry, + cnsVolumes []cnstypes.CnsVolume, allNodeVMs []*cnsvsphere.VirtualMachine) ([]*csi.ListVolumesResponse_Entry, string, string, error) { volumeType := "" @@ -2591,6 +2579,11 @@ func (c *controller) processQueryResultsListVolumes(ctx context.Context, startin log := logger.GetLogger(ctx) var entries []*csi.ListVolumesResponse_Entry + volumeIDToNodeUUIDMap, err := getBlockVolumeIDToNodeUUIDMap(ctx, c, allNodeVMs) + if err != nil { + return entries, nextToken, volumeType, err + } + for i := startingToken; i < len(cnsVolumes); i++ { if cnsVolumes[i].VolumeType == common.FileVolumeType { // If this is multi-VC configuration, then @@ -2611,7 +2604,7 @@ func (c *controller) processQueryResultsListVolumes(ctx context.Context, startin publishedNodeIds := commonco.ContainerOrchestratorUtility.GetNodesForVolumes(ctx, []string{fileVolID}) for volID, nodeName := range publishedNodeIds { if volID == fileVolID && len(nodeName) != 0 { - nodeVMObj, err := c.nodeMgr.GetNodeVMByNameAndUpdateCache(ctx, publishedNodeIds[fileVolID][0]) + nodeVMObj, err := c.nodeMgr.GetNodeByName(ctx, publishedNodeIds[fileVolID][0]) if err != nil { log.Errorf("Failed to get node vm object from the node name, err:%v", err) return entries, nextToken, volumeType, err @@ -2643,21 +2636,9 @@ func (c *controller) processQueryResultsListVolumes(ctx context.Context, startin nodeVMUUID, found := volumeIDToNodeUUIDMap[blockVolID] if found { volCounter += 1 - volumeId := blockVolID - // this check is required as volumeMigrationService is not initialized - // when multi-vc is enabled and there is more than 1 vc - if volumeMigrationService != nil { - migratedVolumePath, err := volumeMigrationService.GetVolumePathFromMigrationServiceCache(ctx, blockVolID) - if err != nil && err == common.ErrNotFound { - log.Debugf("volumeID: %v not found in migration service in-memory cache "+ - "so it's not a migrated in-tree volume", blockVolID) - } else if migratedVolumePath != "" { - volumeId = migratedVolumePath - } - } - // Populate csi.Volume info for the given volume + //Populate csi.Volume info for the given volume blockVolumeInfo := &csi.Volume{ - VolumeId: volumeId, + VolumeId: blockVolID, } // Getting published nodes volStatus := &csi.ListVolumesResponse_VolumeStatus{ @@ -3218,7 +3199,7 @@ func queryAllVolumeSnapshotsForMultiVC(ctx context.Context, c *controller, token CNSSnapshotsForListSnapshots = snapQueryEntries CNSVolumeDetailsMap = cnsVolumeDetailsMap } else { - // fetch snapshots + //fetch snapshots snapQueryEntries, volumeDetails, err := getSnapshotsAndSourceVolumeDetails(ctx, vCenterManager, c.manager.VolumeManager, c.manager.VcenterConfig.Host) if err != nil { diff --git a/pkg/csi/service/vanilla/controller_test.go b/pkg/csi/service/vanilla/controller_test.go index 271a5c0724..ddff1d3bad 100644 --- a/pkg/csi/service/vanilla/controller_test.go +++ b/pkg/csi/service/vanilla/controller_test.go @@ -169,7 +169,7 @@ func configFromEnvOrSim() (*config.Config, func()) { return cfg, func() {} } -func (f *FakeNodeManager) Initialize(ctx context.Context) error { +func (f *FakeNodeManager) Initialize(ctx context.Context, useNodeUuid bool) error { return nil } @@ -223,12 +223,11 @@ func (f *FakeNodeManager) GetSharedDatastoresInK8SCluster(ctx context.Context) ( }, nil } -func (f *FakeNodeManager) GetNodeVMByNameAndUpdateCache(ctx context.Context, - nodeName string) (*cnsvsphere.VirtualMachine, error) { +func (f *FakeNodeManager) GetNodeByName(ctx context.Context, nodeName string) (*cnsvsphere.VirtualMachine, error) { var vm *cnsvsphere.VirtualMachine var t *testing.T if v := os.Getenv("VSPHERE_DATACENTER"); v != "" { - nodeUUID, err := k8s.GetNodeUUID(ctx, f.k8sClient, nodeName) + nodeUUID, err := k8s.GetNodeUUID(ctx, f.k8sClient, nodeName, false) if err != nil { t.Errorf("failed to get providerId from node: %q. Err: %v", nodeName, err) return nil, err @@ -247,16 +246,16 @@ func (f *FakeNodeManager) GetNodeVMByNameAndUpdateCache(ctx context.Context, return vm, nil } -func (f *FakeNodeManager) GetNodeVMByNameOrUUID( +func (f *FakeNodeManager) GetNodeByNameOrUUID( ctx context.Context, nodeNameOrUUID string) (*cnsvsphere.VirtualMachine, error) { - return f.GetNodeVMByNameAndUpdateCache(ctx, nodeNameOrUUID) + return f.GetNodeByName(ctx, nodeNameOrUUID) } func (f *FakeNodeManager) GetNodeNameByUUID(ctx context.Context, nodeUUID string) (string, error) { return "", nil } -func (f *FakeNodeManager) GetNodeVMByUuid(ctx context.Context, nodeUuid string) (*cnsvsphere.VirtualMachine, error) { +func (f *FakeNodeManager) GetNodeByUuid(ctx context.Context, nodeUuid string) (*cnsvsphere.VirtualMachine, error) { var vm *cnsvsphere.VirtualMachine var t *testing.T if v := os.Getenv("VSPHERE_DATACENTER"); v != "" { diff --git a/pkg/csi/service/wcp/controller.go b/pkg/csi/service/wcp/controller.go index a2895a92c2..06c64f1d55 100644 --- a/pkg/csi/service/wcp/controller.go +++ b/pkg/csi/service/wcp/controller.go @@ -25,6 +25,7 @@ import ( "sync" "time" + "github.com/vmware/govmomi/vim25/types" "google.golang.org/protobuf/types/known/timestamppb" "github.com/container-storage-interface/spec/lib/go/csi" @@ -68,8 +69,6 @@ var ( csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME, csi.ControllerServiceCapability_RPC_EXPAND_VOLUME, - csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT, - csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS, } checkCompatibleDataStores = true ) @@ -79,11 +78,8 @@ var getCandidateDatastores = cnsvsphere.GetCandidateDatastoresInCluster // Contains list of clusterComputeResourceMoIds on which supervisor cluster is deployed. var clusterComputeResourceMoIds = make([]string, 0) -var ( - expectedStartingIndex = 0 - cnsVolumeIDs = make([]string, 0) - vmMoidToHostMoid, volumeIDToVMMap map[string]string -) +var expectedStartingIndex = 0 +var cnsVolumeIDs = make([]string, 0) type controller struct { manager *common.Manager @@ -129,7 +125,6 @@ func (c *controller) Init(config *cnsconfig.Config, version string) error { log.Errorf("failed to get VirtualCenterConfig. err=%v", err) return err } - vcenterconfig.ReloadVCConfigForNewClient = true vcManager := cnsvsphere.GetVirtualCenterManager(ctx) vcenter, err := vcManager.RegisterVirtualCenter(ctx, vcenterconfig) if err != nil { @@ -244,8 +239,8 @@ func (c *controller) Init(config *cnsconfig.Config, version string) error { log.Infof("Successfully reloaded configuration from: %q", cfgPath) break } - log.Errorf("failed to reload configuration. will retry again in 60 seconds. err: %+v", reloadConfigErr) - time.Sleep(60 * time.Second) + log.Errorf("failed to reload configuration. will retry again in 5 seconds. err: %+v", reloadConfigErr) + time.Sleep(5 * time.Second) } } // Handling create event for reconnecting to VC when ca file is @@ -323,7 +318,6 @@ func (c *controller) ReloadConfiguration(reconnectToVCFromNewConfig bool) error return err } if newVCConfig != nil { - newVCConfig.ReloadVCConfigForNewClient = true var vcenter *cnsvsphere.VirtualCenter if c.manager.VcenterConfig.Host != newVCConfig.Host || c.manager.VcenterConfig.Username != newVCConfig.Username || @@ -721,17 +715,6 @@ func (c *controller) createBlockVolume(ctx context.Context, req *csi.CreateVolum } } - // Set the Snapshot VolumeContentSource in the CreateVolumeResponse - if contentSourceSnapshotID != "" { - resp.Volume.ContentSource = &csi.VolumeContentSource{ - Type: &csi.VolumeContentSource_Snapshot{ - Snapshot: &csi.VolumeContentSource_SnapshotSource{ - SnapshotId: contentSourceSnapshotID, - }, - }, - } - } - return resp, "", nil } @@ -878,9 +861,6 @@ func (c *controller) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequ log.Debugf("createVolumeInternal: returns fault %q", faultType) if err != nil { - if csifault.IsNonStorageFault(faultType) { - faultType = csifault.AddCsiNonStoragePrefix(ctx, faultType) - } log.Errorf("Operation failed, reporting failure status to Prometheus."+ " Operation Type: %q, Volume Type: %q, Fault Type: %q", prometheus.PrometheusCreateVolumeOpType, volumeType, faultType) @@ -963,9 +943,6 @@ func (c *controller) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequ log.Debugf("deleteVolumeInternal: returns fault %q for volume %q", faultType, req.VolumeId) if err != nil { - if csifault.IsNonStorageFault(faultType) { - faultType = csifault.AddCsiNonStoragePrefix(ctx, faultType) - } log.Errorf("Operation failed, reporting failure status to Prometheus."+ " Operation Type: %q, Volume Type: %q, Fault Type: %q", prometheus.PrometheusDeleteVolumeOpType, volumeType, faultType) @@ -1121,9 +1098,6 @@ func (c *controller) ControllerPublishVolume(ctx context.Context, req *csi.Contr log.Debugf("controllerPublishVolumeInternal: returns fault %q for volume %q", faultType, req.VolumeId) if err != nil { - if csifault.IsNonStorageFault(faultType) { - faultType = csifault.AddCsiNonStoragePrefix(ctx, faultType) - } log.Errorf("Operation failed, reporting failure status to Prometheus."+ " Operation Type: %q, Volume Type: %q, Fault Type: %q", prometheus.PrometheusAttachVolumeOpType, volumeType, faultType) @@ -1273,9 +1247,6 @@ func (c *controller) ControllerUnpublishVolume(ctx context.Context, req *csi.Con log.Debugf("controllerUnpublishVolumeInternal: returns fault %q for volume %q", faultType, req.VolumeId) if err != nil { - if csifault.IsNonStorageFault(faultType) { - faultType = csifault.AddCsiNonStoragePrefix(ctx, faultType) - } log.Errorf("Operation failed, reporting failure status to Prometheus."+ " Operation Type: %q, Volume Type: %q, Fault Type: %q", prometheus.PrometheusDetachVolumeOpType, volumeType, faultType) @@ -1366,13 +1337,6 @@ func (c *controller) ListVolumes(ctx context.Context, req *csi.ListVolumesReques for _, cnsVolume := range cnsQueryVolumes.Volumes { cnsVolumeIDs = append(cnsVolumeIDs, cnsVolume.VolumeId.Id) } - - // Get volume ID to VMMap and vmMoidToHostMoid map - vmMoidToHostMoid, volumeIDToVMMap, err = c.GetVolumeToHostMapping(ctx) - if err != nil { - log.Errorf("failed to get VM MoID to Host MoID map, err:%v", err) - return nil, csifault.CSIInternalFault, status.Error(codes.Internal, "failed to get VM MoID to Host MoID map") - } } // If the difference between the volumes reported by Kubernetes and CNS @@ -1405,7 +1369,7 @@ func (c *controller) ListVolumes(ctx context.Context, req *csi.ListVolumesReques volumeIDs = append(volumeIDs, cnsVolumeIDs[i]) } - response, err := getVolumeIDToVMMap(ctx, volumeIDs, vmMoidToHostMoid, volumeIDToVMMap) + response, err := getVolumeIDToVMMap(ctx, c, volumeIDs) if err != nil { log.Errorf("Error while generating ListVolume response, err:%v", err) return nil, csifault.CSIInternalFault, status.Error(codes.Internal, "Error while generating ListVolume response") @@ -1425,9 +1389,6 @@ func (c *controller) ListVolumes(ctx context.Context, req *csi.ListVolumesReques } resp, faultType, err := controllerListVolumeInternal() if err != nil { - if csifault.IsNonStorageFault(faultType) { - faultType = csifault.AddCsiNonStoragePrefix(ctx, faultType) - } log.Errorf("Operation failed, reporting failure status to Prometheus."+ " Operation Type: %q, Volume Type: %q, Fault Type: %q", prometheus.PrometheusListVolumeOpType, volumeType, faultType) @@ -1437,7 +1398,7 @@ func (c *controller) ListVolumes(ctx context.Context, req *csi.ListVolumesReques prometheus.CsiControlOpsHistVec.WithLabelValues(volumeType, prometheus.PrometheusListVolumeOpType, prometheus.PrometheusPassStatus, faultType).Observe(time.Since(start).Seconds()) } - return resp, err + return resp, nil } func (c *controller) GetCapacity(ctx context.Context, req *csi.GetCapacityRequest) ( @@ -1460,6 +1421,11 @@ func (c *controller) ControllerGetCapabilities(ctx context.Context, req *csi.Con csi.ControllerServiceCapability_RPC_LIST_VOLUMES_PUBLISHED_NODES) } + if commonco.ContainerOrchestratorUtility.IsFSSEnabled(ctx, common.BlockVolumeSnapshot) { + controllerCaps = append(controllerCaps, csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT, + csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS) + } + for _, cap := range controllerCaps { c := &csi.ControllerServiceCapability{ Type: &csi.ControllerServiceCapability_Rpc{ @@ -1498,7 +1464,7 @@ func (c *controller) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshot "cannot snapshot migrated vSphere volume. :%q", volumeID) } volumeType = prometheus.PrometheusBlockVolumeType - // Query capacity in MB for block volume snapshot + // Query capacity in MB and datastore url for block volume snapshot volumeIds := []cnstypes.CnsVolumeId{{Id: volumeID}} cnsVolumeDetailsMap, err := utils.QueryVolumeDetailsUtil(ctx, c.manager.VolumeManager, volumeIds) if err != nil { @@ -1509,15 +1475,52 @@ func (c *controller) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshot "cns query volume did not return the volume: %s", volumeID) } snapshotSizeInMB := cnsVolumeDetailsMap[volumeID].SizeInMB - + datastoreUrl := cnsVolumeDetailsMap[volumeID].DatastoreUrl if cnsVolumeDetailsMap[volumeID].VolumeType != common.BlockVolumeType { return nil, logger.LogNewErrorCodef(log, codes.FailedPrecondition, "queried volume doesn't have the expected volume type. Expected VolumeType: %v. "+ "Queried VolumeType: %v", volumeType, cnsVolumeDetailsMap[volumeID].VolumeType) } + // Check if snapshots number of this volume reaches the granular limit on VSAN/VVOL + maxSnapshotsPerBlockVolume := c.manager.CnsConfig.Snapshot.GlobalMaxSnapshotsPerBlockVolume + log.Infof("The limit of the maximum number of snapshots per block volume is "+ + "set to the global maximum (%v) by default.", maxSnapshotsPerBlockVolume) + if c.manager.CnsConfig.Snapshot.GranularMaxSnapshotsPerBlockVolumeInVSAN > 0 || + c.manager.CnsConfig.Snapshot.GranularMaxSnapshotsPerBlockVolumeInVVOL > 0 { + + var isGranularMaxEnabled bool + if strings.Contains(datastoreUrl, strings.ToLower(string(types.HostFileSystemVolumeFileSystemTypeVsan))) { + if c.manager.CnsConfig.Snapshot.GranularMaxSnapshotsPerBlockVolumeInVSAN > 0 { + maxSnapshotsPerBlockVolume = c.manager.CnsConfig.Snapshot.GranularMaxSnapshotsPerBlockVolumeInVSAN + isGranularMaxEnabled = true + + } + } else if strings.Contains(datastoreUrl, strings.ToLower(string(types.HostFileSystemVolumeFileSystemTypeVVOL))) { + if c.manager.CnsConfig.Snapshot.GranularMaxSnapshotsPerBlockVolumeInVVOL > 0 { + maxSnapshotsPerBlockVolume = c.manager.CnsConfig.Snapshot.GranularMaxSnapshotsPerBlockVolumeInVVOL + isGranularMaxEnabled = true + } + } + + if isGranularMaxEnabled { + log.Infof("The limit of the maximum number of snapshots per block volume on datastore %q is "+ + "overridden by the granular maximum (%v).", datastoreUrl, maxSnapshotsPerBlockVolume) + } + } - // TODO: We may need to add logic to check the limit of max number of snapshots by using - // GlobalMaxSnapshotsPerBlockVolume etc. variables in the future. + // Check if snapshots number of this volume reaches the limit + snapshotList, _, err := common.QueryVolumeSnapshotsByVolumeID(ctx, c.manager.VolumeManager, volumeID, + common.QuerySnapshotLimit) + if err != nil { + return nil, logger.LogNewErrorCodef(log, codes.Internal, + "failed to query snapshots of volume %s for the limit check. Error: %v", volumeID, err) + } + + if len(snapshotList) >= maxSnapshotsPerBlockVolume { + return nil, logger.LogNewErrorCodef(log, codes.FailedPrecondition, + "the number of snapshots on the source volume %s reaches the configured maximum (%v)", + volumeID, c.manager.CnsConfig.Snapshot.GlobalMaxSnapshotsPerBlockVolume) + } // the returned snapshotID below is a combination of CNS VolumeID and CNS SnapshotID concatenated by the "+" // sign. That is, a string of "+". Because, all other CNS snapshot APIs still require both @@ -1749,9 +1752,6 @@ func (c *controller) ControllerExpandVolume(ctx context.Context, req *csi.Contro log.Debugf("controllerExpandVolumeInternal: returns fault %q for volume %q", faultType, req.VolumeId) if err != nil { - if csifault.IsNonStorageFault(faultType) { - faultType = csifault.AddCsiNonStoragePrefix(ctx, faultType) - } log.Errorf("Operation failed, reporting failure status to Prometheus."+ " Operation Type: %q, Volume Type: %q, Fault Type: %q", prometheus.PrometheusExpandVolumeOpType, volumeType, faultType) diff --git a/pkg/csi/service/wcp/controller_helper.go b/pkg/csi/service/wcp/controller_helper.go index dcd7dabc4a..09c0da6674 100644 --- a/pkg/csi/service/wcp/controller_helper.go +++ b/pkg/csi/service/wcp/controller_helper.go @@ -598,8 +598,7 @@ func (c *controller) GetVolumeToHostMapping(ctx context.Context) (map[string]str // getVolumeIDToVMMap returns the csi list volume response by computing the volumeID to nodeNames map for // fake attached volumes and non-fake attached volumes. -func getVolumeIDToVMMap(ctx context.Context, volumeIDs []string, vmMoidToHostMoid, - volumeIDToVMMap map[string]string) (*csi.ListVolumesResponse, error) { +func getVolumeIDToVMMap(ctx context.Context, c *controller, volumeIDs []string) (*csi.ListVolumesResponse, error) { log := logger.GetLogger(ctx) response := &csi.ListVolumesResponse{} @@ -610,7 +609,6 @@ func getVolumeIDToVMMap(ctx context.Context, volumeIDs []string, vmMoidToHostMoi fakeAttachedVolumes = append(fakeAttachedVolumes, volumeID) } } - // Process fake attached volumes log.Debugf("Fake attached volumes %v", fakeAttachedVolumes) volumeIDToNodesMap := commonco.ContainerOrchestratorUtility.GetNodesForVolumes(ctx, fakeAttachedVolumes) @@ -628,6 +626,13 @@ func getVolumeIDToVMMap(ctx context.Context, volumeIDs []string, vmMoidToHostMoi response.Entries = append(response.Entries, entry) } + // Process remaining volumes + vmMoidToHostMoid, volumeIDToVMMap, err := c.GetVolumeToHostMapping(ctx) + if err != nil { + log.Errorf("failed to get VM MoID to Host MoID map, err:%v", err) + return nil, fmt.Errorf("failed to get VM MoID to Host MoID map, err: %v", err) + } + hostNames := commonco.ContainerOrchestratorUtility.GetNodeIDtoNameMap(ctx) if len(hostNames) == 0 { log.Errorf("no hostnames found in the NodeIDtoName map") @@ -667,5 +672,6 @@ func getVolumeIDToVMMap(ctx context.Context, volumeIDs []string, vmMoidToHostMoi } response.Entries = append(response.Entries, entry) } + return response, nil } diff --git a/pkg/csi/service/wcp/controller_test.go b/pkg/csi/service/wcp/controller_test.go index 337f5a1b58..f78f7d21c1 100644 --- a/pkg/csi/service/wcp/controller_test.go +++ b/pkg/csi/service/wcp/controller_test.go @@ -54,11 +54,6 @@ import ( const ( testVolumeName = "test-pvc" testClusterName = "test-cluster" - // TODO: We may need to decide this value by checking GlobalMaxSnapshotsPerBlockVolume - // variable's value when it is set for WCP. - // Currently keeping this as 3, since it is the recommended value of snapshots - // per block volume in vSphere. - maxNumOfSnapshots = 3 ) var ( @@ -129,6 +124,11 @@ func configFromSimWithTLS(tlsConfig *tls.Config, insecureAllowed bool) (*config. Datacenters: cfg.Global.Datacenters, } + // set up the default global maximum of number of snapshots if unset + if cfg.Snapshot.GlobalMaxSnapshotsPerBlockVolume == 0 { + cfg.Snapshot.GlobalMaxSnapshotsPerBlockVolume = config.DefaultGlobalMaxSnapshotsPerBlockVolume + } + return cfg, func() { s.Close() model.Remove() @@ -630,6 +630,7 @@ func TestWCPCreateDeleteSnapshot(t *testing.T) { func TestListSnapshots(t *testing.T) { ct := getControllerTest(t) + numOfSnapshots := ct.config.Snapshot.GlobalMaxSnapshotsPerBlockVolume // Create. params := make(map[string]string) if v := os.Getenv("VSPHERE_DATASTORE_URL"); v != "" { @@ -678,7 +679,7 @@ func TestListSnapshots(t *testing.T) { snapshots := make(map[string]string) var deleteSnapshotList []string - for i := 0; i < maxNumOfSnapshots; i++ { + for i := 0; i < numOfSnapshots; i++ { // Snapshot a volume reqCreateSnapshot := &csi.CreateSnapshotRequest{ SourceVolumeId: volID, @@ -750,6 +751,7 @@ func TestListSnapshots(t *testing.T) { func TestListSnapshotsOnSpecificVolume(t *testing.T) { ct := getControllerTest(t) + numOfSnapshots := ct.config.Snapshot.GlobalMaxSnapshotsPerBlockVolume // Create. params := make(map[string]string) if v := os.Getenv("VSPHERE_DATASTORE_URL"); v != "" { @@ -798,7 +800,7 @@ func TestListSnapshotsOnSpecificVolume(t *testing.T) { snapshots := make(map[string]string) var deleteSnapshotList []string - for i := 0; i < maxNumOfSnapshots; i++ { + for i := 0; i < numOfSnapshots; i++ { // Snapshot a volume reqCreateSnapshot := &csi.CreateSnapshotRequest{ SourceVolumeId: volID, @@ -871,6 +873,7 @@ func TestListSnapshotsOnSpecificVolume(t *testing.T) { func TestListSnapshotsWithToken(t *testing.T) { ct := getControllerTest(t) + numOfSnapshots := ct.config.Snapshot.GlobalMaxSnapshotsPerBlockVolume // Create. params := make(map[string]string) if v := os.Getenv("VSPHERE_DATASTORE_URL"); v != "" { @@ -919,7 +922,7 @@ func TestListSnapshotsWithToken(t *testing.T) { snapshots := make(map[string]string) var deleteSnapshotList []string - for i := 0; i < maxNumOfSnapshots; i++ { + for i := 0; i < numOfSnapshots; i++ { // Snapshot a volume reqCreateSnapshot := &csi.CreateSnapshotRequest{ SourceVolumeId: volID, diff --git a/pkg/csi/service/wcpguest/controller.go b/pkg/csi/service/wcpguest/controller.go index b8c26e7362..ede50bd388 100644 --- a/pkg/csi/service/wcpguest/controller.go +++ b/pkg/csi/service/wcpguest/controller.go @@ -27,7 +27,7 @@ import ( "github.com/container-storage-interface/spec/lib/go/csi" "github.com/davecgh/go-spew/spew" "github.com/fsnotify/fsnotify" - snapshotterClientSet "github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned" + snapshotterClientSet "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned" "github.com/prometheus/client_golang/prometheus/promhttp" vmoperatortypes "github.com/vmware-tanzu/vm-operator-api/api/v1alpha1" "golang.org/x/net/context" @@ -45,7 +45,6 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" "sigs.k8s.io/controller-runtime/pkg/client" - cnsoperatorv1alpha1 "sigs.k8s.io/vsphere-csi-driver/v3/pkg/apis/cnsoperator" cnsfileaccessconfigv1alpha1 "sigs.k8s.io/vsphere-csi-driver/v3/pkg/apis/cnsoperator/cnsfileaccessconfig/v1alpha1" commonconfig "sigs.k8s.io/vsphere-csi-driver/v3/pkg/common/config" @@ -64,8 +63,6 @@ var ( csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME, csi.ControllerServiceCapability_RPC_EXPAND_VOLUME, - csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT, - csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS, } ) @@ -247,7 +244,7 @@ func (c *controller) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequ *csi.CreateVolumeResponse, string, error) { log.Infof("CreateVolume: called with args %+v", *req) - // TODO: If the err is returned by invoking CNS API, then faultType should be + //TODO: If the err is returned by invoking CNS API, then faultType should be // populated by the underlying layer. // If the request failed due to validate the request, "csi.fault.InvalidArgument" will be return. // If thr reqeust failed due to object not found, "csi.fault.NotFound" will be return. @@ -277,15 +274,6 @@ func (c *controller) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequ } volSizeMB := int64(common.RoundUpSize(volSizeBytes, common.MbInBytes)) volumeSource := req.GetVolumeContentSource() - if commonco.ContainerOrchestratorUtility.IsFSSEnabled(ctx, common.BlockVolumeSnapshot) && - volumeSource != nil { - sourceSnapshot := volumeSource.GetSnapshot() - if sourceSnapshot == nil { - return nil, csifault.CSIInvalidArgumentFault, - logger.LogNewErrorCode(log, codes.InvalidArgument, "unsupported VolumeContentSource type") - } - volumeSnapshotName = sourceSnapshot.GetSnapshotId() - } // Get supervisorStorageClass and accessMode var supervisorStorageClass string @@ -315,6 +303,10 @@ func (c *controller) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequ } annotations[common.AnnGuestClusterRequestedTopology] = topologyAnnotation } + if commonco.ContainerOrchestratorUtility.IsFSSEnabled(ctx, common.BlockVolumeSnapshot) && + volumeSource != nil { + volumeSnapshotName = volumeSource.GetSnapshot().GetSnapshotId() + } claim := getPersistentVolumeClaimSpecWithStorageClass(supervisorPVCName, c.supervisorNamespace, diskSize, supervisorStorageClass, getAccessMode(accessMode), annotations, volumeSnapshotName) log.Debugf("PVC claim spec is %+v", spew.Sdump(claim)) @@ -340,29 +332,12 @@ func (c *controller) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequ c.supervisorNamespace, err) log.Error(msg) eventList, err := c.supervisorClient.CoreV1().Events(c.supervisorNamespace).List(ctx, - metav1.ListOptions{ - FieldSelector: "involvedObject.name=" + pvc.Name, - ResourceVersion: pvc.ResourceVersion, - ResourceVersionMatch: metav1.ResourceVersionMatchNotOlderThan, - }) + metav1.ListOptions{FieldSelector: "involvedObject.name=" + pvc.Name}) if err != nil { log.Errorf("Unable to fetch events for pvc %q/%q from supervisor cluster with err: %+v", c.supervisorNamespace, pvc.Name, err) return nil, csifault.CSIInternalFault, status.Errorf(codes.Internal, msg) } - - var failureMessage string - for _, svcPvcEvent := range eventList.Items { - if svcPvcEvent.Type == corev1.EventTypeWarning { - failureMessage = svcPvcEvent.Message - break - } - } - - if failureMessage != "" { - msg = fmt.Sprintf("%s. reason: %s", msg, failureMessage) - } - log.Errorf("Last observed events on the pvc %q/%q in supervisor cluster: %+v", c.supervisorNamespace, pvc.Name, spew.Sdump(eventList.Items)) return nil, csifault.CSIInternalFault, status.Errorf(codes.Internal, msg) @@ -430,9 +405,6 @@ func (c *controller) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequ resp, faultType, err := createVolumeInternal() log.Debugf("createVolumeInternal: returns fault %q", faultType) if err != nil { - if csifault.IsNonStorageFault(faultType) { - faultType = csifault.AddCsiNonStoragePrefix(ctx, faultType) - } log.Errorf("Operation failed, reporting failure status to Prometheus."+ " Operation Type: %q, Volume Type: %q, Fault Type: %q", prometheus.PrometheusCreateVolumeOpType, volumeType, faultType) @@ -459,7 +431,7 @@ func (c *controller) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequ deleteVolumeInternal := func() ( *csi.DeleteVolumeResponse, string, error) { log.Infof("DeleteVolume: called with args: %+v", *req) - // TODO: If the err is returned by invoking CNS API, then faultType should be + //TODO: If the err is returned by invoking CNS API, then faultType should be // populated by the underlying layer. // If the request failed due to validate the request, "csi.fault.InvalidArgument" will be return. // If thr reqeust failed due to object not found, "csi.fault.NotFound" will be return. @@ -509,9 +481,6 @@ func (c *controller) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequ resp, faultType, err := deleteVolumeInternal() log.Debugf("deleteVolumeInternal: returns fault %q for volume %q", faultType, req.VolumeId) if err != nil { - if csifault.IsNonStorageFault(faultType) { - faultType = csifault.AddCsiNonStoragePrefix(ctx, faultType) - } log.Errorf("Operation failed, reporting failure status to Prometheus."+ " Operation Type: %q, Volume Type: %q, Fault Type: %q", prometheus.PrometheusDeleteVolumeOpType, volumeType, faultType) @@ -537,7 +506,7 @@ func (c *controller) ControllerPublishVolume(ctx context.Context, req *csi.Contr controllerPublishVolumeInternal := func() ( *csi.ControllerPublishVolumeResponse, string, error) { log.Infof("ControllerPublishVolume: called with args %+v", *req) - // TODO: If the err is returned by invoking CNS API, then faultType should be + //TODO: If the err is returned by invoking CNS API, then faultType should be // populated by the underlying layer. // If the request failed due to validate the request, "csi.fault.InvalidArgument" will be return. // If thr reqeust failed due to object not found, "csi.fault.NotFound" will be return. @@ -573,9 +542,6 @@ func (c *controller) ControllerPublishVolume(ctx context.Context, req *csi.Contr resp, faultType, err := controllerPublishVolumeInternal() if err != nil { log.Debugf("controllerPublishVolumeInternal: returns fault %q for volume %q", faultType, req.VolumeId) - if csifault.IsNonStorageFault(faultType) { - faultType = csifault.AddCsiNonStoragePrefix(ctx, faultType) - } log.Errorf("Operation failed, reporting failure status to Prometheus."+ " Operation Type: %q, Volume Type: %q, Fault Type: %q", prometheus.PrometheusAttachVolumeOpType, volumeType, faultType) @@ -712,7 +678,7 @@ func controllerPublishForBlockVolume(ctx context.Context, req *csi.ControllerPub log.Debugf("disk UUID %v is set for the volume: %q ", diskUUID, req.VolumeId) } - // return PublishContext with diskUUID of the volume attached to node. + //return PublishContext with diskUUID of the volume attached to node. publishInfo := make(map[string]string) publishInfo[common.AttributeDiskType] = common.DiskTypeBlockVolume publishInfo[common.AttributeFirstClassDiskUUID] = common.FormatDiskUUID(diskUUID) @@ -876,7 +842,7 @@ func (c *controller) ControllerUnpublishVolume(ctx context.Context, req *csi.Con controllerUnpublishVolumeInternal := func() ( *csi.ControllerUnpublishVolumeResponse, string, error) { log.Infof("ControllerUnpublishVolume: called with args %+v", *req) - // TODO: If the err is returned by invoking CNS API, then faultType should be + //TODO: If the err is returned by invoking CNS API, then faultType should be // populated by the underlying layer. // If the request failed due to validate the request, "csi.fault.InvalidArgument" will be return. // If thr reqeust failed due to object not found, "csi.fault.NotFound" will be return. @@ -919,9 +885,6 @@ func (c *controller) ControllerUnpublishVolume(ctx context.Context, req *csi.Con resp, faultType, err := controllerUnpublishVolumeInternal() log.Debugf("controllerUnpublishVolumeInternal: returns fault %q for volume %q", faultType, req.VolumeId) if err != nil { - if csifault.IsNonStorageFault(faultType) { - faultType = csifault.AddCsiNonStoragePrefix(ctx, faultType) - } log.Errorf("Operation failed, reporting failure status to Prometheus."+ " Operation Type: %q, Volume Type: %q, Fault Type: %q", prometheus.PrometheusDetachVolumeOpType, volumeType, faultType) @@ -1168,7 +1131,7 @@ func (c *controller) ControllerExpandVolume(ctx context.Context, req *csi.Contro return nil, csifault.CSIUnimplementedFault, status.Error(codes.Unimplemented, msg) } log.Infof("ControllerExpandVolume: called with args %+v", *req) - // TODO: If the err is returned by invoking CNS API, then faultType should be + //TODO: If the err is returned by invoking CNS API, then faultType should be // populated by the underlying layer. // If the request failed due to validate the request, "csi.fault.InvalidArgument" will be return. // If thr reqeust failed due to object not found, "csi.fault.NotFound" will be return. @@ -1290,9 +1253,6 @@ func (c *controller) ControllerExpandVolume(ctx context.Context, req *csi.Contro resp, faultType, err := controllerExpandVolumeInternal() log.Debugf("controllerExpandVolumeInternal: returns fault %q for volume %q", faultType, req.VolumeId) if err != nil { - if csifault.IsNonStorageFault(faultType) { - faultType = csifault.AddCsiNonStoragePrefix(ctx, faultType) - } log.Errorf("Operation failed, reporting failure status to Prometheus."+ " Operation Type: %q, Volume Type: %q, Fault Type: %q", prometheus.PrometheusExpandVolumeOpType, volumeType, faultType) @@ -1346,6 +1306,10 @@ func (c *controller) ControllerGetCapabilities(ctx context.Context, req *csi.Con ctx = logger.NewContextWithLogger(ctx) log := logger.GetLogger(ctx) log.Infof("ControllerGetCapabilities: called with args %+v", *req) + if commonco.ContainerOrchestratorUtility.IsFSSEnabled(ctx, common.BlockVolumeSnapshot) { + controllerCaps = append(controllerCaps, csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT, + csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS) + } var caps []*csi.ControllerServiceCapability for _, cap := range controllerCaps { c := &csi.ControllerServiceCapability{ @@ -1406,12 +1370,8 @@ func (c *controller) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshot if err != nil { if errors.IsNotFound(err) { // New createSnapshot request on the guest - // Add "csi.vsphere.guest-initiated-csi-snapshot" annotation on VolumeSnapshot CR in - // the supervisor cluster to indicate that snapshot creation is initiated from Guest cluster - annotation := make(map[string]string) - annotation[common.SupervisorVolumeSnapshotAnnotationKey] = "true" supVolumeSnapshot := constructVolumeSnapshotWithVolumeSnapshotClass(supervisorVolumeSnapshotName, - c.supervisorNamespace, supervisorVolumeSnapshotClass, supervisorPVCName, annotation) + c.supervisorNamespace, supervisorVolumeSnapshotClass, supervisorPVCName) log.Infof("Supervisosr VolumeSnapshot Spec: %+v", supVolumeSnapshot) _, err = c.supervisorSnapshotterClient.SnapshotV1().VolumeSnapshots( c.supervisorNamespace).Create(ctx, supVolumeSnapshot, metav1.CreateOptions{}) @@ -1434,7 +1394,7 @@ func (c *controller) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshot // Wait for VolumeSnapshot to be ready to use isReady, vs, err := common.IsVolumeSnapshotReady(ctx, c.supervisorSnapshotterClient, supervisorVolumeSnapshotName, c.supervisorNamespace, - time.Duration(getSnapshotTimeoutInMin(ctx))*time.Minute) + time.Duration(getProvisionTimeoutInMin(ctx))*time.Minute) if !isReady { msg := fmt.Sprintf("volumesnapshot: %s on namespace: %s in supervisor cluster was not Ready. "+ "Error: %+v", supervisorVolumeSnapshotName, c.supervisorNamespace, err) diff --git a/pkg/csi/service/wcpguest/controller_helper.go b/pkg/csi/service/wcpguest/controller_helper.go index a5579cc66b..76532f4bb7 100644 --- a/pkg/csi/service/wcpguest/controller_helper.go +++ b/pkg/csi/service/wcpguest/controller_helper.go @@ -28,7 +28,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "github.com/container-storage-interface/spec/lib/go/csi" - snap "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" + snap "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" "google.golang.org/grpc/codes" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -52,10 +52,6 @@ const ( // Default timeout for resize, used unless overridden by user in // csi-controller YAML. defaultResizeTimeoutInMin = 4 - - // Default timeout for create snapshot, used unless overridden by user in - // csi-controller YAML. - defaultSnapshotTimeoutInMin = 4 ) // validateGuestClusterCreateVolumeRequest is the helper function to validate @@ -232,12 +228,11 @@ func getPersistentVolumeClaimSpecWithStorageClass(pvcName string, namespace stri } func constructVolumeSnapshotWithVolumeSnapshotClass(volumeSnapshotName string, namespace string, - volumeSnapshotClassName string, pvcName string, annotation map[string]string) *snap.VolumeSnapshot { + volumeSnapshotClassName string, pvcName string) *snap.VolumeSnapshot { volumeSnapshot := &snap.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ - Name: volumeSnapshotName, - Namespace: namespace, - Annotations: annotation, + Name: volumeSnapshotName, + Namespace: namespace, }, Spec: snap.VolumeSnapshotSpec{ Source: snap.VolumeSnapshotSource{ @@ -348,30 +343,6 @@ func getProvisionTimeoutInMin(ctx context.Context) int { return provisionTimeoutInMin } -// getSnapshotTimeoutInMin return the timeout for volume snapshot. -// If environment variable SNAPSHOT_TIMEOUT_MINUTES is set and valid, -// return the interval value read from environment variable -// otherwise, use the default timeout 4 mins -func getSnapshotTimeoutInMin(ctx context.Context) int { - log := logger.GetLogger(ctx) - snapshotTimeoutInMin := defaultSnapshotTimeoutInMin - if v := os.Getenv("SNAPSHOT_TIMEOUT_MINUTES"); v != "" { - if value, err := strconv.Atoi(v); err == nil { - if value <= 0 { - log.Warnf(" snapshotTimeout set in env variable SNAPSHOT_TIMEOUT_MINUTES %s "+ - "is equal or less than 0, will use the default timeout", v) - } else { - snapshotTimeoutInMin = value - log.Infof("snapshotTimeout is set to %d minutes", snapshotTimeoutInMin) - } - } else { - log.Warnf("snapshotTimeout set in env variable SNAPSHOT_TIMEOUT_MINUTES %s is invalid, "+ - "will use the default timeout", v) - } - } - return snapshotTimeoutInMin -} - // getResizeTimeoutInMin returns the timeout for volume resize. // If environment variable RESIZE_TIMEOUT_MINUTES is set and valid, // return the interval value read from environment variable diff --git a/pkg/kubernetes/kubernetes.go b/pkg/kubernetes/kubernetes.go index da677dcfbd..75f250769c 100644 --- a/pkg/kubernetes/kubernetes.go +++ b/pkg/kubernetes/kubernetes.go @@ -46,10 +46,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" apiutils "sigs.k8s.io/controller-runtime/pkg/client/apiutil" - snapshotterClientSet "github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned" + snapshotterClientSet "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned" storagev1 "k8s.io/api/storage/v1" cnsoperatorv1alpha1 "sigs.k8s.io/vsphere-csi-driver/v3/pkg/apis/cnsoperator" migrationv1alpha1 "sigs.k8s.io/vsphere-csi-driver/v3/pkg/apis/migration/v1alpha1" + cnsvsphere "sigs.k8s.io/vsphere-csi-driver/v3/pkg/common/cns-lib/vsphere" cnsconfig "sigs.k8s.io/vsphere-csi-driver/v3/pkg/common/config" "sigs.k8s.io/vsphere-csi-driver/v3/pkg/csi/service/logger" "sigs.k8s.io/vsphere-csi-driver/v3/pkg/csi/types" @@ -327,9 +328,21 @@ func CreateKubernetesClientFromConfig(kubeConfigPath string) (clientset.Interfac // If not set, returns node UUID from K8s CSINode API // object. func GetNodeUUID(ctx context.Context, - k8sclient clientset.Interface, nodeName string) (string, error) { + k8sclient clientset.Interface, nodeName string, + useK8sCSINodeObj bool) (string, error) { log := logger.GetLogger(ctx) - log.Infof("GetNodeUUID called for the node: %q", nodeName) + log.Infof("GetNodeUUID called for the node: %q with useK8sCSINodeObj: %t", + nodeName, useK8sCSINodeObj) + if !useK8sCSINodeObj { + node, err := k8sclient.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) + if err != nil { + log.Errorf("failed to get kubernetes node with the name: %q. Err: %v", nodeName, err) + return "", err + } + k8sNodeUUID := cnsvsphere.GetUUIDFromProviderID(node.Spec.ProviderID) + log.Infof("Retrieved node UUID: %q for the node: %q", k8sNodeUUID, nodeName) + return k8sNodeUUID, nil + } node, err := k8sclient.StorageV1().CSINodes().Get(ctx, nodeName, metav1.GetOptions{}) if err != nil { log.Errorf("failed to get K8s CSINode with the name: %q. "+ diff --git a/pkg/syncer/admissionhandler/admissionhandler.go b/pkg/syncer/admissionhandler/admissionhandler.go index 08e52ab54b..a1ff1d92a9 100644 --- a/pkg/syncer/admissionhandler/admissionhandler.go +++ b/pkg/syncer/admissionhandler/admissionhandler.go @@ -140,11 +140,7 @@ func StartWebhookServer(ctx context.Context) error { if clusterFlavor == cnstypes.CnsClusterFlavorWorkload { featureGateTKGSHaEnabled = containerOrchestratorUtility.IsFSSEnabled(ctx, common.TKGsHA) featureGateVolumeHealthEnabled = containerOrchestratorUtility.IsFSSEnabled(ctx, common.VolumeHealth) - featureGateBlockVolumeSnapshotEnabled = containerOrchestratorUtility.IsFSSEnabled(ctx, common.BlockVolumeSnapshot) startCNSCSIWebhookManager(ctx) - } else if clusterFlavor == cnstypes.CnsClusterFlavorGuest { - featureGateBlockVolumeSnapshotEnabled = containerOrchestratorUtility.IsFSSEnabled(ctx, common.BlockVolumeSnapshot) - startPVCSIWebhookManager(ctx) } else if clusterFlavor == cnstypes.CnsClusterFlavorVanilla { if cfg == nil { cfg, err = getWebHookConfig(ctx) @@ -168,17 +164,8 @@ func StartWebhookServer(ctx context.Context) error { cfg.WebHookConfig.Port = defaultWebhookServerPort } server = &http.Server{ - Addr: fmt.Sprintf(":%v", cfg.WebHookConfig.Port), - TLSConfig: &tls.Config{ - Certificates: []tls.Certificate{certs}, - CipherSuites: []uint16{ - tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, - tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, - tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, - tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, - }, - MinVersion: tls.VersionTLS12, - }, + Addr: fmt.Sprintf(":%v", cfg.WebHookConfig.Port), + TLSConfig: &tls.Config{Certificates: []tls.Certificate{certs}}, } // Define http server and server handler. mux := http.NewServeMux() @@ -268,7 +255,7 @@ func validationHandler(w http.ResponseWriter, r *http.Request) { case "StorageClass": admissionResponse = validateStorageClass(ctx, &ar) case "PersistentVolumeClaim": - admissionResponse = validatePVC(ctx, ar.Request) + admissionResponse = validatePVC(ctx, &ar) default: log.Infof("Skipping validation for resource type: %q", ar.Request.Kind.Kind) admissionResponse = &admissionv1.AdmissionResponse{ diff --git a/pkg/syncer/admissionhandler/cnscsi_admissionhandler.go b/pkg/syncer/admissionhandler/cnscsi_admissionhandler.go index 25775126e3..08778b5bfd 100644 --- a/pkg/syncer/admissionhandler/cnscsi_admissionhandler.go +++ b/pkg/syncer/admissionhandler/cnscsi_admissionhandler.go @@ -2,7 +2,6 @@ package admissionhandler import ( "context" - "crypto/tls" "fmt" "os" "strconv" @@ -66,13 +65,6 @@ func startCNSCSIWebhookManager(ctx context.Context) { log.Infof("registering validating webhook with the endpoint %v", ValidationWebhookPath) // we should not allow TLS < 1.2 mgr.GetWebhookServer().TLSMinVersion = WebhookTlsMinVersion - // CipherSuites allows us to specify TLS 1.2 cipher suites that have been recommended by the Security team - mgr.GetWebhookServer().TLSOpts = []func(*tls.Config){ - func(t *tls.Config) { - t.CipherSuites = []uint16{tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, - tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384} - }, - } mgr.GetWebhookServer().Register(ValidationWebhookPath, &webhook.Admission{Handler: &CSISupervisorWebhook{ Client: mgr.GetClient(), clientConfig: mgr.GetConfig(), @@ -105,17 +97,6 @@ func (h *CSISupervisorWebhook) Handle(ctx context.Context, req admission.Request } if featureGateVolumeHealthEnabled { resp = validatePVCAnnotationForVolumeHealth(ctx, req) - if !resp.Allowed { - return - } - } - if featureGateBlockVolumeSnapshotEnabled { - admissionResp := validatePVC(ctx, &req.AdmissionRequest) - resp.AdmissionResponse = *admissionResp.DeepCopy() - } - } else if req.Kind.Kind == "VolumeSnapshot" { - if featureGateBlockVolumeSnapshotEnabled { - resp = validateSnapshotOperationSupervisorRequest(ctx, req) } } return diff --git a/pkg/syncer/admissionhandler/pvcsi_admissionhandler.go b/pkg/syncer/admissionhandler/pvcsi_admissionhandler.go deleted file mode 100644 index 8b29da9263..0000000000 --- a/pkg/syncer/admissionhandler/pvcsi_admissionhandler.go +++ /dev/null @@ -1,98 +0,0 @@ -package admissionhandler - -import ( - "context" - "fmt" - "os" - "strconv" - - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" - crConfig "sigs.k8s.io/controller-runtime/pkg/client/config" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/manager/signals" - "sigs.k8s.io/controller-runtime/pkg/webhook" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - - "sigs.k8s.io/vsphere-csi-driver/v3/pkg/csi/service/logger" -) - -const ( - PVCSIValidationWebhookPath = "/validate" - PVCSIDefaultWebhookPort = 9883 - PVCSIDefaultWebhookMetricsBindAddress = "0" - PVCSIWebhookTlsMinVersion = "1.2" -) - -func getPVCSIWebhookPort() int { - portStr, ok := os.LookupEnv("PVCSI_WEBHOOK_SERVICE_CONTAINER_PORT") - if !ok { - return DefaultWebhookPort - } - - result, err := strconv.ParseInt(portStr, 0, 0) - if err != nil { - panic(fmt.Sprintf("malformed configuration: PVCSI_WEBHOOK_SERVICE_CONTAINER_PORT, expected int: %v", err)) - } - - return int(result) -} - -func getPVCSIMetricsBindAddress() string { - metricsAddr, ok := os.LookupEnv("PVCSI_WEBHOOK_SERVICE_METRICS_BIND_ADDR") - if !ok { - return DefaultWebhookMetricsBindAddress - } - - return metricsAddr -} - -// startPVCSIWebhookManager starts the webhook server in guest cluster -func startPVCSIWebhookManager(ctx context.Context) { - log := logger.GetLogger(ctx) - - webhookPort := getPVCSIWebhookPort() - metricsBindAddress := getPVCSIMetricsBindAddress() - log.Infof("setting up webhook manager with webhookPort %v and metricsBindAddress %v", - webhookPort, metricsBindAddress) - mgr, err := manager.New(crConfig.GetConfigOrDie(), manager.Options{ - MetricsBindAddress: metricsBindAddress, - Port: webhookPort}) - if err != nil { - log.Fatal(err, "unable to set up overall controller manager") - } - - log.Infof("registering validating webhook with the endpoint %v", PVCSIValidationWebhookPath) - // we should not allow TLS < 1.2 - mgr.GetWebhookServer().TLSMinVersion = PVCSIWebhookTlsMinVersion - mgr.GetWebhookServer().Register(PVCSIValidationWebhookPath, &webhook.Admission{Handler: &CSIGuestWebhook{ - Client: mgr.GetClient(), - clientConfig: mgr.GetConfig(), - }}) - - if err := mgr.Start(signals.SetupSignalHandler()); err != nil { - log.Fatal(err, "unable to run the webhook manager") - } -} - -var _ admission.Handler = &CSIGuestWebhook{} - -type CSIGuestWebhook struct { - client.Client - clientConfig *rest.Config -} - -func (h *CSIGuestWebhook) Handle(ctx context.Context, req admission.Request) (resp admission.Response) { - log := logger.GetLogger(ctx) - log.Debugf("PV-CSI validation webhook handler called with request: %+v", req) - defer log.Debugf("PV-CSI validation webhook handler completed for the request: %+v", req) - - resp = admission.Allowed("") - if req.Kind.Kind == "PersistentVolumeClaim" { - if featureGateBlockVolumeSnapshotEnabled { - admissionResp := validatePVC(ctx, &req.AdmissionRequest) - resp.AdmissionResponse = *admissionResp.DeepCopy() - } - } - return -} diff --git a/pkg/syncer/admissionhandler/validatepvc.go b/pkg/syncer/admissionhandler/validatepvc.go index 47c068dabb..a8e0e7aa48 100644 --- a/pkg/syncer/admissionhandler/validatepvc.go +++ b/pkg/syncer/admissionhandler/validatepvc.go @@ -6,7 +6,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" - snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" admissionv1 "k8s.io/api/admission/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -21,7 +21,7 @@ const ( ) // validatePVC helps validate AdmissionReview requests for PersistentVolumeClaim. -func validatePVC(ctx context.Context, req *admissionv1.AdmissionRequest) *admissionv1.AdmissionResponse { +func validatePVC(ctx context.Context, ar *admissionv1.AdmissionReview) *admissionv1.AdmissionResponse { if !featureGateBlockVolumeSnapshotEnabled { // If CSI block volume snapshot is disabled and webhook is running, // skip validation for PersistentVolumeClaim. @@ -30,7 +30,7 @@ func validatePVC(ctx context.Context, req *admissionv1.AdmissionRequest) *admiss } } - if req.Operation != admissionv1.Update && req.Operation != admissionv1.Delete { + if ar.Request.Operation != admissionv1.Update && ar.Request.Operation != admissionv1.Delete { // If AdmissionReview request operation is out of expectation, // skip validation for PersistentVolumeClaim. return &admissionv1.AdmissionResponse{ @@ -39,6 +39,7 @@ func validatePVC(ctx context.Context, req *admissionv1.AdmissionRequest) *admiss } log := logger.GetLogger(ctx) + req := ar.Request var result *metav1.Status allowed := true @@ -48,7 +49,7 @@ func validatePVC(ctx context.Context, req *admissionv1.AdmissionRequest) *admiss log.Debugf("JSON req.OldObject.Raw: %v", string(req.OldObject.Raw)) // req.OldObject is null for CREATE and CONNECT operations. if err := json.Unmarshal(req.OldObject.Raw, &oldPVC); err != nil { - log.Errorf("error deserializing old pvc: %v. skipping validation.", err) + log.Warnf("error deserializing old pvc: %v. skipping validation.", err) return &admissionv1.AdmissionResponse{ // skip validation if there is pvc deserialization error Allowed: true, @@ -70,7 +71,7 @@ func validatePVC(ctx context.Context, req *admissionv1.AdmissionRequest) *admiss log.Debugf("JSON req.Object.Raw: %v", string(req.Object.Raw)) // req.Object is null for DELETE operations. if err := json.Unmarshal(req.Object.Raw, &newPVC); err != nil { - log.Errorf("error deserializing old pvc: %v. skipping validation.", err) + log.Warnf("error deserializing old pvc: %v. skipping validation.", err) return &admissionv1.AdmissionResponse{ // skip validation if there is pvc deserialization error Allowed: true, @@ -151,7 +152,7 @@ func getPVReclaimPolicyForPVC(ctx context.Context, pvc corev1.PersistentVolumeCl pv, err := kubeClient.CoreV1().PersistentVolumes().Get(ctx, pvc.Spec.VolumeName, metav1.GetOptions{}) if err != nil { return result, logger.LogNewErrorf(log, "failed to get PV %v with error: %v. "+ - "Stopping getting reclaim policy for PVC, %s/%s", pvc.Spec.VolumeName, err, pvc.Namespace, pvc.Name) + "Stopping getting reclaim policy for PVC, %s/%s", err, pvc.Spec.VolumeName, pvc.Namespace, pvc.Name) } return pv.Spec.PersistentVolumeReclaimPolicy, nil diff --git a/pkg/syncer/admissionhandler/validatepvc_test.go b/pkg/syncer/admissionhandler/validatepvc_test.go index c18f18844c..2da90344bd 100644 --- a/pkg/syncer/admissionhandler/validatepvc_test.go +++ b/pkg/syncer/admissionhandler/validatepvc_test.go @@ -9,9 +9,9 @@ import ( "k8s.io/client-go/kubernetes/fake" "github.com/agiledragon/gomonkey/v2" - snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" - snapshotterClientSet "github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned" - snapshotclientfake "github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/fake" + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" + snapshotterClientSet "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned" + snapshotclientfake "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/fake" "github.com/stretchr/testify/assert" admissionv1 "k8s.io/api/admission/v1" corev1 "k8s.io/api/core/v1" @@ -429,7 +429,7 @@ func TestValidatePVC(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - actualResponse := validatePVC(ctx, test.admissionReview.Request) + actualResponse := validatePVC(ctx, test.admissionReview) assert.Equal(t, actualResponse, test.expectedResponse) }) } diff --git a/pkg/syncer/admissionhandler/validatepvcannotationfortkgsha.go b/pkg/syncer/admissionhandler/validatepvcannotationfortkgsha.go index 8eaa5b3cee..3a31ef3c32 100644 --- a/pkg/syncer/admissionhandler/validatepvcannotationfortkgsha.go +++ b/pkg/syncer/admissionhandler/validatepvcannotationfortkgsha.go @@ -22,13 +22,9 @@ const ( func validatePVCAnnotationForTKGSHA(ctx context.Context, request admission.Request) admission.Response { log := logger.GetLogger(ctx) log.Debugf("validatePVCAnnotationForTKGSHA called with the request %v", request) - if request.Operation == admissionv1.Delete { - // PVC tkgs ha annotation validation is not required for delete PVC calls - return admission.Allowed("") - } + newPVC := corev1.PersistentVolumeClaim{} if err := json.Unmarshal(request.Object.Raw, &newPVC); err != nil { - log.Errorf("error unmarshalling pvc: %v", err) reason := "skipped validation when failed to deserialize PVC from new request object" log.Warn(reason) return admission.Allowed(reason) @@ -43,7 +39,6 @@ func validatePVCAnnotationForTKGSHA(ctx context.Context, request admission.Reque } else if request.Operation == admissionv1.Update { oldPVC := corev1.PersistentVolumeClaim{} if err := json.Unmarshal(request.OldObject.Raw, &oldPVC); err != nil { - log.Errorf("error unmarshalling pvc: %v", err) reason := "skipped validation when failed to deserialize PVC from old request object" log.Warn(reason) return admission.Allowed(reason) diff --git a/pkg/syncer/admissionhandler/validatepvcannotationforvolumehealth.go b/pkg/syncer/admissionhandler/validatepvcannotationforvolumehealth.go index 113a2c898e..b5994166f3 100644 --- a/pkg/syncer/admissionhandler/validatepvcannotationforvolumehealth.go +++ b/pkg/syncer/admissionhandler/validatepvcannotationforvolumehealth.go @@ -26,13 +26,8 @@ func validatePVCAnnotationForVolumeHealth(ctx context.Context, request admission username := request.UserInfo.Username isCSIServiceAccount := validateCSIServiceAccount(request.UserInfo.Username) log.Debugf("validatePVCAnnotationForVolumeHealth called with the request %v by user: %v", request, username) - if request.Operation == admissionv1.Delete { - // PVC volume health annotation validation is not required for delete PVC calls - return admission.Allowed("") - } newPVC := corev1.PersistentVolumeClaim{} if err := json.Unmarshal(request.Object.Raw, &newPVC); err != nil { - log.Errorf("error unmarshalling pvc: %v", err) reason := "skipped validation when failed to deserialize PVC from new request object" log.Warn(reason) return admission.Allowed(reason) @@ -46,7 +41,6 @@ func validatePVCAnnotationForVolumeHealth(ctx context.Context, request admission } else if request.Operation == admissionv1.Update { oldPVC := corev1.PersistentVolumeClaim{} if err := json.Unmarshal(request.OldObject.Raw, &oldPVC); err != nil { - log.Errorf("error unmarshalling pvc: %v", err) reason := "skipped validation when failed to deserialize PVC from old request object" log.Warn(reason) return admission.Allowed(reason) diff --git a/pkg/syncer/admissionhandler/validatesnapshotoperationsupervisorrequest.go b/pkg/syncer/admissionhandler/validatesnapshotoperationsupervisorrequest.go deleted file mode 100644 index 090f7845b2..0000000000 --- a/pkg/syncer/admissionhandler/validatesnapshotoperationsupervisorrequest.go +++ /dev/null @@ -1,41 +0,0 @@ -package admissionhandler - -import ( - "context" - "encoding/json" - - admissionv1 "k8s.io/api/admission/v1" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - - "sigs.k8s.io/vsphere-csi-driver/v3/pkg/csi/service/common" - "sigs.k8s.io/vsphere-csi-driver/v3/pkg/csi/service/logger" - - snap "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" -) - -const ( - SnapshotOperationNotAllowed = "Snapshot creation initiated directly from the Supervisor cluster " + - "is not supported. Please initiate snapshot creation from the Guest cluster." -) - -// Disallow any opertion on volume snapshot initiated by user directly on the supervisor cluster. -// Currently we only allow snapshot operation initiated from the guest cluster. -func validateSnapshotOperationSupervisorRequest(ctx context.Context, request admission.Request) admission.Response { - log := logger.GetLogger(ctx) - log.Debugf("validateSnapshotOperationSupervisorRequest called with the request %v", request) - newVS := snap.VolumeSnapshot{} - if err := json.Unmarshal(request.Object.Raw, &newVS); err != nil { - reason := "Failed to deserialize VolumeSnapshot from new request object" - log.Warn(reason) - return admission.Denied(reason) - } - - if request.Operation == admissionv1.Create { - if _, annotationFound := newVS.Annotations[common.SupervisorVolumeSnapshotAnnotationKey]; !annotationFound { - return admission.Denied(SnapshotOperationNotAllowed) - } - } - - log.Debugf("validateSnapshotOperationSupervisorRequest completed for the request %v", request) - return admission.Allowed("") -} diff --git a/pkg/syncer/admissionhandler/validatestorageclass.go b/pkg/syncer/admissionhandler/validatestorageclass.go index 03d87b3e19..a7bd1862a9 100644 --- a/pkg/syncer/admissionhandler/validatestorageclass.go +++ b/pkg/syncer/admissionhandler/validatestorageclass.go @@ -43,7 +43,8 @@ var ( ) const ( - migrationParamErrorMessage = "Invalid StorageClass Parameters. " + + volumeExpansionErrorMessage = "AllowVolumeExpansion can not be set to true on the in-tree vSphere StorageClass" + migrationParamErrorMessage = "Invalid StorageClass Parameters. " + "Migration specific parameters should not be used in the StorageClass" ) @@ -74,7 +75,15 @@ func validateStorageClass(ctx context.Context, ar *admissionv1.AdmissionReview) } } log.Infof("Validating StorageClass: %q", sc.Name) - if sc.Provisioner == "csi.vsphere.vmware.com" { + // AllowVolumeExpansion check for kubernetes.io/vsphere-volume provisioner. + if sc.Provisioner == "kubernetes.io/vsphere-volume" { + if sc.AllowVolumeExpansion != nil && *sc.AllowVolumeExpansion { + allowed = false + result = &metav1.Status{ + Reason: volumeExpansionErrorMessage, + } + } + } else if sc.Provisioner == "csi.vsphere.vmware.com" { // Migration parameters check for csi.vsphere.vmware.com provisioner. for param := range sc.Parameters { if unSupportedParameters.Has(param) { diff --git a/pkg/syncer/admissionhandler/validatestorageclass_test.go b/pkg/syncer/admissionhandler/validatestorageclass_test.go index f71da2183a..dcccdedc8c 100644 --- a/pkg/syncer/admissionhandler/validatestorageclass_test.go +++ b/pkg/syncer/admissionhandler/validatestorageclass_test.go @@ -51,12 +51,12 @@ func TestValidateStorageClassForAllowVolumeExpansion(t *testing.T) { "\"volumeBindingMode\": \"Immediate\"\n}"), } admissionResponse := validateStorageClass(ctx, &admissionReview) - if admissionResponse.Allowed { - t.Log("TestValidateStorageClassForAllowVolumeExpansion Passed") - } else { + if !strings.Contains(string(admissionResponse.Result.Reason), volumeExpansionErrorMessage) || + admissionResponse.Allowed { t.Fatalf("TestValidateStorageClassForAllowVolumeExpansion failed. "+ "admissionReview.Request: %v, admissionResponse: %v", admissionReview.Request, admissionResponse) } + t.Log("TestValidateStorageClassForAllowVolumeExpansion Passed") } // TestValidateStorageClassForMigrationParameter is the unit test for validating diff --git a/pkg/syncer/cnsoperator/controller/cnsnodevmattachment/cnsnodevmattachment_controller.go b/pkg/syncer/cnsoperator/controller/cnsnodevmattachment/cnsnodevmattachment_controller.go index d1dec85132..cf95d8bc88 100644 --- a/pkg/syncer/cnsoperator/controller/cnsnodevmattachment/cnsnodevmattachment_controller.go +++ b/pkg/syncer/cnsoperator/controller/cnsnodevmattachment/cnsnodevmattachment_controller.go @@ -630,9 +630,6 @@ func (r *ReconcileCnsNodeVMAttachment) Reconcile(ctx context.Context, // This can happen when reconciler returns reconcile.Result{RequeueAfter: timeout}, the err will be set to nil, // and corresponding faulttype will be set // for this case, we need count it as an attach/detach failure - if csifault.IsNonStorageFault(faulttype) { - faulttype = csifault.AddCsiNonStoragePrefix(ctx, faulttype) - } log.Errorf("Operation failed, reporting failure status to Prometheus."+ " Operation Type: %q, Volume Type: %q, Fault Type: %q", volumeOpType, volumeType, faulttype) diff --git a/pkg/syncer/cnsoperator/controller/csinodetopology/csinodetopology_controller.go b/pkg/syncer/cnsoperator/controller/csinodetopology/csinodetopology_controller.go index 45b0042cb2..531653674e 100644 --- a/pkg/syncer/cnsoperator/controller/csinodetopology/csinodetopology_controller.go +++ b/pkg/syncer/cnsoperator/controller/csinodetopology/csinodetopology_controller.go @@ -110,6 +110,7 @@ func Add(mgr manager.Manager, clusterFlavor cnstypes.CnsClusterFlavor, } } + useNodeUuid := coCommonInterface.IsFSSEnabled(ctx, common.UseCSINodeId) isMultiVCFSSEnabled := coCommonInterface.IsFSSEnabled(ctx, common.MultiVCenterCSITopology) // Initialize kubernetes client. k8sclient, err := k8s.NewClient(ctx) @@ -127,19 +128,20 @@ func Add(mgr manager.Manager, clusterFlavor cnstypes.CnsClusterFlavor, ) recorder := eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: csinodetopologyv1alpha1.GroupName}) - return add(mgr, newReconciler(mgr, configInfo, recorder, + return add(mgr, newReconciler(mgr, configInfo, recorder, useNodeUuid, enableTKGsHAinGuest, isMultiVCFSSEnabled, vmOperatorClient, supervisorNamespace)) } // newReconciler returns a new `reconcile.Reconciler`. func newReconciler(mgr manager.Manager, configInfo *cnsconfig.ConfigurationInfo, recorder record.EventRecorder, - enableTKGsHAinGuest bool, isMultiVCFSSEnabled bool, vmOperatorClient client.Client, + useNodeUuid bool, enableTKGsHAinGuest bool, isMultiVCFSSEnabled bool, vmOperatorClient client.Client, supervisorNamespace string) reconcile.Reconciler { return &ReconcileCSINodeTopology{ client: mgr.GetClient(), scheme: mgr.GetScheme(), configInfo: configInfo, recorder: recorder, + useNodeUuid: useNodeUuid, enableTKGsHAinGuest: enableTKGsHAinGuest, isMultiVCFSSEnabled: isMultiVCFSSEnabled, vmOperatorClient: vmOperatorClient, @@ -205,6 +207,7 @@ type ReconcileCSINodeTopology struct { scheme *runtime.Scheme configInfo *cnsconfig.ConfigurationInfo recorder record.EventRecorder + useNodeUuid bool enableTKGsHAinGuest bool isMultiVCFSSEnabled bool vmOperatorClient client.Client @@ -269,16 +272,16 @@ func (r *ReconcileCSINodeTopology) reconcileForVanilla(ctx context.Context, requ return reconcile.Result{RequeueAfter: timeout}, nil } - if clusterFlavor == cnstypes.CnsClusterFlavorVanilla { + if r.useNodeUuid && clusterFlavor == cnstypes.CnsClusterFlavorVanilla { nodeID = instance.Spec.NodeUUID if nodeID != "" { - nodeVM, err = nodeManager.GetNodeVMAndUpdateCache(ctx, nodeID, nil) + nodeVM, err = nodeManager.GetNode(ctx, nodeID, nil) } else { return reconcile.Result{RequeueAfter: timeout}, nil } } else { nodeID = instance.Spec.NodeID - nodeVM, err = nodeManager.GetNodeVMByNameAndUpdateCache(ctx, nodeID) + nodeVM, err = nodeManager.GetNodeByName(ctx, nodeID) } if err != nil { if err == node.ErrNodeNotFound { diff --git a/pkg/syncer/cnsoperator/controller/csinodetopology/csinodetopology_controller_test.go b/pkg/syncer/cnsoperator/controller/csinodetopology/csinodetopology_controller_test.go index e938b8eb95..455ff839f6 100644 --- a/pkg/syncer/cnsoperator/controller/csinodetopology/csinodetopology_controller_test.go +++ b/pkg/syncer/cnsoperator/controller/csinodetopology/csinodetopology_controller_test.go @@ -157,6 +157,7 @@ func TestCSINodeTopologyControllerForTKGSHA(t *testing.T) { scheme: s, configInfo: &cnsconfig.ConfigurationInfo{}, recorder: record.NewFakeRecorder(testBufferSize), + useNodeUuid: true, enableTKGsHAinGuest: true, vmOperatorClient: fakeVmOperatorClient, supervisorNamespace: testSupervisorNamespace, diff --git a/pkg/syncer/metadatasyncer.go b/pkg/syncer/metadatasyncer.go index 037ce1fc08..b55892812d 100644 --- a/pkg/syncer/metadatasyncer.go +++ b/pkg/syncer/metadatasyncer.go @@ -248,7 +248,6 @@ func InitMetadataSyncer(ctx context.Context, clusterFlavor cnstypes.CnsClusterFl if err != nil { return err } - vCenter.Config.ReloadVCConfigForNewClient = true metadataSyncer.host = vCenter.Config.Host cnsDeletionMap[metadataSyncer.host] = make(map[string]bool) @@ -738,7 +737,7 @@ func startTopologyCRInformer(ctx context.Context, cfg *restclient.Config) error // in the MetadataSyncer.topologyVCMap parameter. func addLabelsToTopologyVCMap(ctx context.Context, nodeTopoObj csinodetopologyv1alpha1.CSINodeTopology) { log := logger.GetLogger(ctx) - nodeVM, err := nodeMgr.GetNodeVMAndUpdateCache(ctx, nodeTopoObj.Spec.NodeUUID, nil) + nodeVM, err := nodeMgr.GetNode(ctx, nodeTopoObj.Spec.NodeUUID, nil) if err != nil { log.Errorf("Node %q is not yet registered in the node manager. Error: %+v", nodeTopoObj.Spec.NodeUUID, err) @@ -855,7 +854,7 @@ func topoCRDeleted(obj interface{}) { // instance in the MetadataSyncer.topologyVCMap parameter. func removeLabelsFromTopologyVCMap(ctx context.Context, nodeTopoObj csinodetopologyv1alpha1.CSINodeTopology) { log := logger.GetLogger(ctx) - nodeVM, err := nodeMgr.GetNodeVMAndUpdateCache(ctx, nodeTopoObj.Spec.NodeUUID, nil) + nodeVM, err := nodeMgr.GetNode(ctx, nodeTopoObj.Spec.NodeUUID, nil) if err != nil { log.Errorf("Node %q is not yet registered in the node manager. Error: %+v", nodeTopoObj.Spec.NodeUUID, err) diff --git a/pkg/syncer/util.go b/pkg/syncer/util.go index 6ea95737f4..42de62a56d 100644 --- a/pkg/syncer/util.go +++ b/pkg/syncer/util.go @@ -361,7 +361,6 @@ func getConfig(ctx context.Context) (*cnsconfig.Config, error) { "vSphere config secret and in immutable ConfigMap") } cfg.Global.ClusterID = clusterID - cnsconfig.GeneratedVanillaClusterID = clusterID } else { if _, err := commonco.ContainerOrchestratorUtility.GetConfigMap(ctx, cnsconfig.ClusterIDConfigMapName, CSINamespace); err == nil { diff --git a/tests/e2e/OWNERS b/tests/e2e/OWNERS index 660fd9dbd4..c3584925bf 100644 --- a/tests/e2e/OWNERS +++ b/tests/e2e/OWNERS @@ -1,3 +1,4 @@ approvers: - openshift-storage-maintainers -component: "Storage / Kubernetes External Components" +component: "Storage" +subcomponent: Kubernetes External Components diff --git a/tests/e2e/config_change_test.go b/tests/e2e/config_change_test.go index 1f82a517e4..f8028c7783 100644 --- a/tests/e2e/config_change_test.go +++ b/tests/e2e/config_change_test.go @@ -23,7 +23,6 @@ var _ bool = ginkgo.Describe("[csi-supervisor] config-change-test", func() { storagePolicyName string ctx context.Context nimbusGeneratedVcPwd string - clientIndex int ) const ( configSecret = "vsphere-config-secret" @@ -45,7 +44,6 @@ var _ bool = ginkgo.Describe("[csi-supervisor] config-change-test", func() { defer cancel() nimbusGeneratedVcPwd = GetAndExpectStringEnvVar(nimbusVcPwd) - clientIndex = 0 }) ginkgo.AfterEach(func() { @@ -101,8 +99,7 @@ var _ bool = ginkgo.Describe("[csi-supervisor] config-change-test", func() { username := vsphereCfg.Global.User currentPassword := vsphereCfg.Global.Password newPassword := e2eTestPassword - err = invokeVCenterChangePassword(username, nimbusGeneratedVcPwd, newPassword, vcAddress, - false, clientIndex) + err = invokeVCenterChangePassword(username, nimbusGeneratedVcPwd, newPassword, vcAddress) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Modifying the password in the secret") @@ -117,8 +114,7 @@ var _ bool = ginkgo.Describe("[csi-supervisor] config-change-test", func() { defer func() { ginkgo.By("Reverting the password change") - err = invokeVCenterChangePassword(username, nimbusGeneratedVcPwd, currentPassword, vcAddress, false, - clientIndex) + err = invokeVCenterChangePassword(username, nimbusGeneratedVcPwd, currentPassword, vcAddress) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Reverting the secret change back to reflect the original password") diff --git a/tests/e2e/config_secret.go b/tests/e2e/config_secret.go index 22b8d2bfc1..7b8a135801 100644 --- a/tests/e2e/config_secret.go +++ b/tests/e2e/config_secret.go @@ -62,7 +62,6 @@ var _ = ginkgo.Describe("Config-Secret", func() { dataCenter string sshClientConfig *ssh.ClientConfig nimbusGeneratedK8sVmPwd string - clusterId string ) ginkgo.BeforeEach(func() { @@ -86,7 +85,6 @@ var _ = ginkgo.Describe("Config-Secret", func() { vCenterIP = e2eVSphere.Config.Global.VCenterHostname vCenterPort = e2eVSphere.Config.Global.VCenterPort dataCenter = e2eVSphere.Config.Global.Datacenters - clusterId = e2eVSphere.Config.Global.ClusterID propagateVal = "false" revertOriginalvCenterUser = false configSecretUser1Alias = configSecretTestUser1 + "@vsphere.local" @@ -128,10 +126,10 @@ var _ = ginkgo.Describe("Config-Secret", func() { ginkgo.By("Reverting back csi-vsphere.conf with its original vcenter user " + "and its credentials") createCsiVsphereSecret(client, ctx, vCenterUIUser, vCenterUIPassword, csiNamespace, vCenterIP, - vCenterPort, dataCenter, "", clusterId) + vCenterPort, dataCenter, "") ginkgo.By("Restart CSI driver") - restartSuccess, err := restartCSIDriver(ctx, client, csiNamespace, csiReplicas) + restartSuccess, err := restartCSIDriver(ctx, client, namespace, csiReplicas) gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") gomega.Expect(err).NotTo(gomega.HaveOccurred()) } @@ -150,8 +148,8 @@ var _ = ginkgo.Describe("Config-Secret", func() { 7. Cleanup all objects created during the test */ - ginkgo.It("Update user credentials in vsphere config secret keeping password same "+ - "for both test users", ginkgo.Label(p1, vsphereConfigSecret, block, file, vanilla), func() { + ginkgo.It("[csi-config-secret-block][csi-config-secret-file] Update user credentials in vsphere config "+ + "secret keeping password same for both test users", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -179,10 +177,10 @@ var _ = ginkgo.Describe("Config-Secret", func() { ginkgo.By("Create vsphere-config-secret file with testuser1 credentials") createCsiVsphereSecret(client, ctx, configSecretUser1Alias, configSecretTestUser1Password, - csiNamespace, vCenterIP, vCenterPort, dataCenter, "", clusterId) + csiNamespace, vCenterIP, vCenterPort, dataCenter, "") ginkgo.By("Restart CSI driver") - restartSuccess, err := restartCSIDriver(ctx, client, csiNamespace, csiReplicas) + restartSuccess, err := restartCSIDriver(ctx, client, namespace, csiReplicas) gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -203,22 +201,22 @@ var _ = ginkgo.Describe("Config-Secret", func() { ginkgo.By("Create vsphere-config-secret file with testuser2 credentials") createCsiVsphereSecret(client, ctx, configSecretUser2Alias, configSecretTestUser1Password, csiNamespace, - vCenterIP, vCenterPort, dataCenter, "", clusterId) + vCenterIP, vCenterPort, dataCenter, "") defer func() { ginkgo.By("Reverting back csi-vsphere.conf with its original vcenter user " + "and its credentials") createCsiVsphereSecret(client, ctx, vCenterUIUser, vCenterUIPassword, csiNamespace, vCenterIP, - vCenterPort, dataCenter, "", clusterId) + vCenterPort, dataCenter, "") revertOriginalvCenterUser = true ginkgo.By("Restart CSI driver") - restartSuccess, err = restartCSIDriver(ctx, client, csiNamespace, csiReplicas) + restartSuccess, err = restartCSIDriver(ctx, client, namespace, csiReplicas) gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() ginkgo.By("Restart CSI driver") - restartSuccess, err = restartCSIDriver(ctx, client, csiNamespace, csiReplicas) + restartSuccess, err = restartCSIDriver(ctx, client, namespace, csiReplicas) gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -249,8 +247,8 @@ var _ = ginkgo.Describe("Config-Secret", func() { 12. Cleanup all objects created during the test */ - ginkgo.It("Change vcenter user password and restart csi controller pod", ginkgo.Label(p0, - vsphereConfigSecret, block, file, vanilla), func() { + ginkgo.It("[csi-config-secret-block][csi-config-secret-file] Change vcenter user password "+ + "and restart csi controller pod", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() testUser1NewPassword := "Admin!123" @@ -280,10 +278,10 @@ var _ = ginkgo.Describe("Config-Secret", func() { ginkgo.By("Update vsphere-config-secret with testuser1 credentials") createCsiVsphereSecret(client, ctx, configSecretUser1Alias, configSecretTestUser1Password, csiNamespace, - vCenterIP, vCenterPort, dataCenter, "", clusterId) + vCenterIP, vCenterPort, dataCenter, "") ginkgo.By("Restart CSI driver") - restartSuccess, err := restartCSIDriver(ctx, client, csiNamespace, csiReplicas) + restartSuccess, err := restartCSIDriver(ctx, client, namespace, csiReplicas) gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -348,30 +346,26 @@ var _ = ginkgo.Describe("Config-Secret", func() { ginkgo.By("Update vsphere-config-secret file with testuser1 updated credentials") createCsiVsphereSecret(client, ctx, configSecretUser1Alias, testUser1NewPassword, csiNamespace, - vCenterIP, vCenterPort, dataCenter, "", clusterId) + vCenterIP, vCenterPort, dataCenter, "") defer func() { ginkgo.By("Reverting back csi-vsphere.conf with its original vcenter user " + "and its credentials") createCsiVsphereSecret(client, ctx, vCenterUIUser, vCenterUIPassword, csiNamespace, - vCenterIP, vCenterPort, dataCenter, "", clusterId) + vCenterIP, vCenterPort, dataCenter, "") revertOriginalvCenterUser = true ginkgo.By("Restart CSI driver") - restartSuccess, err = restartCSIDriver(ctx, client, csiNamespace, csiReplicas) + restartSuccess, err = restartCSIDriver(ctx, client, namespace, csiReplicas) gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() ginkgo.By("Restart CSI driver") - restartSuccess, err = restartCSIDriver(ctx, client, csiNamespace, csiReplicas) + restartSuccess, err = restartCSIDriver(ctx, client, namespace, csiReplicas) gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") gomega.Expect(err).NotTo(gomega.HaveOccurred()) - framework.Logf("Waiting for %v csi controller pods to be in running state", - pollTimeout) - time.Sleep(pollTimeout) - - ginkgo.By("Check csi controller pods running state") + ginkgo.By("Wait for csi controller pods to be in running state") list_of_pods, err := fpod.GetPodsInNamespace(client, csiNamespace, ignoreLabels) gomega.Expect(err).NotTo(gomega.HaveOccurred()) for i := 0; i < len(list_of_pods); i++ { @@ -406,8 +400,8 @@ var _ = ginkgo.Describe("Config-Secret", func() { 7. Cleanup all objects created during the test */ - ginkgo.It("Update user credentials in vsphere config secret keeping password different for both test "+ - "users", ginkgo.Label(p0, vsphereConfigSecret, block, file, vanilla), func() { + ginkgo.It("[csi-config-secret-block][csi-config-secret-file] Update user credentials in vsphere config "+ + "secret keeping password different for both test users", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -435,10 +429,10 @@ var _ = ginkgo.Describe("Config-Secret", func() { ginkgo.By("Update vsphere-config-secret with testuser1 credentials") createCsiVsphereSecret(client, ctx, configSecretUser1Alias, configSecretTestUser1Password, csiNamespace, - vCenterIP, vCenterPort, dataCenter, "", clusterId) + vCenterIP, vCenterPort, dataCenter, "") ginkgo.By("Restart CSI driver") - restartSuccess, err := restartCSIDriver(ctx, client, csiNamespace, csiReplicas) + restartSuccess, err := restartCSIDriver(ctx, client, namespace, csiReplicas) gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -459,22 +453,22 @@ var _ = ginkgo.Describe("Config-Secret", func() { ginkgo.By("Update vsphere-config-secret with testuser2 credentials") createCsiVsphereSecret(client, ctx, configSecretUser2Alias, configSecretTestUser2Password, csiNamespace, - vCenterIP, vCenterPort, dataCenter, "", clusterId) + vCenterIP, vCenterPort, dataCenter, "") defer func() { ginkgo.By("Reverting back csi-vsphere.conf with its original vcenter user " + "and its credentials") createCsiVsphereSecret(client, ctx, vCenterUIUser, vCenterUIPassword, csiNamespace, - vCenterIP, vCenterPort, dataCenter, "", clusterId) + vCenterIP, vCenterPort, dataCenter, "") revertOriginalvCenterUser = true ginkgo.By("Restart CSI driver") - restartSuccess, err = restartCSIDriver(ctx, client, csiNamespace, csiReplicas) + restartSuccess, err = restartCSIDriver(ctx, client, namespace, csiReplicas) gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() ginkgo.By("Restart CSI driver") - restartSuccess, err = restartCSIDriver(ctx, client, csiNamespace, csiReplicas) + restartSuccess, err = restartCSIDriver(ctx, client, namespace, csiReplicas) gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -503,8 +497,8 @@ var _ = ginkgo.Describe("Config-Secret", func() { 9. Cleanup all objects created during the test */ - ginkgo.It("Change vcenter ip to hostname and viceversa in vsphere config "+ - "secret", ginkgo.Label(p0, vsphereConfigSecret, block, file, vanilla), func() { + ginkgo.It("[csi-config-secret-block][csi-config-secret-file] Change vcenter ip to hostname and "+ + "viceversa in vsphere config secret", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -532,10 +526,10 @@ var _ = ginkgo.Describe("Config-Secret", func() { ginkgo.By("Update vsphere-config-secret with testuser1 credentials using vcenter IP") createCsiVsphereSecret(client, ctx, configSecretUser1Alias, configSecretTestUser1Password, csiNamespace, - vCenterIP, vCenterPort, dataCenter, "", clusterId) + vCenterIP, vCenterPort, dataCenter, "") ginkgo.By("Restart CSI driver") - restartSuccess, err := restartCSIDriver(ctx, client, csiNamespace, csiReplicas) + restartSuccess, err := restartCSIDriver(ctx, client, namespace, csiReplicas) gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -561,10 +555,10 @@ var _ = ginkgo.Describe("Config-Secret", func() { ginkgo.By("Update vsphere-config-secret to use vcenter hostname") createCsiVsphereSecret(client, ctx, configSecretUser1Alias, configSecretTestUser1Password, csiNamespace, - vCenterHostName, vCenterPort, dataCenter, "", clusterId) + vCenterHostName, vCenterPort, dataCenter, "") ginkgo.By("Restart CSI driver") - restartSuccess, err = restartCSIDriver(ctx, client, csiNamespace, csiReplicas) + restartSuccess, err = restartCSIDriver(ctx, client, namespace, csiReplicas) gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -576,22 +570,22 @@ var _ = ginkgo.Describe("Config-Secret", func() { ginkgo.By("Update vsphere-config-secret to use vcenter IP") createCsiVsphereSecret(client, ctx, configSecretUser1Alias, configSecretTestUser1Password, csiNamespace, - vCenterIP, vCenterPort, dataCenter, "", clusterId) + vCenterIP, vCenterPort, dataCenter, "") defer func() { ginkgo.By("Reverting back csi-vsphere.conf with its original vcenter user " + "and its credentials") createCsiVsphereSecret(client, ctx, vCenterUIUser, vCenterUIPassword, csiNamespace, vCenterIP, vCenterPort, - dataCenter, "", clusterId) + dataCenter, "") revertOriginalvCenterUser = true ginkgo.By("Restart CSI driver") - restartSuccess, err = restartCSIDriver(ctx, client, csiNamespace, csiReplicas) + restartSuccess, err = restartCSIDriver(ctx, client, namespace, csiReplicas) gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() ginkgo.By("Restart CSI driver") - restartSuccess, err = restartCSIDriver(ctx, client, csiNamespace, csiReplicas) + restartSuccess, err = restartCSIDriver(ctx, client, namespace, csiReplicas) gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -619,8 +613,8 @@ var _ = ginkgo.Describe("Config-Secret", func() { 9. Cleanup all objects created during the test */ - ginkgo.It("Change vcenter user to wrong dummy user and later switch back to "+ - "correct one", ginkgo.Label(p0, vsphereConfigSecret, block, file, vanilla), func() { + ginkgo.It("[csi-config-secret-block][csi-config-secret-file] Change vcenter user to wrong dummy "+ + "user and later switch back to correct one", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() dummyTestUser := "dummyUser@vsphere.local" @@ -649,10 +643,10 @@ var _ = ginkgo.Describe("Config-Secret", func() { ginkgo.By("Update vsphere-config-secret with testuser1 credentials") createCsiVsphereSecret(client, ctx, configSecretUser1Alias, configSecretTestUser1Password, csiNamespace, - vCenterIP, vCenterPort, dataCenter, "", clusterId) + vCenterIP, vCenterPort, dataCenter, "") ginkgo.By("Restart CSI driver") - restartSuccess, err := restartCSIDriver(ctx, client, csiNamespace, csiReplicas) + restartSuccess, err := restartCSIDriver(ctx, client, namespace, csiReplicas) gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -673,7 +667,7 @@ var _ = ginkgo.Describe("Config-Secret", func() { ginkgo.By("Update vsphere-config-secret with dummy user credentials") createCsiVsphereSecret(client, ctx, dummyTestUser, configSecretTestUser1Password, csiNamespace, - vCenterIP, vCenterPort, dataCenter, "", clusterId) + vCenterIP, vCenterPort, dataCenter, "") ginkgo.By("Restart CSI driver") err = updateDeploymentReplicawithWait(client, 0, vSphereCSIControllerPodNamePrefix, csiNamespace) @@ -703,22 +697,22 @@ var _ = ginkgo.Describe("Config-Secret", func() { ginkgo.By("Update vsphere-config-secret with testuser1 credentials") createCsiVsphereSecret(client, ctx, configSecretUser1Alias, configSecretTestUser1Password, csiNamespace, - vCenterIP, vCenterPort, dataCenter, "", clusterId) + vCenterIP, vCenterPort, dataCenter, "") defer func() { ginkgo.By("Reverting back csi-vsphere.conf with its original vcenter user " + "and its credentials") createCsiVsphereSecret(client, ctx, vCenterUIUser, vCenterUIPassword, csiNamespace, - vCenterIP, vCenterPort, dataCenter, "", clusterId) + vCenterIP, vCenterPort, dataCenter, "") revertOriginalvCenterUser = true ginkgo.By("Restart CSI driver") - restartSuccess, err = restartCSIDriver(ctx, client, csiNamespace, csiReplicas) + restartSuccess, err = restartCSIDriver(ctx, client, namespace, csiReplicas) gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() ginkgo.By("Restart CSI driver") - restartSuccess, err = restartCSIDriver(ctx, client, csiNamespace, csiReplicas) + restartSuccess, err = restartCSIDriver(ctx, client, namespace, csiReplicas) gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -748,8 +742,8 @@ var _ = ginkgo.Describe("Config-Secret", func() { 10. Cleanup all objects created during the test */ - ginkgo.It("Add a user without adding required roles and privileges and switch back "+ - "to the correct one", ginkgo.Label(p0, vsphereConfigSecret, block, file, vanilla), func() { + ginkgo.It("[csi-config-secret-block][csi-config-secret-file] Add a user without adding required "+ + "roles and privileges and switch back to the correct one", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -777,10 +771,10 @@ var _ = ginkgo.Describe("Config-Secret", func() { ginkgo.By("Update vsphere-config-secret with testuser1 credentials") createCsiVsphereSecret(client, ctx, configSecretUser1Alias, configSecretTestUser1Password, csiNamespace, - vCenterIP, vCenterPort, dataCenter, "", clusterId) + vCenterIP, vCenterPort, dataCenter, "") ginkgo.By("Restart CSI driver") - restartSuccess, err := restartCSIDriver(ctx, client, csiNamespace, csiReplicas) + restartSuccess, err := restartCSIDriver(ctx, client, namespace, csiReplicas) gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -801,10 +795,10 @@ var _ = ginkgo.Describe("Config-Secret", func() { ginkgo.By("Update vsphere-config-secret with testuser2 credentials") createCsiVsphereSecret(client, ctx, configSecretUser2Alias, configSecretTestUser1Password, csiNamespace, - vCenterIP, vCenterPort, dataCenter, "", clusterId) + vCenterIP, vCenterPort, dataCenter, "") ginkgo.By("Restart CSI driver") - restartSuccess, err = restartCSIDriver(ctx, client, csiNamespace, csiReplicas) + restartSuccess, err = restartCSIDriver(ctx, client, namespace, csiReplicas) gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -824,22 +818,22 @@ var _ = ginkgo.Describe("Config-Secret", func() { ginkgo.By("Update vsphere-config-secret with testuser1 credentials which has required roles and privileges") createCsiVsphereSecret(client, ctx, configSecretUser1Alias, configSecretTestUser1Password, csiNamespace, - vCenterIP, vCenterPort, dataCenter, "", clusterId) + vCenterIP, vCenterPort, dataCenter, "") defer func() { ginkgo.By("Reverting back csi-vsphere.conf with its original vcenter user " + "and its credentials") createCsiVsphereSecret(client, ctx, vCenterUIUser, vCenterUIPassword, csiNamespace, - vCenterIP, vCenterPort, dataCenter, "", clusterId) + vCenterIP, vCenterPort, dataCenter, "") revertOriginalvCenterUser = true ginkgo.By("Restart CSI driver") - restartSuccess, err = restartCSIDriver(ctx, client, csiNamespace, csiReplicas) + restartSuccess, err = restartCSIDriver(ctx, client, namespace, csiReplicas) gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() ginkgo.By("Restart CSI driver") - restartSuccess, err = restartCSIDriver(ctx, client, csiNamespace, csiReplicas) + restartSuccess, err = restartCSIDriver(ctx, client, namespace, csiReplicas) gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -869,8 +863,8 @@ var _ = ginkgo.Describe("Config-Secret", func() { 8. Cleanup all objects created during the test */ - ginkgo.It("Add necessary roles and privileges to the user post CSI driver "+ - "creation", ginkgo.Label(p0, vsphereConfigSecret, block, file, vanilla), func() { + ginkgo.It("[csi-config-secret-block][csi-config-secret-file] Add necessary roles and privileges "+ + "to the user post CSI driver creation", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -898,22 +892,22 @@ var _ = ginkgo.Describe("Config-Secret", func() { ginkgo.By("Update vsphere-config-secret with testuser1 credentials") createCsiVsphereSecret(client, ctx, configSecretUser1Alias, configSecretTestUser1Password, csiNamespace, - vCenterIP, vCenterPort, dataCenter, "", clusterId) + vCenterIP, vCenterPort, dataCenter, "") defer func() { ginkgo.By("Reverting back csi-vsphere.conf with its original vcenter user " + "and its credentials") createCsiVsphereSecret(client, ctx, vCenterUIUser, vCenterUIPassword, csiNamespace, - vCenterIP, vCenterPort, dataCenter, "", clusterId) + vCenterIP, vCenterPort, dataCenter, "") revertOriginalvCenterUser = true ginkgo.By("Restart CSI driver") - restartSuccess, err := restartCSIDriver(ctx, client, csiNamespace, csiReplicas) + restartSuccess, err := restartCSIDriver(ctx, client, namespace, csiReplicas) gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() ginkgo.By("Restart CSI driver") - restartSuccess, err := restartCSIDriver(ctx, client, csiNamespace, csiReplicas) + restartSuccess, err := restartCSIDriver(ctx, client, namespace, csiReplicas) gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -946,7 +940,7 @@ var _ = ginkgo.Describe("Config-Secret", func() { dataCenters, clusters, hosts, vms, datastores, "reuseUser", "reuseRoles") ginkgo.By("Restart CSI driver") - restartSuccess, err = restartCSIDriver(ctx, client, csiNamespace, csiReplicas) + restartSuccess, err = restartCSIDriver(ctx, client, namespace, csiReplicas) gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -981,8 +975,8 @@ var _ = ginkgo.Describe("Config-Secret", func() { 10. Cleanup all objects created during the test */ - ginkgo.It("Add wrong datacenter and switch back to the correct datacenter in vsphere "+ - "config secret file", ginkgo.Label(p1, vsphereConfigSecret, block, file, vanilla), func() { + ginkgo.It("[csi-config-secret-block][csi-config-secret-file] Add wrong datacenter and switch back "+ + "to the correct datacenter in vsphere config secret file", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -1012,10 +1006,10 @@ var _ = ginkgo.Describe("Config-Secret", func() { ginkgo.By("Update vsphere-config-secret with testuser1 credentials") createCsiVsphereSecret(client, ctx, configSecretUser1Alias, configSecretTestUser1Password, csiNamespace, - vCenterIP, vCenterPort, dataCenter, "", clusterId) + vCenterIP, vCenterPort, dataCenter, "") ginkgo.By("Restart CSI driver") - restartSuccess, err := restartCSIDriver(ctx, client, csiNamespace, csiReplicas) + restartSuccess, err := restartCSIDriver(ctx, client, namespace, csiReplicas) gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -1036,7 +1030,7 @@ var _ = ginkgo.Describe("Config-Secret", func() { ginkgo.By("Update vsphere-config-secret with dummy datacenter details") createCsiVsphereSecret(client, ctx, configSecretUser1Alias, configSecretTestUser1Password, csiNamespace, - vCenterIP, vCenterPort, dummyDataCenter, "", clusterId) + vCenterIP, vCenterPort, dummyDataCenter, "") ginkgo.By("Restart CSI driver") err = updateDeploymentReplicawithWait(client, 0, vSphereCSIControllerPodNamePrefix, csiNamespace) @@ -1060,22 +1054,22 @@ var _ = ginkgo.Describe("Config-Secret", func() { ginkgo.By("Update vsphere-config-secret with testuser1 credentials") createCsiVsphereSecret(client, ctx, configSecretUser1Alias, configSecretTestUser1Password, csiNamespace, - vCenterIP, vCenterPort, dataCenter, "", clusterId) + vCenterIP, vCenterPort, dataCenter, "") defer func() { ginkgo.By("Reverting back csi-vsphere.conf with its original vcenter user " + "and its credentials") createCsiVsphereSecret(client, ctx, vCenterUIUser, vCenterUIPassword, csiNamespace, - vCenterIP, vCenterPort, dataCenter, "", clusterId) + vCenterIP, vCenterPort, dataCenter, "") revertOriginalvCenterUser = true ginkgo.By("Restart CSI driver") - restartSuccess, err = restartCSIDriver(ctx, client, csiNamespace, csiReplicas) + restartSuccess, err = restartCSIDriver(ctx, client, namespace, csiReplicas) gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() ginkgo.By("Restart CSI driver") - restartSuccess, err = restartCSIDriver(ctx, client, csiNamespace, csiReplicas) + restartSuccess, err = restartCSIDriver(ctx, client, namespace, csiReplicas) gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -1112,8 +1106,8 @@ var _ = ginkgo.Describe("Config-Secret", func() { 10. Cleanup all objects created during the test */ - ginkgo.It("Add wrong targetvSANFileShareDatastoreURLs and switch back to the correct "+ - "targetvSANFileShareDatastoreURLs", ginkgo.Label(p1, vsphereConfigSecret, file), func() { + ginkgo.It("[csi-config-secret-file] Add wrong targetvSANFileShareDatastoreURLs and switch back to the correct "+ + "targetvSANFileShareDatastoreURLs", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() targetDsURL := GetAndExpectStringEnvVar(envSharedDatastoreURL) @@ -1142,10 +1136,10 @@ var _ = ginkgo.Describe("Config-Secret", func() { ginkgo.By("Update vsphere-config-secret with testuser1 credentials") createCsiVsphereSecret(client, ctx, configSecretUser1Alias, configSecretTestUser1Password, csiNamespace, - vCenterIP, vCenterPort, dataCenter, "", clusterId) + vCenterIP, vCenterPort, dataCenter, "") ginkgo.By("Restart CSI driver") - restartSuccess, err := restartCSIDriver(ctx, client, csiNamespace, csiReplicas) + restartSuccess, err := restartCSIDriver(ctx, client, namespace, csiReplicas) gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -1166,22 +1160,22 @@ var _ = ginkgo.Describe("Config-Secret", func() { ginkgo.By("Update vsphere-config-secret with testuser1 credentials and pass target datastore url") createCsiVsphereSecret(client, ctx, configSecretUser1Alias, configSecretTestUser1Password, csiNamespace, - vCenterIP, vCenterPort, dataCenter, targetDsURL, clusterId) + vCenterIP, vCenterPort, dataCenter, targetDsURL) defer func() { ginkgo.By("Reverting back csi-vsphere.conf with its original vcenter user " + "and its credentials") createCsiVsphereSecret(client, ctx, vCenterUIUser, vCenterUIPassword, csiNamespace, - vCenterIP, vCenterPort, dataCenter, "", clusterId) + vCenterIP, vCenterPort, dataCenter, "") revertOriginalvCenterUser = true ginkgo.By("Restart CSI driver") - restartSuccess, err = restartCSIDriver(ctx, client, csiNamespace, csiReplicas) + restartSuccess, err = restartCSIDriver(ctx, client, namespace, csiReplicas) gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() ginkgo.By("Restart CSI driver") - restartSuccess, err = restartCSIDriver(ctx, client, csiNamespace, csiReplicas) + restartSuccess, err = restartCSIDriver(ctx, client, namespace, csiReplicas) gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -1207,7 +1201,7 @@ var _ = ginkgo.Describe("Config-Secret", func() { 9. Cleanup all objects created during the test */ - ginkgo.It("VC with Custom Port", ginkgo.Label(p1, vsphereConfigSecret, file, block, customPort), func() { + ginkgo.It("[vc-custom-port] VC with Custom Port", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() defaultvCenterPort := "443" @@ -1236,10 +1230,10 @@ var _ = ginkgo.Describe("Config-Secret", func() { ginkgo.By("Create vsphere-config-secret file with testuser1 credentials using default vc port") createCsiVsphereSecret(client, ctx, configSecretUser1Alias, configSecretTestUser1Password, csiNamespace, - vCenterIP, defaultvCenterPort, dataCenter, "", clusterId) + vCenterIP, defaultvCenterPort, dataCenter, "") ginkgo.By("Restart CSI driver") - restartSuccess, err := restartCSIDriver(ctx, client, csiNamespace, csiReplicas) + restartSuccess, err := restartCSIDriver(ctx, client, namespace, csiReplicas) gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -1271,22 +1265,22 @@ var _ = ginkgo.Describe("Config-Secret", func() { ginkgo.By("Create vsphere-config-secret file with testuser1 credentials using non-default port") createCsiVsphereSecret(client, ctx, configSecretUser1Alias, configSecretTestUser1Password, csiNamespace, - vCenterIP, vCenterPort, dataCenter, "", clusterId) + vCenterIP, vCenterPort, dataCenter, "") defer func() { ginkgo.By("Reverting back csi-vsphere.conf with its original vcenter user " + "and its credentials") createCsiVsphereSecret(client, ctx, vCenterUIUser, vCenterUIPassword, csiNamespace, - vCenterIP, vCenterPort, dataCenter, "", clusterId) + vCenterIP, vCenterPort, dataCenter, "") revertOriginalvCenterUser = true ginkgo.By("Restart CSI driver") - restartSuccess, err = restartCSIDriver(ctx, client, csiNamespace, csiReplicas) + restartSuccess, err = restartCSIDriver(ctx, client, namespace, csiReplicas) gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() ginkgo.By("Restart CSI driver") - restartSuccess, err = restartCSIDriver(ctx, client, csiNamespace, csiReplicas) + restartSuccess, err = restartCSIDriver(ctx, client, namespace, csiReplicas) gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -1334,8 +1328,7 @@ var _ = ginkgo.Describe("Config-Secret", func() { 10. Cleanup all objects created during the test */ - ginkgo.It("Modify VC Port and validate the workloads", ginkgo.Label(p1, vsphereConfigSecret, file, block, - customPort), func() { + ginkgo.It("[vc-custom-port] Modify VC Port and validate the workloads", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() dummyvCenterPort := "4444" @@ -1364,10 +1357,10 @@ var _ = ginkgo.Describe("Config-Secret", func() { ginkgo.By("Create vsphere-config-secret file using testuser1 credentials") createCsiVsphereSecret(client, ctx, configSecretUser1Alias, configSecretTestUser1Password, csiNamespace, - vCenterIP, vCenterPort, dataCenter, "", clusterId) + vCenterIP, vCenterPort, dataCenter, "") ginkgo.By("Restart CSI driver") - restartSuccess, err := restartCSIDriver(ctx, client, csiNamespace, csiReplicas) + restartSuccess, err := restartCSIDriver(ctx, client, namespace, csiReplicas) gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -1389,10 +1382,10 @@ var _ = ginkgo.Describe("Config-Secret", func() { ginkgo.By("Update vsphere-config-secret file with dummy vcenter port") createCsiVsphereSecret(client, ctx, configSecretUser1Alias, configSecretTestUser1Password, csiNamespace, - vCenterIP, dummyvCenterPort, dataCenter, "", clusterId) + vCenterIP, dummyvCenterPort, dataCenter, "") ginkgo.By("Restart CSI driver") - restartSuccess, err = restartCSIDriver(ctx, client, csiNamespace, csiReplicas) + restartSuccess, err = restartCSIDriver(ctx, client, namespace, csiReplicas) gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -1412,22 +1405,22 @@ var _ = ginkgo.Describe("Config-Secret", func() { ginkgo.By("Update vsphere-config-secret with correct vCenter credentials") createCsiVsphereSecret(client, ctx, configSecretUser1Alias, configSecretTestUser1Password, csiNamespace, - vCenterIP, vCenterPort, dataCenter, "", clusterId) + vCenterIP, vCenterPort, dataCenter, "") defer func() { ginkgo.By("Reverting back csi-vsphere.conf with its original vcenter user " + "and its credentials") createCsiVsphereSecret(client, ctx, vCenterUIUser, vCenterUIPassword, csiNamespace, - vCenterIP, vCenterPort, dataCenter, "", clusterId) + vCenterIP, vCenterPort, dataCenter, "") revertOriginalvCenterUser = true ginkgo.By("Restart CSI driver") - restartSuccess, err = restartCSIDriver(ctx, client, csiNamespace, csiReplicas) + restartSuccess, err = restartCSIDriver(ctx, client, namespace, csiReplicas) gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() ginkgo.By("Restart CSI driver") - restartSuccess, err = restartCSIDriver(ctx, client, csiNamespace, csiReplicas) + restartSuccess, err = restartCSIDriver(ctx, client, namespace, csiReplicas) gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") gomega.Expect(err).NotTo(gomega.HaveOccurred()) diff --git a/tests/e2e/config_secret_utils.go b/tests/e2e/config_secret_utils.go index a46bfbf502..e10ca24e80 100644 --- a/tests/e2e/config_secret_utils.go +++ b/tests/e2e/config_secret_utils.go @@ -28,7 +28,6 @@ import ( "golang.org/x/crypto/ssh" v1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" @@ -551,20 +550,17 @@ func setSearchlevelPermission(masterIp string, sshClientConfig *ssh.ClientConfig // createCsiVsphereSecret method is used to create csi vsphere secret file func createCsiVsphereSecret(client clientset.Interface, ctx context.Context, testUser string, - password string, csiNamespace string, vCenterIP string, vCenterPort string, - dataCenter string, targetvSANFileShareDatastoreURLs string, clusterID string) { + password string, csiNamespace string, vCenterIP string, + vCenterPort string, dataCenter string, targetvSANFileShareDatastoreURLs string) { currentSecret, err := client.CoreV1().Secrets(csiNamespace).Get(ctx, configSecret, metav1.GetOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) originalConf := string(currentSecret.Data[vSphereCSIConf]) vsphereCfg, err := readConfigFromSecretString(originalConf) - framework.Logf("original config: %v", vsphereCfg) gomega.Expect(err).NotTo(gomega.HaveOccurred()) vsphereCfg.Global.User = testUser vsphereCfg.Global.Password = password vsphereCfg.Global.Datacenters = dataCenter - vsphereCfg.Global.ClusterID = clusterID vsphereCfg.Global.TargetvSANFileShareDatastoreURLs = targetvSANFileShareDatastoreURLs - framework.Logf("updated config: %v", vsphereCfg) modifiedConf, err := writeConfigToSecretString(vsphereCfg) gomega.Expect(err).NotTo(gomega.HaveOccurred()) framework.Logf("Updating the secret to reflect new conf credentials") @@ -804,55 +800,3 @@ func createTestUserAndAssignLimitedRolesAndPrivileges(masterIp string, sshClient } } } - -// verifyClusterIdConfigMapGeneration verifies if cluster id configmap gets generated by -// csi driver in csi namespace -func verifyClusterIdConfigMapGeneration(client clientset.Interface, ctx context.Context, - csiNamespace string, cmToExist bool) { - _, err := client.CoreV1().ConfigMaps(csiNamespace).Get(ctx, vsphereClusterIdConfigMapName, - metav1.GetOptions{}) - if cmToExist && apierrors.IsNotFound(err) { - framework.Logf("Configmap: %s not found in namespace: %s", vsphereClusterIdConfigMapName, csiNamespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } else if !cmToExist && !apierrors.IsNotFound(err) { - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } -} - -// fetchClusterIdFromConfigmap fetches cluster id value from -// auto generated cluster id configmap by csi driver -func fetchClusterIdFromConfigmap(client clientset.Interface, ctx context.Context, - csiNamespace string) string { - clusterIdCm, err := client.CoreV1().ConfigMaps(csiNamespace).Get(ctx, vsphereClusterIdConfigMapName, - metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - data := clusterIdCm.Data - framework.Logf("cluster id configmap: %v", clusterIdCm) - return data["clusterID"] -} - -// recreateVsphereConfigSecret recreates config secret with new config parameters -// and restarts CSI driver -func recreateVsphereConfigSecret(client clientset.Interface, ctx context.Context, - vCenterUIUser string, vCenterUIPassword string, csiNamespace string, vCenterIP string, - clusterId string, vCenterPort string, dataCenter string, csiReplicas int32) { - createCsiVsphereSecret(client, ctx, vCenterUIUser, vCenterUIPassword, csiNamespace, - vCenterIP, vCenterPort, dataCenter, "", clusterId) - - ginkgo.By("Restart CSI driver") - restartSuccess, err := restartCSIDriver(ctx, client, csiNamespace, csiReplicas) - gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) -} - -// getCSIConfigSecretData returns data obtained fom csi config secret -// in namespace where CSI is deployed -func getCSIConfigSecretData(client clientset.Interface, ctx context.Context, - csiNamespace string) e2eTestConfig { - currentSecret, err := client.CoreV1().Secrets(csiNamespace).Get(ctx, configSecret, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - originalConf := string(currentSecret.Data[vSphereCSIConf]) - vsphereCfg, err := readConfigFromSecretString(originalConf) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - return vsphereCfg -} diff --git a/tests/e2e/csi_cns_telemetry_statefulsets.go b/tests/e2e/csi_cns_telemetry_statefulsets.go index 729dde17bd..4e35fa831d 100644 --- a/tests/e2e/csi_cns_telemetry_statefulsets.go +++ b/tests/e2e/csi_cns_telemetry_statefulsets.go @@ -90,11 +90,6 @@ var _ = ginkgo.Describe("[csi-block-vanilla] [csi-file-vanilla] [csi-supervisor] if vanillaCluster { // Reset the cluster distribution value to default value "CSI-Vanilla". setClusterDistribution(ctx, client, vanillaClusterDistribution) - } else if supervisorCluster { - dumpSvcNsEventsOnTestFailure(client, namespace) - } else { - svcClient, svNamespace := getSvcClientAndNamespace() - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) } }) @@ -143,7 +138,7 @@ var _ = ginkgo.Describe("[csi-block-vanilla] [csi-file-vanilla] [csi-supervisor] statefulset := GetStatefulSetFromManifest(namespace) ginkgo.By("Creating statefulset") statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &sc.Name + Annotations["volume.beta.kubernetes.io/storage-class"] = sc.Name CreateStatefulSet(namespace, statefulset, client) replicas := *(statefulset.Spec.Replicas) // Waiting for pods status to be Ready. diff --git a/tests/e2e/csi_cns_telemetry_vc_reboot.go b/tests/e2e/csi_cns_telemetry_vc_reboot.go index d8d843a851..11b20c50a3 100644 --- a/tests/e2e/csi_cns_telemetry_vc_reboot.go +++ b/tests/e2e/csi_cns_telemetry_vc_reboot.go @@ -124,7 +124,7 @@ var _ bool = ginkgo.Describe("[csi-block-vanilla] [csi-file-vanilla] "+ ginkgo.By("Waiting for claim to be in bound phase") pvc, err := fpv.WaitForPVClaimBoundPhase(client, - []*v1.PersistentVolumeClaim{pvclaim}, 2*framework.ClaimProvisionTimeout) + []*v1.PersistentVolumeClaim{pvclaim}, framework.ClaimProvisionTimeout) gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Expect(pvc).NotTo(gomega.BeEmpty()) pv := getPvFromClaim(client, pvclaim.Namespace, pvclaim.Name) @@ -201,7 +201,7 @@ var _ bool = ginkgo.Describe("[csi-block-vanilla] [csi-file-vanilla] "+ ginkgo.By("Waiting for PVC2 claim to be in bound phase") pvc2, err := fpv.WaitForPVClaimBoundPhase(client, - []*v1.PersistentVolumeClaim{pvclaim2}, 2*framework.ClaimProvisionTimeout) + []*v1.PersistentVolumeClaim{pvclaim2}, framework.ClaimProvisionTimeout) gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Expect(pvc2).NotTo(gomega.BeEmpty()) pv2 := getPvFromClaim(client, pvclaim2.Namespace, pvclaim2.Name) @@ -330,7 +330,7 @@ var _ bool = ginkgo.Describe("[csi-block-vanilla] [csi-file-vanilla] "+ ginkgo.By("Waiting for claim to be in bound phase") pvc, err := fpv.WaitForPVClaimBoundPhase(client, - []*v1.PersistentVolumeClaim{pvclaim}, 2*framework.ClaimProvisionTimeout) + []*v1.PersistentVolumeClaim{pvclaim}, framework.ClaimProvisionTimeout) gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Expect(pvc).NotTo(gomega.BeEmpty()) pv := getPvFromClaim(client, pvclaim.Namespace, pvclaim.Name) diff --git a/tests/e2e/csi_snapshot_basic.go b/tests/e2e/csi_snapshot_basic.go index ef6e6dd39d..3d2736fecc 100644 --- a/tests/e2e/csi_snapshot_basic.go +++ b/tests/e2e/csi_snapshot_basic.go @@ -42,27 +42,25 @@ import ( fss "k8s.io/kubernetes/test/e2e/framework/statefulset" admissionapi "k8s.io/pod-security-admission/api" - snapV1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" - snapclient "github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned" + snapV1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" + snapclient "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned" ) -var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { +var _ = ginkgo.Describe("[block-vanilla-snapshot] Volume Snapshot Basic Test", func() { f := framework.NewDefaultFramework("volume-snapshot") f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged var ( - client clientset.Interface - clientNewGc clientset.Interface - c clientset.Interface - namespace string - scParameters map[string]string - datastoreURL string - pandoraSyncWaitTime int - volumeOpsScale int - restConfig *restclient.Config - guestClusterRestConfig *restclient.Config - snapc *snapclient.Clientset - storagePolicyName string - clientIndex int + client clientset.Interface + c clientset.Interface + namespace string + scParameters map[string]string + datastoreURL string + pandoraSyncWaitTime int + pvclaims []*v1.PersistentVolumeClaim + volumeOpsScale int + restConfig *restclient.Config + snapc *snapclient.Clientset + nimbusGeneratedK8sVmPwd string ) ginkgo.BeforeEach(func() { @@ -78,15 +76,9 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { } //Get snapshot client using the rest config - if !guestCluster { - restConfig = getRestConfigClient() - snapc, err = snapclient.NewForConfig(restConfig) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } else { - guestClusterRestConfig = getRestConfigClientForGuestCluster(guestClusterRestConfig) - snapc, err = snapclient.NewForConfig(guestClusterRestConfig) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } + restConfig = getRestConfigClient() + snapc, err = snapclient.NewForConfig(restConfig) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) if os.Getenv(envPandoraSyncWaitTime) != "" { pandoraSyncWaitTime, err = strconv.Atoi(os.Getenv(envPandoraSyncWaitTime)) @@ -113,81 +105,54 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) c = remoteC } - - if guestCluster { - storagePolicyName = GetAndExpectStringEnvVar(envStoragePolicyNameForSharedDatastores) - svcClient, svNamespace := getSvcClientAndNamespace() - setResourceQuota(svcClient, svNamespace, rqLimit) - } - - if !guestCluster { - restConfig = getRestConfigClient() - snapc, err = snapclient.NewForConfig(restConfig) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } else { - guestClusterRestConfig = getRestConfigClientForGuestCluster(guestClusterRestConfig) - snapc, err = snapclient.NewForConfig(guestClusterRestConfig) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }) - - ginkgo.AfterEach(func() { - if guestCluster { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - framework.Logf("Collecting supervisor PVC events before performing PV/PVC cleanup") - eventList, err := svcClient.CoreV1().Events(svcNamespace).List(ctx, metav1.ListOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - for _, item := range eventList.Items { - framework.Logf(fmt.Sprintf(item.Message)) - } - } + nimbusGeneratedK8sVmPwd = GetAndExpectStringEnvVar(nimbusK8sVmPwd) }) /* - Create/Delete snapshot via k8s API using PVC (Dynamic Provisioning) - - 1. Create a storage class (eg: vsan default) and create a pvc using this sc - 2. Create a VolumeSnapshot class with snapshotter as vsphere-csi-driver and set deletionPolicy to Delete - 3. Create a volume-snapshot with labels, using the above snapshot-class and pvc (from step-1) as source - 4. Ensure the snapshot is created, verify using get VolumeSnapshot - 5. Also verify that VolumeSnapshotContent is auto-created - 6. Verify the references to pvc and volume-snapshot on this object - 7. Verify that the VolumeSnapshot has ready-to-use set to True - 8. Verify that the Restore Size set on the snapshot is same as that of the source volume size - 9. Query the snapshot from CNS side using volume id - should pass and return the snapshot entry - 10. Delete the above snapshot from k8s side using kubectl delete, run a get and ensure it is removed - 11. Also ensure that the VolumeSnapshotContent is deleted along with the - volume snapshot as the policy is delete - 12. Query the snapshot from CNS side - should return 0 entries - 13. Cleanup: Delete PVC, SC (validate they are removed) + Create/Delete snapshot via k8s API using PVC (Dynamic Provisioning) + + 1. Create a storage class (eg: vsan default) and create a pvc using this sc + 2. Create a VolumeSnapshot class with snapshotter as vsphere-csi-driver and set deletionPolicy to Delete + 3. Create a volume-snapshot with labels, using the above snapshot-class and pvc (from step-1) as source + 4. Ensure the snapshot is created, verify using get VolumeSnapshot + 5. Also verify that VolumeSnapshotContent is auto-created + 6. Verify the references to pvc and volume-snapshot on this object + 7. Verify that the VolumeSnapshot has ready-to-use set to True + 8. Verify that the Restore Size set on the snapshot is same as that of the source volume size + 9. Query the snapshot from CNS side using volume id - should pass and return the snapshot entry + 10. Delete the above snapshot from k8s side using kubectl delete, run a get and ensure it is removed + 11. Also ensure that the VolumeSnapshotContent is deleted along with the + volume snapshot as the policy is delete + 12. Query the snapshot from CNS side - should return 0 entries + 13. Cleanup: Delete PVC, SC (validate they are removed) */ - ginkgo.It("[block-vanilla-snapshot] [tkg-snapshot] Verify snapshot dynamic provisioning workflow", func() { + ginkgo.It("Verify snapshot dynamic provisioning workflow", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - var volHandle string - if vanillaCluster { - scParameters[scParamDatastoreURL] = datastoreURL - } else if guestCluster { - scParameters[svStorageClassName] = storagePolicyName - } + var storageclass *storagev1.StorageClass + var pvclaim *v1.PersistentVolumeClaim + var err error + var snapshotContentCreated = false ginkgo.By("Create storage class and PVC") - storageclass, err := createStorageClass(client, scParameters, nil, "", "", false, "") + scParameters[scParamDatastoreURL] = datastoreURL + storageclass, pvclaim, err = createPVCAndStorageClass(client, + namespace, nil, scParameters, diskSize, nil, "", false, "") gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer func() { err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() - pvclaim, persistentVolumes := createPVCAndQueryVolumeInCNS(client, namespace, nil, "", - diskSize, storageclass, true) - volHandle = persistentVolumes[0].Spec.CSI.VolumeHandle - if guestCluster { - volHandle = getVolumeIDFromSupervisorCluster(volHandle) - } + ginkgo.By("Expect claim to provision volume successfully") + pvclaims = append(pvclaims, pvclaim) + persistentvolumes, err := fpv.WaitForPVClaimBoundPhase(client, pvclaims, framework.ClaimProvisionTimeout) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + volHandle := persistentvolumes[0].Spec.CSI.VolumeHandle gomega.Expect(volHandle).NotTo(gomega.BeEmpty()) + defer func() { err := fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -195,43 +160,84 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() + // Verify using CNS Query API if VolumeID retrieved from PV is present. + ginkgo.By(fmt.Sprintf("Invoking QueryCNSVolumeWithResult with VolumeID: %s", volHandle)) + queryResult, err := e2eVSphere.queryCNSVolumeWithResult(volHandle) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(queryResult.Volumes).ShouldNot(gomega.BeEmpty()) + gomega.Expect(queryResult.Volumes[0].VolumeId.Id).To(gomega.Equal(volHandle)) + ginkgo.By("Create volume snapshot class") - volumeSnapshotClass, err := createVolumeSnapshotClass(ctx, snapc, deletionPolicy) + volumeSnapshotClass, err := snapc.SnapshotV1().VolumeSnapshotClasses().Create(ctx, + getVolumeSnapshotClassSpec(snapV1.DeletionPolicy("Delete"), nil), metav1.CreateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer func() { - if vanillaCluster { - err = snapc.SnapshotV1().VolumeSnapshotClasses().Delete(ctx, volumeSnapshotClass.Name, - metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } + err := snapc.SnapshotV1().VolumeSnapshotClasses().Delete(ctx, volumeSnapshotClass.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() - ginkgo.By("Create a dynamic volume snapshot") - volumeSnapshot, snapshotContent, snapshotCreated, - snapshotContentCreated, snapshotId, err := createDynamicVolumeSnapshot(ctx, namespace, snapc, volumeSnapshotClass, - pvclaim, volHandle, diskSize) + ginkgo.By("Create a volume snapshot") + volumeSnapshot, err := snapc.SnapshotV1().VolumeSnapshots(namespace).Create(ctx, + getVolumeSnapshotSpec(namespace, volumeSnapshotClass.Name, pvclaim.Name), metav1.CreateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.Logf("Volume snapshot name is : %s", volumeSnapshot.Name) + snapshotCreated := true + defer func() { if snapshotContentCreated { - err = deleteVolumeSnapshotContent(ctx, snapshotContent, snapc, namespace, pandoraSyncWaitTime) + framework.Logf("Deleting volume snapshot content") + err := snapc.SnapshotV1().VolumeSnapshotContents().Delete(ctx, + *volumeSnapshot.Status.BoundVolumeSnapshotContentName, metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } if snapshotCreated { framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeletedWithPandoraWait(ctx, snapc, - *volumeSnapshot.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) + err := snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, volumeSnapshot.Name, metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() - ginkgo.By("Delete dynamic volume snapshot") - snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + ginkgo.By("Verify volume snapshot is created") + volumeSnapshot, err = waitForVolumeSnapshotReadyToUse(*snapc, ctx, namespace, volumeSnapshot.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(volumeSnapshot.Status.RestoreSize.Cmp(resource.MustParse(diskSize))).To(gomega.BeZero()) + + ginkgo.By("Verify volume snapshot content is created") + snapshotContent, err := snapc.SnapshotV1().VolumeSnapshotContents().Get(ctx, + *volumeSnapshot.Status.BoundVolumeSnapshotContentName, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + snapshotContentCreated = true + gomega.Expect(*snapshotContent.Status.ReadyToUse).To(gomega.BeTrue()) + + framework.Logf("Get volume snapshot ID from snapshot handle") + snapshothandle := *snapshotContent.Status.SnapshotHandle + snapshotId := strings.Split(snapshothandle, "+")[1] + + ginkgo.By("Query CNS and check the volume snapshot entry") + err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Delete volume snapshot and verify the snapshot content is deleted") + err = snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, volumeSnapshot.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + snapshotCreated = false + + framework.Logf("Wait till the volume snapshot is deleted") + err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, *volumeSnapshot.Status.BoundVolumeSnapshotContentName) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + snapshotContentCreated = false + + ginkgo.By("Verify snapshot entry is deleted from CNS") + err = verifySnapshotIsDeletedInCNS(volHandle, snapshotId) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + framework.Logf("Deleting volume snapshot Again to check Not found error") + err = snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, volumeSnapshot.Name, metav1.DeleteOptions{}) + if !apierrors.IsNotFound(err) { + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } }) /* @@ -239,19 +245,19 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { 1. Create a storage class (eg: vsan default) and create a pvc using this sc 2. The volumesnapshotclass is set to delete 3. Create a VolumeSnapshotContent using snapshot-handle - a. get snapshotHandle by referring to an existing volume snapshot - b. this snapshot will be created dynamically, and the snapshot-content that is - created by that will be referred to get the snapshotHandle + a. get snapshotHandle by referring to an existing volume snapshot + b. this snapshot will be created dynamically, and the snapshot-content that is + created by that will be referred to get the snapshotHandle 4. Create a volume snapshot using source set to volumeSnapshotContentName above 5. Ensure the snapshot is created, verify using get VolumeSnapshot 6. Verify the restoreSize on the snapshot and the snapshotcontent is set to same as that of the pvcSize 7. Delete the above snapshot, run a get from k8s side and ensure its removed 8. Run QuerySnapshot from CNS side, the backend snapshot should be deleted 9. Also ensure that the VolumeSnapshotContent is deleted along with the - volume snapshot as the policy is delete + volume snapshot as the policy is delete 10. Cleanup the pvc */ - ginkgo.It("[block-vanilla-snapshot] Verify snapshot static provisioning through K8s API workflow", func() { + ginkgo.It("Verify snapshot static provisioning through K8s API workflow", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() var storageclass *storagev1.StorageClass @@ -312,16 +318,22 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { snapshotCreated := true defer func() { + if snapshotContentCreated { + framework.Logf("Deleting volume snapshot content") + err := snapc.SnapshotV1().VolumeSnapshotContents().Delete(ctx, + *volumeSnapshot.Status.BoundVolumeSnapshotContentName, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } + if snapshotCreated { framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) + err := snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, volumeSnapshot.Name, + metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) } - if snapshotContentCreated { - framework.Logf("Deleting volume snapshot content") - deleteVolumeSnapshotContentWithPandoraWait(ctx, snapc, - *volumeSnapshot.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) - } + ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to allow CNS to sync with pandora", pandoraSyncWaitTime)) + time.Sleep(time.Duration(pandoraSyncWaitTime) * time.Second) }() ginkgo.By("Verify volume snapshot is created") @@ -341,7 +353,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { snapshotId := strings.Split(snapshothandle, "+")[1] ginkgo.By("Query CNS and check the volume snapshot entry") - err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId, false) + err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By(fmt.Sprintf("Creating volume snapshot content by snapshotHandle %s", snapshothandle)) @@ -359,8 +371,9 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { defer func() { if snapshotContentCreated2 { - deleteVolumeSnapshotContentWithPandoraWait(ctx, snapc, - snapshotContentNew.ObjectMeta.Name, pandoraSyncWaitTime) + err := snapc.SnapshotV1().VolumeSnapshotContents().Delete(ctx, + snapshotContentNew.ObjectMeta.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() @@ -374,7 +387,8 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { defer func() { if snapshotCreated2 { - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, "static-vs", pandoraSyncWaitTime) + err = snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, "static-vs", metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() @@ -385,7 +399,8 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { framework.Logf("Snapshot details is %+v", staticSnapshot) ginkgo.By("Deleted volume snapshot is created above") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, staticSnapshot.Name, pandoraSyncWaitTime) + err = snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, staticSnapshot.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) snapshotCreated2 = false framework.Logf("Wait till the volume snapshot is deleted") @@ -406,11 +421,11 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { 8. Delete the above snapshot, run a get from k8s side and ensure its removed 9. Run QuerySnapshot from CNS side, the backend snapshot should be deleted 10. Also ensure that the VolumeSnapshotContent is deleted along with the - volume snapshot as the policy is delete + volume snapshot as the policy is delete 11. The snapshot that was created via CNS in step-2 should be deleted as part of k8s snapshot delete 12. Delete the pvc */ - ginkgo.It("[block-vanilla-snapshot] Verify snapshot static provisioning via CNS", func() { + ginkgo.It("Verify snapshot static provisioning via CNS", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() var storageclass *storagev1.StorageClass @@ -475,7 +490,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { }() ginkgo.By("Query CNS and check the volume snapshot entry") - err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId, false) + err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId) gomega.Expect(err).NotTo(gomega.HaveOccurred()) snapshotHandle := volHandle + "+" + snapshotId @@ -494,8 +509,9 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { defer func() { if snapshotContentCreated2 { - deleteVolumeSnapshotContentWithPandoraWait(ctx, snapc, - snapshotContentNew.ObjectMeta.Name, pandoraSyncWaitTime) + err := snapc.SnapshotV1().VolumeSnapshotContents().Delete(ctx, + snapshotContentNew.ObjectMeta.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() @@ -521,7 +537,8 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { framework.Logf("Snapshot details is %+v", staticSnapshot) ginkgo.By("Deleted volume snapshot is created above") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, staticSnapshot.Name, pandoraSyncWaitTime) + err = snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, staticSnapshot.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) snapshotCreated1 = false snapshotCreated2 = false @@ -536,9 +553,9 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { 1. Create a storage class (eg: vsan default) and create a pvc using this sc 2. The volumesnapshotclass is set to delete 3. Create a VolumeSnapshotContent using snapshot-handle with deletion policy Retain - a. get snapshotHandle by referring to an existing volume snapshot - b. this snapshot will be created dynamically, and the snapshot-content that is - created by that will be referred to get the snapshotHandle + a. get snapshotHandle by referring to an existing volume snapshot + b. this snapshot will be created dynamically, and the snapshot-content that is + created by that will be referred to get the snapshotHandle 4. Create a volume snapshot using source set to volumeSnapshotContentName above 5. Ensure the snapshot is created, verify using get VolumeSnapshot 6. Verify the restoreSize on the snapshot and the snapshotcontent is set to same as that of the pvcSize @@ -550,7 +567,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { 12. Delete volume snapshot content 2 13. Cleanup the pvc, volume snapshot class and storage class */ - ginkgo.It("[block-vanilla-snapshot] Verify snapshot static provisioning with deletion policy Retain", func() { + ginkgo.It("Verify snapshot static provisioning with deletion policy Retain", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() var storageclass *storagev1.StorageClass @@ -612,17 +629,16 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { defer func() { if snapshotContentCreated { framework.Logf("Deleting volume snapshot content") - deleteVolumeSnapshotContentWithPandoraWait(ctx, snapc, - *volumeSnapshot.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, *volumeSnapshot.Status.BoundVolumeSnapshotContentName) + err := snapc.SnapshotV1().VolumeSnapshotContents().Delete(ctx, + *volumeSnapshot.Status.BoundVolumeSnapshotContentName, metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } if snapshotCreated { framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) + err := snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, volumeSnapshot.Name, + metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() @@ -643,7 +659,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { snapshotId := strings.Split(snapshothandle, "+")[1] ginkgo.By("Query CNS and check the volume snapshot entry") - err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId, false) + err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ns, err := framework.CreateTestingNS(f.BaseName, client, nil) @@ -664,8 +680,9 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { defer func() { if snapshotContentCreated2 { - deleteVolumeSnapshotContentWithPandoraWait(ctx, snapc, - snapshotContentNew.ObjectMeta.Name, pandoraSyncWaitTime) + err := snapc.SnapshotV1().VolumeSnapshotContents().Delete(ctx, + snapshotContentNew.ObjectMeta.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() @@ -702,8 +719,9 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { gomega.Expect(snapshotContentGetResult.Name).Should(gomega.Equal(snapshotContent2.Name)) framework.Logf("Snapshotcontent name is %s", snapshotContentGetResult.ObjectMeta.Name) - framework.Logf("Deleting volume snapshot 1") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) + framework.Logf("Deleting volume snapshot 1 ") + err = snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, volumeSnapshot.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) snapshotCreated = false framework.Logf("Wait till the volume snapshot is deleted") @@ -712,8 +730,9 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { snapshotContentCreated = false framework.Logf("Delete volume snapshot content 2") - deleteVolumeSnapshotContentWithPandoraWait(ctx, snapc, - snapshotContentGetResult.ObjectMeta.Name, pandoraSyncWaitTime) + err = snapc.SnapshotV1().VolumeSnapshotContents().Delete(ctx, + snapshotContentGetResult.ObjectMeta.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) snapshotContentCreated2 = false }) @@ -730,7 +749,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { 9. Query the Snasphot from CNS side using the volumeId 10. Cleanup the snapshot and delete the volume */ - ginkgo.It("[block-vanilla-snapshot] Verify snapshot static provisioning with deletion policy Retain - test2", func() { + ginkgo.It("Verify snapshot static provisioning with deletion policy Retain - test2", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() var storageclass *storagev1.StorageClass @@ -793,16 +812,15 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { defer func() { if snapshotCreated { framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, snapshot1.Name, pandoraSyncWaitTime) + err := snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, snapshot1.Name, + metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) } if snapshotContentCreated { framework.Logf("Deleting volume snapshot content") - deleteVolumeSnapshotContentWithPandoraWait(ctx, snapc, - contentName, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, contentName) + err := snapc.SnapshotV1().VolumeSnapshotContents().Delete(ctx, + contentName, metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() @@ -825,12 +843,15 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { snapshotId := strings.Split(snapshothandle, "+")[1] ginkgo.By("Query CNS and check the volume snapshot entry") - err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId, false) + err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId) gomega.Expect(err).NotTo(gomega.HaveOccurred()) framework.Logf("Deleting volume snapshot 1 " + snapshot1.Name) - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, snapshot1.Name, pandoraSyncWaitTime) + err = snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, snapshot1.Name, + metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) snapshotCreated = false + time.Sleep(kubeAPIRecoveryTime) _, err = snapc.SnapshotV1().VolumeSnapshots(namespace).Get(ctx, snapshot1.Name, metav1.GetOptions{}) @@ -841,7 +862,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Verify snapshot entry is deleted from CNS") - err = verifySnapshotIsDeletedInCNS(volHandle, snapshotId, false) + err = verifySnapshotIsDeletedInCNS(volHandle, snapshotId) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Verify volume snapshot content is not deleted") @@ -852,8 +873,9 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { framework.Logf("Snapshotcontent name is %s", snapshotContentGetResult.Name) framework.Logf("Delete volume snapshot content") - deleteVolumeSnapshotContentWithPandoraWait(ctx, snapc, - snapshotContentGetResult.Name, pandoraSyncWaitTime) + err = snapc.SnapshotV1().VolumeSnapshotContents().Delete(ctx, + snapshotContentGetResult.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) snapshotContentCreated = false framework.Logf("Wait till the volume snapshot content is deleted") @@ -869,16 +891,14 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { snapshots as source, use the same sc 4. Ensure the pvc gets provisioned and is Bound 5. Attach the pvc to a pod and ensure data from snapshot is available - (file that was written in step.1 should be available) + (file that was written in step.1 should be available) 6. And also write new data to the restored volumes and it should succeed 7. Delete the snapshots and pvcs/pods created in steps 1,2,3 8. Continue to write new data to the restore volumes and it should succeed 9. Create new snapshots on restore volume and verify it succeeds 10. Run cleanup: Delete snapshots, restored-volumes, pods */ - - ginkgo.It("[block-vanilla-snapshot] Volume restore using snapshot a dynamic snapshot b "+ - "pre-provisioned snapshot", func() { + ginkgo.It("Volume restore using snapshot a dynamic snapshot b pre-provisioned snapshot", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() var storageclass *storagev1.StorageClass @@ -940,17 +960,16 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { defer func() { if snapshotContentCreated { framework.Logf("Deleting volume snapshot content") - deleteVolumeSnapshotContentWithPandoraWait(ctx, snapc, - *volumeSnapshot.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, *volumeSnapshot.Status.BoundVolumeSnapshotContentName) + err := snapc.SnapshotV1().VolumeSnapshotContents().Delete(ctx, + *volumeSnapshot.Status.BoundVolumeSnapshotContentName, metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } if snapshotCreated { framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) + err := snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, volumeSnapshot.Name, + metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() @@ -971,7 +990,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { snapshotId := strings.Split(snapshothandle, "+")[1] ginkgo.By("Query CNS and check the volume snapshot entry") - err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId, false) + err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By(fmt.Sprintf("Creating volume snapshot content by snapshotHandle %s", snapshothandle)) @@ -989,8 +1008,9 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { defer func() { if snapshotContentCreated2 { - deleteVolumeSnapshotContentWithPandoraWait(ctx, snapc, - snapshotContentNew.ObjectMeta.Name, pandoraSyncWaitTime) + err := snapc.SnapshotV1().VolumeSnapshotContents().Delete(ctx, + snapshotContentNew.ObjectMeta.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() @@ -1004,7 +1024,8 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { defer func() { if snapshotCreated2 { - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, "static-vs", pandoraSyncWaitTime) + err = snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, "static-vs", metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() @@ -1137,17 +1158,16 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { defer func() { if snapshotContentCreated3 { framework.Logf("Deleting volume snapshot content") - deleteVolumeSnapshotContentWithPandoraWait(ctx, snapc, - *volumeSnapshot.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, *volumeSnapshot.Status.BoundVolumeSnapshotContentName) + err := snapc.SnapshotV1().VolumeSnapshotContents().Delete(ctx, + *volumeSnapshot.Status.BoundVolumeSnapshotContentName, metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } if snapshotCreated3 { framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) + err := snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, volumeSnapshot.Name, + metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() @@ -1169,11 +1189,12 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { snapshotId3 := strings.Split(snapshothandle3, "+")[1] ginkgo.By("Query CNS and check the volume snapshot entry") - err = verifySnapshotIsCreatedInCNS(volHandle2, snapshotId3, false) + err = verifySnapshotIsCreatedInCNS(volHandle2, snapshotId3) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Deleted volume snapshot is created above") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot3.Name, pandoraSyncWaitTime) + err = snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, volumeSnapshot3.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) snapshotCreated3 = false framework.Logf("Wait till the volume snapshot is deleted") @@ -1182,7 +1203,8 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { snapshotContentCreated3 = false ginkgo.By("Deleted volume snapshot is created above") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, staticSnapshot.Name, pandoraSyncWaitTime) + err = snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, staticSnapshot.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) snapshotCreated2 = false framework.Logf("Wait till the volume snapshot is deleted") @@ -1191,7 +1213,8 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { snapshotContentCreated2 = false ginkgo.By("Delete volume snapshot and verify the snapshot content is deleted") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) + err = snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, volumeSnapshot.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) snapshotCreated = false framework.Logf("Wait till the volume snapshot is deleted") @@ -1201,61 +1224,48 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { snapshotContentCreated = false ginkgo.By("Verify snapshot entry is deleted from CNS") - err = verifySnapshotIsDeletedInCNS(volHandle, snapshotId, false) + err = verifySnapshotIsDeletedInCNS(volHandle, snapshotId) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) /* - - Snapshot creation and restore workflow verification with xfs filesystem - 1. Create a storage class with fstype set to XFS and create a pvc using this sc - 2. Create a pod which uses above PVC - 3. Create file1.txt data at mountpath - 4. Create a VolumeSnapshotClass with snapshotter as vsphere-csi-driver and set deletionPolicy to Delete - 5. Create a VolumeSnapshot with labels, using the above snapshot-class and pvc (from step-1) as source - 6. Ensure the snapshot is created, verify using get VolumeSnapshot - 7. Also verify that VolumeSnapshotContent is auto created - 8. Verify that the VolumeSnapshot has ReadyToUse set to True - 9. Query the snapshot from CNS side using volume id to ensure that snapshot is created - 10. Create new PVC using above snapshot as source (restore operation) - 11. Ensure the PVC gets provisioned and is Bound - 12. Attach this PVC to a pod on the same node where source volume is mounted - 13. Ensure that file1.txt from snapshot is available - 14. And write new file file2.txt to the restored volume and it should succeed - 15. Delete the VolumeSnapshot, PVCs and pods created in above steps and ensure it is removed - 16. Query the snapshot from CNS side - it shouldn't be available - 17. Delete SC and VolumeSnapshotClass + Volume restore using snapshot on a different storageclass + 1. Create a sc with thin-provisioned spbm policy, create a pvc and attach the pvc to a pod + 2. Create a dynamically provisioned snapshots using this pvc + 3. create another sc pointing to a different spbm policy (say thick) + 4. Run a restore workflow by giving a different storageclass in the pvc spec + 5. the new storageclass would point to a thick provisioned spbm plocy, + while the source pvc was created usig thin provisioned psp-operatorlicy + 6. cleanup spbm policies, sc's, pvc's */ - ginkgo.It("[block-vanilla-snapshot] [tkg-snapshot] Volume snapshot creation and restoration workflow "+ - "with xfs filesystem", func() { + ginkgo.It("Volume restore using snapshot on a different storageclass", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() + var storageclass *storagev1.StorageClass + var pvclaim *v1.PersistentVolumeClaim + var pvclaims []*v1.PersistentVolumeClaim + var err error + var snapshotContentCreated = false + var snapshotCreated = false - if vanillaCluster { - scParameters[scParamDatastoreURL] = datastoreURL - } else if guestCluster { - scParameters[svStorageClassName] = storagePolicyName - } - - scParameters[scParamFsType] = xfsFSType - - ginkgo.By("Create storage class with xfs filesystem and create PVC") - storageclass, err := createStorageClass(client, scParameters, nil, "", "", true, "") + ginkgo.By("Create storage class and PVC") + scParameters[scParamDatastoreURL] = datastoreURL + storageclass, pvclaim, err = createPVCAndStorageClass(client, + namespace, nil, scParameters, diskSize, nil, "", false, "") gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pvclaim, persistentVolumes := createPVCAndQueryVolumeInCNS(client, namespace, nil, "", diskSize, storageclass, true) - volHandle := persistentVolumes[0].Spec.CSI.VolumeHandle - if guestCluster { - volHandle = getVolumeIDFromSupervisorCluster(volHandle) - } - gomega.Expect(volHandle).NotTo(gomega.BeEmpty()) - defer func() { - err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, - *metav1.NewDeleteOptions(0)) + err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() + ginkgo.By("Expect claim to provision volume successfully") + pvclaims = append(pvclaims, pvclaim) + persistentvolumes, err := fpv.WaitForPVClaimBoundPhase(client, pvclaims, framework.ClaimProvisionTimeout) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + volHandle := persistentvolumes[0].Spec.CSI.VolumeHandle + gomega.Expect(volHandle).NotTo(gomega.BeEmpty()) + defer func() { err := fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -1270,85 +1280,83 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { gomega.Expect(queryResult.Volumes).ShouldNot(gomega.BeEmpty()) gomega.Expect(queryResult.Volumes[0].VolumeId.Id).To(gomega.Equal(volHandle)) - // Create a Pod to use this PVC, and verify volume has been attached - ginkgo.By("Creating pod to attach PV to the node") - pod, err := createPod(client, namespace, nil, []*v1.PersistentVolumeClaim{pvclaim}, false, - execCommand) + ginkgo.By("Create volume snapshot class") + volumeSnapshotClass, err := snapc.SnapshotV1().VolumeSnapshotClasses().Create(ctx, + getVolumeSnapshotClassSpec(snapV1.DeletionPolicy("Delete"), nil), metav1.CreateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) defer func() { - // Delete POD - ginkgo.By(fmt.Sprintf("Deleting the pod %s in namespace %s", pod.Name, namespace)) - err = fpod.DeletePodWithWait(client, pod) + err := snapc.SnapshotV1().VolumeSnapshotClasses().Delete(ctx, volumeSnapshotClass.Name, + metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() - // Verify volume is attached to the node - var vmUUID string - nodeName := pod.Spec.NodeName - - if vanillaCluster { - vmUUID = getNodeUUID(ctx, client, pod.Spec.NodeName) - } else if guestCluster { - vmUUID, err = getVMUUIDFromNodeName(pod.Spec.NodeName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - ginkgo.By(fmt.Sprintf("Verify volume: %s is attached to the node: %s", volHandle, nodeName)) - isDiskAttached, err := e2eVSphere.isVolumeAttachedToVM(client, volHandle, vmUUID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(isDiskAttached).To(gomega.BeTrue(), "Volume is not attached to the node") - - // Verify filesystem used to mount volume inside pod is xfs - ginkgo.By("Verify that filesystem type is xfs as expected") - _, err = framework.LookForStringInPodExec(namespace, pod.Name, []string{"/bin/cat", "/mnt/volume1/fstype"}, - xfsFSType, time.Minute) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - // Create file1.txt at mountpath inside pod - ginkgo.By(fmt.Sprintf("Creating file file1.txt at mountpath inside pod: %v", pod.Name)) - data1 := "This file file1.txt is written by Pod1" - filePath1 := "/mnt/volume1/file1.txt" - writeDataOnFileFromPod(namespace, pod.Name, filePath1, data1) - - ginkgo.By("Create volume snapshot class") - volumeSnapshotClass, err := createVolumeSnapshotClass(ctx, snapc, deletionPolicy) + ginkgo.By("Create a volume snapshot") + volumeSnapshot, err := snapc.SnapshotV1().VolumeSnapshots(namespace).Create(ctx, + getVolumeSnapshotSpec(namespace, volumeSnapshotClass.Name, pvclaim.Name), metav1.CreateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - if vanillaCluster { - err = snapc.SnapshotV1().VolumeSnapshotClasses().Delete(ctx, volumeSnapshotClass.Name, metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }() + framework.Logf("Volume snapshot name is : %s", volumeSnapshot.Name) - ginkgo.By("Create a dynamic volume snapshot") - volumeSnapshot, snapshotContent, snapshotCreated, - snapshotContentCreated, snapshotId, err := createDynamicVolumeSnapshot(ctx, namespace, snapc, volumeSnapshotClass, - pvclaim, volHandle, diskSize) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) defer func() { if snapshotContentCreated { - err = deleteVolumeSnapshotContent(ctx, snapshotContent, snapc, namespace, pandoraSyncWaitTime) + framework.Logf("Deleting volume snapshot content") + err := snapc.SnapshotV1().VolumeSnapshotContents().Delete(ctx, + *volumeSnapshot.Status.BoundVolumeSnapshotContentName, metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } if snapshotCreated { framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeletedWithPandoraWait(ctx, snapc, - *volumeSnapshot.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) + err := snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, volumeSnapshot.Name, + metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() - ginkgo.By("Restore snapshot to new PVC") - pvclaim2, persistentVolumes2, _ := verifyVolumeRestoreOperation(ctx, client, - namespace, storageclass, volumeSnapshot, false) - volHandle2 := persistentVolumes2[0].Spec.CSI.VolumeHandle - if guestCluster { - volHandle2 = getVolumeIDFromSupervisorCluster(volHandle2) - } + ginkgo.By("Verify volume snapshot is created") + volumeSnapshot, err = waitForVolumeSnapshotReadyToUse(*snapc, ctx, namespace, volumeSnapshot.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + snapshotCreated = true + gomega.Expect(volumeSnapshot.Status.RestoreSize.Cmp(resource.MustParse(diskSize))).To(gomega.BeZero()) + + ginkgo.By("Verify volume snapshot content is created") + snapshotContent, err := snapc.SnapshotV1().VolumeSnapshotContents().Get(ctx, + *volumeSnapshot.Status.BoundVolumeSnapshotContentName, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + snapshotContentCreated = true + gomega.Expect(*snapshotContent.Status.ReadyToUse).To(gomega.BeTrue()) + + framework.Logf("Get volume snapshot ID from snapshot handle") + snapshothandle := *snapshotContent.Status.SnapshotHandle + snapshotId := strings.Split(snapshothandle, "+")[1] + + ginkgo.By("Query CNS and check the volume snapshot entry") + err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + scParameters1 := make(map[string]string) + + scParameters1[scParamStoragePolicyName] = "Management Storage Policy - Regular" + + curtime := time.Now().Unix() + randomValue := rand.Int() + val := strconv.FormatInt(int64(randomValue), 10) + val = string(val[1:3]) + curtimestring := strconv.FormatInt(curtime, 10) + scName := "snapshot" + curtimestring + val + storageclass1, err := createStorageClass(client, scParameters1, nil, "", "", false, scName) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + pvcSpec2 := getPersistentVolumeClaimSpecWithDatasource(namespace, diskSize, storageclass1, nil, + v1.ReadWriteOnce, volumeSnapshot.Name, snapshotapigroup) + + pvclaim2, err := fpv.CreatePVC(client, namespace, pvcSpec2) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + persistentvolume2, err := fpv.WaitForPVClaimBoundPhase(client, []*v1.PersistentVolumeClaim{pvclaim2}, + framework.ClaimProvisionTimeout) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + volHandle2 := persistentvolume2[0].Spec.CSI.VolumeHandle gomega.Expect(volHandle2).NotTo(gomega.BeEmpty()) defer func() { @@ -1358,198 +1366,40 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() - ginkgo.By("Creating a pod to attach restored PV on the same node where earlier pod is running") - nodeSelector := make(map[string]string) - nodeSelector["kubernetes.io/hostname"] = nodeName - pod2, err := createPod(client, namespace, nodeSelector, []*v1.PersistentVolumeClaim{pvclaim2}, false, - execCommand) + ginkgo.By("Delete volume snapshot and verify the snapshot content is deleted") + err = snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, volumeSnapshot.Name, metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + snapshotCreated = false - defer func() { - // Delete POD - ginkgo.By(fmt.Sprintf("Deleting the pod %s in namespace %s", pod2.Name, namespace)) - err = fpod.DeletePodWithWait(client, pod2) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Verify that new pod is scheduled on same node where earlier pod is running") - nodeName2 := pod2.Spec.NodeName - gomega.Expect(nodeName == nodeName2).To(gomega.BeTrue(), "Pod is not scheduled on expected node") - - if vanillaCluster { - vmUUID = getNodeUUID(ctx, client, pod2.Spec.NodeName) - } else if guestCluster { - vmUUID, err = getVMUUIDFromNodeName(pod2.Spec.NodeName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - ginkgo.By(fmt.Sprintf("Verify volume: %s is attached to the node: %s", volHandle2, nodeName2)) - isDiskAttached, err = e2eVSphere.isVolumeAttachedToVM(client, volHandle2, vmUUID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(isDiskAttached).To(gomega.BeTrue(), "Volume is not attached to the node") - - // Verify filesystem used to mount volume inside pod is xfs - ginkgo.By("Verify that filesystem type is xfs inside pod which is using restored PVC") - _, err = framework.LookForStringInPodExec(namespace, pod2.Name, []string{"/bin/cat", "/mnt/volume1/fstype"}, - xfsFSType, time.Minute) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - // Ensure that file1.txt is available as expected on the restored PVC - ginkgo.By("Verify that file1.txt data is available as part of snapshot") - output := readFileFromPod(namespace, pod2.Name, filePath1) - gomega.Expect(output == data1+"\n").To(gomega.BeTrue(), - "Pod2 is not able to read file1.txt written before snapshot creation") - - // Create new file file2.txt at mountpath inside pod - ginkgo.By(fmt.Sprintf("Creating file file2.txt at mountpath inside pod: %v", pod2.Name)) - data2 := "This file file2.txt is written by Pod2" - filePath2 := "/mnt/volume1/file2.txt" - writeDataOnFileFromPod(namespace, pod2.Name, filePath2, data2) - - ginkgo.By("Verify that file2.txt data can be successfully read") - output = readFileFromPod(namespace, pod2.Name, filePath2) - gomega.Expect(output == data2+"\n").To(gomega.BeTrue(), "Pod2 is not able to read file2.txt") - - ginkgo.By("Delete dynamic volume snapshot") - snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - - /* - Volume restore using snapshot on a different storageclass - 1. Create a sc with thin-provisioned spbm policy, create a pvc and attach the pvc to a pod - 2. Create a dynamically provisioned snapshots using this pvc - 3. create another sc pointing to a different spbm policy (say thick) - 4. Run a restore workflow by giving a different storageclass in the pvc spec - 5. the new storageclass would point to a thick provisioned spbm plocy, - while the source pvc was created usig thin provisioned psp-operatorlicy - 6. cleanup spbm policies, sc's, pvc's - */ - ginkgo.It("[block-vanilla-snapshot] [tkg-snapshot] Volume restore using snapshot on a different storageclass", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - if vanillaCluster { - scParameters[scParamDatastoreURL] = datastoreURL - } else if guestCluster { - scParameters[svStorageClassName] = storagePolicyName - } - - ginkgo.By("Create storage class and PVC") - storageclass, err := createStorageClass(client, scParameters, nil, "", "", false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - pvclaim, persistentVolumes := createPVCAndQueryVolumeInCNS(client, namespace, nil, "", - diskSize, storageclass, true) - volHandle := persistentVolumes[0].Spec.CSI.VolumeHandle - if guestCluster { - volHandle = getVolumeIDFromSupervisorCluster(volHandle) - } - gomega.Expect(volHandle).NotTo(gomega.BeEmpty()) - - defer func() { - err := fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Create volume snapshot class") - volumeSnapshotClass, err := createVolumeSnapshotClass(ctx, snapc, deletionPolicy) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - if vanillaCluster { - err = snapc.SnapshotV1().VolumeSnapshotClasses().Delete(ctx, volumeSnapshotClass.Name, - metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }() - - ginkgo.By("Create a dynamic volume snapshot") - volumeSnapshot, snapshotContent, snapshotCreated, - snapshotContentCreated, snapshotId, err := createDynamicVolumeSnapshot(ctx, namespace, snapc, volumeSnapshotClass, - pvclaim, volHandle, diskSize) + framework.Logf("Wait till the volume snapshot is deleted") + err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, + *volumeSnapshot.Status.BoundVolumeSnapshotContentName) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - if snapshotContentCreated { - err = deleteVolumeSnapshotContent(ctx, snapshotContent, snapc, namespace, pandoraSyncWaitTime) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - if snapshotCreated { - framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeletedWithPandoraWait(ctx, snapc, - *volumeSnapshot.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }() - - scParameters1 := make(map[string]string) - scParameters1[scParamStoragePolicyName] = "Management Storage Policy - Regular" - - curtime := time.Now().Unix() - randomValue := rand.Int() - val := strconv.FormatInt(int64(randomValue), 10) - val = string(val[1:3]) - curtimestring := strconv.FormatInt(curtime, 10) - scName := "snapshot" + curtimestring + val - var storageclass1 *storagev1.StorageClass - - if vanillaCluster { - storageclass1, err = createStorageClass(client, scParameters1, nil, "", "", false, scName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } else if guestCluster { - scName = GetAndExpectStringEnvVar(envStoragePolicyNameForSharedDatastores2) - storageclass1, err = client.StorageV1().StorageClasses().Get(ctx, scName, metav1.GetOptions{}) - } - - pvclaim2, persistentVolumes2, _ := verifyVolumeRestoreOperation(ctx, client, - namespace, storageclass1, volumeSnapshot, false) - volHandle2 := persistentVolumes2[0].Spec.CSI.VolumeHandle - if guestCluster { - volHandle2 = getVolumeIDFromSupervisorCluster(volHandle2) - } - gomega.Expect(volHandle2).NotTo(gomega.BeEmpty()) - - defer func() { - err := fpv.DeletePersistentVolumeClaim(client, pvclaim2.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volHandle2) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() + snapshotContentCreated = false - ginkgo.By("Delete dynamic volume snapshot") - snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + ginkgo.By("Verify snapshot entry is deleted from CNS") + err = verifySnapshotIsDeletedInCNS(volHandle, snapshotId) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) /* - Delete the namespace hosting the pvcs and volume-snapshots and - recover the data using snapshot-content - 1. Create a sc, create a pvc using this sc on a non-default namesapce - 2. create a dynamic snapshot using the pvc as source - 3. verify volume-snapshot is ready-to-use and volumesnapshotcontent is auto-created - 4. Delete the non-default namespace which should delete all namespaced objects such as pvc, volume-snapshot - 5. Ensure the volumesnapshotcontent object which is cluster-scoped does not get deleted - 6. Also verify we can re-provision a snapshot and restore a volume using - this object on another namespace (could be default too) - 7. This VolumeSnapshotContent is dynamically created. we can't use it for pre-provisioned snapshot. - we would be creating a new VolumeSnapshotContent pointing to the same snapshotHandle - and then create a new VolumeSnapshot to bind with it - 8. Ensure the pvc with source as snapshot creates successfully and is bound - 9. Cleanup the snapshot, pvcs and ns + Delete the namespace hosting the pvcs and volume-snapshots and + recover the data using snapshot-content + 1. Create a sc, create a pvc using this sc on a non-default namesapce + 2. create a dynamic snapshot using the pvc as source + 3. verify volume-snapshot is ready-to-use and volumesnapshotcontent is auto-created + 4. Delete the non-default namespace which should delete all namespaced objects such as pvc, volume-snapshot + 5. Ensure the volumesnapshotcontent object which is cluster-scoped does not get deleted + 6. Also verify we can re-provision a snapshot and restore a volume using + this object on another namespace (could be default too) + 7. This VolumeSnapshotContent is dynamically created. we can't use it for pre-provisioned snapshot. + we would be creating a new VolumeSnapshotContent pointing to the same snapshotHandle + and then create a new VolumeSnapshot to bind with it + 8. Ensure the pvc with source as snapshot creates successfully and is bound + 9. Cleanup the snapshot, pvcs and ns */ - ginkgo.It("[block-vanilla-snapshot] Delete the namespace hosting the pvcs and "+ - "volume-snapshots and recover the data using snapshot-content", func() { + ginkgo.It("Delete the namespace hosting the pvcs and volume-snapshots and "+ + "recover the data using snapshot-content", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() var storageclass *storagev1.StorageClass @@ -1557,6 +1407,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { var pvclaims []*v1.PersistentVolumeClaim var err error var snapshotContentCreated = false + var snapshotCreated = false ginkgo.By("Creating new namespace for the test") namespace1, err := framework.CreateTestingNS(f.BaseName, client, nil) @@ -1613,7 +1464,6 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) defer func() { - ginkgo.By("Delete volume snapshot class") err := snapc.SnapshotV1().VolumeSnapshotClasses().Delete(ctx, volumeSnapshotClass.Name, metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -1624,38 +1474,29 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { getVolumeSnapshotSpec(newNamespaceName, volumeSnapshotClass.Name, pvclaim.Name), metav1.CreateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) framework.Logf("Volume snapshot name is : %s", snapshot1.Name) - snapshotCreated := true - var snapshotContent1Name = "" + defer func() { if !isNamespaceDeleted { - if snapshotCreated { - framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, snapshot1.Name, pandoraSyncWaitTime) - } if snapshotContentCreated { framework.Logf("Deleting volume snapshot content") - deleteVolumeSnapshotContentWithPandoraWait(ctx, snapc, - *snapshot1.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot content is deleted") - err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, *snapshot1.Status.BoundVolumeSnapshotContentName) + err := snapc.SnapshotV1().VolumeSnapshotContents().Delete(ctx, + *snapshot1.Status.BoundVolumeSnapshotContentName, metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } - } - if snapshotContentCreated { - framework.Logf("Deleting volume snapshot content") - deleteVolumeSnapshotContentWithPandoraWait(ctx, snapc, - snapshotContent1Name, pandoraSyncWaitTime) - framework.Logf("Wait till the volume snapshot content is deleted") - err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, *snapshot1.Status.BoundVolumeSnapshotContentName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + if snapshotCreated { + framework.Logf("Deleting volume snapshot") + err := snapc.SnapshotV1().VolumeSnapshots(newNamespaceName).Delete(ctx, + snapshot1.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } } }() ginkgo.By("Verify volume snapshot is Ready to use") snapshot1_updated, err := waitForVolumeSnapshotReadyToUse(*snapc, ctx, newNamespaceName, snapshot1.Name) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + snapshotCreated = true gomega.Expect(snapshot1_updated.Status.RestoreSize.Cmp(resource.MustParse(diskSize))).To(gomega.BeZero()) ginkgo.By("Verify volume snapshot content is created") @@ -1663,7 +1504,6 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { *snapshot1_updated.Status.BoundVolumeSnapshotContentName, metav1.GetOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) snapshotContentCreated = true - snapshotContent1Name = snapshotContent1.Name gomega.Expect(*snapshotContent1.Status.ReadyToUse).To(gomega.BeTrue()) @@ -1672,7 +1512,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { snapshotId := strings.Split(snapshothandle, "+")[1] ginkgo.By("Query CNS and check the volume snapshot entry") - err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId, false) + err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Delete namespace") @@ -1687,7 +1527,6 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { _, err = snapc.SnapshotV1().VolumeSnapshots(newNamespaceName).Get(ctx, snapshot1.Name, metav1.GetOptions{}) gomega.Expect(err).To(gomega.HaveOccurred()) - snapshotCreated = false _, err = snapc.SnapshotV1().VolumeSnapshotContents().Get(ctx, snapshotContent1.Name, metav1.GetOptions{}) @@ -1715,8 +1554,9 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { defer func() { if snapshotContentCreated2 { - deleteVolumeSnapshotContentWithPandoraWait(ctx, snapc, - snapshotContent2_updated.ObjectMeta.Name, pandoraSyncWaitTime) + err := snapc.SnapshotV1().VolumeSnapshotContents().Delete(ctx, + snapshotContent2_updated.ObjectMeta.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() @@ -1730,9 +1570,6 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { defer func() { if snapshotCreated2 { - framework.Logf("Deleting volume snapshot2") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot2.Name, pandoraSyncWaitTime) - err = e2eVSphere.deleteVolumeSnapshotInCNS(volHandle, volumeSnapshot2.Name) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } @@ -1759,15 +1596,6 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { gomega.Expect(volHandle2).NotTo(gomega.BeEmpty()) defer func() { - if snapshotCreated2 { - framework.Logf("Deleting volume snapshot2") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace2Name, volumeSnapshot2.Name, pandoraSyncWaitTime) - - err = e2eVSphere.deleteVolumeSnapshotInCNS(volHandle, volumeSnapshot2.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - snapshotCreated2 = false - } - framework.Logf("Deleting restored PVC") err := fpv.DeletePersistentVolumeClaim(client, pvclaim2.Name, namespace2Name) gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = e2eVSphere.waitForCNSVolumeToBeDeleted(volHandle2) @@ -1775,28 +1603,34 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { }() ginkgo.By("Deleted volume snapshot is created above") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace2Name, volumeSnapshot2.Name, pandoraSyncWaitTime) + err = snapc.SnapshotV1().VolumeSnapshots(namespace2Name).Delete(ctx, + volumeSnapshot2_updated.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) snapshotCreated2 = false + ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to allow CNS to sync with pandora", pandoraSyncWaitTime)) + time.Sleep(time.Duration(pandoraSyncWaitTime) * time.Second) + framework.Logf("Wait till the volume snapshot content is deleted") - deleteVolumeSnapshotContentWithPandoraWait(ctx, snapc, - volumeSnapshot2_updated.ObjectMeta.Name, pandoraSyncWaitTime) + err = snapc.SnapshotV1().VolumeSnapshotContents().Delete(ctx, + volumeSnapshot2_updated.ObjectMeta.Name, metav1.DeleteOptions{}) + gomega.Expect(err).To(gomega.HaveOccurred()) err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, volumeSnapshot2_updated.ObjectMeta.Name) gomega.Expect(err).NotTo(gomega.HaveOccurred()) snapshotContentCreated2 = false framework.Logf("Deleting volume snapshot content 1") - deleteVolumeSnapshotContentWithPandoraWait(ctx, snapc, - snapshotContent1Name, pandoraSyncWaitTime) + err = snapc.SnapshotV1().VolumeSnapshotContents().Delete(ctx, + snapshot1_updated.ObjectMeta.Name, metav1.DeleteOptions{}) + gomega.Expect(err).To(gomega.HaveOccurred()) - err = waitForVolumeSnapshotContentToBeDeletedWithPandoraWait(ctx, snapc, - snapshot1_updated.ObjectMeta.Name, pandoraSyncWaitTime) + err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, snapshot1_updated.ObjectMeta.Name) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - snapshotContentCreated = false + snapshotContentCreated2 = false ginkgo.By("Verify snapshot entry is deleted from CNS") - err = verifySnapshotIsDeletedInCNS(volHandle, snapshotId, false) + err = verifySnapshotIsDeletedInCNS(volHandle, snapshotId) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) @@ -1810,30 +1644,32 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { 6. Delete would return a pass from CSI side (this is expected because CSI is designed to return success even though it cannot find a snapshot in the backend) */ - ginkgo.It("[block-vanilla-snapshot] [tkg-snapshot] Delete a non-existent snapshot", func() { + ginkgo.It("Delete a non-existent snapshot", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - - if vanillaCluster { - scParameters[scParamDatastoreURL] = datastoreURL - } else if guestCluster { - scParameters[svStorageClassName] = storagePolicyName - } + var storageclass *storagev1.StorageClass + var pvclaim *v1.PersistentVolumeClaim + var pvclaims []*v1.PersistentVolumeClaim + var err error + var snapshotContentCreated = false + var snapshotCreated = false ginkgo.By("Create storage class and PVC") - storageclass, err := createStorageClass(client, scParameters, nil, "", "", false, "") + scParameters[scParamDatastoreURL] = datastoreURL + storageclass, pvclaim, err = createPVCAndStorageClass(client, + namespace, nil, scParameters, diskSize, nil, "", false, "") gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer func() { err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() - pvclaim, persistentVolumes := createPVCAndQueryVolumeInCNS(client, namespace, nil, "", - diskSize, storageclass, true) - volHandle := persistentVolumes[0].Spec.CSI.VolumeHandle - if guestCluster { - volHandle = getVolumeIDFromSupervisorCluster(volHandle) - } + ginkgo.By("Expect claim to provision volume successfully") + pvclaims = append(pvclaims, pvclaim) + persistentvolumes, err := fpv.WaitForPVClaimBoundPhase(client, pvclaims, framework.ClaimProvisionTimeout) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + volHandle := persistentvolumes[0].Spec.CSI.VolumeHandle gomega.Expect(volHandle).NotTo(gomega.BeEmpty()) defer func() { @@ -1843,48 +1679,92 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() + // Verify using CNS Query API if VolumeID retrieved from PV is present. + ginkgo.By(fmt.Sprintf("Invoking QueryCNSVolumeWithResult with VolumeID: %s", volHandle)) + queryResult, err := e2eVSphere.queryCNSVolumeWithResult(volHandle) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(queryResult.Volumes).ShouldNot(gomega.BeEmpty()) + gomega.Expect(queryResult.Volumes[0].VolumeId.Id).To(gomega.Equal(volHandle)) + ginkgo.By("Create volume snapshot class") - volumeSnapshotClass, err := createVolumeSnapshotClass(ctx, snapc, deletionPolicy) + volumeSnapshotClass, err := snapc.SnapshotV1().VolumeSnapshotClasses().Create(ctx, + getVolumeSnapshotClassSpec(snapV1.DeletionPolicy("Delete"), nil), metav1.CreateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer func() { - if vanillaCluster { - err = snapc.SnapshotV1().VolumeSnapshotClasses().Delete(ctx, volumeSnapshotClass.Name, metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } + err := snapc.SnapshotV1().VolumeSnapshotClasses().Delete(ctx, + volumeSnapshotClass.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() - ginkgo.By("Create a dynamic volume snapshot") - volumeSnapshot, snapshotContent, snapshotCreated, - snapshotContentCreated, snapshotId, err := createDynamicVolumeSnapshot(ctx, namespace, snapc, volumeSnapshotClass, - pvclaim, volHandle, diskSize) + ginkgo.By("Create a volume snapshot") + volumeSnapshot, err := snapc.SnapshotV1().VolumeSnapshots(namespace).Create(ctx, + getVolumeSnapshotSpec(namespace, volumeSnapshotClass.Name, pvclaim.Name), metav1.CreateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.Logf("Volume snapshot name is : %s", volumeSnapshot.Name) + defer func() { if snapshotContentCreated { - err = deleteVolumeSnapshotContent(ctx, snapshotContent, snapc, namespace, pandoraSyncWaitTime) + framework.Logf("Deleting volume snapshot content") + err := snapc.SnapshotV1().VolumeSnapshotContents().Delete(ctx, + *volumeSnapshot.Status.BoundVolumeSnapshotContentName, metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } if snapshotCreated { framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeletedWithPandoraWait(ctx, snapc, - *volumeSnapshot.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) + err := snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, + volumeSnapshot.Name, metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() + ginkgo.By("Verify volume snapshot is created") + volumeSnapshot_updated, err := waitForVolumeSnapshotReadyToUse(*snapc, ctx, namespace, volumeSnapshot.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + snapshotCreated = true + gomega.Expect(volumeSnapshot_updated.Status.RestoreSize.Cmp(resource.MustParse(diskSize))).To(gomega.BeZero()) + + ginkgo.By("Verify volume snapshot content is created") + snapshotContent_updated, err := snapc.SnapshotV1().VolumeSnapshotContents().Get(ctx, + *volumeSnapshot_updated.Status.BoundVolumeSnapshotContentName, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + snapshotContentCreated = true + gomega.Expect(*snapshotContent_updated.Status.ReadyToUse).To(gomega.BeTrue()) + + framework.Logf("Get volume snapshot ID from snapshot handle") + snapshothandle := *snapshotContent_updated.Status.SnapshotHandle + snapshotId := strings.Split(snapshothandle, "+")[1] + + ginkgo.By("Query CNS and check the volume snapshot entry") + err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + ginkgo.By("Delete snapshot from CNS") err = e2eVSphere.deleteVolumeSnapshotInCNS(volHandle, snapshotId) gomega.Expect(err).NotTo(gomega.HaveOccurred()) snapshotCreated = false - ginkgo.By("Delete dynamic volume snapshot") - snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + ginkgo.By("Verify snapshot entry is deleted from CNS") + err = verifySnapshotIsDeletedInCNS(volHandle, snapshotId) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to allow CNS to sync with pandora", pandoraSyncWaitTime)) + time.Sleep(time.Duration(pandoraSyncWaitTime) * time.Second) + + ginkgo.By("Delete volume snapshot") + err = snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, volumeSnapshot.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + framework.Logf("Deleting volume snapshot content") + err = snapc.SnapshotV1().VolumeSnapshotContents().Delete(ctx, + snapshotContent_updated.Name, metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.Logf("Wait till the volume snapshot content is deleted") + err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, snapshotContent_updated.ObjectMeta.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + snapshotContentCreated = false }) /* @@ -1894,33 +1774,32 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { ensure the default class is picked and honored for snapshot creation 3. Validate the fields after snapshot creation succeeds (snapshotClass, retentionPolicy) */ - ginkgo.It("[block-vanilla-snapshot] [tkg-snapshot] Create snapshots using default "+ - "VolumeSnapshotClass", func() { + ginkgo.It("Create snapshots using default VolumeSnapshotClass", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - - var volumeSnapshotClass *snapV1.VolumeSnapshotClass - - if vanillaCluster { - scParameters[scParamDatastoreURL] = datastoreURL - } else if guestCluster { - scParameters[svStorageClassName] = storagePolicyName - } + var storageclass *storagev1.StorageClass + var pvclaim *v1.PersistentVolumeClaim + var pvclaims []*v1.PersistentVolumeClaim + var err error + var snapshotContentCreated = false + var snapshotCreated = false ginkgo.By("Create storage class and PVC") - storageclass, err := createStorageClass(client, scParameters, nil, "", "", false, "") + scParameters[scParamDatastoreURL] = datastoreURL + storageclass, pvclaim, err = createPVCAndStorageClass(client, + namespace, nil, scParameters, diskSize, nil, "", false, "") gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer func() { err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() - pvclaim, persistentVolumes := createPVCAndQueryVolumeInCNS(client, namespace, nil, "", - diskSize, storageclass, true) - volHandle := persistentVolumes[0].Spec.CSI.VolumeHandle - if guestCluster { - volHandle = getVolumeIDFromSupervisorCluster(volHandle) - } + ginkgo.By("Expect claim to provision volume successfully") + pvclaims = append(pvclaims, pvclaim) + persistentvolumes, err := fpv.WaitForPVClaimBoundPhase(client, pvclaims, framework.ClaimProvisionTimeout) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + volHandle := persistentvolumes[0].Spec.CSI.VolumeHandle gomega.Expect(volHandle).NotTo(gomega.BeEmpty()) defer func() { @@ -1930,70 +1809,87 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() - ginkgo.By("Create volume snapshot class") - if vanillaCluster { - vscSpec := getVolumeSnapshotClassSpec(snapV1.DeletionPolicy("Delete"), nil) - vscSpec.ObjectMeta.Annotations = map[string]string{ - "snapshot.storage.kubernetes.io/is-default-class": "true", - } - volumeSnapshotClass, err = snapc.SnapshotV1().VolumeSnapshotClasses().Create(ctx, - vscSpec, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } else if guestCluster { - restConfig = getRestConfigClient() - snapc, err = snapclient.NewForConfig(restConfig) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - vscSpec := getVolumeSnapshotClassSpec(snapV1.DeletionPolicy("Delete"), nil) - vscSpec.ObjectMeta.Annotations = map[string]string{ - "snapshot.storage.kubernetes.io/is-default-class": "true", - } - volumeSnapshotClass, err = snapc.SnapshotV1().VolumeSnapshotClasses().Create(ctx, - vscSpec, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + // Verify using CNS Query API if VolumeID retrieved from PV is present. + ginkgo.By(fmt.Sprintf("Invoking QueryCNSVolumeWithResult with VolumeID: %s", volHandle)) + queryResult, err := e2eVSphere.queryCNSVolumeWithResult(volHandle) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(queryResult.Volumes).ShouldNot(gomega.BeEmpty()) + gomega.Expect(queryResult.Volumes[0].VolumeId.Id).To(gomega.Equal(volHandle)) + ginkgo.By("Create volume snapshot class") + vscSpec := getVolumeSnapshotClassSpec(snapV1.DeletionPolicy("Delete"), nil) + vscSpec.ObjectMeta.Annotations = map[string]string{ + "snapshot.storage.kubernetes.io/is-default-class": "true", } + volumeSnapshotClass, err := snapc.SnapshotV1().VolumeSnapshotClasses().Create(ctx, + vscSpec, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer func() { - if guestCluster { - restConfig = getRestConfigClient() - snapc, err = snapclient.NewForConfig(restConfig) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } err := snapc.SnapshotV1().VolumeSnapshotClasses().Delete(ctx, volumeSnapshotClass.Name, metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() - ginkgo.By("Create a dynamic volume snapshot") - if guestCluster { - guestClusterRestConfig = getRestConfigClientForGuestCluster(guestClusterRestConfig) - snapc, err = snapclient.NewForConfig(guestClusterRestConfig) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - volumeSnapshot, snapshotContent, snapshotCreated, - snapshotContentCreated, snapshotId, err := createDynamicVolumeSnapshot(ctx, namespace, snapc, volumeSnapshotClass, - pvclaim, volHandle, diskSize) + ginkgo.By("Create a volume snapshot") + volumeSnapshot, err := snapc.SnapshotV1().VolumeSnapshots(namespace).Create(ctx, + getVolumeSnapshotSpecWithoutSC(namespace, pvclaim.Name), metav1.CreateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.Logf("Volume snapshot name is : %s", volumeSnapshot.Name) + defer func() { if snapshotContentCreated { - err = deleteVolumeSnapshotContent(ctx, snapshotContent, snapc, namespace, pandoraSyncWaitTime) + framework.Logf("Deleting volume snapshot content") + err := snapc.SnapshotV1().VolumeSnapshotContents().Delete(ctx, + *volumeSnapshot.Status.BoundVolumeSnapshotContentName, metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } if snapshotCreated { framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeletedWithPandoraWait(ctx, snapc, - *volumeSnapshot.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) + err := snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, + volumeSnapshot.Name, metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() - ginkgo.By("Delete dynamic volume snapshot") - snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + ginkgo.By("Verify volume snapshot is created") + volumeSnapshot, err = waitForVolumeSnapshotReadyToUse(*snapc, ctx, namespace, volumeSnapshot.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + snapshotCreated = true + gomega.Expect(volumeSnapshot.Status.RestoreSize.Cmp(resource.MustParse(diskSize))).To(gomega.BeZero()) + + ginkgo.By("Verify volume snapshot content is created") + snapshotContent, err := snapc.SnapshotV1().VolumeSnapshotContents().Get(ctx, + *volumeSnapshot.Status.BoundVolumeSnapshotContentName, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + snapshotContentCreated = true + gomega.Expect(*snapshotContent.Status.ReadyToUse).To(gomega.BeTrue()) + gomega.Expect(*snapshotContent.Spec.VolumeSnapshotClassName).To(gomega.Equal(volumeSnapshotClass.Name)) + + framework.Logf("Get volume snapshot ID from snapshot handle") + snapshothandle := *snapshotContent.Status.SnapshotHandle + snapshotId := strings.Split(snapshothandle, "+")[1] + + ginkgo.By("Query CNS and check the volume snapshot entry") + err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Delete volume snapshot and verify the snapshot content is deleted") + err = snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, + volumeSnapshot.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + snapshotCreated = false + + framework.Logf("Wait till the volume snapshot is deleted") + err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, + *volumeSnapshot.Status.BoundVolumeSnapshotContentName) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + snapshotContentCreated = false + + ginkgo.By("Verify snapshot entry is deleted from CNS") + err = verifySnapshotIsDeletedInCNS(volHandle, snapshotId) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) /* @@ -2004,31 +1900,34 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { 3. Verify the error 4. Create with exact size and ensure it succeeds */ - ginkgo.It("[block-vanilla-snapshot][tkg-snapshot] Create Volume from snapshot with different size", func() { + ginkgo.It("Create Volume from snapshot with different size", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - - if vanillaCluster { - scParameters[scParamDatastoreURL] = datastoreURL - } else if guestCluster { - scParameters[svStorageClassName] = storagePolicyName - } + var storageclass *storagev1.StorageClass + var pvclaim *v1.PersistentVolumeClaim + var pvclaims []*v1.PersistentVolumeClaim + var err error + var snapshotContentCreated = false + var snapshotCreated = false ginkgo.By("Create storage class and PVC") - storageclass, err := createStorageClass(client, scParameters, nil, "", "", false, "") + scParameters[scParamDatastoreURL] = datastoreURL + storageclass, pvclaim, err = createPVCAndStorageClass(client, + namespace, nil, scParameters, diskSize, nil, "", false, "") gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer func() { err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() - pvclaim, persistentVolumes := createPVCAndQueryVolumeInCNS(client, namespace, nil, "", - diskSize, storageclass, true) - volHandle := persistentVolumes[0].Spec.CSI.VolumeHandle - if guestCluster { - volHandle = getVolumeIDFromSupervisorCluster(volHandle) - } + ginkgo.By("Expect claim to provision volume successfully") + pvclaims = append(pvclaims, pvclaim) + persistentvolumes, err := fpv.WaitForPVClaimBoundPhase(client, pvclaims, framework.ClaimProvisionTimeout) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + volHandle := persistentvolumes[0].Spec.CSI.VolumeHandle gomega.Expect(volHandle).NotTo(gomega.BeEmpty()) + defer func() { err := fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -2036,39 +1935,67 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() + // Verify using CNS Query API if VolumeID retrieved from PV is present. + ginkgo.By(fmt.Sprintf("Invoking QueryCNSVolumeWithResult with VolumeID: %s", volHandle)) + queryResult, err := e2eVSphere.queryCNSVolumeWithResult(volHandle) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(queryResult.Volumes).ShouldNot(gomega.BeEmpty()) + gomega.Expect(queryResult.Volumes[0].VolumeId.Id).To(gomega.Equal(volHandle)) + ginkgo.By("Create volume snapshot class") - volumeSnapshotClass, err := createVolumeSnapshotClass(ctx, snapc, deletionPolicy) + volumeSnapshotClass, err := snapc.SnapshotV1().VolumeSnapshotClasses().Create(ctx, + getVolumeSnapshotClassSpec(snapV1.DeletionPolicy("Delete"), nil), metav1.CreateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer func() { - if vanillaCluster { - err = snapc.SnapshotV1().VolumeSnapshotClasses().Delete(ctx, volumeSnapshotClass.Name, - metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } + err := snapc.SnapshotV1().VolumeSnapshotClasses().Delete(ctx, volumeSnapshotClass.Name, + metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() - ginkgo.By("Create a dynamic volume snapshot") - volumeSnapshot, snapshotContent, snapshotCreated, - snapshotContentCreated, snapshotId, err := createDynamicVolumeSnapshot(ctx, namespace, snapc, volumeSnapshotClass, - pvclaim, volHandle, diskSize) + ginkgo.By("Create a volume snapshot") + volumeSnapshot, err := snapc.SnapshotV1().VolumeSnapshots(namespace).Create(ctx, + getVolumeSnapshotSpec(namespace, volumeSnapshotClass.Name, pvclaim.Name), metav1.CreateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.Logf("Volume snapshot name is : %s", volumeSnapshot.Name) + defer func() { if snapshotContentCreated { - err = deleteVolumeSnapshotContent(ctx, snapshotContent, snapc, namespace, pandoraSyncWaitTime) + framework.Logf("Deleting volume snapshot content") + err := snapc.SnapshotV1().VolumeSnapshotContents().Delete(ctx, + *volumeSnapshot.Status.BoundVolumeSnapshotContentName, metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } if snapshotCreated { framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeletedWithPandoraWait(ctx, snapc, - *volumeSnapshot.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) + err := snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, volumeSnapshot.Name, + metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() + ginkgo.By("Verify volume snapshot is created") + volumeSnapshot, err = waitForVolumeSnapshotReadyToUse(*snapc, ctx, namespace, volumeSnapshot.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + snapshotCreated = true + gomega.Expect(volumeSnapshot.Status.RestoreSize.Cmp(resource.MustParse(diskSize))).To(gomega.BeZero()) + + ginkgo.By("Verify volume snapshot content is created") + snapshotContent, err := snapc.SnapshotV1().VolumeSnapshotContents().Get(ctx, + *volumeSnapshot.Status.BoundVolumeSnapshotContentName, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + snapshotContentCreated = true + gomega.Expect(*snapshotContent.Status.ReadyToUse).To(gomega.BeTrue()) + + framework.Logf("Get volume snapshot ID from snapshot handle") + snapshothandle := *snapshotContent.Status.SnapshotHandle + snapshotId := strings.Split(snapshothandle, "+")[1] + + ginkgo.By("Query CNS and check the volume snapshot entry") + err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + ginkgo.By("Create PVC using the higher size") pvcSpec := getPersistentVolumeClaimSpecWithDatasource(namespace, defaultrqLimit, storageclass, nil, v1.ReadWriteOnce, volumeSnapshot.Name, snapshotapigroup) @@ -2093,41 +2020,40 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) pvc2Deleted = true - if guestCluster { - framework.Logf("Deleting pending PVCs from SVC namespace") - pvcList := getAllPVCFromNamespace(svcClient, svcNamespace) - for _, pvc := range pvcList.Items { - if pvc.Status.Phase == v1.ClaimPending { - framework.ExpectNoError(fpv.DeletePersistentVolumeClaim(svcClient, pvc.Name, svcNamespace), - "Failed to delete PVC", pvc.Name) - } - } - } - ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to allow CNS to sync with pandora", pandoraSyncWaitTime)) time.Sleep(time.Duration(pandoraSyncWaitTime) * time.Second) - ginkgo.By("Delete dynamic volume snapshot") - snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + ginkgo.By("Delete volume snapshot and verify the snapshot content is deleted") + err = snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, volumeSnapshot.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + snapshotCreated = false + + framework.Logf("Wait till the volume snapshot is deleted") + err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, + *volumeSnapshot.Status.BoundVolumeSnapshotContentName) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + snapshotContentCreated = false + + ginkgo.By("Verify snapshot entry is deleted from CNS") + err = verifySnapshotIsDeletedInCNS(volHandle, snapshotId) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) /* - Snapshot workflow for statefulsets - 1. Create a statefulset with 3 replicas using a storageclass with volumeBindingMode set to Immediate - 2. Wait for pvcs to be in Bound state - 3. Wait for pods to be in Running state - 4. Create snapshot on 3rd replica's pvc (pvc as source) - 5. Scale down the statefulset to 2 - 6. Delete the pvc on which snapshot was created - 7. PVC delete succeeds but PV delete will fail as there is snapshot - expected - 8. Create a new PVC with same name (using the snapshot from step-4 as source) - verify a new PV is created - 9. Scale up the statefulset to 3 - 10. Verify if the new pod attaches to the PV created in step-8 - 11. Cleanup the sts and the snapshot + pv that was left behind in step-7 + Snapshot workflow for statefulsets + 1. Create a statefulset with 3 replicas using a storageclass with volumeBindingMode set to Immediate + 2. Wait for pvcs to be in Bound state + 3. Wait for pods to be in Running state + 4. Create snapshot on 3rd replica's pvc (pvc as source) + 5. Scale down the statefulset to 2 + 6. Delete the pvc on which snapshot was created + 7. PVC delete succeeds but PV delete will fail as there is snapshot - expected + 8. Create a new PVC with same name (using the snapshot from step-4 as source) - verify a new PV is created + 9. Scale up the statefulset to 3 + 10. Verify if the new pod attaches to the PV created in step-8 + 11. Cleanup the sts and the snapshot + pv that was left behind in step-7 */ - ginkgo.It("[block-vanilla-snapshot] Snapshot workflow for statefulsets", func() { + ginkgo.It("Snapshot workflow for statefulsets", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() var err error @@ -2160,7 +2086,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { statefulset := GetStatefulSetFromManifest(namespace) ginkgo.By("Creating statefulset") statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &scName + Annotations["volume.beta.kubernetes.io/storage-class"] = scName *statefulset.Spec.Replicas = 2 CreateStatefulSet(namespace, statefulset, client) replicas := *(statefulset.Spec.Replicas) @@ -2270,7 +2196,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { snapshotId1 := strings.Split(snapshothandle1, "+")[1] ginkgo.By("Query CNS and check the volume snapshot entry") - err = verifySnapshotIsCreatedInCNS(volHandle1, snapshotId1, false) + err = verifySnapshotIsCreatedInCNS(volHandle1, snapshotId1) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Create a volume snapshot - 2") @@ -2315,7 +2241,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { snapshotId2 := strings.Split(snapshothandle2, "+")[1] ginkgo.By("Query CNS and check the volume snapshot entry") - err = verifySnapshotIsCreatedInCNS(volHandle2, snapshotId2, false) + err = verifySnapshotIsCreatedInCNS(volHandle2, snapshotId2) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By(fmt.Sprintf("Scaling down statefulsets to number of Replica: %v", replicas-1)) @@ -2351,9 +2277,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { framework.Logf("Wait till the volume snapshot content is deleted") err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, *snapshotToBeDeleted.Status.BoundVolumeSnapshotContentName) - if err != nil { - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } + gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = fpv.DeletePersistentVolumeClaim(client, pvcToDelete.Name, namespace) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -2425,7 +2349,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { snapshotContentCreated2 = false ginkgo.By("Verify snapshot 1 entry is deleted from CNS") - err = verifySnapshotIsDeletedInCNS(volHandle1, snapshotId1, false) + err = verifySnapshotIsDeletedInCNS(volHandle1, snapshotId1) if err != nil { if !apierrors.IsNotFound(err) { gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -2433,7 +2357,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { } ginkgo.By("Verify snapshot 2 entry is deleted from CNS") - err = verifySnapshotIsDeletedInCNS(volHandle2, snapshotId2, false) + err = verifySnapshotIsDeletedInCNS(volHandle2, snapshotId2) if err != nil { if !apierrors.IsNotFound(err) { gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -2447,35 +2371,36 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { 2. Create a dynamic snapshot using above pvc as source 3. Delete this pvc, expect the pvc to be deleted successfully 4. Underlying pv should not be deleted and should have a valid error - calling out that the volume has active snapshots + calling out that the volume has active snapshots (note: the storageclass here is set to Delete retentionPolicy) 5. Expect VolumeFailedDelete error with an appropriate err-msg 6. Run cleanup - delete the snapshots and then delete pv */ - ginkgo.It("[block-vanilla-snapshot] [tkg-snapshot] Volume deletion with existing snapshots", func() { + ginkgo.It("Volume deletion with existing snapshots", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - - if vanillaCluster { - scParameters[scParamDatastoreURL] = datastoreURL - } else if guestCluster { - scParameters[svStorageClassName] = storagePolicyName - } + var storageclass *storagev1.StorageClass + var pvclaim *v1.PersistentVolumeClaim + var pvclaims []*v1.PersistentVolumeClaim + var err error + var snapshotContentCreated = false ginkgo.By("Create storage class and PVC") - storageclass, err := createStorageClass(client, scParameters, nil, "", "", false, "") + scParameters[scParamDatastoreURL] = datastoreURL + storageclass, pvclaim, err = createPVCAndStorageClass(client, + namespace, nil, scParameters, diskSize, nil, "", false, "") gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer func() { err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() - pvclaim, persistentvolumes := createPVCAndQueryVolumeInCNS(client, namespace, nil, "", - diskSize, storageclass, true) + ginkgo.By("Expect claim to provision volume successfully") + pvclaims = append(pvclaims, pvclaim) + persistentvolumes, err := fpv.WaitForPVClaimBoundPhase(client, pvclaims, framework.ClaimProvisionTimeout) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) volHandle := persistentvolumes[0].Spec.CSI.VolumeHandle - if guestCluster { - volHandle = getVolumeIDFromSupervisorCluster(volHandle) - } gomega.Expect(volHandle).NotTo(gomega.BeEmpty()) defer func() { @@ -2487,39 +2412,67 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() + // Verify using CNS Query API if VolumeID retrieved from PV is present. + ginkgo.By(fmt.Sprintf("Invoking QueryCNSVolumeWithResult with VolumeID: %s", volHandle)) + queryResult, err := e2eVSphere.queryCNSVolumeWithResult(volHandle) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(queryResult.Volumes).ShouldNot(gomega.BeEmpty()) + gomega.Expect(queryResult.Volumes[0].VolumeId.Id).To(gomega.Equal(volHandle)) + ginkgo.By("Create volume snapshot class") - volumeSnapshotClass, err := createVolumeSnapshotClass(ctx, snapc, deletionPolicy) + volumeSnapshotClass, err := snapc.SnapshotV1().VolumeSnapshotClasses().Create(ctx, + getVolumeSnapshotClassSpec(snapV1.DeletionPolicy("Delete"), nil), metav1.CreateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer func() { - if vanillaCluster { - err = snapc.SnapshotV1().VolumeSnapshotClasses().Delete(ctx, volumeSnapshotClass.Name, - metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } + err := snapc.SnapshotV1().VolumeSnapshotClasses().Delete(ctx, volumeSnapshotClass.Name, + metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() - ginkgo.By("Create a dynamic volume snapshot") - volumeSnapshot, snapshotContent, snapshotCreated, - snapshotContentCreated, snapshotId, err := createDynamicVolumeSnapshot(ctx, namespace, snapc, volumeSnapshotClass, - pvclaim, volHandle, diskSize) + ginkgo.By("Create a volume snapshot") + volumeSnapshot, err := snapc.SnapshotV1().VolumeSnapshots(namespace).Create(ctx, + getVolumeSnapshotSpec(namespace, volumeSnapshotClass.Name, pvclaim.Name), metav1.CreateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.Logf("Volume snapshot name is : %s", volumeSnapshot.Name) + snapshotCreated := true + defer func() { if snapshotContentCreated { - err = deleteVolumeSnapshotContent(ctx, snapshotContent, snapc, namespace, pandoraSyncWaitTime) + framework.Logf("Deleting volume snapshot content") + err := snapc.SnapshotV1().VolumeSnapshotContents().Delete(ctx, + *volumeSnapshot.Status.BoundVolumeSnapshotContentName, metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } if snapshotCreated { framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeletedWithPandoraWait(ctx, snapc, - *volumeSnapshot.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) + err := snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, + volumeSnapshot.Name, metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() + ginkgo.By("Verify volume snapshot is created") + volumeSnapshot, err = waitForVolumeSnapshotReadyToUse(*snapc, ctx, namespace, volumeSnapshot.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(volumeSnapshot.Status.RestoreSize.Cmp(resource.MustParse(diskSize))).To(gomega.BeZero()) + + ginkgo.By("Verify volume snapshot content is created") + snapshotContent, err := snapc.SnapshotV1().VolumeSnapshotContents().Get(ctx, + *volumeSnapshot.Status.BoundVolumeSnapshotContentName, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + snapshotContentCreated = true + gomega.Expect(*snapshotContent.Status.ReadyToUse).To(gomega.BeTrue()) + + framework.Logf("Get volume snapshot ID from snapshot handle") + snapshothandle := *snapshotContent.Status.SnapshotHandle + snapshotId := strings.Split(snapshothandle, "+")[1] + + ginkgo.By("Query CNS and check the volume snapshot entry") + err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + ginkgo.By("Delete PVC before deleting the snapshot") err = fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) gomega.Expect(err).To(gomega.HaveOccurred()) @@ -2528,9 +2481,19 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { _, err = client.CoreV1().PersistentVolumes().Get(ctx, persistentvolumes[0].Name, metav1.GetOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By("Delete dynamic volume snapshot") - snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + ginkgo.By("Delete volume snapshot and verify the snapshot content is deleted") + err = snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, volumeSnapshot.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + snapshotCreated = false + + framework.Logf("Wait till the volume snapshot is deleted") + err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, + *volumeSnapshot.Status.BoundVolumeSnapshotContentName) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + snapshotContentCreated = false + + ginkgo.By("Verify snapshot entry is deleted from CNS") + err = verifySnapshotIsDeletedInCNS(volHandle, snapshotId) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Delete PV") @@ -2546,30 +2509,32 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) } } + }) /* Create a pre-provisioned snapshot using VolumeSnapshotContent as source - (use VSC which is auto-created by a dynamic provisioned snapshot) + (use VSC which is auto-created by a dynamic provisioned snapshot) 1. create a sc, and pvc using this sc 2. create a dynamic snapshot using above pvc as source 3. verify that it auto-created a VolumeSnapshotContent object 4. create a pre-provisioned snapshot (which uses VolumeSnapshotContent as source) using the VSC from step(3) 5. Ensure this provisioning fails with appropriate error: SnapshotContentMismatch error */ - ginkgo.It("[block-vanilla-snapshot] [tkg-snapshot] Create a pre-provisioned snapshot using "+ - "VolumeSnapshotContent as source", func() { + ginkgo.It("Create a pre-provisioned snapshot using VolumeSnapshotContent as source", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - - if vanillaCluster { - scParameters[scParamDatastoreURL] = datastoreURL - } else if guestCluster { - scParameters[svStorageClassName] = storagePolicyName - } + var storageclass *storagev1.StorageClass + var pvclaim *v1.PersistentVolumeClaim + var pvclaims []*v1.PersistentVolumeClaim + var err error + var snapshotContentCreated = false + var snapshotCreated = false ginkgo.By("Create storage class and PVC") - storageclass, err := createStorageClass(client, scParameters, nil, "", "", true, "") + scParameters[scParamDatastoreURL] = datastoreURL + storageclass, pvclaim, err = createPVCAndStorageClass(client, + namespace, nil, scParameters, diskSize, nil, "", false, "") gomega.Expect(err).NotTo(gomega.HaveOccurred()) defer func() { @@ -2577,11 +2542,11 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() - pvclaim, persistentVolumes := createPVCAndQueryVolumeInCNS(client, namespace, nil, "", diskSize, storageclass, true) - volHandle := persistentVolumes[0].Spec.CSI.VolumeHandle - if guestCluster { - volHandle = getVolumeIDFromSupervisorCluster(volHandle) - } + ginkgo.By("Expect claim to provision volume successfully") + pvclaims = append(pvclaims, pvclaim) + persistentvolumes, err := fpv.WaitForPVClaimBoundPhase(client, pvclaims, framework.ClaimProvisionTimeout) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + volHandle := persistentvolumes[0].Spec.CSI.VolumeHandle gomega.Expect(volHandle).NotTo(gomega.BeEmpty()) defer func() { @@ -2591,47 +2556,78 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() + // Verify using CNS Query API if VolumeID retrieved from PV is present. + ginkgo.By(fmt.Sprintf("Invoking QueryCNSVolumeWithResult with VolumeID: %s", volHandle)) + queryResult, err := e2eVSphere.queryCNSVolumeWithResult(volHandle) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(queryResult.Volumes).ShouldNot(gomega.BeEmpty()) + gomega.Expect(queryResult.Volumes[0].VolumeId.Id).To(gomega.Equal(volHandle)) + ginkgo.By("Create volume snapshot class") - volumeSnapshotClass, err := createVolumeSnapshotClass(ctx, snapc, deletionPolicy) + volumeSnapshotClass, err := snapc.SnapshotV1().VolumeSnapshotClasses().Create(ctx, + getVolumeSnapshotClassSpec(snapV1.DeletionPolicy("Delete"), nil), metav1.CreateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer func() { - if vanillaCluster { - err = snapc.SnapshotV1().VolumeSnapshotClasses().Delete(ctx, volumeSnapshotClass.Name, metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } + err := snapc.SnapshotV1().VolumeSnapshotClasses().Delete(ctx, volumeSnapshotClass.Name, + metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() - ginkgo.By("Create a dynamic volume snapshot") - volumeSnapshot, snapshotContent, snapshotCreated, - snapshotContentCreated, snapshotId, err := createDynamicVolumeSnapshot(ctx, namespace, snapc, volumeSnapshotClass, - pvclaim, volHandle, diskSize) + ginkgo.By("Create a volume snapshot") + volumeSnapshot, err := snapc.SnapshotV1().VolumeSnapshots(namespace).Create(ctx, + getVolumeSnapshotSpec(namespace, volumeSnapshotClass.Name, pvclaim.Name), metav1.CreateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.Logf("Volume snapshot name is : %s", volumeSnapshot.Name) + defer func() { if snapshotContentCreated { - err = deleteVolumeSnapshotContent(ctx, snapshotContent, snapc, namespace, pandoraSyncWaitTime) + framework.Logf("Deleting volume snapshot content") + err := snapc.SnapshotV1().VolumeSnapshotContents().Delete(ctx, + *volumeSnapshot.Status.BoundVolumeSnapshotContentName, metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } if snapshotCreated { framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeletedWithPandoraWait(ctx, snapc, - *volumeSnapshot.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) + err := snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, volumeSnapshot.Name, + metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() + ginkgo.By("Verify volume snapshot is created") + volumeSnapshot, err = waitForVolumeSnapshotReadyToUse(*snapc, ctx, namespace, volumeSnapshot.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + snapshotCreated = true + gomega.Expect(volumeSnapshot.Status.RestoreSize.Cmp(resource.MustParse(diskSize))).To(gomega.BeZero()) + + ginkgo.By("Verify volume snapshot content is created") + snapshotContent, err := snapc.SnapshotV1().VolumeSnapshotContents().Get(ctx, + *volumeSnapshot.Status.BoundVolumeSnapshotContentName, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + snapshotContentCreated = true + gomega.Expect(*snapshotContent.Status.ReadyToUse).To(gomega.BeTrue()) + + framework.Logf("Get volume snapshot ID from snapshot handle") + snapshothandle := *snapshotContent.Status.SnapshotHandle + snapshotId := strings.Split(snapshothandle, "+")[1] + + ginkgo.By("Query CNS and check the volume snapshot entry") + err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + ginkgo.By("Create a volume snapshot2") volumeSnapshot2, err := snapc.SnapshotV1().VolumeSnapshots(namespace).Create(ctx, getVolumeSnapshotSpecByName(namespace, "static-vs", snapshotContent.ObjectMeta.Name), metav1.CreateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) snapshotCreated2 := true + defer func() { if snapshotCreated2 { - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, "static-vs", pandoraSyncWaitTime) + err = snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, "static-vs", metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() @@ -2640,26 +2636,36 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { gomega.Expect(err).To(gomega.HaveOccurred()) framework.Logf("Snapshot details is %+v", staticSnapshot) - ginkgo.By("Delete dynamic volume snapshot") - snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + ginkgo.By("Delete volume snapshot and verify the snapshot content is deleted") + err = snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, volumeSnapshot.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + snapshotCreated = false + + framework.Logf("Wait till the volume snapshot is deleted") + err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, + *volumeSnapshot.Status.BoundVolumeSnapshotContentName) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + snapshotContentCreated = false + + ginkgo.By("Verify snapshot entry is deleted from CNS") + err = verifySnapshotIsDeletedInCNS(volHandle, snapshotId) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) /* - Pre-provisioned snapshot using incorrect/non-existing static snapshot - 1. Create a sc, and pvc using this sc - 2. Create a snapshot for this pvc (use CreateSnapshot CNS API) - 3. Create a VolumeSnapshotContent CR using above snapshot-id, by passing the snapshotHandle - 4. Create a VolumeSnapshot using above content as source - 5. VolumeSnapshot and VolumeSnapshotContent should be created successfully and readToUse set to True - 6. Delete the snapshot created in step-2 (use deleteSnapshots CNS API) - 7. VolumeSnapshot and VolumeSnapshotContent will still have readyToUse set to True - 8. Restore: Create a volume using above pre-provisioned snapshot k8s object - (note the snapshotHandle its pointing to has been deleted) - 9. Volume Create should fail with an appropriate error on k8s side + Pre-provisioned snapshot using incorrect/non-existing static snapshot + 1. Create a sc, and pvc using this sc + 2. Create a snapshot for this pvc (use CreateSnapshot CNS API) + 3. Create a VolumeSnapshotContent CR using above snapshot-id, by passing the snapshotHandle + 4. Create a VolumeSnapshot using above content as source + 5. VolumeSnapshot and VolumeSnapshotContent should be created successfully and readToUse set to True + 6. Delete the snapshot created in step-2 (use deleteSnapshots CNS API) + 7. VolumeSnapshot and VolumeSnapshotContent will still have readyToUse set to True + 8. Restore: Create a volume using above pre-provisioned snapshot k8s object + (note the snapshotHandle its pointing to has been deleted) + 9. Volume Create should fail with an appropriate error on k8s side */ - ginkgo.It("[block-vanilla-snapshot] Pre-provisioned snapshot using incorrect/non-existing static snapshot", func() { + ginkgo.It("Pre-provisioned snapshot using incorrect/non-existing static snapshot", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() var storageclass *storagev1.StorageClass @@ -2724,7 +2730,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { }() ginkgo.By("Query CNS and check the volume snapshot entry") - err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId, false) + err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId) gomega.Expect(err).NotTo(gomega.HaveOccurred()) snapshotHandle := volHandle + "+" + snapshotId @@ -2743,8 +2749,9 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { defer func() { if snapshotContentCreated2 { - deleteVolumeSnapshotContentWithPandoraWait(ctx, snapc, - snapshotContentNew.ObjectMeta.Name, pandoraSyncWaitTime) + err := snapc.SnapshotV1().VolumeSnapshotContents().Delete(ctx, + snapshotContentNew.ObjectMeta.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() @@ -2770,7 +2777,8 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { framework.Logf("Snapshot details is %+v", staticSnapshot) ginkgo.By("Deleted volume snapshot is created above") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, staticSnapshot.Name, pandoraSyncWaitTime) + err = snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, staticSnapshot.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) snapshotCreated1 = false snapshotCreated2 = false @@ -2816,7 +2824,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { 7. Validate the pvc is Bound 8. Cleanup the snapshot and pvc */ - ginkgo.It("[block-vanilla-snapshot] Create a volume from a snapshot that is still not ready-to-use", func() { + ginkgo.It("Create a volume from a snapshot that is still not ready-to-use", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() var storageclass *storagev1.StorageClass @@ -2881,7 +2889,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { }() ginkgo.By("Query CNS and check the volume snapshot entry") - err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId, false) + err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId) gomega.Expect(err).NotTo(gomega.HaveOccurred()) snapshotHandle := volHandle + "+" + snapshotId @@ -2900,8 +2908,9 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { defer func() { if snapshotContentCreated2 { - deleteVolumeSnapshotContentWithPandoraWait(ctx, snapc, - snapshotContentNew.ObjectMeta.Name, pandoraSyncWaitTime) + err := snapc.SnapshotV1().VolumeSnapshotContents().Delete(ctx, + snapshotContentNew.ObjectMeta.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() @@ -2947,7 +2956,8 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { framework.Logf("Snapshot details is %+v", staticSnapshot) ginkgo.By("Deleted volume snapshot is created above") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, staticSnapshot.Name, pandoraSyncWaitTime) + err = snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, staticSnapshot.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) snapshotCreated1 = false snapshotCreated2 = false @@ -2968,18 +2978,18 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { 6. The deployment should succeed and should have the file that was created in step.2 7. Cleanup dep-1 pv snapshots and pvs, delete dep-2 */ - ginkgo.It("[block-vanilla-snapshot] [tkg-snapshot] Snapshot workflow for deployments", func() { + ginkgo.It("Snapshot workflow for deployments", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - - if vanillaCluster { - scParameters[scParamDatastoreURL] = datastoreURL - } else if guestCluster { - scParameters[svStorageClassName] = storagePolicyName - } + var storageclass *storagev1.StorageClass + var pvclaim *v1.PersistentVolumeClaim + var pvclaims []*v1.PersistentVolumeClaim + var err error ginkgo.By("Create storage class and PVC") - storageclass, err := createStorageClass(client, scParameters, nil, "", "", true, "") + scParameters[scParamDatastoreURL] = datastoreURL + storageclass, pvclaim, err = createPVCAndStorageClass(client, + namespace, nil, scParameters, diskSize, nil, "", false, "") gomega.Expect(err).NotTo(gomega.HaveOccurred()) defer func() { @@ -2987,11 +2997,11 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() - pvclaim, persistentVolumes := createPVCAndQueryVolumeInCNS(client, namespace, nil, "", diskSize, storageclass, true) - volHandle := persistentVolumes[0].Spec.CSI.VolumeHandle - if guestCluster { - volHandle = getVolumeIDFromSupervisorCluster(volHandle) - } + ginkgo.By("Expect claim to provision volume successfully") + pvclaims = append(pvclaims, pvclaim) + persistentvolumes, err := fpv.WaitForPVClaimBoundPhase(client, pvclaims, framework.ClaimProvisionTimeout) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + volHandle := persistentvolumes[0].Spec.CSI.VolumeHandle gomega.Expect(volHandle).NotTo(gomega.BeEmpty()) defer func() { @@ -3001,13 +3011,21 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() + // Verify using CNS Query API if VolumeID retrieved from PV is present. + ginkgo.By(fmt.Sprintf("Invoking QueryCNSVolumeWithResult with VolumeID: %s", volHandle)) + queryResult, err := e2eVSphere.queryCNSVolumeWithResult(volHandle) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(queryResult.Volumes).ShouldNot(gomega.BeEmpty()) + gomega.Expect(queryResult.Volumes[0].VolumeId.Id).To(gomega.Equal(volHandle)) + labelsMap := make(map[string]string) labelsMap["app"] = "test" - ginkgo.By("Creating a Deployment using pvc1") + dep, err := createDeployment(ctx, client, 1, labelsMap, nil, namespace, []*v1.PersistentVolumeClaim{pvclaim}, execRWXCommandPod1, false, busyBoxImageOnGcr) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer func() { ginkgo.By("Delete Deployment") err := client.AppsV1().Deployments(namespace).Delete(ctx, dep.Name, metav1.DeleteOptions{}) @@ -3022,37 +3040,63 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Create volume snapshot class") - volumeSnapshotClass, err := createVolumeSnapshotClass(ctx, snapc, deletionPolicy) + volumeSnapshotClass, err := snapc.SnapshotV1().VolumeSnapshotClasses().Create(ctx, + getVolumeSnapshotClassSpec(snapV1.DeletionPolicy("Delete"), nil), metav1.CreateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer func() { - if vanillaCluster { - err = snapc.SnapshotV1().VolumeSnapshotClasses().Delete(ctx, volumeSnapshotClass.Name, metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } + err := snapc.SnapshotV1().VolumeSnapshotClasses().Delete(ctx, + volumeSnapshotClass.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() - ginkgo.By("Create a dynamic volume snapshot") - volumeSnapshot, snapshotContent, snapshotCreated, - snapshotContentCreated, snapshotId, err := createDynamicVolumeSnapshot(ctx, namespace, snapc, volumeSnapshotClass, - pvclaim, volHandle, diskSize) + ginkgo.By("Create a volume snapshot") + volumeSnapshot, err := snapc.SnapshotV1().VolumeSnapshots(namespace).Create(ctx, + getVolumeSnapshotSpec(namespace, volumeSnapshotClass.Name, pvclaim.Name), metav1.CreateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.Logf("Volume snapshot name is : %s", volumeSnapshot.Name) + snapshotCreated := true + snapshotContentCreated := false + defer func() { if snapshotCreated { - err = deleteVolumeSnapshotContent(ctx, snapshotContent, snapc, namespace, pandoraSyncWaitTime) + framework.Logf("Deleting volume snapshot") + err := snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, + volumeSnapshot.Name, metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } if snapshotContentCreated { framework.Logf("Deleting volume snapshot content") - deleteVolumeSnapshotContentWithPandoraWait(ctx, snapc, - *volumeSnapshot.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, *volumeSnapshot.Status.BoundVolumeSnapshotContentName) + err := snapc.SnapshotV1().VolumeSnapshotContents().Delete(ctx, + *volumeSnapshot.Status.BoundVolumeSnapshotContentName, metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } + + ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to allow CNS to sync with pandora", pandoraSyncWaitTime)) + time.Sleep(time.Duration(pandoraSyncWaitTime) * time.Second) }() + ginkgo.By("Verify volume snapshot is created") + volumeSnapshot, err = waitForVolumeSnapshotReadyToUse(*snapc, ctx, namespace, volumeSnapshot.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(volumeSnapshot.Status.RestoreSize.Cmp(resource.MustParse(diskSize))).To(gomega.BeZero()) + + ginkgo.By("Verify volume snapshot content is created") + snapshotContent, err := snapc.SnapshotV1().VolumeSnapshotContents().Get(ctx, + *volumeSnapshot.Status.BoundVolumeSnapshotContentName, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + snapshotContentCreated = true + gomega.Expect(*snapshotContent.Status.ReadyToUse).To(gomega.BeTrue()) + + framework.Logf("Get volume snapshot ID from snapshot handle") + snapshothandle := *snapshotContent.Status.SnapshotHandle + snapshotId := strings.Split(snapshothandle, "+")[1] + + ginkgo.By("Query CNS and check the volume snapshot entry") + err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + ginkgo.By("Create a PVC using the snapshot created above") pvcSpec := getPersistentVolumeClaimSpecWithDatasource(namespace, diskSize, storageclass, nil, v1.ReadWriteOnce, volumeSnapshot.Name, snapshotapigroup) @@ -3064,9 +3108,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) volHandle2 := persistentvolume2[0].Spec.CSI.VolumeHandle gomega.Expect(volHandle2).NotTo(gomega.BeEmpty()) - if guestCluster { - volHandle2 = getVolumeIDFromSupervisorCluster(volHandle2) - } + defer func() { err := fpv.DeletePersistentVolumeClaim(client, pvclaim2.Name, namespace) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -3077,10 +3119,10 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { labelsMap2 := make(map[string]string) labelsMap2["app2"] = "test2" - ginkgo.By("Creating a new deployment from the restored pvc") dep2, err := createDeployment(ctx, client, 1, labelsMap2, nil, namespace, []*v1.PersistentVolumeClaim{pvclaim2}, "", false, busyBoxImageOnGcr) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer func() { ginkgo.By("Delete Deployment-2") err := client.AppsV1().Deployments(namespace).Delete(ctx, dep2.Name, metav1.DeleteOptions{}) @@ -3099,11 +3141,6 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { "cat /mnt/volume1/Pod1.html "} output := framework.RunKubectlOrDie(namespace, cmd...) gomega.Expect(strings.Contains(output, "Hello message from Pod1")).NotTo(gomega.BeFalse()) - - ginkgo.By("Delete dynamic volume snapshot") - snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) /* @@ -3116,31 +3153,31 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { 6. Run resize and it should succeed 7. Cleanup the pvc */ - ginkgo.It("[block-vanilla-snapshot] [tkg-snapshot] Volume offline resize of a volume having snapshots", func() { + ginkgo.It("Volume offline resize of a volume having snapshots", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - - if vanillaCluster { - scParameters[scParamDatastoreURL] = datastoreURL - } else if guestCluster { - scParameters[svStorageClassName] = storagePolicyName - } + var storageclass *storagev1.StorageClass + var pvclaim *v1.PersistentVolumeClaim + var err error ginkgo.By("Create storage class and PVC") - storageclass, err := createStorageClass(client, scParameters, nil, "", "", true, "") + scParameters[scParamDatastoreURL] = datastoreURL + storageclass, pvclaim, err = createPVCAndStorageClass(client, + namespace, nil, scParameters, diskSize, nil, "", true, "") gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer func() { err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() - pvclaim, persistentVolumes := createPVCAndQueryVolumeInCNS(client, namespace, nil, "", - diskSize, storageclass, true) - volHandle := persistentVolumes[0].Spec.CSI.VolumeHandle - if guestCluster { - volHandle = getVolumeIDFromSupervisorCluster(volHandle) - } - gomega.Expect(volHandle).NotTo(gomega.BeEmpty()) + ginkgo.By("Waiting for claim to be in bound phase") + pvc, err := fpv.WaitForPVClaimBoundPhase(client, + []*v1.PersistentVolumeClaim{pvclaim}, framework.ClaimProvisionTimeout) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(pvc).NotTo(gomega.BeEmpty()) + pv := getPvFromClaim(client, pvclaim.Namespace, pvclaim.Name) + volHandle := pv.Spec.CSI.VolumeHandle defer func() { err = fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) @@ -3155,7 +3192,6 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { currentPvcSize := pvclaim.Spec.Resources.Requests[v1.ResourceStorage] newSize := currentPvcSize.DeepCopy() newSize.Add(resource.MustParse("4Gi")) - newDiskSize := "6Gi" framework.Logf("currentPvcSize %v, newSize %v", currentPvcSize, newSize) pvclaim, err = expandPVCSize(pvclaim, newSize, client) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -3193,38 +3229,64 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Create volume snapshot class") - volumeSnapshotClass, err := createVolumeSnapshotClass(ctx, snapc, deletionPolicy) + volumeSnapshotClass, err := snapc.SnapshotV1().VolumeSnapshotClasses().Create(ctx, + getVolumeSnapshotClassSpec(snapV1.DeletionPolicy("Delete"), nil), metav1.CreateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer func() { - if vanillaCluster { - err = snapc.SnapshotV1().VolumeSnapshotClasses().Delete(ctx, volumeSnapshotClass.Name, - metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } + err := snapc.SnapshotV1().VolumeSnapshotClasses().Delete(ctx, + volumeSnapshotClass.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() - ginkgo.By("Create a dynamic volume snapshot") - volumeSnapshot, snapshotContent, snapshotCreated, - snapshotContentCreated, snapshotId, err := createDynamicVolumeSnapshot(ctx, namespace, snapc, volumeSnapshotClass, - pvclaim, volHandle, newDiskSize) + ginkgo.By("Create a volume snapshot") + volumeSnapshot, err := snapc.SnapshotV1().VolumeSnapshots(namespace).Create(ctx, + getVolumeSnapshotSpec(namespace, volumeSnapshotClass.Name, pvclaim.Name), metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.Logf("Volume snapshot name is : %s", volumeSnapshot.Name) + snapshotCreated := true + snapshotContentCreated := false + defer func() { if snapshotContentCreated { - err = deleteVolumeSnapshotContent(ctx, snapshotContent, snapc, namespace, pandoraSyncWaitTime) + framework.Logf("Deleting volume snapshot content") + err := snapc.SnapshotV1().VolumeSnapshotContents().Delete(ctx, + *volumeSnapshot.Status.BoundVolumeSnapshotContentName, metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } if snapshotCreated { framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeletedWithPandoraWait(ctx, snapc, - *volumeSnapshot.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) + err := snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, + volumeSnapshot.Name, metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() - ginkgo.By("Expanding current pvc before deleting volume snapshot") + ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to allow CNS to sync with pandora", pandoraSyncWaitTime)) + time.Sleep(time.Duration(pandoraSyncWaitTime) * time.Second) + + ginkgo.By("Verify volume snapshot is created") + volumeSnapshot, err = waitForVolumeSnapshotReadyToUse(*snapc, ctx, namespace, volumeSnapshot.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(volumeSnapshot.Status.RestoreSize.Cmp(resource.MustParse("6Gi"))).To(gomega.BeZero()) + + ginkgo.By("Verify volume snapshot content is created") + snapshotContent, err := snapc.SnapshotV1().VolumeSnapshotContents().Get(ctx, + *volumeSnapshot.Status.BoundVolumeSnapshotContentName, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + snapshotContentCreated = true + gomega.Expect(*snapshotContent.Status.ReadyToUse).To(gomega.BeTrue()) + + framework.Logf("Get volume snapshot ID from snapshot handle") + snapshothandle := *snapshotContent.Status.SnapshotHandle + snapshotId := strings.Split(snapshothandle, "+")[1] + + ginkgo.By("Query CNS and check the volume snapshot entry") + err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Expanding current pvc") currentPvcSize = pvclaim.Spec.Resources.Requests[v1.ResourceStorage] newSize = currentPvcSize.DeepCopy() newSize.Add(resource.MustParse("6Gi")) @@ -3233,12 +3295,21 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { ginkgo.By("Snapshot webhook does not allow volume expansion on PVC") gomega.Expect(err).To(gomega.HaveOccurred()) - ginkgo.By("Delete dynamic volume snapshot") - snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + ginkgo.By("Deleted volume snapshot is created above") + err = snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, volumeSnapshot.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + snapshotCreated = false + + framework.Logf("Wait till the volume snapshot is deleted") + err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, snapshotContent.ObjectMeta.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + snapshotContentCreated = false + + ginkgo.By("Verify snapshot entry is deleted from CNS") + err = verifySnapshotIsDeletedInCNS(volHandle, snapshotId) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By("Expanding current pvc after deleting volume snapshot") + ginkgo.By("Expanding current pvc") currentPvcSize = pvclaim.Spec.Resources.Requests[v1.ResourceStorage] newSize = currentPvcSize.DeepCopy() newSize.Add(resource.MustParse("6Gi")) @@ -3289,18 +3360,17 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { 6. Run resize and it should succeed 7. Cleanup the pvc */ - ginkgo.It("[block-vanilla-snapshot] [tkg-snapshot] Volume online resize of a volume having snapshots", func() { + ginkgo.It("Volume online resize of a volume having snapshots", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - - if vanillaCluster { - scParameters[scParamDatastoreURL] = datastoreURL - } else if guestCluster { - scParameters[svStorageClassName] = storagePolicyName - } + var storageclass *storagev1.StorageClass + var pvclaim *v1.PersistentVolumeClaim + var err error ginkgo.By("Create storage class and PVC") - storageclass, err := createStorageClass(client, scParameters, nil, "", "", true, "") + scParameters[scParamDatastoreURL] = datastoreURL + storageclass, pvclaim, err = createPVCAndStorageClass(client, + namespace, nil, scParameters, diskSize, nil, "", true, "") gomega.Expect(err).NotTo(gomega.HaveOccurred()) defer func() { @@ -3308,12 +3378,13 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() - pvclaim, persistentVolumes := createPVCAndQueryVolumeInCNS(client, namespace, nil, "", diskSize, storageclass, true) - volHandle := persistentVolumes[0].Spec.CSI.VolumeHandle - if guestCluster { - volHandle = getVolumeIDFromSupervisorCluster(volHandle) - } - gomega.Expect(volHandle).NotTo(gomega.BeEmpty()) + ginkgo.By("Waiting for claim to be in bound phase") + pvc, err := fpv.WaitForPVClaimBoundPhase(client, + []*v1.PersistentVolumeClaim{pvclaim}, framework.ClaimProvisionTimeout) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(pvc).NotTo(gomega.BeEmpty()) + pv := getPvFromClaim(client, pvclaim.Namespace, pvclaim.Name) + volHandle := pv.Spec.CSI.VolumeHandle defer func() { err = fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) @@ -3338,13 +3409,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { var vmUUID string nodeName := pod.Spec.NodeName - - if vanillaCluster { - vmUUID = getNodeUUID(ctx, client, pod.Spec.NodeName) - } else if guestCluster { - vmUUID, err = getVMUUIDFromNodeName(pod.Spec.NodeName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } + vmUUID = getNodeUUID(ctx, client, pod.Spec.NodeName) ginkgo.By(fmt.Sprintf("Verify volume: %s is attached to the node: %s", volHandle, nodeName)) isDiskAttached, err := e2eVSphere.isVolumeAttachedToVM(client, volHandle, vmUUID) @@ -3355,7 +3420,6 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { currentPvcSize := pvclaim.Spec.Resources.Requests[v1.ResourceStorage] newSize := currentPvcSize.DeepCopy() newSize.Add(resource.MustParse("4Gi")) - newDiskSize := "6Gi" framework.Logf("currentPvcSize %v, newSize %v", currentPvcSize, newSize) claims, err := expandPVCSize(pvclaim, newSize, client) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -3385,36 +3449,63 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Create volume snapshot class") - volumeSnapshotClass, err := createVolumeSnapshotClass(ctx, snapc, deletionPolicy) + volumeSnapshotClass, err := snapc.SnapshotV1().VolumeSnapshotClasses().Create(ctx, + getVolumeSnapshotClassSpec(snapV1.DeletionPolicy("Delete"), nil), metav1.CreateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer func() { - if vanillaCluster { - err = snapc.SnapshotV1().VolumeSnapshotClasses().Delete(ctx, volumeSnapshotClass.Name, metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } + err := snapc.SnapshotV1().VolumeSnapshotClasses().Delete(ctx, + volumeSnapshotClass.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() - ginkgo.By("Create a dynamic volume snapshot") - volumeSnapshot, snapshotContent, snapshotCreated, - snapshotContentCreated, snapshotId, err := createDynamicVolumeSnapshot(ctx, namespace, snapc, volumeSnapshotClass, - pvclaim, volHandle, newDiskSize) + ginkgo.By("Create a volume snapshot") + volumeSnapshot, err := snapc.SnapshotV1().VolumeSnapshots(namespace).Create(ctx, + getVolumeSnapshotSpec(namespace, volumeSnapshotClass.Name, pvclaim.Name), metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.Logf("Volume snapshot name is : %s", volumeSnapshot.Name) + snapshotCreated := true + snapshotContentCreated := false + defer func() { if snapshotContentCreated { - err = deleteVolumeSnapshotContent(ctx, snapshotContent, snapc, namespace, pandoraSyncWaitTime) + framework.Logf("Deleting volume snapshot content") + err := snapc.SnapshotV1().VolumeSnapshotContents().Delete(ctx, + *volumeSnapshot.Status.BoundVolumeSnapshotContentName, metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } if snapshotCreated { framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeletedWithPandoraWait(ctx, snapc, - *volumeSnapshot.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) + err := snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, + volumeSnapshot.Name, metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() + ginkgo.By("Verify volume snapshot is created") + volumeSnapshot, err = waitForVolumeSnapshotReadyToUse(*snapc, ctx, namespace, volumeSnapshot.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(volumeSnapshot.Status.RestoreSize.Cmp(resource.MustParse("6Gi"))).To(gomega.BeZero()) + + ginkgo.By("Verify volume snapshot content is created") + snapshotContent, err := snapc.SnapshotV1().VolumeSnapshotContents().Get(ctx, + *volumeSnapshot.Status.BoundVolumeSnapshotContentName, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + snapshotContentCreated = true + gomega.Expect(*snapshotContent.Status.ReadyToUse).To(gomega.BeTrue()) + + framework.Logf("Get volume snapshot ID from snapshot handle") + snapshothandle := *snapshotContent.Status.SnapshotHandle + snapshotId := strings.Split(snapshothandle, "+")[1] + + ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to allow CNS to sync with pandora", pandoraSyncWaitTime)) + time.Sleep(time.Duration(pandoraSyncWaitTime) * time.Second) + + ginkgo.By("Query CNS and check the volume snapshot entry") + err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + // Modify PVC spec to trigger volume expansion currentPvcSize = claims.Spec.Resources.Requests[v1.ResourceStorage] newSize = currentPvcSize.DeepCopy() @@ -3423,9 +3514,18 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { _, err = expandPVCSize(pvclaim, newSize, client) gomega.Expect(err).To(gomega.HaveOccurred()) - ginkgo.By("Delete dynamic volume snapshot") - snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) + ginkgo.By("Deleted volume snapshot is created above") + err = snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, volumeSnapshot.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + snapshotCreated = false + + framework.Logf("Wait till the volume snapshot is deleted") + err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, snapshotContent.ObjectMeta.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + snapshotContentCreated = false + + ginkgo.By("Verify snapshot entry is deleted from CNS") + err = verifySnapshotIsDeletedInCNS(volHandle, snapshotId) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Waiting for file system resize to finish") @@ -3505,7 +3605,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { 7. bring the host back up 8. cleanup the snapshots, restore-pvc and source-pvc */ - ginkgo.It("[block-vanilla-snapshot] Snapshot restore while the Host is Down", func() { + ginkgo.It("Snapshot restore while the Host is Down", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() var storageclass *storagev1.StorageClass @@ -3566,7 +3666,9 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { defer func() { if snapshotCreated { framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, snapshot1.Name, pandoraSyncWaitTime) + err := snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, + snapshot1.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() @@ -3588,7 +3690,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { snapshotId := strings.Split(snapshothandle, "+")[1] ginkgo.By("Query CNS and check the volume snapshot entry") - err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId, false) + err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Identify the host on which the PV resides") @@ -3638,18 +3740,18 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { }) /* - VC reboot with deployment pvcs having snapshot - 1. Create a sc and create 30 pvc's using this sc - 2. Create a deployment using 3 replicas and pvc's pointing to above - 3. Write some files to these PVCs - 4. Create snapshots on all the replica PVCs - 5. Reboot the VC - 6. Ensure the deployment comes up fine and data is available and we can write more data - 7. Create a new deployment, by creating new volumes using the snapshots cut prior to reboot - 8. Ensure the data written in step-4 is intanct - 9. Delete both deployments and. the pvcs + VC reboot with deployment pvcs having snapshot + 1. Create a sc and create 30 pvc's using this sc + 2. Create a deployment using 3 replicas and pvc's pointing to above + 3. Write some files to these PVCs + 4. Create snapshots on all the replica PVCs + 5. Reboot the VC + 6. Ensure the deployment comes up fine and data is available and we can write more data + 7. Create a new deployment, by creating new volumes using the snapshots cut prior to reboot + 8. Ensure the data written in step-4 is intanct + 9. Delete both deployments and. the pvcs */ - ginkgo.It("[block-vanilla-snapshot] VC reboot with deployment pvcs having snapshot", func() { + ginkgo.It("VC reboot with deployment pvcs having snapshot", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() var storageclass *storagev1.StorageClass @@ -3758,8 +3860,12 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { framework.Logf("Deleting volume snapshot") for _, snapshot := range volumesnapshots { - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, snapshot.Name, pandoraSyncWaitTime) + err := snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, + snapshot.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) } + ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to allow CNS to sync with pandora", pandoraSyncWaitTime)) + time.Sleep(time.Duration(pandoraSyncWaitTime) * time.Second) }() ginkgo.By("Rebooting VC") @@ -3775,23 +3881,6 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { // After reboot. bootstrap() - fullSyncWaitTime := 0 - - if os.Getenv(envFullSyncWaitTime) != "" { - fullSyncWaitTime, err = strconv.Atoi(os.Getenv(envFullSyncWaitTime)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - // Full sync interval can be 1 min at minimum so full sync wait time - // has to be more than 120s. - if fullSyncWaitTime < 120 || fullSyncWaitTime > defaultFullSyncWaitTime { - framework.Failf("The FullSync Wait time %v is not set correctly", fullSyncWaitTime) - } - } else { - fullSyncWaitTime = defaultFullSyncWaitTime - } - - ginkgo.By(fmt.Sprintf("Double Sleeping for %v seconds to allow full sync finish", fullSyncWaitTime)) - time.Sleep(time.Duration(2*fullSyncWaitTime) * time.Second) - ginkgo.By("Verify volume snapshot is created") for _, snapshot := range volumesnapshots { snapshot, err = waitForVolumeSnapshotReadyToUse(*snapc, ctx, namespace, snapshot.Name) @@ -3818,7 +3907,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { restoredpvclaims = append(restoredpvclaims, pvclaim2) persistentvolume2, err := fpv.WaitForPVClaimBoundPhase(client, []*v1.PersistentVolumeClaim{pvclaim2}, - framework.ClaimProvisionTimeout*2) + framework.ClaimProvisionTimeout) gomega.Expect(err).NotTo(gomega.HaveOccurred()) volHandle2 := persistentvolume2[0].Spec.CSI.VolumeHandle gomega.Expect(volHandle2).NotTo(gomega.BeEmpty()) @@ -3870,18 +3959,18 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { }) /* - VC password reset during snapshot creation - 1. Create a sc and pvc using this sc - 2. Create a volume snapshot using pvc as source - 3. Verify snapshot created successfully - 4. Change the VC administrator account password - 5. Create another snapshot - creation succeeds with previous csi session - 6. Update the vsphere.conf and the secret under vmware-system-csi ns and wait for 1-2 mins - 7. Create snapshot should succeed - 8. Delete snapshot - 9. Cleanup pvc/sc + VC password reset during snapshot creation + 1. Create a sc and pvc using this sc + 2. Create a volume snapshot using pvc as source + 3. Verify snapshot created successfully + 4. Change the VC administrator account password + 5. Create another snapshot - creation succeeds with previous csi session + 6. Update the vsphere.conf and the secret under vmware-system-csi ns and wait for 1-2 mins + 7. Create snapshot should succeed + 8. Delete snapshot + 9. Cleanup pvc/sc */ - ginkgo.It("[block-vanilla-snapshot] VC password reset during snapshot creation", func() { + ginkgo.It("VC password reset during snapshot creation", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() var storageclass *storagev1.StorageClass @@ -3890,7 +3979,6 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { var err error var snapshotCreated = false var snapshot3Created = false - nimbusGeneratedVcPwd := GetAndExpectStringEnvVar(nimbusVcPwd) ginkgo.By("Create storage class and PVC") scParameters[scParamDatastoreURL] = datastoreURL @@ -3905,7 +3993,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { ginkgo.By("Expect claim to provision volume successfully") pvclaims = append(pvclaims, pvclaim) - persistentvolumes, err := fpv.WaitForPVClaimBoundPhase(client, pvclaims, (2 * framework.ClaimProvisionTimeout)) + persistentvolumes, err := fpv.WaitForPVClaimBoundPhase(client, pvclaims, framework.ClaimProvisionTimeout) gomega.Expect(err).NotTo(gomega.HaveOccurred()) volHandle := persistentvolumes[0].Spec.CSI.VolumeHandle gomega.Expect(volHandle).NotTo(gomega.BeEmpty()) @@ -3944,7 +4032,9 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { defer func() { if snapshotCreated { framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, snapshot1.Name, pandoraSyncWaitTime) + err := snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, + snapshot1.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() @@ -3965,7 +4055,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { snapshotId := strings.Split(snapshothandle, "+")[1] ginkgo.By("Query CNS and check the volume snapshot entry") - err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId, false) + err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Fetching the username and password of the current vcenter session from secret") @@ -3981,17 +4071,14 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { username := vsphereCfg.Global.User originalPassword := vsphereCfg.Global.Password newPassword := e2eTestPassword - ginkgo.By(fmt.Sprintf("Original password %s, new password %s", originalPassword, newPassword)) - err = invokeVCenterChangePassword(username, nimbusGeneratedVcPwd, newPassword, vcAddress, - false, clientIndex) + err = invokeVCenterChangePassword(username, originalPassword, newPassword, vcAddress) gomega.Expect(err).NotTo(gomega.HaveOccurred()) originalVCPasswordChanged := true defer func() { if originalVCPasswordChanged { ginkgo.By("Reverting the password change") - err = invokeVCenterChangePassword(username, nimbusGeneratedVcPwd, originalPassword, - vcAddress, false, clientIndex) + err = invokeVCenterChangePassword(username, newPassword, originalPassword, vcAddress) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() @@ -4004,7 +4091,8 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { defer func() { if snapshot2Created { - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, snapshot2.Name, pandoraSyncWaitTime) + err = snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, snapshot2.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() @@ -4042,7 +4130,9 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { defer func() { if snapshot3Created { framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, snapshot3.Name, pandoraSyncWaitTime) + err := snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, + snapshot3.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() @@ -4063,11 +4153,11 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { snapshotId3 := strings.Split(snapshothandle3, "+")[1] ginkgo.By("Query CNS and check the volume snapshot entry") - err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId3, false) + err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId3) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Reverting the password change") - err = invokeVCenterChangePassword(username, nimbusGeneratedVcPwd, originalPassword, vcAddress, false, clientIndex) + err = invokeVCenterChangePassword(username, newPassword, originalPassword, vcAddress) gomega.Expect(err).NotTo(gomega.HaveOccurred()) originalVCPasswordChanged = false @@ -4101,55 +4191,42 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { }) /* - Multi-master and snapshot workflow - 1. Create a PVC. - 2. Create some dynamic volume snapshots. - 3. Kill csi-snapshotter container when creation of volumesnapshot is going on. - 4. Check if the snapshots go to Bound state. - 5. Create a volume using each of the snapshot. - 6. Kill csi-snapshotter container when restore operation is going on. - 7. Verify pvcs all are in Bound state. - 8. Cleanup all the snapshots and the pvc. + Multi-master and snapshot workflow + 1. Create a multi-master k8s setup (3 masters) + 2. Create a sc and a pvc using this sc + 3. Create a snapshot on the above pvc (dynamic) and also create a volume using this snaphot as source + 4. Immediately, Bring down the master vm where csi controller pod is running + 5. Alternateively we can also stop the kubelet on this node + 6. Verify the snapshot and restore workflow succeeds + 7. validate from k8s side and CNS side + 8. Bring up the node and cleanup restore-pvc, snapshot and source-pvc */ - ginkgo.It("[block-vanilla-snapshot][tkg-snapshot] Multi-master and snapshot workflow", func() { + ginkgo.It("Multi-master and snapshot workflow", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() var storageclass *storagev1.StorageClass var pvclaim *v1.PersistentVolumeClaim + var pvclaims []*v1.PersistentVolumeClaim var err error - var snapshotContentCreated = false - var sshClientConfig, sshWcpConfig *ssh.ClientConfig - var csiControllerPod, k8sMasterIP, svcMasterIp, svcMasterPwd string - var volumeSnapshotNames []string - var volumeSnapshotContents []*snapV1.VolumeSnapshotContent - var snapshotOpsScale = 3 - if guestCluster { - snapshotOpsScale = 5 - } + container_name := "csi-snapshotter" ginkgo.By("Create storage class and PVC") - if vanillaCluster { - scParameters[scParamDatastoreURL] = datastoreURL - storageclass, err = createStorageClass(client, scParameters, nil, "", "", false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } else if guestCluster { - storageclass, err = client.StorageV1().StorageClasses().Get(ctx, storagePolicyName, metav1.GetOptions{}) - if !apierrors.IsNotFound(err) { - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - } + scParameters[scParamDatastoreURL] = datastoreURL + storageclass, pvclaim, err = createPVCAndStorageClass(client, + namespace, nil, scParameters, diskSize, nil, "", false, "") + gomega.Expect(err).NotTo(gomega.HaveOccurred()) defer func() { err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() - pvclaim, persistentVolumes := createPVCAndQueryVolumeInCNS(client, namespace, nil, "", - diskSize, storageclass, true) - volHandle := persistentVolumes[0].Spec.CSI.VolumeHandle - if guestCluster { - volHandle = getVolumeIDFromSupervisorCluster(volHandle) - } + ginkgo.By("Expect claim to provision volume successfully") + pvclaims = append(pvclaims, pvclaim) + + persistentvolumes, err := fpv.WaitForPVClaimBoundPhase(client, pvclaims, framework.ClaimProvisionTimeout) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + volHandle := persistentvolumes[0].Spec.CSI.VolumeHandle gomega.Expect(volHandle).NotTo(gomega.BeEmpty()) defer func() { @@ -4160,10 +4237,18 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() - ginkgo.By("Create volume snapshot class") - volumeSnapshotClass, err := createVolumeSnapshotClass(ctx, snapc, deletionPolicy) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - + // Verify using CNS Query API if VolumeID retrieved from PV is present. + ginkgo.By(fmt.Sprintf("Invoking QueryCNSVolumeWithResult with VolumeID: %s", volHandle)) + queryResult, err := e2eVSphere.queryCNSVolumeWithResult(volHandle) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(queryResult.Volumes).ShouldNot(gomega.BeEmpty()) + gomega.Expect(queryResult.Volumes[0].VolumeId.Id).To(gomega.Equal(volHandle)) + + ginkgo.By("Create volume snapshot class") + volumeSnapshotClass, err := snapc.SnapshotV1().VolumeSnapshotClasses().Create(ctx, + getVolumeSnapshotClassSpec(snapV1.DeletionPolicy("Delete"), nil), metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer func() { ginkgo.By("In defer function deleting volume snapshot class") err := snapc.SnapshotV1().VolumeSnapshotClasses().Delete(ctx, @@ -4171,207 +4256,121 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() - if vanillaCluster { - nimbusGeneratedK8sVmPwd := GetAndExpectStringEnvVar(nimbusK8sVmPwd) - - sshClientConfig = &ssh.ClientConfig{ - User: rootUser, - Auth: []ssh.AuthMethod{ - ssh.Password(nimbusGeneratedK8sVmPwd), - }, - HostKeyCallback: ssh.InsecureIgnoreHostKey(), - } - - /* Get current leader Csi-Controller-Pod where CSI Snapshotter is running and " + - find the master node IP where this Csi-Controller-Pod is running */ - ginkgo.By("Get current leader Csi-Controller-Pod name where csi-snapshotter is running and " + - "find the master node IP where this Csi-Controller-Pod is running") - csiControllerPod, k8sMasterIP, err = getK8sMasterNodeIPWhereContainerLeaderIsRunning(ctx, - c, sshClientConfig, snapshotterContainerName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - framework.Logf("csi-snapshotter leader is in Pod %s "+ - "which is running on master node %s", csiControllerPod, k8sMasterIP) - } else if guestCluster { - svcMasterIp = GetAndExpectStringEnvVar(svcMasterIP) - svcMasterPwd = GetAndExpectStringEnvVar(svcMasterPassword) - framework.Logf("svc master ip: %s", svcMasterIp) - sshWcpConfig = &ssh.ClientConfig{ - User: rootUser, - Auth: []ssh.AuthMethod{ - ssh.Password(svcMasterPwd), - }, - HostKeyCallback: ssh.InsecureIgnoreHostKey(), - } - framework.Logf("sshwcpConfig: %v", sshWcpConfig) - csiControllerPod, k8sMasterIP, err = getK8sMasterNodeIPWhereContainerLeaderIsRunning(ctx, - client, sshWcpConfig, snapshotterContainerName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - framework.Logf("%s leader is running on pod %s "+ - "which is running on master node %s", snapshotterContainerName, csiControllerPod, k8sMasterIP) + sshClientConfig := &ssh.ClientConfig{ + User: "root", + Auth: []ssh.AuthMethod{ + ssh.Password(nimbusGeneratedK8sVmPwd), + }, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), } - for i := 0; i < snapshotOpsScale; i++ { - ginkgo.By("Create a volume snapshot") - framework.Logf("Creating snapshot no: %d", i+1) - volumeSnapshot, err := snapc.SnapshotV1().VolumeSnapshots(namespace).Create(ctx, - getVolumeSnapshotSpec(namespace, volumeSnapshotClass.Name, pvclaim.Name), metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - framework.Logf("Volume snapshot name is : %s", volumeSnapshot.Name) - snapshotCreated := true - volumeSnapshotNames = append(volumeSnapshotNames, volumeSnapshot.Name) - - defer func() { - if snapshotContentCreated { - framework.Logf("Deleting volume snapshot content") - deleteVolumeSnapshotContentWithPandoraWait(ctx, snapc, - *volumeSnapshot.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, *volumeSnapshot.Status.BoundVolumeSnapshotContentName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - if snapshotCreated { - framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) - } - }() - - if i == 1 { - ginkgo.By("Kill container CSI-Snapshotter on the master node where elected leader " + - "csi controller pod is running") + /* Get current leader Csi-Controller-Pod where CSI Provisioner is running and " + + find the master node IP where this Csi-Controller-Pod is running */ + ginkgo.By("Get current leader Csi-Controller-Pod name where csi-snapshotter is running and " + + "find the master node IP where this Csi-Controller-Pod is running") + csi_controller_pod, k8sMasterIP, err := getK8sMasterNodeIPWhereContainerLeaderIsRunning(ctx, + c, sshClientConfig, container_name) + framework.Logf("csi-snapshotter leader is in Pod %s "+ + "which is running on master node %s", csi_controller_pod, k8sMasterIP) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) - if vanillaCluster { - /* Delete elected leader CSI-Controller-Pod where csi-snapshotter is running */ - csipods, err := client.CoreV1().Pods(csiSystemNamespace).List(ctx, metav1.ListOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By("Delete elected leader CSi-Controller-Pod where csi-snapshotter is running") - err = deleteCsiControllerPodWhereLeaderIsRunning(ctx, client, snapshotterContainerName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fpod.WaitForPodsRunningReady(c, csiSystemNamespace, int32(csipods.Size()), - 0, pollTimeoutShort*2, nil) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } else if guestCluster { - err = execStopContainerOnGc(sshWcpConfig, svcMasterIp, - snapshotterContainerName, k8sMasterIP, svcNamespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + ginkgo.By("Create a volume snapshot") + snapshot1, err := snapc.SnapshotV1().VolumeSnapshots(namespace).Create(ctx, + getVolumeSnapshotSpec(namespace, volumeSnapshotClass.Name, pvclaim.Name), metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.Logf("Volume snapshot name is : %s", snapshot1.Name) + snapshotCreated := true + defer func() { + if snapshotCreated { + framework.Logf("In defer function, Deleting volume snapshot") + err := snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, + snapshot1.Name, metav1.DeleteOptions{}) + if err != nil { + if !apierrors.IsNotFound(err) { + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } } } - } + }() - for i := 0; i < snapshotOpsScale; i++ { - ginkgo.By("Verify volume snapshot is created") - framework.Logf("snapshot name: %s", volumeSnapshotNames[i]) - volumeSnapshot, err := waitForVolumeSnapshotReadyToUse(*snapc, ctx, namespace, volumeSnapshotNames[i]) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(volumeSnapshot.Status.RestoreSize.Cmp(resource.MustParse(diskSize))).To(gomega.BeZero()) - framework.Logf("VolumeSnapshot Name: %s", volumeSnapshot.Name) + /* Delete elected leader CSI-Controller-Pod where csi-snapshotter is running */ + csipods, err := client.CoreV1().Pods(csiSystemNamespace).List(ctx, metav1.ListOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + ginkgo.By("Delete elected leader CSi-Controller-Pod where csi-snapshotter is running") + err = deleteCsiControllerPodWhereLeaderIsRunning(ctx, client, csi_controller_pod) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fpod.WaitForPodsRunningReady(c, csiSystemNamespace, int32(csipods.Size()), + 0, pollTimeoutShort*2, nil) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By("Verify volume snapshot content is created") - snapshotContent, err := snapc.SnapshotV1().VolumeSnapshotContents().Get(ctx, - *volumeSnapshot.Status.BoundVolumeSnapshotContentName, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - snapshotContentCreated = true - gomega.Expect(*snapshotContent.Status.ReadyToUse).To(gomega.BeTrue()) - framework.Logf("VolumeSnapshotContent Name: %s", snapshotContent.Name) - volumeSnapshotContents = append(volumeSnapshotContents, snapshotContent) + ginkgo.By("Verify volume snapshot is Ready to use") + snapshot1_updated, err := waitForVolumeSnapshotReadyToUse(*snapc, ctx, namespace, snapshot1.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(snapshot1_updated.Status.RestoreSize.Cmp(resource.MustParse(diskSize))).To(gomega.BeZero()) - framework.Logf("Get volume snapshot ID from snapshot handle") - snapshotId, err := getVolumeSnapshotIdFromSnapshotHandle(ctx, snapshotContent, volumeSnapshotClass, - volHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - framework.Logf("snapshot Id: %s", snapshotId) + ginkgo.By("Verify volume snapshot content is created") + snapshotContent1, err := snapc.SnapshotV1().VolumeSnapshotContents().Get(ctx, + *snapshot1_updated.Status.BoundVolumeSnapshotContentName, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(*snapshotContent1.Status.ReadyToUse).To(gomega.BeTrue()) - ginkgo.By("Query CNS and check the volume snapshot entry") - err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } + framework.Logf("Get volume snapshot ID from snapshot handle") + snapshothandle := *snapshotContent1.Status.SnapshotHandle + snapshotId := strings.Split(snapshothandle, "+")[1] - if vanillaCluster { - /* Get current leader Csi-Controller-Pod where CSI Snapshotter is running and " + - find the master node IP where this Csi-Controller-Pod is running */ - ginkgo.By("Get current leader Csi-Controller-Pod name where csi-snapshotter is running and " + - "find the master node IP where this Csi-Controller-Pod is running") - csiControllerPod, k8sMasterIP, err = getK8sMasterNodeIPWhereContainerLeaderIsRunning(ctx, - c, sshClientConfig, snapshotterContainerName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - framework.Logf("csi-snapshotter leader is in Pod %s "+ - "which is running on master node %s", csiControllerPod, k8sMasterIP) - } else if guestCluster { - framework.Logf("sshwcpConfig: %v", sshWcpConfig) - csiControllerPod, k8sMasterIP, err = getK8sMasterNodeIPWhereContainerLeaderIsRunning(ctx, - client, sshWcpConfig, snapshotterContainerName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - framework.Logf("%s leader is running on pod %s "+ - "which is running on master node %s", snapshotterContainerName, csiControllerPod, k8sMasterIP) - } + ginkgo.By("Query CNS and check the volume snapshot entry") + err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) - for i := 0; i < snapshotOpsScale; i++ { - ginkgo.By("Create PVC from snapshot") - pvcSpec := getPersistentVolumeClaimSpecWithDatasource(namespace, diskSize, storageclass, nil, - v1.ReadWriteOnce, volumeSnapshotNames[i], snapshotapigroup) + ginkgo.By("Create PVC from snapshot") + pvcSpec := getPersistentVolumeClaimSpecWithDatasource(namespace, diskSize, storageclass, nil, + v1.ReadWriteOnce, snapshot1.Name, snapshotapigroup) - pvclaim2, err := fpv.CreatePVC(client, namespace, pvcSpec) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + pvclaim2, err := fpv.CreatePVC(client, namespace, pvcSpec) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) - if i == 1 { - if vanillaCluster { - /* Delete elected leader CSI-Controller-Pod where csi-snapshotter is running */ - csipods, err := client.CoreV1().Pods(csiSystemNamespace).List(ctx, metav1.ListOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By("Delete elected leader CSi-Controller-Pod where csi-snapshotter is running") - err = deleteCsiControllerPodWhereLeaderIsRunning(ctx, client, snapshotterContainerName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fpod.WaitForPodsRunningReady(c, csiSystemNamespace, int32(csipods.Size()), - 0, pollTimeoutShort*2, nil) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } else if guestCluster { - err = execStopContainerOnGc(sshWcpConfig, svcMasterIp, - snapshotterContainerName, k8sMasterIP, svcNamespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - } + csi_controller_pod, _, err = getK8sMasterNodeIPWhereContainerLeaderIsRunning(ctx, + c, sshClientConfig, container_name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + /* Delete elected leader CSI-Controller-Pod where csi-snapshotter is running */ + ginkgo.By("Delete elected leader CSi-Controller-Pod where csi-snapshotter is running") + err = deleteCsiControllerPodWhereLeaderIsRunning(ctx, client, csi_controller_pod) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) - framework.Logf("Waiting for PVCs to come to bound state") - persistentvolumes2, err := fpv.WaitForPVClaimBoundPhase(client, - []*v1.PersistentVolumeClaim{pvclaim2}, framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - volHandle2 := persistentvolumes2[0].Spec.CSI.VolumeHandle - gomega.Expect(volHandle2).NotTo(gomega.BeEmpty()) + persistentvolumes2, err := fpv.WaitForPVClaimBoundPhase(client, + []*v1.PersistentVolumeClaim{pvclaim2}, framework.ClaimProvisionTimeout) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + volHandle2 := persistentvolumes2[0].Spec.CSI.VolumeHandle + gomega.Expect(volHandle2).NotTo(gomega.BeEmpty()) - ginkgo.By("Deleting PVC2") - err = fpv.DeletePersistentVolumeClaim(client, pvclaim2.Name, namespace) + defer func() { + ginkgo.By("In defer function deleting PVC2") + err := fpv.DeletePersistentVolumeClaim(client, pvclaim2.Name, namespace) gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = e2eVSphere.waitForCNSVolumeToBeDeleted(volHandle2) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } + }() - for i := 0; i < snapshotOpsScale; i++ { - framework.Logf("Get volume snapshot ID from snapshot handle") - snapshotId, err := getVolumeSnapshotIdFromSnapshotHandle(ctx, volumeSnapshotContents[i], volumeSnapshotClass, - volHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.Logf("Deleting volume snapshot") + err = snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, + snapshot1.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + snapshotCreated = false - volumeSnapshot, err := snapc.SnapshotV1().VolumeSnapshots(namespace).Get(ctx, - volumeSnapshotNames[i], metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By("Delete volume snapshot") - _, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } + ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to allow CNS to sync with pandora", pandoraSyncWaitTime)) + time.Sleep(time.Duration(pandoraSyncWaitTime) * time.Second) }) /* - Max Snapshots per volume test - 1. Check the default configuration: - 2. Modify global-max-snapshots-per-block-volume field in vsphere-csi.conf - 3. Ensure this can be set to different values and it honors this configuration during snap create - 4. Check behavior when it is set to 0 and 5 as well - 5. Validate creation of additional snapshots beyond the configured - max-snapshots per volume fails - check error returned + Max Snapshots per volume test + 1. Check the default configuration: + 2. Modify global-max-snapshots-per-block-volume field in vsphere-csi.conf + 3. Ensure this can be set to different values and it honors this configuration during snap create + 4. Check behavior when it is set to 0 and 5 as well + 5. Validate creation of additional snapshots beyond the configured + max-snapshots per volume fails - check error returned */ - ginkgo.It("[block-vanilla-snapshot] Max Snapshots per volume test", func() { + ginkgo.It("Max Snapshots per volume test", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() var storageclass *storagev1.StorageClass @@ -4459,14 +4458,16 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { snapshotId := strings.Split(snapshothandle, "+")[1] ginkgo.By("Query CNS and check the volume snapshot entry") - err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId, false) + err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } defer func() { for _, snapName := range snapshotNames { framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, snapName, pandoraSyncWaitTime) + err := snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, + snapName, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() @@ -4478,7 +4479,8 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { defer func() { if snapshot2Created { - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, snapshot2.Name, pandoraSyncWaitTime) + err = snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, snapshot2.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() @@ -4488,11 +4490,12 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { framework.Logf("Snapshot details is %+v", snapshot2) ginkgo.By("Delete failed snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, snapshot2.Name, pandoraSyncWaitTime) + err = snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, snapshot2.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) snapshot2Created = false - ginkgo.By("Modifying the default max snapshots per volume in the secret to 6") - vsphereCfg.Snapshot.GlobalMaxSnapshotsPerBlockVolume = 6 + ginkgo.By("Modifying the default max snapshots per volume in the secret to 5") + vsphereCfg.Snapshot.GlobalMaxSnapshotsPerBlockVolume = 5 modifiedConf, err := writeConfigToSecretString(vsphereCfg) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -4512,24 +4515,6 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() - // Get CSI Controller's replica count from the setup - deployment, err := client.AppsV1().Deployments(csiSystemNamespace).Get(ctx, - vSphereCSIControllerPodNamePrefix, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - csiReplicaCount := *deployment.Spec.Replicas - - ginkgo.By("Bring down csi-controller pod") - bringDownCsiController(client) - isCSIDown := true - defer func() { - if !isCSIDown { - bringUpCsiController(client, csiReplicaCount) - } - }() - - bringUpCsiController(client, csiReplicaCount) - isCSIDown = false - for j := 4; j <= 5; j++ { ginkgo.By(fmt.Sprintf("Create a volume snapshot - %d", j)) snapshot, err := snapc.SnapshotV1().VolumeSnapshots(namespace).Create(ctx, @@ -4554,7 +4539,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { snapshotId := strings.Split(snapshothandle, "+")[1] ginkgo.By("Query CNS and check the volume snapshot entry") - err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId, false) + err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } @@ -4580,55 +4565,54 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { }) /* - Volume snapshot creation when resize is in progress - 1. Create a pvc and resize the pvc (say from 2GB to 4GB) - 2. While the resize operation is in progress, create a snapshot on this volume - 3. Expected behavior: resize operation should succeed and the - snapshot creation should succeed after resize completes + Volume snapshot creation when resize is in progress + 1. Create a pvc and resize the pvc (say from 2GB to 4GB) + 2. While the resize operation is in progress, create a snapshot on this volume + 3. Expected behavior: resize operation should succeed and the + snapshot creation should succeed after resize completes */ - ginkgo.It("[block-vanilla-snapshot][tkg-snapshot] Volume snapshot creation when resize is in progress", func() { + ginkgo.It("Volume snapshot creation when resize is in progress", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - - var volumeSnapshot *snapV1.VolumeSnapshot - var snapshotContent *snapV1.VolumeSnapshotContent - var snapshotCreated, snapshotContentCreated bool - var snapshotId string + var storageclass *storagev1.StorageClass + var pvclaim *v1.PersistentVolumeClaim var err error - if vanillaCluster { - scParameters[scParamDatastoreURL] = datastoreURL - } else if guestCluster { - scParameters[svStorageClassName] = storagePolicyName - } - ginkgo.By("Create storage class and PVC") - storageclass, err := createStorageClass(client, scParameters, nil, "", "", true, "") + scParameters[scParamDatastoreURL] = datastoreURL + storageclass, pvclaim, err = createPVCAndStorageClass(client, + namespace, nil, scParameters, diskSize, nil, "", true, "") gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer func() { err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() - pvclaim, persistentVolumes := createPVCAndQueryVolumeInCNS(client, namespace, nil, "", - diskSize, storageclass, true) - volHandle := persistentVolumes[0].Spec.CSI.VolumeHandle - if guestCluster { - volHandle = getVolumeIDFromSupervisorCluster(volHandle) - } - gomega.Expect(volHandle).NotTo(gomega.BeEmpty()) + ginkgo.By("Waiting for claim to be in bound phase") + pvc, err := fpv.WaitForPVClaimBoundPhase(client, + []*v1.PersistentVolumeClaim{pvclaim}, framework.ClaimProvisionTimeout) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(pvc).NotTo(gomega.BeEmpty()) + pv := getPvFromClaim(client, pvclaim.Namespace, pvclaim.Name) + volHandle := pv.Spec.CSI.VolumeHandle + defer func() { - err := fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volHandle) + err = fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + if volHandle != "" { + err = e2eVSphere.waitForCNSVolumeToBeDeleted(volHandle) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } }() ginkgo.By("Creating pod to attach PV to the node") pod, err := createPod(client, namespace, nil, []*v1.PersistentVolumeClaim{pvclaim}, false, execRWXCommandPod1) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer func() { + // Delete POD ginkgo.By(fmt.Sprintf("Deleting the pod %s in namespace %s", pod.Name, namespace)) err = fpod.DeletePodWithWait(client, pod) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -4636,13 +4620,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { var vmUUID string nodeName := pod.Spec.NodeName - - if vanillaCluster { - vmUUID = getNodeUUID(ctx, client, pod.Spec.NodeName) - } else if guestCluster { - vmUUID, err = getVMUUIDFromNodeName(pod.Spec.NodeName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } + vmUUID = getNodeUUID(ctx, client, pod.Spec.NodeName) ginkgo.By(fmt.Sprintf("Verify volume: %s is attached to the node: %s", volHandle, nodeName)) isDiskAttached, err := e2eVSphere.isVolumeAttachedToVM(client, volHandle, vmUUID) @@ -4653,54 +4631,48 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { currentPvcSize := pvclaim.Spec.Resources.Requests[v1.ResourceStorage] newSize := currentPvcSize.DeepCopy() newSize.Add(resource.MustParse("4Gi")) - newDiskSize := "6Gi" framework.Logf("currentPvcSize %v, newSize %v", currentPvcSize, newSize) claims, err := expandPVCSize(pvclaim, newSize, client) gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Expect(claims).NotTo(gomega.BeNil()) ginkgo.By("Create volume snapshot class") - volumeSnapshotClass, err := createVolumeSnapshotClass(ctx, snapc, deletionPolicy) + volumeSnapshotClass, err := snapc.SnapshotV1().VolumeSnapshotClasses().Create(ctx, + getVolumeSnapshotClassSpec(snapV1.DeletionPolicy("Delete"), nil), metav1.CreateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer func() { - if vanillaCluster { - err = snapc.SnapshotV1().VolumeSnapshotClasses().Delete(ctx, volumeSnapshotClass.Name, - metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } + err := snapc.SnapshotV1().VolumeSnapshotClasses().Delete(ctx, + volumeSnapshotClass.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() - ginkgo.By("Create a dynamic volume snapshot") - if vanillaCluster { - volumeSnapshot, snapshotContent, snapshotCreated, - snapshotContentCreated, snapshotId, err = createDynamicVolumeSnapshot(ctx, namespace, snapc, volumeSnapshotClass, - pvclaim, volHandle, diskSize) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } else if guestCluster { - volumeSnapshot, snapshotContent, snapshotCreated, - snapshotContentCreated, snapshotId, err = createDynamicVolumeSnapshot(ctx, namespace, snapc, volumeSnapshotClass, - pvclaim, volHandle, newDiskSize) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - defer func() { - if snapshotContentCreated { - err = deleteVolumeSnapshotContent(ctx, snapshotContent, snapc, namespace, pandoraSyncWaitTime) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } + ginkgo.By("Create a volume snapshot") + volumeSnapshot, err := snapc.SnapshotV1().VolumeSnapshots(namespace).Create(ctx, + getVolumeSnapshotSpec(namespace, volumeSnapshotClass.Name, pvclaim.Name), metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.Logf("Volume snapshot name is : %s", volumeSnapshot.Name) + snapshotCreated := true + defer func() { if snapshotCreated { framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeletedWithPandoraWait(ctx, snapc, - *volumeSnapshot.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) + err := snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, + volumeSnapshot.Name, metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() - ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to allow CNS to sync with pandora", pandoraSyncWaitTime)) - time.Sleep(time.Duration(pandoraSyncWaitTime) * time.Second) + ginkgo.By("Verify volume snapshot is created") + volumeSnapshot, err = waitForVolumeSnapshotReadyToUse(*snapc, ctx, namespace, volumeSnapshot.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(volumeSnapshot.Status.RestoreSize.Cmp(resource.MustParse("2Gi"))).To(gomega.BeZero()) + + ginkgo.By("Verify volume snapshot content is created") + snapshotContent, err := snapc.SnapshotV1().VolumeSnapshotContents().Get(ctx, + *volumeSnapshot.Status.BoundVolumeSnapshotContentName, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(*snapshotContent.Status.ReadyToUse).To(gomega.BeTrue()) ginkgo.By("Waiting for file system resize to finish") claims, err = waitForFSResize(pvclaim, client) @@ -4724,11 +4696,6 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { queryResult.Volumes[0].BackingObjectDetails.(*cnstypes.CnsBlockBackingDetails).CapacityInMb) } gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Delete dynamic volume snapshot") - snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) /* @@ -4737,7 +4704,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { 2. Create Snapshot class and take a snapshot of the volume 3. Cleanup of snapshot, pvc and sc */ - ginkgo.It("[block-vanilla-snapshot] Volume provision and snapshot creation/restore on VVOL Datastore", func() { + ginkgo.It("Volume provision and snapshot creation/restore on VVOL Datastore", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() var storageclass *storagev1.StorageClass @@ -4806,17 +4773,16 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { defer func() { if snapshotContentCreated { framework.Logf("Deleting volume snapshot content") - deleteVolumeSnapshotContentWithPandoraWait(ctx, snapc, - *volumeSnapshot.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, *volumeSnapshot.Status.BoundVolumeSnapshotContentName) + err := snapc.SnapshotV1().VolumeSnapshotContents().Delete(ctx, + *volumeSnapshot.Status.BoundVolumeSnapshotContentName, metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } if snapshotCreated { framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) + err := snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, volumeSnapshot.Name, + metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() @@ -4838,7 +4804,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { snapshotId := strings.Split(snapshothandle, "+")[1] ginkgo.By("Query CNS and check the volume snapshot entry") - err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId, false) + err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Create PVC from snapshot") @@ -4862,7 +4828,9 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { }() framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) + err = snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, volumeSnapshot.Name, + metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) snapshotCreated = false err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, snapshotContent.ObjectMeta.Name) @@ -4870,7 +4838,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { snapshotContentCreated = false ginkgo.By("Verify snapshot entry is deleted from CNS") - err = verifySnapshotIsDeletedInCNS(volHandle, snapshotId, false) + err = verifySnapshotIsDeletedInCNS(volHandle, snapshotId) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) @@ -4880,7 +4848,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { 2. Create Snapshot class and take a snapshot of the volume 3. Cleanup of snapshot, pvc and sc */ - ginkgo.It("[block-vanilla-snapshot] Volume provision and snapshot creation/restore on VMFS Datastore", func() { + ginkgo.It("Volume provision and snapshot creation/restore on VMFS Datastore", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() var storageclass *storagev1.StorageClass @@ -4949,17 +4917,16 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { defer func() { if snapshotContentCreated { framework.Logf("Deleting volume snapshot content") - deleteVolumeSnapshotContentWithPandoraWait(ctx, snapc, - *volumeSnapshot.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, *volumeSnapshot.Status.BoundVolumeSnapshotContentName) + err := snapc.SnapshotV1().VolumeSnapshotContents().Delete(ctx, + *volumeSnapshot.Status.BoundVolumeSnapshotContentName, metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } if snapshotCreated { framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) + err := snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, volumeSnapshot.Name, + metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() @@ -4981,7 +4948,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { snapshotId := strings.Split(snapshothandle, "+")[1] ginkgo.By("Query CNS and check the volume snapshot entry") - err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId, false) + err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Create PVC from snapshot") @@ -5005,7 +4972,9 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { }() framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) + err = snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, volumeSnapshot.Name, + metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) snapshotCreated = false err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, snapshotContent.ObjectMeta.Name) @@ -5013,20 +4982,20 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { snapshotContentCreated = false ginkgo.By("Verify snapshot entry is deleted from CNS") - err = verifySnapshotIsDeletedInCNS(volHandle, snapshotId, false) + err = verifySnapshotIsDeletedInCNS(volHandle, snapshotId) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) /* - Scale-up creation of snapshots across multiple volumes - - 1. Create a few pvcs (around 25) - 2. Trigger parallel snapshot create calls on all pvcs - 3. Trigger parallel snapshot delete calls on all pvcs - 4. All calls in (2) and (3) should succeed since these are - triggered via k8s API (might take longer time) - 5. Trigger create/delete calls and ensure there are no stale entries left behind - 6. Create multiple volumes from the same snapshot + Scale-up creation of snapshots across multiple volumes + + 1. Create a few pvcs (around 25) + 2. Trigger parallel snapshot create calls on all pvcs + 3. Trigger parallel snapshot delete calls on all pvcs + 4. All calls in (2) and (3) should succeed since these are + triggered via k8s API (might take longer time) + 5. Trigger create/delete calls and ensure there are no stale entries left behind + 6. Create multiple volumes from the same snapshot */ /* @@ -5039,24 +5008,21 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { 4. Volume restore 5. snapshot create/delete workflow */ - ginkgo.It("[block-vanilla-snapshot][tkg-snapshot] Scale-up creation of snapshots across multiple volumes", func() { + ginkgo.It("Scale-up creation of snapshots across multiple volumes", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - var storageclass *storagev1.StorageClass volumesnapshots := make([]*snapV1.VolumeSnapshot, volumeOpsScale) snapshotContents := make([]*snapV1.VolumeSnapshotContent, volumeOpsScale) pvclaims := make([]*v1.PersistentVolumeClaim, volumeOpsScale) pvclaims2 := make([]*v1.PersistentVolumeClaim, volumeOpsScale) + var persistentvolumes []*v1.PersistentVolume + var err error ginkgo.By("Create storage class and PVC") - if vanillaCluster { - scParameters[scParamDatastoreURL] = datastoreURL - } else if guestCluster { - scParameters[svStorageClassName] = storagePolicyName - } + scParameters[scParamDatastoreURL] = datastoreURL curtime := time.Now().Unix() randomValue := rand.Int() @@ -5066,20 +5032,21 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { scName := "snapshot-scale" + curtimestring + val storageclass, err = createStorageClass(client, scParameters, nil, "", "", false, scName) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer func() { err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() ginkgo.By("Create volume snapshot class") - volumeSnapshotClass, err := createVolumeSnapshotClass(ctx, snapc, deletionPolicy) + volumeSnapshotClass, err := snapc.SnapshotV1().VolumeSnapshotClasses().Create(ctx, + getVolumeSnapshotClassSpec(snapV1.DeletionPolicy("Delete"), nil), metav1.CreateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer func() { - if vanillaCluster { - err = snapc.SnapshotV1().VolumeSnapshotClasses().Delete(ctx, volumeSnapshotClass.Name, - metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } + err := snapc.SnapshotV1().VolumeSnapshotClasses().Delete(ctx, volumeSnapshotClass.Name, + metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() ginkgo.By("Creating PVCs using the Storage Class") @@ -5133,7 +5100,9 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { for i := 0; i < volumeOpsScale; i++ { framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumesnapshots[i].Name, pandoraSyncWaitTime) + err = snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, volumesnapshots[i].Name, + metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, snapshotContents[i].ObjectMeta.Name) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -5158,1497 +5127,4 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() { } }() }) - - /* Create/Delete snapshot via k8s API using VolumeSnapshotContent (Pre-Provisioned Snapshots) - - //Steps to create pre-provisioned snapshot in Guest Cluster - - 1. In this approach create a dynamic VolumeSnapshot in Guest with a VolumeSnapshotClass with “Delete” - deletion policy. - 2. Note the VolumeSnapshot name created on the Supervisor. - 3. Explicitly change the deletionPolicy of VolumeSnapshotContent on Guest to “Retain”. - 4. Delete the VolumeSnapshot. This will leave the VolumeSnapshotContent on the Guest as is, - since deletionPolicy was “Retain” - 5. Explicitly delete the VolumeSnapshotContent. - 6. In this approach, we now have Supervisor VolumeSnapshot that doesn’t have a corresponding - VolumeSnapshot-VolumeSnapshotContent on Guest. - 7. Create a VolumeSnapshotContent that points to the Supervisor VolumeSnapshot, and create a - VolumeSnapshot on Guest that point to the VolumeSnapshotContent. - - // TestCase Steps - 1. Create a storage class and create a pvc using this SC - 2. The volumesnapshotclass is set to delete - 3. Create a dynamic volume snapshot - 4. Create a pre-provisoned snapshot following th steps mentioned above - 5. Perform cleanup - */ - - ginkgo.It("[tkg-snapshot] Verify pre-provisioned static snapshot workflow", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - if vanillaCluster { - scParameters[scParamDatastoreURL] = datastoreURL - } else if guestCluster { - scParameters[svStorageClassName] = storagePolicyName - } - - ginkgo.By("Create storage class and PVC") - storageclass, err := createStorageClass(client, scParameters, nil, "", "", true, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - defer func() { - err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - pvclaim, persistentVolumes := createPVCAndQueryVolumeInCNS(client, namespace, nil, "", diskSize, storageclass, true) - volHandle := persistentVolumes[0].Spec.CSI.VolumeHandle - if guestCluster { - volHandle = getVolumeIDFromSupervisorCluster(volHandle) - } - gomega.Expect(volHandle).NotTo(gomega.BeEmpty()) - - defer func() { - err := fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Create volume snapshot class") - volumeSnapshotClass, err := createVolumeSnapshotClass(ctx, snapc, deletionPolicy) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Create a dynamic volume snapshot") - volumeSnapshot, snapshotContent, snapshotCreated, - snapshotContentCreated, dynamicSnapshotId, err := createDynamicVolumeSnapshot(ctx, namespace, snapc, - volumeSnapshotClass, pvclaim, volHandle, diskSize) - defer func() { - if snapshotCreated { - framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeletedWithPandoraWait(ctx, snapc, - *volumeSnapshot.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - if snapshotContentCreated { - err = deleteVolumeSnapshotContent(ctx, snapshotContent, snapc, namespace, pandoraSyncWaitTime) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }() - - framework.Logf("Get volume snapshot handle from Supervisor Cluster") - snapshotId, _, svcVolumeSnapshotName, err := getSnapshotHandleFromSupervisorCluster(ctx, - volumeSnapshotClass, *snapshotContent.Status.SnapshotHandle) - - ginkgo.By("Create pre-provisioned snapshot") - _, staticSnapshot, staticSnapshotContentCreated, - staticSnapshotCreated, err := createPreProvisionedSnapshotInGuestCluster(ctx, volumeSnapshot, snapshotContent, - snapc, namespace, pandoraSyncWaitTime, svcVolumeSnapshotName, diskSize) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - defer func() { - if staticSnapshotCreated { - framework.Logf("Deleting static volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, staticSnapshot.Name, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeletedWithPandoraWait(ctx, snapc, - *staticSnapshot.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - if staticSnapshotContentCreated { - framework.Logf("Deleting static volume snapshot content") - deleteVolumeSnapshotContentWithPandoraWait(ctx, snapc, - *staticSnapshot.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, *staticSnapshot.Status.BoundVolumeSnapshotContentName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }() - - ginkgo.By("Delete pre-provisioned snapshot") - staticSnapshotCreated, staticSnapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - staticSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Delete dynamic volume snapshot") - snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, dynamicSnapshotId) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - - /* - Volume restore using snapshot (a) dynamic snapshot (b) pre-provisioned snapshot - 1. Create a sc, a pvc and attach the pvc to a pod, write a file - 2. Create pre-provisioned and dynamically provisioned snapshots using this pvc - 3. Create new volumes (pvcFromPreProvSS and pvcFromDynamicSS) using these - snapshots as source, use the same sc - 4. Ensure the pvc gets provisioned and is Bound - 5. Attach the pvc to a pod and ensure data from snapshot is available - (file that was written in step.1 should be available) - 6. And also write new data to the restored volumes and it should succeed - 7. Delete the snapshots and pvcs/pods created in steps 1,2,3 - 8. Continue to write new data to the restore volumes and it should succeed - 9. Create new snapshots on restore volume and verify it succeeds - 10. Run cleanup: Delete snapshots, restored-volumes, pods - */ - - ginkgo.It("[tkg-snapshot] Volume restore using dynamic and pre-provisioned snapshot on guest cluster", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - var staticSnapshotCreated, staticSnapshotContentCreated bool - - if vanillaCluster { - scParameters[scParamDatastoreURL] = datastoreURL - } else if guestCluster { - scParameters[svStorageClassName] = storagePolicyName - } - - ginkgo.By("Create storage class and PVC") - storageclass, err := createStorageClass(client, scParameters, nil, "", "", true, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - defer func() { - err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - pvclaim, persistentVolumes := createPVCAndQueryVolumeInCNS(client, namespace, nil, "", diskSize, storageclass, true) - volHandle := persistentVolumes[0].Spec.CSI.VolumeHandle - if guestCluster { - volHandle = getVolumeIDFromSupervisorCluster(volHandle) - } - gomega.Expect(volHandle).NotTo(gomega.BeEmpty()) - - defer func() { - err := fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Create volume snapshot class") - volumeSnapshotClass, err := createVolumeSnapshotClass(ctx, snapc, deletionPolicy) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Create a dynamic volume snapshot") - volumeSnapshot, snapshotContent, snapshotCreated, - snapshotContentCreated, snapshotId, err := createDynamicVolumeSnapshot(ctx, namespace, snapc, volumeSnapshotClass, - pvclaim, volHandle, diskSize) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - if snapshotContentCreated { - err = deleteVolumeSnapshotContent(ctx, snapshotContent, snapc, namespace, pandoraSyncWaitTime) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - if snapshotCreated { - framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeletedWithPandoraWait(ctx, snapc, - *volumeSnapshot.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }() - - ginkgo.By("Restore PVC using dynamic volume snapshot") - pvclaim2, persistentVolumes2, pod := verifyVolumeRestoreOperation(ctx, client, - namespace, storageclass, volumeSnapshot, true) - volHandle2 := persistentVolumes2[0].Spec.CSI.VolumeHandle - if guestCluster { - volHandle2 = getVolumeIDFromSupervisorCluster(volHandle2) - } - gomega.Expect(volHandle2).NotTo(gomega.BeEmpty()) - defer func() { - err := fpv.DeletePersistentVolumeClaim(client, pvclaim2.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volHandle2) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - defer func() { - ginkgo.By(fmt.Sprintf("Deleting the pod %s in namespace %s", pod.Name, namespace)) - err = fpod.DeletePodWithWait(client, pod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - framework.Logf("Get volume snapshot handle from Supervisor Cluster") - _, _, svcVolumeSnapshotName, err := getSnapshotHandleFromSupervisorCluster(ctx, volumeSnapshotClass, - *snapshotContent.Status.SnapshotHandle) - - ginkgo.By("Create pre-provisioned snapshot in Guest Cluster") - _, staticSnapshot, staticSnapshotContentCreated, - staticSnapshotCreated, err := createPreProvisionedSnapshotInGuestCluster(ctx, volumeSnapshot, snapshotContent, - snapc, namespace, pandoraSyncWaitTime, svcVolumeSnapshotName, diskSize) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - if staticSnapshotCreated { - framework.Logf("Deleting static volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, staticSnapshot.Name, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeletedWithPandoraWait(ctx, snapc, - *staticSnapshot.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - if staticSnapshotContentCreated { - framework.Logf("Deleting static volume snapshot content") - deleteVolumeSnapshotContentWithPandoraWait(ctx, snapc, - *staticSnapshot.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, *staticSnapshot.Status.BoundVolumeSnapshotContentName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }() - - ginkgo.By("Restore PVC using pre-provisioned snapshot") - pvclaim3, persistentVolumes3, pod2 := verifyVolumeRestoreOperation(ctx, client, - namespace, storageclass, staticSnapshot, true) - volHandle3 := persistentVolumes3[0].Spec.CSI.VolumeHandle - if guestCluster { - volHandle3 = getVolumeIDFromSupervisorCluster(volHandle3) - } - gomega.Expect(volHandle3).NotTo(gomega.BeEmpty()) - - defer func() { - err := fpv.DeletePersistentVolumeClaim(client, pvclaim3.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volHandle3) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - defer func() { - ginkgo.By(fmt.Sprintf("Deleting the pod %s in namespace %s", pod2.Name, namespace)) - err = fpod.DeletePodWithWait(client, pod2) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Take a snapshot of restored PVC created from dynamic snapshot") - volumeSnapshot3, _, snapshotCreated3, - snapshotContentCreated3, snapshotId3, err := createDynamicVolumeSnapshot(ctx, namespace, snapc, volumeSnapshotClass, - pvclaim2, volHandle, diskSize) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - if snapshotContentCreated3 { - framework.Logf("Deleting volume snapshot content") - deleteVolumeSnapshotContentWithPandoraWait(ctx, snapc, - *volumeSnapshot3.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, *volumeSnapshot3.Status.BoundVolumeSnapshotContentName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - if snapshotCreated3 { - framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot3.Name, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeletedWithPandoraWait(ctx, snapc, - *volumeSnapshot3.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }() - - ginkgo.By("Delete dynamic volume snapshot") - snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot3, pandoraSyncWaitTime, volHandle2, snapshotId3) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Delete pre-provisioned snapshot") - staticSnapshotCreated, staticSnapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - staticSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Delete dynamic volume snapshot") - snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - - /* - Pre-provisioned snapshot using incorrect/non-existing static snapshot - 1. Create a sc, and pvc using this sc - 2. Create a snapshot for this pvc - 3. Create a VolumeSnapshotContent CR using above snapshot-id, by passing the snapshotHandle - 4. Create a VolumeSnapshot using above content as source - 5. VolumeSnapshot and VolumeSnapshotContent should be created successfully and readToUse set to True - 6. Delete the snapshot created in step-4 - 7. Restore: Create a volume using above pre-provisioned snapshot k8s object - (note the snapshotHandle its pointing to has been deleted) - 8. Volume Create should fail with an appropriate error on k8s side - */ - ginkgo.It("[tkg-snapshot] Restore volume using non-existing static snapshot", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - var staticSnapshotCreated, staticSnapshotContentCreated bool - - if vanillaCluster { - scParameters[scParamDatastoreURL] = datastoreURL - } else if guestCluster { - scParameters[svStorageClassName] = storagePolicyName - } - - ginkgo.By("Create storage class and PVC") - storageclass, err := createStorageClass(client, scParameters, nil, "", "", true, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - defer func() { - err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - pvclaim, persistentVolumes := createPVCAndQueryVolumeInCNS(client, namespace, nil, "", diskSize, storageclass, true) - volHandle := persistentVolumes[0].Spec.CSI.VolumeHandle - if guestCluster { - volHandle = getVolumeIDFromSupervisorCluster(volHandle) - } - gomega.Expect(volHandle).NotTo(gomega.BeEmpty()) - - defer func() { - err := fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Create volume snapshot class") - volumeSnapshotClass, err := createVolumeSnapshotClass(ctx, snapc, deletionPolicy) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Create a dynamic volume snapshot") - volumeSnapshot, snapshotContent, snapshotCreated, - snapshotContentCreated, _, err := createDynamicVolumeSnapshot(ctx, namespace, snapc, volumeSnapshotClass, - pvclaim, volHandle, diskSize) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - if snapshotContentCreated { - err = deleteVolumeSnapshotContent(ctx, snapshotContent, snapc, namespace, pandoraSyncWaitTime) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - if snapshotCreated { - framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeletedWithPandoraWait(ctx, snapc, - *volumeSnapshot.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }() - - framework.Logf("Get volume snapshot handle from Supervisor Cluster") - staticSnapshotId, _, svcVolumeSnapshotName, err := getSnapshotHandleFromSupervisorCluster(ctx, - volumeSnapshotClass, *snapshotContent.Status.SnapshotHandle) - - ginkgo.By("Create pre-provisioned snapshot") - _, staticSnapshot, staticSnapshotContentCreated, - staticSnapshotCreated, err := createPreProvisionedSnapshotInGuestCluster(ctx, volumeSnapshot, snapshotContent, - snapc, namespace, pandoraSyncWaitTime, svcVolumeSnapshotName, diskSize) - defer func() { - if staticSnapshotCreated { - framework.Logf("Deleting static volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, staticSnapshot.Name, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeletedWithPandoraWait(ctx, snapc, - *staticSnapshot.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - if staticSnapshotContentCreated { - framework.Logf("Deleting static volume snapshot content") - deleteVolumeSnapshotContentWithPandoraWait(ctx, snapc, - *staticSnapshot.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, *staticSnapshot.Status.BoundVolumeSnapshotContentName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }() - - ginkgo.By("Delete static volume snapshot") - staticSnapshotCreated, staticSnapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - staticSnapshot, pandoraSyncWaitTime, volHandle, staticSnapshotId) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Create PVC using the snapshot deleted") - pvcSpec := getPersistentVolumeClaimSpecWithDatasource(namespace, diskSize, storageclass, nil, - v1.ReadWriteOnce, staticSnapshot.Name, snapshotapigroup) - - pvclaim2, err := fpv.CreatePVC(client, namespace, pvcSpec) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - _, err = fpv.WaitForPVClaimBoundPhase(client, []*v1.PersistentVolumeClaim{pvclaim2}, - framework.ClaimProvisionShortTimeout) - gomega.Expect(err).To(gomega.HaveOccurred()) - - expectedErrMsg := "error getting handle for DataSource Type VolumeSnapshot by Name" - ginkgo.By(fmt.Sprintf("Expected failure message: %+q", expectedErrMsg)) - isFailureFound := checkEventsforError(client, namespace, - metav1.ListOptions{FieldSelector: fmt.Sprintf("involvedObject.name=%s", pvclaim2.Name)}, - expectedErrMsg) - gomega.Expect(isFailureFound).To(gomega.BeTrue(), - fmt.Sprintf("Expected pvc creation failure with error message: %s", expectedErrMsg)) - defer func() { - err := fpv.DeletePersistentVolumeClaim(client, pvclaim2.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - }) - - /* - Create a volume from a snapshot that is still not ready-to-use - 1. Create a pre-provisioned snapshot pointing to a VolumeSnapshotContent - which is still not provisioned (or does not exist) - 2. The snapshot will have status.readyToUse: false and snapshot is in Pending state - 3. Create a volume using the above snapshot as source and ensure the provisioning fails with error: - ProvisioningFailed | snapshot <> not bound - 4. pvc is stuck in Pending - 5. Once the VolumeSnapshotContent is created, snapshot should have status.readyToUse: true - 6. The volume should now get provisioned successfully - 7. Validate the pvc is Bound - 8. Cleanup the snapshot and pvc - */ - ginkgo.It("[tkg-snapshot] Restore volume from a static snapshot that is still not ready-to-use", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - if vanillaCluster { - scParameters[scParamDatastoreURL] = datastoreURL - } else if guestCluster { - scParameters[svStorageClassName] = storagePolicyName - } - - ginkgo.By("Create storage class and PVC") - storageclass, err := createStorageClass(client, scParameters, nil, "", "", true, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - defer func() { - err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - pvclaim, persistentVolumes := createPVCAndQueryVolumeInCNS(client, namespace, nil, "", diskSize, storageclass, true) - volHandle := persistentVolumes[0].Spec.CSI.VolumeHandle - if guestCluster { - volHandle = getVolumeIDFromSupervisorCluster(volHandle) - } - gomega.Expect(volHandle).NotTo(gomega.BeEmpty()) - - defer func() { - err := fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Create volume snapshot class") - volumeSnapshotClass, err := createVolumeSnapshotClass(ctx, snapc, deletionPolicy) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Create a dynamic volume snapshot") - volumeSnapshot, snapshotContent, snapshotCreated, - snapshotContentCreated, _, err := createDynamicVolumeSnapshot(ctx, namespace, snapc, volumeSnapshotClass, - pvclaim, volHandle, diskSize) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - if snapshotContentCreated { - err = deleteVolumeSnapshotContent(ctx, snapshotContent, snapc, namespace, pandoraSyncWaitTime) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - if snapshotCreated { - framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeletedWithPandoraWait(ctx, snapc, - *volumeSnapshot.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }() - - framework.Logf("Get volume snapshot handle from Supervisor Cluster") - snapshotId, _, svcVolumeSnapshotName, err := getSnapshotHandleFromSupervisorCluster(ctx, volumeSnapshotClass, - *snapshotContent.Status.SnapshotHandle) - - ginkgo.By("Create Pre-provisioned snapshot in Guest Cluster") - framework.Logf("Change the deletion policy of VolumeSnapshotContent from Delete to Retain " + - "in Guest Cluster") - updatedSnapshotContent, err := changeDeletionPolicyOfVolumeSnapshotContent(ctx, - snapshotContent, snapc, namespace, snapV1.VolumeSnapshotContentRetain) - - framework.Logf("Delete dynamic volume snapshot from Guest Cluster") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) - - framework.Logf("Delete VolumeSnapshotContent from Guest Cluster explicitly") - err = deleteVolumeSnapshotContent(ctx, updatedSnapshotContent, snapc, namespace, pandoraSyncWaitTime) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - framework.Logf("Creating static VolumeSnapshotContent in Guest Cluster using "+ - "supervisor VolumeSnapshotName %s", svcVolumeSnapshotName) - staticSnapshotContent, err := snapc.SnapshotV1().VolumeSnapshotContents().Create(ctx, - getVolumeSnapshotContentSpec(snapV1.DeletionPolicy("Delete"), svcVolumeSnapshotName, - "static-vs", namespace), metav1.CreateOptions{}) - - framework.Logf("Verify VolumeSnapshotContent is created or not in Guest Cluster") - staticSnapshotContent, err = snapc.SnapshotV1().VolumeSnapshotContents().Get(ctx, - staticSnapshotContent.Name, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - framework.Logf("Snapshotcontent name is %s", staticSnapshotContent.ObjectMeta.Name) - if !*staticSnapshotContent.Status.ReadyToUse { - framework.Logf("VolumeSnapshotContent is not ready to use") - } - - ginkgo.By("Create a static volume snapshot using static snapshotcontent") - staticVolumeSnapshot, err := snapc.SnapshotV1().VolumeSnapshots(namespace).Create(ctx, - getVolumeSnapshotSpecByName(namespace, "static-vs", staticSnapshotContent.ObjectMeta.Name), - metav1.CreateOptions{}) - if err != nil { - framework.Logf("failed to create static volume snapshot: %v", err) - } - framework.Logf("Volume snapshot name is : %s", staticVolumeSnapshot.Name) - - ginkgo.By("Create PVC while snapshot is still provisioning") - pvcSpec := getPersistentVolumeClaimSpecWithDatasource(namespace, diskSize, storageclass, nil, - v1.ReadWriteOnce, "static-vs", snapshotapigroup) - pvclaim2, err := fpv.CreatePVC(client, namespace, pvcSpec) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - persistentvolumes2, err := fpv.WaitForPVClaimBoundPhase(client, - []*v1.PersistentVolumeClaim{pvclaim2}, framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - volHandle2 := persistentvolumes2[0].Spec.CSI.VolumeHandle - gomega.Expect(volHandle2).NotTo(gomega.BeEmpty()) - if guestCluster { - volHandle2 = getVolumeIDFromSupervisorCluster(volHandle2) - } - defer func() { - err := fpv.DeletePersistentVolumeClaim(client, pvclaim2.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volHandle2) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Verify static volume snapshot is created") - staticSnapshot, err := waitForVolumeSnapshotReadyToUse(*snapc, ctx, namespace, staticVolumeSnapshot.Name) - if err != nil { - framework.Logf("failed to wait for volume snapshot: %v", err) - } - if staticSnapshot.Status.RestoreSize.Cmp(resource.MustParse(diskSize)) != 0 { - framework.Logf("expected RestoreSize does not match") - } - framework.Logf("Snapshot details is %+v", staticSnapshot) - - ginkgo.By("Delete pre-provisioned snapshot") - staticSnapshotCreated, staticSnapshotContentCreated, err := deleteVolumeSnapshot(ctx, snapc, namespace, - staticSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - if staticSnapshotCreated { - framework.Logf("Deleting static volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, staticSnapshot.Name, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeletedWithPandoraWait(ctx, snapc, - *staticSnapshot.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - if staticSnapshotContentCreated { - framework.Logf("Deleting static volume snapshot content") - deleteVolumeSnapshotContentWithPandoraWait(ctx, snapc, - *staticSnapshot.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, *staticSnapshot.Status.BoundVolumeSnapshotContentName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }() - - ginkgo.By("Delete dynamic volume snapshot") - snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - - /* - Create snapshot on Supervisor cluster - - Create a Storage Class, and a PVC using this SC in Supervisor cluster. - Create a dynamic snapshot in supervisor using above PVC as source - Snapshot creation should fail with appropriate error message. - Cleanup the snapshots, PVC and SC - */ - - ginkgo.It("[tkg-snapshot] Verify Snapshot creation should fail on supervisor cluster", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - storageclass, err := svcClient.StorageV1().StorageClasses().Get(ctx, storagePolicyName, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - pvclaim, err := fpv.CreatePVC(svcClient, svcNamespace, - getPersistentVolumeClaimSpecWithStorageClass(svcNamespace, diskSize, storageclass, nil, v1.ReadWriteOnce)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Expect claim to provision volume successfully") - persistentvolumes, err := fpv.WaitForPVClaimBoundPhase(svcClient, - []*v1.PersistentVolumeClaim{pvclaim}, framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to provision volume") - volHandle := persistentvolumes[0].Spec.CSI.VolumeHandle - gomega.Expect(volHandle).NotTo(gomega.BeEmpty()) - defer func() { - err = fpv.DeletePersistentVolumeClaim(svcClient, pvclaim.Name, pvclaim.Namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By(fmt.Sprintf("Invoking QueryCNSVolumeWithResult with VolumeID: %s", volHandle)) - queryResult, err := e2eVSphere.queryCNSVolumeWithResult(volHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(queryResult.Volumes).ShouldNot(gomega.BeEmpty()) - gomega.Expect(queryResult.Volumes[0].VolumeId.Id).To(gomega.Equal(volHandle)) - - ginkgo.By("Create volume snapshot class") - volumeSnapshotClass, err := createVolumeSnapshotClass(ctx, snapc, deletionPolicy) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Expected snapshot creation failure on supervisor cluster") - _, err = snapc.SnapshotV1().VolumeSnapshots(svcNamespace).Create(ctx, - getVolumeSnapshotSpec(svcNamespace, volumeSnapshotClass.Name, pvclaim.Name), metav1.CreateOptions{}) - if err != nil { - framework.Logf(err.Error()) - } - }) - - /* - Perform online resize on restored volume - 1. Create a Storage Class, a PVC and attach the PVC to a Pod, write a file - 2. Create dynamically provisioned snapshots using this PVC - 3. Create new volume using this snapshots as source, use the same SC and attach it to a Pod. - 4. Ensure the PVC gets provisioned and is Bound. - 5. Verify the previous snapshot data is intact and write new data to restored volume - 6. Perform online resize on the restored volume and make sure resize should go fine. - 7. Create dynamically provisioned snapshots using the PVC created in step #4 - 8. Verify snapshot size. It should be same as that of restored volume size. - 9. Run cleanup: Delete snapshots, restored-volumes, pods. - */ - ginkgo.It("[tkg-snapshot] Perform online resize on restored volume", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - var storageclass *storagev1.StorageClass - var pvclaim *v1.PersistentVolumeClaim - var err error - var snapshotContentCreated = false - var snapshotCreated = false - - if vanillaCluster { - scParameters[scParamDatastoreURL] = datastoreURL - } else if guestCluster { - scParameters[svStorageClassName] = storagePolicyName - } - - ginkgo.By("Create storage class and PVC") - storageclass, err = createStorageClass(client, scParameters, nil, "", "", true, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - defer func() { - err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - pvclaim, persistentVolumes := createPVCAndQueryVolumeInCNS(client, namespace, nil, "", diskSize, storageclass, true) - volHandle := persistentVolumes[0].Spec.CSI.VolumeHandle - if guestCluster { - volHandle = getVolumeIDFromSupervisorCluster(volHandle) - } - gomega.Expect(volHandle).NotTo(gomega.BeEmpty()) - - defer func() { - ginkgo.By(fmt.Sprintf("Deleting the pvc %s in namespace %s", pvclaim.Name, namespace)) - err := fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Create/Get volume snapshot class") - volumeSnapshotClass, err := createVolumeSnapshotClass(ctx, snapc, deletionPolicy) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - defer func() { - if vanillaCluster { - err := snapc.SnapshotV1().VolumeSnapshotClasses().Delete(ctx, volumeSnapshotClass.Name, - metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }() - - ginkgo.By("Create a volume snapshot") - volumeSnapshot, _, snapshotCreated, - snapshotContentCreated, snapshotId, err := createDynamicVolumeSnapshot(ctx, namespace, snapc, volumeSnapshotClass, - pvclaim, volHandle, diskSize) - - defer func() { - if snapshotContentCreated { - framework.Logf("Deleting volume snapshot content") - deleteVolumeSnapshotContentWithPandoraWait(ctx, snapc, - *volumeSnapshot.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, *volumeSnapshot.Status.BoundVolumeSnapshotContentName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - if snapshotCreated { - framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) - } - }() - - ginkgo.By("Create PVC from Snapshot and verify restore volume operations") - pvclaim2, persistentVolumes2, pod := verifyVolumeRestoreOperation(ctx, client, - namespace, storageclass, volumeSnapshot, true) - volHandle2 := persistentVolumes2[0].Spec.CSI.VolumeHandle - svcPVCName2 := persistentVolumes2[0].Spec.CSI.VolumeHandle - if guestCluster { - volHandle2 = getVolumeIDFromSupervisorCluster(volHandle2) - } - gomega.Expect(volHandle2).NotTo(gomega.BeEmpty()) - - defer func() { - // Delete POD - ginkgo.By(fmt.Sprintf("Deleting the pod %s in namespace %s", pod.Name, namespace)) - err = fpod.DeletePodWithWait(client, pod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By(fmt.Sprintf("Deleting the pvc %s in namespace %s", pvclaim2.Name, namespace)) - err := fpv.DeletePersistentVolumeClaim(client, pvclaim2.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volHandle2) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Perform online resize on the restored volume and make sure resize should go fine") - verifyOnlineVolumeExpansionOnGc(client, namespace, svcPVCName2, volHandle, pvclaim2, pod, f) - - ginkgo.By("Create a volume snapshot from restored volume") - volumeSnapshotFromRestoreVol, snapshotContentFromRestoreVol, snapshotCreated, - snapshotContentCreated, snapshotIdFromRestoreVol, err := createDynamicVolumeSnapshot(ctx, namespace, snapc, - volumeSnapshotClass, pvclaim2, volHandle2, "3Gi") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - framework.Logf("Volume snapshot name is : %s", volumeSnapshotFromRestoreVol.Name) - snapshotCreated = true - - defer func() { - if snapshotContentCreated { - framework.Logf("Deleting volume snapshot content") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = deleteVolumeSnapshotContent(ctx, snapshotContentFromRestoreVol, - snapc, namespace, pandoraSyncWaitTime) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - if snapshotCreated { - framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshotFromRestoreVol.Name, pandoraSyncWaitTime) - } - }() - - framework.Logf("Deleting volume snapshot") - snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshotFromRestoreVol, pandoraSyncWaitTime, volHandle2, snapshotIdFromRestoreVol) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - }) - - /* - Offline relocation of FCD with snapshots - 1. Create a Storage Class, and a PVC. - 2. Ensure the Volume-snapshot and VolumeSnapshotContent is created and Bound - 3. Run FCD relocate on this volume using CNS side APIs - 4. If relocate is supported, create new snapshots after relocate is successful - 5. Verify snapshot status which we took before relocating FCD. - 6. Create new volume using this snapshot as source, use the same SC and attach it to a Pod. - 7. Run cleanup: Delete snapshots, restored-volumes, pods. - */ - ginkgo.It("[tkg-snapshot] Offline relocation of FCD with snapshots", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - var storageclass *storagev1.StorageClass - var pvclaim *v1.PersistentVolumeClaim - var err error - var snapshotContentCreated, snapshotCreated bool - var datastoreUrls []string - - sharedvmfsURL := os.Getenv(envSharedVMFSDatastoreURL) - if sharedvmfsURL == "" { - ginkgo.Skip(fmt.Sprintf("Env %v is missing", envSharedVMFSDatastoreURL)) - } - - sharedVsanDatastoreURL := os.Getenv(envSharedDatastoreURL) - if sharedVsanDatastoreURL == "" { - ginkgo.Skip(fmt.Sprintf("Env %v is missing", envSharedDatastoreURL)) - } - datastoreUrls = append(datastoreUrls, sharedvmfsURL, sharedVsanDatastoreURL) - - storagePolicyName = os.Getenv(envStoragePolicyNameForVsanVmfsDatastores) - if storagePolicyName == "" { - ginkgo.Skip(fmt.Sprintf("Env %v is missing", envStoragePolicyNameForVsanVmfsDatastores)) - } - - ginkgo.By("Create storage class and PVC") - storageclass, err = client.StorageV1().StorageClasses().Get(ctx, storagePolicyName, metav1.GetOptions{}) - if !apierrors.IsNotFound(err) { - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - pvclaim, persistentVolumes := createPVCAndQueryVolumeInCNS(client, namespace, nil, "", - diskSize, storageclass, true) - volHandle := persistentVolumes[0].Spec.CSI.VolumeHandle - if guestCluster { - volHandle = getVolumeIDFromSupervisorCluster(volHandle) - } - gomega.Expect(volHandle).NotTo(gomega.BeEmpty()) - - defer func() { - ginkgo.By(fmt.Sprintf("Deleting the pvc %s in namespace %s", pvclaim.Name, namespace)) - err := fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Create/Get volume snapshot class") - volumeSnapshotClass, err := createVolumeSnapshotClass(ctx, snapc, deletionPolicy) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - defer func() { - if vanillaCluster { - err := snapc.SnapshotV1().VolumeSnapshotClasses().Delete(ctx, volumeSnapshotClass.Name, - metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }() - - ginkgo.By("Create a volume snapshot") - volumeSnapshot, snapshotContent, snapshotCreated, - snapshotContentCreated, snapshotId, err := createDynamicVolumeSnapshot(ctx, namespace, snapc, volumeSnapshotClass, - pvclaim, volHandle, diskSize) - framework.Logf("Volume snapshot name is : %s", volumeSnapshot.Name) - - defer func() { - if snapshotContentCreated { - framework.Logf("Deleting volume snapshot content") - err = deleteVolumeSnapshotContent(ctx, snapshotContent, snapc, namespace, pandoraSyncWaitTime) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - if snapshotCreated { - framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) - } - }() - - ginkgo.By("Verify if VolumeID is created on the given datastores") - dsUrlWhereVolumeIsPresent := fetchDsUrl4CnsVol(e2eVSphere, volHandle) - framework.Logf("Volume: %s is present on %s", volHandle, dsUrlWhereVolumeIsPresent) - e2eVSphere.verifyDatastoreMatch(volHandle, datastoreUrls) - - // Get the destination ds url where the volume will get relocated - destDsUrl := "" - for _, dsurl := range datastoreUrls { - if dsurl != dsUrlWhereVolumeIsPresent { - destDsUrl = dsurl - break - } - } - - ginkgo.By("Relocate FCD to another datastore") - dsRefDest := getDsMoRefFromURL(ctx, destDsUrl) - _, err = e2eVSphere.cnsRelocateVolume(e2eVSphere, ctx, volHandle, dsRefDest, true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Create PVC from snapshot") - pvclaim2, persistentVolumes2, pod := verifyVolumeRestoreOperation(ctx, client, - namespace, storageclass, volumeSnapshot, true) - volHandle2 := persistentVolumes2[0].Spec.CSI.VolumeHandle - if guestCluster { - volHandle2 = getVolumeIDFromSupervisorCluster(volHandle2) - } - gomega.Expect(volHandle2).NotTo(gomega.BeEmpty()) - - defer func() { - // Delete POD - ginkgo.By(fmt.Sprintf("Deleting the pod %s in namespace %s", pod.Name, namespace)) - err = fpod.DeletePodWithWait(client, pod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By(fmt.Sprintf("Deleting the pvc %s in namespace %s", pvclaim2.Name, namespace)) - err := fpv.DeletePersistentVolumeClaim(client, pvclaim2.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volHandle2) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - framework.Logf("Deleting volume snapshot") - snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - }) - - /* - Dynamic snapshot created in one guest cluster and restore it on another guest cluster - 1. Create a SC and PVC using this SC and attach it to Pod. Write some data on it. - 2. Create a volume snapshot using this PVC as source in Guest Cluster GC-1 and bound. - 3. Restore volume snapshot created in step #2 in another Guest Cluster GC-2 - 4. Verify restore volume creation status in another GC fails with appropriate error. - 5. Run cleanup: Delete snapshots, restored-volumes, pods. - */ - ginkgo.It("[tkg-snapshot] Dynamic snapshot created in one guest cluster "+ - "and restore it on another guest cluster", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - var storageclass *storagev1.StorageClass - var pvclaim *v1.PersistentVolumeClaim - var err error - var snapshotContentCreated, snapshotCreated bool - - newGcKubconfigPath := os.Getenv("NEW_GUEST_CLUSTER_KUBE_CONFIG") - if newGcKubconfigPath == "" { - ginkgo.Skip("Env NEW_GUEST_CLUSTER_KUBE_CONFIG is missing") - } - clientNewGc, err = createKubernetesClientFromConfig(newGcKubconfigPath) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), - fmt.Sprintf("Error creating k8s client with %v: %v", newGcKubconfigPath, err)) - - ginkgo.By("Create storage class and PVC") - storageclass, err = client.StorageV1().StorageClasses().Get(ctx, storagePolicyName, metav1.GetOptions{}) - if !apierrors.IsNotFound(err) { - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - pvclaim, persistentVolumes := createPVCAndQueryVolumeInCNS(client, namespace, nil, "", - diskSize, storageclass, true) - volHandle := persistentVolumes[0].Spec.CSI.VolumeHandle - if guestCluster { - volHandle = getVolumeIDFromSupervisorCluster(volHandle) - } - gomega.Expect(volHandle).NotTo(gomega.BeEmpty()) - - defer func() { - ginkgo.By(fmt.Sprintf("Deleting the pvc %s in namespace %s", pvclaim.Name, namespace)) - err := fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Create/Get volume snapshot class") - volumeSnapshotClass, err := createVolumeSnapshotClass(ctx, snapc, deletionPolicy) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - defer func() { - if vanillaCluster { - err := snapc.SnapshotV1().VolumeSnapshotClasses().Delete(ctx, volumeSnapshotClass.Name, - metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }() - - ginkgo.By("Create a volume snapshot") - volumeSnapshot, _, snapshotCreated, - snapshotContentCreated, snapshotId, err := createDynamicVolumeSnapshot(ctx, namespace, snapc, volumeSnapshotClass, - pvclaim, volHandle, diskSize) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - framework.Logf("Volume snapshot name is : %s", volumeSnapshot.Name) - - defer func() { - if snapshotContentCreated { - framework.Logf("Deleting volume snapshot content") - deleteVolumeSnapshotContentWithPandoraWait(ctx, snapc, - *volumeSnapshot.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, *volumeSnapshot.Status.BoundVolumeSnapshotContentName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - if snapshotCreated { - framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) - } - }() - - ginkgo.By("Creating namespace on second GC") - ns, err := framework.CreateTestingNS(f.BaseName, clientNewGc, map[string]string{ - "e2e-framework": f.BaseName, - }) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Error creating namespace on second GC") - - namespaceNewGC := ns.Name - framework.Logf("Created namespace on second GC %v", namespaceNewGC) - defer func() { - err := clientNewGc.CoreV1().Namespaces().Delete(ctx, namespaceNewGC, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Create PVC from snapshot") - pvcSpec := getPersistentVolumeClaimSpecWithDatasource(namespaceNewGC, diskSize, storageclass, nil, - v1.ReadWriteOnce, volumeSnapshot.Name, snapshotapigroup) - - pvclaim2, err := fpv.CreatePVC(clientNewGc, namespaceNewGC, pvcSpec) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - defer func() { - ginkgo.By(fmt.Sprintf("Deleting the pvc %s in namespace %s", pvclaim2.Name, namespace)) - err = fpv.DeletePersistentVolumeClaim(clientNewGc, pvclaim2.Name, namespaceNewGC) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - _, err = fpv.WaitForPVClaimBoundPhase(clientNewGc, - []*v1.PersistentVolumeClaim{pvclaim2}, framework.ClaimProvisionTimeout) - gomega.Expect(err).To(gomega.HaveOccurred()) - expectedErrMsg := "error getting handle for DataSource Type VolumeSnapshot by Name " + volumeSnapshot.Name - framework.Logf("Expected failure message: %+q", expectedErrMsg) - err = waitForEvent(ctx, clientNewGc, namespaceNewGC, expectedErrMsg, pvclaim2.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), fmt.Sprintf("Expected error : %q", expectedErrMsg)) - - ginkgo.By("Delete PVC created from snapshot") - err = fpv.DeletePersistentVolumeClaim(clientNewGc, pvclaim2.Name, namespaceNewGC) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Delete snapshot") - snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - - /* - Volume mode conversion - 1. Create a Storage Class, PVC. - 2. Create Dynamic Provisioned snapshot on above PVC. - 3. Verify VolumeSnapshot and VolumeSnapshotContent status. - 4. Create new volume using snapshot created in step #4, but this time - give access mode like ReadWriteMany or ReadOnlymany or ReadOnlyOncePod) - 5. Restore PVC creation should fail and be stuck in Pending state with appropriate error message. - 6. Perform Cleanup. - */ - ginkgo.It("[tkg-snapshot] Volume mode conversion", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - var storageclass *storagev1.StorageClass - var pvclaim *v1.PersistentVolumeClaim - var err error - var snapshotContentCreated, snapshotCreated bool - - ginkgo.By("Create storage class and PVC") - storageclass, err = client.StorageV1().StorageClasses().Get(ctx, storagePolicyName, metav1.GetOptions{}) - if !apierrors.IsNotFound(err) { - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - pvclaim, persistentVolumes := createPVCAndQueryVolumeInCNS(client, namespace, nil, "", - diskSize, storageclass, true) - volHandle := persistentVolumes[0].Spec.CSI.VolumeHandle - if guestCluster { - volHandle = getVolumeIDFromSupervisorCluster(volHandle) - } - gomega.Expect(volHandle).NotTo(gomega.BeEmpty()) - - defer func() { - ginkgo.By(fmt.Sprintf("Deleting the pvc %s in namespace %s", pvclaim.Name, namespace)) - err := fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Create/Get volume snapshot class") - volumeSnapshotClass, err := createVolumeSnapshotClass(ctx, snapc, deletionPolicy) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - defer func() { - if vanillaCluster { - err := snapc.SnapshotV1().VolumeSnapshotClasses().Delete(ctx, volumeSnapshotClass.Name, - metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }() - - ginkgo.By("Create a volume snapshot") - volumeSnapshot, snapshotContent, snapshotCreated, - snapshotContentCreated, snapshotId, err := createDynamicVolumeSnapshot(ctx, namespace, snapc, volumeSnapshotClass, - pvclaim, volHandle, diskSize) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - framework.Logf("Volume snapshot name is : %s", volumeSnapshot.Name) - - defer func() { - if snapshotContentCreated { - framework.Logf("Deleting volume snapshot content") - err = deleteVolumeSnapshotContent(ctx, snapshotContent, snapc, namespace, pandoraSyncWaitTime) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - if snapshotCreated { - framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) - } - }() - - accessModes := []v1.PersistentVolumeAccessMode{v1.ReadWriteMany, v1.ReadOnlyMany} - - for _, accessMode := range accessModes { - ginkgo.By(fmt.Sprintf("Create PVC from snapshot with %s access mode", accessMode)) - pvcSpec := getPersistentVolumeClaimSpecWithDatasource(namespace, diskSize, storageclass, nil, - accessMode, volumeSnapshot.Name, snapshotapigroup) - - pvclaim2, err := fpv.CreatePVC(client, namespace, pvcSpec) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - _, err = fpv.WaitForPVClaimBoundPhase(client, - []*v1.PersistentVolumeClaim{pvclaim2}, framework.ClaimProvisionTimeout) - framework.Logf("Error from creating pvc with %s accessmode is : %s", accessMode, err.Error()) - gomega.Expect(err).To(gomega.HaveOccurred()) - - expectedErrMsg := "no datastores found to create file volume" - framework.Logf("Expected failure message: %+q", expectedErrMsg) - err = waitForEvent(ctx, client, namespace, expectedErrMsg, pvclaim2.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), fmt.Sprintf("Expected error : %q", expectedErrMsg)) - - ginkgo.By(fmt.Sprintf("Deleting the pvc %s in namespace %s", pvclaim2.Name, namespace)) - err = fpv.DeletePersistentVolumeClaim(client, pvclaim2.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - err = waitForPvcToBeDeleted(ctx, client, pvclaim2.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - } - - framework.Logf("Deleting pending PVCs from SVC namespace") - pvcList := getAllPVCFromNamespace(svcClient, svcNamespace) - for _, pvc := range pvcList.Items { - if pvc.Status.Phase == v1.ClaimPending { - framework.ExpectNoError(fpv.DeletePersistentVolumeClaim(svcClient, pvc.Name, svcNamespace), - "Failed to delete PVC", pvc.Name) - } - } - snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - - /* - Volume snapshot creation on a file-share volume - Create a file-share pvc - Try creating a snapshot on this pvc - Should fail with an appropriate error - */ - - ginkgo.It("[tkg-snapshot] Volume snapshot creation on a file-share volume on a guest cluster", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - scParameters[svStorageClassName] = storagePolicyName - - ginkgo.By("Create storage class and PVC") - storageclass, pvclaim, err := createPVCAndStorageClass(client, - namespace, nil, scParameters, diskSize, nil, "", false, v1.ReadWriteMany) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err = client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Expect claim to fail as invalid storage policy is specified in Storage Class") - framework.ExpectError(fpv.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, - client, pvclaim.Namespace, pvclaim.Name, framework.Poll, framework.ClaimProvisionTimeout)) - expectedErrMsg := "no datastores found to create file volume, vsan file service may be disabled" - err = waitForEvent(ctx, client, namespace, expectedErrMsg, pvclaim.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), fmt.Sprintf("Expected error : %q", expectedErrMsg)) - }) - - /* - PVC/Pod → Snapshot → RestoreVolume/Pod → Snapshot → Restore Vol again/Pod - - Create a Storage Class, a PVC and attach the PVC to a Pod, write a file - Create dynamically provisioned snapshots using this PVC - Create new volume using this snapshots as source, use the same SC - Ensure the PVC gets provisioned and is Bound - Attach the PVC to a Pod and ensure data from snapshot is available (file that was written in step.1 - should be available) - And also write new data to the restored volumes and it should succeed - Take a snapshot of restore volume created in step #3. - Create new volume using the snapshot as source use the same SC created in step #7 - Ensure the PVC gets provisioned and is Bound - Attach the PVC to a Pod and ensure data from snapshot is available (file that was written in - step.1 and step 5 should be available) - And also write new data to the restored volumes and it should succeed - Run cleanup: Delete snapshots, restored-volumes, pods. - */ - - ginkgo.It("[tkg-snapshot] Create restore volume snapshot in consistent order", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - if vanillaCluster { - scParameters[scParamDatastoreURL] = datastoreURL - } else if guestCluster { - scParameters[svStorageClassName] = storagePolicyName - } - - ginkgo.By("Create storage class and PVC") - storageclass, err := createStorageClass(client, scParameters, nil, "", "", false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - pvclaim, persistentVolumes := createPVCAndQueryVolumeInCNS(client, namespace, nil, "", - diskSize, storageclass, true) - volHandle := persistentVolumes[0].Spec.CSI.VolumeHandle - if guestCluster { - volHandle = getVolumeIDFromSupervisorCluster(volHandle) - } - gomega.Expect(volHandle).NotTo(gomega.BeEmpty()) - defer func() { - err := fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Create Pod") - pod, err := createPod(client, namespace, nil, []*v1.PersistentVolumeClaim{pvclaim}, false, - execRWXCommandPod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - ginkgo.By(fmt.Sprintf("Deleting the pod %s in namespace %s", pod.Name, namespace)) - err = fpod.DeletePodWithWait(client, pod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - var vmUUID string - nodeName := pod.Spec.NodeName - - if vanillaCluster { - vmUUID = getNodeUUID(ctx, client, pod.Spec.NodeName) - } else if guestCluster { - vmUUID, err = getVMUUIDFromNodeName(pod.Spec.NodeName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - ginkgo.By(fmt.Sprintf("Verify volume: %s is attached to the node: %s", volHandle, nodeName)) - isDiskAttached, err := e2eVSphere.isVolumeAttachedToVM(client, volHandle, vmUUID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(isDiskAttached).To(gomega.BeTrue(), "Volume is not attached to the node") - - ginkgo.By("Verify the volume is accessible and Read/write is possible") - cmd := []string{"exec", pod.Name, "--namespace=" + namespace, "--", "/bin/sh", "-c", - "cat /mnt/volume1/Pod.html "} - output := framework.RunKubectlOrDie(namespace, cmd...) - gomega.Expect(strings.Contains(output, "Hello message from Pod")).NotTo(gomega.BeFalse()) - - wrtiecmd := []string{"exec", pod.Name, "--namespace=" + namespace, "--", "/bin/sh", "-c", - "echo 'Hello message from test into Pod' > /mnt/volume1/Pod.html"} - framework.RunKubectlOrDie(namespace, wrtiecmd...) - output = framework.RunKubectlOrDie(namespace, cmd...) - gomega.Expect(strings.Contains(output, "Hello message from test into Pod")).NotTo(gomega.BeFalse()) - - ginkgo.By("Create volume snapshot class") - volumeSnapshotClass, err := createVolumeSnapshotClass(ctx, snapc, deletionPolicy) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - if vanillaCluster { - err = snapc.SnapshotV1().VolumeSnapshotClasses().Delete(ctx, volumeSnapshotClass.Name, - metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }() - - ginkgo.By("Create a dynamic volume snapshot") - volumeSnapshot, snapshotContent, snapshotCreated, - snapshotContentCreated, snapshotId, err := createDynamicVolumeSnapshot(ctx, namespace, snapc, volumeSnapshotClass, - pvclaim, volHandle, diskSize) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - if snapshotContentCreated { - err = deleteVolumeSnapshotContent(ctx, snapshotContent, snapc, namespace, pandoraSyncWaitTime) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - if snapshotCreated { - framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeletedWithPandoraWait(ctx, snapc, - *volumeSnapshot.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }() - - ginkgo.By("Restore volume from snapshot created above") - pvcSpec := getPersistentVolumeClaimSpecWithDatasource(namespace, diskSize, storageclass, nil, - v1.ReadWriteOnce, volumeSnapshot.Name, snapshotapigroup) - pvclaim1, err := fpv.CreatePVC(client, namespace, pvcSpec) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - persistentvolume1, err := fpv.WaitForPVClaimBoundPhase(client, []*v1.PersistentVolumeClaim{pvclaim1}, - framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - volHandle1 := persistentvolume1[0].Spec.CSI.VolumeHandle - gomega.Expect(volHandle1).NotTo(gomega.BeEmpty()) - if guestCluster { - volHandle1 = getVolumeIDFromSupervisorCluster(volHandle1) - } - defer func() { - err := fpv.DeletePersistentVolumeClaim(client, pvclaim1.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volHandle1) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Create Pod") - pod1, err := createPod(client, namespace, nil, []*v1.PersistentVolumeClaim{pvclaim1}, false, - execRWXCommandPod1) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - ginkgo.By(fmt.Sprintf("Deleting the pod %s in namespace %s", pod1.Name, namespace)) - err = fpod.DeletePodWithWait(client, pod1) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - if vanillaCluster { - vmUUID = getNodeUUID(ctx, client, pod1.Spec.NodeName) - } else if guestCluster { - vmUUID, err = getVMUUIDFromNodeName(pod1.Spec.NodeName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - ginkgo.By(fmt.Sprintf("Verify volume: %s is attached to the node: %s", volHandle1, pod1.Spec.NodeName)) - isDiskAttached, err = e2eVSphere.isVolumeAttachedToVM(client, volHandle1, vmUUID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(isDiskAttached).To(gomega.BeTrue(), "Volume is not attached to the node") - - ginkgo.By("Verify the volume is accessible and Read/write is possible") - cmd = []string{"exec", pod1.Name, "--namespace=" + namespace, "--", "/bin/sh", "-c", - "cat /mnt/volume1/Pod1.html "} - output = framework.RunKubectlOrDie(namespace, cmd...) - gomega.Expect(strings.Contains(output, "Hello message from Pod1")).NotTo(gomega.BeFalse()) - - wrtiecmd = []string{"exec", pod1.Name, "--namespace=" + namespace, "--", "/bin/sh", "-c", - "echo 'Hello message from test into Pod1' > /mnt/volume1/Pod1.html"} - framework.RunKubectlOrDie(namespace, wrtiecmd...) - output = framework.RunKubectlOrDie(namespace, cmd...) - gomega.Expect(strings.Contains(output, "Hello message from test into Pod1")).NotTo(gomega.BeFalse()) - - ginkgo.By("Create a dynamic volume snapshot") - volumeSnapshot1, snapshotContent1, snapshotCreated1, - snapshotContentCreated1, snapshotId1, err := createDynamicVolumeSnapshot(ctx, namespace, snapc, volumeSnapshotClass, - pvclaim1, volHandle1, diskSize) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - if snapshotContentCreated1 { - err = deleteVolumeSnapshotContent(ctx, snapshotContent1, snapc, namespace, pandoraSyncWaitTime) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - if snapshotCreated1 { - framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot1.Name, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeletedWithPandoraWait(ctx, snapc, - *volumeSnapshot1.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }() - - ginkgo.By("Restore volume from snapshot created above") - pvcSpec = getPersistentVolumeClaimSpecWithDatasource(namespace, diskSize, storageclass, nil, - v1.ReadWriteOnce, volumeSnapshot1.Name, snapshotapigroup) - pvclaim2, err := fpv.CreatePVC(client, namespace, pvcSpec) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - persistentvolume2, err := fpv.WaitForPVClaimBoundPhase(client, []*v1.PersistentVolumeClaim{pvclaim2}, - framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - volHandle2 := persistentvolume2[0].Spec.CSI.VolumeHandle - gomega.Expect(volHandle2).NotTo(gomega.BeEmpty()) - if guestCluster { - volHandle2 = getVolumeIDFromSupervisorCluster(volHandle2) - } - defer func() { - err := fpv.DeletePersistentVolumeClaim(client, pvclaim2.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volHandle2) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Create Pod") - pod2, err := createPod(client, namespace, nil, []*v1.PersistentVolumeClaim{pvclaim2}, false, - execRWXCommandPod2) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - ginkgo.By(fmt.Sprintf("Deleting the pod %s in namespace %s", pod2.Name, namespace)) - err = fpod.DeletePodWithWait(client, pod2) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - if vanillaCluster { - vmUUID = getNodeUUID(ctx, client, pod2.Spec.NodeName) - } else if guestCluster { - vmUUID, err = getVMUUIDFromNodeName(pod2.Spec.NodeName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - ginkgo.By(fmt.Sprintf("Verify volume: %s is attached to the node: %s", volHandle2, pod2.Spec.NodeName)) - isDiskAttached, err = e2eVSphere.isVolumeAttachedToVM(client, volHandle2, vmUUID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(isDiskAttached).To(gomega.BeTrue(), "Volume is not attached to the node") - - ginkgo.By("Verify the volume is accessible and Read/write is possible") - cmd = []string{"exec", pod2.Name, "--namespace=" + namespace, "--", "/bin/sh", "-c", - "cat /mnt/volume1/Pod2.html "} - output = framework.RunKubectlOrDie(namespace, cmd...) - gomega.Expect(strings.Contains(output, "Hello message from Pod2")).NotTo(gomega.BeFalse()) - - wrtiecmd = []string{"exec", pod2.Name, "--namespace=" + namespace, "--", "/bin/sh", "-c", - "echo 'Hello message from test into Pod2' > /mnt/volume1/Pod2.html"} - framework.RunKubectlOrDie(namespace, wrtiecmd...) - output = framework.RunKubectlOrDie(namespace, cmd...) - gomega.Expect(strings.Contains(output, "Hello message from test into Pod2")).NotTo(gomega.BeFalse()) - - ginkgo.By("Delete dynamic volume snapshot") - snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volHandle, snapshotId) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Delete dynamic volume snapshot") - snapshotCreated1, snapshotContentCreated1, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot1, pandoraSyncWaitTime, volHandle1, snapshotId1) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) }) diff --git a/tests/e2e/csi_snapshot_file_volume.go b/tests/e2e/csi_snapshot_file_volume.go index 7b6a0f1e11..f3468467cc 100644 --- a/tests/e2e/csi_snapshot_file_volume.go +++ b/tests/e2e/csi_snapshot_file_volume.go @@ -16,8 +16,6 @@ package e2e import ( "context" "fmt" - "os" - "strconv" ginkgo "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" @@ -31,21 +29,20 @@ import ( fnodes "k8s.io/kubernetes/test/e2e/framework/node" fpv "k8s.io/kubernetes/test/e2e/framework/pv" - snapV1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" - snapclient "github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned" + snapV1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" + snapclient "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned" ) var _ = ginkgo.Describe("[file-vanilla-snapshot] Volume Snapshot file volume Test", func() { f := framework.NewDefaultFramework("file-snapshot") var ( - client clientset.Interface - namespace string - scParameters map[string]string - datastoreURL string - pvclaims []*v1.PersistentVolumeClaim - restConfig *restclient.Config - snapc *snapclient.Clientset - pandoraSyncWaitTime int + client clientset.Interface + namespace string + scParameters map[string]string + datastoreURL string + pvclaims []*v1.PersistentVolumeClaim + restConfig *restclient.Config + snapc *snapclient.Clientset ) ginkgo.BeforeEach(func() { @@ -64,13 +61,6 @@ var _ = ginkgo.Describe("[file-vanilla-snapshot] Volume Snapshot file volume Tes restConfig = getRestConfigClient() snapc, err = snapclient.NewForConfig(restConfig) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - if os.Getenv(envPandoraSyncWaitTime) != "" { - pandoraSyncWaitTime, err = strconv.Atoi(os.Getenv(envPandoraSyncWaitTime)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } else { - pandoraSyncWaitTime = defaultPandoraSyncWaitTime - } }) /* @@ -137,7 +127,8 @@ var _ = ginkgo.Describe("[file-vanilla-snapshot] Volume Snapshot file volume Tes defer func() { if snapshotCreated { framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) + err := snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, volumeSnapshot.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() diff --git a/tests/e2e/csi_snapshot_negative.go b/tests/e2e/csi_snapshot_negative.go index 55d844b596..c72fc72fcb 100644 --- a/tests/e2e/csi_snapshot_negative.go +++ b/tests/e2e/csi_snapshot_negative.go @@ -36,27 +36,26 @@ import ( fpv "k8s.io/kubernetes/test/e2e/framework/pv" admissionapi "k8s.io/pod-security-admission/api" - snapV1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" - snapclient "github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned" + snapV1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" + snapclient "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned" ) var _ = ginkgo.Describe("[block-snapshot-negative] Volume Snapshot Fault-Injection Test", func() { f := framework.NewDefaultFramework("file-snapshot") f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged var ( - client clientset.Interface - csiNamespace string - csiReplicas int32 - isServiceStopped bool - namespace string - scParameters map[string]string - datastoreURL string - fullSyncWaitTime int - pvclaims []*v1.PersistentVolumeClaim - restConfig *restclient.Config - snapc *snapclient.Clientset - serviceName string - pandoraSyncWaitTime int + client clientset.Interface + csiNamespace string + csiReplicas int32 + isServiceStopped bool + namespace string + scParameters map[string]string + datastoreURL string + fullSyncWaitTime int + pvclaims []*v1.PersistentVolumeClaim + restConfig *restclient.Config + snapc *snapclient.Clientset + serviceName string ) ginkgo.BeforeEach(func() { @@ -94,13 +93,6 @@ var _ = ginkgo.Describe("[block-snapshot-negative] Volume Snapshot Fault-Injecti ctx, vSphereCSIControllerPodNamePrefix, metav1.GetOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) csiReplicas = *csiDeployment.Spec.Replicas - - if os.Getenv(envPandoraSyncWaitTime) != "" { - pandoraSyncWaitTime, err = strconv.Atoi(os.Getenv(envPandoraSyncWaitTime)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } else { - pandoraSyncWaitTime = defaultPandoraSyncWaitTime - } }) ginkgo.AfterEach(func() { @@ -207,7 +199,8 @@ var _ = ginkgo.Describe("[block-snapshot-negative] Volume Snapshot Fault-Injecti defer func() { if snapshotCreated { framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) + err := snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, volumeSnapshot.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() @@ -229,38 +222,38 @@ var _ = ginkgo.Describe("[block-snapshot-negative] Volume Snapshot Fault-Injecti ginkgo.It("create volume snapshot when hostd goes down", func() { serviceName = hostdServiceName snapshotOperationWhileServiceDown(serviceName, namespace, client, snapc, datastoreURL, - fullSyncWaitTime, isServiceStopped, true, csiReplicas, pandoraSyncWaitTime) + fullSyncWaitTime, isServiceStopped, true, csiReplicas) }) ginkgo.It("create volume snapshot when CSI restarts", func() { serviceName = "CSI" snapshotOperationWhileServiceDown(serviceName, namespace, client, snapc, datastoreURL, - fullSyncWaitTime, isServiceStopped, true, csiReplicas, pandoraSyncWaitTime) + fullSyncWaitTime, isServiceStopped, true, csiReplicas) }) ginkgo.It("create volume snapshot when VPXD goes down", func() { serviceName = vpxdServiceName snapshotOperationWhileServiceDownNegative(serviceName, namespace, client, snapc, datastoreURL, - fullSyncWaitTime, isServiceStopped, csiReplicas, pandoraSyncWaitTime) + fullSyncWaitTime, isServiceStopped, csiReplicas) }) ginkgo.It("create volume snapshot when CNS goes down", func() { serviceName = vsanhealthServiceName snapshotOperationWhileServiceDown(serviceName, namespace, client, snapc, datastoreURL, - fullSyncWaitTime, isServiceStopped, false, csiReplicas, pandoraSyncWaitTime) + fullSyncWaitTime, isServiceStopped, false, csiReplicas) }) ginkgo.It("create volume snapshot when SPS goes down", func() { serviceName = spsServiceName snapshotOperationWhileServiceDown(serviceName, namespace, client, snapc, datastoreURL, - fullSyncWaitTime, isServiceStopped, true, csiReplicas, pandoraSyncWaitTime) + fullSyncWaitTime, isServiceStopped, true, csiReplicas) }) }) // snapshotOperationWhileServiceDown creates the volumesnapshot while the services is down func snapshotOperationWhileServiceDown(serviceName string, namespace string, client clientset.Interface, snapc *snapclient.Clientset, datastoreURL string, - fullSyncWaitTime int, isServiceStopped bool, isSnapshotCreated bool, csiReplicas int32, pandoraSyncWaitTime int) { + fullSyncWaitTime int, isServiceStopped bool, isSnapshotCreated bool, csiReplicas int32) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() var storageclass *storagev1.StorageClass @@ -325,7 +318,9 @@ func snapshotOperationWhileServiceDown(serviceName string, namespace string, defer func() { if snapshotCreated { framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, snapshot.Name, pandoraSyncWaitTime) + err := snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, + snapshot.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) framework.Logf("Wait till the volume snapshot is deleted") err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, snapshot.ObjectMeta.Name) @@ -335,29 +330,26 @@ func snapshotOperationWhileServiceDown(serviceName string, namespace string, if snapshotContentCreated { framework.Logf("Deleting volume snapshot content") - deleteVolumeSnapshotContentWithPandoraWait(ctx, snapc, - *snapshot.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, *snapshot.Status.BoundVolumeSnapshotContentName) + err := snapc.SnapshotV1().VolumeSnapshotContents().Delete(ctx, + *snapshot.Status.BoundVolumeSnapshotContentName, metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() if serviceName == "CSI" { ginkgo.By("Stopping CSI driver") - isServiceStopped, err = stopCSIPods(ctx, client, csiSystemNamespace) + isServiceStopped, err = stopCSIPods(ctx, client) gomega.Expect(err).NotTo(gomega.HaveOccurred()) defer func() { if isServiceStopped { framework.Logf("Starting CSI driver") - isServiceStopped, err = startCSIPods(ctx, client, csiReplicas, csiSystemNamespace) + isServiceStopped, err = startCSIPods(ctx, client, csiReplicas) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() framework.Logf("Starting CSI driver") - isServiceStopped, err = startCSIPods(ctx, client, csiReplicas, csiSystemNamespace) + isServiceStopped, err = startCSIPods(ctx, client, csiReplicas) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to allow full sync finish", fullSyncWaitTime)) @@ -451,17 +443,17 @@ func snapshotOperationWhileServiceDown(serviceName string, namespace string, if serviceName == "CSI" { ginkgo.By("Stopping CSI driver") - isServiceStopped, err = stopCSIPods(ctx, client, csiSystemNamespace) + isServiceStopped, err = stopCSIPods(ctx, client) gomega.Expect(err).NotTo(gomega.HaveOccurred()) defer func() { if isServiceStopped { framework.Logf("Starting CSI driver") - isServiceStopped, err = startCSIPods(ctx, client, csiReplicas, csiSystemNamespace) + isServiceStopped, err = startCSIPods(ctx, client, csiReplicas) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() framework.Logf("Starting CSI driver") - isServiceStopped, err = startCSIPods(ctx, client, csiReplicas, csiSystemNamespace) + isServiceStopped, err = startCSIPods(ctx, client, csiReplicas) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to allow full sync finish", fullSyncWaitTime)) @@ -547,24 +539,25 @@ func snapshotOperationWhileServiceDown(serviceName string, namespace string, } ginkgo.By("Deleted volume snapshot is created above") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, snapshot.Name, pandoraSyncWaitTime) + err = snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, snapshot.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) snapshotCreated = false if isSnapshotCreated { if serviceName == "CSI" { ginkgo.By("Stopping CSI driver") - isServiceStopped, err = stopCSIPods(ctx, client, csiSystemNamespace) + isServiceStopped, err = stopCSIPods(ctx, client) gomega.Expect(err).NotTo(gomega.HaveOccurred()) defer func() { if isServiceStopped { framework.Logf("Starting CSI driver") - isServiceStopped, err = startCSIPods(ctx, client, csiReplicas, csiSystemNamespace) + isServiceStopped, err = startCSIPods(ctx, client, csiReplicas) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() framework.Logf("Starting CSI driver") - isServiceStopped, err = startCSIPods(ctx, client, csiReplicas, csiSystemNamespace) + isServiceStopped, err = startCSIPods(ctx, client, csiReplicas) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to allow full sync finish", fullSyncWaitTime)) @@ -646,7 +639,7 @@ func snapshotOperationWhileServiceDown(serviceName string, namespace string, // snapshotOperationWhileServiceDownNegative creates the volumesnapshot while the services is down func snapshotOperationWhileServiceDownNegative(serviceName string, namespace string, client clientset.Interface, snapc *snapclient.Clientset, datastoreURL string, - fullSyncWaitTime int, isServiceStopped bool, csiReplicas int32, pandoraSyncWaitTime int) { + fullSyncWaitTime int, isServiceStopped bool, csiReplicas int32) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() var storageclass *storagev1.StorageClass @@ -709,7 +702,9 @@ func snapshotOperationWhileServiceDownNegative(serviceName string, namespace str defer func() { if snapshotCreated { framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, snapshot.Name, pandoraSyncWaitTime) + err := snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, + snapshot.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) framework.Logf("Wait till the volume snapshot is deleted") err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, snapshot.ObjectMeta.Name) @@ -719,19 +714,19 @@ func snapshotOperationWhileServiceDownNegative(serviceName string, namespace str if serviceName == "CSI" { ginkgo.By("Stopping CSI driver") - isServiceStopped, err = stopCSIPods(ctx, client, csiSystemNamespace) + isServiceStopped, err = stopCSIPods(ctx, client) gomega.Expect(err).NotTo(gomega.HaveOccurred()) defer func() { if isServiceStopped { framework.Logf("Starting CSI driver") - isServiceStopped, err = startCSIPods(ctx, client, csiReplicas, csiSystemNamespace) + isServiceStopped, err = startCSIPods(ctx, client, csiReplicas) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() framework.Logf("Starting CSI driver") - isServiceStopped, err = startCSIPods(ctx, client, csiReplicas, csiSystemNamespace) + isServiceStopped, err = startCSIPods(ctx, client, csiReplicas) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to allow full sync finish", fullSyncWaitTime)) diff --git a/tests/e2e/csi_snapshot_utils.go b/tests/e2e/csi_snapshot_utils.go deleted file mode 100644 index f14d024c42..0000000000 --- a/tests/e2e/csi_snapshot_utils.go +++ /dev/null @@ -1,725 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2e - -import ( - "context" - "fmt" - "strings" - "sync" - "time" - - snapV1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" - snapclient "github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned" - ginkgo "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" - storagev1 "k8s.io/api/storage/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/kubernetes/test/e2e/framework" - fpv "k8s.io/kubernetes/test/e2e/framework/pv" -) - -// getVolumeSnapshotClassSpec returns a spec for the volume snapshot class -func getVolumeSnapshotClassSpec(deletionPolicy snapV1.DeletionPolicy, - parameters map[string]string) *snapV1.VolumeSnapshotClass { - var volumesnapshotclass = &snapV1.VolumeSnapshotClass{ - TypeMeta: metav1.TypeMeta{ - Kind: "VolumeSnapshotClass", - }, - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "volumesnapshot-", - }, - Driver: e2evSphereCSIDriverName, - DeletionPolicy: deletionPolicy, - } - - volumesnapshotclass.Parameters = parameters - return volumesnapshotclass -} - -// getVolumeSnapshotSpec returns a spec for the volume snapshot -func getVolumeSnapshotSpec(namespace string, snapshotclassname string, pvcName string) *snapV1.VolumeSnapshot { - var volumesnapshotSpec = &snapV1.VolumeSnapshot{ - TypeMeta: metav1.TypeMeta{ - Kind: "VolumeSnapshot", - }, - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "snapshot-", - Namespace: namespace, - }, - Spec: snapV1.VolumeSnapshotSpec{ - VolumeSnapshotClassName: &snapshotclassname, - Source: snapV1.VolumeSnapshotSource{ - PersistentVolumeClaimName: &pvcName, - }, - }, - } - return volumesnapshotSpec -} - -// waitForVolumeSnapshotReadyToUse waits for the volume's snapshot to be in ReadyToUse -func waitForVolumeSnapshotReadyToUse(client snapclient.Clientset, ctx context.Context, namespace string, - name string) (*snapV1.VolumeSnapshot, error) { - var volumeSnapshot *snapV1.VolumeSnapshot - var err error - waitErr := wait.PollImmediate(poll, pollTimeout*2, func() (bool, error) { - volumeSnapshot, err = client.SnapshotV1().VolumeSnapshots(namespace).Get(ctx, name, metav1.GetOptions{}) - if err != nil { - return false, fmt.Errorf("error fetching volumesnapshot details : %v", err) - } - if volumeSnapshot.Status != nil && *volumeSnapshot.Status.ReadyToUse { - return true, nil - } - return false, nil - }) - return volumeSnapshot, waitErr -} - -// waitForVolumeSnapshotContentToBeDeleted wait till the volume snapshot content is deleted -func waitForVolumeSnapshotContentToBeDeleted(client snapclient.Clientset, ctx context.Context, - name string) error { - var err error - waitErr := wait.PollImmediate(poll, 2*pollTimeout, func() (bool, error) { - _, err = client.SnapshotV1().VolumeSnapshotContents().Get(ctx, name, metav1.GetOptions{}) - if err != nil { - if apierrors.IsNotFound(err) { - return true, nil - } else { - return false, fmt.Errorf("error fetching volumesnapshotcontent details : %v", err) - } - } - return false, nil - }) - return waitErr -} - -// deleteVolumeSnapshotWithPandoraWait deletes Volume Snapshot with Pandora wait for CNS to sync -func deleteVolumeSnapshotWithPandoraWait(ctx context.Context, snapc *snapclient.Clientset, - namespace string, snapshotName string, pandoraSyncWaitTime int) { - err := snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, snapshotName, - metav1.DeleteOptions{}) - if !apierrors.IsNotFound(err) { - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to allow CNS to sync with pandora", pandoraSyncWaitTime)) - time.Sleep(time.Duration(pandoraSyncWaitTime) * time.Second) -} - -// deleteVolumeSnapshotContentWithPandoraWait deletes Volume Snapshot Content with Pandora wait for CNS to sync -func deleteVolumeSnapshotContentWithPandoraWait(ctx context.Context, snapc *snapclient.Clientset, - snapshotContentName string, pandoraSyncWaitTime int) { - err := snapc.SnapshotV1().VolumeSnapshotContents().Delete(ctx, snapshotContentName, metav1.DeleteOptions{}) - if !apierrors.IsNotFound(err) { - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to allow CNS to sync with pandora", pandoraSyncWaitTime)) - time.Sleep(time.Duration(pandoraSyncWaitTime) * time.Second) - - err = waitForVolumeSnapshotContentToBeDeletedWithPandoraWait(ctx, snapc, snapshotContentName, pandoraSyncWaitTime) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) -} - -// waitForVolumeSnapshotContentToBeDeletedWithPandoraWait wait till the volume snapshot content is deleted -func waitForVolumeSnapshotContentToBeDeletedWithPandoraWait(ctx context.Context, snapc *snapclient.Clientset, - name string, pandoraSyncWaitTime int) error { - var err error - waitErr := wait.PollImmediate(poll, 2*pollTimeout, func() (bool, error) { - _, err = snapc.SnapshotV1().VolumeSnapshotContents().Get(ctx, name, metav1.GetOptions{}) - if err != nil { - if apierrors.IsNotFound(err) { - ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to allow CNS to sync with pandora", pandoraSyncWaitTime)) - time.Sleep(time.Duration(pandoraSyncWaitTime) * time.Second) - return true, nil - } else { - return false, fmt.Errorf("error fetching volumesnapshotcontent details : %v", err) - } - } - return false, err - }) - return waitErr -} - -// waitForCNSSnapshotToBeDeleted wait till the give snapshot is deleted from CNS -func waitForCNSSnapshotToBeDeleted(volumeId string, snapshotId string) error { - var err error - waitErr := wait.PollImmediate(poll, pollTimeout, func() (bool, error) { - err = verifySnapshotIsDeletedInCNS(volumeId, snapshotId, false) - if err != nil { - if strings.Contains(err.Error(), "snapshot entry is still present") { - return false, nil - } - return false, err - } - framework.Logf("Snapshot with ID: %v for volume with ID: %v is deleted from CNS now...", snapshotId, volumeId) - return true, nil - }) - return waitErr -} - -// getVolumeSnapshotContentSpec returns a spec for the volume snapshot content -func getVolumeSnapshotContentSpec(deletionPolicy snapV1.DeletionPolicy, snapshotHandle string, - futureSnapshotName string, namespace string) *snapV1.VolumeSnapshotContent { - var volumesnapshotContentSpec = &snapV1.VolumeSnapshotContent{ - TypeMeta: metav1.TypeMeta{ - Kind: "VolumeSnapshotContent", - }, - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "snapshotcontent-", - }, - Spec: snapV1.VolumeSnapshotContentSpec{ - DeletionPolicy: deletionPolicy, - Driver: e2evSphereCSIDriverName, - Source: snapV1.VolumeSnapshotContentSource{ - SnapshotHandle: &snapshotHandle, - }, - VolumeSnapshotRef: v1.ObjectReference{ - Name: futureSnapshotName, - Namespace: namespace, - }, - }, - } - return volumesnapshotContentSpec -} - -// getVolumeSnapshotSpecByName returns a spec for the volume snapshot by name -func getVolumeSnapshotSpecByName(namespace string, snapshotName string, - snapshotcontentname string) *snapV1.VolumeSnapshot { - var volumesnapshotSpec = &snapV1.VolumeSnapshot{ - TypeMeta: metav1.TypeMeta{ - Kind: "VolumeSnapshot", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: snapshotName, - Namespace: namespace, - }, - Spec: snapV1.VolumeSnapshotSpec{ - Source: snapV1.VolumeSnapshotSource{ - VolumeSnapshotContentName: &snapshotcontentname, - }, - }, - } - return volumesnapshotSpec -} - -// createSnapshotInParallel creates snapshot for a given pvc -// in a given namespace -func createSnapshotInParallel(ctx context.Context, namespace string, - snapc *snapclient.Clientset, pvcName string, volumeSnapClassName string, - ch chan *snapV1.VolumeSnapshot, lock *sync.Mutex, wg *sync.WaitGroup) { - defer wg.Done() - framework.Logf("Waiting for a few seconds for IO to happen to pod") - time.Sleep(time.Duration(10) * time.Second) - volumeSnapshot, err := snapc.SnapshotV1().VolumeSnapshots(namespace).Create(ctx, - getVolumeSnapshotSpec(namespace, volumeSnapClassName, pvcName), metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - framework.Logf("Volume snapshot name is : %s", volumeSnapshot.Name) - lock.Lock() - ch <- volumeSnapshot - lock.Unlock() -} - -// getSnapshotHandleFromSupervisorCluster fetches the SnapshotHandle from Supervisor Cluster -func getSnapshotHandleFromSupervisorCluster(ctx context.Context, - volumeSnapshotClass *snapV1.VolumeSnapshotClass, snapshothandle string) (string, string, string, error) { - var snapc *snapclient.Clientset - var err error - if k8senv := GetAndExpectStringEnvVar("SUPERVISOR_CLUSTER_KUBE_CONFIG"); k8senv != "" { - restConfig, err := clientcmd.BuildConfigFromFlags("", k8senv) - if err != nil { - return "", "", "", err - } - snapc, err = snapclient.NewForConfig(restConfig) - if err != nil { - return "", "", "", err - } - } - - svNamespace := GetAndExpectStringEnvVar(envSupervisorClusterNamespace) - - volumeSnapshot, err := snapc.SnapshotV1().VolumeSnapshots(svNamespace).Get(ctx, snapshothandle, - metav1.GetOptions{}) - if err != nil { - return "", "", "", err - } - - snapshotContent, err := snapc.SnapshotV1().VolumeSnapshotContents().Get(ctx, - *volumeSnapshot.Status.BoundVolumeSnapshotContentName, metav1.GetOptions{}) - if err != nil { - return "", "", "", err - } - - svcSnapshotHandle := *snapshotContent.Status.SnapshotHandle - snapshotID := strings.Split(svcSnapshotHandle, "+")[1] - - svcVolumeSnapshotName := volumeSnapshot.Name - - return snapshotID, svcSnapshotHandle, svcVolumeSnapshotName, nil -} - -// getRestConfigClient returns rest config client for Guest Cluster -func getRestConfigClientForGuestCluster(guestClusterRestConfig *rest.Config) *rest.Config { - var err error - if guestClusterRestConfig == nil { - if k8senv := GetAndExpectStringEnvVar("KUBECONFIG"); k8senv != "" { - guestClusterRestConfig, err = clientcmd.BuildConfigFromFlags("", k8senv) - } - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - } - return guestClusterRestConfig -} - -// deleteVolumeSnapshot deletes volume snapshot from K8s side and CNS side -func deleteVolumeSnapshot(ctx context.Context, snapc *snapclient.Clientset, namespace string, - volumeSnapshot *snapV1.VolumeSnapshot, pandoraSyncWaitTime int, - volHandle string, snapshotID string) (bool, bool, error) { - var err error - - framework.Logf("Delete volume snapshot and verify the snapshot content is deleted") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) - snapshotCreated := false - - framework.Logf("Wait until the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, *volumeSnapshot.Status.BoundVolumeSnapshotContentName) - if err != nil { - return snapshotCreated, false, err - } - snapshotContentCreated := false - - framework.Logf("Verify snapshot entry %v is deleted from CNS for volume %v", snapshotID, volHandle) - err = waitForCNSSnapshotToBeDeleted(volHandle, snapshotID) - if err != nil { - return snapshotCreated, snapshotContentCreated, err - } - - framework.Logf("Verify snapshot entry is deleted from CNS") - err = verifySnapshotIsDeletedInCNS(volHandle, snapshotID, false) - if err != nil { - return snapshotCreated, snapshotContentCreated, err - } - - framework.Logf("Deleting volume snapshot again to check 'Not found' error") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) - - return snapshotCreated, snapshotContentCreated, nil -} - -// getVolumeSnapshotIdFromSnapshotHandle fetches VolumeSnapshotId From SnapshotHandle -func getVolumeSnapshotIdFromSnapshotHandle(ctx context.Context, snapshotContent *snapV1.VolumeSnapshotContent, - volumeSnapshotClass *snapV1.VolumeSnapshotClass, volHandle string) (string, error) { - var snapshotID string - var err error - if vanillaCluster { - snapshotHandle := *snapshotContent.Status.SnapshotHandle - snapshotID = strings.Split(snapshotHandle, "+")[1] - } else if guestCluster { - snapshotHandle := *snapshotContent.Status.SnapshotHandle - snapshotID, _, _, err = getSnapshotHandleFromSupervisorCluster(ctx, volumeSnapshotClass, snapshotHandle) - if err != nil { - return "", err - } - } - return snapshotID, nil -} - -// createVolumeSnapshotClass creates VSC for a Vanilla cluster and -// fetches VSC for a Guest or Supervisor Cluster -func createVolumeSnapshotClass(ctx context.Context, snapc *snapclient.Clientset, - deletionPolicy string) (*snapV1.VolumeSnapshotClass, error) { - var volumeSnapshotClass *snapV1.VolumeSnapshotClass - var err error - if vanillaCluster { - volumeSnapshotClass, err = snapc.SnapshotV1().VolumeSnapshotClasses().Create(ctx, - getVolumeSnapshotClassSpec(snapV1.DeletionPolicy(deletionPolicy), nil), metav1.CreateOptions{}) - if err != nil { - return nil, err - } - } else if guestCluster || supervisorCluster { - var volumeSnapshotClassName string - if deletionPolicy == "Delete" { - volumeSnapshotClassName = GetAndExpectStringEnvVar(envVolSnapClassDel) - } else { - framework.Failf("%s volume snapshotclass is not supported"+ - " in Supervisor or Guest Cluster", deletionPolicy) - } - waitErr := wait.PollImmediate(poll, pollTimeout, func() (bool, error) { - volumeSnapshotClass, err = snapc.SnapshotV1().VolumeSnapshotClasses().Get(ctx, - volumeSnapshotClassName, metav1.GetOptions{}) - framework.Logf("volumesnapshotclass %v, err:%v", volumeSnapshotClass, err) - if !apierrors.IsNotFound(err) && err != nil { - return false, fmt.Errorf("couldn't find "+ - "snapshotclass: %s due to error: %v", volumeSnapshotClassName, err) - } - if volumeSnapshotClass.Name != "" { - framework.Logf("Found volumesnapshotclass %s", volumeSnapshotClassName) - return true, nil - } - framework.Logf("waiting to get volumesnapshotclass %s", volumeSnapshotClassName) - return false, nil - }) - if waitErr == wait.ErrWaitTimeout { - return nil, fmt.Errorf("couldn't find volumesnapshotclass: %s in SVC", volumeSnapshotClassName) - } - } - return volumeSnapshotClass, nil -} - -// createDynamicVolumeSnapshot util creates dynamic volume snapshot for a volume -func createDynamicVolumeSnapshot(ctx context.Context, namespace string, - snapc *snapclient.Clientset, volumeSnapshotClass *snapV1.VolumeSnapshotClass, - pvclaim *v1.PersistentVolumeClaim, volHandle string, diskSize string) (*snapV1.VolumeSnapshot, - *snapV1.VolumeSnapshotContent, bool, bool, string, error) { - - volumeSnapshot, err := snapc.SnapshotV1().VolumeSnapshots(namespace).Create(ctx, - getVolumeSnapshotSpec(namespace, volumeSnapshotClass.Name, pvclaim.Name), metav1.CreateOptions{}) - if err != nil { - return nil, nil, false, false, "", err - } - framework.Logf("Volume snapshot name is : %s", volumeSnapshot.Name) - - ginkgo.By("Verify volume snapshot is created") - volumeSnapshot, err = waitForVolumeSnapshotReadyToUse(*snapc, ctx, namespace, volumeSnapshot.Name) - if err != nil { - return nil, nil, false, false, "", err - } - - snapshotCreated := true - if volumeSnapshot.Status.RestoreSize.Cmp(resource.MustParse(diskSize)) != 0 { - return nil, nil, false, false, "", fmt.Errorf("unexpected restore size") - } - - ginkgo.By("Verify volume snapshot content is created") - snapshotContent, err := snapc.SnapshotV1().VolumeSnapshotContents().Get(ctx, - *volumeSnapshot.Status.BoundVolumeSnapshotContentName, metav1.GetOptions{}) - if err != nil { - return nil, nil, false, false, "", err - } - snapshotContentCreated := true - snapshotContent, err = waitForVolumeSnapshotContentReadyToUse(*snapc, ctx, snapshotContent.Name) - if err != nil { - return nil, nil, false, false, "", fmt.Errorf("volume snapshot content is not ready to use") - } - - framework.Logf("Get volume snapshot ID from snapshot handle") - snapshotId, err := getVolumeSnapshotIdFromSnapshotHandle(ctx, snapshotContent, volumeSnapshotClass, - volHandle) - if err != nil { - return nil, nil, false, false, "", err - } - - ginkgo.By("Query CNS and check the volume snapshot entry") - err = waitForCNSSnapshotToBeCreated(volHandle, snapshotId) - if err != nil { - return nil, nil, false, false, snapshotId, err - } - - return volumeSnapshot, snapshotContent, snapshotCreated, snapshotContentCreated, snapshotId, nil -} - -// getPersistentVolumeClaimSpecWithDatasource return the PersistentVolumeClaim -// spec with specified storage class. -func getPersistentVolumeClaimSpecWithDatasource(namespace string, ds string, storageclass *storagev1.StorageClass, - pvclaimlabels map[string]string, accessMode v1.PersistentVolumeAccessMode, - datasourceName string, snapshotapigroup string) *v1.PersistentVolumeClaim { - disksize := diskSize - if ds != "" { - disksize = ds - } - if accessMode == "" { - // If accessMode is not specified, set the default accessMode. - accessMode = v1.ReadWriteOnce - } - claim := &v1.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "pvc-", - Namespace: namespace, - }, - Spec: v1.PersistentVolumeClaimSpec{ - AccessModes: []v1.PersistentVolumeAccessMode{ - accessMode, - }, - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceName(v1.ResourceStorage): resource.MustParse(disksize), - }, - }, - StorageClassName: &(storageclass.Name), - DataSource: &v1.TypedLocalObjectReference{ - APIGroup: &snapshotapigroup, - Kind: "VolumeSnapshot", - Name: datasourceName, - }, - }, - } - - if pvclaimlabels != nil { - claim.Labels = pvclaimlabels - } - - return claim -} - -/* -changeDeletionPolicyOfVolumeSnapshotContentOnGuest changes the deletion policy -of volume snapshot content from delete to retain in Guest Cluster -*/ -func changeDeletionPolicyOfVolumeSnapshotContent(ctx context.Context, - snapshotContent *snapV1.VolumeSnapshotContent, snapc *snapclient.Clientset, - namespace string, policyName snapV1.DeletionPolicy) (*snapV1.VolumeSnapshotContent, error) { - - // Retrieve the latest version of the object - latestSnapshotContent, err := snapc.SnapshotV1().VolumeSnapshotContents().Get(ctx, snapshotContent.Name, - metav1.GetOptions{}) - if err != nil { - return nil, err - } - - // Apply changes to the latest version - latestSnapshotContent.Spec.DeletionPolicy = policyName - - // Update the object - updatedSnapshotContent, err := snapc.SnapshotV1().VolumeSnapshotContents().Update(ctx, - latestSnapshotContent, metav1.UpdateOptions{}) - if err != nil { - return nil, err - } - - return updatedSnapshotContent, nil -} - -/* deleteVolumeSnapshotContent deletes volume snapshot content explicitly on Guest cluster */ -func deleteVolumeSnapshotContent(ctx context.Context, updatedSnapshotContent *snapV1.VolumeSnapshotContent, - snapc *snapclient.Clientset, namespace string, pandoraSyncWaitTime int) error { - - framework.Logf("Delete volume snapshot content") - deleteVolumeSnapshotContentWithPandoraWait(ctx, snapc, updatedSnapshotContent.Name, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot content is deleted") - err := waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, updatedSnapshotContent.Name) - if err != nil { - return err - } - return nil -} - -/* createPreProvisionedSnapshotInGuestCluster created pre-provisioned snaphot on Guest cluster */ -func createPreProvisionedSnapshotInGuestCluster(ctx context.Context, volumeSnapshot *snapV1.VolumeSnapshot, - updatedSnapshotContent *snapV1.VolumeSnapshotContent, - snapc *snapclient.Clientset, namespace string, pandoraSyncWaitTime int, - svcVolumeSnapshotName string, diskSize string) (*snapV1.VolumeSnapshotContent, - *snapV1.VolumeSnapshot, bool, bool, error) { - - framework.Logf("Change the deletion policy of VolumeSnapshotContent from Delete to Retain " + - "in Guest Cluster") - updatedSnapshotContent, err := changeDeletionPolicyOfVolumeSnapshotContent(ctx, updatedSnapshotContent, - snapc, namespace, snapV1.VolumeSnapshotContentRetain) - if err != nil { - return nil, nil, false, false, fmt.Errorf("failed to change deletion policy of VolumeSnapshotContent: %v", err) - } - - framework.Logf("Delete dynamic volume snapshot from Guest Cluster") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) - - framework.Logf("Delete VolumeSnapshotContent from Guest Cluster explicitly") - err = deleteVolumeSnapshotContent(ctx, updatedSnapshotContent, snapc, namespace, pandoraSyncWaitTime) - if err != nil { - return nil, nil, false, false, fmt.Errorf("failed to delete VolumeSnapshotContent: %v", err) - } - - framework.Logf(fmt.Sprintf("Creating static VolumeSnapshotContent in Guest Cluster using "+ - "supervisor VolumeSnapshotName %s", svcVolumeSnapshotName)) - staticSnapshotContent, err := snapc.SnapshotV1().VolumeSnapshotContents().Create(ctx, - getVolumeSnapshotContentSpec(snapV1.DeletionPolicy("Delete"), svcVolumeSnapshotName, - "static-vs", namespace), metav1.CreateOptions{}) - if err != nil { - return nil, nil, false, false, fmt.Errorf("failed to create static VolumeSnapshotContent: %v", err) - } - - framework.Logf("Verify VolumeSnapshotContent is created or not in Guest Cluster") - staticSnapshotContent, err = snapc.SnapshotV1().VolumeSnapshotContents().Get(ctx, - staticSnapshotContent.Name, metav1.GetOptions{}) - if err != nil { - return nil, nil, false, false, fmt.Errorf("failed to get static VolumeSnapshotContent: %v", err) - } - framework.Logf("Snapshotcontent name is %s", staticSnapshotContent.ObjectMeta.Name) - - staticSnapshotContent, err = waitForVolumeSnapshotContentReadyToUse(*snapc, ctx, staticSnapshotContent.Name) - if err != nil { - return nil, nil, false, false, fmt.Errorf("volume snapshot content is not ready to use") - } - staticSnapshotContentCreated := true - - ginkgo.By("Create a static volume snapshot by static snapshotcontent") - staticVolumeSnapshot, err := snapc.SnapshotV1().VolumeSnapshots(namespace).Create(ctx, - getVolumeSnapshotSpecByName(namespace, "static-vs", - staticSnapshotContent.ObjectMeta.Name), metav1.CreateOptions{}) - if err != nil { - return nil, nil, false, false, fmt.Errorf("failed to create static volume snapshot: %v", err) - } - framework.Logf("Volume snapshot name is : %s", staticVolumeSnapshot.Name) - - ginkgo.By("Verify static volume snapshot is created") - staticSnapshot, err := waitForVolumeSnapshotReadyToUse(*snapc, ctx, namespace, staticVolumeSnapshot.Name) - if err != nil { - return nil, nil, false, false, fmt.Errorf("volumeSnapshot is still not ready to use: %v", err) - } - if staticSnapshot.Status.RestoreSize.Cmp(resource.MustParse(diskSize)) != 0 { - return nil, nil, false, false, fmt.Errorf("expected RestoreSize does not match") - } - framework.Logf("Snapshot details is %+v", staticSnapshot) - staticSnapshotCreated := true - - return staticSnapshotContent, staticSnapshot, staticSnapshotContentCreated, staticSnapshotCreated, nil -} - -// verifyVolumeRestoreOperation verifies if volume(PVC) restore from given snapshot -// and creates pod and checks attach volume operation if verifyPodCreation is set to true -func verifyVolumeRestoreOperation(ctx context.Context, client clientset.Interface, - namespace string, storageclass *storagev1.StorageClass, - volumeSnapshot *snapV1.VolumeSnapshot, - verifyPodCreation bool) (*v1.PersistentVolumeClaim, []*v1.PersistentVolume, *v1.Pod) { - - ginkgo.By("Create PVC from snapshot") - pvcSpec := getPersistentVolumeClaimSpecWithDatasource(namespace, diskSize, storageclass, nil, - v1.ReadWriteOnce, volumeSnapshot.Name, snapshotapigroup) - - pvclaim2, err := fpv.CreatePVC(client, namespace, pvcSpec) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - persistentvolumes2, err := fpv.WaitForPVClaimBoundPhase(client, - []*v1.PersistentVolumeClaim{pvclaim2}, framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - volHandle2 := persistentvolumes2[0].Spec.CSI.VolumeHandle - if guestCluster { - volHandle2 = getVolumeIDFromSupervisorCluster(volHandle2) - } - gomega.Expect(volHandle2).NotTo(gomega.BeEmpty()) - - if verifyPodCreation { - // Create a Pod to use this PVC, and verify volume has been attached - ginkgo.By("Creating pod to attach PV to the node") - pod, err := createPod(client, namespace, nil, []*v1.PersistentVolumeClaim{pvclaim2}, false, - execRWXCommandPod1) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - var vmUUID string - nodeName := pod.Spec.NodeName - - if vanillaCluster { - vmUUID = getNodeUUID(ctx, client, pod.Spec.NodeName) - } else if guestCluster { - vmUUID, err = getVMUUIDFromNodeName(pod.Spec.NodeName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - ginkgo.By(fmt.Sprintf("Verify volume: %s is attached to the node: %s", volHandle2, nodeName)) - isDiskAttached, err := e2eVSphere.isVolumeAttachedToVM(client, volHandle2, vmUUID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(isDiskAttached).To(gomega.BeTrue(), "Volume is not attached to the node") - - ginkgo.By("Verify the volume is accessible and Read/write is possible") - cmd := []string{"exec", pod.Name, "--namespace=" + namespace, "--", "/bin/sh", "-c", - "cat /mnt/volume1/Pod1.html "} - output := framework.RunKubectlOrDie(namespace, cmd...) - gomega.Expect(strings.Contains(output, "Hello message from Pod1")).NotTo(gomega.BeFalse()) - - wrtiecmd := []string{"exec", pod.Name, "--namespace=" + namespace, "--", "/bin/sh", "-c", - "echo 'Hello message from test into Pod1' > /mnt/volume1/Pod1.html"} - framework.RunKubectlOrDie(namespace, wrtiecmd...) - output = framework.RunKubectlOrDie(namespace, cmd...) - gomega.Expect(strings.Contains(output, "Hello message from test into Pod1")).NotTo(gomega.BeFalse()) - return pvclaim2, persistentvolumes2, pod - } - return pvclaim2, persistentvolumes2, nil -} - -// createPVCAndQueryVolumeInCNS creates PVc with a given storage class on a given namespace -// and verifies cns metadata of that volume if verifyCNSVolume is set to true -func createPVCAndQueryVolumeInCNS(client clientset.Interface, namespace string, - pvclaimLabels map[string]string, accessMode v1.PersistentVolumeAccessMode, - ds string, storageclass *storagev1.StorageClass, - verifyCNSVolume bool) (*v1.PersistentVolumeClaim, []*v1.PersistentVolume) { - pvclaim, err := createPVC(client, namespace, pvclaimLabels, ds, storageclass, accessMode) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Expect claim to provision volume successfully") - persistentvolumes, err := fpv.WaitForPVClaimBoundPhase(client, - []*v1.PersistentVolumeClaim{pvclaim}, framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - volHandle := persistentvolumes[0].Spec.CSI.VolumeHandle - if guestCluster { - volHandle = getVolumeIDFromSupervisorCluster(volHandle) - } - gomega.Expect(volHandle).NotTo(gomega.BeEmpty()) - - if verifyCNSVolume { - // Verify using CNS Query API if VolumeID retrieved from PV is present. - ginkgo.By(fmt.Sprintf("Invoking QueryCNSVolumeWithResult with VolumeID: %s", volHandle)) - queryResult, err := e2eVSphere.queryCNSVolumeWithResult(volHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(queryResult.Volumes).ShouldNot(gomega.BeEmpty()) - gomega.Expect(queryResult.Volumes[0].VolumeId.Id).To(gomega.Equal(volHandle)) - } - return pvclaim, persistentvolumes -} - -// waitForVolumeSnapshotContentReadyToUse waits for the volume's snapshot content to be in ReadyToUse -func waitForVolumeSnapshotContentReadyToUse(client snapclient.Clientset, ctx context.Context, - name string) (*snapV1.VolumeSnapshotContent, error) { - var volumeSnapshotContent *snapV1.VolumeSnapshotContent - var err error - - waitErr := wait.PollImmediate(poll, pollTimeout*2, func() (bool, error) { - volumeSnapshotContent, err = client.SnapshotV1().VolumeSnapshotContents().Get(ctx, name, metav1.GetOptions{}) - framework.Logf("volumesnapshotcontent details: %v", volumeSnapshotContent) - if err != nil { - return false, fmt.Errorf("error fetching volumesnapshotcontent details : %v", err) - } - if volumeSnapshotContent.Status != nil && *volumeSnapshotContent.Status.ReadyToUse { - framework.Logf("%s volume snapshotContent is in ready state", name) - return true, nil - } - return false, nil - }) - return volumeSnapshotContent, waitErr -} - -// waitForCNSSnapshotToBeCreated wait till the give snapshot is created in CNS -func waitForCNSSnapshotToBeCreated(volumeId string, snapshotId string) error { - var err error - waitErr := wait.PollImmediate(poll, pollTimeout*2, func() (bool, error) { - err = verifySnapshotIsCreatedInCNS(volumeId, snapshotId, false) - if err != nil { - if strings.Contains(err.Error(), "snapshot entry is not present in CNS") { - return false, nil - } - return false, err - } - framework.Logf("Snapshot with ID: %v for volume with ID: %v is created in CNS now...", snapshotId, volumeId) - return true, nil - }) - return waitErr -} diff --git a/tests/e2e/csi_static_provisioning_basic.go b/tests/e2e/csi_static_provisioning_basic.go index 5830b1da6d..fc9c8b3a84 100644 --- a/tests/e2e/csi_static_provisioning_basic.go +++ b/tests/e2e/csi_static_provisioning_basic.go @@ -52,7 +52,7 @@ var _ = ginkgo.Describe("Basic Static Provisioning", func() { f := framework.NewDefaultFramework("e2e-csistaticprovision") f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged - framework.TestContext.DeleteNamespace = true + var ( client clientset.Interface namespace string @@ -165,10 +165,6 @@ var _ = ginkgo.Describe("Basic Static Provisioning", func() { if guestCluster { svcClient, svNamespace := getSvcClientAndNamespace() setResourceQuota(svcClient, svNamespace, defaultrqLimit) - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) - } - if supervisorCluster { - dumpSvcNsEventsOnTestFailure(client, namespace) } }) diff --git a/tests/e2e/data_persistence.go b/tests/e2e/data_persistence.go index e1685ffa6f..15e4430d21 100644 --- a/tests/e2e/data_persistence.go +++ b/tests/e2e/data_persistence.go @@ -110,15 +110,11 @@ var _ = ginkgo.Describe("Data Persistence", func() { if guestCluster { svcClient, svNamespace := getSvcClientAndNamespace() setResourceQuota(svcClient, svNamespace, defaultrqLimit) - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) - } - if supervisorCluster { - dumpSvcNsEventsOnTestFailure(client, namespace) } }) ginkgo.It("[csi-block-vanilla] [csi-supervisor] [csi-guest] [csi-block-vanilla-parallelized] "+ - "Should create and delete pod with the same volume source and data", func() { + "Should create and delete pod with the same volume source", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() var sc *storagev1.StorageClass diff --git a/tests/e2e/docs/supervisor_cluster_setup.md b/tests/e2e/docs/supervisor_cluster_setup.md index aae8ea62d6..8f2c5bc4c6 100644 --- a/tests/e2e/docs/supervisor_cluster_setup.md +++ b/tests/e2e/docs/supervisor_cluster_setup.md @@ -98,10 +98,6 @@ datacenters should be comma separated if deployed on multi-datacenters export SHARED_NFS_DATASTORE_URL="" #shared VMFS datastore url export SHARED_VMFS_DATASTORE_URL="" - #For vsan direct tests for spbm policy driven allocation tests, set following variables - export USE_VSAN_DIRECT_DATASTORE_IN_WCP="VSAN_DIRECT" - export SHARED_VSAND_DATASTORE_URL="" - export SHARED_VSAND_DATASTORE2_URL="" export BUSYBOX_IMAGE="" ### To run full sync test, need do extra following steps diff --git a/tests/e2e/e2e_common.go b/tests/e2e/e2e_common.go index c06c1cd663..d8a58224f0 100644 --- a/tests/e2e/e2e_common.go +++ b/tests/e2e/e2e_common.go @@ -77,12 +77,8 @@ const ( envSharedNFSDatastoreURL = "SHARED_NFS_DATASTORE_URL" envSharedVMFSDatastoreURL = "SHARED_VMFS_DATASTORE_URL" envSharedVMFSDatastore2URL = "SHARED_VMFS_DATASTORE2_URL" - envVsanDirectSetup = "USE_VSAN_DIRECT_DATASTORE_IN_WCP" - envVsanDDatastoreURL = "SHARED_VSAND_DATASTORE_URL" - envVsanDDatastore2URL = "SHARED_VSAND_DATASTORE2_URL" envStoragePolicyNameForNonSharedDatastores = "STORAGE_POLICY_FOR_NONSHARED_DATASTORES" envStoragePolicyNameForSharedDatastores = "STORAGE_POLICY_FOR_SHARED_DATASTORES" - envStoragePolicyNameForVsanVmfsDatastores = "STORAGE_POLICY_FOR_VSAN_VMFS_DATASTORES" envStoragePolicyNameForSharedDatastores2 = "STORAGE_POLICY_FOR_SHARED_DATASTORES_2" envStoragePolicyNameFromInaccessibleZone = "STORAGE_POLICY_FROM_INACCESSIBLE_ZONE" envStoragePolicyNameWithThickProvision = "STORAGE_POLICY_WITH_THICK_PROVISIONING" @@ -96,11 +92,8 @@ const ( envVmdkDiskURL = "DISK_URL_PATH" envVolumeOperationsScale = "VOLUME_OPS_SCALE" envComputeClusterName = "COMPUTE_CLUSTER_NAME" - envTKGImage = "TKG_IMAGE_NAME" execCommand = "/bin/df -T /mnt/volume1 | " + "/bin/awk 'FNR == 2 {print $2}' > /mnt/volume1/fstype && while true ; do sleep 2 ; done" - execRWXCommandPod = "echo 'Hello message from Pod' > /mnt/volume1/Pod.html && " + - "chmod o+rX /mnt /mnt/volume1/Pod.html && while true ; do sleep 2 ; done" execRWXCommandPod1 = "echo 'Hello message from Pod1' > /mnt/volume1/Pod1.html && " + "chmod o+rX /mnt /mnt/volume1/Pod1.html && while true ; do sleep 2 ; done" execRWXCommandPod2 = "echo 'Hello message from Pod2' > /mnt/volume1/Pod2.html && " + @@ -199,8 +192,8 @@ const ( waitTimeForCNSNodeVMAttachmentReconciler = 30 * time.Second wcpServiceName = "wcp" vmcWcpHost = "10.2.224.24" //This is the LB IP of VMC WCP and its constant - devopsTKG = "test-cluster-e2e-script-2" - cloudadminTKG = "test-cluster-e2e-script-3" + devopsTKG = "test-cluster-e2e-script" + cloudadminTKG = "test-cluster-e2e-script-1" vmOperatorAPI = "/apis/vmoperator.vmware.com/v1alpha1/" devopsUser = "testuser" zoneKey = "failure-domain.beta.kubernetes.io/zone" @@ -221,70 +214,17 @@ const ( topologyLength = 5 tkgshaTopologyLevels = 1 vmcPrdEndpoint = "https://vmc.vmware.com/vmc/api/orgs/" - vsphereClusterIdConfigMapName = "vsphere-csi-cluster-id" authAPI = "https://console.cloud.vmware.com/csp/gateway/am/api/auth" + "/api-tokens/authorize" ) -/* -// test suite labels - -flaky -> label include the testcases which fails intermittently -disruptive -> label include the testcases which are disruptive in nature -vanilla -> label include the testcases for block, file, configSecret, topology etc. -stable -> label include the testcases which do not fail -longRunning -> label include the testcases which takes longer time for completion -p0 -> label include the testcases which are P0 -p1 -> label include the testcases which are P1 -p2 -> label include the testcases which are P2 -semiAutomated -> label include the testcases which are semi-automated -newTests -> label include the testcases which are newly automated -core -> label include the testcases specific to block or file -level2 -> label include the level-2 topology testcases or pipeline specific -level5 -> label include the level-5 topology testcases -customPort -> label include the testcases running on vCenter custom port -deprecated ->label include the testcases which are no longer in execution -*/ -const ( - flaky = "flaky" - disruptive = "disruptive" - wcp = "wcp" - tkg = "tkg" - vanilla = "vanilla" - topology = "topology" - preferential = "preferential" - vsphereConfigSecret = "vsphereConfigSecret" - snapshot = "snapshot" - stable = "stable" - newTests = "newTests" - multiVc = "multiVc" - block = "block" - file = "file" - core = "core" - p0 = "p0" - p1 = "p1" - p2 = "p2" - vsanStretch = "vsanStretch" - longRunning = "longRunning" - deprecated = "deprecated" - vmc = "vmc" - tkgsHA = "tkgsHA" - thickThin = "thickThin" - customPort = "customPort" - windows = "windows" - semiAutomated = "semiAutomated" - level2 = "level2" - level5 = "level5" -) - // The following variables are required to know cluster type to run common e2e // tests. These variables will be set once during test suites initialization. var ( - vanillaCluster bool - supervisorCluster bool - guestCluster bool - rwxAccessMode bool - wcpVsanDirectCluster bool + vanillaCluster bool + supervisorCluster bool + guestCluster bool + rwxAccessMode bool ) // For busybox pod image @@ -303,6 +243,7 @@ var ( migratedPluginAnnotation = "storage.alpha.kubernetes.io/migrated-plugins" pvcAnnotationStorageProvisioner = "volume.beta.kubernetes.io/storage-provisioner" pvAnnotationProvisionedBy = "pv.kubernetes.io/provisioned-by" + scAnnotation4Statefulset = "volume.beta.kubernetes.io/storage-class" nodeMapper = &NodeMapper{} ) @@ -319,6 +260,11 @@ var ( configSecretTestUser2 = "testuser2" ) +// CSI Internal FSSs +var ( + useCsiNodeID = "use-csinode-id" +) + // Nimbus generated passwords var ( nimbusK8sVmPwd = "NIMBUS_K8S_VM_PWD" @@ -346,24 +292,6 @@ var ( datastoreClusterMap = "DATASTORE_CLUSTER_MAP" ) -// For multivc -var ( - envSharedDatastoreURLVC1 = "SHARED_VSPHERE_DATASTORE_URL_VC1" - envSharedDatastoreURLVC2 = "SHARED_VSPHERE_DATASTORE_URL_VC2" - envStoragePolicyNameToDeleteLater = "STORAGE_POLICY_TO_DELETE_LATER" - envMultiVCSetupType = "MULTI_VC_SETUP_TYPE" - envStoragePolicyNameVC1 = "STORAGE_POLICY_VC1" - envStoragePolicyNameInVC1VC2 = "STORAGE_POLICY_NAME_COMMON_IN_VC1_VC2" - envPreferredDatastoreUrlVC1 = "PREFERRED_DATASTORE_URL_VC1" - envPreferredDatastoreUrlVC2 = "PREFERRED_DATASTORE_URL_VC2" -) - -// VolumeSnapshotClass env variables for tkg-snapshot -var ( - envVolSnapClassDel = "VOLUME_SNAPSHOT_CLASS_DELETE" - deletionPolicy = "Delete" -) - // GetAndExpectStringEnvVar parses a string from env variable. func GetAndExpectStringEnvVar(varName string) string { varValue := os.Getenv(varName) diff --git a/tests/e2e/file_volume_statefulsets.go b/tests/e2e/file_volume_statefulsets.go index 8b2c531545..1303c1cfb3 100644 --- a/tests/e2e/file_volume_statefulsets.go +++ b/tests/e2e/file_volume_statefulsets.go @@ -25,7 +25,6 @@ import ( ginkgo "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" - "golang.org/x/crypto/ssh" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -33,7 +32,6 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" fpod "k8s.io/kubernetes/test/e2e/framework/pod" - fpv "k8s.io/kubernetes/test/e2e/framework/pv" fss "k8s.io/kubernetes/test/e2e/framework/statefulset" admissionapi "k8s.io/pod-security-admission/api" ) @@ -118,7 +116,7 @@ var _ = ginkgo.Describe("[csi-file-vanilla] File Volume statefulset", func() { statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1].Spec.AccessModes[0] = v1.ReadWriteMany statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &scName + Annotations["volume.beta.kubernetes.io/storage-class"] = scName CreateStatefulSet(namespace, statefulset, client) replicas := *(statefulset.Spec.Replicas) // Waiting for pods status to be Ready @@ -284,7 +282,7 @@ var _ = ginkgo.Describe("[csi-file-vanilla] File Volume statefulset", func() { statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1].Spec.AccessModes[0] = v1.ReadWriteMany statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &scName + Annotations["volume.beta.kubernetes.io/storage-class"] = scName ginkgo.By("Creating statefulset") CreateStatefulSet(namespace, statefulset, client) replicas := *(statefulset.Spec.Replicas) @@ -443,7 +441,7 @@ var _ = ginkgo.Describe("[csi-file-vanilla] File Volume statefulset", func() { statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1].Spec.AccessModes[0] = v1.ReadWriteMany statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &scName + Annotations["volume.beta.kubernetes.io/storage-class"] = scName CreateStatefulSet(namespace, statefulset, client) replicas := *(statefulset.Spec.Replicas) // Waiting for pods status to be Ready @@ -559,7 +557,7 @@ var _ = ginkgo.Describe("[csi-file-vanilla] File Volume statefulset", func() { statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1].Spec.AccessModes[0] = v1.ReadWriteMany statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &scName + Annotations["volume.beta.kubernetes.io/storage-class"] = scName CreateStatefulSet(namespace, statefulset, client) replicas := *(statefulset.Spec.Replicas) // Waiting for pods status to be Ready @@ -701,118 +699,4 @@ var _ = ginkgo.Describe("[csi-file-vanilla] File Volume statefulset", func() { gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(replicas)).To(gomega.BeTrue(), "Number of Pods in the statefulset should match with number of replicas") }) - - /* - Verify List volume Response on vsphere-csi-controller logs - Note: ist volume Threshold is set to 1 , and query limit set to 3 - 1. Create SC - 2. Create statefull set with 3 replica - 3. Bring down the CSI driver replica to 1 , so that it is easy to validate the List volume Response. - 4. Wait for all the PVC to reach bound and PODs to reach running state. - 5. Verify the Listvolume response in logs. It should contain all the 3 volumeID's noted in step 5 - 6. Scale up the Statefullset replica to 5 and validate the Pagination. - The 1st List volume Response will have the "token for next set:" - 7. Delete All the volumes - 8. Verify list volume response for 0 volume. - 9. Clean up the statefull set - 10. Increase the CSI driver replica to 3 - - */ - ginkgo.It("List-volumeResponseFor-fileVolumes", func() { - curtime := time.Now().Unix() - randomValue := rand.Int() - val := strconv.FormatInt(int64(randomValue), 10) - val = string(val[1:3]) - curtimestring := strconv.FormatInt(curtime, 10) - scName := "nginx-sc-default-" + curtimestring + val - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - var volumesBeforeScaleUp []string - containerName := "vsphere-csi-controller" - - ginkgo.By("scale down CSI driver POD to 1 , so that it will" + - "be easy to validate all Listvolume response on one driver POD") - collectPodLogs(ctx, client, csiSystemNamespace) - scaledownCSIDriver, err := scaleCSIDriver(ctx, client, namespace, 1) - gomega.Expect(scaledownCSIDriver).To(gomega.BeTrue(), "csi driver scaledown is not successful") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - ginkgo.By("Scale up the csi-driver replica to 3") - success, err := scaleCSIDriver(ctx, client, namespace, 3) - gomega.Expect(success).To(gomega.BeTrue(), "csi driver scale up to 3 replica not successful") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Creating StorageClass for Statefulset") - scParameters[scParamFsType] = nfs4FSType - scSpec := getVSphereStorageClassSpec(scName, scParameters, nil, "", "", false) - sc, err := client.StorageV1().StorageClasses().Create(ctx, scSpec, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err := client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Creating service") - service := CreateService(namespace, client) - defer func() { - deleteService(namespace, client, service) - }() - ginkgo.By("Creating statefulset with replica 3 and a deployment") - statefulset, _, volumesBeforeScaleUp := createStsDeployment(ctx, client, namespace, sc, true, - false, 0, "", v1.ReadWriteMany) - replicas := *(statefulset.Spec.Replicas) - - //List volume responses will show up in the interval of every 1 minute. - time.Sleep(pollTimeoutShort) - nimbusGeneratedK8sVmPwd := GetAndExpectStringEnvVar(nimbusK8sVmPwd) - sshClientConfig := &ssh.ClientConfig{ - User: "root", - Auth: []ssh.AuthMethod{ - ssh.Password(nimbusGeneratedK8sVmPwd), - }, - HostKeyCallback: ssh.InsecureIgnoreHostKey(), - } - - ginkgo.By("Validate ListVolume Response for all the volumes") - logMessage := "List volume response: entries:" - _, _, err = getCSIPodWhereListVolumeResponseIsPresent(ctx, client, sshClientConfig, - containerName, logMessage, volumesBeforeScaleUp) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - replicas = replicas + 2 - scaleUpStsAndVerifyPodMetadata(ctx, client, namespace, statefulset, - replicas, true, true) - - time.Sleep(pollTimeoutShort) - ginkgo.By("Validate pagination") - logMessage = "token for next set: 3" - _, _, err = getCSIPodWhereListVolumeResponseIsPresent(ctx, client, sshClientConfig, containerName, logMessage, nil) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - replicas = 0 - ginkgo.By(fmt.Sprintf("Scaling down statefulsets to number of Replica: %v", replicas)) - _, scaledownErr := fss.Scale(client, statefulset, replicas) - gomega.Expect(scaledownErr).NotTo(gomega.HaveOccurred()) - fss.WaitForStatusReplicas(client, statefulset, replicas) - ssPodsAfterScaleDown := fss.GetPodList(client, statefulset) - gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(replicas)).To(gomega.BeTrue(), - "Number of Pods in the statefulset should match with number of replicas") - - pvcList := getAllPVCFromNamespace(client, namespace) - for _, pvc := range pvcList.Items { - framework.ExpectNoError(fpv.DeletePersistentVolumeClaim(client, pvc.Name, namespace), - "Failed to delete PVC", pvc.Name) - } - //List volume responses will show up in the interval of every 1 minute. - //To see the empty response, It is required to wait for 1 min after deleteting all the PVC's - time.Sleep(pollTimeoutShort) - - ginkgo.By("Validate ListVolume Response when no volumes are present") - logMessage = "ListVolumes served 0 results" - - _, _, err = getCSIPodWhereListVolumeResponseIsPresent(ctx, client, sshClientConfig, containerName, logMessage, nil) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) }) diff --git a/tests/e2e/fullsync_test_for_block_volume.go b/tests/e2e/fullsync_test_for_block_volume.go index d85b55252d..0e8d65e8a4 100644 --- a/tests/e2e/fullsync_test_for_block_volume.go +++ b/tests/e2e/fullsync_test_for_block_volume.go @@ -127,16 +127,11 @@ var _ bool = ginkgo.Describe("full-sync-test", func() { vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + sshdPort ctx, cancel := context.WithCancel(context.Background()) defer cancel() - if isVsanHealthServiceStopped { - startVCServiceWait4VPs(ctx, vcAddress, vsanhealthServiceName, &isVsanHealthServiceStopped) - } if supervisorCluster { deleteResourceQuota(client, namespace) - dumpSvcNsEventsOnTestFailure(client, namespace) } - if guestCluster { - svcClient, svNamespace := getSvcClientAndNamespace() - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) + if isVsanHealthServiceStopped { + startVCServiceWait4VPs(ctx, vcAddress, vsanhealthServiceName, &isVsanHealthServiceStopped) } }) @@ -798,7 +793,7 @@ var _ bool = ginkgo.Describe("full-sync-test", func() { ginkgo.By("create a pvc pvc1, wait for pvc bound to pv") volHandle, pvc, pv, storageclass := createSCwithVolumeExpansionTrueAndDynamicPVC( - f, client, "", storagePolicyName, namespace, ext4FSType) + f, client, "", storagePolicyName, namespace) defer func() { if !supervisorCluster { err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) @@ -815,7 +810,7 @@ var _ bool = ginkgo.Describe("full-sync-test", func() { }() ginkgo.By("create a pod pod1, using pvc1") - pod, _ := createPODandVerifyVolumeMount(ctx, f, client, namespace, pvc, volHandle, "") + pod, _ := createPODandVerifyVolumeMount(ctx, f, client, namespace, pvc, volHandle) defer func() { err := fpod.DeletePodWithWait(client, pod) gomega.Expect(err).NotTo(gomega.HaveOccurred()) diff --git a/tests/e2e/gc_block_resize_retain_policy.go b/tests/e2e/gc_block_resize_retain_policy.go index 35eb64ddb4..93866bbfbe 100644 --- a/tests/e2e/gc_block_resize_retain_policy.go +++ b/tests/e2e/gc_block_resize_retain_policy.go @@ -161,7 +161,6 @@ var _ = ginkgo.Describe("[csi-guest] Volume Expansion Tests with reclaimation po gomega.Expect(err).NotTo(gomega.HaveOccurred()) svcClient, svNamespace := getSvcClientAndNamespace() setResourceQuota(svcClient, svNamespace, defaultrqLimit) - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) }) // Combined: diff --git a/tests/e2e/gc_block_volume_expansion.go b/tests/e2e/gc_block_volume_expansion.go index 6d480951dc..780ec1e85c 100644 --- a/tests/e2e/gc_block_volume_expansion.go +++ b/tests/e2e/gc_block_volume_expansion.go @@ -146,9 +146,8 @@ var _ = ginkgo.Describe("[csi-guest] Volume Expansion Test", func() { setResourceQuota(svcClient, svNamespace, defaultrqLimit) if isGCCSIDeploymentPODdown { - _ = updateDeploymentReplica(client, 3, vSphereCSIControllerPodNamePrefix, csiSystemNamespace) + _ = updateDeploymentReplica(client, 1, vSphereCSIControllerPodNamePrefix, csiSystemNamespace) } - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) }) // Verify offline expansion triggers FS resize. diff --git a/tests/e2e/gc_cns_nodevm_attachment.go b/tests/e2e/gc_cns_nodevm_attachment.go index 30fb071ad4..2a9caa272f 100644 --- a/tests/e2e/gc_cns_nodevm_attachment.go +++ b/tests/e2e/gc_cns_nodevm_attachment.go @@ -67,7 +67,6 @@ var _ = ginkgo.Describe("[csi-guest] CnsNodeVmAttachment persistence", func() { ginkgo.AfterEach(func() { svcClient, svNamespace := getSvcClientAndNamespace() setResourceQuota(svcClient, svNamespace, defaultrqLimit) - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) }) // Test-1 (Attach/Detach) diff --git a/tests/e2e/gc_file_share_negative.go b/tests/e2e/gc_file_share_negative.go index 77b26ad209..f8f04fba82 100644 --- a/tests/e2e/gc_file_share_negative.go +++ b/tests/e2e/gc_file_share_negative.go @@ -59,7 +59,6 @@ var _ = ginkgo.Describe("[csi-guest] File Share on Non File Service enabled setu ginkgo.AfterEach(func() { svcClient, svNamespace := getSvcClientAndNamespace() setResourceQuota(svcClient, svNamespace, defaultrqLimit) - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) }) /* diff --git a/tests/e2e/gc_full_sync.go b/tests/e2e/gc_full_sync.go index 1292f04842..67bdff05eb 100644 --- a/tests/e2e/gc_full_sync.go +++ b/tests/e2e/gc_full_sync.go @@ -83,7 +83,6 @@ var _ = ginkgo.Describe("[csi-guest] Guest cluster fullsync tests", func() { ginkgo.AfterEach(func() { svcClient, svNamespace := getSvcClientAndNamespace() setResourceQuota(svcClient, svNamespace, defaultrqLimit) - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) }) // Steps: diff --git a/tests/e2e/gc_metadata_syncer.go b/tests/e2e/gc_metadata_syncer.go index 8738a7f4ba..bd15153b82 100644 --- a/tests/e2e/gc_metadata_syncer.go +++ b/tests/e2e/gc_metadata_syncer.go @@ -96,7 +96,6 @@ var _ = ginkgo.Describe("[csi-guest] pvCSI metadata syncer tests", func() { ginkgo.By(fmt.Sprintln("Starting vsan-health on the vCenter host")) startVCServiceWait4VPs(ctx, vcAddress, vsanhealthServiceName, &isVsanHealthServiceStopped) } - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) }) // Steps: diff --git a/tests/e2e/gc_rwx_basic.go b/tests/e2e/gc_rwx_basic.go index 0e63c1e1ad..3923f0559e 100644 --- a/tests/e2e/gc_rwx_basic.go +++ b/tests/e2e/gc_rwx_basic.go @@ -44,7 +44,6 @@ var _ = ginkgo.Describe("[rwm-csi-tkg] Basic File Volume Provision Test", func() scParameters map[string]string storagePolicyName string ) - ginkgo.BeforeEach(func() { client = f.ClientSet namespace = getNamespaceToRunTests(f) @@ -58,11 +57,6 @@ var _ = ginkgo.Describe("[rwm-csi-tkg] Basic File Volume Provision Test", func() } }) - ginkgo.AfterEach(func() { - svcClient, svNamespace := getSvcClientAndNamespace() - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) - }) - /* Test to verify file volume provision - basic tests. diff --git a/tests/e2e/gc_rwx_deployments.go b/tests/e2e/gc_rwx_deployments.go index 16ed6c3672..0daef0b13b 100644 --- a/tests/e2e/gc_rwx_deployments.go +++ b/tests/e2e/gc_rwx_deployments.go @@ -67,7 +67,6 @@ var _ = ginkgo.Describe("[rwm-csi-tkg] File Volume Provision with Deployments", ginkgo.AfterEach(func() { svcClient, svNamespace := getSvcClientAndNamespace() setResourceQuota(svcClient, svNamespace, defaultrqLimit) - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) }) /* diff --git a/tests/e2e/gc_rwx_destructive.go b/tests/e2e/gc_rwx_destructive.go index 676e45f944..77f039d72b 100644 --- a/tests/e2e/gc_rwx_destructive.go +++ b/tests/e2e/gc_rwx_destructive.go @@ -68,7 +68,6 @@ var _ = ginkgo.Describe("[rwm-csi-destructive-tkg] Statefulsets with File Volume ginkgo.AfterEach(func() { svcClient, svNamespace := getSvcClientAndNamespace() setResourceQuota(svcClient, svNamespace, defaultrqLimit) - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) }) /* @@ -168,11 +167,10 @@ var _ = ginkgo.Describe("[rwm-csi-destructive-tkg] Statefulsets with File Volume statefulset := GetStatefulSetFromManifest(namespace) ginkgo.By("Creating statefulset") - scName := defaultNginxStorageClassName statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1].Spec.AccessModes[0] = v1.ReadWriteMany statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &scName + Annotations["volume.beta.kubernetes.io/storage-class"] = defaultNginxStorageClassName *statefulset.Spec.Replicas = 2 CreateStatefulSet(namespace, statefulset, clientNewGc) replicas := *(statefulset.Spec.Replicas) diff --git a/tests/e2e/gc_rwx_multi_gc.go b/tests/e2e/gc_rwx_multi_gc.go index d5e4088f75..fbb01ab52a 100644 --- a/tests/e2e/gc_rwx_multi_gc.go +++ b/tests/e2e/gc_rwx_multi_gc.go @@ -69,7 +69,6 @@ var _ = ginkgo.Describe("[rwm-csi-tkg] Volume Provision Across TKG clusters", fu ginkgo.AfterEach(func() { svcClient, svNamespace := getSvcClientAndNamespace() setResourceQuota(svcClient, svNamespace, defaultrqLimit) - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) }) /* diff --git a/tests/e2e/gc_rwx_multi_ns_gc.go b/tests/e2e/gc_rwx_multi_ns_gc.go index ad1e4cf627..0fd13aa3c0 100644 --- a/tests/e2e/gc_rwx_multi_ns_gc.go +++ b/tests/e2e/gc_rwx_multi_ns_gc.go @@ -66,7 +66,6 @@ var _ = ginkgo.Describe("[rwm-csi-tkg] Volume Provision Across Namespace", func( ginkgo.AfterEach(func() { svcClient, svNamespace := getSvcClientAndNamespace() setResourceQuota(svcClient, svNamespace, defaultrqLimit) - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) }) /* diff --git a/tests/e2e/gc_rwx_non_vsan_datastore.go b/tests/e2e/gc_rwx_non_vsan_datastore.go index 10d57b1182..8550934fc4 100644 --- a/tests/e2e/gc_rwx_non_vsan_datastore.go +++ b/tests/e2e/gc_rwx_non_vsan_datastore.go @@ -39,7 +39,6 @@ var _ = ginkgo.Describe("[rwm-csi-tkg] File Volume Provision with Non-VSAN datas scParameters map[string]string nonVsanStoragePolicyName string ) - ginkgo.BeforeEach(func() { client = f.ClientSet namespace = getNamespaceToRunTests(f) @@ -53,11 +52,6 @@ var _ = ginkgo.Describe("[rwm-csi-tkg] File Volume Provision with Non-VSAN datas } }) - ginkgo.AfterEach(func() { - svcClient, svNamespace := getSvcClientAndNamespace() - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) - }) - /* Test to verify file volume provision on non-vsan datastore - ReadWriteMany diff --git a/tests/e2e/gc_rwx_operation_storm.go b/tests/e2e/gc_rwx_operation_storm.go index 2bf07fc77d..69403a929e 100644 --- a/tests/e2e/gc_rwx_operation_storm.go +++ b/tests/e2e/gc_rwx_operation_storm.go @@ -74,7 +74,6 @@ var _ = ginkgo.Describe("[rwm-csi-tkg] File Volume Operation storm Test", func() ginkgo.AfterEach(func() { svcClient, svNamespace := getSvcClientAndNamespace() setResourceQuota(svcClient, svNamespace, defaultrqLimit) - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) }) /* diff --git a/tests/e2e/gc_rwx_parallel_claim.go b/tests/e2e/gc_rwx_parallel_claim.go index cc7db8ba7d..6cf43919ba 100644 --- a/tests/e2e/gc_rwx_parallel_claim.go +++ b/tests/e2e/gc_rwx_parallel_claim.go @@ -65,7 +65,6 @@ var _ = ginkgo.Describe("[rwm-csi-tkg] PVCs claiming the available resource in p ginkgo.AfterEach(func() { svcClient, svcNamespace := getSvcClientAndNamespace() setResourceQuota(svcClient, svcNamespace, rqLimit) - dumpSvcNsEventsOnTestFailure(svcClient, svcNamespace) }) /* diff --git a/tests/e2e/gc_rwx_readonly.go b/tests/e2e/gc_rwx_readonly.go index 84f1625a99..c36457ffc6 100644 --- a/tests/e2e/gc_rwx_readonly.go +++ b/tests/e2e/gc_rwx_readonly.go @@ -57,11 +57,6 @@ var _ = ginkgo.Describe("[rwm-csi-tkg] File Volume Test for ReadOnlyMany", func( } }) - ginkgo.AfterEach(func() { - svcClient, svNamespace := getSvcClientAndNamespace() - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) - }) - /* Test to verify Pod restricts write into PVC diff --git a/tests/e2e/gc_rwx_reclaim_policy.go b/tests/e2e/gc_rwx_reclaim_policy.go index ea8e18883e..c8f9410199 100644 --- a/tests/e2e/gc_rwx_reclaim_policy.go +++ b/tests/e2e/gc_rwx_reclaim_policy.go @@ -67,7 +67,6 @@ var _ = ginkgo.Describe("File Volume Test for Reclaim Policy", func() { ginkgo.AfterEach(func() { svcClient, svNamespace := getSvcClientAndNamespace() setResourceQuota(svcClient, svNamespace, defaultrqLimit) - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) }) /* diff --git a/tests/e2e/gc_rwx_security_context.go b/tests/e2e/gc_rwx_security_context.go index fe8ee96f43..cd468bd996 100644 --- a/tests/e2e/gc_rwx_security_context.go +++ b/tests/e2e/gc_rwx_security_context.go @@ -65,7 +65,6 @@ var _ = ginkgo.Describe("File Volume Test with security context", func() { ginkgo.AfterEach(func() { svcClient, svNamespace := getSvcClientAndNamespace() setResourceQuota(svcClient, svNamespace, defaultrqLimit) - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) }) /* diff --git a/tests/e2e/gc_rwx_service_down.go b/tests/e2e/gc_rwx_service_down.go index 9a3cac0929..8bcd5342f8 100644 --- a/tests/e2e/gc_rwx_service_down.go +++ b/tests/e2e/gc_rwx_service_down.go @@ -87,7 +87,6 @@ var _ = ginkgo.Describe("File Volume Test on Service down", func() { vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + sshdPort startVCServiceWait4VPs(ctx, vcAddress, vsanhealthServiceName, &isVsanHealthServiceStopped) } - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) }) /* diff --git a/tests/e2e/gc_rwx_statefulsets.go b/tests/e2e/gc_rwx_statefulsets.go index b1af58b47b..fd00b067b7 100644 --- a/tests/e2e/gc_rwx_statefulsets.go +++ b/tests/e2e/gc_rwx_statefulsets.go @@ -81,7 +81,6 @@ var _ = ginkgo.Describe("[rwm-csi-tkg] File Volume Provision with Statefulsets", } svcClient, svNamespace := getSvcClientAndNamespace() setResourceQuota(svcClient, svNamespace, defaultrqLimit) - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) }) /* @@ -138,11 +137,10 @@ var _ = ginkgo.Describe("[rwm-csi-tkg] File Volume Provision with Statefulsets", }() statefulset := GetStatefulSetFromManifest(namespace) ginkgo.By("Creating statefulset") - scName := defaultNginxStorageClassName statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1].Spec.AccessModes[0] = v1.ReadWriteMany statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &scName + Annotations["volume.beta.kubernetes.io/storage-class"] = defaultNginxStorageClassName CreateStatefulSet(namespace, statefulset, client) replicas := *(statefulset.Spec.Replicas) @@ -433,12 +431,11 @@ var _ = ginkgo.Describe("[rwm-csi-tkg] File Volume Provision with Statefulsets", }() statefulset := GetStatefulSetFromManifest(namespace) ginkgo.By("Creating statefulset") - scName := defaultNginxStorageClassName statefulset.Spec.PodManagementPolicy = apps.ParallelPodManagement statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1].Spec.AccessModes[0] = v1.ReadWriteMany statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &scName + Annotations["volume.beta.kubernetes.io/storage-class"] = defaultNginxStorageClassName CreateStatefulSet(namespace, statefulset, client) replicas := *(statefulset.Spec.Replicas) diff --git a/tests/e2e/gc_rwx_static_provision.go b/tests/e2e/gc_rwx_static_provision.go index f0c691c3a7..0aedd70e51 100644 --- a/tests/e2e/gc_rwx_static_provision.go +++ b/tests/e2e/gc_rwx_static_provision.go @@ -57,11 +57,6 @@ var _ = ginkgo.Describe("[rwm-csi-tkg] File Volume static Provision Test", func( } }) - ginkgo.AfterEach(func() { - svcClient, svNamespace := getSvcClientAndNamespace() - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) - }) - /* Test to verify static volume provision. diff --git a/tests/e2e/gc_rwx_syncer.go b/tests/e2e/gc_rwx_syncer.go index c3d5ade970..190fdb9e69 100644 --- a/tests/e2e/gc_rwx_syncer.go +++ b/tests/e2e/gc_rwx_syncer.go @@ -74,7 +74,6 @@ var _ = ginkgo.Describe("[rwm-csi-tkg] File Volume Test for label updates", func ginkgo.By(fmt.Sprintf("Starting %v on the vCenter host", vsanhealthServiceName)) startVCServiceWait4VPs(ctx, vcAddress, vsanhealthServiceName, &isVsanHealthServiceStopped) } - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) }) /* diff --git a/tests/e2e/gc_rwx_tkg_scale.go b/tests/e2e/gc_rwx_tkg_scale.go index e5ab92ab2f..bd8299b618 100644 --- a/tests/e2e/gc_rwx_tkg_scale.go +++ b/tests/e2e/gc_rwx_tkg_scale.go @@ -78,7 +78,6 @@ var _ = ginkgo.Describe("[rwm-csi-tkg] TKG RWX for STS with GC worker nodes scal } svcClient, svNamespace := getSvcClientAndNamespace() setResourceQuota(svcClient, svNamespace, defaultrqLimit) - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) }) /* @@ -136,11 +135,10 @@ var _ = ginkgo.Describe("[rwm-csi-tkg] TKG RWX for STS with GC worker nodes scal statefulset := GetStatefulSetFromManifest(namespace) ginkgo.By("Creating statefulset") - scName := defaultNginxStorageClassName statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1].Spec.AccessModes[0] = v1.ReadWriteMany statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &scName + Annotations["volume.beta.kubernetes.io/storage-class"] = defaultNginxStorageClassName CreateStatefulSet(namespace, statefulset, client) replicas := *(statefulset.Spec.Replicas) @@ -379,12 +377,11 @@ var _ = ginkgo.Describe("[rwm-csi-tkg] TKG RWX for STS with GC worker nodes scal }() statefulset := GetStatefulSetFromManifest(namespace) ginkgo.By("Creating statefulset") - scName := defaultNginxStorageClassName statefulset.Spec.PodManagementPolicy = apps.ParallelPodManagement statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1].Spec.AccessModes[0] = v1.ReadWriteMany statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &scName + Annotations["volume.beta.kubernetes.io/storage-class"] = defaultNginxStorageClassName CreateStatefulSet(namespace, statefulset, client) replicas := *(statefulset.Spec.Replicas) diff --git a/tests/e2e/gc_rwx_volume_health.go b/tests/e2e/gc_rwx_volume_health.go index 71e598c224..b2bd9d09ab 100644 --- a/tests/e2e/gc_rwx_volume_health.go +++ b/tests/e2e/gc_rwx_volume_health.go @@ -73,8 +73,6 @@ var _ = ginkgo.Describe("File Volume Test volume health plumbing", func() { ginkgo.By(fmt.Sprintf("Starting %v on the vCenter host", vsanhealthServiceName)) startVCServiceWait4VPs(ctx, vcAddress, vsanhealthServiceName, &isVsanHealthServiceStopped) } - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) - dumpSvcNsEventsOnTestFailure(svcClient, csiSystemNamespace) }) /* diff --git a/tests/e2e/improved_csi_idempotency.go b/tests/e2e/improved_csi_idempotency.go index 28e4ea1f0b..e39630d785 100644 --- a/tests/e2e/improved_csi_idempotency.go +++ b/tests/e2e/improved_csi_idempotency.go @@ -124,6 +124,14 @@ var _ = ginkgo.Describe("[csi-block-vanilla] [csi-file-vanilla] "+ ginkgo.AfterEach(func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() + + if supervisorCluster { + deleteResourceQuota(client, namespace) + } + if guestCluster { + svcClient, svNamespace := getSvcClientAndNamespace() + setResourceQuota(svcClient, svNamespace, defaultrqLimit) + } if isServiceStopped { if serviceName == "CSI" { framework.Logf("Starting CSI driver") @@ -141,7 +149,7 @@ var _ = ginkgo.Describe("[csi-block-vanilla] [csi-file-vanilla] "+ gomega.Expect(err).NotTo(gomega.HaveOccurred()) } else if serviceName == hostdServiceName { framework.Logf("In afterEach function to start the hostd service on all hosts") - hostIPs := getAllHostsIP(ctx, true) + hostIPs := getAllHostsIP(ctx) for _, hostIP := range hostIPs { startHostDOnHost(ctx, hostIP) } @@ -157,16 +165,6 @@ var _ = ginkgo.Describe("[csi-block-vanilla] [csi-file-vanilla] "+ ginkgo.By(fmt.Sprintf("Resetting provisioner time interval to %s sec", defaultProvisionerTimeInSec)) updateCSIDeploymentProvisionerTimeout(c, csiSystemNamespace, defaultProvisionerTimeInSec) - - if supervisorCluster { - deleteResourceQuota(client, namespace) - dumpSvcNsEventsOnTestFailure(client, namespace) - } - if guestCluster { - svcClient, svNamespace := getSvcClientAndNamespace() - setResourceQuota(svcClient, svNamespace, defaultrqLimit) - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) - } }) /* @@ -398,7 +396,7 @@ func createVolumesByReducingProvisionerTime(namespace string, client clientset.I ginkgo.By("Waiting for all claims to be in bound state") persistentvolumes, err = fpv.WaitForPVClaimBoundPhase(client, pvclaims, - 2*framework.ClaimProvisionTimeout) + framework.ClaimProvisionTimeout) gomega.Expect(err).NotTo(gomega.HaveOccurred()) // TODO: Add a logic to check for the no orphan volumes @@ -509,18 +507,18 @@ func createVolumeWithServiceDown(serviceName string, namespace string, client cl csiReplicaCount := *deployment.Spec.Replicas ginkgo.By("Stopping CSI driver") - isServiceStopped, err = stopCSIPods(ctx, c, csiSystemNamespace) + isServiceStopped, err = stopCSIPods(ctx, c) gomega.Expect(err).NotTo(gomega.HaveOccurred()) defer func() { if isServiceStopped { framework.Logf("Starting CSI driver") - isServiceStopped, err = startCSIPods(ctx, c, csiReplicaCount, csiSystemNamespace) + isServiceStopped, err = startCSIPods(ctx, c, csiReplicaCount) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() framework.Logf("Starting CSI driver") - isServiceStopped, err = startCSIPods(ctx, c, csiReplicaCount, csiSystemNamespace) + isServiceStopped, err = startCSIPods(ctx, c, csiReplicaCount) gomega.Expect(err).NotTo(gomega.HaveOccurred()) if os.Getenv(envFullSyncWaitTime) != "" { @@ -538,7 +536,7 @@ func createVolumeWithServiceDown(serviceName string, namespace string, client cl time.Sleep(time.Duration(fullSyncWaitTime) * time.Second) } else if serviceName == hostdServiceName { ginkgo.By("Fetch IPs for the all the hosts in the cluster") - hostIPs := getAllHostsIP(ctx, true) + hostIPs := getAllHostsIP(ctx) isServiceStopped = true var wg sync.WaitGroup @@ -605,7 +603,7 @@ func createVolumeWithServiceDown(serviceName string, namespace string, client cl ginkgo.By("Waiting for all claims to be in bound state") persistentvolumes, err = fpv.WaitForPVClaimBoundPhase(client, pvclaims, - 2*framework.ClaimProvisionTimeout) + framework.ClaimProvisionTimeout) gomega.Expect(err).NotTo(gomega.HaveOccurred()) // TODO: Add a logic to check for the no orphan volumes @@ -710,7 +708,7 @@ func extendVolumeWithServiceDown(serviceName string, namespace string, client cl ginkgo.By("Waiting for all claims to be in bound state") persistentvolumes, err = fpv.WaitForPVClaimBoundPhase(client, pvclaims, - 2*framework.ClaimProvisionTimeout) + framework.ClaimProvisionTimeout) gomega.Expect(err).NotTo(gomega.HaveOccurred()) // TODO: Add a logic to check for the no orphan volumes @@ -766,19 +764,19 @@ func extendVolumeWithServiceDown(serviceName string, namespace string, client cl csiReplicaCount := *deployment.Spec.Replicas ginkgo.By("Stopping CSI driver") - isServiceStopped, err = stopCSIPods(ctx, c, csiSystemNamespace) + isServiceStopped, err = stopCSIPods(ctx, c) gomega.Expect(err).NotTo(gomega.HaveOccurred()) defer func() { if isServiceStopped { framework.Logf("Starting CSI driver") - isServiceStopped, err = startCSIPods(ctx, c, csiReplicaCount, csiSystemNamespace) + isServiceStopped, err = startCSIPods(ctx, c, csiReplicaCount) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() framework.Logf("Starting CSI driver") - isServiceStopped, err = startCSIPods(ctx, c, csiReplicaCount, csiSystemNamespace) + isServiceStopped, err = startCSIPods(ctx, c, csiReplicaCount) gomega.Expect(err).NotTo(gomega.HaveOccurred()) if os.Getenv(envFullSyncWaitTime) != "" { @@ -839,7 +837,6 @@ func extendVolumeWithServiceDown(serviceName string, namespace string, client cl // stopHostD is a function for waitGroup to run stop hostd parallelly func stopHostD(ctx context.Context, addr string, wg *sync.WaitGroup) { - defer ginkgo.GinkgoRecover() defer wg.Done() stopHostDOnHost(ctx, addr) } diff --git a/tests/e2e/invalid_topology_values.go b/tests/e2e/invalid_topology_values.go index 81a82ece1b..35b05abc50 100644 --- a/tests/e2e/invalid_topology_values.go +++ b/tests/e2e/invalid_topology_values.go @@ -96,7 +96,7 @@ var _ = ginkgo.Describe("[csi-topology-vanilla] Topology-Aware-Provisioning-With // Get the event list and verify if it contains expected error message eventList, _ := client.CoreV1().Events(pvclaim.Namespace).List(ctx, metav1.ListOptions{}) gomega.Expect(eventList.Items).NotTo(gomega.BeEmpty()) - expectedErrMsg := "No compatible datastores found for accessibility requirements" + expectedErrMsg := "failed to get shared datastores for topology requirement" err = waitForEvent(ctx, client, namespace, expectedErrMsg, pvclaim.Name) gomega.Expect(err).NotTo(gomega.HaveOccurred(), fmt.Sprintf("Expected error : %q", expectedErrMsg)) }) @@ -137,7 +137,7 @@ var _ = ginkgo.Describe("[csi-topology-vanilla] Topology-Aware-Provisioning-With // Get the event list and verify if it contains expected error message eventList, _ := client.CoreV1().Events(pvclaim.Namespace).List(ctx, metav1.ListOptions{}) gomega.Expect(eventList.Items).NotTo(gomega.BeEmpty()) - expectedErrMsg := "No compatible datastores found for accessibility requirements" + expectedErrMsg := "failed to get shared datastores for topology requirement" err = waitForEvent(ctx, client, namespace, expectedErrMsg, pvclaim.Name) gomega.Expect(err).NotTo(gomega.HaveOccurred(), fmt.Sprintf("Expected error : %q", expectedErrMsg)) }) @@ -177,7 +177,7 @@ var _ = ginkgo.Describe("[csi-topology-vanilla] Topology-Aware-Provisioning-With // Get the event list and verify if it contains expected error message eventList, _ := client.CoreV1().Events(pvclaim.Namespace).List(ctx, metav1.ListOptions{}) gomega.Expect(eventList.Items).NotTo(gomega.BeEmpty()) - expectedErrMsg := "No compatible datastores found for accessibility requirements" + expectedErrMsg := "failed to get shared datastores for topology requirement" err = waitForEvent(ctx, client, namespace, expectedErrMsg, pvclaim.Name) gomega.Expect(err).NotTo(gomega.HaveOccurred(), fmt.Sprintf("Expected error %q", expectedErrMsg)) }) diff --git a/tests/e2e/labelupdates.go b/tests/e2e/labelupdates.go index 092e5e7a16..af6988e7f8 100644 --- a/tests/e2e/labelupdates.go +++ b/tests/e2e/labelupdates.go @@ -116,11 +116,6 @@ var _ bool = ginkgo.Describe("[csi-block-vanilla] [csi-block-vanilla-parallelize ginkgo.AfterEach(func() { if supervisorCluster { deleteResourceQuota(client, namespace) - dumpSvcNsEventsOnTestFailure(client, namespace) - } - if guestCluster { - svcClient, svNamespace := getSvcClientAndNamespace() - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) } }) @@ -472,13 +467,10 @@ var _ bool = ginkgo.Describe("[csi-block-vanilla] [csi-block-vanilla-parallelize ginkgo.By(fmt.Sprintf("Deleting pvc %s in namespace %s", pvc.Name, pvc.Namespace)) err = client.CoreV1().PersistentVolumeClaims(namespace).Delete(ctx, pvc.Name, *metav1.NewDeleteOptions(0)) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + // Waiting for some time for PVC to be deleted correctly + ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to allow PVC deletion", oneMinuteWaitTimeInSeconds)) + time.Sleep(time.Duration(oneMinuteWaitTimeInSeconds) * time.Second) - // Waiting for PVC to be deleted correctly - ginkgo.By("Verify if PVC is deleted from namespace") - err = waitForPvcToBeDeleted(ctx, client, pvc.Name, pvc.Namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Verify labels are not present in cns") _, err = e2eVSphere.getLabelsForCNSVolume(pv.Spec.CSI.VolumeHandle, string(cnstypes.CnsKubernetesEntityTypePVC), pvc.Name, namespace) gomega.Expect(err).To(gomega.HaveOccurred()) @@ -687,7 +679,7 @@ var _ bool = ginkgo.Describe("[csi-block-vanilla] [csi-block-vanilla-parallelize ginkgo.By("Creating statefulset") statefulset := GetStatefulSetFromManifest(namespace) statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &storageClassName + Annotations["volume.beta.kubernetes.io/storage-class"] = storageClassName CreateStatefulSet(namespace, statefulset, client) defer func() { ginkgo.By(fmt.Sprintf("Deleting all statefulsets in namespace: %v", namespace)) diff --git a/tests/e2e/multi_master_k8s.go b/tests/e2e/multi_master_k8s.go index a6d68272a5..3d92a40513 100644 --- a/tests/e2e/multi_master_k8s.go +++ b/tests/e2e/multi_master_k8s.go @@ -116,13 +116,6 @@ var _ = ginkgo.Describe("[csi-multi-master-block-e2e]", func() { ginkgo.By("Waiting for old vsphere-csi-controller pod to be removed") err = waitForControllerDeletion(client, controllerNamespace) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - if supervisorCluster { - dumpSvcNsEventsOnTestFailure(client, namespace) - } else { - svcClient, svNamespace := getSvcClientAndNamespace() - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) - } }) /* diff --git a/tests/e2e/multi_vc.go b/tests/e2e/multi_vc.go deleted file mode 100644 index a06ab52f74..0000000000 --- a/tests/e2e/multi_vc.go +++ /dev/null @@ -1,1832 +0,0 @@ -/* - Copyright 2023 The Kubernetes Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package e2e - -import ( - "context" - "fmt" - "os" - "strconv" - "strings" - "sync" - "time" - - ginkgo "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" - cnstypes "github.com/vmware/govmomi/cns/types" - "golang.org/x/crypto/ssh" - v1 "k8s.io/api/core/v1" - storagev1 "k8s.io/api/storage/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/kubernetes/test/e2e/framework" - fnodes "k8s.io/kubernetes/test/e2e/framework/node" - fpod "k8s.io/kubernetes/test/e2e/framework/pod" - fpv "k8s.io/kubernetes/test/e2e/framework/pv" - fss "k8s.io/kubernetes/test/e2e/framework/statefulset" - admissionapi "k8s.io/pod-security-admission/api" -) - -var _ = ginkgo.Describe("[csi-multi-vc-topology] Multi-VC", func() { - f := framework.NewDefaultFramework("csi-multi-vc") - f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged - var ( - client clientset.Interface - namespace string - allowedTopologies []v1.TopologySelectorLabelRequirement - bindingMode storagev1.VolumeBindingMode - allowedTopologyLen int - nodeAffinityToSet bool - parallelStatefulSetCreation bool - stsReplicas int32 - parallelPodPolicy bool - scParameters map[string]string - topValStartIndex int - topValEndIndex int - topkeyStartIndex int - datastoreURLVC1 string - datastoreURLVC2 string - podAntiAffinityToSet bool - sshClientConfig *ssh.ClientConfig - nimbusGeneratedK8sVmPwd string - allMasterIps []string - masterIp string - scaleUpReplicaCount int32 - scaleDownReplicaCount int32 - multiVCSetupType string - isVsanHealthServiceStopped bool - stsScaleUp bool - stsScaleDown bool - verifyTopologyAffinity bool - storagePolicyInVc1 string - storagePolicyInVc1Vc2 string - storagePolicyToDelete string - isSPSServiceStopped bool - isStorageProfileDeleted bool - ) - ginkgo.BeforeEach(func() { - var cancel context.CancelFunc - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - client = f.ClientSet - namespace = f.Namespace.Name - - multiVCbootstrap() - - stsScaleUp = true - stsScaleDown = true - verifyTopologyAffinity = true - - sc, err := client.StorageV1().StorageClasses().Get(ctx, defaultNginxStorageClassName, metav1.GetOptions{}) - if err == nil && sc != nil { - gomega.Expect(client.StorageV1().StorageClasses().Delete(ctx, sc.Name, - *metav1.NewDeleteOptions(0))).NotTo(gomega.HaveOccurred()) - } - - nodeList, err := fnodes.GetReadySchedulableNodes(f.ClientSet) - framework.ExpectNoError(err, "Unable to find ready and schedulable Node") - if !(len(nodeList.Items) > 0) { - framework.Failf("Unable to find ready and schedulable Node") - } - - topologyMap := GetAndExpectStringEnvVar(topologyMap) - allowedTopologies = createAllowedTopolgies(topologyMap, topologyLength) - bindingMode = storagev1.VolumeBindingWaitForFirstConsumer - scParameters = make(map[string]string) - storagePolicyInVc1 = GetAndExpectStringEnvVar(envStoragePolicyNameVC1) - storagePolicyInVc1Vc2 = GetAndExpectStringEnvVar(envStoragePolicyNameInVC1VC2) - storagePolicyToDelete = GetAndExpectStringEnvVar(envStoragePolicyNameToDeleteLater) - datastoreURLVC1 = GetAndExpectStringEnvVar(envSharedDatastoreURLVC1) - datastoreURLVC2 = GetAndExpectStringEnvVar(envSharedDatastoreURLVC2) - nimbusGeneratedK8sVmPwd = GetAndExpectStringEnvVar(nimbusK8sVmPwd) - multiVCSetupType = GetAndExpectStringEnvVar(envMultiVCSetupType) - - sshClientConfig = &ssh.ClientConfig{ - User: "root", - Auth: []ssh.AuthMethod{ - ssh.Password(nimbusGeneratedK8sVmPwd), - }, - HostKeyCallback: ssh.InsecureIgnoreHostKey(), - } - - // fetching k8s master ip - allMasterIps = getK8sMasterIPs(ctx, client) - masterIp = allMasterIps[0] - - }) - - ginkgo.AfterEach(func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - ginkgo.By(fmt.Sprintf("Deleting all statefulsets in namespace: %v", namespace)) - fss.DeleteAllStatefulSets(client, namespace) - ginkgo.By(fmt.Sprintf("Deleting service nginx in namespace: %v", namespace)) - err := client.CoreV1().Services(namespace).Delete(ctx, servicename, *metav1.NewDeleteOptions(0)) - if !apierrors.IsNotFound(err) { - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - framework.Logf("Perform cleanup of any left over stale PVs") - allPvs, err := client.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - for _, pv := range allPvs.Items { - err := client.CoreV1().PersistentVolumes().Delete(ctx, pv.Name, metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - if isVsanHealthServiceStopped { - vCenterHostname := strings.Split(multiVCe2eVSphere.multivcConfig.Global.VCenterHostname, ",") - vcAddress := vCenterHostname[0] + ":" + sshdPort - framework.Logf("Bringing vsanhealth up before terminating the test") - startVCServiceWait4VPs(ctx, vcAddress, vsanhealthServiceName, &isVsanHealthServiceStopped) - } - - if isSPSServiceStopped { - vCenterHostname := strings.Split(multiVCe2eVSphere.multivcConfig.Global.VCenterHostname, ",") - vcAddress := vCenterHostname[0] + ":" + sshdPort - framework.Logf("Bringing sps up before terminating the test") - startVCServiceWait4VPs(ctx, vcAddress, spsServiceName, &isSPSServiceStopped) - isSPSServiceStopped = false - } - - if isStorageProfileDeleted { - clientIndex := 0 - err = createStorageProfile(masterIp, sshClientConfig, storagePolicyToDelete, clientIndex) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }) - - /* TESTCASE-1 - - Stateful set with SC contains default parameters and WFC binding mode and - specific nodeaffinity details in statefullset - - Steps: - 1. Create SC default values so all the AZ's should be consided for provisioning. - 2. Create Statefulset with parallel pod management policy - 3. Wait for PVC to reach bound state and POD to reach Running state - 4. Volumes should get distributed among the nodes which are mentioned in node affinity of - Statefullset yaml - 5. Make sure common validation points are met - a) Verify the PV node affinity details should have appropriate node details - b) The Pods should be running on the appropriate nodes - c) CNS metadata - 6. Scale-up /Scale-down the statefulset and verify the common validation points on newly - created statefullset - 7. Clean up the data - */ - - ginkgo.It("Workload creation on a multivc environment with sts specified with node affinity "+ - "and SC with no allowed topology", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - parallelPodPolicy = true - nodeAffinityToSet = true - stsReplicas = 3 - scaleUpReplicaCount = 5 - scaleDownReplicaCount = 2 - - if multiVCSetupType == "multi-2vc-setup" { - /* here in 2-VC setup statefulset node affinity is taken as k8s-zone -> zone-1,zone-2,zone-3 - i.e VC1 allowed topology and VC2 partial allowed topology - */ - allowedTopologyLen = 3 - topValStartIndex = 0 - topValEndIndex = 3 - } else if multiVCSetupType == "multi-3vc-setup" { - /* here in 3-VC setup, statefulset node affinity is taken as k8s-zone -> zone-3 - i.e only VC3 allowed topology - */ - allowedTopologyLen = 1 - topValStartIndex = 2 - topValEndIndex = 3 - } - - ginkgo.By("Set specific allowed topology") - allowedTopologies = setSpecificAllowedTopology(allowedTopologies, topkeyStartIndex, topValStartIndex, - topValEndIndex) - - ginkgo.By("Create StorageClass with no allowed topolgies specified and with WFC binding mode") - scSpec := getVSphereStorageClassSpec(defaultNginxStorageClassName, nil, nil, "", - bindingMode, false) - sc, err := client.StorageV1().StorageClasses().Create(ctx, scSpec, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err = client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Create StatefulSet and verify pv affinity and pod affinity details") - service, statefulset, err := createStafeulSetAndVerifyPVAndPodNodeAffinty(ctx, client, namespace, - parallelPodPolicy, stsReplicas, nodeAffinityToSet, allowedTopologies, allowedTopologyLen, - podAntiAffinityToSet, parallelStatefulSetCreation, false, "", "", nil, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - fss.DeleteAllStatefulSets(client, namespace) - deleteService(namespace, client, service) - }() - - ginkgo.By("Perform scaleup/scaledown operation on statefulsets and " + - "verify pv affinity and pod affinity") - err = performScalingOnStatefulSetAndVerifyPvNodeAffinity(ctx, client, scaleUpReplicaCount, - scaleDownReplicaCount, statefulset, parallelStatefulSetCreation, namespace, - allowedTopologies, stsScaleUp, stsScaleDown, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - }) - - /* TESTCASE-2 - - Deploy workload with allowed topology details in SC which should contain all the AZ's - so that workload will get distributed among all the VC's - - Steps: - 1. Create SC with allowedTopology details contains more than one Availability zone details - 2. Create statefulset with replica-5 - 3. Wait for PVC to reach bound state and POD to reach Running state - 4. Volumes should get distributed among all the Availability zones - 5. Make sure common validation points are met - a) Verify the PV node affinity details should have appropriate Node details - b) The Pods should be running on the appropriate nodes - c) CNS metadata - 6. Scale-up/Scale-down the statefulset and verify the common validation points on newly - created statefullset - 7. Clean up the data - */ - - ginkgo.It("Workload creation when all allowed topology specified in SC on a "+ - "multivc environment", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - /* here we are considering all the allowed topologies of VC1 and VC2 in case of 2-VC setup. - i.e. k8s-zone -> zone-1,zone-2,zone-3,zone-4,zone-5 - - And, all the allowed topologies of VC1, VC2 and VC3 are considered in case of 3-VC setup - i.e k8s-zone -> zone-1,zone-2,zone-3 - */ - - stsReplicas = 5 - scaleUpReplicaCount = 7 - scaleDownReplicaCount = 3 - - ginkgo.By("Create StorageClass with specific allowed topolgies details") - scSpec := getVSphereStorageClassSpec(defaultNginxStorageClassName, nil, allowedTopologies, "", - "", false) - sc, err := client.StorageV1().StorageClasses().Create(ctx, scSpec, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err = client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Create StatefulSet and verify pv affinity and pod affinity details") - service, statefulset, err := createStafeulSetAndVerifyPVAndPodNodeAffinty(ctx, client, namespace, parallelPodPolicy, - stsReplicas, nodeAffinityToSet, allowedTopologies, allowedTopologyLen, podAntiAffinityToSet, - parallelStatefulSetCreation, false, "", "", nil, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - fss.DeleteAllStatefulSets(client, namespace) - deleteService(namespace, client, service) - }() - - ginkgo.By("Perform scaleup/scaledown operation on statefulsets and " + - "verify pv affinity and pod affinity") - err = performScalingOnStatefulSetAndVerifyPvNodeAffinity(ctx, client, scaleUpReplicaCount, - scaleDownReplicaCount, statefulset, parallelStatefulSetCreation, namespace, - allowedTopologies, stsScaleUp, stsScaleDown, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - - /* - TESTCASE-3 - Deploy workload with Specific storage policy name available in Single VC - - Steps: - 1. Create a SC with storage policy name available in single VC - 2. Create statefulset with replica-5 - 3. Wait for PVC to reach bound state and POD to reach Running state - 4. Volumes should get created under appropriate nodes which is accessible to the storage Policy - 5. Make sure common verification Points met in PVC, PV ad POD - a) Verify the PV node affinity details should have appropriate Node details - b) The Pods should be running on the appropriate nodes - c) CNS metadata - 6. Scale-up/Scale-down the statefulset and verify the common validation points on newly created statefullset - 7. Clean up the data - */ - - ginkgo.It("Workload creation when specific storage policy of any single VC is given in SC", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - /* here we are considering storage policy of VC1 and the allowed topology is k8s-zone -> zone-1 - in case of 2-VC setup and 3-VC setup - */ - - stsReplicas = 5 - scParameters[scParamStoragePolicyName] = storagePolicyInVc1 - topValStartIndex = 0 - topValEndIndex = 1 - scaleUpReplicaCount = 9 - scaleDownReplicaCount = 2 - - ginkgo.By("Set specific allowed topology") - allowedTopologies = setSpecificAllowedTopology(allowedTopologies, topkeyStartIndex, topValStartIndex, - topValEndIndex) - - ginkgo.By("Create StorageClass with storage policy specified") - scSpec := getVSphereStorageClassSpec(defaultNginxStorageClassName, scParameters, nil, "", - "", false) - sc, err := client.StorageV1().StorageClasses().Create(ctx, scSpec, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err = client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Create StatefulSet and verify pv affinity and pod affinity details") - service, statefulset, err := createStafeulSetAndVerifyPVAndPodNodeAffinty(ctx, client, namespace, parallelPodPolicy, - stsReplicas, nodeAffinityToSet, allowedTopologies, allowedTopologyLen, podAntiAffinityToSet, - parallelStatefulSetCreation, false, "", "", nil, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - fss.DeleteAllStatefulSets(client, namespace) - deleteService(namespace, client, service) - }() - - ginkgo.By("Perform scaleup/scaledown operation on statefulsets and " + - "verify pv affinity and pod affinity") - err = performScalingOnStatefulSetAndVerifyPvNodeAffinity(ctx, client, scaleUpReplicaCount, - scaleDownReplicaCount, statefulset, parallelStatefulSetCreation, namespace, - allowedTopologies, stsScaleUp, stsScaleDown, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - - /* TESTCASE-4 - Same Policy is available in two VC's - - Steps: - 1. Create a SC with Storage policy name available in VC1 and VC2 - 2. Create two Statefulset with replica-3 - 3. Wait for PVC to reach bound state and POD to reach Running state - 4. Since both the VCs have the same storage policy, volume should get distributed among all the - availability zones - 5. Make sure common verification Points met in PVC, PV ad POD - a) Verify the PV node affinity details should have appropriate Node details - b) The POD's should be running on the appropriate nodes - c) CNS metadata - 6. Scale-up/scale-down the statefulset and verify the common validation points on newly - created statefullset - 7. Clean up the data - */ - - ginkgo.It("Workload creation when storage policy available in multivc setup is given in SC", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - /* In case of 2-VC setup, we are considering storage policy of VC1 and VC2 and the allowed - topology is k8s-zone -> zone-1, zone-2 i.e VC1 allowed topology and partial allowed topology - of VC2 - - In case of 3-VC setup, we are considering storage policy of VC1 and VC2 and the allowed - topology is k8s-zone -> zone-1, zone-2 i.e VC1 and VC2 allowed topologies. - */ - - stsReplicas = 3 - scParameters[scParamStoragePolicyName] = storagePolicyInVc1Vc2 - topValStartIndex = 0 - topValEndIndex = 2 - sts_count := 2 - parallelStatefulSetCreation = true - scaleUpReplicaCount = 7 - scaleDownReplicaCount = 2 - - ginkgo.By("Set specific allowed topology") - allowedTopologies = setSpecificAllowedTopology(allowedTopologies, topkeyStartIndex, topValStartIndex, - topValEndIndex) - - ginkgo.By("Create StorageClass with storage policy specified") - scSpec := getVSphereStorageClassSpec(defaultNginxStorageClassName, scParameters, nil, "", - "", false) - sc, err := client.StorageV1().StorageClasses().Create(ctx, scSpec, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err = client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Create service") - service := CreateService(namespace, client) - defer func() { - deleteService(namespace, client, service) - }() - - ginkgo.By("Create 2 StatefulSet with replica count 5") - statefulSets := createParallelStatefulSetSpec(namespace, sts_count, stsReplicas) - var wg sync.WaitGroup - wg.Add(sts_count) - for i := 0; i < len(statefulSets); i++ { - go createParallelStatefulSets(client, namespace, statefulSets[i], - stsReplicas, &wg) - - } - wg.Wait() - - ginkgo.By("Waiting for StatefulSets Pods to be in Ready State") - err = waitForStsPodsToBeInReadyRunningState(ctx, client, namespace, statefulSets) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - deleteAllStsAndPodsPVCsInNamespace(ctx, client, namespace) - }() - - ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") - for i := 0; i < len(statefulSets); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulSets[i], - namespace, allowedTopologies, parallelStatefulSetCreation, true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - ginkgo.By("Perform scaleup/scaledown operation on statefulset and " + - "verify pv and pod affinity details") - err = performScalingOnStatefulSetAndVerifyPvNodeAffinity(ctx, client, scaleUpReplicaCount, - scaleDownReplicaCount, statefulSets[0], parallelStatefulSetCreation, namespace, - allowedTopologies, stsScaleUp, stsScaleDown, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - - /* TESTCASE-5 - Pod affiity tests - - Steps: - 1. Create SC default values so all the AZ's should be considered for provisioning. - 2. Create statefulset with Pod affinity rules such a way that each AZ should get atleast 1 statefulset - 3. Wait for PVC to bound and POD to reach running state - 4. Verify the stateful set distribution - 5. Make sure common verification Points met in PVC, PV ad POD - a) Verify the PV node affinity details should have appropriate Node details - b) The Pods should be running on the appropriate nodes - c) CNS metadata - 6. Scale-up/Scale-down the statefulset and verify the common validation points on newly created - statefullset - 7. Clean up data - */ - - ginkgo.It("Workload creation on a multivc environment with sts specified with pod affinity "+ - "and SC with no allowed topology", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - stsReplicas = 5 - podAntiAffinityToSet = true - scaleUpReplicaCount = 8 - scaleDownReplicaCount = 1 - - ginkgo.By("Create StorageClass with storage policy specified") - scSpec := getVSphereStorageClassSpec(defaultNginxStorageClassName, nil, nil, "", - "", false) - sc, err := client.StorageV1().StorageClasses().Create(ctx, scSpec, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err = client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Create StatefulSet and verify pv affinity and pod affinity details") - service, statefulset, err := createStafeulSetAndVerifyPVAndPodNodeAffinty(ctx, client, namespace, parallelPodPolicy, - stsReplicas, nodeAffinityToSet, allowedTopologies, allowedTopologyLen, podAntiAffinityToSet, - parallelStatefulSetCreation, false, "", "", nil, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - fss.DeleteAllStatefulSets(client, namespace) - deleteService(namespace, client, service) - }() - - ginkgo.By("Perform scaleup/scaledown operation on statefulsets and " + - "verify pv affinity and pod affinity") - err = performScalingOnStatefulSetAndVerifyPvNodeAffinity(ctx, client, scaleUpReplicaCount, - scaleDownReplicaCount, statefulset, parallelStatefulSetCreation, namespace, - allowedTopologies, stsScaleUp, stsScaleDown, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - - /* TESTCASE-6 - Deploy workload with allowed topology and Datastore URL - - Steps: - 1. Create a SC with allowed topology and appropriate datastore url - 2. Create Statefulset with replica-5 - 3. Wait for PVC to reach bound state and POD to reach Running state - 4. Volumes should get created under appropriate availability zone and on the specified datastore - 5. Make sure common validation points are met - 6. Verify the PV node affinity details should have appropriate node details - 7. The Pods should be running on the appropriate nodes - 8. Scale-up/Scale-down the statefulset - 9. Verify the node affinity details also verify the Pod details - 10. Clean up the data - */ - - ginkgo.It("Deploy workload with allowed topology and datastore url on a multivc environment", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - /* here, we are considering datastore url of VC1 in case of both 2-VC and 3-VC multi setup - so the allowed topology will be considered as - k8s-zone -> zone-1 - */ - - stsReplicas = 5 - scParameters[scParamDatastoreURL] = datastoreURLVC1 - topValStartIndex = 0 - topValEndIndex = 1 - scaleDownReplicaCount = 3 - scaleUpReplicaCount = 6 - - ginkgo.By("Set specific allowed topology") - allowedTopologies = setSpecificAllowedTopology(allowedTopologies, topkeyStartIndex, topValStartIndex, - topValEndIndex) - - ginkgo.By("Create StorageClass with allowed topology, storage-policy and with datastore url") - scSpec := getVSphereStorageClassSpec(defaultNginxStorageClassName, scParameters, allowedTopologies, "", - bindingMode, false) - sc, err := client.StorageV1().StorageClasses().Create(ctx, scSpec, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err = client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Create StatefulSet and verify pv affinity and pod affinity details") - service, statefulset, err := createStafeulSetAndVerifyPVAndPodNodeAffinty(ctx, client, namespace, parallelPodPolicy, - stsReplicas, nodeAffinityToSet, allowedTopologies, allowedTopologyLen, podAntiAffinityToSet, - parallelStatefulSetCreation, false, "", "", nil, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - fss.DeleteAllStatefulSets(client, namespace) - deleteService(namespace, client, service) - }() - - ginkgo.By("Perform scaleup/scaledown operation on statefulsets and " + - "verify pv affinity and pod affinity") - err = performScalingOnStatefulSetAndVerifyPvNodeAffinity(ctx, client, scaleUpReplicaCount, - scaleDownReplicaCount, statefulset, parallelStatefulSetCreation, namespace, - allowedTopologies, stsScaleUp, stsScaleDown, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - - /* TESTCASE-7 - Deploy workload with allowed topology details in SC specific to VC1 with Immediate Binding - - Steps: - 1. Create SC with allowedTopology details set to VC1 availability zone - 2. Create statefulset with replica-3 - 3. Wait for PVC to reach bound state and POD to reach Running state - 4. Make sure common validation points are met - a) Verify the PV node affinity details should have appropriate Node details - b) The POD's should be running on the appropriate nodes which are present in VC1 - 5. Scale-up/scale-down the statefulset - 6. Verify the node affinity details. Verify the POD details. All the pods should come up on the - nodes of VC1 - 7. Clean up the data - */ - - ginkgo.It("Deploy workload with allowed topology details in SC specific "+ - "to VC1 with Immediate Binding", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - /* here, we are considering allowed topology of VC1 in case of both 2-VC and 3-VC multi setup - so the allowed topology will be considered as - k8s-zone -> zone-1 - */ - - stsReplicas = 3 - topValStartIndex = 0 - topValEndIndex = 1 - scaleDownReplicaCount = 0 - scaleUpReplicaCount = 6 - - ginkgo.By("Set specific allowed topology") - allowedTopologies = setSpecificAllowedTopology(allowedTopologies, topkeyStartIndex, topValStartIndex, - topValEndIndex) - - ginkgo.By("Create StorageClass with allowed topology details of VC1") - scSpec := getVSphereStorageClassSpec(defaultNginxStorageClassName, nil, allowedTopologies, "", - "", false) - sc, err := client.StorageV1().StorageClasses().Create(ctx, scSpec, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err = client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Create StatefulSet and verify pv affinity and pod affinity details") - service, statefulset, err := createStafeulSetAndVerifyPVAndPodNodeAffinty(ctx, client, namespace, parallelPodPolicy, - stsReplicas, nodeAffinityToSet, allowedTopologies, allowedTopologyLen, podAntiAffinityToSet, - parallelStatefulSetCreation, false, "", "", nil, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - fss.DeleteAllStatefulSets(client, namespace) - deleteService(namespace, client, service) - }() - - ginkgo.By("Perform scaleup/scaledown operation on statefulsets and " + - "verify pv affinity and pod affinity") - err = performScalingOnStatefulSetAndVerifyPvNodeAffinity(ctx, client, scaleUpReplicaCount, - scaleDownReplicaCount, statefulset, parallelStatefulSetCreation, namespace, - allowedTopologies, stsScaleUp, stsScaleDown, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - - /* TESTCASE-8 - Deploy workload with allowed topology details in SC specific to VC2 + WFC binding mode + - default pod management policy - - Steps: - 1. Create SC with allowedTopology details set to VC2's Availability Zone - 2. Create statefulset with replica-3 - 3. Wait for PVC to reach bound state and Pod to reach running state - 4. Make sure common validation points are met - a) Verify the PV node affinity details should have appropriate node details - b) The Pods should be running on the appropriate nodes which are present in VC2 - 5. Scale-up/scale-down the statefulset - 6. Verify the node affinity details. Verify the pod details. All the pods should come up on the nodes of - VC2 - 7. Validate CNS metadata on appropriate VC - 8. Clean up the data - */ - - ginkgo.It("Deploy workload with allowed topology details in SC specific to VC2 with WFC "+ - "binding mode and with default pod management policy", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - stsReplicas = 3 - scaleDownReplicaCount = 2 - scaleUpReplicaCount = 5 - - if multiVCSetupType == "multi-2vc-setup" { - // here, we are considering partial allowed topology of VC2 i.e k8s-zone -> zone-3,zone-4 - topValStartIndex = 2 - topValEndIndex = 4 - } else if multiVCSetupType == "multi-3vc-setup" { - // here, we are considering allowed topology of VC2 i.e k8s-zone -> zone-2 - topValStartIndex = 1 - topValEndIndex = 2 - } - - ginkgo.By("Set specific allowed topology") - allowedTopologies = setSpecificAllowedTopology(allowedTopologies, topkeyStartIndex, topValStartIndex, - topValEndIndex) - - ginkgo.By("Create StorageClass with allowed topology details of VC2") - scSpec := getVSphereStorageClassSpec(defaultNginxStorageClassName, nil, allowedTopologies, "", - bindingMode, false) - sc, err := client.StorageV1().StorageClasses().Create(ctx, scSpec, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err = client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Create StatefulSet and verify pv affinity and pod affinity details") - service, statefulset, err := createStafeulSetAndVerifyPVAndPodNodeAffinty(ctx, client, namespace, parallelPodPolicy, - stsReplicas, nodeAffinityToSet, allowedTopologies, allowedTopologyLen, podAntiAffinityToSet, - parallelStatefulSetCreation, false, "", "", nil, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - fss.DeleteAllStatefulSets(client, namespace) - deleteService(namespace, client, service) - }() - - ginkgo.By("Perform scaleup/scaledown operation on statefulsets and " + - "verify pv affinity and pod affinity") - err = performScalingOnStatefulSetAndVerifyPvNodeAffinity(ctx, client, scaleUpReplicaCount, - scaleDownReplicaCount, statefulset, parallelStatefulSetCreation, namespace, - allowedTopologies, stsScaleUp, stsScaleDown, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - - /* TESTCASE-9 - Deploy workload with allowed topology details in SC specific to VC3 + parallel pod management policy - - Steps: - 1. Create SC with allowedTopology details set to VC3 availability zone - 2. Create statefulset with replica-3 - 3. Wait for PVC to reach bound state and POD to reach Running state - 4. Make sure common validation points are met - a) Verify the PV node affinity details should have appropriate Node details - b) The Pod should be running on the appropriate nodes which are present in VC3 - 5. Scale-up /scale-down the statefulset - 6. Verify the node affinity details and also verify the pod details. All the pods should come up on the nodes of - VC3 - 7. Clean up the data - */ - - ginkgo.It("Deploy workload with allowed topology details in SC specific to VC3 with "+ - "parallel pod management policy", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - stsReplicas = 3 - parallelPodPolicy = true - scaleDownReplicaCount = 1 - scaleUpReplicaCount = 6 - - if multiVCSetupType == "multi-2vc-setup" { - /* here, For 2-VC setup we will consider all allowed topology of VC1 and VC2 - i.e. k8s-zone -> zone-1,zone-2,zone-3,zone-4 */ - topValStartIndex = 0 - topValEndIndex = 4 - } else if multiVCSetupType == "multi-3vc-setup" { - /* here, For 3-VC setup we will consider allowed topology of VC3 - i.e. k8s-zone ->zone-3 */ - topValStartIndex = 2 - topValEndIndex = 3 - } - - ginkgo.By("Set specific allowed topology") - allowedTopologies = setSpecificAllowedTopology(allowedTopologies, topkeyStartIndex, topValStartIndex, - topValEndIndex) - - ginkgo.By("Create StorageClass with allowed topology details of VC3") - scSpec := getVSphereStorageClassSpec(defaultNginxStorageClassName, nil, allowedTopologies, "", - bindingMode, false) - sc, err := client.StorageV1().StorageClasses().Create(ctx, scSpec, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err = client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Create StatefulSet and verify pv affinity and pod affinity details") - service, statefulset, err := createStafeulSetAndVerifyPVAndPodNodeAffinty(ctx, client, namespace, parallelPodPolicy, - stsReplicas, nodeAffinityToSet, allowedTopologies, allowedTopologyLen, podAntiAffinityToSet, - parallelStatefulSetCreation, false, "", "", nil, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - fss.DeleteAllStatefulSets(client, namespace) - deleteService(namespace, client, service) - }() - - ginkgo.By("Perform scaleup/scaledown operation on statefulsets and " + - "verify pv affinity and pod affinity") - err = performScalingOnStatefulSetAndVerifyPvNodeAffinity(ctx, client, scaleUpReplicaCount, - scaleDownReplicaCount, statefulset, parallelStatefulSetCreation, namespace, - allowedTopologies, stsScaleUp, stsScaleDown, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - - /* - TESTCASE-10 - Deploy workload with default SC parameters with WaitForFirstConsumer - - Steps: - 1. Create a storage class with default parameters - a) SC1 with WFC - b) SC2 with Immediate - 2. Create statefulset with replica-5 using SC1 - 3. Cretate few dynamic PVCs using SC2 and create Pods using the same PVCs - 4. Wait for PVC to reach bound state and POD to reach Running state - 5. Make sure common validation points are met - a) Volumes should get distributed among all the availability zones - b) Verify the PV node affinity details should have appropriate Node details - c) The Pods should be running on the appropriate nodes - 6. Scale-up/scale-down the statefulset - 7. Clean up the data - */ - - ginkgo.It("Deploy workload with default SC parameters with WaitForFirstConsumer", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - stsReplicas = 5 - pvcCount := 5 - var podList []*v1.Pod - scaleDownReplicaCount = 3 - scaleUpReplicaCount = 7 - parallelPodPolicy = true - parallelStatefulSetCreation = true - - /* here, For 2-VC setup, we are considering all the allowed topologies of VC1 and VC2 - i.e. k8s-zone -> zone-1,zone-2,zone-3,zone-4,zone-5 - - For 3-VC setup, we are considering all the allowed topologies of VC1, VC2 and VC3 - i.e. k8s-zone -> zone-1,zone-2,zone-3 - */ - - ginkgo.By("Create StorageClass with default parameters using WFC binding mode") - scSpec := getVSphereStorageClassSpec(defaultNginxStorageClassName, nil, nil, "", - bindingMode, false) - sc, err := client.StorageV1().StorageClasses().Create(ctx, scSpec, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err = client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Create StatefulSet and verify pv affinity and pod affinity details") - service, statefulset, err := createStafeulSetAndVerifyPVAndPodNodeAffinty(ctx, client, namespace, - parallelPodPolicy, stsReplicas, nodeAffinityToSet, allowedTopologies, allowedTopologyLen, - podAntiAffinityToSet, parallelStatefulSetCreation, false, "", "", nil, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - deleteAllStsAndPodsPVCsInNamespace(ctx, client, namespace) - deleteService(namespace, client, service) - }() - - ginkgo.By("Create StorageClass with default parameters using Immediate binding mode") - storageclass, err := createStorageClass(client, nil, nil, "", "", false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - ginkgo.By("Delete Storage Class") - err = client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Trigger multiple PVCs") - pvclaimsList := createMultiplePVCsInParallel(ctx, client, namespace, storageclass, pvcCount, nil) - - ginkgo.By("Verify PVC claim to be in bound phase and create POD for each PVC") - for i := 0; i < len(pvclaimsList); i++ { - var pvclaims []*v1.PersistentVolumeClaim - pvc, err := fpv.WaitForPVClaimBoundPhase(client, - []*v1.PersistentVolumeClaim{pvclaimsList[i]}, framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(pvc).NotTo(gomega.BeEmpty()) - - pv := getPvFromClaim(client, pvclaimsList[i].Namespace, pvclaimsList[i].Name) - - ginkgo.By("Creating Pod") - pvclaims = append(pvclaims, pvclaimsList[i]) - pod, err := createPod(client, namespace, nil, pvclaims, false, "") - podList = append(podList, pod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By(fmt.Sprintf("Verify volume: %s is attached to the node: %s", - pv.Spec.CSI.VolumeHandle, pod.Spec.NodeName)) - vmUUID := getNodeUUID(ctx, client, pod.Spec.NodeName) - isDiskAttached, err := multiVCe2eVSphere.verifyVolumeIsAttachedToVMInMultiVC(client, - pv.Spec.CSI.VolumeHandle, vmUUID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(isDiskAttached).To(gomega.BeTrue(), "Volume is not attached") - } - defer func() { - ginkgo.By("Deleting PVC's and PV's") - for i := 0; i < len(pvclaimsList); i++ { - pv := getPvFromClaim(client, pvclaimsList[i].Namespace, pvclaimsList[i].Name) - err = fpv.DeletePersistentVolumeClaim(client, pvclaimsList[i].Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - framework.ExpectNoError(fpv.WaitForPersistentVolumeDeleted(client, pv.Name, poll, pollTimeoutShort)) - err = multiVCe2eVSphere.waitForCNSVolumeToBeDeletedInMultiVC(pv.Spec.CSI.VolumeHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }() - defer func() { - for i := 0; i < len(podList); i++ { - ginkgo.By(fmt.Sprintf("Deleting the pod %s in namespace %s", podList[i].Name, namespace)) - err = fpod.DeletePodWithWait(client, podList[i]) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - ginkgo.By("Verify volume is detached from the node") - for i := 0; i < len(pvclaimsList); i++ { - pv := getPvFromClaim(client, pvclaimsList[i].Namespace, pvclaimsList[i].Name) - isDiskDetached, err := multiVCe2eVSphere.waitForVolumeDetachedFromNodeInMultiVC(client, - pv.Spec.CSI.VolumeHandle, podList[i].Spec.NodeName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(isDiskDetached).To(gomega.BeTrue(), - fmt.Sprintf("Volume %q is not detached from the node", pv.Spec.CSI.VolumeHandle)) - } - }() - - ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") - for i := 0; i < len(podList); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStandalonePodLevel5(ctx, client, podList[i], - namespace, allowedTopologies, true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - ginkgo.By("Perform scaleup/scaledown operation on statefulsets and " + - "verify pv affinity and pod affinity") - err = performScalingOnStatefulSetAndVerifyPvNodeAffinity(ctx, client, scaleUpReplicaCount, - scaleDownReplicaCount, statefulset, parallelStatefulSetCreation, namespace, - allowedTopologies, stsScaleUp, stsScaleDown, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - - /* TESTCASE-11 - Create SC with single AllowedTopologyLabel - - Steps: - 1. Create a SC with specific topology details which is available in any one VC - 2. Create statefulset with replica-5 - 3. Wait for PVC to reach Bound state and Pod to reach Running state - 4. Make sure common validation points are met - a) Volumes should get created under appropriate zone - b) Verify the PV node affinity details should have appropriate node details - c) The Pods should be running on the appropriate nodes - 5. Scale-up/scale-down the statefulset - 6. Verify the node affinity details and also verify the pod details - 7. Clean up the data - */ - - ginkgo.It("Create SC with single allowed topology label on a multivc environment", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - /* Considering partial allowed topology of VC2 in case of 2-VC setup i.e. k8s-zone -> zone-2 - And, allowed topology of VC2 in case of 3-VC setup i.e. k8s-zone -> zone-2 - */ - - stsReplicas = 5 - topValStartIndex = 1 - topValEndIndex = 2 - parallelPodPolicy = true - scaleDownReplicaCount = 2 - scaleUpReplicaCount = 5 - - ginkgo.By("Set specific allowed topology") - allowedTopologies = setSpecificAllowedTopology(allowedTopologies, topkeyStartIndex, topValStartIndex, - topValEndIndex) - - ginkgo.By("Create StorageClass with allowed topology details of VC2") - scSpec := getVSphereStorageClassSpec(defaultNginxStorageClassName, nil, allowedTopologies, "", - bindingMode, false) - sc, err := client.StorageV1().StorageClasses().Create(ctx, scSpec, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err = client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Create StatefulSet and verify pv affinity and pod affinity details") - service, statefulset, err := createStafeulSetAndVerifyPVAndPodNodeAffinty(ctx, client, namespace, parallelPodPolicy, - stsReplicas, nodeAffinityToSet, allowedTopologies, allowedTopologyLen, podAntiAffinityToSet, - parallelStatefulSetCreation, false, "", "", nil, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - fss.DeleteAllStatefulSets(client, namespace) - deleteService(namespace, client, service) - }() - - ginkgo.By("Perform scaleup/scaledown operation on statefulsets and " + - "verify pv affinity and pod affinity") - err = performScalingOnStatefulSetAndVerifyPvNodeAffinity(ctx, client, scaleUpReplicaCount, - scaleDownReplicaCount, statefulset, parallelStatefulSetCreation, namespace, - allowedTopologies, stsScaleUp, stsScaleDown, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - - /* TESTCASE-12 - Create PVC using the wrong StoragePolicy name. Consider partially matching storage policy - - Steps: - 1. Use the partially matching storage policy and create PVC - 2. PVC should not go to bound, appropriate error should be shown - 3. Perform cleanup - */ - - ginkgo.It("PVC creation failure when wrong storage policy name is specified in SC", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - storagePolicyName := "shared-ds-polic" - scParameters[scParamStoragePolicyName] = storagePolicyName - - storageclass, pvclaim, err := createPVCAndStorageClass(client, - namespace, nil, scParameters, "", nil, "", false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err = client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Expect claim to fail as invalid storage policy is specified in Storage Class") - framework.ExpectError(fpv.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, - client, pvclaim.Namespace, pvclaim.Name, framework.Poll, framework.ClaimProvisionTimeout)) - expectedErrMsg := "failed to create volume" - err = waitForEvent(ctx, client, namespace, expectedErrMsg, pvclaim.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), fmt.Sprintf("Expected error : %q", expectedErrMsg)) - }) - - /* - TESTCASE-13 - Deploy workload With allowed topology of VC1 and datastore url which is in VC2 - - Steps: - 1. Create SC with allowed topology which matches VC1 details and datastore url which is in VC2 - 2. PVC should not go to bound, appropriate error should be shown - */ - - ginkgo.It("Deploy workload with allowed topology of VC1 and datastore url of VC2", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - topValStartIndex = 0 - topValEndIndex = 1 - scParameters[scParamDatastoreURL] = datastoreURLVC2 - - ginkgo.By("Set specific allowed topology") - allowedTopologies = setSpecificAllowedTopology(allowedTopologies, topkeyStartIndex, topValStartIndex, - topValEndIndex) - - storageclass, pvclaim, err := createPVCAndStorageClass(client, - namespace, nil, scParameters, "", allowedTopologies, "", false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err = client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Expect claim to fail as non-compatible datastore url is specified in Storage Class") - framework.ExpectError(fpv.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, - client, pvclaim.Namespace, pvclaim.Name, framework.Poll, framework.ClaimProvisionTimeout)) - expectedErrMsg := "failed to create volume" - err = waitForEvent(ctx, client, namespace, expectedErrMsg, pvclaim.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), fmt.Sprintf("Expected error : %q", expectedErrMsg)) - }) - - /* - TESTCASE-14 - Create storage policy in VC1 and VC2 and create storage class with the same and delete Storage policy - in VC1, expected to go to VC2 - - Steps: - 1. Create Storage policy in VC1 and VC2 - 2. Create Storage class with above policy - 3. Delete storage policy from VC1 - 4. Create statefulSet - 5. Expected to provision volume on VC2 - 6. Make sure common validation points are met - 7. Clear data - */ - - ginkgo.It("Create storage policy in multivc and later delete storage policy from one of the VC", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - topValStartIndex = 1 - topValEndIndex = 2 - stsReplicas = 3 - clientIndex := 0 - - scParameters[scParamStoragePolicyName] = storagePolicyToDelete - - /* Considering partial allowed topology of VC2 in case of 2-VC setup i.e. k8s-zone -> zone-2 - And, allowed topology of VC2 in case of 3-VC setup i.e. k8s-zone -> zone-2 - */ - - ginkgo.By("Set specific allowed topology") - allowedTopologies = setSpecificAllowedTopology(allowedTopologies, topkeyStartIndex, - topValStartIndex, topValEndIndex) - - ginkgo.By("Create StorageClass") - scSpec := getVSphereStorageClassSpec(defaultNginxStorageClassName, scParameters, nil, "", - "", false) - sc, err := client.StorageV1().StorageClasses().Create(ctx, scSpec, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err = client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Delete Storage Policy created in VC1") - err = deleteStorageProfile(masterIp, sshClientConfig, storagePolicyToDelete, clientIndex) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - isStorageProfileDeleted = true - defer func() { - if isStorageProfileDeleted { - err = createStorageProfile(masterIp, sshClientConfig, storagePolicyToDelete, clientIndex) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - isStorageProfileDeleted = false - } - }() - - ginkgo.By("Create StatefulSet and verify pv affinity and pod affinity details") - service, _, err := createStafeulSetAndVerifyPVAndPodNodeAffinty(ctx, client, namespace, parallelPodPolicy, - stsReplicas, nodeAffinityToSet, allowedTopologies, allowedTopologyLen, podAntiAffinityToSet, - parallelStatefulSetCreation, false, "", "", nil, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - fss.DeleteAllStatefulSets(client, namespace) - deleteService(namespace, client, service) - }() - - }) - - /* - TESTCASE-16 - Create Deployment pod using SC with allowed topology set to Specific VC - - Steps: - 1. Create SC with allowedTopology details set to any one VC with Availability Zone - 2. Create PVC using above SC - 3. Wait for PVC to reach bound state - 4. Create deployment and wait for POD to reach Running state - 5. Make sure common validation points are met on PV,PVC and POD - a) Verify the PV node affinity details should have appropriate Node details - b) The POD's should be running on the appropriate nodes which are present in VC1 - c) Verify the node affinity details also verify the POD details. - All the POd's should come up on the nodes of VC - 6. Clean up the data - */ - - ginkgo.It("Create Deployment pod using SC with allowed topology set to specific VC", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - topValStartIndex = 2 - topValEndIndex = 3 - var lables = make(map[string]string) - lables["app"] = "nginx" - replica := 1 - - /* here considering partial allowed topology of VC2 in case of 2VC setup i.e k8s-zone -> zone-3 - here considering allowed topology of VC3 in case of 3VC setup i.e k8s-zone -> zone-3 - */ - - ginkgo.By("Set specific allowed topology") - allowedTopologies = setSpecificAllowedTopology(allowedTopologies, topkeyStartIndex, topValStartIndex, - topValEndIndex) - - ginkgo.By("Create StorageClass and PVC for Deployment") - sc, pvclaim, err := createPVCAndStorageClass(client, namespace, nil, - nil, diskSize, allowedTopologies, "", false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - // Wait for PVC to be in Bound phase - pvclaims = append(pvclaims, pvclaim) - persistentvolumes, err := fpv.WaitForPVClaimBoundPhase(client, pvclaims, - framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - volHandle := persistentvolumes[0].Spec.CSI.VolumeHandle - gomega.Expect(volHandle).NotTo(gomega.BeEmpty()) - defer func() { - err := client.StorageV1().StorageClasses().Delete(ctx, sc.Name, - *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - defer func() { - err := fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = multiVCe2eVSphere.waitForCNSVolumeToBeDeletedInMultiVC(volHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pvclaim = nil - }() - - ginkgo.By("Create Deployments") - deployment, err := createDeployment(ctx, client, int32(replica), lables, - nil, namespace, pvclaims, "", false, nginxImage) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - framework.Logf("Delete deployment set") - err := client.AppsV1().Deployments(namespace).Delete(ctx, deployment.Name, - *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Verify PV node affinity and that the PODS are running " + - "on appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForDeploymentSetsLevel5(ctx, client, deployment, - namespace, allowedTopologies, false, true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - - /* - TESTCASE-17 - Create SC with invalid allowed topology details - NEGETIVE - - Steps: - 1. Create SC with invalid label details - 2. Create PVC, Pvc should not reach bound state. It should throuw error - */ - - ginkgo.It("Create SC with invalid allowed topology details", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - topValStartIndex = 1 - topValEndIndex = 1 - - ginkgo.By("Set invalid allowed topology for SC") - allowedTopologies = setSpecificAllowedTopology(allowedTopologies, topkeyStartIndex, topValStartIndex, - topValEndIndex) - allowedTopologies[0].Values = []string{"new-zone"} - - storageclass, err := createStorageClass(client, nil, allowedTopologies, "", "", false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, - *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - pvc, err := createPVC(client, namespace, nil, "", storageclass, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - if pvc != nil { - ginkgo.By("Delete the PVC") - err = fpv.DeletePersistentVolumeClaim(client, pvc.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }() - - ginkgo.By("Expect claim to fail as invalid topology label is specified in Storage Class") - framework.ExpectError(fpv.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, - client, pvc.Namespace, pvc.Name, framework.Poll, framework.ClaimProvisionTimeout)) - expectedErrMsg := "failed to fetch vCenter associated with topology segments" - err = waitForEvent(ctx, client, namespace, expectedErrMsg, pvc.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), fmt.Sprintf("Expected error : %q", expectedErrMsg)) - }) - - /* - TESTCASE-18 - Verify online and offline Volume expansion - - Steps: - 1. Create SC default values, so all the AZ's should be considered for provisioning. - 2. Create PVC and wait for it to bound - 3. Edit PVC and trigger offline volume expansion - 4. Create POD , Wait for POD to reach running state - 5. Describe PVC and verify that new size should be updated on PVC - 6. Edit the same PVC again to test Online volume expansion - 7. Wait for resize to complete - 8. Verify the newly updated size on PV and PVC - 9. Clean up the data - */ - - ginkgo.It("Offline and online volume expansion on a multivc setup", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - var pvclaims []*v1.PersistentVolumeClaim - - ginkgo.By("Create StorageClass") - storageclass, pvclaim, err := createPVCAndStorageClass(client, namespace, nil, nil, "", - nil, "", true, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Expect claim to provision volume successfully") - pvclaims = append(pvclaims, pvclaim) - pv, err := fpv.WaitForPVClaimBoundPhase(client, pvclaims, framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - volHandle := pv[0].Spec.CSI.VolumeHandle - gomega.Expect(volHandle).NotTo(gomega.BeEmpty()) - - defer func() { - err := fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = multiVCe2eVSphere.waitForCNSVolumeToBeDeletedInMultiVC(volHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Perform offline volume expansion on PVC") - err = performOfflineVolumeExpansin(client, pvclaim, volHandle, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Creating pod to attach PV to the node") - pod, err := createPod(client, namespace, nil, []*v1.PersistentVolumeClaim{pvclaim}, false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - var vmUUID string - nodeName := pod.Spec.NodeName - - if vanillaCluster { - vmUUID = getNodeUUID(ctx, client, pod.Spec.NodeName) - } - - ginkgo.By(fmt.Sprintf("Verify volume: %s is attached to the node: %s", volHandle, nodeName)) - isDiskAttached, err := multiVCe2eVSphere.verifyVolumeIsAttachedToVMInMultiVC(client, volHandle, vmUUID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(isDiskAttached).To(gomega.BeTrue(), "Volume is not attached to the node") - defer func() { - ginkgo.By(fmt.Sprintf("Deleting the pod %s in namespace %s", pod.Name, namespace)) - err = fpod.DeletePodWithWait(client, pod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Verify volume is detached from the node") - isDiskDetached, err := multiVCe2eVSphere.waitForVolumeDetachedFromNodeInMultiVC(client, - pv[0].Spec.CSI.VolumeHandle, pod.Spec.NodeName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(isDiskDetached).To(gomega.BeTrue(), - fmt.Sprintf("Volume %q is not detached from the node", pv[0].Spec.CSI.VolumeHandle)) - }() - - ginkgo.By("Perform online volume expansion on PVC") - err = performOnlineVolumeExpansin(f, client, pvclaim, namespace, pod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Verify PV node affinity and that the PODS are running " + - "on appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStandalonePodLevel5(ctx, client, pod, - namespace, allowedTopologies, true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - - /* - TESTCASE-19 - Create a workload and try to reboot one of the VC - NEGETIVE - - Steps: - 1. Create SC default values, so all the AZ's should be considered for provisioning. - 2. Reboot any one VC1 - 3. Create 3 statefulset each with 5 replica - 4. Since one VC1 is rebooting, volume should get provisioned on another VC till the VC1 Comes up - 5. Wait for the statefulset, PVC's and POD's should be in up and running state - 6. After the VC came to running state scale up/down the statefull sets - 7. Newly created statefull set should get distributed among both the VC's - 8. Make sure common validation points are met on PV,PVC and POD - 9. Perform Cleanup - */ - - ginkgo.It("Create workload and reboot one of the VC", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - stsReplicas = 5 - sts_count := 3 - parallelStatefulSetCreation = true - scaleUpReplicaCount = 9 - scaleDownReplicaCount = 1 - - ginkgo.By("Create StorageClass") - scSpec := getVSphereStorageClassSpec(defaultNginxStorageClassName, nil, nil, "", - "", false) - sc, err := client.StorageV1().StorageClasses().Create(ctx, scSpec, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err = client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Create service") - service := CreateService(namespace, client) - defer func() { - deleteService(namespace, client, service) - }() - - ginkgo.By("Rebooting VC1") - vCenterHostname := strings.Split(multiVCe2eVSphere.multivcConfig.Global.VCenterHostname, ",") - vcAddress := vCenterHostname[0] + ":" + sshdPort - framework.Logf("vcAddress - %s ", vcAddress) - err = invokeVCenterReboot(vcAddress) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = waitForHostToBeUp(vCenterHostname[0]) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By("Done with reboot") - - ginkgo.By("Create 3 StatefulSet with replica count 5") - statefulSets := createParallelStatefulSetSpec(namespace, sts_count, stsReplicas) - var wg sync.WaitGroup - wg.Add(sts_count) - for i := 0; i < len(statefulSets); i++ { - go createParallelStatefulSets(client, namespace, statefulSets[i], - stsReplicas, &wg) - - } - wg.Wait() - - essentialServices := []string{spsServiceName, vsanhealthServiceName, vpxdServiceName} - checkVcenterServicesRunning(ctx, vcAddress, essentialServices) - - //After reboot - multiVCbootstrap() - - ginkgo.By("Waiting for StatefulSets Pods to be in Ready State") - err = waitForStsPodsToBeInReadyRunningState(ctx, client, namespace, statefulSets) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - deleteAllStsAndPodsPVCsInNamespace(ctx, client, namespace) - }() - - ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") - for i := 0; i < len(statefulSets); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulSets[i], - namespace, allowedTopologies, parallelStatefulSetCreation, true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - ginkgo.By("Perform scaleup/scaledown operation on statefulsets and " + - "verify pv affinity and pod affinity") - err = performScalingOnStatefulSetAndVerifyPvNodeAffinity(ctx, client, scaleUpReplicaCount, - scaleDownReplicaCount, statefulSets[0], parallelStatefulSetCreation, namespace, - allowedTopologies, stsScaleUp, stsScaleDown, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - - /* TESTCASE-20 - Storage policy is present in VC1 and VC1 is under reboot - - Steps: - 1. Create SC with specific storage policy , Which is present in VC1 - 2. Create Statefulsets using the above SC and reboot VC1 at the same time - 3. Since VC1 is under reboot , volume creation should be in pendig state till the VC1 is up - 4. Once the VC1 is up , all the volumes should come up on the worker nodes which is in VC1 - 5. Make sure common validation points are met on PV,PVC and POD - 6. Clean up the data - */ - - ginkgo.It("Storage policy is present in VC1 and VC1 is under reboot", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - stsReplicas = 3 - scParameters[scParamStoragePolicyName] = storagePolicyInVc1 - parallelStatefulSetCreation = true - scaleDownReplicaCount = 2 - scaleUpReplicaCount = 7 - topValStartIndex = 0 - topValEndIndex = 1 - sts_count := 2 - - // here, we are considering the allowed topology of VC1 i.e. k8s-zone -> zone-1 - - ginkgo.By("Set specific allowed topology") - allowedTopologies = setSpecificAllowedTopology(allowedTopologies, topkeyStartIndex, topValStartIndex, - topValEndIndex) - - ginkgo.By("Create StorageClass with storage policy specified") - scSpec := getVSphereStorageClassSpec(defaultNginxStorageClassName, scParameters, nil, "", - "", false) - sc, err := client.StorageV1().StorageClasses().Create(ctx, scSpec, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err = client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Create service") - service := CreateService(namespace, client) - defer func() { - deleteService(namespace, client, service) - }() - - ginkgo.By("Rebooting VC1") - vCenterHostname := strings.Split(multiVCe2eVSphere.multivcConfig.Global.VCenterHostname, ",") - vcAddress := vCenterHostname[0] + ":" + sshdPort - framework.Logf("vcAddress - %s ", vcAddress) - err = invokeVCenterReboot(vcAddress) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = waitForHostToBeUp(vCenterHostname[0]) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By("Done with reboot") - - ginkgo.By("Create 3 StatefulSet with replica count 3") - statefulSets := createParallelStatefulSetSpec(namespace, sts_count, stsReplicas) - var wg sync.WaitGroup - wg.Add(sts_count) - for i := 0; i < len(statefulSets); i++ { - go createParallelStatefulSets(client, namespace, statefulSets[i], - stsReplicas, &wg) - - } - wg.Wait() - - essentialServices := []string{spsServiceName, vsanhealthServiceName, vpxdServiceName} - checkVcenterServicesRunning(ctx, vcAddress, essentialServices) - - //After reboot - multiVCbootstrap() - - ginkgo.By("Waiting for StatefulSets Pods to be in Ready State") - err = waitForStsPodsToBeInReadyRunningState(ctx, client, namespace, statefulSets) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - deleteAllStsAndPodsPVCsInNamespace(ctx, client, namespace) - }() - - ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") - for i := 0; i < len(statefulSets); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulSets[i], - namespace, allowedTopologies, parallelStatefulSetCreation, true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - ginkgo.By("Perform scaleup/scaledown operation on statefulsets and " + - "verify pv affinity and pod affinity") - for i := 0; i < len(statefulSets); i++ { - err = performScalingOnStatefulSetAndVerifyPvNodeAffinity(ctx, client, scaleUpReplicaCount, - scaleDownReplicaCount, statefulSets[i], parallelStatefulSetCreation, namespace, - allowedTopologies, stsScaleUp, stsScaleDown, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }) - - /* TESTCASE-21 - VSAN-health down on VC1 - - Steps: - 1. Create SC default values, so all the AZ's should be considered for provisioning. - 2. Create dynamic PVC's and Add labels to PVC's and PV's - 3. Bring down the VSAN-health on VC1 - 4. Create two statefulset each with replica 5 - 5. Since VSAN-health on VC1 is down all the volumes should get created under VC2 availability zones - 6. Verify the PV node affinity and the nodes on which Pods have come should be appropriate - 7. Update the label on PV and PVC - 8. Bring up the VSAN-health - 9. Scale up both the statefull set to 10 - 10. Wait for 2 Full sync cycle - 11. Verify CNS-metadata for the volumes that are created - 12. Verify that the volume provisioning should resume on VC1 as well - 13. Make sure common validation points are met on PV,PVC and POD - 14. Scale down the statefull sets to 1 - 15. Clean up the data - */ - - ginkgo.It("Create workloads when VSAN-health is down on VC1", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - stsReplicas = 5 - pvcCount := 3 - scaleDownReplicaCount = 1 - scaleUpReplicaCount = 10 - parallelPodPolicy = true - parallelStatefulSetCreation = true - labelKey := "volId" - labelValue := "pvcVolume" - labels := make(map[string]string) - labels[labelKey] = labelValue - sts_count := 2 - var fullSyncWaitTime int - var err error - - // Read full-sync value. - if os.Getenv(envFullSyncWaitTime) != "" { - fullSyncWaitTime, err = strconv.Atoi(os.Getenv(envFullSyncWaitTime)) - framework.Logf("Full-Sync interval time value is = %v", fullSyncWaitTime) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - /* here, For 2-VC setup, we are considering all the allowed topologies of VC1 and VC2 - i.e. k8s-zone -> zone-1,zone-2,zone-3,zone-4,zone-5 - - For 3-VC setup, we are considering all the allowed topologies of VC1, VC2 and VC3 - i.e. k8s-zone -> zone-1,zone-2,zone-3 - */ - - ginkgo.By("Create StorageClass with default parameters") - scSpec := getVSphereStorageClassSpec(defaultNginxStorageClassName, nil, nil, "", - "", false) - sc, err := client.StorageV1().StorageClasses().Create(ctx, scSpec, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err = client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Trigger multiple PVCs") - pvclaimsList := createMultiplePVCsInParallel(ctx, client, namespace, sc, pvcCount, labels) - - ginkgo.By("Verify PVC claim to be in bound phase and create POD for each PVC") - for i := 0; i < len(pvclaimsList); i++ { - pvc, err := fpv.WaitForPVClaimBoundPhase(client, - []*v1.PersistentVolumeClaim{pvclaimsList[i]}, framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(pvc).NotTo(gomega.BeEmpty()) - - } - defer func() { - ginkgo.By("Deleting PVC's and PV's") - for i := 0; i < len(pvclaimsList); i++ { - pv := getPvFromClaim(client, pvclaimsList[i].Namespace, pvclaimsList[i].Name) - err = fpv.DeletePersistentVolumeClaim(client, pvclaimsList[i].Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - framework.ExpectNoError(fpv.WaitForPersistentVolumeDeleted(client, pv.Name, poll, pollTimeoutShort)) - err = multiVCe2eVSphere.waitForCNSVolumeToBeDeletedInMultiVC(pv.Spec.CSI.VolumeHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - deleteAllStsAndPodsPVCsInNamespace(ctx, client, namespace) - }() - - ginkgo.By("Bring down Vsan-health service on VC1") - vCenterHostname := strings.Split(multiVCe2eVSphere.multivcConfig.Global.VCenterHostname, ",") - vcAddress := vCenterHostname[0] + ":" + sshdPort - framework.Logf("vcAddress - %s ", vcAddress) - err = invokeVCenterServiceControl(stopOperation, vsanhealthServiceName, vcAddress) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - isVsanHealthServiceStopped = true - defer func() { - if isVsanHealthServiceStopped { - framework.Logf("Bringing vsanhealth up before terminating the test") - startVCServiceWait4VPs(ctx, vcAddress, vsanhealthServiceName, &isVsanHealthServiceStopped) - } - }() - - ginkgo.By("Create 2 StatefulSet with replica count 5 when vsan-health is down") - statefulSets := createParallelStatefulSetSpec(namespace, sts_count, stsReplicas) - var wg sync.WaitGroup - wg.Add(sts_count) - for i := 0; i < len(statefulSets); i++ { - go createParallelStatefulSets(client, namespace, statefulSets[i], - stsReplicas, &wg) - } - wg.Wait() - - labelKey = "volIdNew" - labelValue = "pvcVolumeNew" - labels = make(map[string]string) - labels[labelKey] = labelValue - - ginkgo.By("Updating labels for PVC and PV") - for i := 0; i < len(pvclaimsList); i++ { - pv := getPvFromClaim(client, pvclaimsList[i].Namespace, pvclaimsList[i].Name) - framework.Logf("Updating labels %+v for pvc %s in namespace %s", labels, pvclaimsList[i].Name, - pvclaimsList[i].Namespace) - pvc, err := client.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvclaimsList[i].Name, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pvc.Labels = labels - _, err = client.CoreV1().PersistentVolumeClaims(namespace).Update(ctx, pvc, metav1.UpdateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - framework.Logf("Updating labels %+v for pv %s", labels, pv.Name) - pv.Labels = labels - _, err = client.CoreV1().PersistentVolumes().Update(ctx, pv, metav1.UpdateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - ginkgo.By(fmt.Sprintln("Starting vsan-health on the vCenter host")) - startVCServiceWait4VPs(ctx, vcAddress, vsanhealthServiceName, &isVsanHealthServiceStopped) - - ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to allow full sync finish", fullSyncWaitTime)) - time.Sleep(time.Duration(fullSyncWaitTime) * time.Second) - - ginkgo.By("Waiting for labels to be updated for PVC and PV") - for i := 0; i < len(pvclaimsList); i++ { - pv := getPvFromClaim(client, pvclaimsList[i].Namespace, pvclaimsList[i].Name) - - framework.Logf("Waiting for labels %+v to be updated for pvc %s in namespace %s", - labels, pvclaimsList[i].Name, pvclaimsList[i].Namespace) - err = multiVCe2eVSphere.waitForLabelsToBeUpdatedInMultiVC(pv.Spec.CSI.VolumeHandle, labels, - string(cnstypes.CnsKubernetesEntityTypePVC), pvclaimsList[i].Name, pvclaimsList[i].Namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - framework.Logf("Waiting for labels %+v to be updated for pv %s", labels, pv.Name) - err = multiVCe2eVSphere.waitForLabelsToBeUpdatedInMultiVC(pv.Spec.CSI.VolumeHandle, labels, - string(cnstypes.CnsKubernetesEntityTypePV), pv.Name, pv.Namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - ginkgo.By("Waiting for StatefulSets Pods to be in Ready State") - err = waitForStsPodsToBeInReadyRunningState(ctx, client, namespace, statefulSets) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") - for i := 0; i < len(statefulSets); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulSets[i], - namespace, allowedTopologies, parallelStatefulSetCreation, true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - ginkgo.By("Perform scaleup/scaledown operation on statefulsets and " + - "verify pv affinity and pod affinity") - err = performScalingOnStatefulSetAndVerifyPvNodeAffinity(ctx, client, scaleUpReplicaCount, - scaleDownReplicaCount, statefulSets[0], parallelStatefulSetCreation, namespace, - allowedTopologies, stsScaleUp, stsScaleDown, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - - /* TESTCASE-22 - SPS down on VC2 + use storage policy while creating statefulset - - Steps: - 1. Create SC with storage policy which is in both VCs i.e. VC1 and VC2 - 2. Create Statefulset using the same storage policy - 3. Bring down the SPS on VC1 - 4. create two statefulset each with replica 5 - 5. Since SPS on VC1 is down all the volumes should get created under VC2 availability zones - [Note: if any VC or its service is down, volume provisioning will no go through on those - Pods which are provisioned on the VC where service is down] - till all VCs came to up and running state] - 6. Verify the PV node affinity and the nodes on which Pods have come should be appropriate - 7. Bring up the SPS service - 8. Scale up both the Statefulset to 10 - 9. Verify that the volume provisioning should resume on VC1 as well - 10. Make sure common validation points are met on PV,PVC and POD - 11. Scale down the statefull sets to 1 - 12. Clean up the data - */ - - ginkgo.It("Create workloads with storage policy given in SC and when sps service is down", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - /* here we are considering storage policy of VC1 and VC2 so - the allowed topology to be considered as for 2-VC and 3-VC setup is k8s-zone -> zone-1,zone-2 - - */ - - stsReplicas = 5 - scParameters[scParamStoragePolicyName] = storagePolicyInVc1Vc2 - topValStartIndex = 0 - topValEndIndex = 2 - scaleUpReplicaCount = 10 - scaleDownReplicaCount = 1 - sts_count := 2 - parallelStatefulSetCreation = true - parallelPodPolicy = true - - ginkgo.By("Set specific allowed topology") - allowedTopologies = setSpecificAllowedTopology(allowedTopologies, topkeyStartIndex, topValStartIndex, - topValEndIndex) - - ginkgo.By("Create StorageClass with storage policy specified") - scSpec := getVSphereStorageClassSpec(defaultNginxStorageClassName, scParameters, nil, "", - "", false) - sc, err := client.StorageV1().StorageClasses().Create(ctx, scSpec, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err = client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Create StatefulSet and verify pv affinity and pod affinity details") - service, statefulset, err := createStafeulSetAndVerifyPVAndPodNodeAffinty(ctx, client, namespace, parallelPodPolicy, - stsReplicas, nodeAffinityToSet, allowedTopologies, allowedTopologyLen, podAntiAffinityToSet, - parallelStatefulSetCreation, false, "", "", nil, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - deleteAllStsAndPodsPVCsInNamespace(ctx, client, namespace) - deleteService(namespace, client, service) - }() - - ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, - namespace, allowedTopologies, parallelStatefulSetCreation, true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Bring down SPS service") - vCenterHostname := strings.Split(multiVCe2eVSphere.multivcConfig.Global.VCenterHostname, ",") - vcAddress := vCenterHostname[0] + ":" + sshdPort - framework.Logf("vcAddress - %s ", vcAddress) - err = invokeVCenterServiceControl(stopOperation, spsServiceName, vcAddress) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - isSPSServiceStopped = true - err = waitVCenterServiceToBeInState(spsServiceName, vcAddress, svcStoppedMessage) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - if isSPSServiceStopped { - framework.Logf("Bringing sps up before terminating the test") - startVCServiceWait4VPs(ctx, vcAddress, spsServiceName, &isSPSServiceStopped) - isSPSServiceStopped = false - } - }() - - ginkgo.By("Create 2 StatefulSet with replica count 5 when sps-service is down") - statefulSets := createParallelStatefulSetSpec(namespace, sts_count, stsReplicas) - var wg sync.WaitGroup - wg.Add(sts_count) - for i := 0; i < len(statefulSets); i++ { - go createParallelStatefulSets(client, namespace, statefulSets[i], - stsReplicas, &wg) - } - wg.Wait() - - ginkgo.By("Bringup SPS service") - startVCServiceWait4VPs(ctx, vcAddress, spsServiceName, &isSPSServiceStopped) - - framework.Logf("Waiting for %v seconds for testbed to be in the normal state", pollTimeoutShort) - time.Sleep(pollTimeoutShort) - - ginkgo.By("Waiting for StatefulSets Pods to be in Ready State") - err = waitForStsPodsToBeInReadyRunningState(ctx, client, namespace, statefulSets) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") - for i := 0; i < len(statefulSets); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulSets[i], - namespace, allowedTopologies, parallelStatefulSetCreation, true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - ginkgo.By("Perform scaleup/scaledown operation on statefulsets and " + - "verify pv affinity and pod affinity") - err = performScalingOnStatefulSetAndVerifyPvNodeAffinity(ctx, client, scaleUpReplicaCount, - scaleDownReplicaCount, statefulSets[0], parallelStatefulSetCreation, namespace, - allowedTopologies, stsScaleUp, stsScaleDown, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - }) -}) diff --git a/tests/e2e/multi_vc_bootstrap.go b/tests/e2e/multi_vc_bootstrap.go deleted file mode 100644 index ff66311705..0000000000 --- a/tests/e2e/multi_vc_bootstrap.go +++ /dev/null @@ -1,54 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2e - -import ( - "context" - - "github.com/onsi/gomega" - "k8s.io/kubernetes/test/e2e/framework" - "k8s.io/kubernetes/test/e2e/framework/testfiles" -) - -var multiVCe2eVSphere multiVCvSphere -var multiVCtestConfig *e2eTestConfig - -/* -multiVCbootstrap function takes care of initializing necessary tests context for e2e tests -*/ -func multiVCbootstrap(withoutDc ...bool) { - var err error - multiVCtestConfig, err = getConfig() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - if len(withoutDc) > 0 { - if withoutDc[0] { - (*multiVCtestConfig).Global.Datacenters = "" - } - } - multiVCe2eVSphere = multiVCvSphere{ - multivcConfig: multiVCtestConfig, - } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - // connects and verifies multiple VC connections - connectMultiVC(ctx, &multiVCe2eVSphere) - - if framework.TestContext.RepoRoot != "" { - testfiles.AddFileSource(testfiles.RootFileSource{Root: framework.TestContext.RepoRoot}) - } - framework.TestContext.Provider = "vsphere" -} diff --git a/tests/e2e/multi_vc_config_secret.go b/tests/e2e/multi_vc_config_secret.go deleted file mode 100644 index d4014d7956..0000000000 --- a/tests/e2e/multi_vc_config_secret.go +++ /dev/null @@ -1,1058 +0,0 @@ -/* - Copyright 2023 The Kubernetes Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package e2e - -import ( - "context" - "fmt" - "strings" - "sync" - "time" - - ginkgo "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" - "golang.org/x/crypto/ssh" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/kubernetes/test/e2e/framework" - fnodes "k8s.io/kubernetes/test/e2e/framework/node" - fpod "k8s.io/kubernetes/test/e2e/framework/pod" - fpv "k8s.io/kubernetes/test/e2e/framework/pv" - fss "k8s.io/kubernetes/test/e2e/framework/statefulset" - admissionapi "k8s.io/pod-security-admission/api" -) - -var _ = ginkgo.Describe("[csi-multi-vc-config-secret] Multi-VC-Config-Secret", func() { - f := framework.NewDefaultFramework("multi-vc-config-secret") - f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged - var ( - client clientset.Interface - namespace string - csiNamespace string - csiReplicas int32 - podAntiAffinityToSet bool - stsScaleUp bool - stsScaleDown bool - verifyTopologyAffinity bool - allowedTopologies []v1.TopologySelectorLabelRequirement - scaleUpReplicaCount int32 - scaleDownReplicaCount int32 - allowedTopologyLen int - nodeAffinityToSet bool - parallelStatefulSetCreation bool - stsReplicas int32 - parallelPodPolicy bool - originalVC1PasswordChanged bool - originalVC3PasswordChanged bool - vCenterIP string - vCenterUser string - vCenterPassword string - vCenterPort string - dataCenter string - err error - revertToOriginalVsphereConf bool - multiVCSetupType string - allMasterIps []string - sshClientConfig *ssh.ClientConfig - nimbusGeneratedK8sVmPwd string - revertToOriginalCsiYaml bool - newNamespace *v1.Namespace - ) - - ginkgo.BeforeEach(func() { - var cancel context.CancelFunc - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - client = f.ClientSet - namespace = f.Namespace.Name - multiVCbootstrap() - - sc, err := client.StorageV1().StorageClasses().Get(ctx, defaultNginxStorageClassName, metav1.GetOptions{}) - if err == nil && sc != nil { - gomega.Expect(client.StorageV1().StorageClasses().Delete(ctx, sc.Name, - *metav1.NewDeleteOptions(0))).NotTo(gomega.HaveOccurred()) - } - - nodeList, err := fnodes.GetReadySchedulableNodes(f.ClientSet) - framework.ExpectNoError(err, "Unable to find ready and schedulable Node") - if !(len(nodeList.Items) > 0) { - framework.Failf("Unable to find ready and schedulable Node") - } - - verifyTopologyAffinity = true - stsScaleUp = true - stsScaleDown = true - - // read namespace - csiNamespace = GetAndExpectStringEnvVar(envCSINamespace) - csiDeployment, err := client.AppsV1().Deployments(csiNamespace).Get( - ctx, vSphereCSIControllerPodNamePrefix, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - csiReplicas = *csiDeployment.Spec.Replicas - - // read testbed topology map - topologyMap := GetAndExpectStringEnvVar(topologyMap) - allowedTopologies = createAllowedTopolgies(topologyMap, topologyLength) - - // save original vsphere conf credentials in temp variable - vCenterIP = multiVCe2eVSphere.multivcConfig.Global.VCenterHostname - vCenterUser = multiVCe2eVSphere.multivcConfig.Global.User - vCenterPassword = multiVCe2eVSphere.multivcConfig.Global.Password - vCenterPort = multiVCe2eVSphere.multivcConfig.Global.VCenterPort - dataCenter = multiVCe2eVSphere.multivcConfig.Global.Datacenters - - // read type of multi-vc setup - multiVCSetupType = GetAndExpectStringEnvVar(envMultiVCSetupType) - nimbusGeneratedK8sVmPwd = GetAndExpectStringEnvVar(nimbusK8sVmPwd) - - sshClientConfig = &ssh.ClientConfig{ - User: "root", - Auth: []ssh.AuthMethod{ - ssh.Password(nimbusGeneratedK8sVmPwd), - }, - HostKeyCallback: ssh.InsecureIgnoreHostKey(), - } - - // fetching k8s master ip - allMasterIps = getK8sMasterIPs(ctx, client) - }) - - ginkgo.AfterEach(func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - if revertToOriginalVsphereConf { - ginkgo.By("Reverting back csi-vsphere.conf with its original vcenter user " + - "and its credentials") - vsphereCfg, err := readVsphereConfSecret(client, ctx, csiNamespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - vsphereCfg.Global.VCenterHostname = vCenterIP - vsphereCfg.Global.User = vCenterUser - vsphereCfg.Global.Password = vCenterPassword - vsphereCfg.Global.VCenterPort = vCenterPort - vsphereCfg.Global.Datacenters = dataCenter - err = writeNewDataAndUpdateVsphereConfSecret(client, ctx, csiNamespace, vsphereCfg) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Restart CSI driver") - restartSuccess, err := restartCSIDriver(ctx, client, csiNamespace, csiReplicas) - gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - if originalVC1PasswordChanged { - clientIndex := 0 - vcAddress := strings.Split(vCenterIP, ",")[0] + ":" + sshdPort - username := strings.Split(vCenterUser, ",")[0] - originalPassword := strings.Split(vCenterPassword, ",")[0] - newPassword := e2eTestPassword - ginkgo.By("Reverting the password change") - err = invokeVCenterChangePassword(username, newPassword, originalPassword, vcAddress, true, - clientIndex) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - if originalVC3PasswordChanged { - clientIndex2 := 2 - vcAddress3 := strings.Split(vCenterIP, ",")[2] + ":" + sshdPort - username3 := strings.Split(vCenterUser, ",")[2] - originalPassword3 := strings.Split(vCenterPassword, ",")[2] - newPassword3 := "Admin!23" - ginkgo.By("Reverting the password change") - err = invokeVCenterChangePassword(username3, newPassword3, originalPassword3, vcAddress3, true, - clientIndex2) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - originalVC3PasswordChanged = false - } - - if revertToOriginalCsiYaml { - vsphereCfg, err := readVsphereConfSecret(client, ctx, csiNamespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - vsphereCfg.Global.VCenterHostname = vCenterIP - vsphereCfg.Global.User = vCenterUser - vsphereCfg.Global.Password = vCenterPassword - vsphereCfg.Global.VCenterPort = vCenterPort - vsphereCfg.Global.Datacenters = dataCenter - - ginkgo.By("Recreate config secret on a default csi system namespace") - err = deleteVsphereConfigSecret(client, ctx, newNamespace.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = createVsphereConfigSecret(csiNamespace, vsphereCfg, sshClientConfig, allMasterIps) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By("Revert vsphere CSI driver on a default csi system namespace") - err = setNewNameSpaceInCsiYaml(client, sshClientConfig, newNamespace.Name, csiNamespace, allMasterIps) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }) - - /* TESTCASE-1 - Change VC password on one of the VC, also update the vsphere-csi-secret - - // Steps - 1. Create SC with all topologies present in allowed Topology list - 2. Change the VC UI password for any one VC, Update the same in "CSI config secret" file and re-create the - secret - 3. Re-start the CSI driver - 4. Wait for some time, CSI will auto identify the change in vsphere-secret file and get updated - 5. Create Statefulset. PVCs and Pods should be in bound and running state. - 6. Make sure all the common verification points are met - a) Verify node affinity on all the PV's - b) Verify that POD should be up and running on the appropriate nodes - 7. Scale-up/Scale-down the statefulset - 8. Clean up the data - */ - - ginkgo.It("Change vCenter password on one of the multi-vc setup and update the same "+ - "in csi vsphere conf", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - clientIndex := 0 - stsReplicas = 3 - scaleUpReplicaCount = 5 - scaleDownReplicaCount = 2 - - ginkgo.By("Create StorageClass with all allowed topolgies set") - scSpec := getVSphereStorageClassSpec(defaultNginxStorageClassName, nil, allowedTopologies, "", - "", false) - sc, err := client.StorageV1().StorageClasses().Create(ctx, scSpec, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err = client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - // read original vsphere config secret - vsphereCfg, err := readVsphereConfSecret(client, ctx, csiNamespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - vcAddress := strings.Split(vsphereCfg.Global.VCenterHostname, ",")[0] + ":" + sshdPort - framework.Logf("vcAddress - %s ", vcAddress) - username := strings.Split(vsphereCfg.Global.User, ",")[0] - originalPassword := strings.Split(vsphereCfg.Global.Password, ",")[0] - newPassword := e2eTestPassword - ginkgo.By(fmt.Sprintf("Original password %s, new password %s", originalPassword, newPassword)) - - ginkgo.By("Changing password on the vCenter VC1 host") - err = invokeVCenterChangePassword(username, originalPassword, newPassword, vcAddress, true, clientIndex) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - originalVC1PasswordChanged = true - - // here 0 indicates that we are updating only VC1 password - ginkgo.By("Create vsphere-config-secret file with new VC1 password") - passwordList := strings.Split(vsphereCfg.Global.Password, ",") - passwordList[0] = newPassword - vsphereCfg.Global.Password = strings.Join(passwordList, ",") - - // here we are writing new password of VC1 and later updating vsphere config secret - err = writeNewDataAndUpdateVsphereConfSecret(client, ctx, csiNamespace, vsphereCfg) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - revertToOriginalVsphereConf = true - defer func() { - if originalVC1PasswordChanged { - ginkgo.By("Reverting the password change") - err = invokeVCenterChangePassword(username, newPassword, originalPassword, vcAddress, true, - clientIndex) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - originalVC1PasswordChanged = false - } - - if revertToOriginalVsphereConf { - ginkgo.By("Reverting back csi-vsphere.conf with its original vcenter user " + - "and its credentials") - vsphereCfg.Global.VCenterHostname = vCenterIP - vsphereCfg.Global.User = vCenterUser - vsphereCfg.Global.Password = vCenterPassword - vsphereCfg.Global.VCenterPort = vCenterPort - vsphereCfg.Global.Datacenters = dataCenter - err = writeNewDataAndUpdateVsphereConfSecret(client, ctx, csiNamespace, vsphereCfg) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - revertToOriginalVsphereConf = false - - ginkgo.By("Restart CSI driver") - restartSuccess, err := restartCSIDriver(ctx, client, csiNamespace, csiReplicas) - gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - }() - - ginkgo.By("Restart CSI driver") - restartSuccess, err := restartCSIDriver(ctx, client, csiNamespace, csiReplicas) - gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Create StatefulSet and verify pv affinity and pod affinity details") - service, statefulset, err := createStafeulSetAndVerifyPVAndPodNodeAffinty(ctx, client, namespace, - parallelPodPolicy, stsReplicas, nodeAffinityToSet, allowedTopologies, allowedTopologyLen, - podAntiAffinityToSet, parallelStatefulSetCreation, false, "", "", nil, verifyTopologyAffinity) - defer func() { - fss.DeleteAllStatefulSets(client, namespace) - deleteService(namespace, client, service) - }() - - ginkgo.By("Perform scaleup/scaledown operation on statefulsets and " + - "verify pv affinity and pod affinity") - err = performScalingOnStatefulSetAndVerifyPvNodeAffinity(ctx, client, scaleUpReplicaCount, - scaleDownReplicaCount, statefulset, parallelStatefulSetCreation, namespace, - allowedTopologies, stsScaleUp, stsScaleDown, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - - /* Testcase-2 - copy same VC details twice in csi-vsphere conf - NEGETIVE - - In csi-vsphere conf, copy VC1's details twice , mention VC2 details once and create secret - Observe the system behaviour , Expectation is CSI pod's should show CLBO or should show error - */ - - ginkgo.It("Copy same vCenter details twice in csi vsphere conf in a multi-vc setup", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - // read original vsphere config secret - vsphereCfg, err := readVsphereConfSecret(client, ctx, csiNamespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - /* here we are updating only vCenter IP and vCenter Password, rest all other vCenter credentials will - remain same - VC1 and VC3 credentials will be same in conf secret, VC2 will be having its exact credentials - - Note: For this scenario, we will not be doing any restart of service - */ - - ginkgo.By("Create vsphere-config-secret file with same VC credential details 2 times") - if multiVCSetupType == "multi-2vc-setup" { - // copying VC1 IP to VC2 IP - vCenterIPList := strings.Split(vsphereCfg.Global.VCenterHostname, ",") - vCenterIPList[1] = vCenterIPList[0] - vsphereCfg.Global.VCenterHostname = strings.Join(vCenterIPList, ",") - - // assigning new Password to VC2 - passwordList := strings.Split(vsphereCfg.Global.Password, ",") - passwordList[1] = e2eTestPassword - vsphereCfg.Global.Password = strings.Join(passwordList, ",") - } else if multiVCSetupType == "multi-3vc-setup" { - // copying VC1 IP to VC3 IP - vCenterIPList := strings.Split(vsphereCfg.Global.VCenterHostname, ",") - vCenterIPList[2] = vCenterIPList[0] - vsphereCfg.Global.VCenterHostname = strings.Join(vCenterIPList, ",") - - // assigning new Password to VC3 - passwordList := strings.Split(vsphereCfg.Global.Password, ",") - passwordList[2] = e2eTestPassword - vsphereCfg.Global.Password = strings.Join(passwordList, ",") - } - - /* here we are copying VC1 credentials to VC3 credentials in case of 3-VC setup and - VC1 credentials to VC2 credentials in case of 2-VC setup and later updating vsphere config secret */ - - err = writeNewDataAndUpdateVsphereConfSecret(client, ctx, csiNamespace, vsphereCfg) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - revertToOriginalVsphereConf = true - defer func() { - if revertToOriginalVsphereConf { - ginkgo.By("Reverting back csi-vsphere.conf with its original vcenter user " + - "and its credentials") - vsphereCfg.Global.VCenterHostname = vCenterIP - vsphereCfg.Global.User = vCenterUser - vsphereCfg.Global.Password = vCenterPassword - vsphereCfg.Global.VCenterPort = vCenterPort - vsphereCfg.Global.Datacenters = dataCenter - err = writeNewDataAndUpdateVsphereConfSecret(client, ctx, csiNamespace, vsphereCfg) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - revertToOriginalVsphereConf = false - - ginkgo.By("Restart CSI driver") - restartSuccess, err := restartCSIDriver(ctx, client, csiNamespace, csiReplicas) - gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - }() - - framework.Logf("Wait for %v for csi to auto-detect the config secret changes", pollTimeout) - time.Sleep(pollTimeout) - - framework.Logf("Verify CSI Pods are in CLBO state") - deploymentPods, err := client.AppsV1().Deployments(csiNamespace).Get(ctx, vSphereCSIControllerPodNamePrefix, - metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - if deploymentPods.Status.UnavailableReplicas != 0 { - framework.Logf("CSI Pods are in CLBO state") - } - - ginkgo.By("Try to create a PVC verify that it is stuck in pending state") - storageclass, pvclaim, err := createPVCAndStorageClass(client, namespace, nil, nil, "", - allowedTopologies, "", false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err = client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Expect claim to fail as invalid storage policy is specified in Storage Class") - framework.ExpectError(fpv.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, - client, pvclaim.Namespace, pvclaim.Name, framework.Poll, framework.ClaimProvisionTimeout)) - expectedErrMsg := "waiting for a volume to be created" - err = waitForEvent(ctx, client, namespace, expectedErrMsg, pvclaim.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), fmt.Sprintf("Expected error : %q", expectedErrMsg)) - }) - - /* Testcase-3 - In csi-vsphere conf, use VC-hostname instead of VC-IP for one VC and try to switch the same during - a workload vcreation - - 1. In csi-vsphere conf, for VC1 use VC-IP, for VC2 use VC-hostname - 2. Create SC - 3. Create Statefulset of replica 10 - 4. Try to switch the VC-ip and VC-hostname in config secret and re-create - 5. Reboot CSI-driver to consider the new change - 6. Make sure Statefulset creation should be successful - 7. Verify the node affinity rules - 8. Verify the list-volume response in CSI logs - [This will be covered in list volume testcase] - 9. Clean up the data - */ - - ginkgo.It("Use VC-hostname instead of VC-IP for one VC and try to switch the same during"+ - "a workload vcreation in a multivc setup", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - stsReplicas = 10 - scaleUpReplicaCount = 5 - scaleDownReplicaCount = 2 - - ginkgo.By("Create StorageClass with all allowed topolgies set") - scSpec := getVSphereStorageClassSpec(defaultNginxStorageClassName, nil, allowedTopologies, "", - "", false) - sc, err := client.StorageV1().StorageClasses().Create(ctx, scSpec, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err = client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - // read original vsphere config secret - vsphereCfg, err := readVsphereConfSecret(client, ctx, csiNamespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Use VC-IP for VC1 and VC-hostname for VC2 in a multi VC setup") - vCenterList := strings.Split(vsphereCfg.Global.VCenterHostname, ",") - vCenterIPVC2 := vCenterList[1] - - ginkgo.By("Fetch vcenter hotsname for VC2") - vCenterHostName := getVcenterHostName(vCenterList[1]) - vCenterList[1] = vCenterHostName - vsphereCfg.Global.VCenterHostname = strings.Join(vCenterList, ",") - - // here we are writing hostname of VC2 and later updating vsphere config secret - err = writeNewDataAndUpdateVsphereConfSecret(client, ctx, csiNamespace, vsphereCfg) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - revertToOriginalVsphereConf = true - defer func() { - if revertToOriginalVsphereConf { - ginkgo.By("Reverting back csi-vsphere.conf with its original vcenter user " + - "and its credentials") - vsphereCfg.Global.VCenterHostname = vCenterIP - vsphereCfg.Global.User = vCenterUser - vsphereCfg.Global.Password = vCenterPassword - vsphereCfg.Global.VCenterPort = vCenterPort - vsphereCfg.Global.Datacenters = dataCenter - err = writeNewDataAndUpdateVsphereConfSecret(client, ctx, csiNamespace, vsphereCfg) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - revertToOriginalVsphereConf = false - - ginkgo.By("Restart CSI driver") - restartSuccess, err := restartCSIDriver(ctx, client, csiNamespace, csiReplicas) - gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - }() - - ginkgo.By("Restart CSI driver") - restartSuccess, err := restartCSIDriver(ctx, client, csiNamespace, csiReplicas) - gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Create StatefulSet and verify pv affinity and pod affinity details") - service, statefulset, err := createStafeulSetAndVerifyPVAndPodNodeAffinty(ctx, client, namespace, - parallelPodPolicy, stsReplicas, nodeAffinityToSet, allowedTopologies, allowedTopologyLen, - podAntiAffinityToSet, parallelStatefulSetCreation, false, "", "", nil, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - deleteService(namespace, client, service) - fss.DeleteAllStatefulSets(client, namespace) - }() - - ginkgo.By("Use VC-IP for VC2 and VC-hostname for VC1 in a multi VC setup") - ginkgo.By("Fetch vcenter hotsname of VC1") - vCenterHostNameVC1 := getVcenterHostName(vCenterList[0]) - vCenterList[0] = vCenterHostNameVC1 - vCenterList[1] = vCenterIPVC2 - vsphereCfg.Global.VCenterHostname = strings.Join(vCenterList, ",") - - // here we are writing hostname of VC1 and later updating vsphere config secret - err = writeNewDataAndUpdateVsphereConfSecret(client, ctx, csiNamespace, vsphereCfg) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - revertToOriginalVsphereConf = true - - ginkgo.By("Restart CSI driver") - restartSuccess, err = restartCSIDriver(ctx, client, csiNamespace, csiReplicas) - gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Perform scaleup/scaledown operation on statefulsets and " + - "verify pv affinity and pod affinity") - err = performScalingOnStatefulSetAndVerifyPvNodeAffinity(ctx, client, scaleUpReplicaCount, - scaleDownReplicaCount, statefulset, parallelStatefulSetCreation, namespace, - allowedTopologies, stsScaleUp, stsScaleDown, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - - /* Testcase-4 - Install CSI driver on different namespace and restart CSI-controller and node daemon sets in - between the statefulset creation - - 1. Create SC with allowedTopology details contains all availability zones - 2. Create 3 Statefulset with replica-5 - 3. Re-start the CSI controller Pod and node-daemon sets - 4. Wait for PVC to reach bound state and POD to reach Running state - 5. Volumes should get distributed among all the Availability zones - 6. Verify the PV node affinity details should have appropriate node details - 7. The Pods should be running on the appropriate nodes - 8. Scale-up/scale-down the statefulset - 9. Verify the node affinity details also verify the POD details - 10. Clean up the data - */ - - ginkgo.It("Install CSI driver on different namespace and restart CSI-controller and node daemon sets"+ - "in between the statefulset creation", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - stsReplicas = 5 - sts_count := 3 - ignoreLabels := make(map[string]string) - parallelStatefulSetCreation = true - - // read original vsphere config secret - vsphereCfg, err := readVsphereConfSecret(client, ctx, csiNamespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Create a new namespace") - newNamespace, err = createNamespace(client, ctx, "test-ns") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - ginkgo.By("Delete newly created namespace") - err = deleteNamespace(client, ctx, newNamespace.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Delete config secret created on csi namespace") - err = deleteVsphereConfigSecret(client, ctx, csiNamespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Create config secret on a new namespace") - err = createVsphereConfigSecret(newNamespace.Name, vsphereCfg, sshClientConfig, allMasterIps) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Install vsphere CSI driver on different test namespace") - err = setNewNameSpaceInCsiYaml(client, sshClientConfig, csiNamespace, newNamespace.Name, allMasterIps) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - revertToOriginalCsiYaml = true - defer func() { - if revertToOriginalCsiYaml { - ginkgo.By("Recreate config secret on a default csi system namespace") - err = deleteVsphereConfigSecret(client, ctx, newNamespace.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - err = createVsphereConfigSecret(csiNamespace, vsphereCfg, sshClientConfig, allMasterIps) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Revert vsphere CSI driver on a system csi namespace") - err = setNewNameSpaceInCsiYaml(client, sshClientConfig, newNamespace.Name, csiNamespace, allMasterIps) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - revertToOriginalCsiYaml = false - } - }() - - ginkgo.By("Create StorageClass with all allowed topolgies set") - scSpec := getVSphereStorageClassSpec(defaultNginxStorageClassName, nil, allowedTopologies, "", - "", false) - sc, err := client.StorageV1().StorageClasses().Create(ctx, scSpec, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err = client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Creating service") - service := CreateService(namespace, client) - defer func() { - deleteService(namespace, client, service) - }() - - ginkgo.By("Creating multiple StatefulSets specs in parallel") - statefulSets := createParallelStatefulSetSpec(namespace, sts_count, stsReplicas) - - ginkgo.By("Trigger multiple StatefulSets creation in parallel") - var wg sync.WaitGroup - wg.Add(sts_count) - for i := 0; i < len(statefulSets); i++ { - go createParallelStatefulSets(client, namespace, statefulSets[i], stsReplicas, &wg) - if i == 1 { - ginkgo.By("Restart CSI driver") - restartSuccess, err := restartCSIDriver(ctx, client, newNamespace.Name, csiReplicas) - gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - } - wg.Wait() - - ginkgo.By("Waiting for StatefulSets Pods to be in Ready State") - err = waitForStsPodsToBeInReadyRunningState(ctx, client, namespace, statefulSets) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - deleteAllStsAndPodsPVCsInNamespace(ctx, client, namespace) - }() - - ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") - for i := 0; i < len(statefulSets); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, - statefulSets[i], namespace, allowedTopologies, parallelStatefulSetCreation, true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - ginkgo.By("Perform scaleup/scaledown operation on statefulset and verify pv and pod affinity details") - for i := 0; i < len(statefulSets); i++ { - if i == 0 { - stsScaleUp = false - scaleDownReplicaCount = 3 - framework.Logf("Scale down StatefulSet1 replica count to 3") - err = performScalingOnStatefulSetAndVerifyPvNodeAffinity(ctx, client, scaleUpReplicaCount, - scaleDownReplicaCount, statefulSets[i], parallelStatefulSetCreation, namespace, - allowedTopologies, stsScaleUp, stsScaleDown, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - if i == 1 { - scaleUpReplicaCount = 9 - stsScaleDown = false - framework.Logf("Scale up StatefulSet2 replica count to 9") - err = performScalingOnStatefulSetAndVerifyPvNodeAffinity(ctx, client, scaleUpReplicaCount, - scaleDownReplicaCount, statefulSets[i], parallelStatefulSetCreation, namespace, - allowedTopologies, stsScaleUp, stsScaleDown, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - } - if i == 2 { - framework.Logf("Scale up StatefulSet3 replica count to 13 and later scale it down " + - "to replica count 2 and in between restart node daemon set") - scaleUpReplicaCount = 13 - scaleDownReplicaCount = 2 - - // Fetch the number of CSI pods running before restart - list_of_pods, err := fpod.GetPodsInNamespace(client, newNamespace.Name, ignoreLabels) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - num_csi_pods := len(list_of_pods) - - // Collecting and dumping csi pod logs before restrating CSI daemonset - collectPodLogs(ctx, client, newNamespace.Name) - - // Restart CSI daemonset - ginkgo.By("Restart Daemonset") - cmd := []string{"rollout", "restart", "daemonset/vsphere-csi-node", "--namespace=" + newNamespace.Name} - framework.RunKubectlOrDie(newNamespace.Name, cmd...) - - ginkgo.By("Waiting for daemon set rollout status to finish") - statusCheck := []string{"rollout", "status", "daemonset/vsphere-csi-node", "--namespace=" + newNamespace.Name} - framework.RunKubectlOrDie(newNamespace.Name, statusCheck...) - - // wait for csi Pods to be in running ready state - err = fpod.WaitForPodsRunningReady(client, newNamespace.Name, int32(num_csi_pods), 0, pollTimeout, ignoreLabels) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - err = performScalingOnStatefulSetAndVerifyPvNodeAffinity(ctx, client, scaleUpReplicaCount, - scaleDownReplicaCount, statefulSets[i], parallelStatefulSetCreation, namespace, - allowedTopologies, stsScaleUp, stsScaleDown, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - } - - }) - - /* Testcase-5 - In vsphere-config, keep different passwords on each VC and check Statefulset creation and reboot VC - - 1. Set different passwords to VC's in config secret . - 2. Re-start csi driver pods - 3. Wait for some time for csi to pick the changes on vsphere-secret file - 4. Create Statefulsets - 5. Reboot any one VC - 6. Revert the original VC password and vsphere conf, but this time do not restart csi driver - 6. Scale up/Scale-down the statefulset - 7. Verify the node affinity - 8. Clean up the data - */ - - ginkgo.It("Keep different passwords on each VC and check Statefulset creation and reboot VC", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - stsReplicas = 3 - scaleUpReplicaCount = 5 - scaleDownReplicaCount = 2 - var clientIndex2 int - var username3, newPassword3, originalPassword3, vcAddress3 string - - // read original vsphere config secret - vsphereCfg, err := readVsphereConfSecret(client, ctx, csiNamespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Create StorageClass with all allowed topolgies set") - scSpec := getVSphereStorageClassSpec(defaultNginxStorageClassName, nil, allowedTopologies, "", - "", false) - sc, err := client.StorageV1().StorageClasses().Create(ctx, scSpec, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err = client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - // read VC1 credentials - vcAddress1 := strings.Split(vsphereCfg.Global.VCenterHostname, ",")[0] + ":" + sshdPort - username1 := strings.Split(vsphereCfg.Global.User, ",")[0] - originalPassword1 := strings.Split(vsphereCfg.Global.Password, ",")[0] - newPassword1 := "E2E-test-password!23" - - ginkgo.By("Changing password on the vCenter VC1 host") - clientIndex0 := 0 - err = invokeVCenterChangePassword(username1, originalPassword1, newPassword1, vcAddress1, true, clientIndex0) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - originalVC1PasswordChanged = true - - if multiVCSetupType == "multi-3vc-setup" { - // read VC3 credentials - vcAddress3 = strings.Split(vsphereCfg.Global.VCenterHostname, ",")[2] + ":" + sshdPort - username3 = strings.Split(vsphereCfg.Global.User, ",")[2] - originalPassword3 = strings.Split(vsphereCfg.Global.Password, ",")[2] - newPassword3 = "Admin!23" - - ginkgo.By("Changing password on the vCenter VC3 host") - clientIndex2 = 2 - err = invokeVCenterChangePassword(username3, originalPassword3, newPassword3, vcAddress3, true, clientIndex2) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - originalVC3PasswordChanged = true - - ginkgo.By("Create vsphere-config-secret file with new VC1 and new VC2 password in 3-VC setup") - passwordList := strings.Split(vsphereCfg.Global.Password, ",") - passwordList[0] = newPassword1 - passwordList[2] = newPassword3 - vsphereCfg.Global.Password = strings.Join(passwordList, ",") - } else if multiVCSetupType == "multi-2vc-setup" { - ginkgo.By("Create vsphere-config-secret file with new VC1 password in 2-VC setup") - passwordList := strings.Split(vsphereCfg.Global.Password, ",") - passwordList[0] = newPassword1 - vsphereCfg.Global.Password = strings.Join(passwordList, ",") - } - - // here we are writing new password of VC1 and VC2 and later updating vsphere config secret - err = writeNewDataAndUpdateVsphereConfSecret(client, ctx, csiNamespace, vsphereCfg) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - revertToOriginalVsphereConf = true - defer func() { - if originalVC1PasswordChanged { - ginkgo.By("Reverting the password change") - err = invokeVCenterChangePassword(username1, newPassword1, originalPassword1, vcAddress1, true, - clientIndex0) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - originalVC1PasswordChanged = false - } - - if multiVCSetupType == "multi-3vc-setup" { - if originalVC3PasswordChanged { - ginkgo.By("Reverting the password change") - err = invokeVCenterChangePassword(username3, newPassword3, originalPassword3, vcAddress3, true, - clientIndex2) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - originalVC3PasswordChanged = false - } - } - - if revertToOriginalVsphereConf { - ginkgo.By("Reverting back csi-vsphere.conf with its original vcenter user " + - "and its credentials") - vsphereCfg.Global.VCenterHostname = vCenterIP - vsphereCfg.Global.User = vCenterUser - vsphereCfg.Global.Password = vCenterPassword - vsphereCfg.Global.VCenterPort = vCenterPort - vsphereCfg.Global.Datacenters = dataCenter - err = writeNewDataAndUpdateVsphereConfSecret(client, ctx, csiNamespace, vsphereCfg) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - revertToOriginalVsphereConf = false - - ginkgo.By("Restart CSI driver") - restartSuccess, err := restartCSIDriver(ctx, client, csiNamespace, csiReplicas) - gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - }() - - ginkgo.By("Restart CSI driver") - restartSuccess, err := restartCSIDriver(ctx, client, csiNamespace, csiReplicas) - gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Create StatefulSet and verify pv affinity and pod affinity details") - service, statefulset, err := createStafeulSetAndVerifyPVAndPodNodeAffinty(ctx, client, namespace, - parallelPodPolicy, stsReplicas, nodeAffinityToSet, allowedTopologies, allowedTopologyLen, - podAntiAffinityToSet, parallelStatefulSetCreation, false, "", "", nil, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - fss.DeleteAllStatefulSets(client, namespace) - deleteService(namespace, client, service) - }() - - ginkgo.By("Rebooting VC2") - vCenterHostname := strings.Split(multiVCe2eVSphere.multivcConfig.Global.VCenterHostname, ",") - vcAddress := vCenterHostname[1] + ":" + sshdPort - framework.Logf("vcAddress - %s ", vcAddress) - err = invokeVCenterReboot(vcAddress) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = waitForHostToBeUp(vCenterHostname[1]) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By("Done with reboot") - - essentialServices := []string{spsServiceName, vsanhealthServiceName, vpxdServiceName} - checkVcenterServicesRunning(ctx, vcAddress, essentialServices) - - ginkgo.By("Reverting the password change on VC1") - err = invokeVCenterChangePassword(username1, newPassword1, originalPassword1, vcAddress1, true, - clientIndex0) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - originalVC1PasswordChanged = false - - if multiVCSetupType == "multi-3vc-setup" { - ginkgo.By("Reverting the password change on VC3") - err = invokeVCenterChangePassword(username3, newPassword3, originalPassword3, vcAddress3, true, - clientIndex2) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - originalVC3PasswordChanged = false - } - - ginkgo.By("Reverting back csi-vsphere.conf with its original vcenter user " + - "and its credentials") - vsphereCfg.Global.VCenterHostname = vCenterIP - vsphereCfg.Global.User = vCenterUser - vsphereCfg.Global.Password = vCenterPassword - vsphereCfg.Global.VCenterPort = vCenterPort - vsphereCfg.Global.Datacenters = dataCenter - err = writeNewDataAndUpdateVsphereConfSecret(client, ctx, csiNamespace, vsphereCfg) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - revertToOriginalVsphereConf = false - - framework.Logf("Wait for %v for csi driver to auto-detect the changes", pollTimeout) - time.Sleep(pollTimeout) - - ginkgo.By("Perform scaleup/scaledown operation on statefulsets and " + - "verify pv affinity and pod affinity") - err = performScalingOnStatefulSetAndVerifyPvNodeAffinity(ctx, client, scaleUpReplicaCount, - scaleDownReplicaCount, statefulset, parallelStatefulSetCreation, namespace, - allowedTopologies, stsScaleUp, stsScaleDown, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - - /*Testcase-6 - Change VC in the UI but not on the vsphere secret - 1. Create SC - 2. Without updating the vsphere secret on the VC password - 3. Create a statefulset - 4. Until driver restarts all the workflows will go fine - 5. Statefulset creation should be successful - 6. Restart the driver - 7. There should be error while provisioning volume on the VC on which password has updated - 8. Clean up the data - */ - - ginkgo.It("Change VC in the UI but not on the vsphere secret and verify "+ - "volume creation workflow on a multivc setup", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - clientIndex := 0 - - ginkgo.By("Changing password on the vCenter VC1 host") - // read original vsphere config secret - vsphereCfg, err := readVsphereConfSecret(client, ctx, csiNamespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - vcAddress := strings.Split(vsphereCfg.Global.VCenterHostname, ",")[0] + ":" + sshdPort - framework.Logf("vcAddress - %s ", vcAddress) - username := strings.Split(vsphereCfg.Global.User, ",")[0] - originalPassword := strings.Split(vsphereCfg.Global.Password, ",")[0] - newPassword := e2eTestPassword - ginkgo.By(fmt.Sprintf("Original password %s, new password %s", originalPassword, newPassword)) - err = invokeVCenterChangePassword(username, originalPassword, newPassword, vcAddress, true, clientIndex) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - originalVC1PasswordChanged = true - defer func() { - if originalVC1PasswordChanged { - ginkgo.By("Reverting the password change") - err = invokeVCenterChangePassword(username, newPassword, originalPassword, vcAddress, true, - clientIndex) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - originalVC1PasswordChanged = false - } - }() - - framework.Logf("Wait for %v for csi driver to auto-detect the changes", pollTimeout) - time.Sleep(pollTimeout) - - framework.Logf("Verify CSI Pods are in CLBO state") - deploymentPods, err := client.AppsV1().Deployments(csiNamespace).Get(ctx, vSphereCSIControllerPodNamePrefix, - metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - if deploymentPods.Status.UnavailableReplicas != 0 { - framework.Logf("CSI Pods are in CLBO state") - } - - ginkgo.By("Try to create a PVC verify that it is stuck in pending state") - storageclass, pvclaim, err := createPVCAndStorageClass(client, namespace, nil, nil, "", - allowedTopologies, "", false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err = client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Expect claim to fail as invalid storage policy is specified in Storage Class") - framework.ExpectError(fpv.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, - client, pvclaim.Namespace, pvclaim.Name, framework.Poll, framework.ClaimProvisionTimeout)) - expectedErrMsg := "waiting for a volume to be created" - err = waitForEvent(ctx, client, namespace, expectedErrMsg, pvclaim.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), fmt.Sprintf("Expected error : %q", expectedErrMsg)) - }) - - /* Testcase-7 - Make a wrong entry in vsphere conf for one of the VC and create a secret - Logs will have error messages, But driver still be up and running . - until driver restarts , volume creation will be fine - After re-starting CSI driver it should throw error, should not come to running state until the error in - vsphere-secret is fixed - */ - - ginkgo.It("Add any wrong entry in vsphere conf and verify csi pods behaviour", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - wrongPortNoVC1 := "337" - - // read original vsphere config secret - vsphereCfg, err := readVsphereConfSecret(client, ctx, csiNamespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Create vsphere-config-secret file with wrong port number in VC1") - portList := strings.Split(vsphereCfg.Global.VCenterPort, ",") - portList[0] = wrongPortNoVC1 - vsphereCfg.Global.VCenterPort = strings.Join(portList, ",") - - err = writeNewDataAndUpdateVsphereConfSecret(client, ctx, csiNamespace, vsphereCfg) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - revertToOriginalVsphereConf = true - defer func() { - if revertToOriginalVsphereConf { - ginkgo.By("Reverting back csi-vsphere.conf with its original vcenter user " + - "and its credentials") - vsphereCfg.Global.VCenterHostname = vCenterIP - vsphereCfg.Global.User = vCenterUser - vsphereCfg.Global.Password = vCenterPassword - vsphereCfg.Global.VCenterPort = vCenterPort - vsphereCfg.Global.Datacenters = dataCenter - err = writeNewDataAndUpdateVsphereConfSecret(client, ctx, csiNamespace, vsphereCfg) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - revertToOriginalVsphereConf = false - - ginkgo.By("Restart CSI driver") - restartSuccess, err := restartCSIDriver(ctx, client, csiNamespace, csiReplicas) - gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - }() - - framework.Logf("Wait for %v to see the CSI Pods ready running status after wrong entry in vsphere conf", pollTimeout) - time.Sleep(pollTimeout) - - framework.Logf("Verify CSI Pods status") - deploymentPods, err := client.AppsV1().Deployments(csiNamespace).Get(ctx, - vSphereCSIControllerPodNamePrefix, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - if deploymentPods.Status.AvailableReplicas == csiReplicas { - framework.Logf("CSi Pods are in ready running state") - } - - ginkgo.By("Restart CSI controller pod") - err = updateDeploymentReplicawithWait(client, 0, vSphereCSIControllerPodNamePrefix, csiNamespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = updateDeploymentReplicawithWait(client, csiReplicas, vSphereCSIControllerPodNamePrefix, csiNamespace) - if err != nil { - if strings.Contains(err.Error(), "error waiting for deployment") { - framework.Logf("csi pods are not in ready state") - } else { - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - } - - framework.Logf("Verify CSI Pods status now") - deploymentPods, err = client.AppsV1().Deployments(csiNamespace).Get(ctx, - vSphereCSIControllerPodNamePrefix, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - if deploymentPods.Status.UnavailableReplicas != csiReplicas { - framework.Logf("CSi Pods are not ready or in CLBO state with %d unavilable csi pod replica") - } - - ginkgo.By("Try to create a PVC verify that it is stuck in pending state") - storageclass, pvclaim, err := createPVCAndStorageClass(client, namespace, nil, nil, "", - allowedTopologies, "", false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err = client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Expect claim to fail as invalid storage policy is specified in Storage Class") - framework.ExpectError(fpv.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, - client, pvclaim.Namespace, pvclaim.Name, framework.Poll, framework.ClaimProvisionTimeout)) - expectedErrMsg := "waiting for a volume to be created" - err = waitForEvent(ctx, client, namespace, expectedErrMsg, pvclaim.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), fmt.Sprintf("Expected error : %q", expectedErrMsg)) - }) - -}) diff --git a/tests/e2e/multi_vc_connection.go b/tests/e2e/multi_vc_connection.go deleted file mode 100644 index 52dffd25d2..0000000000 --- a/tests/e2e/multi_vc_connection.go +++ /dev/null @@ -1,101 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2e - -import ( - "context" - "fmt" - neturl "net/url" - "strings" - - gomega "github.com/onsi/gomega" - "github.com/vmware/govmomi" - "github.com/vmware/govmomi/session" - "github.com/vmware/govmomi/vim25" - "k8s.io/kubernetes/test/e2e/framework" -) - -/* -connectMultiVC helps make a connection to a multiple vCenter Server. No actions are taken if a connection -exists and alive. Otherwise, a new client will be created. -*/ -func connectMultiVC(ctx context.Context, vs *multiVCvSphere) { - clientLock.Lock() - defer clientLock.Unlock() - if vs.multiVcClient == nil { - framework.Logf("Creating new VC session") - vs.multiVcClient = newClientForMultiVC(ctx, vs) - } - for i := 0; i < len(vs.multiVcClient); i++ { - manager := session.NewManager(vs.multiVcClient[i].Client) - userSession, err := manager.UserSession(ctx) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - if userSession != nil { - continue - } else { - framework.Logf("Current session is not valid or not authenticated, trying to logout from it") - err = vs.multiVcClient[i].Logout(ctx) - if err != nil { - framework.Logf("Ignoring the log out error: %v", err) - } - framework.Logf("Creating new client session after attempting to logout from existing session") - vs.multiVcClient = newClientForMultiVC(ctx, vs) - } - } -} - -/* -newClientForMultiVC creates a new client for vSphere connection on a multivc environment -*/ -func newClientForMultiVC(ctx context.Context, vs *multiVCvSphere) []*govmomi.Client { - var clients []*govmomi.Client - configUser := strings.Split(vs.multivcConfig.Global.User, ",") - configPwd := strings.Split(vs.multivcConfig.Global.Password, ",") - configvCenterHostname := strings.Split(vs.multivcConfig.Global.VCenterHostname, ",") - configvCenterPort := strings.Split(vs.multivcConfig.Global.VCenterPort, ",") - for i := 0; i < len(configvCenterHostname); i++ { - framework.Logf("https://%s:%s/sdk", configvCenterHostname[i], configvCenterPort[i]) - url, err := neturl.Parse(fmt.Sprintf("https://%s:%s/sdk", - configvCenterHostname[i], configvCenterPort[i])) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - url.User = neturl.UserPassword(configUser[i], configPwd[i]) - client, err := govmomi.NewClient(ctx, url, true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = client.UseServiceVersion(vsanNamespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - client.RoundTripper = vim25.Retry(client.RoundTripper, vim25.TemporaryNetworkError(roundTripperDefaultCount)) - clients = append(clients, client) - } - return clients -} - -/* -connectMultiVcCns creates a CNS client for the virtual center for a multivc environment -*/ -func connectMultiVcCns(ctx context.Context, vs *multiVCvSphere) error { - var err error - clientMutex.Lock() - defer clientMutex.Unlock() - if vs.multiVcCnsClient == nil { - vs.multiVcCnsClient = make([]*cnsClient, len(vs.multiVcClient)) - for i := 0; i < len(vs.multiVcClient); i++ { - vs.multiVcCnsClient[i], err = newCnsClient(ctx, vs.multiVcClient[i].Client) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - } - return nil -} diff --git a/tests/e2e/multi_vc_multi_replica.go b/tests/e2e/multi_vc_multi_replica.go deleted file mode 100644 index ca4fe2d268..0000000000 --- a/tests/e2e/multi_vc_multi_replica.go +++ /dev/null @@ -1,259 +0,0 @@ -/* - Copyright 2023 The Kubernetes Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package e2e - -import ( - "context" - "fmt" - "sync" - - ginkgo "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" - "golang.org/x/crypto/ssh" - v1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/kubernetes/test/e2e/framework" - fnodes "k8s.io/kubernetes/test/e2e/framework/node" - fss "k8s.io/kubernetes/test/e2e/framework/statefulset" - admissionapi "k8s.io/pod-security-admission/api" -) - -var _ = ginkgo.Describe("[csi-multi-vc-topology] Multi-VC-Replica", func() { - f := framework.NewDefaultFramework("csi-multi-vc") - f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged - var ( - client clientset.Interface - namespace string - allowedTopologies []v1.TopologySelectorLabelRequirement - sshClientConfig *ssh.ClientConfig - nimbusGeneratedK8sVmPwd string - statefulSetReplicaCount int32 - k8sVersion string - stsScaleUp bool - stsScaleDown bool - verifyTopologyAffinity bool - parallelStatefulSetCreation bool - scaleUpReplicaCount int32 - scaleDownReplicaCount int32 - ) - ginkgo.BeforeEach(func() { - var cancel context.CancelFunc - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - client = f.ClientSet - namespace = f.Namespace.Name - - multiVCbootstrap() - - stsScaleUp = true - stsScaleDown = true - verifyTopologyAffinity = true - parallelStatefulSetCreation = true - - sc, err := client.StorageV1().StorageClasses().Get(ctx, defaultNginxStorageClassName, metav1.GetOptions{}) - if err == nil && sc != nil { - gomega.Expect(client.StorageV1().StorageClasses().Delete(ctx, sc.Name, - *metav1.NewDeleteOptions(0))).NotTo(gomega.HaveOccurred()) - } - - nodeList, err := fnodes.GetReadySchedulableNodes(f.ClientSet) - framework.ExpectNoError(err, "Unable to find ready and schedulable Node") - if !(len(nodeList.Items) > 0) { - framework.Failf("Unable to find ready and schedulable Node") - } - - topologyMap := GetAndExpectStringEnvVar(topologyMap) - allowedTopologies = createAllowedTopolgies(topologyMap, topologyLength) - nimbusGeneratedK8sVmPwd = GetAndExpectStringEnvVar(nimbusK8sVmPwd) - - sshClientConfig = &ssh.ClientConfig{ - User: "root", - Auth: []ssh.AuthMethod{ - ssh.Password(nimbusGeneratedK8sVmPwd), - }, - HostKeyCallback: ssh.InsecureIgnoreHostKey(), - } - - // fetching k8s version - v, err := client.Discovery().ServerVersion() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - k8sVersion = v.Major + "." + v.Minor - - }) - - ginkgo.AfterEach(func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - ginkgo.By(fmt.Sprintf("Deleting all statefulsets in namespace: %v", namespace)) - fss.DeleteAllStatefulSets(client, namespace) - ginkgo.By(fmt.Sprintf("Deleting service nginx in namespace: %v", namespace)) - err := client.CoreV1().Services(namespace).Delete(ctx, servicename, *metav1.NewDeleteOptions(0)) - if !apierrors.IsNotFound(err) { - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - framework.Logf("Perform cleanup of any left over stale PVs") - allPvs, err := client.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - for _, pv := range allPvs.Items { - err = client.CoreV1().PersistentVolumes().Delete(ctx, pv.Name, metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }) - - /* - Verify the behaviour when CSI Provisioner, CSI Attacher, Vsphere syncer is deleted repeatedly - during workload creation - - 1. Identify the CSI-Controller-Pod where CSI Provisioner, CSI Attacher and vsphere-syncer are the leader - 2. Create SC with allowed topology set to different availability zones spread across multiple VC's - 3. Create three Statefulsets each with replica 5 - 4. While the Statefulsets is creating PVCs and Pods, kill CSI-Provisioner, CSI-attacher identified - in the step 1 - 5. Wait for some time for all the PVCs and Pods to come up - 6. Verify the PV node affinity and the nodes on which Pods have come should be appropriate - 7. Scale-up/Scale-down the Statefulset and kill the vsphere-syncer identified in the step 1 - 8. Wait for some time, Statefulset workload is patched with proper count - 9. Make sure common validation points are met on PV,PVC and POD - 10. Clean up the data - */ - - ginkgo.It("Verify behaviour when CSI-Provisioner, CSI-Attacher, Vsphere-Syncer is "+ - "deleted repeatedly during workload creation in multivc", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - sts_count := 3 - statefulSetReplicaCount = 5 - - ginkgo.By("Get current leader where CSI-Provisioner, CSI-Attacher and " + - "Vsphere-Syncer is running and find the master node IP where these containers are running") - csiProvisionerLeader, csiProvisionerControlIp, err := getK8sMasterNodeIPWhereContainerLeaderIsRunning(ctx, - client, sshClientConfig, provisionerContainerName) - framework.Logf("CSI-Provisioner is running on Leader Pod %s "+ - "which is running on master node %s", csiProvisionerLeader, csiProvisionerControlIp) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - csiAttacherLeaderleader, csiAttacherControlIp, err := getK8sMasterNodeIPWhereContainerLeaderIsRunning(ctx, - client, sshClientConfig, attacherContainerName) - framework.Logf("CSI-Attacher is running on Leader Pod %s "+ - "which is running on master node %s", csiAttacherLeaderleader, csiAttacherControlIp) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - vsphereSyncerLeader, vsphereSyncerControlIp, err := getK8sMasterNodeIPWhereContainerLeaderIsRunning(ctx, - client, sshClientConfig, syncerContainerName) - framework.Logf("Vsphere-Syncer is running on Leader Pod %s "+ - "which is running on master node %s", vsphereSyncerLeader, vsphereSyncerControlIp) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Create SC with allowed topology spread across multiple VCs") - scSpec := getVSphereStorageClassSpec(defaultNginxStorageClassName, nil, allowedTopologies, "", - "", false) - sc, err := client.StorageV1().StorageClasses().Create(ctx, scSpec, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err = client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Creating service") - service := CreateService(namespace, client) - defer func() { - deleteService(namespace, client, service) - }() - - ginkgo.By("Creating multiple StatefulSets specs in parallel") - statefulSets := createParallelStatefulSetSpec(namespace, sts_count, statefulSetReplicaCount) - - ginkgo.By("Trigger multiple StatefulSets creation in parallel. During StatefulSets " + - "creation, kill CSI-Provisioner, CSI-Attacher container in between") - var wg sync.WaitGroup - wg.Add(sts_count) - for i := 0; i < len(statefulSets); i++ { - go createParallelStatefulSets(client, namespace, statefulSets[i], statefulSetReplicaCount, &wg) - if i == 1 { - ginkgo.By("Kill CSI-Provisioner container") - err = execDockerPauseNKillOnContainer(sshClientConfig, csiProvisionerControlIp, provisionerContainerName, - k8sVersion) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - if i == 2 { - ginkgo.By("Kill CSI-Attacher container") - err = execDockerPauseNKillOnContainer(sshClientConfig, csiAttacherControlIp, attacherContainerName, - k8sVersion) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - } - wg.Wait() - - ginkgo.By("Waiting for StatefulSets Pods to be in Ready State") - err = waitForStsPodsToBeInReadyRunningState(ctx, client, namespace, statefulSets) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - deleteAllStsAndPodsPVCsInNamespace(ctx, client, namespace) - }() - - ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") - for i := 0; i < len(statefulSets); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, - statefulSets[i], namespace, allowedTopologies, parallelStatefulSetCreation, true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - ginkgo.By("Perform scaleup/scaledown operation on statefulset and verify pv and pod affinity details") - for i := 0; i < len(statefulSets); i++ { - if i == 0 { - stsScaleUp = false - scaleDownReplicaCount = 3 - framework.Logf("Scale down StatefulSet1 replica count to 3") - err = performScalingOnStatefulSetAndVerifyPvNodeAffinity(ctx, client, scaleUpReplicaCount, - scaleDownReplicaCount, statefulSets[i], parallelStatefulSetCreation, namespace, - allowedTopologies, stsScaleUp, stsScaleDown, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - if i == 1 { - scaleUpReplicaCount = 9 - stsScaleDown = false - framework.Logf("Scale up StatefulSet2 replica count to 9 and in between " + - "kill vsphere syncer container") - err = performScalingOnStatefulSetAndVerifyPvNodeAffinity(ctx, client, scaleUpReplicaCount, - scaleDownReplicaCount, statefulSets[i], parallelStatefulSetCreation, namespace, - allowedTopologies, stsScaleUp, stsScaleDown, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - framework.Logf("Kill Vsphere-Syncer container") - err = execDockerPauseNKillOnContainer(sshClientConfig, vsphereSyncerControlIp, syncerContainerName, - k8sVersion) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - } - if i == 2 { - framework.Logf("Scale up StatefulSet3 replica count to 7 and later scale down" + - "the replica count to 2") - scaleUpReplicaCount = 7 - scaleDownReplicaCount = 2 - err = performScalingOnStatefulSetAndVerifyPvNodeAffinity(ctx, client, scaleUpReplicaCount, - scaleDownReplicaCount, statefulSets[i], parallelStatefulSetCreation, namespace, - allowedTopologies, stsScaleUp, stsScaleDown, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - } - }) -}) diff --git a/tests/e2e/multi_vc_preferential_topology.go b/tests/e2e/multi_vc_preferential_topology.go deleted file mode 100644 index 3ea2765570..0000000000 --- a/tests/e2e/multi_vc_preferential_topology.go +++ /dev/null @@ -1,791 +0,0 @@ -/* - Copyright 2023 The Kubernetes Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package e2e - -import ( - "context" - "fmt" - "os" - "strconv" - "strings" - "sync" - "time" - - "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" - "golang.org/x/crypto/ssh" - v1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/kubernetes/test/e2e/framework" - fnodes "k8s.io/kubernetes/test/e2e/framework/node" - fpod "k8s.io/kubernetes/test/e2e/framework/pod" - fpv "k8s.io/kubernetes/test/e2e/framework/pv" - fss "k8s.io/kubernetes/test/e2e/framework/statefulset" - admissionapi "k8s.io/pod-security-admission/api" - - snapclient "github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned" -) - -var _ = ginkgo.Describe("[csi-multi-vc-preferential-topology] Multi-VC-Preferential-Topology", func() { - f := framework.NewDefaultFramework("multi-vc-preferential-topology") - f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged - var ( - client clientset.Interface - namespace string - preferredDatastoreChosen int - allMasterIps []string - masterIp string - preferredDatastorePaths []string - allowedTopologyRacks []string - sshClientConfig *ssh.ClientConfig - nimbusGeneratedK8sVmPwd string - allowedTopologies []v1.TopologySelectorLabelRequirement - ClusterdatastoreListVc []map[string]string - ClusterdatastoreListVc1 map[string]string - ClusterdatastoreListVc2 map[string]string - ClusterdatastoreListVc3 map[string]string - parallelStatefulSetCreation bool - stsReplicas int32 - scaleDownReplicaCount int32 - scaleUpReplicaCount int32 - stsScaleUp bool - stsScaleDown bool - verifyTopologyAffinity bool - topValStartIndex int - topValEndIndex int - topkeyStartIndex int - scParameters map[string]string - storagePolicyInVc1Vc2 string - allowedTopologyLen int - parallelPodPolicy bool - nodeAffinityToSet bool - podAntiAffinityToSet bool - snapc *snapclient.Clientset - pandoraSyncWaitTime int - err error - csiNamespace string - csiReplicas int32 - ) - - ginkgo.BeforeEach(func() { - var cancel context.CancelFunc - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - client = f.ClientSet - namespace = f.Namespace.Name - - multiVCbootstrap() - - sc, err := client.StorageV1().StorageClasses().Get(ctx, defaultNginxStorageClassName, metav1.GetOptions{}) - if err == nil && sc != nil { - gomega.Expect(client.StorageV1().StorageClasses().Delete(ctx, sc.Name, - *metav1.NewDeleteOptions(0))).NotTo(gomega.HaveOccurred()) - } - nodeList, err := fnodes.GetReadySchedulableNodes(f.ClientSet) - framework.ExpectNoError(err, "Unable to find ready and schedulable Node") - if !(len(nodeList.Items) > 0) { - framework.Failf("Unable to find ready and schedulable Node") - } - - nimbusGeneratedK8sVmPwd = GetAndExpectStringEnvVar(nimbusK8sVmPwd) - sshClientConfig = &ssh.ClientConfig{ - User: "root", - Auth: []ssh.AuthMethod{ - ssh.Password(nimbusGeneratedK8sVmPwd), - }, - HostKeyCallback: ssh.InsecureIgnoreHostKey(), - } - - stsScaleUp = true - stsScaleDown = true - verifyTopologyAffinity = true - scParameters = make(map[string]string) - storagePolicyInVc1Vc2 = GetAndExpectStringEnvVar(envStoragePolicyNameInVC1VC2) - topologyMap := GetAndExpectStringEnvVar(topologyMap) - allowedTopologies = createAllowedTopolgies(topologyMap, topologyLength) - - //Get snapshot client using the rest config - restConfig = getRestConfigClient() - snapc, err = snapclient.NewForConfig(restConfig) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - // fetching k8s master ip - allMasterIps = getK8sMasterIPs(ctx, client) - masterIp = allMasterIps[0] - - // fetching cluster details - clientIndex := 0 - clusterComputeResource, _, err = getClusterNameForMultiVC(ctx, &multiVCe2eVSphere, clientIndex) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - // fetching list of datastores available in different VCs - ClusterdatastoreListVc1, ClusterdatastoreListVc2, - ClusterdatastoreListVc3, err = getDatastoresListFromMultiVCs(masterIp, sshClientConfig, - clusterComputeResource[0], true) - ClusterdatastoreListVc = append(ClusterdatastoreListVc, ClusterdatastoreListVc1, - ClusterdatastoreListVc2, ClusterdatastoreListVc3) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - if os.Getenv(envPandoraSyncWaitTime) != "" { - pandoraSyncWaitTime, err = strconv.Atoi(os.Getenv(envPandoraSyncWaitTime)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } else { - pandoraSyncWaitTime = defaultPandoraSyncWaitTime - } - - allowedTopologyRacks = nil - for i := 0; i < len(allowedTopologies); i++ { - for j := 0; j < len(allowedTopologies[i].Values); j++ { - allowedTopologyRacks = append(allowedTopologyRacks, allowedTopologies[i].Values[j]) - } - } - - // read namespace - csiNamespace = GetAndExpectStringEnvVar(envCSINamespace) - csiDeployment, err := client.AppsV1().Deployments(csiNamespace).Get( - ctx, vSphereCSIControllerPodNamePrefix, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - csiReplicas = *csiDeployment.Spec.Replicas - - //set preferred datatsore time interval - setPreferredDatastoreTimeInterval(client, ctx, csiNamespace, csiReplicas, true) - }) - - ginkgo.AfterEach(func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - ginkgo.By(fmt.Sprintf("Deleting all statefulsets in namespace: %v", namespace)) - fss.DeleteAllStatefulSets(client, namespace) - ginkgo.By(fmt.Sprintf("Deleting service nginx in namespace: %v", namespace)) - err := client.CoreV1().Services(namespace).Delete(ctx, servicename, *metav1.NewDeleteOptions(0)) - if !apierrors.IsNotFound(err) { - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - framework.Logf("Perform preferred datastore tags cleanup after test completion") - err = deleteTagCreatedForPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks, - true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - framework.Logf("Recreate preferred datastore tags post cleanup") - err = createTagForPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks, true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - }) - - /* Testcase-1: - Add preferential tag in all the Availability zone's of VC1 and VC2 → change the preference during - execution - - 1. Create SC default parameters without any topology requirement. - 2. In each availability zone for any one datastore add preferential tag in VC1 and VC2 - 3. Create 3 statefulset with 10 replica's - 4. Wait for all the PVC to bound and pods to reach running state - 5. Verify that since the preferred datastore is available, Volume should get created on the datastores - which has the preferencce set - 6. Make sure common validation points are met on PV,PVC and POD - 7. Change the Preference in any 2 datastores - 8. Scale up the statefulset to 15 replica - 9. The volumes should get provision on the datastores which has the preference - 10. Clear the data - */ - ginkgo.It("TestTag one datastore as preferred each in VC1 and VC2 and verify it is honored", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - parallelStatefulSetCreation = true - preferredDatastoreChosen = 1 - preferredDatastorePaths = nil - sts_count := 3 - stsReplicas = 10 - var dsUrls []string - scaleUpReplicaCount = 15 - stsScaleDown = false - - ginkgo.By("Create SC with default parameters") - scSpec := getVSphereStorageClassSpec(defaultNginxStorageClassName, nil, nil, "", - "", false) - sc, err := client.StorageV1().StorageClasses().Create(ctx, scSpec, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err = client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - // choose preferred datastore - ginkgo.By("Tag preferred datastore for volume provisioning in VC1 and VC2") - for i := 0; i < 2; i++ { - paths, err := tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologies[0].Values[i], - preferredDatastoreChosen, ClusterdatastoreListVc[i], nil, true, i) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - preferredDatastorePaths = append(preferredDatastorePaths, paths...) - - // Get the length of the paths for the current iteration - pathsLen := len(paths) - - for j := 0; j < pathsLen; j++ { - // Calculate the index for ClusterdatastoreListVc based on the current iteration - index := i + j - - if val, ok := ClusterdatastoreListVc[index][paths[j]]; ok { - dsUrls = append(dsUrls, val) - } - } - } - - framework.Logf("Waiting for %v for preferred datastore to get refreshed in the environment", - preferredDatastoreTimeOutInterval) - time.Sleep(preferredDatastoreTimeOutInterval) - - ginkgo.By("Create service") - service := CreateService(namespace, client) - defer func() { - deleteService(namespace, client, service) - }() - - ginkgo.By("Create 3 statefulset with 10 replicas") - statefulSets := createParallelStatefulSetSpec(namespace, sts_count, stsReplicas) - var wg sync.WaitGroup - wg.Add(sts_count) - for i := 0; i < len(statefulSets); i++ { - go createParallelStatefulSets(client, namespace, statefulSets[i], - stsReplicas, &wg) - - } - wg.Wait() - - ginkgo.By("Waiting for StatefulSets Pods to be in Ready State") - err = waitForStsPodsToBeInReadyRunningState(ctx, client, namespace, statefulSets) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - deleteAllStsAndPodsPVCsInNamespace(ctx, client, namespace) - }() - - ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") - for i := 0; i < len(statefulSets); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulSets[i], - namespace, allowedTopologies, parallelStatefulSetCreation, true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - ginkgo.By("Verify volume is provisioned on the preferred datatsore") - for i := 0; i < len(statefulSets); i++ { - err = verifyVolumeProvisioningForStatefulSet(ctx, client, statefulSets[i], namespace, - preferredDatastorePaths, nil, true, true, true, dsUrls) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - ginkgo.By("Remove preferred datatsore tag which is chosen for volume provisioning") - for i := 0; i < len(preferredDatastorePaths); i++ { - err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[i], - allowedTopologies[0].Values[i], true, i) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - var preferredDatastorePathsNew []string - ginkgo.By("Tag new preferred datastore for volume provisioning in VC1 and VC2") - for i := 0; i < 2; i++ { - paths, err := tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologies[0].Values[i], - preferredDatastoreChosen, ClusterdatastoreListVc[i], preferredDatastorePaths, true, i) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - preferredDatastorePathsNew = append(preferredDatastorePathsNew, paths...) - pathsLen := len(paths) - for j := 0; j < pathsLen; j++ { - index := i + j - if val, ok := ClusterdatastoreListVc[index][paths[j]]; ok { - dsUrls = append(dsUrls, val) - } - } - } - preferredDatastorePaths = append(preferredDatastorePaths, preferredDatastorePathsNew...) - defer func() { - ginkgo.By("Remove preferred datastore tag") - for i := 0; i < len(preferredDatastorePathsNew); i++ { - err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePathsNew[i], - allowedTopologies[0].Values[i], true, i) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }() - - ginkgo.By("Perform scaleup/scaledown operation on statefulsets and " + - "verify pv affinity and pod affinity") - for i := 0; i < len(statefulSets); i++ { - err = performScalingOnStatefulSetAndVerifyPvNodeAffinity(ctx, client, scaleUpReplicaCount, - scaleDownReplicaCount, statefulSets[i], parallelStatefulSetCreation, namespace, - allowedTopologies, stsScaleUp, stsScaleDown, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - ginkgo.By("Verify volume is provisioned on the preferred datatsore") - for i := 0; i < len(statefulSets); i++ { - err = verifyVolumeProvisioningForStatefulSet(ctx, client, statefulSets[i], namespace, - preferredDatastorePaths, nil, true, true, true, dsUrls) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }) - - /* Testcase-2: - Create SC with storage policy available in VC1 and VC2, set the preference in VC1 datastore only - - Steps// - 1. Create storage policy with same name on both VC1 and VC2 - 2. Add preference tag on the datastore which is on VC1 only - 3. Create statefulset using the above policy - 4. Since the preference tag is added in VC1, volume provisioning should happned on VC1's datastore only - [no, first preference will be given to Storage Policy mentioned in the Storage Class] - 5. Make sure common validation points are met on PV,PVC and POD - 6. Reboot VC1 - 7. Scale up the stateful set to replica 15 → What should be the behaviour here - 8. Since the VC1 is presently in reboot state, new volumes should start coming up on VC2 - Once VC1 is up again the datastore preference should take preference - [no, until all VCs comes up PVC provision will be stuck in Pending state] - 9. Verify the node affinity on all PV's - 10. Make sure POD has come up on appropriate nodes . - 11. Clean up the data - */ - - ginkgo.It("Create SC with storage policy available in VC1 and VC2 and set the "+ - "preference in VC1 datastore only", func() { - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - /* here we are considering storage policy of VC1 and VC2 and the allowed topology is k8s-zone -> zone-1 - in case of 2-VC setup and 3-VC setup - */ - - stsReplicas = 1 - scParameters[scParamStoragePolicyName] = storagePolicyInVc1Vc2 - topValStartIndex = 0 - topValEndIndex = 2 - var dsUrls []string - stsScaleDown = false - scaleUpReplicaCount = 7 - var multiVcClientIndex = 0 - preferredDatastoreChosen = 1 - preferredDatastorePaths = nil - - datastoreURLVC1 := GetAndExpectStringEnvVar(envPreferredDatastoreUrlVC1) - datastoreURLVC2 := GetAndExpectStringEnvVar(envPreferredDatastoreUrlVC2) - - /* - fetching datstore details passed in the storage policy - Note: Since storage policy is specified in the SC, the first preference for volume provisioning - will be given to the datastore given in the storage profile and second preference will then be - given to the preferred datastore chosen - */ - for _, vCenterList := range ClusterdatastoreListVc { - for _, url := range vCenterList { - if url == datastoreURLVC1 || url == datastoreURLVC2 { - dsUrls = append(dsUrls, url) - } - } - } - - ginkgo.By("Set specific allowed topology") - allowedTopologies = setSpecificAllowedTopology(allowedTopologies, topkeyStartIndex, topValStartIndex, - topValEndIndex) - - ginkgo.By("Create StorageClass with storage policy specified") - scSpec := getVSphereStorageClassSpec(defaultNginxStorageClassName, scParameters, allowedTopologies, "", - "", false) - sc, err := client.StorageV1().StorageClasses().Create(ctx, scSpec, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err = client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - // choose preferred datastore - ginkgo.By("Tag preferred datastore for volume provisioning in VC1") - preferredDatastorePaths, err := tagPreferredDatastore(masterIp, sshClientConfig, - allowedTopologies[0].Values[0], - preferredDatastoreChosen, ClusterdatastoreListVc[0], nil, true, multiVcClientIndex) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pathsLen := len(preferredDatastorePaths) - for j := 0; j < pathsLen; j++ { - if val, ok := ClusterdatastoreListVc[0][preferredDatastorePaths[j]]; ok { - dsUrls = append(dsUrls, val) - } - } - defer func() { - ginkgo.By("Remove preferred datastore tag") - for i := 0; i < len(preferredDatastorePaths); i++ { - err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[i], - allowedTopologies[0].Values[0], true, i) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }() - - framework.Logf("Waiting for %v for preferred datastore to get refreshed in the environment", - preferredDatastoreTimeOutInterval) - time.Sleep(preferredDatastoreTimeOutInterval) - - ginkgo.By("Create StatefulSet and verify pv affinity and pod affinity details") - service, statefulset, err := createStafeulSetAndVerifyPVAndPodNodeAffinty(ctx, client, namespace, - parallelPodPolicy, stsReplicas, nodeAffinityToSet, allowedTopologies, allowedTopologyLen, - podAntiAffinityToSet, parallelStatefulSetCreation, false, "", "", sc, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - deleteAllStsAndPodsPVCsInNamespace(ctx, client, namespace) - deleteService(namespace, client, service) - }() - - ginkgo.By("Verify volume is provisioned on the preferred datatsore") - err = verifyVolumeProvisioningForStatefulSet(ctx, client, statefulset, namespace, - preferredDatastorePaths, nil, false, false, true, dsUrls) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Rebooting VC") - vCenterHostname := strings.Split(multiVCe2eVSphere.multivcConfig.Global.VCenterHostname, ",") - vcAddress := vCenterHostname[0] + ":" + sshdPort - framework.Logf("vcAddress - %s ", vcAddress) - err = invokeVCenterReboot(vcAddress) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = waitForHostToBeUp(vCenterHostname[0]) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By("Done with reboot") - - essentialServices := []string{spsServiceName, vsanhealthServiceName, vpxdServiceName} - checkVcenterServicesRunning(ctx, vcAddress, essentialServices) - - //After reboot - multiVCbootstrap() - - ginkgo.By("Perform scaleup/scaledown operation on statefulsets and " + - "verify pv affinity and pod affinity") - err = performScalingOnStatefulSetAndVerifyPvNodeAffinity(ctx, client, scaleUpReplicaCount, - scaleDownReplicaCount, statefulset, parallelStatefulSetCreation, namespace, - allowedTopologies, stsScaleUp, stsScaleDown, verifyTopologyAffinity) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Verify volume is provisioned on the preferred datatsore") - err = verifyVolumeProvisioningForStatefulSet(ctx, client, statefulset, namespace, - preferredDatastorePaths, nil, false, false, true, dsUrls) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - - /* Testcase-3: - Create/Restore Snapshot of PVC using single datastore preference - - Steps// - - 1. Assign preferential tag to any one datastore under any one VC - 2. Create SC with allowed topology set to all the volumes - [here, for verification of snapshot, considering single allowed topology of VC3 ] - 3. Create PVC-1 with the above SC - 4. Wait for PVC-1 to reach Bound state. - 5. Describe PV-1 and verify node affinity details - 6. Verify volume should be provisioned on the selected preferred datastore - 7. Create SnapshotClass, Snapshot of PVC-1. - 8. Verify snapshot state. It should be in ready-to-use state. - 9. Verify snapshot should be created on the preferred datastore. - 10. Restore snapshot to create PVC-2 - 11. Wait for PVC-2 to reach Bound state. - 12. Describe PV-2 and verify node affinity details - 13. Verify volume should be provisioned on the selected preferred datastore - 14. Create Pod from restored PVC-2. - 15. Make sure common validation points are met on PV,PVC and POD - 16. Make sure POD is running on the same node as mentioned in the node affinity details. - 17. Perform Cleanup. Delete Snapshot, Pod, PVC, SC - 18. Remove datastore preference tags as part of cleanup. - */ - - ginkgo.It("Assign preferred datatsore to any one VC and verify create restore snapshot", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - preferredDatastoreChosen = 1 - preferredDatastorePaths = nil - var dsUrls []string - var multiVcClientIndex = 2 - topValStartIndex = 2 - topValEndIndex = 3 - - // Considering k8s-zone -> zone-3 i.e. VC3 allowed topology - ginkgo.By("Set specific allowed topology") - allowedTopologies = setSpecificAllowedTopology(allowedTopologies, topkeyStartIndex, topValStartIndex, - topValEndIndex) - - // choose preferred datastore - ginkgo.By("Tag preferred datastore for volume provisioning in VC3") - preferredDatastorePaths, err = tagPreferredDatastore(masterIp, sshClientConfig, - allowedTopologies[0].Values[0], - preferredDatastoreChosen, ClusterdatastoreListVc[2], nil, true, multiVcClientIndex) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pathsLen := len(preferredDatastorePaths) - for j := 0; j < pathsLen; j++ { - if val, ok := ClusterdatastoreListVc[2][preferredDatastorePaths[j]]; ok { - dsUrls = append(dsUrls, val) - } - } - defer func() { - ginkgo.By("Remove preferred datastore tag") - err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[0], - allowedTopologies[0].Values[0], true, multiVcClientIndex) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - framework.Logf("Waiting for %v for preferred datastore to get refreshed in the environment", - preferredDatastoreTimeOutInterval) - time.Sleep(preferredDatastoreTimeOutInterval) - - ginkgo.By("Create StorageClass and PVC") - storageclass, pvclaim, err := createPVCAndStorageClass(client, namespace, nil, - nil, diskSize, allowedTopologies, "", false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err = client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, - *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - // Wait for PVC to be in Bound phase - persistentvolumes, err := fpv.WaitForPVClaimBoundPhase(client, []*v1.PersistentVolumeClaim{pvclaim}, - framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - volHandle := persistentvolumes[0].Spec.CSI.VolumeHandle - gomega.Expect(volHandle).NotTo(gomega.BeEmpty()) - defer func() { - err = fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = multiVCe2eVSphere.waitForCNSVolumeToBeDeletedInMultiVC(volHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pvclaim = nil - }() - - ginkgo.By(fmt.Sprintf("Invoking QueryCNSVolumeWithResult with VolumeID: %s", volHandle)) - queryResult, err := multiVCe2eVSphere.queryCNSVolumeWithResultInMultiVC(volHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(queryResult.Volumes).ShouldNot(gomega.BeEmpty()) - gomega.Expect(queryResult.Volumes[0].VolumeId.Id).To(gomega.Equal(volHandle)) - - ginkgo.By("Create volume snapshot class, volume snapshot") - volumeSnapshot, volumeSnapshotClass, snapshotId := createSnapshotClassAndVolSnapshot(ctx, snapc, namespace, - pvclaim, volHandle, false, true) - defer func() { - ginkgo.By("Perform cleanup of snapshot created") - performCleanUpForSnapshotCreated(ctx, snapc, namespace, volHandle, volumeSnapshot, snapshotId, - volumeSnapshotClass, pandoraSyncWaitTime, true) - }() - - ginkgo.By("Create PVC from snapshot") - pvcSpec := getPersistentVolumeClaimSpecWithDatasource(namespace, diskSize, storageclass, nil, - v1.ReadWriteOnce, volumeSnapshot.Name, snapshotapigroup) - pvclaim2, err := fpv.CreatePVC(client, namespace, pvcSpec) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - persistentvolumes2, err := fpv.WaitForPVClaimBoundPhase(client, - []*v1.PersistentVolumeClaim{pvclaim2}, framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - volHandle2 := persistentvolumes2[0].Spec.CSI.VolumeHandle - gomega.Expect(volHandle2).NotTo(gomega.BeEmpty()) - defer func() { - err := fpv.DeletePersistentVolumeClaim(client, pvclaim2.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = multiVCe2eVSphere.waitForCNSVolumeToBeDeletedInMultiVC(volHandle2) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Creating pod") - pod, err := createPod(client, namespace, nil, []*v1.PersistentVolumeClaim{pvclaim2}, false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - ginkgo.By(fmt.Sprintf("Deleting the pod %s in namespace %s", pod.Name, namespace)) - err = fpod.DeletePodWithWait(client, pod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Verify volume is detached from the node") - isDiskDetached, err := multiVCe2eVSphere.waitForVolumeDetachedFromNodeInMultiVC(client, - volHandle2, pod.Spec.NodeName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(isDiskDetached).To(gomega.BeTrue(), - fmt.Sprintf("Volume %q is not detached from the node %q", volHandle2, - pod.Spec.NodeName)) - }() - - // verifying volume provisioning - ginkgo.By("Verify volume is provisioned on the preferred datatsore") - verifyVolumeProvisioningForStandalonePods(ctx, client, pod, namespace, preferredDatastorePaths, - ClusterdatastoreListVc[2], true, dsUrls) - - ginkgo.By("Verify PV node affinity and that the PODS are running on " + - "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStandalonePodLevel5(ctx, client, pod, - namespace, allowedTopologies, true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - - /* Testcase-4 - - Create/Restore Snapshot of PVC, datastore preference change - - 1. Assign preferential tag to any one datastore under any one VC - 2. Create SC with allowed topology set to all the volumes - 3. Create PVC-1 with the above SC - 4. Wait for PVC-1 to reach Bound state. - 5. Describe PV-1 and verify node affinity details - 6. Verify volume should be provisioned on the selected preferred datastore - 7. Make sure common validation points are met on PV,PVC and POD - 8. Change datastore preference(ex- from NFS-2 to vSAN-2) - 9. Create SnapshotClass, Snapshot of PVC-1 - 10. Verify snapshot state. It should be in ready-to-use state. - 11. Restore snapshot to create PVC-2 - 12. PVC-2 should get stuck in Pending state and proper error message should be displayed. - 13. Perform Cleanup. Delete Snapshot, Pod, PVC, SC - 14. Remove datastore preference tags as part of cleanup. - */ - - ginkgo.It("Assign preferred datatsore to any one VC and verify create restore snapshot "+ - "and later change datastore preference", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - preferredDatastoreChosen = 1 - preferredDatastorePaths = nil - var dsUrls []string - var multiVcClientIndex = 2 - topValStartIndex = 2 - topValEndIndex = 3 - - // Considering k8s-zone -> zone-3 i.e. VC3 allowed topology - ginkgo.By("Set specific allowed topology") - allowedTopologies = setSpecificAllowedTopology(allowedTopologies, topkeyStartIndex, topValStartIndex, - topValEndIndex) - - // choose preferred datastore - ginkgo.By("Tag preferred datastore for volume provisioning in VC3") - preferredDatastorePaths, err = tagPreferredDatastore(masterIp, sshClientConfig, - allowedTopologies[0].Values[0], - preferredDatastoreChosen, ClusterdatastoreListVc[2], nil, true, multiVcClientIndex) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pathsLen := len(preferredDatastorePaths) - for j := 0; j < pathsLen; j++ { - if val, ok := ClusterdatastoreListVc[2][preferredDatastorePaths[j]]; ok { - dsUrls = append(dsUrls, val) - } - } - defer func() { - ginkgo.By("Remove preferred datastore tag") - err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[0], - allowedTopologies[0].Values[0], true, multiVcClientIndex) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - framework.Logf("Waiting for %v for preferred datastore to get refreshed in the environment", - preferredDatastoreTimeOutInterval) - time.Sleep(preferredDatastoreTimeOutInterval) - - ginkgo.By("Create StorageClass and PVC") - storageclass, pvclaim, err := createPVCAndStorageClass(client, namespace, nil, - nil, diskSize, allowedTopologies, "", false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, - *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - // Wait for PVC to be in Bound phase - persistentvolumes, err := fpv.WaitForPVClaimBoundPhase(client, []*v1.PersistentVolumeClaim{pvclaim}, - framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - volHandle := persistentvolumes[0].Spec.CSI.VolumeHandle - gomega.Expect(volHandle).NotTo(gomega.BeEmpty()) - defer func() { - err := fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = multiVCe2eVSphere.waitForCNSVolumeToBeDeletedInMultiVC(volHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pvclaim = nil - }() - - ginkgo.By(fmt.Sprintf("Invoking QueryCNSVolumeWithResult with VolumeID: %s", volHandle)) - queryResult, err := multiVCe2eVSphere.queryCNSVolumeWithResultInMultiVC(volHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(queryResult.Volumes).ShouldNot(gomega.BeEmpty()) - gomega.Expect(queryResult.Volumes[0].VolumeId.Id).To(gomega.Equal(volHandle)) - - ginkgo.By("Creating pod") - pod, err := createPod(client, namespace, nil, []*v1.PersistentVolumeClaim{pvclaim}, false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - ginkgo.By(fmt.Sprintf("Deleting the pod %s in namespace %s", pod.Name, namespace)) - err = fpod.DeletePodWithWait(client, pod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Verify volume is detached from the node") - isDiskDetached, err := multiVCe2eVSphere.waitForVolumeDetachedFromNodeInMultiVC(client, - volHandle, pod.Spec.NodeName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(isDiskDetached).To(gomega.BeTrue(), - fmt.Sprintf("Volume %q is not detached from the node %q", volHandle, - pod.Spec.NodeName)) - }() - - // verifying volume provisioning - ginkgo.By("Verify volume is provisioned on the preferred datatsore") - verifyVolumeProvisioningForStandalonePods(ctx, client, pod, namespace, preferredDatastorePaths, - ClusterdatastoreListVc[2], true, dsUrls) - - ginkgo.By("Remove preferred datastore tag chosen for volume provisioning") - err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[0], - allowedTopologyRacks[2], true, multiVcClientIndex) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - // choose preferred datastore - ginkgo.By("Tag preferred datastore for volume provisioning in VC3") - preferredDatastorePaths, err = tagPreferredDatastore(masterIp, sshClientConfig, - allowedTopologies[0].Values[0], preferredDatastoreChosen, ClusterdatastoreListVc[2], - preferredDatastorePaths, true, multiVcClientIndex) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - framework.Logf("Waiting for %v for preferred datastore to get refreshed in the environment", - preferredDatastoreTimeOutInterval) - time.Sleep(preferredDatastoreTimeOutInterval) - - ginkgo.By("Create volume snapshot class, volume snapshot") - volumeSnapshot, volumeSnapshotClass, snapshotId := createSnapshotClassAndVolSnapshot(ctx, snapc, namespace, - pvclaim, volHandle, false, true) - defer func() { - ginkgo.By("Perform cleanup of snapshot created") - performCleanUpForSnapshotCreated(ctx, snapc, namespace, volHandle, volumeSnapshot, snapshotId, - volumeSnapshotClass, pandoraSyncWaitTime, true) - }() - - ginkgo.By("Create PVC from snapshot") - pvcSpec := getPersistentVolumeClaimSpecWithDatasource(namespace, diskSize, storageclass, nil, - v1.ReadWriteOnce, volumeSnapshot.Name, snapshotapigroup) - pvclaim2, err := fpv.CreatePVC(client, namespace, pvcSpec) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Expect claim to fail provisioning volume within the topology") - err = fpv.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, - client, pvclaim2.Namespace, pvclaim2.Name, framework.Poll, framework.ClaimProvisionTimeout) - gomega.Expect(err).To(gomega.HaveOccurred()) - expectedErrMsg := "failed to get the compatible shared datastore for create volume from snapshot" - err = waitForEvent(ctx, client, namespace, expectedErrMsg, pvclaim2.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), fmt.Sprintf("Expected error : %q", expectedErrMsg)) - defer func() { - err = fpv.DeletePersistentVolumeClaim(client, pvclaim2.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - }) -}) diff --git a/tests/e2e/multi_vc_utils.go b/tests/e2e/multi_vc_utils.go deleted file mode 100644 index f9ce6f5939..0000000000 --- a/tests/e2e/multi_vc_utils.go +++ /dev/null @@ -1,1016 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2e - -import ( - "context" - "errors" - "fmt" - "strconv" - "strings" - "time" - - ginkgo "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" - "github.com/vmware/govmomi/cns" - cnsmethods "github.com/vmware/govmomi/cns/methods" - cnstypes "github.com/vmware/govmomi/cns/types" - "github.com/vmware/govmomi/find" - "github.com/vmware/govmomi/object" - vim25types "github.com/vmware/govmomi/vim25/types" - "golang.org/x/crypto/ssh" - - appsv1 "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" - storagev1 "k8s.io/api/storage/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - labels "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/wait" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/kubernetes/test/e2e/framework" - fpod "k8s.io/kubernetes/test/e2e/framework/pod" - fssh "k8s.io/kubernetes/test/e2e/framework/ssh" - fss "k8s.io/kubernetes/test/e2e/framework/statefulset" -) - -/* -createCustomisedStatefulSets util methods creates statefulset as per the user's -specific requirement and returns the customised statefulset -*/ -func createCustomisedStatefulSets(client clientset.Interface, namespace string, - isParallelPodMgmtPolicy bool, replicas int32, nodeAffinityToSet bool, - allowedTopologies []v1.TopologySelectorLabelRequirement, allowedTopologyLen int, - podAntiAffinityToSet bool, modifyStsSpec bool, stsName string, - accessMode v1.PersistentVolumeAccessMode, sc *storagev1.StorageClass) *appsv1.StatefulSet { - framework.Logf("Preparing StatefulSet Spec") - statefulset := GetStatefulSetFromManifest(namespace) - - if accessMode == "" { - // If accessMode is not specified, set the default accessMode. - accessMode = v1.ReadWriteOnce - } - - if modifyStsSpec { - statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Annotations["volume.beta.kubernetes.io/storage-class"] = sc.Name - statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1].Spec.AccessModes[0] = - accessMode - statefulset.Name = stsName - statefulset.Spec.Template.Labels["app"] = statefulset.Name - statefulset.Spec.Selector.MatchLabels["app"] = statefulset.Name - } - if nodeAffinityToSet { - nodeSelectorTerms := getNodeSelectorTerms(allowedTopologies) - statefulset.Spec.Template.Spec.Affinity = new(v1.Affinity) - statefulset.Spec.Template.Spec.Affinity.NodeAffinity = new(v1.NodeAffinity) - statefulset.Spec.Template.Spec.Affinity.NodeAffinity. - RequiredDuringSchedulingIgnoredDuringExecution = new(v1.NodeSelector) - statefulset.Spec.Template.Spec.Affinity.NodeAffinity. - RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = nodeSelectorTerms - } - if podAntiAffinityToSet { - statefulset.Spec.Template.Spec.Affinity = &v1.Affinity{ - PodAntiAffinity: &v1.PodAntiAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ - { - LabelSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "key": "app", - }, - }, - TopologyKey: "topology.kubernetes.io/zone", - }, - }, - }, - } - - } - if isParallelPodMgmtPolicy { - statefulset.Spec.PodManagementPolicy = appsv1.ParallelPodManagement - } - statefulset.Spec.Replicas = &replicas - - framework.Logf("Creating statefulset") - CreateStatefulSet(namespace, statefulset, client) - - framework.Logf("Wait for StatefulSet pods to be in up and running state") - fss.WaitForStatusReadyReplicas(client, statefulset, replicas) - gomega.Expect(fss.CheckMount(client, statefulset, mountPath)).NotTo(gomega.HaveOccurred()) - ssPodsBeforeScaleDown := fss.GetPodList(client, statefulset) - gomega.Expect(ssPodsBeforeScaleDown.Items).NotTo(gomega.BeEmpty(), - fmt.Sprintf("Unable to get list of Pods from the Statefulset: %v", statefulset.Name)) - gomega.Expect(len(ssPodsBeforeScaleDown.Items) == int(replicas)).To(gomega.BeTrue(), - "Number of Pods in the statefulset should match with number of replicas") - - return statefulset -} - -/* -setSpecificAllowedTopology returns topology map with specific topology fields and values -in a multi-vc setup -*/ -func setSpecificAllowedTopology(allowedTopologies []v1.TopologySelectorLabelRequirement, - topkeyStartIndex int, startIndex int, endIndex int) []v1.TopologySelectorLabelRequirement { - var allowedTopologiesMap []v1.TopologySelectorLabelRequirement - specifiedAllowedTopology := v1.TopologySelectorLabelRequirement{ - Key: allowedTopologies[topkeyStartIndex].Key, - Values: allowedTopologies[topkeyStartIndex].Values[startIndex:endIndex], - } - allowedTopologiesMap = append(allowedTopologiesMap, specifiedAllowedTopology) - - return allowedTopologiesMap -} - -/* -If we have multiple statefulsets, deployment Pods, PVCs/PVs created on a given namespace and for performing -cleanup of these multiple sts creation, deleteAllStsAndPodsPVCsInNamespace is used -*/ -func deleteAllStsAndPodsPVCsInNamespace(ctx context.Context, c clientset.Interface, ns string) { - StatefulSetPoll := 10 * time.Second - StatefulSetTimeout := 10 * time.Minute - ssList, err := c.AppsV1().StatefulSets(ns).List(context.TODO(), - metav1.ListOptions{LabelSelector: labels.Everything().String()}) - framework.ExpectNoError(err) - errList := []string{} - for i := range ssList.Items { - ss := &ssList.Items[i] - var err error - if ss, err = scaleStatefulSetPods(c, ss, 0); err != nil { - errList = append(errList, fmt.Sprintf("%v", err)) - } - fss.WaitForStatusReplicas(c, ss, 0) - framework.Logf("Deleting statefulset %v", ss.Name) - if err := c.AppsV1().StatefulSets(ss.Namespace).Delete(context.TODO(), ss.Name, - metav1.DeleteOptions{OrphanDependents: new(bool)}); err != nil { - errList = append(errList, fmt.Sprintf("%v", err)) - } - } - pvNames := sets.NewString() - pvcPollErr := wait.PollImmediate(StatefulSetPoll, StatefulSetTimeout, func() (bool, error) { - pvcList, err := c.CoreV1().PersistentVolumeClaims(ns).List(context.TODO(), - metav1.ListOptions{LabelSelector: labels.Everything().String()}) - if err != nil { - framework.Logf("WARNING: Failed to list pvcs, retrying %v", err) - return false, nil - } - for _, pvc := range pvcList.Items { - pvNames.Insert(pvc.Spec.VolumeName) - framework.Logf("Deleting pvc: %v with volume %v", pvc.Name, pvc.Spec.VolumeName) - if err := c.CoreV1().PersistentVolumeClaims(ns).Delete(context.TODO(), pvc.Name, - metav1.DeleteOptions{}); err != nil { - return false, nil - } - } - return true, nil - }) - if pvcPollErr != nil { - errList = append(errList, "Timeout waiting for pvc deletion.") - } - - pollErr := wait.PollImmediate(StatefulSetPoll, StatefulSetTimeout, func() (bool, error) { - pvList, err := c.CoreV1().PersistentVolumes().List(context.TODO(), - metav1.ListOptions{LabelSelector: labels.Everything().String()}) - if err != nil { - framework.Logf("WARNING: Failed to list pvs, retrying %v", err) - return false, nil - } - waitingFor := []string{} - for _, pv := range pvList.Items { - if pvNames.Has(pv.Name) { - waitingFor = append(waitingFor, fmt.Sprintf("%v: %+v", pv.Name, pv.Status)) - } - } - if len(waitingFor) == 0 { - return true, nil - } - framework.Logf("Still waiting for pvs of statefulset to disappear:\n%v", strings.Join(waitingFor, "\n")) - return false, nil - }) - if pollErr != nil { - errList = append(errList, "Timeout waiting for pv provisioner to delete pvs, this might mean the test leaked pvs.") - - } - if len(errList) != 0 { - framework.ExpectNoError(fmt.Errorf("%v", strings.Join(errList, "\n"))) - } - - framework.Logf("Deleting Deployment Pods and its PVCs") - depList, err := c.AppsV1().Deployments(ns).List( - ctx, metav1.ListOptions{LabelSelector: labels.Everything().String()}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - for _, deployment := range depList.Items { - dep := &deployment - err = updateDeploymentReplicawithWait(c, 0, dep.Name, ns) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - deletePolicy := metav1.DeletePropagationForeground - err = c.AppsV1().Deployments(ns).Delete(ctx, dep.Name, metav1.DeleteOptions{PropagationPolicy: &deletePolicy}) - if err != nil { - if apierrors.IsNotFound(err) { - return - } else { - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - } - } -} - -/* -verifyVolumeMetadataInCNSForMultiVC verifies container volume metadata is matching the -one is CNS cache on a multivc environment. -*/ -func verifyVolumeMetadataInCNSForMultiVC(vs *multiVCvSphere, volumeID string, - PersistentVolumeClaimName string, PersistentVolumeName string, - PodName string, Labels ...vim25types.KeyValue) error { - queryResult, err := vs.queryCNSVolumeWithResultInMultiVC(volumeID) - if err != nil { - return err - } - gomega.Expect(queryResult.Volumes).ShouldNot(gomega.BeEmpty()) - if len(queryResult.Volumes) != 1 || queryResult.Volumes[0].VolumeId.Id != volumeID { - return fmt.Errorf("failed to query cns volume %s", volumeID) - } - for _, metadata := range queryResult.Volumes[0].Metadata.EntityMetadata { - kubernetesMetadata := metadata.(*cnstypes.CnsKubernetesEntityMetadata) - if kubernetesMetadata.EntityType == "POD" && kubernetesMetadata.EntityName != PodName { - return fmt.Errorf("entity Pod with name %s not found for volume %s", PodName, volumeID) - } else if kubernetesMetadata.EntityType == "PERSISTENT_VOLUME" && - kubernetesMetadata.EntityName != PersistentVolumeName { - return fmt.Errorf("entity PV with name %s not found for volume %s", PersistentVolumeName, volumeID) - } else if kubernetesMetadata.EntityType == "PERSISTENT_VOLUME_CLAIM" && - kubernetesMetadata.EntityName != PersistentVolumeClaimName { - return fmt.Errorf("entity PVC with name %s not found for volume %s", PersistentVolumeClaimName, volumeID) - } - } - labelMap := make(map[string]string) - for _, e := range queryResult.Volumes[0].Metadata.EntityMetadata { - if e == nil { - continue - } - if e.GetCnsEntityMetadata().Labels == nil { - continue - } - for _, al := range e.GetCnsEntityMetadata().Labels { - labelMap[al.Key] = al.Value - } - for _, el := range Labels { - if val, ok := labelMap[el.Key]; ok { - gomega.Expect(el.Value == val).To(gomega.BeTrue(), - fmt.Sprintf("Actual label Value of the statically provisioned PV is %s but expected is %s", - val, el.Value)) - } else { - return fmt.Errorf("label(%s:%s) is expected in the provisioned PV but its not found", el.Key, el.Value) - } - } - } - ginkgo.By(fmt.Sprintf("successfully verified metadata of the volume %q", volumeID)) - return nil -} - -// govc login cmd for multivc setups -func govcLoginCmdForMultiVC(i int) string { - configUser := strings.Split(multiVCe2eVSphere.multivcConfig.Global.User, ",") - configPwd := strings.Split(multiVCe2eVSphere.multivcConfig.Global.Password, ",") - configvCenterHostname := strings.Split(multiVCe2eVSphere.multivcConfig.Global.VCenterHostname, ",") - configvCenterPort := strings.Split(multiVCe2eVSphere.multivcConfig.Global.VCenterPort, ",") - - loginCmd := "export GOVC_INSECURE=1;" - loginCmd += fmt.Sprintf("export GOVC_URL='https://%s:%s@%s:%s';", - configUser[i], configPwd[i], configvCenterHostname[i], configvCenterPort[i]) - return loginCmd -} - -/*deletes storage profile deletes the storage profile*/ -func deleteStorageProfile(masterIp string, sshClientConfig *ssh.ClientConfig, - storagePolicyName string, clientIndex int) error { - removeStoragePolicy := govcLoginCmdForMultiVC(clientIndex) + - "govc storage.policy.rm " + storagePolicyName - framework.Logf("Remove storage policy: %s ", removeStoragePolicy) - removeStoragePolicytRes, err := sshExec(sshClientConfig, masterIp, removeStoragePolicy) - if err != nil && removeStoragePolicytRes.Code != 0 { - fssh.LogResult(removeStoragePolicytRes) - return fmt.Errorf("couldn't execute command: %s on host: %v , error: %s", - removeStoragePolicy, masterIp, err) - } - return nil -} - -/*deletes storage profile deletes the storage profile*/ -func createStorageProfile(masterIp string, sshClientConfig *ssh.ClientConfig, - storagePolicyName string, clientIndex int) error { - createStoragePolicy := govcLoginCmdForMultiVC(clientIndex) + - "govc storage.policy.create -category=shared-cat-todelete1 -tag=shared-tag-todelete1 " + storagePolicyName - framework.Logf("Create storage policy: %s ", createStoragePolicy) - createStoragePolicytRes, err := sshExec(sshClientConfig, masterIp, createStoragePolicy) - if err != nil && createStoragePolicytRes.Code != 0 { - fssh.LogResult(createStoragePolicytRes) - return fmt.Errorf("couldn't execute command: %s on host: %v , error: %s", - createStoragePolicy, masterIp, err) - } - return nil -} - -/* -performScalingOnStatefulSetAndVerifyPvNodeAffinity accepts 3 bool values - one for scaleup, -second for scale down and third bool value to check node and pod topology affinites -*/ -func performScalingOnStatefulSetAndVerifyPvNodeAffinity(ctx context.Context, client clientset.Interface, - scaleUpReplicaCount int32, scaleDownReplicaCount int32, statefulset *appsv1.StatefulSet, - parallelStatefulSetCreation bool, namespace string, - allowedTopologies []v1.TopologySelectorLabelRequirement, stsScaleUp bool, stsScaleDown bool, - verifyTopologyAffinity bool) error { - - if stsScaleDown { - framework.Logf("Scale down statefulset replica") - err := scaleDownStatefulSetPod(ctx, client, statefulset, namespace, scaleDownReplicaCount, - parallelStatefulSetCreation, true) - if err != nil { - return fmt.Errorf("error scaling down statefulset: %v", err) - } - } - - if stsScaleUp { - framework.Logf("Scale up statefulset replica") - err := scaleUpStatefulSetPod(ctx, client, statefulset, namespace, scaleUpReplicaCount, - parallelStatefulSetCreation, true) - if err != nil { - return fmt.Errorf("error scaling up statefulset: %v", err) - } - } - - if verifyTopologyAffinity { - framework.Logf("Verify PV node affinity and that the PODS are running on appropriate node") - err := verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, - namespace, allowedTopologies, parallelStatefulSetCreation, true) - if err != nil { - return fmt.Errorf("error verifying PV node affinity and POD node details: %v", err) - } - } - - return nil -} - -/* -createStafeulSetAndVerifyPVAndPodNodeAffinty creates user specified statefulset and -further checks the node and volumes affinities -*/ -func createStafeulSetAndVerifyPVAndPodNodeAffinty(ctx context.Context, client clientset.Interface, - namespace string, parallelPodPolicy bool, replicas int32, nodeAffinityToSet bool, - allowedTopologies []v1.TopologySelectorLabelRequirement, allowedTopologyLen int, - podAntiAffinityToSet bool, parallelStatefulSetCreation bool, modifyStsSpec bool, - stsName string, accessMode v1.PersistentVolumeAccessMode, - sc *storagev1.StorageClass, verifyTopologyAffinity bool) (*v1.Service, *appsv1.StatefulSet, error) { - - ginkgo.By("Create service") - service := CreateService(namespace, client) - - framework.Logf("Create StatefulSet") - statefulset := createCustomisedStatefulSets(client, namespace, parallelPodPolicy, - replicas, nodeAffinityToSet, allowedTopologies, allowedTopologyLen, podAntiAffinityToSet, modifyStsSpec, - "", "", nil) - - if verifyTopologyAffinity { - framework.Logf("Verify PV node affinity and that the PODS are running on appropriate node") - err := verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, - namespace, allowedTopologies, parallelStatefulSetCreation, true) - if err != nil { - return nil, nil, fmt.Errorf("error verifying PV node affinity and POD node details: %v", err) - } - } - - return service, statefulset, nil -} - -/* -performOfflineVolumeExpansin performs offline volume expansion on the PVC passed as an input -*/ -func performOfflineVolumeExpansin(client clientset.Interface, - pvclaim *v1.PersistentVolumeClaim, volHandle string, namespace string) error { - - ginkgo.By("Expanding current pvc, performing offline volume expansion") - currentPvcSize := pvclaim.Spec.Resources.Requests[v1.ResourceStorage] - newSize := currentPvcSize.DeepCopy() - newSize.Add(resource.MustParse("1Gi")) - framework.Logf("currentPvcSize %v, newSize %v", currentPvcSize, newSize) - expandedPvc, err := expandPVCSize(pvclaim, newSize, client) - if err != nil { - return fmt.Errorf("error expanding PVC size: %v", err) - } - if expandedPvc == nil { - return errors.New("expanded PVC is nil") - } - - pvcSize := expandedPvc.Spec.Resources.Requests[v1.ResourceStorage] - if pvcSize.Cmp(newSize) != 0 { - return fmt.Errorf("error updating PVC size %q", expandedPvc.Name) - } - - ginkgo.By("Waiting for controller volume resize to finish") - err = waitForPvResizeForGivenPvc(expandedPvc, client, totalResizeWaitPeriod) - if err != nil { - return fmt.Errorf("error waiting for controller volume resize: %v", err) - } - - ginkgo.By("Checking for conditions on PVC") - _, err = waitForPVCToReachFileSystemResizePendingCondition(client, namespace, expandedPvc.Name, pollTimeout) - if err != nil { - return fmt.Errorf("error waiting for PVC conditions: %v", err) - } - - ginkgo.By(fmt.Sprintf("Invoking QueryCNSVolumeWithResult with VolumeID: %s", volHandle)) - queryResult, err := multiVCe2eVSphere.queryCNSVolumeWithResultInMultiVC(volHandle) - if err != nil { - return fmt.Errorf("error querying CNS volume: %v", err) - } - - if len(queryResult.Volumes) == 0 { - return errors.New("queryCNSVolumeWithResult returned no volume") - } - - ginkgo.By("Verifying disk size requested in volume expansion is honored") - newSizeInMb := int64(3072) - if queryResult.Volumes[0].BackingObjectDetails.(*cnstypes.CnsBlockBackingDetails).CapacityInMb != newSizeInMb { - return errors.New("wrong disk size after volume expansion") - } - - return nil -} - -/* -performOnlineVolumeExpansin performs online volume expansion on the Pod passed as an input -*/ -func performOnlineVolumeExpansin(f *framework.Framework, client clientset.Interface, - pvclaim *v1.PersistentVolumeClaim, namespace string, pod *v1.Pod) error { - - ginkgo.By("Waiting for file system resize to finish") - expandedPVC, err := waitForFSResize(pvclaim, client) - if err != nil { - return fmt.Errorf("error waiting for file system resize: %v", err) - } - - pvcConditions := expandedPVC.Status.Conditions - expectEqual(len(pvcConditions), 0, "pvc should not have conditions") - - var fsSize int64 - - ginkgo.By("Verify filesystem size for mount point /mnt/volume1") - fsSize, err = getFSSizeMb(f, pod) - if err != nil { - return fmt.Errorf("error getting file system size: %v", err) - } - framework.Logf("File system size after expansion : %s", fsSize) - - if fsSize < diskSizeInMb { - return fmt.Errorf("error updating filesystem size for %q. Resulting filesystem size is %d", expandedPVC.Name, fsSize) - } - ginkgo.By("File system resize finished successfully") - - return nil -} - -/* -This util getClusterNameForMultiVC will return the cluster details of anyone VC passed to this util -*/ -func getClusterNameForMultiVC(ctx context.Context, vs *multiVCvSphere, - clientIndex int) ([]*object.ClusterComputeResource, - *VsanClient, error) { - - var vsanHealthClient *VsanClient - var err error - c := newClientForMultiVC(ctx, vs) - - datacenter := strings.Split(multiVCe2eVSphere.multivcConfig.Global.Datacenters, ",") - - for i, client := range c { - if clientIndex == i { - vsanHealthClient, err = newVsanHealthSvcClient(ctx, client.Client) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - } - - finder := find.NewFinder(vsanHealthClient.vim25Client, false) - dc, err := finder.Datacenter(ctx, datacenter[0]) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - finder.SetDatacenter(dc) - - clusterComputeResource, err := finder.ClusterComputeResourceList(ctx, "*") - framework.Logf("clusterComputeResource %v", clusterComputeResource) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - return clusterComputeResource, vsanHealthClient, err -} - -/* -This util verifyPreferredDatastoreMatchInMultiVC will compare the prefrence of datatsore with the -actual datatsore and expected datastore and will return a bool value if both actual and expected datatsore -gets matched else will return false -This util will basically be used to check where exactly the volume provisioning has happened -*/ -func (vs *multiVCvSphere) verifyPreferredDatastoreMatchInMultiVC(volumeID string, dsUrls []string) bool { - framework.Logf("Invoking QueryCNSVolumeWithResult with VolumeID: %s", volumeID) - queryResult, err := vs.queryCNSVolumeWithResultInMultiVC(volumeID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(queryResult.Volumes).ShouldNot(gomega.BeEmpty()) - actualDatastoreUrl := queryResult.Volumes[0].DatastoreUrl - flag := false - for _, dsUrl := range dsUrls { - if actualDatastoreUrl == dsUrl { - flag = true - return flag - } - } - return flag -} - -/* -queryCNSVolumeSnapshotWithResultInMultiVC Call CnsQuerySnapshots and returns CnsSnapshotQueryResult -to client -*/ -func (vs *multiVCvSphere) queryCNSVolumeSnapshotWithResultInMultiVC(fcdID string, - snapshotId string) (*cnstypes.CnsSnapshotQueryResult, error) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - var snapshotSpec []cnstypes.CnsSnapshotQuerySpec - var taskResult *cnstypes.CnsSnapshotQueryResult - snapshotSpec = append(snapshotSpec, cnstypes.CnsSnapshotQuerySpec{ - VolumeId: cnstypes.CnsVolumeId{ - Id: fcdID, - }, - SnapshotId: &cnstypes.CnsSnapshotId{ - Id: snapshotId, - }, - }) - - queryFilter := cnstypes.CnsSnapshotQueryFilter{ - SnapshotQuerySpecs: snapshotSpec, - Cursor: &cnstypes.CnsCursor{ - Offset: 0, - Limit: 100, - }, - } - - req := cnstypes.CnsQuerySnapshots{ - This: cnsVolumeManagerInstance, - SnapshotQueryFilter: queryFilter, - } - - for i := 0; i < len(vs.multiVcCnsClient); i++ { - res, err := cnsmethods.CnsQuerySnapshots(ctx, vs.multiVcCnsClient[i].Client, &req) - if err != nil { - return nil, err - } - - task, err := object.NewTask(vs.multiVcClient[i].Client, res.Returnval), nil - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - taskInfo, err := cns.GetTaskInfo(ctx, task) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - taskResult, err = cns.GetQuerySnapshotsTaskResult(ctx, taskInfo) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - if taskResult.Entries[0].Snapshot.SnapshotId.Id == snapshotId { - return taskResult, nil - } - } - return taskResult, nil -} - -/* -getDatastoresListFromMultiVCs util will fetch the list of datastores available in all -the multi-vc setup -This util will return key-value combination of datastore-name:datastore-url of all the 3 VCs available -in a multi-vc setup -*/ -func getDatastoresListFromMultiVCs(masterIp string, sshClientConfig *ssh.ClientConfig, - cluster *object.ClusterComputeResource, isMultiVcSetup bool) (map[string]string, map[string]string, - map[string]string, error) { - ClusterdatastoreListMapVc1 := make(map[string]string) - ClusterdatastoreListMapVc2 := make(map[string]string) - ClusterdatastoreListMapVc3 := make(map[string]string) - - configvCenterHostname := strings.Split(multiVCe2eVSphere.multivcConfig.Global.VCenterHostname, ",") - for i := 0; i < len(configvCenterHostname); i++ { - datastoreListByVC := govcLoginCmdForMultiVC(i) + - "govc object.collect -s -d ' ' " + cluster.InventoryPath + " host | xargs govc datastore.info -H | " + - "grep 'Path\\|URL' | tr -s [:space:]" - - framework.Logf("cmd : %s ", datastoreListByVC) - result, err := sshExec(sshClientConfig, masterIp, datastoreListByVC) - if err != nil && result.Code != 0 { - fssh.LogResult(result) - return nil, nil, nil, fmt.Errorf("couldn't execute command: %s on host: %v , error: %s", - datastoreListByVC, masterIp, err) - } - - datastoreList := strings.Split(result.Stdout, "\n") - - ClusterdatastoreListMap := make(map[string]string) // Empty the map - - for i := 0; i < len(datastoreList)-1; i = i + 2 { - key := strings.ReplaceAll(datastoreList[i], " Path: ", "") - value := strings.ReplaceAll(datastoreList[i+1], " URL: ", "") - ClusterdatastoreListMap[key] = value - } - - if i == 0 { - ClusterdatastoreListMapVc1 = ClusterdatastoreListMap - } else if i == 1 { - ClusterdatastoreListMapVc2 = ClusterdatastoreListMap - } else if i == 2 { - ClusterdatastoreListMapVc3 = ClusterdatastoreListMap - } - } - - return ClusterdatastoreListMapVc1, ClusterdatastoreListMapVc2, ClusterdatastoreListMapVc3, nil -} - -/* -readVsphereConfCredentialsInMultiVCcSetup util accepts a vsphere conf parameter, reads all the values -of vsphere conf and returns a testConf file -*/ -func readVsphereConfCredentialsInMultiVcSetup(cfg string) (e2eTestConfig, error) { - var config e2eTestConfig - - virtualCenters := make([]string, 0) - userList := make([]string, 0) - passwordList := make([]string, 0) - portList := make([]string, 0) - dataCenterList := make([]string, 0) - - key, value := "", "" - lines := strings.Split(cfg, "\n") - for index, line := range lines { - if index == 0 { - // Skip [Global]. - continue - } - words := strings.Split(line, " = ") - if strings.Contains(words[0], "topology-categories=") { - words = strings.Split(line, "=") - } - - if len(words) == 1 { - if strings.Contains(words[0], "Snapshot") { - continue - } - if strings.Contains(words[0], "Labels") { - continue - } - words = strings.Split(line, " ") - if strings.Contains(words[0], "VirtualCenter") { - value = words[1] - value = strings.TrimSuffix(value, "]") - value = trimQuotes(value) - config.Global.VCenterHostname = value - virtualCenters = append(virtualCenters, value) - } - continue - } - key = words[0] - value = trimQuotes(words[1]) - var strconvErr error - switch key { - case "insecure-flag": - if strings.Contains(value, "true") { - config.Global.InsecureFlag = true - } else { - config.Global.InsecureFlag = false - } - case "cluster-id": - config.Global.ClusterID = value - case "cluster-distribution": - config.Global.ClusterDistribution = value - case "user": - config.Global.User = value - userList = append(userList, value) - case "password": - config.Global.Password = value - passwordList = append(passwordList, value) - case "datacenters": - config.Global.Datacenters = value - dataCenterList = append(dataCenterList, value) - case "port": - config.Global.VCenterPort = value - portList = append(portList, value) - case "cnsregistervolumes-cleanup-intervalinmin": - config.Global.CnsRegisterVolumesCleanupIntervalInMin, strconvErr = strconv.Atoi(value) - if strconvErr != nil { - return config, fmt.Errorf("invalid value for cnsregistervolumes-cleanup-intervalinmin: %s", value) - } - case "topology-categories": - config.Labels.TopologyCategories = value - case "global-max-snapshots-per-block-volume": - config.Snapshot.GlobalMaxSnapshotsPerBlockVolume, strconvErr = strconv.Atoi(value) - if strconvErr != nil { - return config, fmt.Errorf("invalid value for global-max-snapshots-per-block-volume: %s", value) - } - case "csi-fetch-preferred-datastores-intervalinmin": - config.Global.CSIFetchPreferredDatastoresIntervalInMin, strconvErr = strconv.Atoi(value) - if strconvErr != nil { - return config, fmt.Errorf("invalid value for csi-fetch-preferred-datastores-intervalinmin: %s", value) - } - case "targetvSANFileShareDatastoreURLs": - config.Global.TargetvSANFileShareDatastoreURLs = value - case "query-limit": - config.Global.QueryLimit, strconvErr = strconv.Atoi(value) - if strconvErr != nil { - return config, fmt.Errorf("invalid value for query-limit: %s", value) - } - case "list-volume-threshold": - config.Global.ListVolumeThreshold, strconvErr = strconv.Atoi(value) - if strconvErr != nil { - return config, fmt.Errorf("invalid value for list-volume-threshold: %s", value) - } - default: - return config, fmt.Errorf("unknown key %s in the input string", key) - } - } - - config.Global.VCenterHostname = strings.Join(virtualCenters, ",") - config.Global.User = strings.Join(userList, ",") - config.Global.Password = strings.Join(passwordList, ",") - config.Global.VCenterPort = strings.Join(portList, ",") - config.Global.Datacenters = strings.Join(dataCenterList, ",") - - return config, nil -} - -/* -writeNewDataAndUpdateVsphereConfSecret uitl edit the vsphere conf and returns the updated -vsphere config sceret -*/ -func writeNewDataAndUpdateVsphereConfSecret(client clientset.Interface, ctx context.Context, - csiNamespace string, cfg e2eTestConfig) error { - var result string - - // fetch current secret - currentSecret, err := client.CoreV1().Secrets(csiNamespace).Get(ctx, configSecret, metav1.GetOptions{}) - if err != nil { - return err - } - - // modify vshere conf file - vCenterHostnames := strings.Split(cfg.Global.VCenterHostname, ",") - users := strings.Split(cfg.Global.User, ",") - passwords := strings.Split(cfg.Global.Password, ",") - dataCenters := strings.Split(cfg.Global.Datacenters, ",") - ports := strings.Split(cfg.Global.VCenterPort, ",") - - result += fmt.Sprintf("[Global]\ncluster-distribution = \"%s\"\n"+ - "csi-fetch-preferred-datastores-intervalinmin = %d\n"+ - "query-limit = %d\nlist-volume-threshold = %d\n\n", - cfg.Global.ClusterDistribution, cfg.Global.CSIFetchPreferredDatastoresIntervalInMin, cfg.Global.QueryLimit, - cfg.Global.ListVolumeThreshold) - for i := 0; i < len(vCenterHostnames); i++ { - result += fmt.Sprintf("[VirtualCenter \"%s\"]\ninsecure-flag = \"%t\"\nuser = \"%s\"\npassword = \"%s\"\n"+ - "port = \"%s\"\ndatacenters = \"%s\"\n\n", - vCenterHostnames[i], cfg.Global.InsecureFlag, users[i], passwords[i], ports[i], dataCenters[i]) - } - - result += fmt.Sprintf("[Snapshot]\nglobal-max-snapshots-per-block-volume = %d\n\n", - cfg.Snapshot.GlobalMaxSnapshotsPerBlockVolume) - result += fmt.Sprintf("[Labels]\ntopology-categories = \"%s\"\n", cfg.Labels.TopologyCategories) - - framework.Logf(result) - - // update config secret with newly updated vshere conf file - framework.Logf("Updating the secret to reflect new conf credentials") - currentSecret.Data[vSphereCSIConf] = []byte(result) - _, err = client.CoreV1().Secrets(csiNamespace).Update(ctx, currentSecret, metav1.UpdateOptions{}) - if err != nil { - return err - } - - return nil -} - -// readVsphereConfSecret method is used to read csi vsphere conf file -func readVsphereConfSecret(client clientset.Interface, ctx context.Context, - csiNamespace string) (e2eTestConfig, error) { - - // fetch current secret - currentSecret, err := client.CoreV1().Secrets(csiNamespace).Get(ctx, configSecret, metav1.GetOptions{}) - if err != nil { - return e2eTestConfig{}, err - } - - // read vsphere conf - originalConf := string(currentSecret.Data[vSphereCSIConf]) - vsphereCfg, err := readVsphereConfCredentialsInMultiVcSetup(originalConf) - if err != nil { - return e2eTestConfig{}, err - } - - return vsphereCfg, nil -} - -/* -setNewNameSpaceInCsiYaml util installs the csi yaml in new namespace -*/ -func setNewNameSpaceInCsiYaml(client clientset.Interface, sshClientConfig *ssh.ClientConfig, originalNS string, - newNS string, allMasterIps []string) error { - - var controlIp string - ignoreLabels := make(map[string]string) - - for _, masterIp := range allMasterIps { - deleteCsiYaml := "kubectl delete -f vsphere-csi-driver.yaml" - framework.Logf("Delete csi driver yaml: %s ", deleteCsiYaml) - deleteCsi, err := sshExec(sshClientConfig, masterIp, deleteCsiYaml) - if err != nil && deleteCsi.Code != 0 { - if strings.Contains(err.Error(), "does not exist") { - framework.Logf("Retry other master nodes") - continue - } else { - gomega.Expect(err).NotTo(gomega.HaveOccurred(), "couldn't execute command on host: %v , error: %s", - masterIp, err) - } - } - if err == nil { - controlIp = masterIp - break - } - } - - findAndSetVal := "sed -i 's/" + originalNS + "/" + newNS + "/g' " + "vsphere-csi-driver.yaml" - framework.Logf("Set test namespace to csi yaml: %s ", findAndSetVal) - setVal, err := sshExec(sshClientConfig, controlIp, findAndSetVal) - if err != nil && setVal.Code != 0 { - fssh.LogResult(setVal) - return fmt.Errorf("couldn't execute command: %s on host: %v , error: %s", - findAndSetVal, controlIp, err) - } - - applyCsiYaml := "kubectl apply -f vsphere-csi-driver.yaml" - framework.Logf("Apply updated csi yaml: %s ", applyCsiYaml) - applyCsi, err := sshExec(sshClientConfig, controlIp, applyCsiYaml) - if err != nil && applyCsi.Code != 0 { - fssh.LogResult(applyCsi) - return fmt.Errorf("couldn't execute command: %s on host: %v , error: %s", - applyCsiYaml, controlIp, err) - } - - // Wait for the CSI Pods to be up and Running - list_of_pods, err := fpod.GetPodsInNamespace(client, newNS, ignoreLabels) - if err != nil { - return err - } - num_csi_pods := len(list_of_pods) - err = fpod.WaitForPodsRunningReady(client, newNS, int32(num_csi_pods), 0, - pollTimeout, ignoreLabels) - if err != nil { - return err - } - return nil -} - -/* -deleteVsphereConfigSecret deletes vsphere config secret -*/ -func deleteVsphereConfigSecret(client clientset.Interface, ctx context.Context, - originalNS string) error { - - // get current secret - currentSecret, err := client.CoreV1().Secrets(originalNS).Get(ctx, configSecret, metav1.GetOptions{}) - if err != nil { - return err - } - - // delete current secret - err = client.CoreV1().Secrets(originalNS).Delete(ctx, currentSecret.Name, metav1.DeleteOptions{}) - if err != nil { - return err - } - - return nil -} - -/* -createNamespaceSpec util creates a spec required for creating a namespace -*/ -func createNamespaceSpec(nsName string) *v1.Namespace { - var namespace = &v1.Namespace{ - TypeMeta: metav1.TypeMeta{ - Kind: "Namespace", - }, - ObjectMeta: metav1.ObjectMeta{ - GenerateName: nsName, - }, - } - return namespace -} - -/* -createNamespace creates a namespace -*/ -func createNamespace(client clientset.Interface, ctx context.Context, nsName string) (*v1.Namespace, error) { - - framework.Logf("Create namespace") - namespaceSpec := createNamespaceSpec(nsName) - namespace, err := client.CoreV1().Namespaces().Create(ctx, namespaceSpec, metav1.CreateOptions{}) - if err != nil { - return nil, err - } - return namespace, nil -} - -/* -createVsphereConfigSecret util creates csi vsphere conf and later creates a new config secret -*/ -func createVsphereConfigSecret(namespace string, cfg e2eTestConfig, sshClientConfig *ssh.ClientConfig, - allMasterIps []string) error { - - var conf string - var controlIp string - - for _, masterIp := range allMasterIps { - readCsiYaml := "ls -l vsphere-csi-driver.yaml" - framework.Logf("list csi driver yaml: %s ", readCsiYaml) - grepCsiNs, err := sshExec(sshClientConfig, masterIp, readCsiYaml) - if err != nil && grepCsiNs.Code != 0 { - if strings.Contains(err.Error(), "No such file or directory") { - framework.Logf("Retry other master nodes") - continue - } else { - gomega.Expect(err).NotTo(gomega.HaveOccurred(), "couldn't execute command on host: %v , error: %s", - masterIp, err) - } - } - if err == nil { - controlIp = masterIp - break - } - } - - vCenterHostnames := strings.Split(cfg.Global.VCenterHostname, ",") - users := strings.Split(cfg.Global.User, ",") - passwords := strings.Split(cfg.Global.Password, ",") - dataCenters := strings.Split(cfg.Global.Datacenters, ",") - ports := strings.Split(cfg.Global.VCenterPort, ",") - - conf = fmt.Sprintf("tee csi-vsphere.conf >/dev/null < 0 { - for _, snap := range snaps { - framework.Logf("Delete volume snapshot %v", snap.Name) - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, snap.Name, pandoraSyncWaitTime) - } - for i, snapshotId := range snapIDs { - framework.Logf("Verify snapshot entry %v is deleted from CNS for volume %v", snapshotId, volIds[i]) - err = waitForCNSSnapshotToBeDeleted(volIds[i], snapshotId) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - } - }() - - ginkgo.By("Change volume allocation of pol1 and pol2 to EZT and apply the changes") - ginkgo.By("Verify the volume allocation type changes fail") - for i, volId := range volIds { - framework.Logf("updating policy %v with %v allocation type", policyNames[i/2], eztAllocType) - err = updateVmfsPolicyAlloctype(ctx, pc, eztAllocType, policyNames[i/2], policyIds[i/2]) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - framework.Logf( - "trying to reconfigure volume %v with policy %v which is expected to fail", volId, policyNames[i/2]) - err = e2eVSphere.reconfigPolicy(ctx, volId, policyIds[i/2].UniqueId) - framework.Logf("reconfigure volume %v with policy %v errored out with:\n%v", volId, policyNames[i/2], err) - gomega.Expect(err).To(gomega.HaveOccurred()) - } - - ginkgo.By("Delete snapshots created in step 6") - for _, snap := range snaps { - framework.Logf("Delete volume snapshot %v", snap.Name) - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, snap.Name, pandoraSyncWaitTime) - } - for i, snapshotId := range snapIDs { - framework.Logf("Verify snapshot entry %v is deleted from CNS for volume %v", snapshotId, volIds[i]) - err = waitForCNSSnapshotToBeDeleted(volIds[i], snapshotId) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - snaps = []*snapV1.VolumeSnapshot{} - ginkgo.By("Apply volume allocation type changes again and verify that it is successful this time") - for i, volId := range volIds { - framework.Logf("trying to reconfigure volume %v with policy %v", volId, policyNames[i/2]) - err = e2eVSphere.reconfigPolicy(ctx, volIds[i], policyIds[i/2].UniqueId) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - }) - - /* - Verify expansion during Thin -> EZT, LZT -> EZT conversion (should take >vpxd task timeout) - Steps for offline volumes: - 1 Create 2 SPBM policies with thin, and LZT volume allocation respectively, say pol1, pol2 - 2 Create 2 SCs each with a SPBM policy created from step 1 - 3 Create a PVC of 10g using each of the SCs created from step 2 - 4 Wait for PVCs created in step 3 to be bound - 5 Change volume allocation of pol1 and pol2 to EZT and opt for immediate updation - 6 While updation in step 5 is still running, expand all PVCs created in step 3 such that resize should take - more than vpxd task timeout for PVCs updated to EZT allocation - 7 Wait for all PVCs created in step 3 to reach FileSystemResizePending state - 8 Delete the PVCs created in step 3 - 9 Delete the SCs created in step 2 - 10 Deleted the SPBM policies created in step 1 - - Steps for online volumes: - 1 Create 2 SPBM policies with thin, and LZT volume allocation respectively, say pol1, pol2 - 2 Create 2 SCs each with a SPBM policy created from step 1 - 3 Create a PVC of 10g using each of the SCs created from step 2 - 4 Wait for PVCs created in step 3 to be bound - 5 Create pods using PVCs created in step 4 - 6 Change volume allocation of pol1 and pol2 to EZT and opt for immediate updation - 7 While updation in step 5 is still running, expand all PVCs created in step 3 such that resize should take - more than vpxd task timeout for PVCs updated to EZT allocation - 8 Wait for file system resize to complete on all PVCs created in step 3 - 9 Delete pods created in step 4 - 10 Delete the PVCs created in step 3 - 11 Delete the SCs created in step 2 - 12 Deleted the SPBM policies created in step 1 - */ - ginkgo.It("[csi-block-vanilla][csi-guest][csi-supervisor] Verify expansion during Thin -> EZT, LZT -> EZT"+ - " conversion (should take >vpxd task timeout)", func() { - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - sharedvmfsURL := os.Getenv(envSharedVMFSDatastoreURL) - if sharedvmfsURL == "" { - ginkgo.Skip(fmt.Sprintf("Env %v is missing", envSharedVMFSDatastoreURL)) - } - - scParameters := make(map[string]string) - policyNames := []string{} - policyIds := []*pbmtypes.PbmProfileId{} - pvcs := []*v1.PersistentVolumeClaim{} - scs := []*storagev1.StorageClass{} - pvcs2d := [][]*v1.PersistentVolumeClaim{} - largeSize := os.Getenv(envDiskSizeLarge) - pvc10g := "10Gi" - if largeSize == "" { - largeSize = diskSizeLarge - } - - rand.New(rand.NewSource(time.Now().UnixNano())) - suffix := fmt.Sprintf("-%v-%v", time.Now().UnixNano(), rand.Intn(10000)) - categoryName := "category" + suffix - tagName := "tag" + suffix - - catID, tagID := createCategoryNTag(ctx, categoryName, tagName) - defer func() { - deleteCategoryNTag(ctx, catID, tagID) - }() - - attachTagToDS(ctx, tagID, sharedvmfsURL) - defer func() { - detachTagFromDS(ctx, tagID, sharedvmfsURL) - }() - - allocationTypes := []string{ - thinAllocType, - lztAllocType, - } - - ginkgo.By("create 2 SPBM policies with thin, LZT volume allocation respectively") - for _, at := range allocationTypes { - policyID, policyName := createVmfsStoragePolicy( - ctx, pc, at, map[string]string{categoryName: tagName}) - defer func() { - deleteStoragePolicy(ctx, pc, policyID) - }() - scParameters[scParamStoragePolicyName] = policyName - policyNames = append(policyNames, policyName) - policyIds = append(policyIds, policyID) - } - - defer func() { - for _, policyID := range policyIds { - deleteStoragePolicy(ctx, pc, policyID) - } - }() - - ginkgo.By("Create 2 SCs each with a SPBM policy created from step 1") - if vanillaCluster { - for i, policyName := range policyNames { - scParameters[scParamStoragePolicyName] = policyName - policyNames = append(policyNames, policyName) - policyIds = append(policyIds, policyIds[i]) - storageclass, err := createStorageClass(client, - scParameters, nil, "", "", true, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - scs = append(scs, storageclass) - } - } else if supervisorCluster { - assignPolicyToWcpNamespace(client, ctx, namespace, policyNames) - for _, policyName := range policyNames { - createResourceQuota(client, namespace, rqLimit, policyName) - storageclass, err := client.StorageV1().StorageClasses().Get(ctx, policyName, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - scs = append(scs, storageclass) - } - } else if guestCluster { - svcClient, svNamespace := getSvcClientAndNamespace() - assignPolicyToWcpNamespace(svcClient, ctx, svNamespace, policyNames) - for _, policyName := range policyNames { - createResourceQuota(svcClient, svNamespace, rqLimit, policyName) - storageclass, err := svcClient.StorageV1().StorageClasses().Get(ctx, policyName, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - scs = append(scs, storageclass) - } - } - - defer func() { - ginkgo.By("Delete the SCs created in step 2") - if vanillaCluster { - for _, sc := range scs { - err := client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - } - }() - - ginkgo.By("Create two PVCs of 10g using each of the SCs created from step 2") - for _, sc := range scs { - pvclaim, err := createPVC(client, namespace, nil, pvc10g, sc, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pvcs = append(pvcs, pvclaim) - pvclaim2, err := createPVC(client, namespace, nil, pvc10g, sc, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pvcs = append(pvcs, pvclaim2) - // one pvc for online case and one more pvc for offline case - pvcs2d = append(pvcs2d, []*v1.PersistentVolumeClaim{pvclaim}) - } - - ginkgo.By("Verify the PVCs created in step 3 are bound") - pvs, err := fpv.WaitForPVClaimBoundPhase(client, pvcs, framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - volIds := []string{} - ginkgo.By("Verify that the created CNS volumes are compliant and have correct policy id") - for i, pv := range pvs { - volumeID := pv.Spec.CSI.VolumeHandle - if guestCluster { - volumeID = getVolumeIDFromSupervisorCluster(pvs[0].Spec.CSI.VolumeHandle) - gomega.Expect(volumeID).NotTo(gomega.BeEmpty()) - } - storagePolicyMatches, err := e2eVSphere.VerifySpbmPolicyOfVolume(volumeID, policyNames[i/2]) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(storagePolicyMatches).To(gomega.BeTrue(), "storage policy verification failed") - e2eVSphere.verifyVolumeCompliance(volumeID, true) - e2eVSphere.verifyDatastoreMatch(volumeID, []string{sharedvmfsURL}) - volIds = append(volIds, volumeID) - } - - defer func() { - ginkgo.By("Delete the PVCs created in step 3") - for _, pvc := range pvcs { - err := fpv.DeletePersistentVolumeClaim(client, pvc.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - for _, volId := range volIds { - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volId) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }() - - // Create a Pod to use this PVC, and verify volume has been attached - ginkgo.By("Creating 2 pods to attach 2 PVCs") - pods := createMultiplePods(ctx, client, pvcs2d, true) - - defer func() { - ginkgo.By("Delete pods") - deletePodsAndWaitForVolsToDetach(ctx, client, pods, true) - }() - - fsSizes := []int64{} - for _, pod := range pods { - originalSizeInMb, err := getFSSizeMb(f, pod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - fsSizes = append(fsSizes, originalSizeInMb) - } - - ginkgo.By("Change volume allocation of pol1 and pol2 to EZT") - for i, policyId := range policyIds { - err = updateVmfsPolicyAlloctype(ctx, pc, eztAllocType, policyNames[i], policyId) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - ginkgo.By("Apply the volume allocation changes and while it is still running, expand PVCs created to a" + - " large size such that the resize takes more than vpxd task timeout") - var wg sync.WaitGroup - wg.Add(len(volIds)) - n := resource.MustParse(largeSize) - sizeInInt, b := n.AsInt64() - gomega.Expect(b).To(gomega.BeTrue()) - // since we are creating 4 volumes reducing the size of each expansion by half to start with - newSize := *(resource.NewQuantity(sizeInInt/2, resource.BinarySI)) - start := time.Now() - for i, volId := range volIds { - go reconfigPolicyParallel(ctx, volId, policyIds[i/2].UniqueId, &wg) - } - wg.Wait() - - wg.Add(len(volIds)) - for _, pvc := range pvcs { - go resize(client, pvc, pvc.Spec.Resources.Requests[v1.ResourceStorage], newSize, &wg) - } - wg.Wait() - for i := range pvcs { - err = waitForPvResize(pvs[i], client, newSize, totalResizeWaitPeriod*2) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), "resize exceeded timeout") - } - elapsed := time.Since(start) - - ginkgo.By("Verify PVC expansion took longer than vpxd timeout") - gomega.Expect(elapsed > time.Second*time.Duration(vpxdReducedTaskTimeoutSecsInt)).To( - gomega.BeTrue(), "PVC expansion was faster than vpxd timeout") - - ginkgo.By("Wait and verify the file system resize on pvcs") - for i := range pods { - framework.Logf("Waiting for file system resize to finish for pvc %v", pvcs[i*2].Name) - pvcs[i*2], err = waitForFSResize(pvcs[i*2], client) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - fsSize, err := getFSSizeMb(f, pods[i]) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - framework.Logf("File system size after expansion : %v", fsSize) - gomega.Expect(fsSize > fsSizes[i]).To(gomega.BeTrue(), - fmt.Sprintf( - "filesystem size %v is not > than before expansion %v for pvc %q", - fsSize, fsSizes[i], pvcs[i*2].Name)) - - framework.Logf("File system resize finished successfully for pvc %v", pvcs[i*2].Name) - } - - }) - - /* - Relocate from vsand datastore to vmfs datastore and vice versa - Steps: - 1. Create a SPBM policy with EZT volume allocation on vmfs datastore. - 2. Create a SC, say sc1 - 3. Create a large PVC pvc1 using SC created in step 2, this should take more than 40 mins - 4. Verify that pvc1 is bound and backend fcd is on vsand datastore. - 5. Create a pod, say pod1 using pvc1. - 6. write some data to the volume. - 7. delete pod1. - 8. Relocate fcd to vmfs ds. - 9. Recreate pod1 - 10.Verify pod1 is running and pvc1 is accessible and verify the data written in step 6 - 11.Delete pod1 - 12. Relocate fcd to vsanDirect ds. - 13. Recreate pod1 - 14.Verify pod1 is running and pvc1 is accessible and verify the data written in step 6 - 15. Delete pod1 - 16. Delete pvc1, sc1 - 17. Delete SPBM policies created - */ - ginkgo.It("[csi-wcp-vsan-direct] Relocate from vmfs datastore to vsand datastore "+ - "and vice versa", func() { - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - sharedvmfsURL := os.Getenv(envSharedVMFSDatastoreURL) - if sharedvmfsURL == "" { - ginkgo.Skip(fmt.Sprintf("Env %v is missing", envSharedVMFSDatastoreURL)) - } - - vsanDDatstoreURL := os.Getenv(envVsanDDatastoreURL) - if sharedvmfsURL == "" { - ginkgo.Skip(fmt.Sprintf("Env %v is missing", envVsanDDatastoreURL)) - } - - datastoreUrls := []string{sharedvmfsURL, vsanDDatstoreURL} - policyNames := []string{} - pvcs := []*v1.PersistentVolumeClaim{} - scs := []*storagev1.StorageClass{} - - rand.New(rand.NewSource(time.Now().UnixNano())) - - suffix := fmt.Sprintf("-%v-%v", time.Now().UnixNano(), rand.Intn(10000)) - categoryName := "category" + suffix - tagName := "tag" + suffix - - catID, tagID := createCategoryNTag(ctx, categoryName, tagName) - defer func() { - deleteCategoryNTag(ctx, catID, tagID) - }() - - attachTagToDS(ctx, tagID, sharedvmfsURL) - defer func() { - detachTagFromDS(ctx, tagID, sharedvmfsURL) - }() - - attachTagToDS(ctx, tagID, vsanDDatstoreURL) - defer func() { - detachTagFromDS(ctx, tagID, vsanDDatstoreURL) - }() - - ginkgo.By("create SPBM policy with EZT volume allocation") - ginkgo.By("create a storage class with a SPBM policy created from step 1") - ginkgo.By("create a PVC each using the storage policy created from step 2") - var pvclaim *v1.PersistentVolumeClaim - var pod *v1.Pod - - policyID, policyName := createStoragePolicyWithSharedVmfsNVsand( - ctx, pc, eztAllocType, map[string]string{categoryName: tagName}) - defer func() { - deleteStoragePolicy(ctx, pc, policyID) - }() - policyNames = append(policyNames, policyName) - - assignPolicyToWcpNamespace(client, ctx, namespace, policyNames) - createResourceQuota(client, namespace, rqLimit, policyName) - storageclass, err := client.StorageV1().StorageClasses().Get(ctx, policyName, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pvclaim, pod = createVsanDPvcAndPod(client, ctx, - namespace, storageclass, eztVsandPvcName, eztVsandPodName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pvcs = append(pvcs, pvclaim) - scs = append(scs, storageclass) - - defer func() { - ginkgo.By("Delete the SCs created in step 2") - for _, sc := range scs { - err := client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }() - - ginkgo.By("Verify the PVCs created in step 3 are bound") - pvs, err := fpv.WaitForPVClaimBoundPhase(client, pvcs, framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Verify that the created CNS volumes are compliant and have correct policy id") - - volumeID := pvs[0].Spec.CSI.VolumeHandle - storagePolicyExists, err := e2eVSphere.VerifySpbmPolicyOfVolume(volumeID, policyNames[0]) - e2eVSphere.verifyVolumeCompliance(volumeID, true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(storagePolicyExists).To(gomega.BeTrue(), "storage policy verification failed") - - defer func() { - ginkgo.By("Delete the pod created") - err = fpod.DeletePodWithWait(client, pod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Delete the PVCs created in step 3") - for i, pvc := range pvcs { - err := fpv.DeletePersistentVolumeClaim(client, pvc.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(pvs[i].Spec.CSI.VolumeHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }() - - err = fpod.DeletePodWithWait(client, pod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Verify if VolumeID is created on the given datastores") - dsUrlWhereVolumeIsPresent := fetchDsUrl4CnsVol(e2eVSphere, volumeID) - framework.Logf("Volume is present on %s", dsUrlWhereVolumeIsPresent) - srcVsandDsUrl := dsUrlWhereVolumeIsPresent - e2eVSphere.verifyDatastoreMatch(volumeID, datastoreUrls) - - // Get the destination ds url where the volume will get relocated - destDsUrl := "" - for _, dsurl := range datastoreUrls { - if dsurl != dsUrlWhereVolumeIsPresent { - destDsUrl = dsurl - } - } - - dsRefDest := getDsMoRefFromURL(ctx, destDsUrl) - _, err = e2eVSphere.cnsRelocateVolume(e2eVSphere, ctx, volumeID, dsRefDest) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - e2eVSphere.verifyDatastoreMatch(volumeID, []string{destDsUrl}) - - storagePolicyExists, err = e2eVSphere.VerifySpbmPolicyOfVolume(volumeID, policyNames[0]) - e2eVSphere.verifyVolumeCompliance(volumeID, true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(storagePolicyExists).To(gomega.BeTrue(), "storage policy verification failed") - - ginkgo.By("Creating a pod") - podSpec := getVsanDPodSpec(namespace, nil, []*v1.PersistentVolumeClaim{pvclaim}, false, execCommand, eztVsandPodName) - pod, err = client.CoreV1().Pods(namespace).Create(ctx, podSpec, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - err = fpod.WaitForPodNameRunningInNamespace(client, pod.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Relocate back to vsand datstore") - - err = fpod.DeletePodWithWait(client, pod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - dsRefDest = getDsMoRefFromURL(ctx, srcVsandDsUrl) - _, err = e2eVSphere.cnsRelocateVolume(e2eVSphere, ctx, volumeID, dsRefDest) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - e2eVSphere.verifyDatastoreMatch(volumeID, []string{srcVsandDsUrl}) - - storagePolicyExists, err = e2eVSphere.VerifySpbmPolicyOfVolume(volumeID, policyNames[0]) - e2eVSphere.verifyVolumeCompliance(volumeID, true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(storagePolicyExists).To(gomega.BeTrue(), "storage policy verification failed") - - ginkgo.By("Creating a pod") - podSpec = getVsanDPodSpec(namespace, nil, []*v1.PersistentVolumeClaim{pvclaim}, false, execCommand, eztVsandPodName) - pod, err = client.CoreV1().Pods(namespace).Create(ctx, podSpec, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - err = fpod.WaitForPodNameRunningInNamespace(client, pod.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Delete pods created in step 5") - deletePodsAndWaitForVolsToDetach(ctx, client, []*v1.Pod{pod}, true) - - }) - - /* - Start attached volume's conversion and relocation in parallel - Steps for offline volumes: - 1. Create a SPBM policy with lzt volume allocation for vmfs datastore. - 2. Create SC using policy created in step 1 - 3. Create PVC using SC created in step 2 - 4. Verify that pvc created in step 3 are bound - 5. Create a pod, say pod1 using pvc created in step 4. - 6. Start writing some IO to pod. - 7. Delete pod1. - 8. Relocate CNS volume corresponding to pvc from step 3 to a different datastore. - 9. While relocation is running perform volume conversion. - 10. Verify relocation was successful. - 11. Verify online volume conversion is successful. - 12. Delete all the objects created during the test. - - Steps for online volumes: - 1. Create a SPBM policy with lzt volume allocation for vmfs datastore. - 2. Create SC using policy created in step 1 - 3. Create PVC using SC created in step 2 - 4. Verify that pvc created in step 3 are bound - 5. Create a pod, say pod1 using pvc created in step 4. - 6. Start writing some IO to pod which run in parallel to steps 6-7. - 7. Relocate CNS volume corresponding to pvc from step 3 to a different datastore. - 8. While relocation is running perform volume conversion. - 9. Verify the IO written so far. - 10. Verify relocation was successful. - 11. Verify online volume conversion is successful. - 12. Delete all the objects created during the test. - */ - ginkgo.It("[csi-block-vanilla][csi-block-vanilla-parallelized]"+ - " Start attached volume's conversion and relocation in parallel", func() { - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - sharedvmfsURL, sharedvmfs2URL := "", "" - var datastoreUrls []string - var policyName string - volIdToDsUrlMap := make(map[string]string) - volIdToCnsRelocateVolTask := make(map[string]*object.Task) - - sharedvmfsURL = os.Getenv(envSharedVMFSDatastoreURL) - if sharedvmfsURL == "" { - ginkgo.Skip(fmt.Sprintf("Env %v is missing", envSharedVMFSDatastoreURL)) - } - - sharedvmfs2URL = os.Getenv(envSharedVMFSDatastore2URL) - if sharedvmfsURL == "" { - ginkgo.Skip(fmt.Sprintf("Env %v is missing", envSharedVMFSDatastore2URL)) - } - datastoreUrls = append(datastoreUrls, sharedvmfsURL, sharedvmfs2URL) - - scParameters := make(map[string]string) - policyNames := []string{} - pvcs := []*v1.PersistentVolumeClaim{} - pvclaims2d := [][]*v1.PersistentVolumeClaim{} - - rand.New(rand.NewSource(time.Now().UnixNano())) - suffix := fmt.Sprintf("-%v-%v", time.Now().UnixNano(), rand.Intn(10000)) - categoryName := "category" + suffix - tagName := "tag" + suffix - - catID, tagID := createCategoryNTag(ctx, categoryName, tagName) - defer func() { - deleteCategoryNTag(ctx, catID, tagID) - }() - - attachTagToDS(ctx, tagID, sharedvmfsURL) - defer func() { - detachTagFromDS(ctx, tagID, sharedvmfsURL) - }() - - attachTagToDS(ctx, tagID, sharedvmfs2URL) - defer func() { - detachTagFromDS(ctx, tagID, sharedvmfs2URL) - }() - - ginkgo.By("create SPBM policy with lzt volume allocation") - ginkgo.By("create a storage class with a SPBM policy created from step 1") - ginkgo.By("create a PVC each using the storage policy created from step 2") - var storageclass *storagev1.StorageClass - var pvclaim *v1.PersistentVolumeClaim - var err error - var policyID *pbmtypes.PbmProfileId - - policyID, policyName = createVmfsStoragePolicy( - ctx, pc, lztAllocType, map[string]string{categoryName: tagName}) - defer func() { - deleteStoragePolicy(ctx, pc, policyID) - }() - policyNames = append(policyNames, policyName) - - framework.Logf("CNS_TEST: Running for vanilla k8s setup") - scParameters[scParamStoragePolicyName] = policyName - storageclass, pvclaim, err = createPVCAndStorageClass(client, - namespace, nil, scParameters, "", nil, "", true, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pvclaim2, err := createPVC(client, namespace, nil, "", storageclass, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pvcs = append(pvcs, pvclaim, pvclaim2) - pvclaims2d = append(pvclaims2d, []*v1.PersistentVolumeClaim{pvclaim}) - pvclaims2d = append(pvclaims2d, []*v1.PersistentVolumeClaim{pvclaim2}) - - defer func() { - if vanillaCluster { - ginkgo.By("Delete the SCs created in step 2") - err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }() - - ginkgo.By("Verify the PVCs created in step 3 are bound") - pvs, err := fpv.WaitForPVClaimBoundPhase(client, pvcs, framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - volIds := []string{} - ginkgo.By("Verify that the created CNS volumes are compliant and have correct policy id") - for i, pv := range pvs { - volumeID := pv.Spec.CSI.VolumeHandle - if guestCluster { - volumeID = getVolumeIDFromSupervisorCluster(pv.Spec.CSI.VolumeHandle) - gomega.Expect(volumeID).NotTo(gomega.BeEmpty()) - } - storagePolicyMatches, err := e2eVSphere.VerifySpbmPolicyOfVolume(volumeID, policyNames[i/2]) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(storagePolicyMatches).To(gomega.BeTrue(), "storage policy verification failed") - e2eVSphere.verifyVolumeCompliance(volumeID, true) - volIds = append(volIds, volumeID) - } - - defer func() { - ginkgo.By("Delete the PVCs created in step 3") - for i, pvc := range pvcs { - err := fpv.DeletePersistentVolumeClaim(client, pvc.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volIds[i]) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - }() - - ginkgo.By("Create pods with using the PVCs created in step 3 and wait for them to be ready") - ginkgo.By("verify we can read and write on the PVCs") - pods := createMultiplePods(ctx, client, pvclaims2d, true) - defer func() { - ginkgo.By("Delete the pod created") - deletePodsAndWaitForVolsToDetach(ctx, client, pods, true) - }() - - ginkgo.By("Verify if VolumeID is created on the given datastores") - // Get the destination ds url where the volume will get relocated - destDsUrl := "" - for _, volId := range volIds { - dsUrlWhereVolumeIsPresent := fetchDsUrl4CnsVol(e2eVSphere, volId) - framework.Logf("Volume is present on %s for volume: %s", dsUrlWhereVolumeIsPresent, volId) - e2eVSphere.verifyDatastoreMatch(volId, datastoreUrls) - for _, dsurl := range datastoreUrls { - if dsurl != dsUrlWhereVolumeIsPresent { - destDsUrl = dsurl - } - } - framework.Logf("dest url: %s", destDsUrl) - volIdToDsUrlMap[volId] = destDsUrl - } - - rand.New(rand.NewSource(time.Now().Unix())) - testdataFile := fmt.Sprintf("/tmp/testdata_%v_%v", time.Now().Unix(), rand.Intn(1000)) - ginkgo.By(fmt.Sprintf("Creating a 100mb test data file %v", testdataFile)) - op, err := exec.Command("dd", "if=/dev/urandom", fmt.Sprintf("of=%v", testdataFile), - "bs=1M", "count=100").Output() - fmt.Println(op) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - op, err = exec.Command("rm", "-f", testdataFile).Output() - fmt.Println(op) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("delete pod1") - deletePodsAndWaitForVolsToDetach(ctx, client, []*v1.Pod{pods[1]}, true) - - ginkgo.By("Updating policy volume allocation from lzt -> ezt") - err = updateVmfsPolicyAlloctype(ctx, pc, eztAllocType, policyName, policyID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Start relocation of volume to a different datastore") - for _, volId := range volIds { - dsRefDest := getDsMoRefFromURL(ctx, volIdToDsUrlMap[volId]) - task, err := e2eVSphere.cnsRelocateVolume(e2eVSphere, ctx, volId, dsRefDest, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - volIdToCnsRelocateVolTask[volId] = task - framework.Logf("Waiting for a few seconds for relocation to be started properly on VC") - time.Sleep(time.Duration(10) * time.Second) - } - - ginkgo.By("Perform volume conversion and write IO to pod while relocate volume to different datastore") - var wg sync.WaitGroup - wg.Add(1 + len(volIds)) - go writeKnownData2PodInParallel(f, pods[0], testdataFile, &wg) - for _, volId := range volIds { - go reconfigPolicyParallel(ctx, volId, policyID.UniqueId, &wg) - } - wg.Wait() - - ginkgo.By("Verify the data on the PVCs match what was written in step 7") - verifyKnownDataInPod(f, pods[0], testdataFile) - - for _, volId := range volIds { - ginkgo.By(fmt.Sprintf("Wait for relocation task to complete for volumeID: %s", volId)) - waitForCNSTaskToComplete(ctx, volIdToCnsRelocateVolTask[volId]) - ginkgo.By("Verify relocation of volume is successful") - e2eVSphere.verifyDatastoreMatch(volId, []string{volIdToDsUrlMap[volId]}) - storagePolicyExists, err := e2eVSphere.VerifySpbmPolicyOfVolume(volId, policyNames[0]) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(storagePolicyExists).To(gomega.BeTrue(), "storage policy verification failed") - e2eVSphere.verifyVolumeCompliance(volId, true) - } - - ginkgo.By("Delete the pod created") - err = fpod.DeletePodWithWait(client, pods[0]) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - }) - - /* - Start attached volume's conversion and relocation of volume with updation of its metadata in parallel - Steps for offline volumes: - 1. Create a SPBM policy with lzt volume allocation for vmfs datastore. - 2. Create SC using policy created in step 1 - 3. Create PVC using SC created in step 2 - 4. Verify that pvc created in step 3 are bound - 5. Create a pod, say pod1 using pvc created in step 4. - 6. Start writing some IO to pod. - 7. Delete pod1. - 8. Relocate CNS volume corresponding to pvc from step 3 to a different datastore. - 9. While relocation is running add labels to PV and PVC - in parallel with volume conversion. - 10. Verify relocation was successful. - 11. Verify online volume conversion is successful. - 12. Delete all the objects created during the test. - - Steps for online volumes: - 1. Create a SPBM policy with lzt volume allocation for vmfs datastore. - 2. Create SC using policy created in step 1 - 3. Create PVC using SC created in step 2 - 4. Verify that pvc created in step 3 are bound - 5. Create a pod, say pod1 using pvc created in step 4. - 6. Start writing some IO to pod which run in parallel to steps 6-7. - 7. Relocate CNS volume corresponding to pvc from step 3 to a different datastore. - 8. While relocation is running add labels to PV and PVC - in parallel with volume conversion. - 9. Verify the IO written so far. - 10. Verify relocation was successful. - 11. Verify online volume conversion is successful. - 12. Delete all the objects created during the test. - */ - ginkgo.It("[csi-block-vanilla][csi-block-vanilla-parallelized]"+ - " Start attached volume's conversion and relocation of volume"+ - " with updation of its metadata in parallel", func() { - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - sharedvmfsURL, sharedvmfs2URL := "", "" - var datastoreUrls []string - var policyName string - volIdToDsUrlMap := make(map[string]string) - labels := make(map[string]string) - labels[labelKey] = labelValue - volIdToCnsRelocateVolTask := make(map[string]*object.Task) - - sharedvmfsURL = os.Getenv(envSharedVMFSDatastoreURL) - if sharedvmfsURL == "" { - ginkgo.Skip(fmt.Sprintf("Env %v is missing", envSharedVMFSDatastoreURL)) - } - - sharedvmfs2URL = os.Getenv(envSharedVMFSDatastore2URL) - if sharedvmfsURL == "" { - ginkgo.Skip(fmt.Sprintf("Env %v is missing", envSharedVMFSDatastore2URL)) - } - datastoreUrls = append(datastoreUrls, sharedvmfsURL, sharedvmfs2URL) - - scParameters := make(map[string]string) - policyNames := []string{} - pvcs := []*v1.PersistentVolumeClaim{} - pvclaims2d := [][]*v1.PersistentVolumeClaim{} - - rand.New(rand.NewSource(time.Now().UnixNano())) - suffix := fmt.Sprintf("-%v-%v", time.Now().UnixNano(), rand.Intn(10000)) - categoryName := "category" + suffix - tagName := "tag" + suffix - - catID, tagID := createCategoryNTag(ctx, categoryName, tagName) - defer func() { - deleteCategoryNTag(ctx, catID, tagID) - }() - - attachTagToDS(ctx, tagID, sharedvmfsURL) - defer func() { - detachTagFromDS(ctx, tagID, sharedvmfsURL) - }() - - attachTagToDS(ctx, tagID, sharedvmfs2URL) - defer func() { - detachTagFromDS(ctx, tagID, sharedvmfs2URL) - }() - - ginkgo.By("create SPBM policy with lzt volume allocation") - ginkgo.By("create a storage class with a SPBM policy created from step 1") - ginkgo.By("create a PVC each using the storage policy created from step 2") - var storageclass *storagev1.StorageClass - var pvclaim *v1.PersistentVolumeClaim - var err error - var policyID *pbmtypes.PbmProfileId - - policyID, policyName = createVmfsStoragePolicy( - ctx, pc, lztAllocType, map[string]string{categoryName: tagName}) - defer func() { - deleteStoragePolicy(ctx, pc, policyID) - }() - policyNames = append(policyNames, policyName) - - framework.Logf("CNS_TEST: Running for vanilla k8s setup") - scParameters[scParamStoragePolicyName] = policyName - storageclass, pvclaim, err = createPVCAndStorageClass(client, - namespace, nil, scParameters, "", nil, "", true, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pvclaim2, err := createPVC(client, namespace, nil, "", storageclass, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pvcs = append(pvcs, pvclaim, pvclaim2) - pvclaims2d = append(pvclaims2d, []*v1.PersistentVolumeClaim{pvclaim}) - pvclaims2d = append(pvclaims2d, []*v1.PersistentVolumeClaim{pvclaim2}) - - defer func() { - if vanillaCluster { - ginkgo.By("Delete the SCs created in step 2") - err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }() - - ginkgo.By("Verify the PVCs created in step 3 are bound") - pvs, err := fpv.WaitForPVClaimBoundPhase(client, pvcs, framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - volIds := []string{} - ginkgo.By("Verify that the created CNS volumes are compliant and have correct policy id") - for i, pv := range pvs { - volumeID := pv.Spec.CSI.VolumeHandle - if guestCluster { - volumeID = getVolumeIDFromSupervisorCluster(pv.Spec.CSI.VolumeHandle) - gomega.Expect(volumeID).NotTo(gomega.BeEmpty()) - } - storagePolicyMatches, err := e2eVSphere.VerifySpbmPolicyOfVolume(volumeID, policyNames[i/2]) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(storagePolicyMatches).To(gomega.BeTrue(), "storage policy verification failed") - e2eVSphere.verifyVolumeCompliance(volumeID, true) - volIds = append(volIds, volumeID) - } - - defer func() { - ginkgo.By("Delete the PVCs created in step 3") - for i, pvc := range pvcs { - err := fpv.DeletePersistentVolumeClaim(client, pvc.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volIds[i]) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - }() - - ginkgo.By("Create pods with using the PVCs created in step 3 and wait for them to be ready") - ginkgo.By("verify we can read and write on the PVCs") - pods := createMultiplePods(ctx, client, pvclaims2d, true) - defer func() { - ginkgo.By("Delete the pod created") - deletePodsAndWaitForVolsToDetach(ctx, client, pods, true) - }() - - ginkgo.By("Verify if VolumeID is created on the given datastores") - // Get the destination ds url where the volume will get relocated - destDsUrl := "" - for _, volId := range volIds { - dsUrlWhereVolumeIsPresent := fetchDsUrl4CnsVol(e2eVSphere, volId) - framework.Logf("Volume is present on %s for volume: %s", dsUrlWhereVolumeIsPresent, volId) - e2eVSphere.verifyDatastoreMatch(volId, datastoreUrls) - for _, dsurl := range datastoreUrls { - if dsurl != dsUrlWhereVolumeIsPresent { - destDsUrl = dsurl - } - } - framework.Logf("dest url: %s", destDsUrl) - volIdToDsUrlMap[volId] = destDsUrl - } - - rand.New(rand.NewSource(time.Now().Unix())) - testdataFile := fmt.Sprintf("/tmp/testdata_%v_%v", time.Now().Unix(), rand.Intn(1000)) - ginkgo.By(fmt.Sprintf("Creating a 100mb test data file %v", testdataFile)) - op, err := exec.Command("dd", "if=/dev/urandom", fmt.Sprintf("of=%v", testdataFile), - "bs=1M", "count=100").Output() - fmt.Println(op) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - op, err = exec.Command("rm", "-f", testdataFile).Output() - fmt.Println(op) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("delete pod1") - deletePodsAndWaitForVolsToDetach(ctx, client, []*v1.Pod{pods[1]}, true) - - ginkgo.By("Start relocation of volume to a different datastore") - for _, volId := range volIds { - dsRefDest := getDsMoRefFromURL(ctx, volIdToDsUrlMap[volId]) - task, err := e2eVSphere.cnsRelocateVolume(e2eVSphere, ctx, volId, dsRefDest, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - volIdToCnsRelocateVolTask[volId] = task - framework.Logf("Waiting for a few seconds for relocation to be started properly on VC") - time.Sleep(time.Duration(10) * time.Second) - } - - ginkgo.By("Add labels to volumes and write IO to pod while relocating volume to different datastore") - err = updateVmfsPolicyAlloctype(ctx, pc, eztAllocType, policyName, policyID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - var wg sync.WaitGroup - wg.Add(3 + len(volIds)) - go writeKnownData2PodInParallel(f, pods[0], testdataFile, &wg) - go updatePvcLabelsInParallel(ctx, client, namespace, labels, pvcs, &wg) - go updatePvLabelsInParallel(ctx, client, namespace, labels, pvs, &wg) - for _, volId := range volIds { - go reconfigPolicyParallel(ctx, volId, policyID.UniqueId, &wg) - - } - wg.Wait() - - for _, pvclaim := range pvcs { - ginkgo.By(fmt.Sprintf("Waiting for labels %+v to be updated for pvc %s in namespace %s", - labels, pvclaim.Name, namespace)) - pv := getPvFromClaim(client, namespace, pvclaim.Name) - err = e2eVSphere.waitForLabelsToBeUpdated(pv.Spec.CSI.VolumeHandle, labels, - string(cnstypes.CnsKubernetesEntityTypePVC), pvclaim.Name, pvclaim.Namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By(fmt.Sprintf("Waiting for labels %+v to be updated for pv %s", - labels, pv.Name)) - err = e2eVSphere.waitForLabelsToBeUpdated(pv.Spec.CSI.VolumeHandle, labels, - string(cnstypes.CnsKubernetesEntityTypePV), pv.Name, pv.Namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - ginkgo.By("Verify the data on the PVCs match what was written in step 7") - verifyKnownDataInPod(f, pods[0], testdataFile) - - for _, volId := range volIds { - ginkgo.By(fmt.Sprintf("Wait for relocation task to complete for volumeID: %s", volId)) - waitForCNSTaskToComplete(ctx, volIdToCnsRelocateVolTask[volId]) - ginkgo.By("Verify relocation of volume is successful") - e2eVSphere.verifyDatastoreMatch(volId, []string{volIdToDsUrlMap[volId]}) - storagePolicyExists, err := e2eVSphere.VerifySpbmPolicyOfVolume(volId, policyNames[0]) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(storagePolicyExists).To(gomega.BeTrue(), "storage policy verification failed") - e2eVSphere.verifyVolumeCompliance(volId, true) - } - - ginkgo.By("Delete the pod created") - err = fpod.DeletePodWithWait(client, pods[0]) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - - /* - Start attached volume's conversion while creation of snapshot and - relocation of volume in parallel - Steps for offline volumes: - 1. Create a SPBM policy with lzt volume allocation for vmfs datastore. - 2. Create SC using policy created in step 1 - 3. Create PVC using SC created in step 2 - 4. Verify that pvc created in step 3 are bound - 5. Create a pod, say pod1 using pvc created in step 4. - 6. Start writing some IO to pod. - 7. Delete pod1. - 8. Relocate CNS volume corresponding to pvc from step 3 to a different datastore. - 9. While relocation is running perform volume conversion - and create a snapshot in parallel. - 10. Verify relocation was successful. - 11. Verify online volume conversion is successful. - 12. Delete all the objects created during the test. - - Steps for online volumes: - 1. Create a SPBM policy with lzt volume allocation for vmfs datastore. - 2. Create SC using policy created in step 1 - 3. Create PVC using SC created in step 2 - 4. Verify that pvc created in step 3 are bound - 5. Create a pod, say pod1 using pvc created in step 4. - 6. Start writing some IO to pod which run in parallel to steps 6-7. - 7. Relocate CNS volume corresponding to pvc from step 3 to a different datastore. - 8. While relocation is running perform volume conversion - and create a snapshot in parallel. - 9. Verify the IO written so far. - 10. Verify relocation was successful. - 11. Verify online volume conversion is successful. - 12. Delete all the objects created during the test. - */ - ginkgo.It("[csi-block-vanilla][csi-block-vanilla-parallelized]"+ - " Start attached volume's conversion while creation of snapshot and"+ - " relocation of volume in parallel", func() { - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - sharedvmfsURL, sharedvmfs2URL := "", "" - var datastoreUrls []string - var policyName string - volIdToDsUrlMap := make(map[string]string) - volIdToCnsRelocateVolTask := make(map[string]*object.Task) - snapToVolIdMap := make(map[string]string) - - sharedvmfsURL = os.Getenv(envSharedVMFSDatastoreURL) - if sharedvmfsURL == "" { - ginkgo.Skip(fmt.Sprintf("Env %v is missing", envSharedVMFSDatastoreURL)) - } - - sharedvmfs2URL = os.Getenv(envSharedVMFSDatastore2URL) - if sharedvmfsURL == "" { - ginkgo.Skip(fmt.Sprintf("Env %v is missing", envSharedVMFSDatastore2URL)) - } - datastoreUrls = append(datastoreUrls, sharedvmfsURL, sharedvmfs2URL) - - scParameters := make(map[string]string) - policyNames := []string{} - pvcs := []*v1.PersistentVolumeClaim{} - pvclaims2d := [][]*v1.PersistentVolumeClaim{} - - rand.New(rand.NewSource(time.Now().UnixNano())) - suffix := fmt.Sprintf("-%v-%v", time.Now().UnixNano(), rand.Intn(10000)) - categoryName := "category" + suffix - tagName := "tag" + suffix - - catID, tagID := createCategoryNTag(ctx, categoryName, tagName) - defer func() { - deleteCategoryNTag(ctx, catID, tagID) - }() - - attachTagToDS(ctx, tagID, sharedvmfsURL) - defer func() { - detachTagFromDS(ctx, tagID, sharedvmfsURL) - }() - - attachTagToDS(ctx, tagID, sharedvmfs2URL) - defer func() { - detachTagFromDS(ctx, tagID, sharedvmfs2URL) - }() - - ginkgo.By("create SPBM policy with lzt volume allocation") - ginkgo.By("create a storage class with a SPBM policy created from step 1") - ginkgo.By("create a PVC each using the storage policy created from step 2") - var storageclass *storagev1.StorageClass - var pvclaim *v1.PersistentVolumeClaim - var err error - var policyID *pbmtypes.PbmProfileId - - policyID, policyName = createVmfsStoragePolicy( - ctx, pc, lztAllocType, map[string]string{categoryName: tagName}) - defer func() { - deleteStoragePolicy(ctx, pc, policyID) - }() - policyNames = append(policyNames, policyName) - - framework.Logf("CNS_TEST: Running for vanilla k8s setup") - scParameters[scParamStoragePolicyName] = policyName - storageclass, pvclaim, err = createPVCAndStorageClass(client, - namespace, nil, scParameters, "", nil, "", true, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pvclaim2, err := createPVC(client, namespace, nil, "", storageclass, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pvcs = append(pvcs, pvclaim, pvclaim2) - pvclaims2d = append(pvclaims2d, []*v1.PersistentVolumeClaim{pvclaim}) - pvclaims2d = append(pvclaims2d, []*v1.PersistentVolumeClaim{pvclaim2}) - - defer func() { - if vanillaCluster { - ginkgo.By("Delete the SCs created in step 2") - err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }() - - ginkgo.By("Verify the PVCs created in step 3 are bound") - pvs, err := fpv.WaitForPVClaimBoundPhase(client, pvcs, framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - volIds := []string{} - ginkgo.By("Verify that the created CNS volumes are compliant and have correct policy id") - for i, pv := range pvs { - volumeID := pv.Spec.CSI.VolumeHandle - if guestCluster { - volumeID = getVolumeIDFromSupervisorCluster(pv.Spec.CSI.VolumeHandle) - gomega.Expect(volumeID).NotTo(gomega.BeEmpty()) - } - storagePolicyMatches, err := e2eVSphere.VerifySpbmPolicyOfVolume(volumeID, policyNames[i/2]) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(storagePolicyMatches).To(gomega.BeTrue(), "storage policy verification failed") - e2eVSphere.verifyVolumeCompliance(volumeID, true) - volIds = append(volIds, volumeID) - } - - defer func() { - ginkgo.By("Delete the PVCs created in step 3") - for i, pvc := range pvcs { - err := fpv.DeletePersistentVolumeClaim(client, pvc.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volIds[i]) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - }() - - ginkgo.By("Create pods with using the PVCs created in step 3 and wait for them to be ready") - ginkgo.By("verify we can read and write on the PVCs") - pods := createMultiplePods(ctx, client, pvclaims2d, true) - defer func() { - ginkgo.By("Delete the pod created") - deletePodsAndWaitForVolsToDetach(ctx, client, pods, true) - }() - - ginkgo.By("Verify if VolumeID is created on the given datastores") - // Get the destination ds url where the volume will get relocated - destDsUrl := "" - for _, volId := range volIds { - dsUrlWhereVolumeIsPresent := fetchDsUrl4CnsVol(e2eVSphere, volId) - framework.Logf("Volume is present on %s for volume: %s", dsUrlWhereVolumeIsPresent, volId) - e2eVSphere.verifyDatastoreMatch(volId, datastoreUrls) - for _, dsurl := range datastoreUrls { - if dsurl != dsUrlWhereVolumeIsPresent { - destDsUrl = dsurl - } - } - framework.Logf("dest url: %s", destDsUrl) - volIdToDsUrlMap[volId] = destDsUrl - } - - snaps := []*snapV1.VolumeSnapshot{} - //Get snapshot client using the rest config - restConfig := getRestConfigClient() - snapc, err := snapclient.NewForConfig(restConfig) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Create volume snapshot class") - volumeSnapshotClass, err := snapc.SnapshotV1().VolumeSnapshotClasses().Create(ctx, - getVolumeSnapshotClassSpec(snapV1.DeletionPolicy("Delete"), nil), metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - framework.Logf("Volume snapshot class with name %q created", volumeSnapshotClass.Name) - - defer func() { - err := snapc.SnapshotV1().VolumeSnapshotClasses().Delete( - ctx, volumeSnapshotClass.Name, metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - rand.New(rand.NewSource(time.Now().Unix())) - testdataFile := fmt.Sprintf("/tmp/testdata_%v_%v", time.Now().Unix(), rand.Intn(1000)) - ginkgo.By(fmt.Sprintf("Creating a 100mb test data file %v", testdataFile)) - op, err := exec.Command("dd", "if=/dev/urandom", fmt.Sprintf("of=%v", testdataFile), - "bs=1M", "count=100").Output() - fmt.Println(op) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - op, err = exec.Command("rm", "-f", testdataFile).Output() - fmt.Println(op) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("delete pod1") - deletePodsAndWaitForVolsToDetach(ctx, client, []*v1.Pod{pods[1]}, true) - - ginkgo.By("Start relocation of volume to a different datastore") - for _, volId := range volIds { - dsRefDest := getDsMoRefFromURL(ctx, volIdToDsUrlMap[volId]) - task, err := e2eVSphere.cnsRelocateVolume(e2eVSphere, ctx, volId, dsRefDest, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - volIdToCnsRelocateVolTask[volId] = task - framework.Logf("Waiting for a few seconds for relocation to be started properly on VC") - time.Sleep(time.Duration(10) * time.Second) - } - - ginkgo.By("Perform volume conversion and write IO to pod and" + - " create a snapshot while relocating volume to different datastore") - err = updateVmfsPolicyAlloctype(ctx, pc, eztAllocType, policyName, policyID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - var wg sync.WaitGroup - ch := make(chan *snapV1.VolumeSnapshot) - lock := &sync.Mutex{} - wg.Add(1 + 2*len(volIds)) - go writeKnownData2PodInParallel(f, pods[0], testdataFile, &wg) - for i := range volIds { - go reconfigPolicyParallel(ctx, volIds[i], policyID.UniqueId, &wg) - go createSnapshotInParallel(ctx, namespace, snapc, pvcs[i].Name, volumeSnapshotClass.Name, - ch, lock, &wg) - go func(volID string) { - for v := range ch { - snaps = append(snaps, v) - snapToVolIdMap[v.Name] = volID - } - }(volIds[i]) - } - wg.Wait() - - ginkgo.By("Verify the data on the PVCs match what was written in step 7") - verifyKnownDataInPod(f, pods[0], testdataFile) - - for _, snap := range snaps { - volumeSnapshot := snap - ginkgo.By("Verify volume snapshot is created") - volumeSnapshot, err = waitForVolumeSnapshotReadyToUse(*snapc, ctx, namespace, volumeSnapshot.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - framework.Logf("snapshot restore size is : %s", volumeSnapshot.Status.RestoreSize.String()) - gomega.Expect(volumeSnapshot.Status.RestoreSize.Cmp(pvclaim.Spec.Resources.Requests[v1.ResourceStorage])).To( - gomega.BeZero()) - ginkgo.By("Verify volume snapshot content is created") - snapshotContent, err := snapc.SnapshotV1().VolumeSnapshotContents().Get(ctx, - *volumeSnapshot.Status.BoundVolumeSnapshotContentName, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(*snapshotContent.Status.ReadyToUse).To(gomega.BeTrue()) - - framework.Logf("Get volume snapshot ID from snapshot handle") - snapshothandle := *snapshotContent.Status.SnapshotHandle - snapshotId := strings.Split(snapshothandle, "+")[1] - - defer func() { - framework.Logf("Delete volume snapshot %v", volumeSnapshot.Name) - err = snapc.SnapshotV1().VolumeSnapshots(namespace).Delete( - ctx, volumeSnapshot.Name, metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - framework.Logf("Verify snapshot entry %v is deleted from CNS for volume %v", snapshotId, - snapToVolIdMap[volumeSnapshot.Name]) - err = waitForCNSSnapshotToBeDeleted(snapToVolIdMap[volumeSnapshot.Name], snapshotId) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Query CNS and check the volume snapshot entry") - err = waitForCNSSnapshotToBeCreated(snapToVolIdMap[volumeSnapshot.Name], snapshotId) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - for _, volId := range volIds { - ginkgo.By(fmt.Sprintf("Wait for relocation task to complete for volumeID: %s", volId)) - waitForCNSTaskToComplete(ctx, volIdToCnsRelocateVolTask[volId]) - ginkgo.By("Verify relocation of volume is successful") - e2eVSphere.verifyDatastoreMatch(volId, []string{volIdToDsUrlMap[volId]}) - storagePolicyExists, err := e2eVSphere.VerifySpbmPolicyOfVolume(volId, policyNames[0]) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(storagePolicyExists).To(gomega.BeTrue(), "storage policy verification failed") - e2eVSphere.verifyVolumeCompliance(volId, true) - } - - ginkgo.By("Delete the pod created") - err = fpod.DeletePodWithWait(client, pods[0]) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - }) - -}) - -// fillVolumesInPods fills the volumes in pods after leaving 100m for FS metadata -func fillVolumeInPods(f *framework.Framework, pods []*v1.Pod) { - for _, pod := range pods { - size, err := getFSSizeMb(f, pod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - writeRandomDataOnPod(pod, size-100) // leaving 100m for FS metadata - } -} +// fillVolumesInPods fills the volumes in pods after leaving 100m for FS metadata +func fillVolumeInPods(f *framework.Framework, pods []*v1.Pod) { + for _, pod := range pods { + size, err := getFSSizeMb(f, pod) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + writeRandomDataOnPod(pod, size-100) // leaving 100m for FS metadata + } +} // writeRandomDataOnPod runs dd on the given pod and write count in Mib func writeRandomDataOnPod(pod *v1.Pod, count int64) { @@ -3121,7 +1474,6 @@ func setVpxdTaskTimeout(ctx context.Context, taskTimeout int) { func writeKnownData2PodInParallel( f *framework.Framework, pod *v1.Pod, testdataFile string, wg *sync.WaitGroup, size ...int64) { - defer ginkgo.GinkgoRecover() defer wg.Done() writeKnownData2Pod(f, pod, testdataFile, size...) } @@ -3165,7 +1517,7 @@ func verifyKnownDataInPod(f *framework.Framework, pod *v1.Pod, testdataFile stri "/bin/sh", "-c", "dd if=/mnt/volume1/f1 of=/mnt/volume1/testdata bs=1M count=100 skip=" + skip} _ = framework.RunKubectlOrDie(pod.Namespace, cmd...) _ = framework.RunKubectlOrDie(pod.Namespace, "cp", - fmt.Sprintf("%v/%v:mnt/volume1/testdata", pod.Namespace, pod.Name), + fmt.Sprintf("%v/%v:/mnt/volume1/testdata", pod.Namespace, pod.Name), testdataFile+pod.Name) framework.Logf("Running diff with source file and file from pod %v for 100M starting %vM", pod.Name, skip) op, err := exec.Command("diff", testdataFile, testdataFile+pod.Name).Output() @@ -3174,10 +1526,3 @@ func verifyKnownDataInPod(f *framework.Framework, pod *v1.Pod, testdataFile stri gomega.Expect(len(op)).To(gomega.BeZero()) } } - -func reconfigPolicyParallel(ctx context.Context, volID string, policyId string, wg *sync.WaitGroup) { - defer ginkgo.GinkgoRecover() - defer wg.Done() - err := e2eVSphere.reconfigPolicy(ctx, volID, policyId) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) -} diff --git a/tests/e2e/preferential_topology.go b/tests/e2e/preferential_topology.go index ae2e1e2d01..3bf78831d4 100644 --- a/tests/e2e/preferential_topology.go +++ b/tests/e2e/preferential_topology.go @@ -78,7 +78,6 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision vcAddress string sshClientConfig *ssh.ClientConfig nimbusGeneratedK8sVmPwd string - clientIndex int ) ginkgo.BeforeEach(func() { @@ -173,9 +172,7 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision } //set preferred datatsore time interval - setPreferredDatastoreTimeInterval(client, ctx, csiNamespace, csiReplicas, false) - - clientIndex = 0 + setPreferredDatastoreTimeInterval(client, ctx, csiNamespace, namespace, csiReplicas) }) @@ -190,11 +187,11 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision gomega.Expect(err).NotTo(gomega.HaveOccurred()) } framework.Logf("Perform preferred datastore tags cleanup after test completion") - err = deleteTagCreatedForPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks, false) + err = deleteTagCreatedForPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks) gomega.Expect(err).NotTo(gomega.HaveOccurred()) framework.Logf("Recreate preferred datastore tags post cleanup") - err = createTagForPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks, false) + err = createTagForPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks) gomega.Expect(err).NotTo(gomega.HaveOccurred()) if isSPSServiceStopped { @@ -233,8 +230,7 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision 18. Perform cleanup. Delete StatefulSet, PVC, PV and SC. 19. Remove datastore preference tags as part of cleanup. */ - ginkgo.It("Tag single preferred datastore each in rack-1 and rack-2 "+ - "and verify it is honored", ginkgo.Label(p0, topology, preferential, block, vanilla, level5), func() { + ginkgo.It("Tag single preferred datastore each in rack-1 and rack-2 and verify it is honored", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() preferredDatastoreChosen = 1 @@ -243,13 +239,13 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision // choose preferred datastore ginkgo.By("Tag preferred datastore for volume provisioning in rack-2(cluster-2))") preferredDatastorePaths, err = tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[1], - preferredDatastoreChosen, nonShareddatastoreListMapRack2, nil, false, clientIndex) + preferredDatastoreChosen, nonShareddatastoreListMapRack2, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) preferredDatastoreRack2 := preferredDatastorePaths[0] defer func() { ginkgo.By("Remove preferred datastore tag") err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastoreRack2, - allowedTopologyRacks[1], false, clientIndex) + allowedTopologyRacks[1]) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() @@ -296,24 +292,23 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision // verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") err = verifyVolumeProvisioningForStatefulSet(ctx, client, statefulset, namespace, preferredDatastorePaths, - nonShareddatastoreListMapRack2, false, false, false, nil) + nonShareddatastoreListMapRack2, false, false) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, - namespace, allowedTopologyForRack2, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, + namespace, allowedTopologyForRack2, false) // choose preferred datastore in rack-1 ginkgo.By("Tag preferred datatstore for volume provisioning in rack-1(cluster-1))") preferredDatastorePaths, err = tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[0], - preferredDatastoreChosen, nonShareddatastoreListMapRack1, nil, false, clientIndex) + preferredDatastoreChosen, nonShareddatastoreListMapRack1, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) defer func() { ginkgo.By("Remove preferred datastore tag") err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[0], - allowedTopologyRacks[0], false, clientIndex) + allowedTopologyRacks[0]) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() @@ -370,13 +365,12 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision // verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") verifyVolumeProvisioningForStandalonePods(ctx, client, pod, namespace, preferredDatastorePaths, - nonShareddatastoreListMapRack1, false, nil) + nonShareddatastoreListMapRack1) ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate " + "node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStandalonePodLevel5(ctx, client, pod, namespace, - allowedTopologyForRack1, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsFoStandalonePodLevel5(ctx, client, pod, namespace, + allowedTopologyForRack1) }) /* @@ -413,19 +407,19 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision // choose preferred datastore ginkgo.By("Tag preferred datastore for volume provisioning in rack-3(cluster-3))") preferredDatastorePaths, err := tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[2], - preferredDatastoreChosen, nonShareddatastoreListMapRack3, nil, false, clientIndex) + preferredDatastoreChosen, nonShareddatastoreListMapRack3, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) preferredDatastore, err := tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[2], - preferredDatastoreChosen, shareddatastoreListMap, nil, false, clientIndex) + preferredDatastoreChosen, shareddatastoreListMap, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) preferredDatastorePaths = append(preferredDatastorePaths, preferredDatastore...) defer func() { ginkgo.By("Remove preferred datatsore tag") for i := 0; i < len(preferredDatastorePaths); i++ { err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[i], - allowedTopologyRacks[2], false, clientIndex) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + allowedTopologyRacks[2]) } + gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() framework.Logf("Waiting for %v for preferred datastore to get refreshed in the environment", @@ -471,32 +465,29 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision //verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") err = verifyVolumeProvisioningForStatefulSet(ctx, client, statefulset, namespace, preferredDatastorePaths, - rack3DatastoreListMap, false, false, false, nil) + rack3DatastoreListMap, false, false) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, - namespace, allowedTopologyForRack3, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, + namespace, allowedTopologyForRack3, false) // perform statefulset scaleup replicas = 10 ginkgo.By("Scale up statefulset replica count from 3 to 10") - err = scaleUpStatefulSetPod(ctx, client, statefulset, namespace, replicas, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleUpStatefulSetPod(ctx, client, statefulset, namespace, replicas, false) //verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") err = verifyVolumeProvisioningForStatefulSet(ctx, client, statefulset, namespace, preferredDatastorePaths, - rack3DatastoreListMap, false, false, false, nil) + rack3DatastoreListMap, false, false) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, - namespace, allowedTopologyForRack3, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, + namespace, allowedTopologyForRack3, false) }) /* @@ -536,18 +527,18 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision // choose preferred datastore ginkgo.By("Tag preferred datatsore for volume provisioning in rack-2(cluster-2))") preferredDatastorePaths, err := tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[1], - preferredDatastoreChosen, nonShareddatastoreListMapRack2, nil, false, clientIndex) + preferredDatastoreChosen, nonShareddatastoreListMapRack2, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) preferredDatastoreChosen = 1 preferredDatastore, err := tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[1], - preferredDatastoreChosen, shareddatastoreListMap, nil, false, clientIndex) + preferredDatastoreChosen, shareddatastoreListMap, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) preferredDatastorePaths = append(preferredDatastorePaths, preferredDatastore...) defer func() { ginkgo.By("Remove preferred datastore tag") for i := 0; i < len(preferredDatastorePaths); i++ { err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[i], - allowedTopologyRacks[1], false, clientIndex) + allowedTopologyRacks[1]) } gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() @@ -596,38 +587,34 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision //verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") err = verifyVolumeProvisioningForStatefulSet(ctx, client, statefulset, namespace, preferredDatastorePaths, - rack2DatastoreListMap, false, false, false, nil) + rack2DatastoreListMap, false, false) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, - namespace, allowedTopologyForRack2, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, + namespace, allowedTopologyForRack2, false) // perform statefulset scaleup replicas = 10 ginkgo.By("Scale up statefulset replica count from 3 to 10") - err = scaleUpStatefulSetPod(ctx, client, statefulset, namespace, replicas, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleUpStatefulSetPod(ctx, client, statefulset, namespace, replicas, false) //verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") err = verifyVolumeProvisioningForStatefulSet(ctx, client, statefulset, namespace, preferredDatastorePaths, - rack2DatastoreListMap, false, false, false, nil) + rack2DatastoreListMap, false, false) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, - namespace, allowedTopologyForRack2, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, + namespace, allowedTopologyForRack2, false) // perform statefulset scaledown replicas = 5 ginkgo.By("Scale down statefulset replica count from 10 to 5") - err = scaleDownStatefulSetPod(ctx, client, statefulset, namespace, replicas, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleDownStatefulSetPod(ctx, client, statefulset, namespace, replicas, false) }) /* @@ -658,12 +645,12 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision // choose preferred datastore ginkgo.By("Tag preferred datatsore for volume provisioning in rack-1(cluster-1))") preferredDatastorePaths, err := tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[0], - preferredDatastoreChosen, shareddatastoreListMap, nil, false, clientIndex) + preferredDatastoreChosen, shareddatastoreListMap, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) defer func() { ginkgo.By("Remove preferred datatsore tag") err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[0], - allowedTopologyRacks[0], false, clientIndex) + allowedTopologyRacks[0]) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() @@ -711,14 +698,13 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision //verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") err = verifyVolumeProvisioningForStatefulSet(ctx, client, statefulset, namespace, preferredDatastorePaths, - shareddatastoreListMap, true, false, false, nil) + shareddatastoreListMap, true, false) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, - namespace, allowedTopologies, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, + namespace, allowedTopologies, false) }) /* @@ -749,7 +735,7 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision // choose preferred datastore ginkgo.By("Tag preferred datatsore for volume provisioning in rack-2(cluster-2))") preferredDatastorePaths, err := tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[0], - preferredDatastoreChosen, shareddatastoreListMap, nil, false, clientIndex) + preferredDatastoreChosen, shareddatastoreListMap, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = tagSameDatastoreAsPreferenceToDifferentRacks(masterIp, sshClientConfig, allowedTopologyRacks[1], preferredDatastoreChosen, preferredDatastorePaths) @@ -758,7 +744,7 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision ginkgo.By("Remove preferred datatsore tag") for j := 0; j < len(allowedTopologyRacks)-1; j++ { err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[0], - allowedTopologyRacks[j], false, clientIndex) + allowedTopologyRacks[j]) } gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() @@ -806,14 +792,13 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision //verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") err = verifyVolumeProvisioningForStatefulSet(ctx, client, statefulset, namespace, preferredDatastorePaths, - shareddatastoreListMap, false, false, false, nil) + shareddatastoreListMap, false, false) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, - namespace, allowedTopologyForRack2, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, + namespace, allowedTopologyForRack2, false) }) /* @@ -847,7 +832,7 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision // choose preferred datastore ginkgo.By("Tag preferred datatsore for volume provisioning") preferredDatastorePaths, err = tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[0], - preferredDatastoreChosen, shareddatastoreListMap, nil, false, clientIndex) + preferredDatastoreChosen, shareddatastoreListMap, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) for i := 1; i < len(allowedTopologyRacks); i++ { err = tagSameDatastoreAsPreferenceToDifferentRacks(masterIp, sshClientConfig, allowedTopologyRacks[i], @@ -858,7 +843,7 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision ginkgo.By("Remove preferred datatsore tag") for j := 0; j < len(allowedTopologyRacks); j++ { err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[0], - allowedTopologyRacks[j], false, clientIndex) + allowedTopologyRacks[j]) } gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() @@ -904,14 +889,13 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision //verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") err = verifyVolumeProvisioningForStatefulSet(ctx, client, statefulset, namespace, preferredDatastorePaths, - shareddatastoreListMap, true, false, false, nil) + shareddatastoreListMap, true, false) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, - namespace, allowedTopologies, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, + namespace, allowedTopologies, false) }) /* @@ -941,12 +925,12 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision // choose preferred datastore ginkgo.By("Tag preferred datastore for volume provisioning") preferredDatastorePaths, err = tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[2], - preferredDatastoreChosen, shareddatastoreListMap, nil, false, clientIndex) + preferredDatastoreChosen, shareddatastoreListMap, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) defer func() { ginkgo.By("Remove the preferred datastore tag") err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[0], - allowedTopologyRacks[2], false, clientIndex) + allowedTopologyRacks[2]) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() @@ -991,14 +975,13 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision //verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") err = verifyVolumeProvisioningForStatefulSet(ctx, client, statefulset, namespace, preferredDatastorePaths, - shareddatastoreListMap, true, false, false, nil) + shareddatastoreListMap, true, false) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, - namespace, allowedTopologies, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, + namespace, allowedTopologies, false) }) /* @@ -1036,7 +1019,7 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision // choose preferred datastore ginkgo.By("Tag preferred datastore for volume provisioning in rack-1(cluster-1))") preferredDatastorePaths, err = tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[0], - preferredDatastoreChosen, nonShareddatastoreListMapRack1, nil, false, clientIndex) + preferredDatastoreChosen, nonShareddatastoreListMapRack1, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) framework.Logf("Waiting for %v for preferred datastore to get refreshed in the environment", @@ -1081,28 +1064,27 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision //verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") err = verifyVolumeProvisioningForStatefulSet(ctx, client, sts1, namespace, preferredDatastorePaths, - nonShareddatastoreListMapRack1, false, false, false, nil) + nonShareddatastoreListMapRack1, false, false) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, sts1, - namespace, allowedTopologyForRack1, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, sts1, + namespace, allowedTopologyForRack1, true) ginkgo.By("Remove preferred datatsore tag which is chosen for volume provisioning") err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[0], - allowedTopologyRacks[0], false, clientIndex) + allowedTopologyRacks[0]) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Tag new preferred datastore from rack-1 for volume provisioning") preferredDatastorePaths, err = tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[0], - preferredDatastoreChosen, shareddatastoreListMap, nil, false, clientIndex) + preferredDatastoreChosen, shareddatastoreListMap, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) defer func() { ginkgo.By("Remove the datastore preference chosen for volume provisioning") err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[0], - allowedTopologyRacks[0], false, clientIndex) + allowedTopologyRacks[0]) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() @@ -1155,13 +1137,12 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision // verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") verifyVolumeProvisioningForStandalonePods(ctx, client, pod, namespace, preferredDatastorePaths, - shareddatastoreListMap, false, nil) + shareddatastoreListMap) ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate " + "node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStandalonePodLevel5(ctx, client, pod, namespace, - allowedTopologyForRack1, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsFoStandalonePodLevel5(ctx, client, pod, namespace, + allowedTopologyForRack1) }) @@ -1206,7 +1187,7 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision // choose preferred datastore ginkgo.By("Tag preferred datastore for volume provisioning in rack-3(cluster-3))") preferredDatastorePaths, err = tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[2], - preferredDatastoreChosen, nonShareddatastoreListMapRack3, nil, false, clientIndex) + preferredDatastoreChosen, nonShareddatastoreListMapRack3, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) preferredDatastore := preferredDatastorePaths @@ -1252,29 +1233,28 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision //verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") err = verifyVolumeProvisioningForStatefulSet(ctx, client, sts1, namespace, preferredDatastorePaths, - nonShareddatastoreListMapRack3, false, false, false, nil) + nonShareddatastoreListMapRack3, false, false) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, sts1, - namespace, allowedTopologyForRack3, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, sts1, + namespace, allowedTopologyForRack3, false) ginkgo.By("Remove preferred datatsore tag chosen for volume provisioning") err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[0], - allowedTopologyRacks[2], false, clientIndex) + allowedTopologyRacks[2]) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Tag new preferred datatsore for volume provisioning in rack-3") preferredDatastorePaths, err = tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[2], - preferredDatastoreChosen, nonShareddatastoreListMapRack3, preferredDatastorePaths, false, clientIndex) + preferredDatastoreChosen, nonShareddatastoreListMapRack3, preferredDatastorePaths) gomega.Expect(err).NotTo(gomega.HaveOccurred()) defer func() { ginkgo.By("Remove preferred datatsore tag") for i := 0; i < len(preferredDatastorePaths); i++ { err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[i], - allowedTopologyRacks[2], false, clientIndex) + allowedTopologyRacks[2]) } gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() @@ -1328,32 +1308,29 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision // verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") verifyVolumeProvisioningForStandalonePods(ctx, client, pod, namespace, preferredDatastorePaths, - nonShareddatastoreListMapRack3, false, nil) + nonShareddatastoreListMapRack3) ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate " + "node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStandalonePodLevel5(ctx, client, pod, namespace, - allowedTopologyForRack3, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsFoStandalonePodLevel5(ctx, client, pod, namespace, + allowedTopologyForRack3) // perform statefulset scaleup sts1Replicas = 10 ginkgo.By("Scale up statefulset replica count from 3 to 10") preferredDatastorePaths = append(preferredDatastorePaths, preferredDatastore...) - err = scaleUpStatefulSetPod(ctx, client, sts1, namespace, sts1Replicas, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleUpStatefulSetPod(ctx, client, sts1, namespace, sts1Replicas, false) //verifying volume is provisioned on the preferred datastore ginkgo.By("Verify volume is provisioned on the specified datatsore") err = verifyVolumeProvisioningForStatefulSet(ctx, client, sts1, namespace, preferredDatastorePaths, - nonShareddatastoreListMapRack3, false, true, false, nil) + nonShareddatastoreListMapRack3, false, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, sts1, - namespace, allowedTopologyForRack3, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, sts1, + namespace, allowedTopologyForRack3, true) }) /* @@ -1403,7 +1380,7 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision // choose preferred datastore ginkgo.By("Tag preferred datastore for volume provisioning in rack-1(cluster-1))") preferredDatastorePaths, err = tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[0], - preferredDatastoreChosen, shareddatastoreListMap, nil, false, clientIndex) + preferredDatastoreChosen, shareddatastoreListMap, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) framework.Logf("Waiting for %v for preferred datastore to get refreshed in the environment", @@ -1448,23 +1425,22 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision // verify volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") err = verifyVolumeProvisioningForStatefulSet(ctx, client, sts1, namespace, preferredDatastorePaths, - shareddatastoreListMap, false, false, false, nil) + shareddatastoreListMap, false, false) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, sts1, - namespace, allowedTopologyForRack1, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, sts1, + namespace, allowedTopologyForRack1, false) ginkgo.By("Remove preferred datastore tag chosen for volume provisioning") err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[0], - allowedTopologyRacks[0], false, clientIndex) + allowedTopologyRacks[0]) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Tag new preferred datatsore for volume provisioning in rack-1(cluster-1)") preferredDatastore, err := tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[0], - preferredDatastoreChosen, nonShareddatastoreListMapRack1, nil, false, clientIndex) + preferredDatastoreChosen, nonShareddatastoreListMapRack1, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) preferredDatastorePaths = append(preferredDatastorePaths, preferredDatastore...) @@ -1475,44 +1451,41 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision // perform statefulset scaleup sts1Replicas = 13 ginkgo.By("Scale up statefulset replica count from 3 to 13") - err = scaleUpStatefulSetPod(ctx, client, sts1, namespace, sts1Replicas, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleUpStatefulSetPod(ctx, client, sts1, namespace, sts1Replicas, false) // verifying volume provisioning ginkgo.By("Verify volume is provisioned on the specified datatsore") err = verifyVolumeProvisioningForStatefulSet(ctx, client, sts1, namespace, preferredDatastorePaths, - rack1DatastoreListMap, false, false, false, nil) + rack1DatastoreListMap, false, false) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, sts1, - namespace, allowedTopologyForRack1, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, sts1, + namespace, allowedTopologyForRack1, false) // perform statefulset scaledown sts1Replicas = 6 ginkgo.By("Scale down statefulset replica count from 13 to 6") - err = scaleDownStatefulSetPod(ctx, client, sts1, namespace, sts1Replicas, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleDownStatefulSetPod(ctx, client, sts1, namespace, sts1Replicas, false) ginkgo.By("Remove preferred datastore tag chosen for volume provisioning") err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[1], - allowedTopologyRacks[0], false, clientIndex) + allowedTopologyRacks[0]) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Tag new datastore chosen for volume provisioning") preferredDatastore, err = tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[0], - preferredDatastoreChosen, nonShareddatastoreListMapRack1, preferredDatastorePaths, false, clientIndex) + preferredDatastoreChosen, nonShareddatastoreListMapRack1, preferredDatastorePaths) gomega.Expect(err).NotTo(gomega.HaveOccurred()) preferredDatastorePaths = append(preferredDatastorePaths, preferredDatastore...) defer func() { ginkgo.By("Remove preferred datastore tags chosen for volume provisioning") for i := 0; i < len(preferredDatastorePaths); i++ { err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[i], - allowedTopologyRacks[0], false, clientIndex) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + allowedTopologyRacks[0]) } + gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() framework.Logf("Waiting for %v for preferred datastore to get refreshed in the environment", @@ -1522,20 +1495,18 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision // perform statefulset scaleup sts1Replicas = 20 ginkgo.By("Scale up statefulset replica count from 6 to 20") - err = scaleUpStatefulSetPod(ctx, client, sts1, namespace, sts1Replicas, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleUpStatefulSetPod(ctx, client, sts1, namespace, sts1Replicas, false) //verifying volume provisioning ginkgo.By("Verify volume is provisioned on the specified datatsore") err = verifyVolumeProvisioningForStatefulSet(ctx, client, sts1, namespace, preferredDatastorePaths, - rack1DatastoreListMap, false, false, false, nil) + rack1DatastoreListMap, false, false) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, sts1, - namespace, allowedTopologyForRack1, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, sts1, + namespace, allowedTopologyForRack1, false) }) /* @@ -1595,7 +1566,7 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision // choose preferred datastore ginkgo.By("Tag preferred datatsore for volume provisioning in rack-1(cluster-1)") preferredDatastorePaths, err = tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[0], - preferredDatastoreChosen, nonShareddatastoreListMapRack1, nil, false, clientIndex) + preferredDatastoreChosen, nonShareddatastoreListMapRack1, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) framework.Logf("Waiting for %v for preferred datastore to get refreshed in the environment", @@ -1639,23 +1610,22 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision // verify volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") err = verifyVolumeProvisioningForStatefulSet(ctx, client, sts1, namespace, preferredDatastorePaths, - nonShareddatastoreListMapRack1, true, false, false, nil) + nonShareddatastoreListMapRack1, true, false) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, sts1, - namespace, allowedTopologies, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, sts1, + namespace, allowedTopologies, false) ginkgo.By("Remove preferred datatsore tag which was chosen for volume provisioning in rack-1") err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[0], - allowedTopologyRacks[0], false, clientIndex) + allowedTopologyRacks[0]) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Tag new preferred datatsore for volume provisioning in rack-2(cluster-2)") preferredDatastorePaths, err = tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[1], - preferredDatastoreChosen, nonShareddatastoreListMapRack2, nil, false, clientIndex) + preferredDatastoreChosen, nonShareddatastoreListMapRack2, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) framework.Logf("Waiting for %v for preferred datastore to get refreshed in the environment", @@ -1665,34 +1635,32 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision // perform statefulset scaleup sts1Replicas = 7 ginkgo.By("Scale up statefulset replica count from 3 to 7") - err = scaleUpStatefulSetPod(ctx, client, sts1, namespace, sts1Replicas, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleUpStatefulSetPod(ctx, client, sts1, namespace, sts1Replicas, false) //verify volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") err = verifyVolumeProvisioningForStatefulSet(ctx, client, sts1, namespace, preferredDatastorePaths, - nonShareddatastoreListMapRack2, true, false, false, nil) + nonShareddatastoreListMapRack2, true, false) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, sts1, - namespace, allowedTopologies, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, sts1, + namespace, allowedTopologies, false) ginkgo.By("Remove the datastore preference chosen for volume provisioning") err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[0], - allowedTopologyRacks[1], false, clientIndex) + allowedTopologyRacks[1]) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Tag new preferred datatsore for volume provisioning in rack-3(cluster-3)") preferredDatastorePaths, err = tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[2], - preferredDatastoreChosen, nonShareddatastoreListMapRack3, nil, false, clientIndex) + preferredDatastoreChosen, nonShareddatastoreListMapRack3, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) defer func() { ginkgo.By("Remove the datastore preference chosen for volume provisioning") err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[0], - allowedTopologyRacks[2], false, clientIndex) + allowedTopologyRacks[2]) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() @@ -1704,20 +1672,18 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision // perform statefulset scaleup sts1Replicas = 13 ginkgo.By("Scale up statefulset replica count from 7 to 13") - err = scaleUpStatefulSetPod(ctx, client, sts1, namespace, sts1Replicas, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleUpStatefulSetPod(ctx, client, sts1, namespace, sts1Replicas, false) // verify volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") err = verifyVolumeProvisioningForStatefulSet(ctx, client, sts1, namespace, preferredDatastorePaths, - nonShareddatastoreListMapRack3, true, false, false, nil) + nonShareddatastoreListMapRack3, true, false) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, sts1, - namespace, allowedTopologies, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, sts1, + namespace, allowedTopologies, false) }) /* @@ -1754,20 +1720,20 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision // choose preferred datastore ginkgo.By("Tag preferred datatsore for volume provisioning in rack-2(cluster-2)") preferredDatastorePaths, err = tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[1], - preferredDatastoreChosen, nonShareddatastoreListMapRack2, nil, false, clientIndex) + preferredDatastoreChosen, nonShareddatastoreListMapRack2, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) preferredDatastoreChosen = 1 preferredDatastore, err := tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[1], - preferredDatastoreChosen, shareddatastoreListMap, nil, false, clientIndex) + preferredDatastoreChosen, shareddatastoreListMap, nil) preferredDatastorePaths = append(preferredDatastorePaths, preferredDatastore...) gomega.Expect(err).NotTo(gomega.HaveOccurred()) defer func() { ginkgo.By("Remove preferred datastore tag") for i := 0; i < len(preferredDatastorePaths); i++ { err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[i], - allowedTopologyRacks[1], false, clientIndex) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + allowedTopologyRacks[1]) } + gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() framework.Logf("Waiting for %v for preferred datastore to get refreshed in the environment", @@ -1811,14 +1777,13 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision // verify volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") err = verifyVolumeProvisioningForStatefulSet(ctx, client, sts1, namespace, preferredDatastorePaths, - rack2DatastoreListMap, false, false, false, nil) + rack2DatastoreListMap, false, false) gomega.Expect(err).NotTo(gomega.HaveOccurred()) // Verify affinity details ginkgo.By("Verify node and pv topology affinity details") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, sts1, - namespace, allowedTopologyForRack2, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, sts1, + namespace, allowedTopologyForRack2, false) }) /* @@ -1851,17 +1816,17 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision // choose preferred datastore ginkgo.By("Tag preferred datatsore for volume provisioning in rack-2(cluster-2)") preferredDatastorePaths, err = tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[1], - preferredDatastoreChosen, nonShareddatastoreListMapRack2, nil, false, clientIndex) + preferredDatastoreChosen, nonShareddatastoreListMapRack2, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) preferredDatastore, err := tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[1], - preferredDatastoreChosen, shareddatastoreListMap, nil, false, clientIndex) + preferredDatastoreChosen, shareddatastoreListMap, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) preferredDatastorePaths = append(preferredDatastorePaths, preferredDatastore...) defer func() { ginkgo.By("Remove preferred datastore tag") for i := 0; i < len(preferredDatastorePaths); i++ { err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[i], - allowedTopologyRacks[1], false, clientIndex) + allowedTopologyRacks[1]) } gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() @@ -1930,12 +1895,12 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision // choose preferred datastore ginkgo.By("Tag preferred datatsore for volume provisioning in rack-2(cluster-2)") preferredDatastorePaths, err = tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[1], - preferredDatastoreChosen, shareddatastoreListMap, nil, false, clientIndex) + preferredDatastoreChosen, shareddatastoreListMap, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) defer func() { ginkgo.By("Remove preferred datastore tag") err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[0], - allowedTopologyRacks[1], false, clientIndex) + allowedTopologyRacks[1]) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() @@ -1991,13 +1956,12 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision // verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") verifyVolumeProvisioningForStandalonePods(ctx, client, pod, namespace, storagePolicyDs, - nonShareddatastoreListMapRack2, false, nil) + nonShareddatastoreListMapRack2) ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate " + "node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStandalonePodLevel5(ctx, client, pod, namespace, - allowedTopologyForRack2, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsFoStandalonePodLevel5(ctx, client, pod, namespace, + allowedTopologyForRack2) }) /* @@ -2035,24 +1999,24 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision // choose preferred datastore ginkgo.By("Tag different preferred datatsores in different racks") preferredDatastore1, err := tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[0], - preferredDatastoreChosen, nonShareddatastoreListMapRack1, nil, false, clientIndex) + preferredDatastoreChosen, nonShareddatastoreListMapRack1, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) preferredDatastorePaths = append(preferredDatastorePaths, preferredDatastore1...) preferredDatastore2, err := tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[1], - preferredDatastoreChosen, nonShareddatastoreListMapRack2, nil, false, clientIndex) + preferredDatastoreChosen, nonShareddatastoreListMapRack2, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) preferredDatastorePaths = append(preferredDatastorePaths, preferredDatastore2...) preferredDatastore3, err := tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[2], - preferredDatastoreChosen, shareddatastoreListMap, nil, false, clientIndex) + preferredDatastoreChosen, shareddatastoreListMap, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) preferredDatastorePaths = append(preferredDatastorePaths, preferredDatastore3...) defer func() { for i := 0; i < len(allowedTopologyRacks); i++ { err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[i], - allowedTopologyRacks[i], false, clientIndex) + allowedTopologyRacks[i]) } gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() @@ -2115,32 +2079,29 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision //verify volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") err = verifyVolumeProvisioningForStatefulSet(ctx, client, sts1, namespace, preferredDatastorePaths, - allDatastoresListMap, true, false, false, nil) + allDatastoresListMap, true, false) gomega.Expect(err).NotTo(gomega.HaveOccurred()) // Verify affinity details ginkgo.By("Verify node and pv topology affinity details") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, sts1, - namespace, allowedTopologies, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, sts1, + namespace, allowedTopologies, false) // perform statefulset scaleup sts1Replicas = 13 ginkgo.By("Scale up statefulset replica count from 7 to 13") - err = scaleUpStatefulSetPod(ctx, client, sts1, namespace, sts1Replicas, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleUpStatefulSetPod(ctx, client, sts1, namespace, sts1Replicas, false) //verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") err = verifyVolumeProvisioningForStatefulSet(ctx, client, sts1, namespace, preferredDatastorePaths, - allDatastoresListMap, true, false, false, nil) + allDatastoresListMap, true, false) gomega.Expect(err).NotTo(gomega.HaveOccurred()) // Verify affinity details ginkgo.By("Verify node and pv topology affinity details") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, sts1, - namespace, allowedTopologies, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, sts1, + namespace, allowedTopologies, false) }) /* @@ -2169,17 +2130,17 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision // choose preferred datastore ginkgo.By("Tag rack-1 to preferred datatsore which is accessible only on rack-2") preferredDatastorePaths, err = tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[0], - preferredDatastoreChosen, nonShareddatastoreListMapRack2, nil, false, clientIndex) + preferredDatastoreChosen, nonShareddatastoreListMapRack2, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) preferredDatastoreChosen = 1 preferredDatastore, err := tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[0], - preferredDatastoreChosen, shareddatastoreListMap, nil, false, clientIndex) + preferredDatastoreChosen, shareddatastoreListMap, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) preferredDatastorePaths = append(preferredDatastorePaths, preferredDatastore...) defer func() { for i := 0; i < len(preferredDatastorePaths); i++ { err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[i], - allowedTopologyRacks[0], false, clientIndex) + allowedTopologyRacks[0]) } gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() @@ -2242,13 +2203,12 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision // verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") verifyVolumeProvisioningForStandalonePods(ctx, client, pod, namespace, preferredDatastorePaths, - rack2DatastoreListMap, false, nil) + rack2DatastoreListMap) ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate " + "node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStandalonePodLevel5(ctx, client, pod, namespace, - allowedTopologies, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsFoStandalonePodLevel5(ctx, client, pod, namespace, + allowedTopologies) }) /* @@ -2293,7 +2253,7 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision // choose preferred datastore ginkgo.By("Assign Tags to preferred datatsore for volume provisioning in rack-2(cluster-2)") preferredDatastorePaths, err = tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[0], - preferredDatastoreChosen, shareddatastoreListMap, nil, false, clientIndex) + preferredDatastoreChosen, shareddatastoreListMap, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) for i := 1; i < len(allowedTopologyRacks); i++ { err = tagSameDatastoreAsPreferenceToDifferentRacks(masterIp, sshClientConfig, allowedTopologyRacks[i], @@ -2303,7 +2263,7 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision defer func() { for i := 0; i < len(allowedTopologyRacks); i++ { err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[0], - allowedTopologyRacks[i], false, clientIndex) + allowedTopologyRacks[i]) } gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() @@ -2360,13 +2320,12 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision // verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") verifyVolumeProvisioningForStandalonePods(ctx, client, pod, namespace, storagePolicyDs, - nonShareddatastoreListMapRack2, false, nil) + nonShareddatastoreListMapRack2) ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate " + "node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStandalonePodLevel5(ctx, client, pod, namespace, - allowedTopologyForRack2, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsFoStandalonePodLevel5(ctx, client, pod, namespace, + allowedTopologyForRack2) }) /* @@ -2395,11 +2354,11 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision // choose preferred datastore ginkgo.By("Assign Tags to preferred datatsore for volume provisioning") preferredDatastorePaths, err = tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[0], - preferredDatastoreChosen, nonShareddatastoreListMapRack1, nil, false, clientIndex) + preferredDatastoreChosen, nonShareddatastoreListMapRack1, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) defer func() { err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[0], - allowedTopologyRacks[0], false, clientIndex) + allowedTopologyRacks[0]) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() @@ -2421,7 +2380,7 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision ginkgo.By("Expect claim to fail provisioning volume") framework.ExpectError(fpv.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, pvclaim.Namespace, pvclaim.Name, pollTimeoutShort, framework.PollShortTimeout)) - expectedErrMsg := "failed to create volume" + expectedErrMsg := "failed to get shared datastores for topology requirement" err = waitForEvent(ctx, client, namespace, expectedErrMsg, pvclaim.Name) gomega.Expect(err).NotTo(gomega.HaveOccurred(), fmt.Sprintf("Expected error : %q", expectedErrMsg)) }) @@ -2460,12 +2419,12 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision // choose preferred datastore ginkgo.By("Assign Tag to preferred datatsore for volume provisioning in rack-2(cluster-2)") preferredDatastorePaths, err = tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[1], - preferredDatastoreChosen, nonShareddatastoreListMapRack2, nil, false, clientIndex) + preferredDatastoreChosen, nonShareddatastoreListMapRack2, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) defer func() { for i := 0; i < len(preferredDatastorePaths); i++ { err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[i], - allowedTopologyRacks[1], false, clientIndex) + allowedTopologyRacks[1]) } gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() @@ -2593,14 +2552,13 @@ var _ = ginkgo.Describe("[Preferential-Topology] Preferential-Topology-Provision ginkgo.By("Verify volume provisioning for Pod-1/Pod-2") for i := 0; i < len(podList); i++ { verifyVolumeProvisioningForStandalonePods(ctx, client, podList[i], namespace, - preferredDatastorePaths, nonShareddatastoreListMapRack2, false, nil) + preferredDatastorePaths, nonShareddatastoreListMapRack2) } ginkgo.By("Verify pv and pod node affinity details for pv-1/pod-1 and pv-2/pod-2") for i := 0; i < len(podList); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStandalonePodLevel5(ctx, client, podList[i], - namespace, allowedTopologyForRack2, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsFoStandalonePodLevel5(ctx, client, podList[i], + namespace, allowedTopologyForRack2) } }) diff --git a/tests/e2e/preferential_topology_disruptive.go b/tests/e2e/preferential_topology_disruptive.go index 3693c8681b..ca0ef87d08 100644 --- a/tests/e2e/preferential_topology_disruptive.go +++ b/tests/e2e/preferential_topology_disruptive.go @@ -73,7 +73,6 @@ var _ = ginkgo.Describe("[Disruptive-Preferential-Topology] Preferential-Topolog csiNamespace string sshClientConfig *ssh.ClientConfig nimbusGeneratedK8sVmPwd string - clientIndex int ) ginkgo.BeforeEach(func() { var cancel context.CancelFunc @@ -168,9 +167,7 @@ var _ = ginkgo.Describe("[Disruptive-Preferential-Topology] Preferential-Topolog topologyLength, leafNode, leafNodeTag1) //set preferred datatsore time interval - setPreferredDatastoreTimeInterval(client, ctx, csiNamespace, csiReplicas, false) - - clientIndex = 0 + setPreferredDatastoreTimeInterval(client, ctx, csiNamespace, namespace, csiReplicas) }) @@ -185,11 +182,11 @@ var _ = ginkgo.Describe("[Disruptive-Preferential-Topology] Preferential-Topolog gomega.Expect(err).NotTo(gomega.HaveOccurred()) } framework.Logf("Perform preferred datastore tags cleanup after test completion") - err = deleteTagCreatedForPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks, false) + err = deleteTagCreatedForPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks) gomega.Expect(err).NotTo(gomega.HaveOccurred()) framework.Logf("Recreate preferred datastore tags post cleanup") - err = createTagForPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks, false) + err = createTagForPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) @@ -253,7 +250,7 @@ var _ = ginkgo.Describe("[Disruptive-Preferential-Topology] Preferential-Topolog ginkgo.By("Tag preferred datastore for volume provisioning") for i := 0; i < len(allowedTopologyRacks); i++ { preferredDatastorePath, err := tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[i], - preferredDatastoreChosen, datastorestMap[i], nil, false, clientIndex) + preferredDatastoreChosen, datastorestMap[i], nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) preferredDatastorePaths = append(preferredDatastorePaths, preferredDatastorePath...) } @@ -305,26 +302,25 @@ var _ = ginkgo.Describe("[Disruptive-Preferential-Topology] Preferential-Topolog //verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") err = verifyVolumeProvisioningForStatefulSet(ctx, client, statefulset, namespace, preferredDatastorePaths, - allDatastoresListMap, true, false, false, nil) + allDatastoresListMap, true, false) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, - namespace, allowedTopologies, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, + namespace, allowedTopologies, false) ginkgo.By("Remove preferred datastore tag chosen for volume provisioning") for i := 0; i < len(allowedTopologyRacks); i++ { err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[i], - allowedTopologyRacks[i], false, clientIndex) + allowedTopologyRacks[i]) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } ginkgo.By("Tag new preferred datatsore for volume provisioning") for i := 0; i < len(allowedTopologyRacks); i++ { preferredDatastorePath, err := tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[i], - preferredDatastoreChosen, datastorestMap[i], preferredDatastorePaths, false, clientIndex) + preferredDatastoreChosen, datastorestMap[i], preferredDatastorePaths) gomega.Expect(err).NotTo(gomega.HaveOccurred()) preferredDatastorePaths = append(preferredDatastorePaths, preferredDatastorePath...) preferredDatastorePathsToDel = append(preferredDatastorePathsToDel, preferredDatastorePath...) @@ -333,7 +329,7 @@ var _ = ginkgo.Describe("[Disruptive-Preferential-Topology] Preferential-Topolog ginkgo.By("Preferred datastore tags cleanup") for i := 0; i < len(allowedTopologyRacks); i++ { err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePathsToDel[i], - allowedTopologyRacks[i], false, clientIndex) + allowedTopologyRacks[i]) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() @@ -377,8 +373,7 @@ var _ = ginkgo.Describe("[Disruptive-Preferential-Topology] Preferential-Topolog ginkgo.By("Scale up statefulset replica and verify the replica count") replicas = 10 - err = scaleUpStatefulSetPod(ctx, client, statefulset, namespace, replicas, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleUpStatefulSetPod(ctx, client, statefulset, namespace, replicas, true) ssPodsAfterScaleDown := GetListOfPodsInSts(client, statefulset) gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(replicas)).To(gomega.BeTrue(), "Number of Pods in the statefulset should match with number of replicas") @@ -386,37 +381,35 @@ var _ = ginkgo.Describe("[Disruptive-Preferential-Topology] Preferential-Topolog //verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") err = verifyVolumeProvisioningForStatefulSet(ctx, client, statefulset, namespace, preferredDatastorePaths, - allDatastoresListMap, true, false, false, nil) + allDatastoresListMap, true, false) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, - namespace, allowedTopologies, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, + namespace, allowedTopologies, false) ginkgo.By("Remove preferred datastore tag in rack-2(cluster-2)") err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[4], - allowedTopologyRacks[1], false, clientIndex) + allowedTopologyRacks[1]) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Tag new preferred datatsore for volume provisioning") preferredDatastoreRack2New, err := tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[1], - preferredDatastoreChosen, shareddatastoreListMap, nil, false, clientIndex) + preferredDatastoreChosen, shareddatastoreListMap, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) preferredDatastorePaths = append(preferredDatastorePaths, preferredDatastoreRack2New...) defer func() { ginkgo.By("Remove preferred datastore tag") err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastoreRack2New[0], - allowedTopologyRacks[1], false, clientIndex) + allowedTopologyRacks[1]) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() // Scale down statefulSets replica count replicas = 5 ginkgo.By("Scale down statefulset replica and verify the replica count") - err = scaleDownStatefulSetPod(ctx, client, statefulset, namespace, replicas, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleDownStatefulSetPod(ctx, client, statefulset, namespace, replicas, true) ssPodsAfterScaleDown = GetListOfPodsInSts(client, statefulset) gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(replicas)).To(gomega.BeTrue(), "Number of Pods in the statefulset should match with number of replicas") @@ -443,8 +436,7 @@ var _ = ginkgo.Describe("[Disruptive-Preferential-Topology] Preferential-Topolog // Scale up statefulSets replicas count ginkgo.By("Scale up statefulset replica and verify the replica count") replicas = 13 - err = scaleUpStatefulSetPod(ctx, client, statefulset, namespace, replicas, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleUpStatefulSetPod(ctx, client, statefulset, namespace, replicas, true) ssPodsAfterScaleDown = GetListOfPodsInSts(client, statefulset) gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(replicas)).To(gomega.BeTrue(), "Number of Pods in the statefulset should match with number of replicas") @@ -452,14 +444,13 @@ var _ = ginkgo.Describe("[Disruptive-Preferential-Topology] Preferential-Topolog //verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") err = verifyVolumeProvisioningForStatefulSet(ctx, client, statefulset, namespace, preferredDatastorePaths, - allDatastoresListMap, true, false, false, nil) + allDatastoresListMap, true, false) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, - namespace, allowedTopologies, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, + namespace, allowedTopologies, false) }) /* @@ -503,14 +494,14 @@ var _ = ginkgo.Describe("[Disruptive-Preferential-Topology] Preferential-Topolog ginkgo.By("Tag preferred datastore for volume provisioning in rack-2(cluster-2)") preferredDatastore1, err := tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[1], - preferredDatastoreChosen, nonShareddatastoreListMapRack2, nil, false, clientIndex) + preferredDatastoreChosen, nonShareddatastoreListMapRack2, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) if !strings.Contains(preferredDatastore1[0], "nfs") { err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastore1[0], - allowedTopologyRacks[1], false, clientIndex) + allowedTopologyRacks[1]) gomega.Expect(err).NotTo(gomega.HaveOccurred()) preferredDatastore1, err = tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[1], - preferredDatastoreChosen, nonShareddatastoreListMapRack2, preferredDatastore1, false, clientIndex) + preferredDatastoreChosen, nonShareddatastoreListMapRack2, preferredDatastore1) gomega.Expect(err).NotTo(gomega.HaveOccurred()) preferredDatastorePaths = append(preferredDatastorePaths, preferredDatastore1...) } else { @@ -518,14 +509,14 @@ var _ = ginkgo.Describe("[Disruptive-Preferential-Topology] Preferential-Topolog } preferredDatastore2, err := tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[1], - preferredDatastoreChosen, shareddatastoreListMap, nil, false, clientIndex) + preferredDatastoreChosen, shareddatastoreListMap, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) preferredDatastorePaths = append(preferredDatastorePaths, preferredDatastore2...) defer func() { ginkgo.By("Remove preferred datastore tag") for i := 0; i < len(preferredDatastorePaths); i++ { err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[i], - allowedTopologyRacks[1], false, clientIndex) + allowedTopologyRacks[1]) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() @@ -566,14 +557,13 @@ var _ = ginkgo.Describe("[Disruptive-Preferential-Topology] Preferential-Topolog //verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") err = verifyVolumeProvisioningForStatefulSet(ctx, client, statefulset, namespace, preferredDatastorePaths, - rack2DatastoreListMap, false, false, false, nil) + rack2DatastoreListMap, false, false) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, - namespace, allowedTopologyForRack2, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, + namespace, allowedTopologyForRack2, false) ginkgo.By("Migrate the worker vms residing on the nfs datatsore before " + "making datastore inaccessible") @@ -625,62 +615,56 @@ var _ = ginkgo.Describe("[Disruptive-Preferential-Topology] Preferential-Topolog // verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") verifyVolumeProvisioningForStandalonePods(ctx, client, pod, namespace, preferredDatastorePaths, - rack2DatastoreListMap, false, nil) + rack2DatastoreListMap) ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate " + "node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStandalonePodLevel5(ctx, client, pod, namespace, - allowedTopologyForRack2, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsFoStandalonePodLevel5(ctx, client, pod, namespace, + allowedTopologyForRack2) // perform statefulset scaleup replicas = 15 ginkgo.By("Scale up statefulset replica count from 10 to 15") - err = scaleUpStatefulSetPod(ctx, client, statefulset, namespace, replicas, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleUpStatefulSetPod(ctx, client, statefulset, namespace, replicas, false) //verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") err = verifyVolumeProvisioningForStatefulSet(ctx, client, statefulset, namespace, preferredDatastorePaths, - rack2DatastoreListMap, false, false, false, nil) + rack2DatastoreListMap, false, false) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, - namespace, allowedTopologyForRack2, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, + namespace, allowedTopologyForRack2, false) ginkgo.By("Power on the inaccessible datastore") datastoreOp = "on" powerOnPreferredDatastore(datastoreName, datastoreOp) ginkgo.By("Remove preferred datastore tag chosen for volume provisioning") - err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastore1[0], - allowedTopologyRacks[1], false, clientIndex) + err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastore1[0], allowedTopologyRacks[1]) gomega.Expect(err).NotTo(gomega.HaveOccurred()) newPreferredDatastore, err := tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[1], - preferredDatastoreChosen, nonShareddatastoreListMapRack2, preferredDatastorePaths, false, clientIndex) + preferredDatastoreChosen, nonShareddatastoreListMapRack2, preferredDatastorePaths) preferredDatastorePaths = append(preferredDatastorePaths, newPreferredDatastore...) // perform statefulset scaleup replicas = 20 ginkgo.By("Scale up statefulset replica count from 15 to 20") - err = scaleUpStatefulSetPod(ctx, client, statefulset, namespace, replicas, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleUpStatefulSetPod(ctx, client, statefulset, namespace, replicas, false) //verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") err = verifyVolumeProvisioningForStatefulSet(ctx, client, statefulset, namespace, preferredDatastorePaths, - rack2DatastoreListMap, false, false, false, nil) + rack2DatastoreListMap, false, false) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, - namespace, allowedTopologyForRack2, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, + namespace, allowedTopologyForRack2, false) }) /* @@ -724,14 +708,14 @@ var _ = ginkgo.Describe("[Disruptive-Preferential-Topology] Preferential-Topolog ginkgo.By("Tag preferred datastore for volume provisioning in rack-1(cluster-1)") preferredDatastore1, err := tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[0], - preferredDatastoreChosen, nonShareddatastoreListMapRack1, nil, false, clientIndex) + preferredDatastoreChosen, nonShareddatastoreListMapRack1, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) if !strings.Contains(preferredDatastore1[0], "nfs") { err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastore1[0], - allowedTopologyRacks[0], false, clientIndex) + allowedTopologyRacks[0]) gomega.Expect(err).NotTo(gomega.HaveOccurred()) preferredDatastore1, err = tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[0], - preferredDatastoreChosen, nonShareddatastoreListMapRack1, preferredDatastore1, false, clientIndex) + preferredDatastoreChosen, nonShareddatastoreListMapRack1, preferredDatastore1) gomega.Expect(err).NotTo(gomega.HaveOccurred()) preferredDatastorePaths = append(preferredDatastorePaths, preferredDatastore1...) } else { @@ -739,14 +723,14 @@ var _ = ginkgo.Describe("[Disruptive-Preferential-Topology] Preferential-Topolog } preferredDatastore2, err := tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[0], - preferredDatastoreChosen, shareddatastoreListMap, nil, false, clientIndex) + preferredDatastoreChosen, shareddatastoreListMap, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) preferredDatastorePaths = append(preferredDatastorePaths, preferredDatastore2...) defer func() { ginkgo.By("Remove preferred datastore tag") for i := 0; i < len(preferredDatastorePaths); i++ { err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[i], - allowedTopologyRacks[0], false, clientIndex) + allowedTopologyRacks[0]) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() @@ -788,14 +772,13 @@ var _ = ginkgo.Describe("[Disruptive-Preferential-Topology] Preferential-Topolog //verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") err = verifyVolumeProvisioningForStatefulSet(ctx, client, statefulset, namespace, preferredDatastorePaths, - rack1DatastoreListMap, false, false, false, nil) + rack1DatastoreListMap, false, false) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, - namespace, allowedTopologyForRack1, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, + namespace, allowedTopologyForRack1, false) ginkgo.By("Migrate all the worker vms residing on the preferred datatsore before " + "putting it into maintenance mode") @@ -852,31 +835,28 @@ var _ = ginkgo.Describe("[Disruptive-Preferential-Topology] Preferential-Topolog // verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") verifyVolumeProvisioningForStandalonePods(ctx, client, pod, namespace, preferredDatastorePaths, - rack1DatastoreListMap, false, nil) + rack1DatastoreListMap) ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate " + "node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStandalonePodLevel5(ctx, client, pod, namespace, - allowedTopologyForRack1, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsFoStandalonePodLevel5(ctx, client, pod, namespace, + allowedTopologyForRack1) // perform statefulset scaleup replicas = 15 ginkgo.By("Scale up statefulset replica count from 10 to 15") - err = scaleUpStatefulSetPod(ctx, client, statefulset, namespace, replicas, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleUpStatefulSetPod(ctx, client, statefulset, namespace, replicas, false) //verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") err = verifyVolumeProvisioningForStatefulSet(ctx, client, statefulset, namespace, preferredDatastorePaths, - rack1DatastoreListMap, false, false, false, nil) + rack1DatastoreListMap, false, false) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, - namespace, allowedTopologyForRack1, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, + namespace, allowedTopologyForRack1, false) ginkgo.By("Exit datastore from Maintenance mode") err = exitDatastoreFromMaintenanceMode(masterIp, sshClientConfig, dataCenters, preferredDatastore1[0]) @@ -884,31 +864,28 @@ var _ = ginkgo.Describe("[Disruptive-Preferential-Topology] Preferential-Topolog isDatastoreInMaintenanceMode = false ginkgo.By("Remove preferred datastore tag chosen for volume provisioning") - err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastore1[0], - allowedTopologyRacks[0], false, clientIndex) + err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastore1[0], allowedTopologyRacks[0]) gomega.Expect(err).NotTo(gomega.HaveOccurred()) newPreferredDatastore, err := tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[0], - preferredDatastoreChosen, nonShareddatastoreListMapRack1, preferredDatastorePaths, false, clientIndex) + preferredDatastoreChosen, nonShareddatastoreListMapRack1, preferredDatastorePaths) preferredDatastorePaths = append(preferredDatastorePaths, newPreferredDatastore...) // perform statefulset scaleup replicas = 20 ginkgo.By("Scale up statefulset replica count from 15 to 20") - err = scaleUpStatefulSetPod(ctx, client, statefulset, namespace, replicas, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleUpStatefulSetPod(ctx, client, statefulset, namespace, replicas, false) //verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") err = verifyVolumeProvisioningForStatefulSet(ctx, client, statefulset, namespace, preferredDatastorePaths, - rack1DatastoreListMap, false, false, false, nil) + rack1DatastoreListMap, false, false) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, - namespace, allowedTopologyForRack1, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, + namespace, allowedTopologyForRack1, false) }) /* @@ -952,14 +929,14 @@ var _ = ginkgo.Describe("[Disruptive-Preferential-Topology] Preferential-Topolog ginkgo.By("Tag preferred datastore for volume provisioning in rack-2(cluster-2)") preferredDatastore1, err := tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[1], - preferredDatastoreChosen, nonShareddatastoreListMapRack2, nil, false, clientIndex) + preferredDatastoreChosen, nonShareddatastoreListMapRack2, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) if !strings.Contains(preferredDatastore1[0], "nfs") { err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastore1[0], - allowedTopologyRacks[1], false, clientIndex) + allowedTopologyRacks[1]) gomega.Expect(err).NotTo(gomega.HaveOccurred()) preferredDatastore1, err = tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[1], - preferredDatastoreChosen, nonShareddatastoreListMapRack2, preferredDatastore1, false, clientIndex) + preferredDatastoreChosen, nonShareddatastoreListMapRack2, preferredDatastore1) gomega.Expect(err).NotTo(gomega.HaveOccurred()) preferredDatastorePaths = append(preferredDatastorePaths, preferredDatastore1...) } else { @@ -967,7 +944,7 @@ var _ = ginkgo.Describe("[Disruptive-Preferential-Topology] Preferential-Topolog } preferredDatastore2, err := tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[1], - preferredDatastoreChosen, shareddatastoreListMap, nil, false, clientIndex) + preferredDatastoreChosen, shareddatastoreListMap, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) preferredDatastorePaths = append(preferredDatastorePaths, preferredDatastore2...) @@ -975,7 +952,7 @@ var _ = ginkgo.Describe("[Disruptive-Preferential-Topology] Preferential-Topolog ginkgo.By("Remove preferred datastore tag") for i := 0; i < len(preferredDatastorePaths); i++ { err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[i], - allowedTopologyRacks[1], false, clientIndex) + allowedTopologyRacks[1]) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } @@ -1018,14 +995,13 @@ var _ = ginkgo.Describe("[Disruptive-Preferential-Topology] Preferential-Topolog //verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") err = verifyVolumeProvisioningForStatefulSet(ctx, client, statefulset, namespace, preferredDatastorePaths, - rack2DatastoreListMap, false, false, false, nil) + rack2DatastoreListMap, false, false) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, - namespace, allowedTopologyForRack2, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, + namespace, allowedTopologyForRack2, false) ginkgo.By("Migrate all the worker vms residing on the nfs datatsore before " + "making datastore inaccessible") @@ -1078,61 +1054,55 @@ var _ = ginkgo.Describe("[Disruptive-Preferential-Topology] Preferential-Topolog // verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") verifyVolumeProvisioningForStandalonePods(ctx, client, pod, namespace, preferredDatastorePaths, - rack2DatastoreListMap, false, nil) + rack2DatastoreListMap) ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate " + "node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStandalonePodLevel5(ctx, client, pod, namespace, - allowedTopologyForRack2, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsFoStandalonePodLevel5(ctx, client, pod, namespace, + allowedTopologyForRack2) // perform statefulset scaleup replicas = 15 ginkgo.By("Scale up statefulset replica count from 10 to 15") - err = scaleUpStatefulSetPod(ctx, client, statefulset, namespace, replicas, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleUpStatefulSetPod(ctx, client, statefulset, namespace, replicas, false) //verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") err = verifyVolumeProvisioningForStatefulSet(ctx, client, statefulset, namespace, preferredDatastorePaths, - rack2DatastoreListMap, false, false, false, nil) + rack2DatastoreListMap, false, false) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, - namespace, allowedTopologyForRack2, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, + namespace, allowedTopologyForRack2, false) ginkgo.By("Power on the suspended datastore") datastoreOp = "on" powerOnPreferredDatastore(datastoreName, datastoreOp) ginkgo.By("Remove preferred datastore tag chosen for volume provisioning") - err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastore1[0], - allowedTopologyRacks[1], false, clientIndex) + err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastore1[0], allowedTopologyRacks[1]) gomega.Expect(err).NotTo(gomega.HaveOccurred()) newPreferredDatastore, err := tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[1], - preferredDatastoreChosen, nonShareddatastoreListMapRack2, preferredDatastorePaths, false, clientIndex) + preferredDatastoreChosen, nonShareddatastoreListMapRack2, preferredDatastorePaths) preferredDatastorePaths = append(preferredDatastorePaths, newPreferredDatastore...) // perform statefulset scaleup replicas = 20 ginkgo.By("Scale up statefulset replica count from 15 to 20") - err = scaleUpStatefulSetPod(ctx, client, statefulset, namespace, replicas, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleUpStatefulSetPod(ctx, client, statefulset, namespace, replicas, false) //verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") err = verifyVolumeProvisioningForStatefulSet(ctx, client, statefulset, namespace, preferredDatastorePaths, - rack2DatastoreListMap, false, false, false, nil) + rack2DatastoreListMap, false, false) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, - namespace, allowedTopologyForRack2, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, + namespace, allowedTopologyForRack2, false) }) }) diff --git a/tests/e2e/preferential_topology_snapshot.go b/tests/e2e/preferential_topology_snapshot.go index 1dd73787b9..4231e8cbf5 100644 --- a/tests/e2e/preferential_topology_snapshot.go +++ b/tests/e2e/preferential_topology_snapshot.go @@ -38,7 +38,7 @@ import ( fss "k8s.io/kubernetes/test/e2e/framework/statefulset" admissionapi "k8s.io/pod-security-admission/api" - snapclient "github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned" + snapclient "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned" ) var _ = ginkgo.Describe("[Preferential-Topology-Snapshot] Preferential Topology Volume Snapshot tests", func() { @@ -81,7 +81,6 @@ var _ = ginkgo.Describe("[Preferential-Topology-Snapshot] Preferential Topology csiNamespace string sshClientConfig *ssh.ClientConfig nimbusGeneratedK8sVmPwd string - clientIndex int ) ginkgo.BeforeEach(func() { @@ -188,9 +187,7 @@ var _ = ginkgo.Describe("[Preferential-Topology-Snapshot] Preferential Topology csiReplicas = *csiDeployment.Spec.Replicas //set preferred datatsore time interval - setPreferredDatastoreTimeInterval(client, ctx, csiNamespace, csiReplicas, false) - - clientIndex = 0 + setPreferredDatastoreTimeInterval(client, ctx, csiNamespace, namespace, csiReplicas) }) ginkgo.AfterEach(func() { @@ -204,11 +201,11 @@ var _ = ginkgo.Describe("[Preferential-Topology-Snapshot] Preferential Topology gomega.Expect(err).NotTo(gomega.HaveOccurred()) } framework.Logf("Perform preferred datastore tags cleanup after test completion") - err = deleteTagCreatedForPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks, false) + err = deleteTagCreatedForPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks) gomega.Expect(err).NotTo(gomega.HaveOccurred()) framework.Logf("Recreate preferred datastore tags post cleanup") - err = createTagForPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks, false) + err = createTagForPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) @@ -245,12 +242,12 @@ var _ = ginkgo.Describe("[Preferential-Topology-Snapshot] Preferential Topology // choose preferred datastore ginkgo.By("Tag preferred datastore for volume provisioning in rack-1(cluster-1))") preferredDatastorePaths, err = tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[0], - preferredDatastoreChosen, nonShareddatastoreListMapRack1, nil, false, clientIndex) + preferredDatastoreChosen, nonShareddatastoreListMapRack1, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) defer func() { ginkgo.By("Remove preferred datastore tag") err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[0], - allowedTopologyRacks[0], false, clientIndex) + allowedTopologyRacks[0]) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() @@ -290,11 +287,11 @@ var _ = ginkgo.Describe("[Preferential-Topology-Snapshot] Preferential Topology ginkgo.By("Create volume snapshot class, volume snapshot") volumeSnapshot, volumeSnapshotClass, snapshotId := createSnapshotClassAndVolSnapshot(ctx, snapc, namespace, - pvclaim, volHandle, false, false) + pvclaim, volHandle, false) defer func() { ginkgo.By("Perform cleanup of snapshot created") performCleanUpForSnapshotCreated(ctx, snapc, namespace, volHandle, volumeSnapshot, snapshotId, - volumeSnapshotClass, pandoraSyncWaitTime, false) + volumeSnapshotClass) }() ginkgo.By("Create PVC from snapshot") @@ -335,13 +332,12 @@ var _ = ginkgo.Describe("[Preferential-Topology-Snapshot] Preferential Topology // verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") verifyVolumeProvisioningForStandalonePods(ctx, client, pod, namespace, preferredDatastorePaths, - nonShareddatastoreListMapRack1, false, nil) + nonShareddatastoreListMapRack1) ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStandalonePodLevel5(ctx, client, pod, namespace, - allowedTopologyForRack1, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsFoStandalonePodLevel5(ctx, client, pod, namespace, + allowedTopologyForRack1) }) /* @@ -373,7 +369,7 @@ var _ = ginkgo.Describe("[Preferential-Topology-Snapshot] Preferential Topology // choose preferred datastore ginkgo.By("Tag preferred datastore for volume provisioning in rack-2(cluster-2))") preferredDatastorePaths, err = tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[1], - preferredDatastoreChosen, nonShareddatastoreListMapRack2, nil, false, clientIndex) + preferredDatastoreChosen, nonShareddatastoreListMapRack2, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) framework.Logf("Waiting for %v for preferred datastore to get refreshed in the environment", @@ -413,26 +409,26 @@ var _ = ginkgo.Describe("[Preferential-Topology-Snapshot] Preferential Topology ginkgo.By("Create volume snapshot class, volume snapshot") volumeSnapshot, volumeSnapshotClass, snapshotId := createSnapshotClassAndVolSnapshot(ctx, snapc, namespace, - pvclaim, volHandle, false, false) + pvclaim, volHandle, false) defer func() { ginkgo.By("Perform cleanup of snapshot created") performCleanUpForSnapshotCreated(ctx, snapc, namespace, volHandle, volumeSnapshot, snapshotId, - volumeSnapshotClass, pandoraSyncWaitTime, false) + volumeSnapshotClass) }() ginkgo.By("Remove preferred datastore tag chosen for volume provisioning") err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[0], - allowedTopologyRacks[1], false, clientIndex) + allowedTopologyRacks[1]) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Tag new preferred datatsore for volume provisioning") preferredDatastorePaths, err = tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[1], - preferredDatastoreChosen, nonShareddatastoreListMapRack2, preferredDatastorePaths, false, clientIndex) + preferredDatastoreChosen, nonShareddatastoreListMapRack2, preferredDatastorePaths) gomega.Expect(err).NotTo(gomega.HaveOccurred()) defer func() { ginkgo.By("Remove preferred datastore tag") err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[0], - allowedTopologyRacks[1], false, clientIndex) + allowedTopologyRacks[1]) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() @@ -447,10 +443,9 @@ var _ = ginkgo.Describe("[Preferential-Topology-Snapshot] Preferential Topology gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Expect claim to fail provisioning volume within the topology") - err = fpv.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, - client, pvclaim2.Namespace, pvclaim2.Name, framework.Poll, framework.ClaimProvisionTimeout) - gomega.Expect(err).To(gomega.HaveOccurred()) - expectedErrMsg := "failed to get the compatible shared datastore for create volume from snapshot" + framework.ExpectError(fpv.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, + client, pvclaim2.Namespace, pvclaim2.Name, pollTimeoutShort, framework.PollShortTimeout)) + expectedErrMsg := "failed to get the compatible datastore for create volume from snapshot" err = waitForEvent(ctx, client, namespace, expectedErrMsg, pvclaim2.Name) gomega.Expect(err).NotTo(gomega.HaveOccurred(), fmt.Sprintf("Expected error : %q", expectedErrMsg)) defer func() { @@ -532,13 +527,13 @@ var _ = ginkgo.Describe("[Preferential-Topology-Snapshot] Preferential Topology ginkgo.By("Tag different preferred datastore for volume provisioning in different racks") for i := 0; i < len(allowedTopologyRacks); i++ { preferredDatastorePath, err := tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[i], - preferredDatastoreChosen, datastorestMap[i], nil, false, clientIndex) + preferredDatastoreChosen, datastorestMap[i], nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) preferredDatastorePaths = append(preferredDatastorePaths, preferredDatastorePath...) } sharedPreferredDatastorePaths, err := tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[0], - preferredDatastoreChosen, shareddatastoreListMap, nil, false, clientIndex) + preferredDatastoreChosen, shareddatastoreListMap, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) preferredDatastorePaths = append(preferredDatastorePaths, sharedPreferredDatastorePaths...) for i := 1; i < len(allowedTopologyRacks); i++ { @@ -549,7 +544,7 @@ var _ = ginkgo.Describe("[Preferential-Topology-Snapshot] Preferential Topology defer func() { for i := 0; i < len(allowedTopologyRacks); i++ { err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, sharedPreferredDatastorePaths[0], - allowedTopologyRacks[i], false, clientIndex) + allowedTopologyRacks[i]) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } @@ -677,24 +672,23 @@ var _ = ginkgo.Describe("[Preferential-Topology-Snapshot] Preferential Topology ginkgo.By("Verify volume is provisioned on the preferred datatsore") for i := 0; i < len(podList); i++ { verifyVolumeProvisioningForStandalonePods(ctx, client, podList[i], namespace, - preferredDatastorePaths, allDatastoresListMap, false, nil) + preferredDatastorePaths, allDatastoresListMap) } ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") for i := 0; i < len(podList); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStandalonePodLevel5(ctx, client, podList[i], - namespace, allowedTopologies, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsFoStandalonePodLevel5(ctx, client, podList[i], + namespace, allowedTopologies) } ginkgo.By("Create volume snapshot class, volume snapshot") volumeSnapshot1, volumeSnapshotClass1, snapshotId1 := createSnapshotClassAndVolSnapshot(ctx, snapc, namespace, - pvclaim1, volHandle1, false, false) + pvclaim1, volHandle1, false) defer func() { ginkgo.By("Perform cleanup of snapshot created") performCleanUpForSnapshotCreated(ctx, snapc, namespace, volHandle1, volumeSnapshot1, snapshotId1, - volumeSnapshotClass1, pandoraSyncWaitTime, false) + volumeSnapshotClass1) }() ginkgo.By("Create PVC-3 from snapshot") @@ -715,7 +709,7 @@ var _ = ginkgo.Describe("[Preferential-Topology-Snapshot] Preferential Topology gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Verify snapshot entry is deleted from CNS") - err = verifySnapshotIsDeletedInCNS(volHandle3, snapshotId1, false) + err = verifySnapshotIsDeletedInCNS(volHandle3, snapshotId1) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() @@ -739,18 +733,17 @@ var _ = ginkgo.Describe("[Preferential-Topology-Snapshot] Preferential Topology // verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") verifyVolumeProvisioningForStandalonePods(ctx, client, pod3, namespace, preferredDatastorePaths, - allDatastoresListMap, false, nil) + allDatastoresListMap) ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStandalonePodLevel5(ctx, client, pod3, namespace, - allowedTopologies, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsFoStandalonePodLevel5(ctx, client, pod3, namespace, + allowedTopologies) ginkgo.By("Remove preferred datastore tag chosen for volume provisioning") for i := 0; i < len(allowedTopologyRacks); i++ { err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[i], - allowedTopologyRacks[i], false, clientIndex) + allowedTopologyRacks[i]) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } @@ -758,14 +751,14 @@ var _ = ginkgo.Describe("[Preferential-Topology-Snapshot] Preferential Topology ginkgo.By("Assign new preferred datastore tags for volume provisioning in different racks") for i := 0; i < len(allowedTopologyRacks); i++ { preferredDatastorePath, err := tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[i], - preferredDatastoreChosen, datastorestMap[i], preferredDatastorePaths, false, clientIndex) + preferredDatastoreChosen, datastorestMap[i], preferredDatastorePaths) gomega.Expect(err).NotTo(gomega.HaveOccurred()) preferredDatastorePathsNew = append(preferredDatastorePathsNew, preferredDatastorePath...) } defer func() { for i := 0; i < len(allowedTopologyRacks); i++ { err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePathsNew[i], - allowedTopologyRacks[i], false, clientIndex) + allowedTopologyRacks[i]) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() @@ -834,19 +827,18 @@ var _ = ginkgo.Describe("[Preferential-Topology-Snapshot] Preferential Topology // verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") verifyVolumeProvisioningForStandalonePods(ctx, client, pod4, namespace, preferredDatastorePathsNew, - allDatastoresListMap, false, nil) + allDatastoresListMap) ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStandalonePodLevel5(ctx, client, pod4, namespace, - allowedTopologies, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsFoStandalonePodLevel5(ctx, client, pod4, namespace, + allowedTopologies) volumeSnapshot2, volumeSnapshotClass2, snapshotId2 := createSnapshotClassAndVolSnapshot(ctx, snapc, namespace, - pvclaim4, volHandle4, false, false) + pvclaim4, volHandle4, false) defer func() { performCleanUpForSnapshotCreated(ctx, snapc, namespace, pv4.Spec.CSI.VolumeHandle, volumeSnapshot2, - snapshotId2, volumeSnapshotClass2, pandoraSyncWaitTime, false) + snapshotId2, volumeSnapshotClass2) }() ginkgo.By("Create PVC-5 from snapshot") @@ -887,13 +879,12 @@ var _ = ginkgo.Describe("[Preferential-Topology-Snapshot] Preferential Topology // verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") verifyVolumeProvisioningForStandalonePods(ctx, client, pod5, namespace, preferredDatastorePathsNew, - allDatastoresListMap, false, nil) + allDatastoresListMap) ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStandalonePodLevel5(ctx, client, pod5, namespace, - allowedTopologies, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsFoStandalonePodLevel5(ctx, client, pod5, namespace, + allowedTopologies) }) /* @@ -929,11 +920,11 @@ var _ = ginkgo.Describe("[Preferential-Topology-Snapshot] Preferential Topology ginkgo.By("Tag preferred datastore for volume provisioning in rack-3(cluster-3)") preferredDatastorePaths, err = tagPreferredDatastore(masterIp, sshClientConfig, allowedTopologyRacks[2], - preferredDatastoreChosen, nonShareddatastoreListMapRack3, nil, false, clientIndex) + preferredDatastoreChosen, nonShareddatastoreListMapRack3, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) defer func() { err = detachTagCreatedOnPreferredDatastore(masterIp, sshClientConfig, preferredDatastorePaths[0], - allowedTopologyRacks[2], false, clientIndex) + allowedTopologyRacks[2]) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() @@ -978,14 +969,13 @@ var _ = ginkgo.Describe("[Preferential-Topology-Snapshot] Preferential Topology //verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") err = verifyVolumeProvisioningForStatefulSet(ctx, client, statefulset, namespace, preferredDatastorePaths, - nonShareddatastoreListMapRack3, false, false, false, nil) + nonShareddatastoreListMapRack3, false, false) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, - namespace, allowedTopologyForRack3, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, + namespace, allowedTopologyForRack3, false) framework.Logf("Fetching pod 3, pvc3 and pv3 details") pod3, err := client.CoreV1().Pods(namespace).Get(ctx, @@ -1005,11 +995,11 @@ var _ = ginkgo.Describe("[Preferential-Topology-Snapshot] Preferential Topology ginkgo.By("Create volume snapshot class, volume snapshot") volumeSnapshot, volumeSnapshotClass, snapshotId := createSnapshotClassAndVolSnapshot(ctx, snapc, namespace, - pvclaim3, volHandle3, true, false) + pvclaim3, volHandle3, true) defer func() { ginkgo.By("Perform cleanup of snapshot created") performCleanUpForSnapshotCreated(ctx, snapc, namespace, volHandle3, volumeSnapshot, snapshotId, - volumeSnapshotClass, pandoraSyncWaitTime, false) + volumeSnapshotClass) }() ginkgo.By(fmt.Sprintf("Scaling down statefulsets to number of Replica: %v", replicas-1)) @@ -1053,13 +1043,12 @@ var _ = ginkgo.Describe("[Preferential-Topology-Snapshot] Preferential Topology //verifying volume provisioning ginkgo.By("Verify volume is provisioned on the preferred datatsore") err = verifyVolumeProvisioningForStatefulSet(ctx, client, statefulset, namespace, preferredDatastorePaths, - nonShareddatastoreListMapRack3, false, false, false, nil) + nonShareddatastoreListMapRack3, false, false) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, - namespace, allowedTopologyForRack3, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, + namespace, allowedTopologyForRack3, false) }) }) diff --git a/tests/e2e/preferential_topology_utils.go b/tests/e2e/preferential_topology_utils.go index 949ab1d30e..9210995ffb 100644 --- a/tests/e2e/preferential_topology_utils.go +++ b/tests/e2e/preferential_topology_utils.go @@ -21,14 +21,15 @@ import ( "fmt" "strings" - snapV1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" - snapclient "github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned" + snapV1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" + snapclient "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" "github.com/vmware/govmomi/object" "golang.org/x/crypto/ssh" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" @@ -109,45 +110,24 @@ attachTagToPreferredDatastore method is used to attach the preferred tag to the datastore chosen for volume provisioning */ func attachTagToPreferredDatastore(masterIp string, sshClientConfig *ssh.ClientConfig, - datastore string, tagName string, isMultiVC bool, clientIndexForMultiVC int) error { - var attachTagCat string - if !isMultiVC { - attachTagCat = govcLoginCmd() + - "govc tags.attach -c " + preferredDSCat + " " + tagName + " " + "'" + datastore + "'" - framework.Logf("cmd to attach tag to preferred datastore: %s ", attachTagCat) - attachTagCatRes, err := sshExec(sshClientConfig, masterIp, attachTagCat) - if err != nil && attachTagCatRes.Code != 0 { - fssh.LogResult(attachTagCatRes) - return fmt.Errorf("couldn't execute command: %s on host: %v , error: %s", - attachTagCat, masterIp, err) - } - return nil - } else { - attachTagCat = govcLoginCmdForMultiVC(clientIndexForMultiVC) + - "govc tags.attach -c " + preferredDSCat + " " + tagName + " " + "'" + datastore + "'" - framework.Logf("cmd to attach tag to preferred datastore: %s ", attachTagCat) - attachTagCatRes, err := sshExec(sshClientConfig, masterIp, attachTagCat) - if err != nil && attachTagCatRes.Code != 0 { - fssh.LogResult(attachTagCatRes) - return fmt.Errorf("couldn't execute command: %s on host: %v , error: %s", - attachTagCat, masterIp, err) - } - return nil + datastore string, tagName string) error { + attachTagCat := govcLoginCmd() + + "govc tags.attach -c " + preferredDSCat + " " + tagName + " " + "'" + datastore + "'" + framework.Logf("cmd to attach tag to preferred datastore: %s ", attachTagCat) + attachTagCatRes, err := sshExec(sshClientConfig, masterIp, attachTagCat) + if err != nil && attachTagCatRes.Code != 0 { + fssh.LogResult(attachTagCatRes) + return fmt.Errorf("couldn't execute command: %s on host: %v , error: %s", + attachTagCat, masterIp, err) } + return nil } /* detachTagCreatedOnPreferredDatastore is used to detach the tag created on preferred datastore */ func detachTagCreatedOnPreferredDatastore(masterIp string, sshClientConfig *ssh.ClientConfig, - datastore string, tagName string, isMultiVcSetup bool, clientIndex int) error { - var detachTagCat string - if !isMultiVcSetup { - detachTagCat = govcLoginCmd() + - "govc tags.detach -c " + preferredDSCat + " " + tagName + " " + "'" + datastore + "'" - - } else { - detachTagCat = govcLoginCmdForMultiVC(clientIndex) + - "govc tags.detach -c " + preferredDSCat + " " + tagName + " " + "'" + datastore + "'" - } + datastore string, tagName string) error { + detachTagCat := govcLoginCmd() + + "govc tags.detach -c " + preferredDSCat + " " + tagName + " " + "'" + datastore + "'" framework.Logf("cmd to detach the tag assigned to preferred datastore: %s ", detachTagCat) detachTagCatRes, err := sshExec(sshClientConfig, masterIp, detachTagCat) if err != nil && detachTagCatRes.Code != 0 { @@ -239,8 +219,7 @@ chosen preferred datastore or not for statefulsets func verifyVolumeProvisioningForStatefulSet(ctx context.Context, client clientset.Interface, statefulset *appsv1.StatefulSet, namespace string, datastoreNames []string, datastoreListMap map[string]string, - multipleAllowedTopology bool, parallelStatefulSetCreation bool, - isMultiVcSetup bool, multiVCDsUrls []string) error { + multipleAllowedTopology bool, parallelStatefulSetCreation bool) error { counter := 0 stsPodCount := 0 var dsUrls []string @@ -251,36 +230,22 @@ func verifyVolumeProvisioningForStatefulSet(ctx context.Context, ssPodsBeforeScaleDown = fss.GetPodList(client, statefulset) } stsPodCount = len(ssPodsBeforeScaleDown.Items) - if !isMultiVcSetup { - for i := 0; i < len(datastoreNames); i++ { - if val, ok := datastoreListMap[datastoreNames[i]]; ok { - dsUrls = append(dsUrls, val) - } + for i := 0; i < len(datastoreNames); i++ { + if val, ok := datastoreListMap[datastoreNames[i]]; ok { + dsUrls = append(dsUrls, val) } } for _, sspod := range ssPodsBeforeScaleDown.Items { _, err := client.CoreV1().Pods(namespace).Get(ctx, sspod.Name, metav1.GetOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) for _, volumespec := range sspod.Spec.Volumes { - var isPreferred bool if volumespec.PersistentVolumeClaim != nil { pv := getPvFromClaim(client, statefulset.Namespace, volumespec.PersistentVolumeClaim.ClaimName) - if !isMultiVcSetup { - isPreferred = e2eVSphere.verifyPreferredDatastoreMatch(pv.Spec.CSI.VolumeHandle, dsUrls) - if isPreferred { - framework.Logf("volume %s is created on preferred datastore %v", pv.Spec.CSI.VolumeHandle, dsUrls) - counter = counter + 1 - } - } else { - isPreferred = multiVCe2eVSphere.verifyPreferredDatastoreMatchInMultiVC(pv.Spec.CSI.VolumeHandle, - multiVCDsUrls) - if isPreferred { - framework.Logf("volume %s is created on preferred datastore %v", pv.Spec.CSI.VolumeHandle, - multiVCDsUrls) - counter = counter + 1 - } + isPreferred := e2eVSphere.verifyPreferredDatastoreMatch(pv.Spec.CSI.VolumeHandle, dsUrls) + if isPreferred { + framework.Logf("volume %s is created on preferred datastore %v", pv.Spec.CSI.VolumeHandle, dsUrls) + counter = counter + 1 } - } } } @@ -305,8 +270,7 @@ chosen preferred datastore or not for standalone pods */ func verifyVolumeProvisioningForStandalonePods(ctx context.Context, client clientset.Interface, pod *v1.Pod, - namespace string, datastoreNames []string, datastoreListMap map[string]string, - isMultiVcSetup bool, multiVCDsUrls []string) { + namespace string, datastoreNames []string, datastoreListMap map[string]string) { var flag bool = false var dsUrls []string for i := 0; i < len(datastoreNames); i++ { @@ -317,18 +281,10 @@ func verifyVolumeProvisioningForStandalonePods(ctx context.Context, for _, volumespec := range pod.Spec.Volumes { if volumespec.PersistentVolumeClaim != nil { pv := getPvFromClaim(client, pod.Namespace, volumespec.PersistentVolumeClaim.ClaimName) - if !isMultiVcSetup { - isPreferred := e2eVSphere.verifyPreferredDatastoreMatch(pv.Spec.CSI.VolumeHandle, dsUrls) - if isPreferred { - framework.Logf("volume %s is created on preferred datastore %v", pv.Spec.CSI.VolumeHandle, dsUrls) - flag = true - } - } else { - isPreferred := multiVCe2eVSphere.verifyPreferredDatastoreMatchInMultiVC(pv.Spec.CSI.VolumeHandle, multiVCDsUrls) - if isPreferred { - framework.Logf("volume %s is created on preferred datastore %v", pv.Spec.CSI.VolumeHandle, multiVCDsUrls) - flag = true - } + isPreferred := e2eVSphere.verifyPreferredDatastoreMatch(pv.Spec.CSI.VolumeHandle, dsUrls) + if isPreferred { + framework.Logf("volume %s is created on preferred datastore %v", pv.Spec.CSI.VolumeHandle, dsUrls) + flag = true } } } @@ -346,7 +302,7 @@ func tagSameDatastoreAsPreferenceToDifferentRacks(masterIp string, sshClientConf i := 0 for j := 0; j < len(datastoreNames); j++ { i = i + 1 - err := attachTagToPreferredDatastore(masterIp, sshClientConfig, datastoreNames[j], zoneValue, false, 0) + err := attachTagToPreferredDatastore(masterIp, sshClientConfig, datastoreNames[j], zoneValue) if err != nil { return err } @@ -361,22 +317,14 @@ func tagSameDatastoreAsPreferenceToDifferentRacks(masterIp string, sshClientConf tagPreferredDatastore method is used to tag the datastore which is chosen for volume provisioning */ func tagPreferredDatastore(masterIp string, sshClientConfig *ssh.ClientConfig, zoneValue string, itr int, - datastoreListMap map[string]string, datastoreNames []string, - isMultiVcSetup bool, clientIndex int) ([]string, error) { + datastoreListMap map[string]string, datastoreNames []string) ([]string, error) { var preferredDatastorePaths []string - var err error i := 0 if datastoreNames == nil { for dsName := range datastoreListMap { i = i + 1 preferredDatastorePaths = append(preferredDatastorePaths, dsName) - if !isMultiVcSetup { - err = attachTagToPreferredDatastore(masterIp, sshClientConfig, dsName, zoneValue, - false, clientIndex) - } else { - err = attachTagToPreferredDatastore(masterIp, sshClientConfig, dsName, zoneValue, - true, clientIndex) - } + err := attachTagToPreferredDatastore(masterIp, sshClientConfig, dsName, zoneValue) if err != nil { return preferredDatastorePaths, err } @@ -389,13 +337,7 @@ func tagPreferredDatastore(masterIp string, sshClientConfig *ssh.ClientConfig, z for dsName := range datastoreListMap { if !slices.Contains(datastoreNames, dsName) { preferredDatastorePaths = append(preferredDatastorePaths, dsName) - if !isMultiVcSetup { - err = attachTagToPreferredDatastore(masterIp, sshClientConfig, dsName, zoneValue, - false, clientIndex) - } else { - err = attachTagToPreferredDatastore(masterIp, sshClientConfig, dsName, zoneValue, - true, clientIndex) - } + err := attachTagToPreferredDatastore(masterIp, sshClientConfig, dsName, zoneValue) if err != nil { return preferredDatastorePaths, err } @@ -412,11 +354,12 @@ func tagPreferredDatastore(masterIp string, sshClientConfig *ssh.ClientConfig, z // restartCSIDriver method restarts the csi driver func restartCSIDriver(ctx context.Context, client clientset.Interface, namespace string, csiReplicas int32) (bool, error) { - isServiceStopped, err := stopCSIPods(ctx, client, namespace) + isServiceStopped, err := stopCSIPods(ctx, client) if err != nil { return isServiceStopped, err } - isServiceStarted, err := startCSIPods(ctx, client, csiReplicas, namespace) + isServiceStarted, err := startCSIPods(ctx, client, csiReplicas) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) if err != nil { return isServiceStarted, err } @@ -428,40 +371,20 @@ setPreferredDatastoreTimeInterval method is used to set the time interval at whi datastores are refreshed in the environment */ func setPreferredDatastoreTimeInterval(client clientset.Interface, ctx context.Context, - csiNamespace string, csiReplicas int32, isMultiVcSetup bool) { - - var modifiedConf string - var vsphereCfg e2eTestConfig - - // read current secret + csiNamespace string, namespace string, csiReplicas int32) { currentSecret, err := client.CoreV1().Secrets(csiNamespace).Get(ctx, configSecret, metav1.GetOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - // read original conf originalConf := string(currentSecret.Data[vSphereCSIConf]) - - if !isMultiVcSetup { - vsphereCfg, err = readConfigFromSecretString(originalConf) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } else { - vsphereCfg, err = readVsphereConfCredentialsInMultiVcSetup(originalConf) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - + vsphereCfg, err := readConfigFromSecretString(originalConf) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) if vsphereCfg.Global.CSIFetchPreferredDatastoresIntervalInMin == 0 { vsphereCfg.Global.CSIFetchPreferredDatastoresIntervalInMin = preferredDatastoreRefreshTimeInterval - if !isMultiVcSetup { - modifiedConf, err = writeConfigToSecretString(vsphereCfg) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By("Updating the secret to reflect new changes") - currentSecret.Data[vSphereCSIConf] = []byte(modifiedConf) - _, err = client.CoreV1().Secrets(csiNamespace).Update(ctx, currentSecret, metav1.UpdateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } else { - err = writeNewDataAndUpdateVsphereConfSecret(client, ctx, csiNamespace, vsphereCfg) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - + modifiedConf, err := writeConfigToSecretString(vsphereCfg) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + ginkgo.By("Updating the secret to reflect new changes") + currentSecret.Data[vSphereCSIConf] = []byte(modifiedConf) + _, err = client.CoreV1().Secrets(csiNamespace).Update(ctx, currentSecret, metav1.UpdateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) // restart csi driver restartSuccess, err := restartCSIDriver(ctx, client, csiNamespace, csiReplicas) gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") @@ -486,15 +409,10 @@ func getNonSharedDatastoresInCluster(ClusterdatastoreListMap map[string]string, } // deleteTagCreatedForPreferredDatastore method is used to delete the tag created on preferred datastore -func deleteTagCreatedForPreferredDatastore(masterIp string, sshClientConfig *ssh.ClientConfig, - tagName []string, isMultiVcSetup bool) error { - var deleteTagCat string +func deleteTagCreatedForPreferredDatastore(masterIp string, sshClientConfig *ssh.ClientConfig, tagName []string) error { for i := 0; i < len(tagName); i++ { - if !isMultiVcSetup { - deleteTagCat = govcLoginCmd() + "govc tags.rm -f -c " + preferredDSCat + " " + tagName[i] - } else { - deleteTagCat = govcLoginCmdForMultiVC(i) + "govc tags.rm -f -c " + preferredDSCat + " " + tagName[i] - } + deleteTagCat := govcLoginCmd() + + "govc tags.rm -f -c " + preferredDSCat + " " + tagName[i] framework.Logf("Deleting tag created for preferred datastore: %s ", deleteTagCat) deleteTagCatRes, err := sshExec(sshClientConfig, masterIp, deleteTagCat) if err != nil && deleteTagCatRes.Code != 0 { @@ -507,18 +425,10 @@ func deleteTagCreatedForPreferredDatastore(masterIp string, sshClientConfig *ssh } // createTagForPreferredDatastore method is used to create tag required for choosing preferred datastore -func createTagForPreferredDatastore(masterIp string, sshClientConfig *ssh.ClientConfig, - tagName []string, isMultiVcSetup bool) error { - var createTagCat string +func createTagForPreferredDatastore(masterIp string, sshClientConfig *ssh.ClientConfig, tagName []string) error { for i := 0; i < len(tagName); i++ { - if !isMultiVcSetup { - createTagCat = govcLoginCmd() + - "govc tags.create -d '" + preferredTagDesc + "' -c " + preferredDSCat + " " + tagName[i] - } else { - createTagCat = govcLoginCmdForMultiVC(i) + - "govc tags.create -d '" + preferredTagDesc + "' -c " + preferredDSCat + " " + tagName[i] - framework.Logf(createTagCat) - } + createTagCat := govcLoginCmd() + + "govc tags.create -d '" + preferredTagDesc + "' -c " + preferredDSCat + " " + tagName[i] framework.Logf("Creating tag for preferred datastore: %s ", createTagCat) createTagCatRes, err := sshExec(sshClientConfig, masterIp, createTagCat) if err != nil && createTagCatRes.Code != 0 { @@ -536,7 +446,7 @@ volume snapshot and to verify if volume snapshot has created or not */ func createSnapshotClassAndVolSnapshot(ctx context.Context, snapc *snapclient.Clientset, namespace string, pvclaim *v1.PersistentVolumeClaim, - volHandle string, stsPvc bool, isMultiVcSetup bool) (*snapV1.VolumeSnapshot, *snapV1.VolumeSnapshotClass, string) { + volHandle string, stsPvc bool) (*snapV1.VolumeSnapshot, *snapV1.VolumeSnapshotClass, string) { framework.Logf("Create volume snapshot class") volumeSnapshotClass, err := snapc.SnapshotV1().VolumeSnapshotClasses().Create(ctx, @@ -569,13 +479,8 @@ func createSnapshotClassAndVolSnapshot(ctx context.Context, snapc *snapclient.Cl snapshotId := strings.Split(snapshothandle, "+")[1] framework.Logf("Query CNS and check the volume snapshot entry") - if !isMultiVcSetup { - err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } else { - err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId, true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } + err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) return volumeSnapshot, volumeSnapshotClass, snapshotId } @@ -586,27 +491,25 @@ snapshot class, volume snapshot created for pvc post testcase completion */ func performCleanUpForSnapshotCreated(ctx context.Context, snapc *snapclient.Clientset, namespace string, volHandle string, volumeSnapshot *snapV1.VolumeSnapshot, snapshotId string, - volumeSnapshotClass *snapV1.VolumeSnapshotClass, pandoraSyncWaitTime int, isMultiVcSetup bool) { + volumeSnapshotClass *snapV1.VolumeSnapshotClass) { framework.Logf("Delete volume snapshot and verify the snapshot content is deleted") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) + err := snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, volumeSnapshot.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) framework.Logf("Wait till the volume snapshot is deleted") - err := waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, *volumeSnapshot.Status.BoundVolumeSnapshotContentName) + err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, *volumeSnapshot.Status.BoundVolumeSnapshotContentName) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - if !isMultiVcSetup { - framework.Logf("Verify snapshot entry is deleted from CNS") - err = verifySnapshotIsDeletedInCNS(volHandle, snapshotId, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } else { - framework.Logf("Verify snapshot entry is deleted from CNS") - err = verifySnapshotIsDeletedInCNS(volHandle, snapshotId, true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } + framework.Logf("Verify snapshot entry is deleted from CNS") + err = verifySnapshotIsDeletedInCNS(volHandle, snapshotId) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) framework.Logf("Deleting volume snapshot Again to check Not found error") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) + err = snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, volumeSnapshot.Name, metav1.DeleteOptions{}) + if !apierrors.IsNotFound(err) { + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } framework.Logf("Deleting volume snapshot class") err = snapc.SnapshotV1().VolumeSnapshotClasses().Delete(ctx, volumeSnapshotClass.Name, metav1.DeleteOptions{}) diff --git a/tests/e2e/preupgrade_datasetup.go b/tests/e2e/preupgrade_datasetup.go index b7a1e51564..c75957180a 100644 --- a/tests/e2e/preupgrade_datasetup.go +++ b/tests/e2e/preupgrade_datasetup.go @@ -92,7 +92,7 @@ var _ = ginkgo.Describe("PreUpgrade datasetup Test", func() { statefulset := GetStatefulSetFromManifest(namespace) ginkgo.By("Creating statefulset") statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &storageClassName + Annotations["volume.beta.kubernetes.io/storage-class"] = storageClassName CreateStatefulSet(namespace, statefulset, client) replicas := *(statefulset.Spec.Replicas) // Waiting for pods status to be Ready diff --git a/tests/e2e/prevent_duplicate_cluster_ids.go b/tests/e2e/prevent_duplicate_cluster_ids.go deleted file mode 100644 index a780c85537..0000000000 --- a/tests/e2e/prevent_duplicate_cluster_ids.go +++ /dev/null @@ -1,949 +0,0 @@ -/* - Copyright 2023 The Kubernetes Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package e2e - -import ( - "context" - "fmt" - "math/rand" - "strings" - - ginkgo "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" - "golang.org/x/crypto/ssh" - v1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/kubernetes/test/e2e/framework" - fnodes "k8s.io/kubernetes/test/e2e/framework/node" - fpod "k8s.io/kubernetes/test/e2e/framework/pod" - fpv "k8s.io/kubernetes/test/e2e/framework/pv" - fssh "k8s.io/kubernetes/test/e2e/framework/ssh" - admissionapi "k8s.io/pod-security-admission/api" -) - -var _ = ginkgo.Describe("Prevent duplicate cluster ID", func() { - f := framework.NewDefaultFramework("cluster-id-test") - f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged - var ( - client clientset.Interface - namespace string - csiNamespace string - csiReplicas int32 - vCenterUIUser string - vCenterUIPassword string - clusterId string - revertToOriginalVsphereSecret bool - vCenterIP string - vCenterPort string - dataCenter string - scParameters map[string]string - accessMode v1.PersistentVolumeAccessMode - sshClientConfig *ssh.ClientConfig - nimbusGeneratedK8sVmPwd string - ) - - ginkgo.BeforeEach(func() { - var cancel context.CancelFunc - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - client = f.ClientSet - namespace = f.Namespace.Name - bootstrap() - nodeList, err := fnodes.GetReadySchedulableNodes(f.ClientSet) - framework.ExpectNoError(err, "Unable to find ready and schedulable Node") - if !(len(nodeList.Items) > 0) { - framework.Failf("Unable to find ready and schedulable Node") - } - scParameters = make(map[string]string) - accessMode = v1.ReadWriteOnce - // fetching required parameters - - csiNamespace = GetAndExpectStringEnvVar(envCSINamespace) - csiDeployment, err := client.AppsV1().Deployments(csiNamespace).Get( - ctx, vSphereCSIControllerPodNamePrefix, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - csiReplicas = *csiDeployment.Spec.Replicas - - vsphereCfg := getCSIConfigSecretData(client, ctx, csiNamespace) - vCenterUIUser = vsphereCfg.Global.User - vCenterUIPassword = vsphereCfg.Global.Password - dataCenter = vsphereCfg.Global.Datacenters - clusterId = vsphereCfg.Global.ClusterID - vCenterIP = e2eVSphere.Config.Global.VCenterHostname - vCenterPort = e2eVSphere.Config.Global.VCenterPort - framework.Logf("clusterId: %v", clusterId) - revertToOriginalVsphereSecret = false - nimbusGeneratedK8sVmPwd = GetAndExpectStringEnvVar(nimbusK8sVmPwd) - - sshClientConfig = &ssh.ClientConfig{ - User: "root", - Auth: []ssh.AuthMethod{ - ssh.Password(nimbusGeneratedK8sVmPwd), - }, - HostKeyCallback: ssh.InsecureIgnoreHostKey(), - } - }) - - ginkgo.AfterEach(func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - if !revertToOriginalVsphereSecret { - ginkgo.By("Delete vsphere-csi-cluster-id configmap if it exists") - _, err := client.CoreV1().ConfigMaps(csiNamespace).Get(ctx, - vsphereClusterIdConfigMapName, metav1.GetOptions{}) - if !apierrors.IsNotFound(err) { - err = client.CoreV1().ConfigMaps(csiNamespace).Delete(ctx, - vsphereClusterIdConfigMapName, metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - ginkgo.By("Reverting back to original vsphere secret") - framework.Logf("clusterId: %v", clusterId) - recreateVsphereConfigSecret(client, ctx, vCenterUIUser, vCenterUIPassword, csiNamespace, vCenterIP, - clusterId, vCenterPort, dataCenter, csiReplicas) - } - }) - - /* - Generate unique cluster id through configmap and create workloads - 1. Create vsphere config secret with no cluster id field. - 2. Validate that "vsphere-csi-cluster-id" configmap is generated with a unique cluster id. - 3. Create statefulset with replica 3 and a deployment. - 4. Verify all PVCs are in bound state and pods are in running state. - 5. Scale sts replica to 5. - 6. Verify cns metadata and check if cluster id is populated in cns metadata. - 7. Clean up the sts, deployment, pods and PVCs. - - */ - ginkgo.It("[csi-config-secret-block][csi-config-secret-file] Generate unique cluster id through configmap"+ - " and create workloads", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - framework.Logf("CNS_TEST: Running for vanilla k8s setup") - - ginkgo.By("Creating csi config secret with no cluster id field set") - recreateVsphereConfigSecret(client, ctx, vCenterUIUser, vCenterUIPassword, csiNamespace, vCenterIP, - "", vCenterPort, dataCenter, csiReplicas) - - defer func() { - if !revertToOriginalVsphereSecret { - ginkgo.By("Delete vsphere-csi-cluster-id configmap if it exists") - _, err := client.CoreV1().ConfigMaps(csiNamespace).Get(ctx, - vsphereClusterIdConfigMapName, metav1.GetOptions{}) - if !apierrors.IsNotFound(err) { - err = client.CoreV1().ConfigMaps(csiNamespace).Delete(ctx, - vsphereClusterIdConfigMapName, metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - ginkgo.By("Reverting back to original vsphere secret") - framework.Logf("clusterId: %v", clusterId) - recreateVsphereConfigSecret(client, ctx, vCenterUIUser, vCenterUIPassword, csiNamespace, vCenterIP, - clusterId, vCenterPort, dataCenter, csiReplicas) - } - revertToOriginalVsphereSecret = true - }() - - ginkgo.By("Verify cluster id configmap is auto generated by csi driver") - verifyClusterIdConfigMapGeneration(client, ctx, csiNamespace, true) - clusterID := fetchClusterIdFromConfigmap(client, ctx, csiNamespace) - framework.Logf("clusterID: %v", clusterID) - - ginkgo.By("Creating Storage Class") - if rwxAccessMode { - scParameters[scParamFsType] = nfs4FSType - } - sc, err := createStorageClass(client, scParameters, nil, "", "", false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - ginkgo.By("Delete Storage Class") - err = client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Creating service") - service := CreateService(namespace, client) - defer func() { - deleteService(namespace, client, service) - }() - // Check if it is file volumes setups - if rwxAccessMode { - accessMode = v1.ReadWriteMany - } - ginkgo.By("Creating statefulset with replica 3 and a deployment") - statefulset, deployment, _ := createStsDeployment(ctx, client, namespace, sc, true, - false, 0, "", accessMode) - replicas := *(statefulset.Spec.Replicas) - - defer func() { - scaleDownNDeleteStsDeploymentsInNamespace(ctx, client, namespace) - pvcs, err := client.CoreV1().PersistentVolumeClaims(namespace).List(ctx, metav1.ListOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - for _, claim := range pvcs.Items { - pv := getPvFromClaim(client, namespace, claim.Name) - err := fpv.DeletePersistentVolumeClaim(client, claim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By("Verify it's PV and corresponding volumes are deleted from CNS") - err = fpv.WaitForPersistentVolumeDeleted(client, pv.Name, poll, - pollTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - volumeHandle := pv.Spec.CSI.VolumeHandle - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volumeHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), - fmt.Sprintf("Volume: %s should not be present in the CNS after it is deleted from "+ - "kubernetes", volumeHandle)) - } - }() - - // Scale up replicas of statefulset and verify CNS entries for volumes - scaleUpStsAndVerifyPodMetadata(ctx, client, namespace, statefulset, - replicas+2, true, true) - verifyVolumeMetadataOnDeployments(ctx, client, deployment, namespace, nil, nil, - nil, "") - err = checkClusterIdValueOnWorkloads(&e2eVSphere, client, ctx, namespace, clusterID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - scaleDownNDeleteStsDeploymentsInNamespace(ctx, client, namespace) - - }) - - /* - Modify unique cluster id value in Configmap - 1. Create vsphere config secret with no cluster id field. - 2. Validate that "vsphere-csi-cluster-id" configmap is generated with a unique cluster id. - 3. Change the cluster id value in "vsphere-csi-cluster-id" configmap, which should throw a proper error. - - */ - ginkgo.It("[csi-config-secret-block][csi-config-secret-file] Modify unique cluster id value in"+ - " Configmap", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - framework.Logf("CNS_TEST: Running for vanilla k8s setup") - - ginkgo.By("Creating csi config secret with no cluster id field set") - recreateVsphereConfigSecret(client, ctx, vCenterUIUser, vCenterUIPassword, csiNamespace, vCenterIP, - "", vCenterPort, dataCenter, csiReplicas) - - defer func() { - if !revertToOriginalVsphereSecret { - ginkgo.By("Delete vsphere-csi-cluster-id configmap if it exists") - _, err := client.CoreV1().ConfigMaps(csiNamespace).Get(ctx, - vsphereClusterIdConfigMapName, metav1.GetOptions{}) - if !apierrors.IsNotFound(err) { - err = client.CoreV1().ConfigMaps(csiNamespace).Delete(ctx, - vsphereClusterIdConfigMapName, metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - ginkgo.By("Reverting back to original vsphere secret") - framework.Logf("clusterId: %v", clusterId) - recreateVsphereConfigSecret(client, ctx, vCenterUIUser, vCenterUIPassword, csiNamespace, vCenterIP, - clusterId, vCenterPort, dataCenter, csiReplicas) - } - revertToOriginalVsphereSecret = true - }() - - ginkgo.By("Verify cluster id configmap is auto generated by csi driver") - verifyClusterIdConfigMapGeneration(client, ctx, csiNamespace, true) - - ginkgo.By("Modify unique cluster id value in Configmap") - clusterIdCm, err := client.CoreV1().ConfigMaps(csiNamespace).Get(ctx, vsphereClusterIdConfigMapName, - metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - clusterIdCm.Data["clusterID"] = "cluster1" - _, err = client.CoreV1().ConfigMaps(csiNamespace).Update(ctx, clusterIdCm, - metav1.UpdateOptions{}) - framework.Logf("Error from updating cluster id value in configmap is : %v", err.Error()) - gomega.Expect(err).To(gomega.HaveOccurred()) - - }) - - /* - Generate cluster id and then set cluster id in vsphere config secret - and remove cluster id field in vsphere config secret - 1. Create vsphere config secret with no cluster id field. - 2. Validate that "vsphere-csi-cluster-id" configmap is generated with a unique cluster id. - 3. Create statefulset with replica 3 and a deployment. - 4. Verify all PVCs are in bound state and pods are in running state. - 5. Delete config secret and create config secret with cluster id set and restart csi driver. - 6. Check if "vsphere-csi-cluster-id" configmap still exists. - 7. Verify CSI pods go into crashing state with a proper error message. - 8. Remove cluster id field from vsphere config secret and restart csi driver. - 9. Verify csi pods are in running state. - 10. Scale sts replica to 5. - 11. Verify cns metadata and check if cluster id is populated in cns metadata. - 12. Clean up the sts, deployment, pods and PVCs. - - */ - ginkgo.It("[csi-config-secret-block][csi-config-secret-file] Generate cluster id and then set cluster id "+ - "in vsphere config secret and remove cluster id field in vsphere config secret", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - framework.Logf("CNS_TEST: Running for vanilla k8s setup") - - ginkgo.By("Creating csi config secret with no cluster id field set") - recreateVsphereConfigSecret(client, ctx, vCenterUIUser, vCenterUIPassword, csiNamespace, vCenterIP, - "", vCenterPort, dataCenter, csiReplicas) - defer func() { - if !revertToOriginalVsphereSecret { - ginkgo.By("Delete vsphere-csi-cluster-id configmap if it exists") - _, err := client.CoreV1().ConfigMaps(csiNamespace).Get(ctx, - vsphereClusterIdConfigMapName, metav1.GetOptions{}) - if !apierrors.IsNotFound(err) { - err = client.CoreV1().ConfigMaps(csiNamespace).Delete(ctx, - vsphereClusterIdConfigMapName, metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - ginkgo.By("Reverting back to original vsphere secret") - framework.Logf("clusterId: %v", clusterId) - recreateVsphereConfigSecret(client, ctx, vCenterUIUser, vCenterUIPassword, csiNamespace, vCenterIP, - clusterId, vCenterPort, dataCenter, csiReplicas) - } - revertToOriginalVsphereSecret = true - }() - - ginkgo.By("Verify cluster id configmap is auto generated by csi driver") - verifyClusterIdConfigMapGeneration(client, ctx, csiNamespace, true) - clusterID := fetchClusterIdFromConfigmap(client, ctx, csiNamespace) - framework.Logf("clusterID: %v", clusterID) - - ginkgo.By("Creating Storage Class") - if rwxAccessMode { - scParameters[scParamFsType] = nfs4FSType - } - sc, err := createStorageClass(client, scParameters, nil, "", "", false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - ginkgo.By("Delete Storage Class") - err = client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Creating service") - service := CreateService(namespace, client) - defer func() { - deleteService(namespace, client, service) - }() - // Check if it is file volumes setups - if rwxAccessMode { - accessMode = v1.ReadWriteMany - } - ginkgo.By("Creating statefulset with replica 3 and a deployment") - statefulset, deployment, _ := createStsDeployment(ctx, client, namespace, sc, true, - false, 0, "", accessMode) - replicas := *(statefulset.Spec.Replicas) - - defer func() { - scaleDownNDeleteStsDeploymentsInNamespace(ctx, client, namespace) - pvcs, err := client.CoreV1().PersistentVolumeClaims(namespace).List(ctx, metav1.ListOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - for _, claim := range pvcs.Items { - pv := getPvFromClaim(client, namespace, claim.Name) - err := fpv.DeletePersistentVolumeClaim(client, claim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By("Verify it's PV and corresponding volumes are deleted from CNS") - err = fpv.WaitForPersistentVolumeDeleted(client, pv.Name, poll, - pollTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - volumeHandle := pv.Spec.CSI.VolumeHandle - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volumeHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), - fmt.Sprintf("Volume: %s should not be present in the CNS after it is deleted from "+ - "kubernetes", volumeHandle)) - } - }() - - ginkgo.By("Creating csi config secret with cluster id field set") - createCsiVsphereSecret(client, ctx, vCenterUIUser, vCenterUIPassword, csiNamespace, - vCenterIP, vCenterPort, dataCenter, "", "cluster1") - - ginkgo.By("Restart CSI driver") - _, err = restartCSIDriver(ctx, client, csiNamespace, csiReplicas) - gomega.Expect(err).To(gomega.HaveOccurred()) - ginkgo.By("Verify that cluster id configmap still exists") - verifyClusterIdConfigMapGeneration(client, ctx, csiNamespace, true) - - ginkgo.By("Check if csi pods are in crashing state after recreation of secret with proper message") - csipods, err := client.CoreV1().Pods(csiNamespace).List(ctx, metav1.ListOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - k8sMasterIPs := getK8sMasterIPs(ctx, client) - k8sMasterIP := k8sMasterIPs[0] - var csiPodName string - for _, csiPod := range csipods.Items { - if strings.Contains(csiPod.Name, vSphereCSIControllerPodNamePrefix) { - csiPodName = csiPod.Name - break - } - } - errMessage := "Please remove the cluster ID from vSphere Config Secret." - grepCmdForErrMsg := "echo `kubectl logs " + csiPodName + " -n " + - csiSystemNamespace + " --allContainers" + " | grep " + "'" + errMessage - - framework.Logf("Invoking command '%v' on host %v", grepCmdForErrMsg, - k8sMasterIP) - result, err := sshExec(sshClientConfig, k8sMasterIP, - grepCmdForErrMsg) - if err != nil || result.Code != 0 { - fssh.LogResult(result) - gomega.Expect(err).To(gomega.HaveOccurred(), fmt.Sprintf("couldn't execute command: %s on host: %v , error: %s", - grepCmdForErrMsg, k8sMasterIP, err)) - } - if result.Stdout != "" { - framework.Logf("CSI pods are in crashing state with proper error message") - } else { - framework.Logf("CSI pods are in crashing state with improper error message") - gomega.Expect(err).To(gomega.HaveOccurred()) - } - - ginkgo.By("Remove cluster id field from vsphere config secret and verify csi pods are in running state") - recreateVsphereConfigSecret(client, ctx, vCenterUIUser, vCenterUIPassword, csiNamespace, vCenterIP, - "", vCenterPort, dataCenter, csiReplicas) - newclusterID := fetchClusterIdFromConfigmap(client, ctx, csiNamespace) - if clusterID != newclusterID { - framework.Failf("New clusterID should not be generated") - } - csipods, err = client.CoreV1().Pods(csiNamespace).List(ctx, metav1.ListOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fpod.WaitForPodsRunningReady(client, csiNamespace, int32(csipods.Size()), 0, pollTimeout, nil) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Scale up replicas of statefulset and verify CNS entries for volumes") - scaleUpStsAndVerifyPodMetadata(ctx, client, namespace, statefulset, - replicas+2, true, true) - verifyVolumeMetadataOnDeployments(ctx, client, deployment, namespace, nil, nil, - nil, "") - err = checkClusterIdValueOnWorkloads(&e2eVSphere, client, ctx, namespace, clusterID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - scaleDownNDeleteStsDeploymentsInNamespace(ctx, client, namespace) - - }) - - /* - Recreate vsphere config secret multiple times - 1. Create vsphere config secret with no cluster id field. - 2. Validate that "vsphere-csi-cluster-id" configmap is generated with a unique cluster id. - 3. Create statefulset with replica 3 and a deployment. - 4. Verify all PVCs are in bound state and pods are in running state. - 5. Delete and create vsphere config secret multiple times(3-4 times atleast continuously). - 6. Verify "vsphere-csi-cluster-id" configmap still exists with same cluster id. - 7. Scale sts replica to 5. - 8. Verify cns metadata and check if cluster id is populated in cns metadata. - 9. Clean up the sts, deployment, pods and PVCs. - - */ - ginkgo.It("[csi-config-secret-block][csi-config-secret-file] Recreate vsphere config secret multiple"+ - " times", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - framework.Logf("CNS_TEST: Running for vanilla k8s setup") - - ginkgo.By("Creating csi config secret with no cluster id field set") - recreateVsphereConfigSecret(client, ctx, vCenterUIUser, vCenterUIPassword, csiNamespace, vCenterIP, - "", vCenterPort, dataCenter, csiReplicas) - - defer func() { - if !revertToOriginalVsphereSecret { - ginkgo.By("Delete vsphere-csi-cluster-id configmap if it exists") - _, err := client.CoreV1().ConfigMaps(csiNamespace).Get(ctx, - vsphereClusterIdConfigMapName, metav1.GetOptions{}) - if !apierrors.IsNotFound(err) { - err = client.CoreV1().ConfigMaps(csiNamespace).Delete(ctx, - vsphereClusterIdConfigMapName, metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - ginkgo.By("Reverting back to original vsphere secret") - framework.Logf("clusterId: %v", clusterId) - recreateVsphereConfigSecret(client, ctx, vCenterUIUser, vCenterUIPassword, csiNamespace, vCenterIP, - clusterId, vCenterPort, dataCenter, csiReplicas) - } - revertToOriginalVsphereSecret = true - }() - - ginkgo.By("Verify cluster id configmap is auto generated by csi driver") - verifyClusterIdConfigMapGeneration(client, ctx, csiNamespace, true) - clusterID := fetchClusterIdFromConfigmap(client, ctx, csiNamespace) - framework.Logf("clusterID: %v", clusterID) - - ginkgo.By("Creating Storage Class") - if rwxAccessMode { - scParameters[scParamFsType] = nfs4FSType - } - sc, err := createStorageClass(client, scParameters, nil, "", "", false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - ginkgo.By("Delete Storage Class") - err = client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Creating service") - service := CreateService(namespace, client) - defer func() { - deleteService(namespace, client, service) - }() - // Check if it is file volumes setups - if rwxAccessMode { - accessMode = v1.ReadWriteMany - } - ginkgo.By("Creating statefulset with replica 3 and a deployment") - statefulset, deployment, _ := createStsDeployment(ctx, client, namespace, sc, true, - false, 0, "", accessMode) - replicas := *(statefulset.Spec.Replicas) - - defer func() { - scaleDownNDeleteStsDeploymentsInNamespace(ctx, client, namespace) - pvcs, err := client.CoreV1().PersistentVolumeClaims(namespace).List(ctx, metav1.ListOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - for _, claim := range pvcs.Items { - pv := getPvFromClaim(client, namespace, claim.Name) - err := fpv.DeletePersistentVolumeClaim(client, claim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By("Verify it's PV and corresponding volumes are deleted from CNS") - err = fpv.WaitForPersistentVolumeDeleted(client, pv.Name, poll, - pollTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - volumeHandle := pv.Spec.CSI.VolumeHandle - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volumeHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), - fmt.Sprintf("Volume: %s should not be present in the CNS after it is deleted from "+ - "kubernetes", volumeHandle)) - } - }() - - ginkgo.By("Recreating CSI config secret multiple times to verify if a" + - "new cluster id configmap gets auto generated") - for i := 0; i < 3; i++ { - recreateVsphereConfigSecret(client, ctx, vCenterUIUser, vCenterUIPassword, csiNamespace, vCenterIP, - "", vCenterPort, dataCenter, csiReplicas) - } - newclusterID := fetchClusterIdFromConfigmap(client, ctx, csiNamespace) - if clusterID != newclusterID { - framework.Failf("New clusterID should not be generated") - } - - ginkgo.By("Scale up replicas of statefulset and verify CNS entries for volumes") - scaleUpStsAndVerifyPodMetadata(ctx, client, namespace, statefulset, - replicas+2, true, true) - verifyVolumeMetadataOnDeployments(ctx, client, deployment, namespace, nil, nil, - nil, "") - err = checkClusterIdValueOnWorkloads(&e2eVSphere, client, ctx, namespace, clusterID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - scaleDownNDeleteStsDeploymentsInNamespace(ctx, client, namespace) - - }) - - /* - Create vsphere config secret with cluster id value set with special characters - 1. Special characters in the user-provided value for cluster-id key in vsphere config secret and try to create PVC - a. cluster-id" key and value including special characters - example "#1$k8s" - b. "cluster-id" key and value as maximum length of characters - 2. The CNS metadata for the PVC should have a cluster id value set. - */ - ginkgo.It("[csi-config-secret-block][csi-config-secret-file] Create vsphere config secret with cluster "+ - "id value set with special characters", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - framework.Logf("CNS_TEST: Running for vanilla k8s setup") - ginkgo.By("Delete vsphere-csi-cluster-id configmap if it exists") - _, err := client.CoreV1().ConfigMaps(csiNamespace).Get(ctx, - vsphereClusterIdConfigMapName, metav1.GetOptions{}) - if !apierrors.IsNotFound(err) { - err = client.CoreV1().ConfigMaps(csiNamespace).Delete(ctx, - vsphereClusterIdConfigMapName, metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - ginkgo.By("Creating csi config secret with cluster id value with some special characters field set") - clusterID := "#1$k8s" - recreateVsphereConfigSecret(client, ctx, vCenterUIUser, vCenterUIPassword, csiNamespace, vCenterIP, - clusterID, vCenterPort, dataCenter, csiReplicas) - - defer func() { - if !revertToOriginalVsphereSecret { - ginkgo.By("Delete vsphere-csi-cluster-id configmap if it exists") - _, err := client.CoreV1().ConfigMaps(csiNamespace).Get(ctx, - vsphereClusterIdConfigMapName, metav1.GetOptions{}) - if !apierrors.IsNotFound(err) { - err = client.CoreV1().ConfigMaps(csiNamespace).Delete(ctx, - vsphereClusterIdConfigMapName, metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - ginkgo.By("Reverting back to original vsphere secret") - framework.Logf("clusterId: %v", clusterId) - recreateVsphereConfigSecret(client, ctx, vCenterUIUser, vCenterUIPassword, csiNamespace, vCenterIP, - clusterId, vCenterPort, dataCenter, csiReplicas) - } - revertToOriginalVsphereSecret = true - }() - - ginkgo.By("Verify cluster id configmap is not auto generated by csi driver") - verifyClusterIdConfigMapGeneration(client, ctx, csiNamespace, false) - - ginkgo.By("Creating Storage Class and PVC") - if rwxAccessMode { - scParameters[scParamFsType] = nfs4FSType - accessMode = v1.ReadWriteMany - } - sc, pvclaim, err := createPVCAndStorageClass(client, - namespace, nil, scParameters, diskSize, nil, "", false, accessMode) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - defer func() { - ginkgo.By("Delete Storage Class") - err = client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - pvs, err := fpv.WaitForPVClaimBoundPhase(client, []*v1.PersistentVolumeClaim{pvclaim}, - framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - queryResult1, err := e2eVSphere.queryCNSVolumeWithResult(pvs[0].Spec.CSI.VolumeHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(len(queryResult1.Volumes) > 0).To(gomega.BeTrue()) - - framework.Logf("Cluster ID value on CNS is %s", - queryResult1.Volumes[0].Metadata.ContainerClusterArray[0].ClusterId) - gomega.Expect(queryResult1.Volumes[0].Metadata.ContainerClusterArray[0].ClusterId).Should( - gomega.Equal(clusterID), "Wrong/empty cluster id name present") - - defer func() { - pvcs, err := client.CoreV1().PersistentVolumeClaims(namespace).List(ctx, metav1.ListOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - for _, claim := range pvcs.Items { - pv := getPvFromClaim(client, namespace, claim.Name) - err := fpv.DeletePersistentVolumeClaim(client, claim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By("Verify it's PV and corresponding volumes are deleted from CNS") - err = fpv.WaitForPersistentVolumeDeleted(client, pvs[0].Name, poll, - pollTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - volumeHandle := pv.Spec.CSI.VolumeHandle - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volumeHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), - fmt.Sprintf("Volume: %s should not be present in the CNS after it is deleted from "+ - "kubernetes", volumeHandle)) - } - }() - - err = fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By("Verify it's PV and corresponding volumes are deleted from CNS") - err = fpv.WaitForPersistentVolumeDeleted(client, pvs[0].Name, poll, - pollTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - volumeHandle := pvs[0].Spec.CSI.VolumeHandle - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volumeHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), - fmt.Sprintf("Volume: %s should not be present in the CNS after it is deleted from "+ - "kubernetes", volumeHandle)) - - letters := []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") - // 64 is the max length supported for cluster id value - n := 64 - c := make([]rune, n) - for i := range c { - c[i] = letters[rand.Intn(len(letters))] - } - clusterID = string(c) - - recreateVsphereConfigSecret(client, ctx, vCenterUIUser, vCenterUIPassword, csiNamespace, vCenterIP, - clusterID, vCenterPort, dataCenter, csiReplicas) - - ginkgo.By("Verify cluster id configmap is not auto generated by csi driver") - verifyClusterIdConfigMapGeneration(client, ctx, csiNamespace, false) - - sc, pvclaim, err = createPVCAndStorageClass(client, - namespace, nil, scParameters, diskSize, nil, "", false, accessMode) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - pvs, err = fpv.WaitForPVClaimBoundPhase(client, []*v1.PersistentVolumeClaim{pvclaim}, framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - queryResult2, err := e2eVSphere.queryCNSVolumeWithResult(pvs[0].Spec.CSI.VolumeHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(len(queryResult2.Volumes) > 0).To(gomega.BeTrue()) - - framework.Logf("Cluster ID value on CNS is %s", - queryResult2.Volumes[0].Metadata.ContainerClusterArray[0].ClusterId) - gomega.Expect(queryResult2.Volumes[0].Metadata.ContainerClusterArray[0].ClusterId).Should( - gomega.Equal(clusterID), "Wrong/empty cluster id name present") - - }) - - /* - Restart CSI pods multiple times - 1. Create vsphere config secret with no cluster id field. - 2. Validate that "vsphere-csi-cluster-id" configmap is generated with a unique cluster id. - 3. Create statefulset with replica 3 and a deployment. - 4. Verify all PVCs are in bound state and pods are in running state. - 5. Delete and create vsphere config secret multiple times(3-4 times atleast continuously). - 6. Verify "vsphere-csi-cluster-id" configmap still exists with same cluster id. - 7. Scale sts replica to 5. - 8. Verify cns metadata and check if cluster id is populated in cns metadata. - 9. Clean up the sts, deployment, pods and PVCs. - */ - ginkgo.It("[csi-config-secret-block][csi-config-secret-file] Restart CSI pods multiple"+ - " times", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - framework.Logf("CNS_TEST: Running for vanilla k8s setup") - ginkgo.By("Creating csi config secret with no cluster id field set") - recreateVsphereConfigSecret(client, ctx, vCenterUIUser, vCenterUIPassword, csiNamespace, vCenterIP, - "", vCenterPort, dataCenter, csiReplicas) - defer func() { - if !revertToOriginalVsphereSecret { - ginkgo.By("Delete vsphere-csi-cluster-id configmap if it exists") - _, err := client.CoreV1().ConfigMaps(csiNamespace).Get(ctx, - vsphereClusterIdConfigMapName, metav1.GetOptions{}) - if !apierrors.IsNotFound(err) { - err = client.CoreV1().ConfigMaps(csiNamespace).Delete(ctx, - vsphereClusterIdConfigMapName, metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - ginkgo.By("Reverting back to original vsphere secret") - framework.Logf("clusterId: %v", clusterId) - recreateVsphereConfigSecret(client, ctx, vCenterUIUser, vCenterUIPassword, csiNamespace, vCenterIP, - clusterId, vCenterPort, dataCenter, csiReplicas) - } - revertToOriginalVsphereSecret = true - }() - - ginkgo.By("Verify cluster id configmap is auto generated by csi driver") - verifyClusterIdConfigMapGeneration(client, ctx, csiNamespace, true) - clusterID := fetchClusterIdFromConfigmap(client, ctx, csiNamespace) - framework.Logf("clusterID: %v", clusterID) - - ginkgo.By("Creating Storage Class") - if rwxAccessMode { - scParameters[scParamFsType] = nfs4FSType - } - sc, err := createStorageClass(client, scParameters, nil, "", "", false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - ginkgo.By("Delete Storage Class") - err = client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Creating service") - service := CreateService(namespace, client) - defer func() { - deleteService(namespace, client, service) - }() - // Check if it is file volumes setups - if rwxAccessMode { - accessMode = v1.ReadWriteMany - } - ginkgo.By("Creating statefulset with replica 3 and a deployment") - statefulset, deployment, _ := createStsDeployment(ctx, client, namespace, sc, true, - false, 0, "", accessMode) - replicas := *(statefulset.Spec.Replicas) - - defer func() { - scaleDownNDeleteStsDeploymentsInNamespace(ctx, client, namespace) - pvcs, err := client.CoreV1().PersistentVolumeClaims(namespace).List(ctx, metav1.ListOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - for _, claim := range pvcs.Items { - pv := getPvFromClaim(client, namespace, claim.Name) - err := fpv.DeletePersistentVolumeClaim(client, claim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By("Verify it's PV and corresponding volumes are deleted from CNS") - err = fpv.WaitForPersistentVolumeDeleted(client, pv.Name, poll, - pollTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - volumeHandle := pv.Spec.CSI.VolumeHandle - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volumeHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), - fmt.Sprintf("Volume: %s should not be present in the CNS after it is deleted from "+ - "kubernetes", volumeHandle)) - } - }() - for i := 0; i < 3; i++ { - restartSuccess, err := restartCSIDriver(ctx, client, csiNamespace, csiReplicas) - gomega.Expect(restartSuccess).To(gomega.BeTrue(), "csi driver restart not successful") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - ginkgo.By("Scale up replicas of statefulset and verify CNS entries for volumes") - scaleUpStsAndVerifyPodMetadata(ctx, client, namespace, statefulset, - replicas+2, true, true) - verifyVolumeMetadataOnDeployments(ctx, client, deployment, namespace, nil, nil, - nil, "") - err = checkClusterIdValueOnWorkloads(&e2eVSphere, client, ctx, namespace, clusterID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - scaleDownNDeleteStsDeploymentsInNamespace(ctx, client, namespace) - - }) - - /* - Delete CSI driver - 1. Create vsphere config secret with no cluster id field. - 2. Validate that "vsphere-csi-cluster-id" configmap is generated with a unique cluster id. - 3. Create statefulset with replica 3 and a deployment. - 4. Verify all PVCs are in bound state and pods are in running state. - 5. Delete and create vsphere config secret multiple times(3-4 times atleast continuously). - 6. Verify "vsphere-csi-cluster-id" configmap still exists with same cluster id. - 7. Scale sts replica to 5. - 8. Verify cns metadata and check if cluster id is populated in cns metadata. - 9. Clean up the sts, deployment, pods and PVCs. - */ - ginkgo.It("[csi-config-secret-block][csi-config-secret-file] Delete CSI"+ - " driver", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - framework.Logf("CNS_TEST: Running for vanilla k8s setup") - - ginkgo.By("Creating csi config secret with no cluster id field set") - recreateVsphereConfigSecret(client, ctx, vCenterUIUser, vCenterUIPassword, csiNamespace, vCenterIP, - "", vCenterPort, dataCenter, csiReplicas) - - defer func() { - if !revertToOriginalVsphereSecret { - ginkgo.By("Delete vsphere-csi-cluster-id configmap if it exists") - _, err := client.CoreV1().ConfigMaps(csiNamespace).Get(ctx, - vsphereClusterIdConfigMapName, metav1.GetOptions{}) - if !apierrors.IsNotFound(err) { - err = client.CoreV1().ConfigMaps(csiNamespace).Delete(ctx, - vsphereClusterIdConfigMapName, metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - ginkgo.By("Reverting back to original vsphere secret") - framework.Logf("clusterId: %v", clusterId) - recreateVsphereConfigSecret(client, ctx, vCenterUIUser, vCenterUIPassword, csiNamespace, vCenterIP, - clusterId, vCenterPort, dataCenter, csiReplicas) - } - revertToOriginalVsphereSecret = true - }() - ginkgo.By("Verify cluster id configmap is auto generated by csi driver") - verifyClusterIdConfigMapGeneration(client, ctx, csiNamespace, true) - clusterID := fetchClusterIdFromConfigmap(client, ctx, csiNamespace) - framework.Logf("clusterID: %v", clusterID) - - ginkgo.By("Creating Storage Class") - if rwxAccessMode { - scParameters[scParamFsType] = nfs4FSType - } - sc, err := createStorageClass(client, scParameters, nil, "", "", false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - ginkgo.By("Delete Storage Class") - err = client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Creating service") - service := CreateService(namespace, client) - defer func() { - deleteService(namespace, client, service) - }() - // Check if it is file volumes setups - if rwxAccessMode { - accessMode = v1.ReadWriteMany - } - ginkgo.By("Creating statefulset with replica 3 and a deployment") - statefulset, deployment, _ := createStsDeployment(ctx, client, namespace, sc, true, - false, 0, "", accessMode) - replicas := *(statefulset.Spec.Replicas) - - defer func() { - scaleDownNDeleteStsDeploymentsInNamespace(ctx, client, namespace) - pvcs, err := client.CoreV1().PersistentVolumeClaims(namespace).List(ctx, metav1.ListOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - for _, claim := range pvcs.Items { - pv := getPvFromClaim(client, namespace, claim.Name) - err := fpv.DeletePersistentVolumeClaim(client, claim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By("Verify it's PV and corresponding volumes are deleted from CNS") - err = fpv.WaitForPersistentVolumeDeleted(client, pv.Name, poll, - pollTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - volumeHandle := pv.Spec.CSI.VolumeHandle - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volumeHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), - fmt.Sprintf("Volume: %s should not be present in the CNS after it is deleted from "+ - "kubernetes", volumeHandle)) - } - }() - - var ignoreLabels map[string]string - list_of_pods, err := fpod.GetPodsInNamespace(client, csiSystemNamespace, ignoreLabels) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - allMasterIps := getK8sMasterIPs(ctx, client) - masterIP := "" - filePath := "/root/vsphere-csi-driver.yaml" - cmd := "kubectl delete -f " + filePath - for _, k8sMasterIP := range allMasterIps { - framework.Logf("Invoking command '%v' on host %v", cmd, - k8sMasterIP) - result, err := sshExec(sshClientConfig, k8sMasterIP, - cmd) - fssh.LogResult(result) - if err == nil { - framework.Logf("File exists on %s", k8sMasterIP) - masterIP = k8sMasterIP - break - } - } - - for _, pod := range list_of_pods { - err = fpod.WaitForPodNotFoundInNamespace(client, pod.Name, csiNamespace, pollTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), fmt.Sprintf("pod %q was not deleted: %v", pod.Name, err)) - } - - cmd = "kubectl apply -f " + filePath - framework.Logf("Invoking command '%v' on host %v", cmd, - masterIP) - result, err := sshExec(sshClientConfig, masterIP, - cmd) - if err != nil || result.Code != 0 { - fssh.LogResult(result) - gomega.Expect(err).To(gomega.HaveOccurred(), fmt.Sprintf("couldn't execute command: %s on host: %v , error: %s", - cmd, masterIP, err)) - } - - // Wait for the CSI Pods to be up and Running - list_of_pods, err = fpod.GetPodsInNamespace(client, csiSystemNamespace, ignoreLabels) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - num_csi_pods := len(list_of_pods) - err = fpod.WaitForPodsRunningReady(client, csiSystemNamespace, int32(num_csi_pods), 0, - pollTimeout, ignoreLabels) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - newclusterID := fetchClusterIdFromConfigmap(client, ctx, csiNamespace) - if clusterID != newclusterID { - framework.Failf("New clusterID should not be generated") - } - - ginkgo.By("Scale up replicas of statefulset and verify CNS entries for volumes") - scaleUpStsAndVerifyPodMetadata(ctx, client, namespace, statefulset, - replicas+2, true, true) - verifyVolumeMetadataOnDeployments(ctx, client, deployment, namespace, nil, nil, - nil, "") - err = checkClusterIdValueOnWorkloads(&e2eVSphere, client, ctx, namespace, clusterID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - scaleDownNDeleteStsDeploymentsInNamespace(ctx, client, namespace) - - }) - -}) diff --git a/tests/e2e/provision_with_multiple_zones.go b/tests/e2e/provision_with_multiple_zones.go index 9a19524dae..6609e33fa7 100644 --- a/tests/e2e/provision_with_multiple_zones.go +++ b/tests/e2e/provision_with_multiple_zones.go @@ -166,7 +166,7 @@ var _ = ginkgo.Describe("[csi-topology-vanilla] Topology-Aware-Provisioning-With in the allowed topology //Steps 1. Create SC with multiple Zone and region details specified in the SC - 2. Create statefulset with replica 3 using the above SC + 2. Create statefulset with replica 5 using the above SC 3. Wait for PV, PVC to bound 4. Statefulset should get distributed across zones. 5. Describe PV and verify node affinity details should contain both @@ -199,17 +199,11 @@ var _ = ginkgo.Describe("[csi-topology-vanilla] Topology-Aware-Provisioning-With gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() - ginkgo.By("Creating service") - service := CreateService(namespace, client) - defer func() { - deleteService(namespace, client, service) - }() - - // Creating statefulset with 3 replicas - ginkgo.By("Creating statefulset with 3 replica") + // Creating statefulset with 5 replicas + ginkgo.By("Creating statefulset with 5 replica") statefulset := GetStatefulSetFromManifest(namespace) ginkgo.By("Creating statefulset") - var replica int32 = 3 + var replica int32 = 5 statefulset.Spec.Replicas = &replica replicas := *(statefulset.Spec.Replicas) CreateStatefulSet(namespace, statefulset, client) diff --git a/tests/e2e/raw_block_volume.go b/tests/e2e/raw_block_volume.go deleted file mode 100644 index de8e1d0fdf..0000000000 --- a/tests/e2e/raw_block_volume.go +++ /dev/null @@ -1,1307 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2e - -import ( - "context" - "fmt" - "math/rand" - "os" - "os/exec" - "strconv" - "strings" - "time" - - snapclient "github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned" - ginkgo "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" - cnstypes "github.com/vmware/govmomi/cns/types" - "github.com/vmware/govmomi/find" - "github.com/vmware/govmomi/object" - "github.com/vmware/govmomi/vim25/types" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clientset "k8s.io/client-go/kubernetes" - restclient "k8s.io/client-go/rest" - "k8s.io/kubernetes/test/e2e/framework" - fnodes "k8s.io/kubernetes/test/e2e/framework/node" - fpod "k8s.io/kubernetes/test/e2e/framework/pod" - fpv "k8s.io/kubernetes/test/e2e/framework/pv" - fss "k8s.io/kubernetes/test/e2e/framework/statefulset" - admissionapi "k8s.io/pod-security-admission/api" -) - -const ( - statefulset_volname string = "block-vol" - statefulset_devicePath string = "/dev/testblk" - pod_devicePathPrefix string = "/mnt/volume" - volsizeInMiBBeforeExpansion int64 = 2048 -) - -var _ = ginkgo.Describe("raw block volume support", func() { - - f := framework.NewDefaultFramework("e2e-raw-block-volume") - f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged - var ( - namespace string - client clientset.Interface - defaultDatacenter *object.Datacenter - datastoreURL string - scParameters map[string]string - storageClassName string - storagePolicyName string - svcPVCName string - rawBlockVolumeMode = corev1.PersistentVolumeBlock - pandoraSyncWaitTime int - deleteFCDRequired bool - fcdID string - pv *corev1.PersistentVolume - pvc *corev1.PersistentVolumeClaim - snapc *snapclient.Clientset - restConfig *restclient.Config - guestClusterRestConfig *restclient.Config - ) - - ginkgo.BeforeEach(func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - namespace = getNamespaceToRunTests(f) - client = f.ClientSet - bootstrap() - nodeList, err := fnodes.GetReadySchedulableNodes(f.ClientSet) - framework.ExpectNoError(err, "Unable to find ready and schedulable Node") - if !(len(nodeList.Items) > 0) { - framework.Failf("Unable to find ready and schedulable Node") - } - - //Get snapshot client using the rest config - if !guestCluster { - restConfig = getRestConfigClient() - snapc, err = snapclient.NewForConfig(restConfig) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } else { - guestClusterRestConfig = getRestConfigClientForGuestCluster(guestClusterRestConfig) - snapc, err = snapclient.NewForConfig(guestClusterRestConfig) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - sc, err := client.StorageV1().StorageClasses().Get(ctx, defaultNginxStorageClassName, metav1.GetOptions{}) - if err == nil && sc != nil { - gomega.Expect(client.StorageV1().StorageClasses().Delete(ctx, sc.Name, - *metav1.NewDeleteOptions(0))).NotTo(gomega.HaveOccurred()) - } - scParameters = make(map[string]string) - storagePolicyName = GetAndExpectStringEnvVar(envStoragePolicyNameForSharedDatastores) - - if os.Getenv(envPandoraSyncWaitTime) != "" { - pandoraSyncWaitTime, err = strconv.Atoi(os.Getenv(envPandoraSyncWaitTime)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } else { - pandoraSyncWaitTime = defaultPandoraSyncWaitTime - } - deleteFCDRequired = false - - var datacenters []string - datastoreURL = GetAndExpectStringEnvVar(envSharedDatastoreURL) - finder := find.NewFinder(e2eVSphere.Client.Client, false) - cfg, err := getConfig() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - dcList := strings.Split(cfg.Global.Datacenters, ",") - for _, dc := range dcList { - dcName := strings.TrimSpace(dc) - if dcName != "" { - datacenters = append(datacenters, dcName) - } - } - for _, dc := range datacenters { - defaultDatacenter, err = finder.Datacenter(ctx, dc) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - finder.SetDatacenter(defaultDatacenter) - defaultDatastore, err = getDatastoreByURL(ctx, datastoreURL, defaultDatacenter) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - if guestCluster { - svcClient, svNamespace := getSvcClientAndNamespace() - setResourceQuota(svcClient, svNamespace, rqLimit) - } - }) - - ginkgo.AfterEach(func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - ginkgo.By(fmt.Sprintf("Deleting all statefulsets and/or deployments in namespace: %v", namespace)) - scaleDownNDeleteStsDeploymentsInNamespace(ctx, client, namespace) - ginkgo.By(fmt.Sprintf("Deleting service nginx in namespace: %v", namespace)) - err := client.CoreV1().Services(namespace).Delete(ctx, servicename, *metav1.NewDeleteOptions(0)) - if !apierrors.IsNotFound(err) { - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - if supervisorCluster { - dumpSvcNsEventsOnTestFailure(client, namespace) - } - if guestCluster { - svcClient, svNamespace := getSvcClientAndNamespace() - setResourceQuota(svcClient, svNamespace, defaultrqLimit) - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) - } - if deleteFCDRequired { - ginkgo.By(fmt.Sprintf("Deleting FCD: %s", fcdID)) - err := e2eVSphere.deleteFCD(ctx, fcdID, defaultDatastore.Reference()) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - }) - - /* - Test statefulset scaleup/scaledown operations with raw block volume - Steps - 1. Create a storage class. - 2. Create nginx service. - 3. Create nginx statefulsets with 3 replicas and using raw block volume. - 4. Wait until all Pods are ready and PVCs are bounded with PV. - 5. Scale down statefulsets to 2 replicas. - 6. Scale up statefulsets to 3 replicas. - 7. Scale down statefulsets to 0 replicas and delete all pods. - 8. Delete all PVCs from the tests namespace. - 9. Delete the storage class. - */ - ginkgo.It("[csi-block-vanilla] [csi-block-vanilla-parallelized] [csi-guest]"+ - "Statefulset testing with raw block volume and default podManagementPolicy", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - storageClassName = defaultNginxStorageClassName - if vanillaCluster { - ginkgo.By("CNS_TEST: Running for vanilla k8s setup") - } else if guestCluster { - ginkgo.By("CNS_TEST: Running for GC setup") - storageClassName = defaultNginxStorageClassName - scParameters[svStorageClassName] = storagePolicyName - } - - ginkgo.By("Creating StorageClass for Statefulset") - scSpec := getVSphereStorageClassSpec(storageClassName, scParameters, nil, "", "", false) - sc, err := client.StorageV1().StorageClasses().Create(ctx, scSpec, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err := client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Creating service") - service := CreateService(namespace, client) - defer func() { - deleteService(namespace, client, service) - }() - - ginkgo.By("Creating statefulset with raw block volume") - scName := defaultNginxStorageClassName - statefulset := GetStatefulSetFromManifest(namespace) - statefulset.Spec.Template.Spec.Containers[len(statefulset.Spec.Template.Spec.Containers)-1].VolumeMounts = nil - statefulset.Spec.Template.Spec.Containers[len(statefulset.Spec.Template.Spec.Containers)-1]. - VolumeDevices = []corev1.VolumeDevice{ - { - Name: statefulset_volname, - DevicePath: statefulset_devicePath, - }, - } - statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &scName - statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1].ObjectMeta.Name = - statefulset_volname - statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1].Spec.VolumeMode = - &rawBlockVolumeMode - CreateStatefulSet(namespace, statefulset, client) - replicas := *(statefulset.Spec.Replicas) - defer func() { - ginkgo.By(fmt.Sprintf("Deleting all statefulsets in namespace: %v", namespace)) - fss.DeleteAllStatefulSets(client, namespace) - }() - - // Waiting for pods status to be Ready - fss.WaitForStatusReadyReplicas(client, statefulset, replicas) - // Check if raw device available inside all pods of statefulset - gomega.Expect(CheckDevice(client, statefulset, statefulset_devicePath)).NotTo(gomega.HaveOccurred()) - - ssPodsBeforeScaleDown := fss.GetPodList(client, statefulset) - gomega.Expect(ssPodsBeforeScaleDown.Items).NotTo(gomega.BeEmpty(), - fmt.Sprintf("Unable to get list of Pods from the Statefulset: %v", statefulset.Name)) - gomega.Expect(len(ssPodsBeforeScaleDown.Items) == int(replicas)).To(gomega.BeTrue(), - "Number of Pods in the statefulset should match with number of replicas") - - // Get the list of Volumes attached to Pods before scale down - var volumesBeforeScaleDown []string - for _, sspod := range ssPodsBeforeScaleDown.Items { - _, err := client.CoreV1().Pods(namespace).Get(ctx, sspod.Name, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - for _, volumespec := range sspod.Spec.Volumes { - if volumespec.PersistentVolumeClaim != nil { - pv := getPvFromClaim(client, statefulset.Namespace, volumespec.PersistentVolumeClaim.ClaimName) - volumeID := pv.Spec.CSI.VolumeHandle - if guestCluster { - volumeID = getVolumeIDFromSupervisorCluster(pv.Spec.CSI.VolumeHandle) - } - volumesBeforeScaleDown = append(volumesBeforeScaleDown, volumeID) - // Verify the attached volume match the one in CNS cache - queryResult, err := e2eVSphere.queryCNSVolumeWithResult(volumeID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(len(queryResult.Volumes) > 0).To(gomega.BeTrue()) - } - } - } - - ginkgo.By(fmt.Sprintf("Scaling down statefulsets to number of Replica: %v", replicas-1)) - _, scaledownErr := fss.Scale(client, statefulset, replicas-1) - gomega.Expect(scaledownErr).NotTo(gomega.HaveOccurred()) - fss.WaitForStatusReadyReplicas(client, statefulset, replicas-1) - ssPodsAfterScaleDown := fss.GetPodList(client, statefulset) - gomega.Expect(ssPodsAfterScaleDown.Items).NotTo(gomega.BeEmpty(), - fmt.Sprintf("Unable to get list of Pods from the Statefulset: %v", statefulset.Name)) - gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(replicas-1)).To(gomega.BeTrue(), - "Number of Pods in the statefulset should match with number of replicas") - - // After scale down, verify vSphere volumes are detached from deleted pods - ginkgo.By("Verify Volumes are detached from Nodes after Statefulsets is scaled down") - for _, sspod := range ssPodsBeforeScaleDown.Items { - _, err := client.CoreV1().Pods(namespace).Get(ctx, sspod.Name, metav1.GetOptions{}) - if err != nil { - gomega.Expect(apierrors.IsNotFound(err), gomega.BeTrue()) - for _, volumespec := range sspod.Spec.Volumes { - if volumespec.PersistentVolumeClaim != nil { - pv := getPvFromClaim(client, statefulset.Namespace, volumespec.PersistentVolumeClaim.ClaimName) - volumeID := pv.Spec.CSI.VolumeHandle - if guestCluster { - volumeID = getVolumeIDFromSupervisorCluster(pv.Spec.CSI.VolumeHandle) - } - isDiskDetached, err := e2eVSphere.waitForVolumeDetachedFromNode( - client, volumeID, sspod.Spec.NodeName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(isDiskDetached).To(gomega.BeTrue(), - fmt.Sprintf("Volume %q is not detached from the node %q", - volumeID, sspod.Spec.NodeName)) - } - } - } - } - - // After scale down, verify the attached volumes match those in CNS Cache - for _, sspod := range ssPodsAfterScaleDown.Items { - _, err := client.CoreV1().Pods(namespace).Get(ctx, sspod.Name, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - for _, volumespec := range sspod.Spec.Volumes { - if volumespec.PersistentVolumeClaim != nil { - pv := getPvFromClaim(client, statefulset.Namespace, volumespec.PersistentVolumeClaim.ClaimName) - volumeID := pv.Spec.CSI.VolumeHandle - if guestCluster { - volumeID = getVolumeIDFromSupervisorCluster(pv.Spec.CSI.VolumeHandle) - } - queryResult, err := e2eVSphere.queryCNSVolumeWithResult(volumeID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(len(queryResult.Volumes) > 0).To(gomega.BeTrue()) - } - } - } - - ginkgo.By(fmt.Sprintf("Scaling up statefulsets to number of Replica: %v", replicas)) - _, scaleupErr := fss.Scale(client, statefulset, replicas) - gomega.Expect(scaleupErr).NotTo(gomega.HaveOccurred()) - fss.WaitForStatusReplicas(client, statefulset, replicas) - fss.WaitForStatusReadyReplicas(client, statefulset, replicas) - - ssPodsAfterScaleUp := fss.GetPodList(client, statefulset) - gomega.Expect(ssPodsAfterScaleUp.Items).NotTo(gomega.BeEmpty(), - fmt.Sprintf("Unable to get list of Pods from the Statefulset: %v", statefulset.Name)) - gomega.Expect(len(ssPodsAfterScaleUp.Items) == int(replicas)).To(gomega.BeTrue(), - "Number of Pods in the statefulset should match with number of replicas") - - // After scale up, verify all vSphere volumes are attached to node VMs. - ginkgo.By("Verify all volumes are attached to Nodes after Statefulsets is scaled up") - for _, sspod := range ssPodsAfterScaleUp.Items { - err := fpod.WaitTimeoutForPodReadyInNamespace(client, sspod.Name, statefulset.Namespace, pollTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pod, err := client.CoreV1().Pods(namespace).Get(ctx, sspod.Name, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - for _, volumespec := range pod.Spec.Volumes { - if volumespec.PersistentVolumeClaim != nil { - pv := getPvFromClaim(client, statefulset.Namespace, volumespec.PersistentVolumeClaim.ClaimName) - volumeID := pv.Spec.CSI.VolumeHandle - if guestCluster { - volumeID = getVolumeIDFromSupervisorCluster(pv.Spec.CSI.VolumeHandle) - } - ginkgo.By("Verify scale up operation should not introduced new volume") - gomega.Expect(contains(volumesBeforeScaleDown, volumeID)).To(gomega.BeTrue()) - ginkgo.By(fmt.Sprintf("Verify volume: %s is attached to the node: %s", - volumeID, sspod.Spec.NodeName)) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - var vmUUID string - if vanillaCluster { - vmUUID = getNodeUUID(ctx, client, pod.Spec.NodeName) - } else if guestCluster { - vmUUID, err = getVMUUIDFromNodeName(pod.Spec.NodeName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - isDiskAttached, err := e2eVSphere.isVolumeAttachedToVM(client, volumeID, vmUUID) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Disk is not attached to the node") - gomega.Expect(isDiskAttached).To(gomega.BeTrue(), "Disk is not attached") - ginkgo.By("After scale up, verify the attached volumes match those in CNS Cache") - queryResult, err := e2eVSphere.queryCNSVolumeWithResult(volumeID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(len(queryResult.Volumes) > 0).To(gomega.BeTrue()) - } - } - } - - replicas = 0 - ginkgo.By(fmt.Sprintf("Scaling down statefulsets to number of Replica: %v", replicas)) - _, scaledownErr = fss.Scale(client, statefulset, replicas) - gomega.Expect(scaledownErr).NotTo(gomega.HaveOccurred()) - fss.WaitForStatusReplicas(client, statefulset, replicas) - ssPodsAfterScaleDown = fss.GetPodList(client, statefulset) - gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(replicas)).To(gomega.BeTrue(), - "Number of Pods in the statefulset should match with number of replicas") - }) - - /* - Test dynamic volume provisioning with raw block volume - Steps - 1. Create a PVC. - 2. Create pod and wait for pod to become ready. - 3. Verify volume is attached. - 4. Write some test data to raw block device inside pod. - 5. Verify the data written on the volume correctly. - 6. Delete pod. - 7. Create a new pod using the previously created volume and wait for pod to - become ready. - 8. Verify previously written data using a read on volume. - 9. Write some new test data and verify it. - 10. Delete pod. - 11. Wait for volume to be detached. - */ - ginkgo.It("[csi-block-vanilla] [csi-guest] [csi-block-vanilla-parallelized] "+ - "Should create and delete pod with the same raw block volume", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - ginkgo.By("Creating Storage Class and PVC") - // Decide which test setup is available to run. - if vanillaCluster { - ginkgo.By("CNS_TEST: Running for vanilla k8s setup") - } else if guestCluster { - ginkgo.By("CNS_TEST: Running for GC setup") - scParameters[svStorageClassName] = storagePolicyName - } - sc, err := createStorageClass(client, scParameters, nil, "", "", false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err := client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Creating raw block PVC") - pvcspec := getPersistentVolumeClaimSpecWithStorageClass(namespace, "", sc, nil, "") - pvcspec.Spec.VolumeMode = &rawBlockVolumeMode - pvc, err = fpv.CreatePVC(client, namespace, pvcspec) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), fmt.Sprintf("Failed to create pvc with err: %v", err)) - - ginkgo.By(fmt.Sprintf("Waiting for claim %s to be in bound phase", pvc.Name)) - pvs, err := fpv.WaitForPVClaimBoundPhase(client, []*corev1.PersistentVolumeClaim{pvc}, - framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(pvs).NotTo(gomega.BeEmpty()) - pv := pvs[0] - volumeID := pv.Spec.CSI.VolumeHandle - if guestCluster { - // svcPVCName refers to PVC Name in the supervisor cluster. - svcPVCName = volumeID - volumeID = getVolumeIDFromSupervisorCluster(svcPVCName) - gomega.Expect(volumeID).NotTo(gomega.BeEmpty()) - } - defer func() { - err := fpv.DeletePersistentVolumeClaim(client, pvc.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fpv.WaitForPersistentVolumeDeleted(client, pv.Name, poll, pollTimeoutShort) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volumeID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Creating pod") - pod, err := createPod(client, namespace, nil, []*corev1.PersistentVolumeClaim{pvc}, false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By(fmt.Sprintf("Verify volume: %s is attached to the node: %s", - volumeID, pod.Spec.NodeName)) - var vmUUID string - if vanillaCluster { - vmUUID = getNodeUUID(ctx, client, pod.Spec.NodeName) - } else if guestCluster { - vmUUID, err = getVMUUIDFromNodeName(pod.Spec.NodeName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - verifyCRDInSupervisorWithWait(ctx, f, pod.Spec.NodeName+"-"+svcPVCName, - crdCNSNodeVMAttachment, crdVersion, crdGroup, true) - } - isDiskAttached, err := e2eVSphere.isVolumeAttachedToVM(client, volumeID, vmUUID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(isDiskAttached).To(gomega.BeTrue(), fmt.Sprintf("Volume is not attached to the node, %s", vmUUID)) - - // Write and read some data on raw block volume inside the pod. - // Use same devicePath for raw block volume here as used inside podSpec by createPod(). - // Refer setVolumes() for more information on naming of devicePath. - volumeIndex := 1 - devicePath := fmt.Sprintf("%v%v", pod_devicePathPrefix, volumeIndex) - rand.New(rand.NewSource(time.Now().Unix())) - testdataFile := fmt.Sprintf("/tmp/testdata_%v_%v", time.Now().Unix(), rand.Intn(1000)) - ginkgo.By(fmt.Sprintf("Creating a 1mb test data file %v", testdataFile)) - op, err := exec.Command("dd", "if=/dev/urandom", fmt.Sprintf("of=%v", testdataFile), - "bs=1M", "count=1").Output() - fmt.Println(op) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - op, err = exec.Command("rm", "-f", testdataFile).Output() - fmt.Println(op) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By(fmt.Sprintf("Write and read data on raw volume attached to: %v at path %v", pod.Name, - pod.Spec.Containers[0].VolumeDevices[0].DevicePath)) - verifyIOOnRawBlockVolume(namespace, pod.Name, devicePath, testdataFile, 0, 1) - - ginkgo.By("Deleting the pod") - err = fpod.DeletePodWithWait(client, pod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By("Verify volume is detached from the node") - isDiskDetached, err := e2eVSphere.waitForVolumeDetachedFromNode(client, - volumeID, pod.Spec.NodeName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(isDiskDetached).To(gomega.BeTrue(), - fmt.Sprintf("Volume %q is not detached from the node %q", volumeID, pod.Spec.NodeName)) - if guestCluster { - ginkgo.By("Waiting for CnsNodeVMAttachment controller to reconcile resource") - verifyCRDInSupervisorWithWait(ctx, f, pod.Spec.NodeName+"-"+svcPVCName, - crdCNSNodeVMAttachment, crdVersion, crdGroup, false) - } - - ginkgo.By("Creating a new pod using the same volume") - pod, err = createPod(client, namespace, nil, []*corev1.PersistentVolumeClaim{pvc}, false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By(fmt.Sprintf("Verify volume: %s is attached to the node: %s", - volumeID, pod.Spec.NodeName)) - - if vanillaCluster { - vmUUID = getNodeUUID(ctx, client, pod.Spec.NodeName) - } else if guestCluster { - vmUUID, err = getVMUUIDFromNodeName(pod.Spec.NodeName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - verifyCRDInSupervisorWithWait(ctx, f, pod.Spec.NodeName+"-"+svcPVCName, - crdCNSNodeVMAttachment, crdVersion, crdGroup, true) - } - isDiskAttached, err = e2eVSphere.isVolumeAttachedToVM(client, volumeID, vmUUID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(isDiskAttached).To(gomega.BeTrue(), "Volume is not attached to the node") - - // Verify previously written data. Later perform another write and verify it. - ginkgo.By(fmt.Sprintf("Verify previously written data on raw volume attached to: %v at path %v", pod.Name, - pod.Spec.Containers[0].VolumeDevices[0].DevicePath)) - verifyDataFromRawBlockVolume(namespace, pod.Name, devicePath, testdataFile, 0, 1) - ginkgo.By(fmt.Sprintf("Writing new 1mb test data in file %v", testdataFile)) - op, err = exec.Command("dd", "if=/dev/urandom", fmt.Sprintf("of=%v", testdataFile), - "bs=1M", "count=1").Output() - fmt.Println(op) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By(fmt.Sprintf("Write and read new data on raw volume attached to: %v at path %v", pod.Name, - pod.Spec.Containers[0].VolumeDevices[0].DevicePath)) - verifyIOOnRawBlockVolume(namespace, pod.Name, devicePath, testdataFile, 0, 1) - - ginkgo.By("Deleting the pod") - err = fpod.DeletePodWithWait(client, pod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Verify volume is detached from the node") - isDiskDetached, err = e2eVSphere.waitForVolumeDetachedFromNode(client, - volumeID, pod.Spec.NodeName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(isDiskDetached).To(gomega.BeTrue(), - fmt.Sprintf("Volume %q is not detached from the node %q", volumeID, pod.Spec.NodeName)) - if guestCluster { - ginkgo.By("Waiting for 30 seconds to allow CnsNodeVMAttachment controller to reconcile resource") - time.Sleep(waitTimeForCNSNodeVMAttachmentReconciler) - verifyCRDInSupervisorWithWait(ctx, f, pod.Spec.NodeName+"-"+svcPVCName, - crdCNSNodeVMAttachment, crdVersion, crdGroup, false) - } - }) - - /* - Test static volume provisioning with raw block volume - Steps: - 1. Create FCD and wait for fcd to allow syncing with pandora. - 2. Create PV Spec with volumeID set to FCDID created in Step-1, and - PersistentVolumeReclaimPolicy is set to Delete. - 3. Create PVC with the storage request set to PV's storage capacity. - 4. Wait for PV and PVC to bound. - 5. Create a POD. - 6. Verify volume is attached to the node and volume is accessible in the pod. - 7. Verify container volume metadata is present in CNS cache. - 8. Delete POD. - 9. Verify volume is detached from the node. - 10. Delete PVC. - 11. Verify PV is deleted automatically. - */ - ginkgo.It("[csi-block-vanilla] [csi-block-vanilla-parallelized] Verify basic static provisioning workflow"+ - " with raw block volume", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - ginkgo.By("Creating FCD Disk") - fcdID, err := e2eVSphere.createFCD(ctx, "BasicStaticFCD", diskSizeInMb, defaultDatastore.Reference()) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - deleteFCDRequired = true - - ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to allow newly created FCD:%s to sync with pandora", - pandoraSyncWaitTime, fcdID)) - time.Sleep(time.Duration(pandoraSyncWaitTime) * time.Second) - - // Creating label for PV. - // PVC will use this label as Selector to find PV. - staticPVLabels := make(map[string]string) - staticPVLabels["fcd-id"] = fcdID - - ginkgo.By("Creating raw block PV") - pv = getPersistentVolumeSpec(fcdID, corev1.PersistentVolumeReclaimDelete, staticPVLabels, "") - pv.Spec.VolumeMode = &rawBlockVolumeMode - pv, err = client.CoreV1().PersistentVolumes().Create(ctx, pv, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeCreated(pv.Spec.CSI.VolumeHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err := fpv.DeletePersistentVolume(client, pv.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Creating raw block PVC") - pvc = getPersistentVolumeClaimSpec(namespace, staticPVLabels, pv.Name) - pvc.Spec.VolumeMode = &rawBlockVolumeMode - pvc, err = client.CoreV1().PersistentVolumeClaims(namespace).Create(ctx, pvc, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - // Wait for PV and PVC to Bind. - framework.ExpectNoError(fpv.WaitOnPVandPVC(client, framework.NewTimeoutContextWithDefaults(), namespace, pv, pvc)) - defer func() { - err := fpv.DeletePersistentVolumeClaim(client, pvc.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fpv.WaitForPersistentVolumeDeleted(client, pv.Name, poll, pollTimeoutShort) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(pv.Spec.CSI.VolumeHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - // Set deleteFCDRequired to false. - // After PV, PVC is in the bind state, Deleting PVC should delete - // container volume. So no need to delete FCD directly using vSphere - // API call. - deleteFCDRequired = false - - ginkgo.By("Verifying CNS entry is present in cache") - _, err = e2eVSphere.queryCNSVolumeWithResult(pv.Spec.CSI.VolumeHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Creating the Pod") - var pvclaims []*corev1.PersistentVolumeClaim - pvclaims = append(pvclaims, pvc) - pod, err := createPod(client, namespace, nil, pvclaims, false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By(fmt.Sprintf("Verify volume: %s is attached to the node: %s", pv.Spec.CSI.VolumeHandle, pod.Spec.NodeName)) - vmUUID := getNodeUUID(ctx, client, pod.Spec.NodeName) - isDiskAttached, err := e2eVSphere.isVolumeAttachedToVM(client, pv.Spec.CSI.VolumeHandle, vmUUID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(isDiskAttached).To(gomega.BeTrue(), "Volume is not attached") - - // Write and read some data on raw block volume inside the pod. - // Use same devicePath for raw block volume here as used inside podSpec by createPod(). - // Refer setVolumes() for more information on naming of devicePath. - volumeIndex := 1 - devicePath := fmt.Sprintf("%v%v", pod_devicePathPrefix, volumeIndex) - rand.New(rand.NewSource(time.Now().Unix())) - testdataFile := fmt.Sprintf("/tmp/testdata_%v_%v", time.Now().Unix(), rand.Intn(1000)) - ginkgo.By(fmt.Sprintf("Creating a 1mb test data file %v", testdataFile)) - op, err := exec.Command("dd", "if=/dev/urandom", fmt.Sprintf("of=%v", testdataFile), - "bs=1M", "count=1").Output() - fmt.Println(op) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - op, err = exec.Command("rm", "-f", testdataFile).Output() - fmt.Println(op) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - ginkgo.By(fmt.Sprintf("Write and read data on raw volume attached to: %v at path %v", pod.Name, - pod.Spec.Containers[0].VolumeDevices[0].DevicePath)) - verifyIOOnRawBlockVolume(namespace, pod.Name, devicePath, testdataFile, 0, 1) - - ginkgo.By("Verify container volume metadata is present in CNS cache") - ginkgo.By(fmt.Sprintf("Invoking QueryCNSVolume with VolumeID: %s", pv.Spec.CSI.VolumeHandle)) - _, err = e2eVSphere.queryCNSVolumeWithResult(pv.Spec.CSI.VolumeHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - labels := []types.KeyValue{{Key: "fcd-id", Value: fcdID}} - ginkgo.By("Verify container volume metadata is matching the one in CNS cache") - err = verifyVolumeMetadataInCNS(&e2eVSphere, pv.Spec.CSI.VolumeHandle, - pvc.Name, pv.ObjectMeta.Name, pod.Name, labels...) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Deleting the Pod") - framework.ExpectNoError(fpod.DeletePodWithWait(client, pod), "Failed to delete pod", pod.Name) - - ginkgo.By(fmt.Sprintf("Verify volume %q is detached from the node: %s", pv.Spec.CSI.VolumeHandle, pod.Spec.NodeName)) - isDiskDetached, err := e2eVSphere.waitForVolumeDetachedFromNode(client, pv.Spec.CSI.VolumeHandle, pod.Spec.NodeName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(isDiskDetached).To(gomega.BeTrue(), "Volume is not detached from the node") - }) - - /* - Test online volume expansion on dynamic raw block volume - Steps: - 1. Create StorageClass with allowVolumeExpansion set to true. - 2. Create raw block PVC which uses the StorageClass created in step 1. - 3. Wait for PV to be provisioned. - 4. Wait for PVC's status to become Bound and note down the size - 5. Create a Pod using the above created PVC - 6. Modify PVC's size to trigger online volume expansion - 7. verify the PVC status will change to "FilesystemResizePending". Wait till the status is removed - 8. Verify the resized PVC by doing CNS query - 9. Make sure data is intact on the PV mounted on the pod - 10. Make sure file system has increased - - */ - ginkgo.It("[csi-block-vanilla] [csi-block-vanilla-parallelized] [csi-guest] "+ - "Verify online volume expansion on dynamic raw block volume", func() { - ginkgo.By("Invoking Test for online Volume Expansion on raw block volume") - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - ginkgo.By("Create StorageClass with allowVolumeExpansion set to true") - sharedVSANDatastoreURL := GetAndExpectStringEnvVar(envSharedDatastoreURL) - if vanillaCluster { - ginkgo.By("CNS_TEST: Running for vanilla k8s setup") - scParameters[scParamDatastoreURL] = sharedVSANDatastoreURL - } else if guestCluster { - ginkgo.By("CNS_TEST: Running for GC setup") - scParameters[svStorageClassName] = storagePolicyName - } - scParameters[scParamFsType] = ext4FSType - sc, err := createStorageClass(client, scParameters, nil, "", "", true, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err := client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Creating raw block PVC") - pvcspec := getPersistentVolumeClaimSpecWithStorageClass(namespace, "", sc, nil, "") - pvcspec.Spec.VolumeMode = &rawBlockVolumeMode - pvc, err = fpv.CreatePVC(client, namespace, pvcspec) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), fmt.Sprintf("Failed to create pvc with err: %v", err)) - - ginkgo.By(fmt.Sprintf("Waiting for claim %s to be in bound phase", pvc.Name)) - pvs, err := fpv.WaitForPVClaimBoundPhase(client, []*corev1.PersistentVolumeClaim{pvc}, - framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(pvs).NotTo(gomega.BeEmpty()) - pv = pvs[0] - volumeID := pv.Spec.CSI.VolumeHandle - if guestCluster { - // svcPVCName refers to PVC Name in the supervisor cluster. - svcPVCName = volumeID - volumeID = getVolumeIDFromSupervisorCluster(svcPVCName) - gomega.Expect(volumeID).NotTo(gomega.BeEmpty()) - } - defer func() { - err := fpv.DeletePersistentVolumeClaim(client, pvc.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fpv.WaitForPersistentVolumeDeleted(client, pv.Name, poll, pollTimeoutShort) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volumeID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Create Pod using the above raw block PVC") - var pvclaims []*corev1.PersistentVolumeClaim - pvclaims = append(pvclaims, pvc) - pod, err := createPod(client, namespace, nil, pvclaims, false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By(fmt.Sprintf("Verify volume: %s is attached to the node: %s", - volumeID, pod.Spec.NodeName)) - var vmUUID string - if vanillaCluster { - vmUUID = getNodeUUID(ctx, client, pod.Spec.NodeName) - } else if guestCluster { - vmUUID, err = getVMUUIDFromNodeName(pod.Spec.NodeName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - verifyCRDInSupervisorWithWait(ctx, f, pod.Spec.NodeName+"-"+svcPVCName, - crdCNSNodeVMAttachment, crdVersion, crdGroup, true) - } - isDiskAttached, err := e2eVSphere.isVolumeAttachedToVM(client, volumeID, vmUUID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(isDiskAttached).To(gomega.BeTrue(), fmt.Sprintf("Volume is not attached to the node, %s", vmUUID)) - - // Get the size of block device and verify if device is accessible by performing write and read. - // Write and read some data on raw block volume inside the pod. - // Use same devicePath for raw block volume here as used inside podSpec by createPod(). - // Refer setVolumes() for more information on naming of devicePath. - volumeIndex := 1 - devicePath := fmt.Sprintf("%v%v", pod_devicePathPrefix, volumeIndex) - ginkgo.By(fmt.Sprintf("Check size for block device at devicePath %v before expansion", devicePath)) - originalBlockDevSize, err := getBlockDevSizeInBytes(f, namespace, pod, devicePath) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - rand.New(rand.NewSource(time.Now().Unix())) - testdataFile := fmt.Sprintf("/tmp/testdata_%v_%v", time.Now().Unix(), rand.Intn(1000)) - ginkgo.By(fmt.Sprintf("Creating a 1mb test data file %v", testdataFile)) - op, err := exec.Command("dd", "if=/dev/urandom", fmt.Sprintf("of=%v", testdataFile), - "bs=1M", "count=1").Output() - fmt.Println(op) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - op, err = exec.Command("rm", "-f", testdataFile).Output() - fmt.Println(op) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - ginkgo.By(fmt.Sprintf("Write and read data on raw volume attached to: %v at path %v", pod.Name, - pod.Spec.Containers[0].VolumeDevices[0].DevicePath)) - verifyIOOnRawBlockVolume(namespace, pod.Name, devicePath, testdataFile, 0, 1) - - defer func() { - // Delete Pod. - ginkgo.By(fmt.Sprintf("Deleting the pod %s in namespace %s", pod.Name, namespace)) - err := fpod.DeletePodWithWait(client, pod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - isDiskDetached, err := e2eVSphere.waitForVolumeDetachedFromNode(client, - volumeID, pod.Spec.NodeName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(isDiskDetached).To(gomega.BeTrue(), - fmt.Sprintf("Volume %q is not detached from the node %q", volumeID, pod.Spec.NodeName)) - if guestCluster { - ginkgo.By("Waiting for 30 seconds to allow CnsNodeVMAttachment controller to reconcile resource") - time.Sleep(waitTimeForCNSNodeVMAttachmentReconciler) - verifyCRDInSupervisorWithWait(ctx, f, pod.Spec.NodeName+"-"+svcPVCName, - crdCNSNodeVMAttachment, crdVersion, crdGroup, false) - } - }() - - ginkgo.By("Increase PVC size and verify online volume resize") - increaseSizeOfPvcAttachedToPod(f, client, namespace, pvc, pod) - - ginkgo.By("Wait for block device size to be updated inside pod after expansion") - isPvcExpandedInsidePod := false - for !isPvcExpandedInsidePod { - blockDevSize, err := getBlockDevSizeInBytes(f, namespace, pod, devicePath) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - if blockDevSize > originalBlockDevSize { - ginkgo.By("Volume size updated inside pod successfully") - isPvcExpandedInsidePod = true - } else { - ginkgo.By(fmt.Sprintf("updating volume size for %q. Resulting volume size is %d", pvc.Name, blockDevSize)) - time.Sleep(30 * time.Second) - } - } - - // Verify original data on raw block volume after expansion - ginkgo.By(fmt.Sprintf("Verify previously written data on raw volume attached to: %v at path %v", pod.Name, - pod.Spec.Containers[0].VolumeDevices[0].DevicePath)) - verifyDataFromRawBlockVolume(namespace, pod.Name, devicePath, testdataFile, 0, 1) - // Write data on expanded space to verify the expansion is successful and accessible. - ginkgo.By(fmt.Sprintf("Writing new 1mb test data in file %v", testdataFile)) - op, err = exec.Command("dd", "if=/dev/urandom", fmt.Sprintf("of=%v", testdataFile), - "bs=1M", "count=1").Output() - fmt.Println(op) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By(fmt.Sprintf("Write testdata of 1MB size from offset=%v on raw volume at path %v inside pod %v", - volsizeInMiBBeforeExpansion, pod.Spec.Containers[0].VolumeDevices[0].DevicePath, pod.Name)) - verifyIOOnRawBlockVolume(namespace, pod.Name, devicePath, testdataFile, - volsizeInMiBBeforeExpansion, 1) - }) - - /* - Test to verify volume expansion is supported if allowVolumeExpansion - is true in StorageClass, PVC is created and offline and not attached - to a Pod before the expansion. - Steps - 1. Create StorageClass with allowVolumeExpansion set to true. - 2. Create raw block PVC which uses the StorageClass created in step 1. - 3. Wait for PV to be provisioned. - 4. Wait for PVC's status to become Bound. - 5. Create pod using PVC on specific node. - 6. Wait for Disk to be attached to the node. - 7. Write some data to raw block PVC. - 8. Detach the volume. - 9. Modify PVC's size to trigger offline volume expansion. - 10. Create pod again using PVC on specific node. - 11. Wait for Disk to be attached to the node. - 12. Verify data written on PVC before expansion. - 13. Delete pod and Wait for Volume Disk to be detached from the Node. - 14. Delete PVC, PV and Storage Class. - */ - ginkgo.It("[csi-block-vanilla] [csi-block-vanilla-parallelized] [csi-guest] "+ - "Verify offline volume expansion with raw block volume", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - ginkgo.By("Invoking Test for Offline Volume Expansion") - // Create Storage class and PVC - scParameters := make(map[string]string) - scParameters[scParamFsType] = ext4FSType - - // Create a StorageClass that sets allowVolumeExpansion to true - ginkgo.By("Creating Storage Class with allowVolumeExpansion = true") - if vanillaCluster { - ginkgo.By("CNS_TEST: Running for vanilla k8s setup") - } else if guestCluster { - ginkgo.By("CNS_TEST: Running for GC setup") - scParameters[svStorageClassName] = storagePolicyName - } - sc, err := createStorageClass(client, scParameters, nil, "", "", true, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err := client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Creating raw block PVC") - pvcspec := getPersistentVolumeClaimSpecWithStorageClass(namespace, "", sc, nil, "") - pvcspec.Spec.VolumeMode = &rawBlockVolumeMode - pvc, err = fpv.CreatePVC(client, namespace, pvcspec) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), fmt.Sprintf("Failed to create pvc with err: %v", err)) - - // Waiting for PVC to be bound - var pvclaims []*corev1.PersistentVolumeClaim - pvclaims = append(pvclaims, pvc) - ginkgo.By("Waiting for all claims to be in bound state") - pvs, err := fpv.WaitForPVClaimBoundPhase(client, pvclaims, framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pv := pvs[0] - volumeID := pv.Spec.CSI.VolumeHandle - svcPVCName := pv.Spec.CSI.VolumeHandle - if guestCluster { - volumeID = getVolumeIDFromSupervisorCluster(volumeID) - gomega.Expect(volumeID).NotTo(gomega.BeEmpty()) - } - defer func() { - err := fpv.DeletePersistentVolumeClaim(client, pvc.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fpv.WaitForPersistentVolumeDeleted(client, pv.Name, poll, pollTimeoutShort) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volumeID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - // Create a Pod to use this PVC, and verify volume has been attached - ginkgo.By("Creating pod to attach PV to the node") - pod, err := createPod(client, namespace, nil, []*corev1.PersistentVolumeClaim{pvc}, false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - var vmUUID string - ginkgo.By(fmt.Sprintf("Verify volume: %s is attached to the node: %s", volumeID, pod.Spec.NodeName)) - if vanillaCluster { - vmUUID = getNodeUUID(ctx, client, pod.Spec.NodeName) - } else if guestCluster { - vmUUID, err = getVMUUIDFromNodeName(pod.Spec.NodeName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - isDiskAttached, err := e2eVSphere.isVolumeAttachedToVM(client, volumeID, vmUUID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(isDiskAttached).To(gomega.BeTrue(), "Volume is not attached to the node") - - // Get the size of block device and verify if device is accessible by performing write and read. - volumeIndex := 1 - devicePath := fmt.Sprintf("%v%v", pod_devicePathPrefix, volumeIndex) - ginkgo.By(fmt.Sprintf("Check size for block device at devicePath %v before expansion", devicePath)) - originalBlockDevSize, err := getBlockDevSizeInBytes(f, namespace, pod, devicePath) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - rand.New(rand.NewSource(time.Now().Unix())) - testdataFile := fmt.Sprintf("/tmp/testdata_%v_%v", time.Now().Unix(), rand.Intn(1000)) - ginkgo.By(fmt.Sprintf("Creating a 1mb test data file %v", testdataFile)) - op, err := exec.Command("dd", "if=/dev/urandom", fmt.Sprintf("of=%v", testdataFile), - "bs=1M", "count=1").Output() - fmt.Println(op) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - op, err = exec.Command("rm", "-f", testdataFile).Output() - fmt.Println(op) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - ginkgo.By(fmt.Sprintf("Write and read data on raw volume attached to: %v at path %v", pod.Name, - pod.Spec.Containers[0].VolumeDevices[0].DevicePath)) - verifyIOOnRawBlockVolume(namespace, pod.Name, devicePath, testdataFile, 0, 1) - - // Delete POD - ginkgo.By(fmt.Sprintf("Deleting the pod %s in namespace %s", pod.Name, namespace)) - err = fpod.DeletePodWithWait(client, pod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Verify volume is detached from the node") - isDiskDetached, err := e2eVSphere.waitForVolumeDetachedFromNode( - client, volumeID, pod.Spec.NodeName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(isDiskDetached).To(gomega.BeTrue(), - fmt.Sprintf("Volume %q is not detached from the node %q", volumeID, pod.Spec.NodeName)) - if guestCluster { - ginkgo.By("Waiting for 30 seconds to allow CnsNodeVMAttachment controller to reconcile resource") - time.Sleep(waitTimeForCNSNodeVMAttachmentReconciler) - verifyCRDInSupervisorWithWait(ctx, f, pod.Spec.NodeName+"-"+svcPVCName, - crdCNSNodeVMAttachment, crdVersion, crdGroup, false) - } - - // Modify PVC spec to trigger volume expansion - // We expand the PVC while no pod is using it to ensure offline expansion - ginkgo.By("Expanding current pvc") - currentPvcSize := pvc.Spec.Resources.Requests[corev1.ResourceStorage] - newSize := currentPvcSize.DeepCopy() - newSize.Add(resource.MustParse("1Gi")) - framework.Logf("currentPvcSize %v, newSize %v", currentPvcSize, newSize) - pvc, err = expandPVCSize(pvc, newSize, client) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(pvc).NotTo(gomega.BeNil()) - - pvcSize := pvc.Spec.Resources.Requests[corev1.ResourceStorage] - if pvcSize.Cmp(newSize) != 0 { - framework.Failf("error updating pvc size %q", pvc.Name) - } - if guestCluster { - ginkgo.By("Checking for PVC request size change on SVC PVC") - b, err := verifyPvcRequestedSizeUpdateInSupervisorWithWait(svcPVCName, newSize) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(b).To(gomega.BeTrue()) - } - - ginkgo.By("Waiting for controller volume resize to finish") - err = waitForPvResizeForGivenPvc(pvc, client, totalResizeWaitPeriod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - if guestCluster { - ginkgo.By("Checking for resize on SVC PV") - verifyPVSizeinSupervisor(svcPVCName, newSize) - } - - ginkgo.By(fmt.Sprintf("Invoking QueryCNSVolumeWithResult with VolumeID: %s", volumeID)) - queryResult, err := e2eVSphere.queryCNSVolumeWithResult(volumeID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - if len(queryResult.Volumes) == 0 { - err = fmt.Errorf("queryCNSVolumeWithResult returned no volume") - } - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By("Verifying disk size requested in volume expansion is honored") - newSizeInMb := int64(3072) - if queryResult.Volumes[0].BackingObjectDetails.(*cnstypes.CnsBlockBackingDetails).CapacityInMb != newSizeInMb { - err = fmt.Errorf("got wrong disk size after volume expansion") - } - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - // Create a new Pod to use this PVC, and verify volume has been attached - ginkgo.By("Creating a new pod to attach PV again to the node") - pod, err = createPod(client, namespace, nil, []*corev1.PersistentVolumeClaim{pvc}, false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By(fmt.Sprintf("Verify volume after expansion: %s is attached to the node: %s", - volumeID, pod.Spec.NodeName)) - if vanillaCluster { - vmUUID = getNodeUUID(ctx, client, pod.Spec.NodeName) - } else if guestCluster { - vmUUID, err = getVMUUIDFromNodeName(pod.Spec.NodeName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - isDiskAttached, err = e2eVSphere.isVolumeAttachedToVM(client, volumeID, vmUUID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(isDiskAttached).To(gomega.BeTrue(), "Volume is not attached to the node") - - pvcConditions := pvc.Status.Conditions - expectEqual(len(pvcConditions), 0, "pvc should not have conditions") - - ginkgo.By("Verify block device size inside pod after expansion") - blockDevSize, err := getBlockDevSizeInBytes(f, namespace, pod, devicePath) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - if blockDevSize <= originalBlockDevSize { - framework.Failf("error updating volume size for %q. Resulting volume size is %d", pvc.Name, blockDevSize) - } - ginkgo.By("Resized volume attached successfully") - - // Verify original data on raw block volume after expansion - ginkgo.By(fmt.Sprintf("Verify previously written data on raw volume attached to: %v at path %v", pod.Name, - pod.Spec.Containers[0].VolumeDevices[0].DevicePath)) - verifyDataFromRawBlockVolume(namespace, pod.Name, devicePath, testdataFile, 0, 1) - // Write data on expanded space to verify the expansion is successful and accessible. - ginkgo.By(fmt.Sprintf("Writing new 1mb test data in file %v", testdataFile)) - op, err = exec.Command("dd", "if=/dev/urandom", fmt.Sprintf("of=%v", testdataFile), - "bs=1M", "count=1").Output() - fmt.Println(op) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By(fmt.Sprintf("Write testdata of 1MB size from offset=%v on raw volume at path %v inside pod %v", - volsizeInMiBBeforeExpansion, pod.Spec.Containers[0].VolumeDevices[0].DevicePath, pod.Name)) - verifyIOOnRawBlockVolume(namespace, pod.Name, devicePath, testdataFile, - volsizeInMiBBeforeExpansion, 1) - - // Delete POD - ginkgo.By(fmt.Sprintf("Deleting the new pod %s in namespace %s after expansion", pod.Name, namespace)) - err = fpod.DeletePodWithWait(client, pod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Verify volume is detached from the node after expansion") - isDiskDetached, err = e2eVSphere.waitForVolumeDetachedFromNode(client, volumeID, pod.Spec.NodeName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(isDiskDetached).To(gomega.BeTrue(), - fmt.Sprintf("Volume %q is not detached from the node %q", volumeID, pod.Spec.NodeName)) - if guestCluster { - ginkgo.By("Waiting for 30 seconds to allow CnsNodeVMAttachment controller to reconcile resource") - time.Sleep(waitTimeForCNSNodeVMAttachmentReconciler) - verifyCRDInSupervisorWithWait(ctx, f, pod.Spec.NodeName+"-"+svcPVCName, - crdCNSNodeVMAttachment, crdVersion, crdGroup, false) - } - }) - - /* - Test snapshot restore operation with raw block volume - Steps: - 1. Create a storage class (eg: vsan default) and create a pvc using this sc - 2. Write some data on source volume - 3. Create a VolumeSnapshot class with snapshotter as vsphere-csi-driver and set deletionPolicy to Delete - 4. Create a volume-snapshot with labels, using the above snapshot-class and pvc (from step-1) as source - 5. Ensure the snapshot is created, verify using get VolumeSnapshot - 6. Also verify that VolumeSnapshotContent is auto-created - 7. Verify the references to pvc and volume-snapshot on this object - 8. Verify that the VolumeSnapshot has ready-to-use set to True - 9. Verify that the Restore Size set on the snapshot is same as that of the source volume size - 10. Query the snapshot from CNS side using volume id - should pass and return the snapshot entry - 11. Restore the snapshot to another pvc - 12. Verify previous data written on source volume is present on restored volume - 13. Delete the above snapshot from k8s side using kubectl delete, run a get and ensure it is removed - 14. Also ensure that the VolumeSnapshotContent is deleted along with the - volume snapshot as the policy is delete - 15. Query the snapshot from CNS side - should return 0 entries - 16. Cleanup: Delete PVC, SC (validate they are removed) - */ - ginkgo.It("[block-vanilla-snapshot] [tkg-snapshot] Verify snapshot dynamic provisioning workflow with "+ - "raw block volume", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - if vanillaCluster { - scParameters[scParamDatastoreURL] = datastoreURL - } else if guestCluster { - scParameters[svStorageClassName] = storagePolicyName - } - - ginkgo.By("Create storage class") - sc, err := createStorageClass(client, scParameters, nil, "", "", false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err := client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Creating source raw block PVC") - pvcspec := getPersistentVolumeClaimSpecWithStorageClass(namespace, "", sc, nil, "") - pvcspec.Spec.VolumeMode = &rawBlockVolumeMode - pvc1, err := fpv.CreatePVC(client, namespace, pvcspec) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), fmt.Sprintf("Failed to create pvc with err: %v", err)) - - ginkgo.By("Expect source volume claim to provision volume successfully") - pvs, err := fpv.WaitForPVClaimBoundPhase(client, []*corev1.PersistentVolumeClaim{pvc1}, - framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - volumeID := pvs[0].Spec.CSI.VolumeHandle - gomega.Expect(volumeID).NotTo(gomega.BeEmpty()) - if guestCluster { - volumeID = getVolumeIDFromSupervisorCluster(volumeID) - } - - defer func() { - err := fpv.DeletePersistentVolumeClaim(client, pvc1.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volumeID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - // Verify using CNS Query API if VolumeID retrieved from PV is present. - ginkgo.By(fmt.Sprintf("Invoking QueryCNSVolumeWithResult with VolumeID: %s", volumeID)) - queryResult, err := e2eVSphere.queryCNSVolumeWithResult(volumeID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(queryResult.Volumes).ShouldNot(gomega.BeEmpty()) - gomega.Expect(queryResult.Volumes[0].VolumeId.Id).To(gomega.Equal(volumeID)) - - ginkgo.By("Creating pod to attach source PV to the node") - pod1, err := createPod(client, namespace, nil, []*corev1.PersistentVolumeClaim{pvc1}, - false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - defer func() { - // Delete POD - ginkgo.By(fmt.Sprintf("Deleting the pod %s in namespace %s", pod1.Name, namespace)) - err = fpod.DeletePodWithWait(client, pod1) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - var vmUUID string - nodeName := pod1.Spec.NodeName - if vanillaCluster { - vmUUID = getNodeUUID(ctx, client, pod1.Spec.NodeName) - } else if guestCluster { - vmUUID, err = getVMUUIDFromNodeName(pod1.Spec.NodeName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - ginkgo.By(fmt.Sprintf("Verify volume: %s is attached to the node: %s", volumeID, nodeName)) - isDiskAttached, err := e2eVSphere.isVolumeAttachedToVM(client, volumeID, vmUUID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(isDiskAttached).To(gomega.BeTrue(), "Volume is not attached to the node") - - // Write and read some data on raw block volume inside the pod. - // Use same devicePath for raw block volume here as used inside podSpec by createPod(). - // Refer setVolumes() for more information on naming of devicePath. - volumeIndex := 1 - devicePath := fmt.Sprintf("%v%v", pod_devicePathPrefix, volumeIndex) - rand.New(rand.NewSource(time.Now().Unix())) - testdataFile := fmt.Sprintf("/tmp/testdata_%v_%v", time.Now().Unix(), rand.Intn(1000)) - ginkgo.By(fmt.Sprintf("Creating a 1mb test data file %v", testdataFile)) - op, err := exec.Command("dd", "if=/dev/urandom", fmt.Sprintf("of=%v", testdataFile), - "bs=1M", "count=1").Output() - fmt.Println(op) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - op, err = exec.Command("rm", "-f", testdataFile).Output() - fmt.Println(op) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - ginkgo.By(fmt.Sprintf("Write and read data on source raw volume attached to: %v at path %v", pod1.Name, - pod1.Spec.Containers[0].VolumeDevices[0].DevicePath)) - verifyIOOnRawBlockVolume(namespace, pod1.Name, devicePath, testdataFile, 0, 1) - - ginkgo.By("Create volume snapshot class") - volumeSnapshotClass, err := createVolumeSnapshotClass(ctx, snapc, deletionPolicy) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - if vanillaCluster { - err = snapc.SnapshotV1().VolumeSnapshotClasses().Delete(ctx, volumeSnapshotClass.Name, - metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }() - - ginkgo.By("Create a dynamic volume snapshot") - volumeSnapshot, snapshotContent, snapshotCreated, - snapshotContentCreated, snapshotId, err := createDynamicVolumeSnapshot(ctx, namespace, snapc, volumeSnapshotClass, - pvc1, volumeID, diskSize) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - if snapshotContentCreated { - framework.Logf("Deleting volume snapshot content") - err = deleteVolumeSnapshotContent(ctx, snapshotContent, snapc, namespace, pandoraSyncWaitTime) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - if snapshotCreated { - framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeletedWithPandoraWait(ctx, snapc, - *volumeSnapshot.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }() - - // Restore volumeSnapshot to another PVC - ginkgo.By("Restore volume snapshot to another raw block PVC") - restorePvcSpec := getPersistentVolumeClaimSpecWithDatasource(namespace, diskSize, sc, nil, - corev1.ReadWriteOnce, volumeSnapshot.Name, snapshotapigroup) - restorePvcSpec.Spec.VolumeMode = &rawBlockVolumeMode - restoredPvc, err := fpv.CreatePVC(client, namespace, restorePvcSpec) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - restoredPvs, err := fpv.WaitForPVClaimBoundPhase(client, - []*corev1.PersistentVolumeClaim{restoredPvc}, - framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - volumeID2 := restoredPvs[0].Spec.CSI.VolumeHandle - gomega.Expect(volumeID2).NotTo(gomega.BeEmpty()) - if guestCluster { - volumeID2 = getVolumeIDFromSupervisorCluster(volumeID2) - } - defer func() { - ginkgo.By("Deleting the restored PVC") - err := fpv.DeletePersistentVolumeClaim(client, restoredPvc.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By("Wait for the restored PVC to disappear in CNS") - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volumeID2) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - // Create a Pod to use this PVC, and verify volume has been attached - ginkgo.By("Creating pod to attach restored PVC to the node") - pod2, err := createPod(client, namespace, nil, []*corev1.PersistentVolumeClaim{restoredPvc}, false, - "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - nodeName = pod2.Spec.NodeName - if vanillaCluster { - vmUUID = getNodeUUID(ctx, client, pod2.Spec.NodeName) - } else if guestCluster { - vmUUID, err = getVMUUIDFromNodeName(pod2.Spec.NodeName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - ginkgo.By(fmt.Sprintf("Verify volume: %s is attached to the node: %s", volumeID2, nodeName)) - isDiskAttached, err = e2eVSphere.isVolumeAttachedToVM(client, volumeID2, vmUUID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(isDiskAttached).To(gomega.BeTrue(), "Volume is not attached to the node") - defer func() { - // Delete POD - ginkgo.By(fmt.Sprintf("Deleting the pod %s in namespace %s", pod2.Name, namespace)) - err = fpod.DeletePodWithWait(client, pod2) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By(fmt.Sprintf("Verify volume: %s is attached to the node: %s", volumeID2, nodeName)) - isDiskDetached, err := e2eVSphere.waitForVolumeDetachedFromNode(client, volumeID2, nodeName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(isDiskDetached).To(gomega.BeTrue(), - fmt.Sprintf("Volume %q is not detached from the node %q", volumeID2, nodeName)) - if guestCluster { - ginkgo.By("Waiting for 30 seconds to allow CnsNodeVMAttachment controller to reconcile resource") - time.Sleep(waitTimeForCNSNodeVMAttachmentReconciler) - verifyCRDInSupervisorWithWait(ctx, f, pod2.Spec.NodeName+"-"+svcPVCName, - crdCNSNodeVMAttachment, crdVersion, crdGroup, false) - } - }() - - // Verify previously written data. Later perform another write and verify it. - ginkgo.By(fmt.Sprintf("Verify previously written data on restored volume attached to: %v at path %v", pod2.Name, - pod2.Spec.Containers[0].VolumeDevices[0].DevicePath)) - verifyDataFromRawBlockVolume(namespace, pod2.Name, devicePath, testdataFile, 0, 1) - ginkgo.By(fmt.Sprintf("Writing new 1mb test data in file %v", testdataFile)) - op, err = exec.Command("dd", "if=/dev/urandom", fmt.Sprintf("of=%v", testdataFile), - "bs=1M", "count=1").Output() - fmt.Println(op) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By(fmt.Sprintf("Write and read data on restored volume attached to: %v at path %v", pod2.Name, - pod2.Spec.Containers[0].VolumeDevices[0].DevicePath)) - verifyIOOnRawBlockVolume(namespace, pod2.Name, devicePath, testdataFile, 0, 1) - - ginkgo.By("Delete dyanmic volume snapshot") - snapshotCreated, snapshotContentCreated, err = deleteVolumeSnapshot(ctx, snapc, namespace, - volumeSnapshot, pandoraSyncWaitTime, volumeID, snapshotId) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) -}) diff --git a/tests/e2e/staging_env_basic.go b/tests/e2e/staging_env_basic.go index b2bf68f20d..75cc15f911 100644 --- a/tests/e2e/staging_env_basic.go +++ b/tests/e2e/staging_env_basic.go @@ -190,7 +190,7 @@ var _ = ginkgo.Describe("[csi-supervisor-staging] Tests for WCP env with minimal statefulset.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort = port statefulset.Spec.Template.Spec.Containers[0].Ports[0].Name = "web" + val + strconv.Itoa(min) statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &storagePolicyName + Annotations["volume.beta.kubernetes.io/storage-class"] = storagePolicyName CreateStatefulSet(namespace, statefulset, client) replicas := *(statefulset.Spec.Replicas) @@ -423,7 +423,7 @@ var _ = ginkgo.Describe("[csi-supervisor-staging] Tests for WCP env with minimal statefulset.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort = port statefulset.Spec.Template.Spec.Containers[0].Ports[0].Name = "web" + val + strconv.Itoa(min) statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &storagePolicyName + Annotations["volume.beta.kubernetes.io/storage-class"] = storagePolicyName CreateStatefulSet(namespace, statefulset, client) replicas := *(statefulset.Spec.Replicas) @@ -662,7 +662,7 @@ var _ = ginkgo.Describe("[csi-supervisor-staging] Tests for WCP env with minimal statefulset.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort = port statefulset.Spec.Template.Spec.Containers[0].Ports[0].Name = "web" + val + strconv.Itoa(min) statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &storagePolicyName + Annotations["volume.beta.kubernetes.io/storage-class"] = storagePolicyName CreateStatefulSet(namespace, statefulset, client) replicas := *(statefulset.Spec.Replicas) @@ -733,7 +733,7 @@ var _ = ginkgo.Describe("[csi-supervisor-staging] Tests for WCP env with minimal statefulset = GetResizedStatefulSetFromManifest(namespace) ginkgo.By("Creating statefulset") statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &storagePolicyName + Annotations["volume.beta.kubernetes.io/storage-class"] = storagePolicyName CreateStatefulSet(namespace, statefulset, client) replicas = *(statefulset.Spec.Replicas) diff --git a/tests/e2e/statefulset_with_topology.go b/tests/e2e/statefulset_with_topology.go index df4f6bfb07..d233bf92a5 100644 --- a/tests/e2e/statefulset_with_topology.go +++ b/tests/e2e/statefulset_with_topology.go @@ -157,12 +157,6 @@ var _ = ginkgo.Describe("[csi-topology-vanilla] Topology-Aware-Provisioning-With gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() - ginkgo.By("Creating service") - service := CreateService(namespace, client) - defer func() { - deleteService(namespace, client, service) - }() - // Creating statefulset with 3 replicas ginkgo.By("Creating statefulset with 3 replica") statefulset := GetStatefulSetFromManifest(namespace) diff --git a/tests/e2e/statefulset_xfs.go b/tests/e2e/statefulset_xfs.go index a0614609fe..457b7a3fb7 100644 --- a/tests/e2e/statefulset_xfs.go +++ b/tests/e2e/statefulset_xfs.go @@ -94,7 +94,7 @@ var _ = ginkgo.Describe("[csi-block-vanilla] [csi-block-vanilla-parallelized] st ginkgo.By("Creating statefulset") statefulset := GetStatefulSetFromManifest(namespace) statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &storageClassName + Annotations["volume.beta.kubernetes.io/storage-class"] = storageClassName CreateStatefulSet(namespace, statefulset, client) replicas := *(statefulset.Spec.Replicas) // Waiting for pods status to be Ready diff --git a/tests/e2e/statefulsets.go b/tests/e2e/statefulsets.go index 37e3aa61e3..58ef0eb432 100644 --- a/tests/e2e/statefulsets.go +++ b/tests/e2e/statefulsets.go @@ -64,11 +64,13 @@ var _ = ginkgo.Describe("statefulset", func() { f := framework.NewDefaultFramework("e2e-vsphere-statefulset") f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged var ( - namespace string - client clientset.Interface - storagePolicyName string - scParameters map[string]string - storageClassName string + namespace string + client clientset.Interface + storagePolicyName string + scParameters map[string]string + storageClassName string + sshClientConfig *ssh.ClientConfig + nimbusGeneratedK8sVmPwd string ) ginkgo.BeforeEach(func() { @@ -82,6 +84,15 @@ var _ = ginkgo.Describe("statefulset", func() { gomega.Expect(client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0))).NotTo(gomega.HaveOccurred()) } + nimbusGeneratedK8sVmPwd = GetAndExpectStringEnvVar(nimbusK8sVmPwd) + + sshClientConfig = &ssh.ClientConfig{ + User: "root", + Auth: []ssh.AuthMethod{ + ssh.Password(nimbusGeneratedK8sVmPwd), + }, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + } scParameters = make(map[string]string) storagePolicyName = GetAndExpectStringEnvVar(envStoragePolicyNameForSharedDatastores) @@ -137,7 +148,7 @@ var _ = ginkgo.Describe("statefulset", func() { statefulset := GetStatefulSetFromManifest(namespace) ginkgo.By("Creating statefulset") statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &storageClassName + Annotations["volume.beta.kubernetes.io/storage-class"] = storageClassName CreateStatefulSet(namespace, statefulset, client) replicas := *(statefulset.Spec.Replicas) // Waiting for pods status to be Ready @@ -335,7 +346,7 @@ var _ = ginkgo.Describe("statefulset", func() { *(statefulset.Spec.Replicas) = 8 statefulset.Spec.PodManagementPolicy = apps.ParallelPodManagement statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &storageClassName + Annotations["volume.beta.kubernetes.io/storage-class"] = storageClassName ginkgo.By("Creating statefulset") CreateStatefulSet(namespace, statefulset, client) replicas := *(statefulset.Spec.Replicas) @@ -533,7 +544,7 @@ var _ = ginkgo.Describe("statefulset", func() { statefulset := GetStatefulSetFromManifest(namespace) ginkgo.By("Creating statefulset") statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &storageClassName + Annotations["volume.beta.kubernetes.io/storage-class"] = storageClassName CreateStatefulSet(namespace, statefulset, client) replicas := *(statefulset.Spec.Replicas) // Waiting for pods status to be Ready @@ -592,7 +603,7 @@ var _ = ginkgo.Describe("statefulset", func() { statefulset = GetResizedStatefulSetFromManifest(namespace) ginkgo.By("Creating statefulset") statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &storageClassName + Annotations["volume.beta.kubernetes.io/storage-class"] = storageClassName CreateStatefulSet(namespace, statefulset, client) replicas = *(statefulset.Spec.Replicas) @@ -648,7 +659,7 @@ var _ = ginkgo.Describe("statefulset", func() { }) /* - Verify List volume Response on vsphere-csi-controller logs + Verify List volume Response on vsphere-ccsi-controller logs Note: ist volume Threshold is set to 1 , and query limit set to 3 1. Create SC 2. Create statefull set with 3 replica @@ -665,12 +676,10 @@ var _ = ginkgo.Describe("statefulset", func() { 12. Inncrease the CSI driver replica to 3 */ - ginkgo.It("[csi-block-vanilla] [csi-supervisor] ListVolumeResponse Validation", func() { + ginkgo.It("[csi-block-vanilla] ListVolumeResponse Validation", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - var svcMasterPswd string var volumesBeforeScaleUp []string - var sshClientConfig *ssh.ClientConfig containerName := "vsphere-csi-controller" ginkgo.By("Creating StorageClass for Statefulset") // decide which test setup is available to run @@ -679,13 +688,12 @@ var _ = ginkgo.Describe("statefulset", func() { scParameters = nil storageClassName = "nginx-sc-default" } else { - storageClassName = defaultNginxStorageClassName - ginkgo.By("Running for WCP setup") - + ginkgo.By("CNS_TEST: Running for WCP setup") profileID := e2eVSphere.GetSpbmPolicyID(storagePolicyName) scParameters[scParamStoragePolicyID] = profileID + storageClassName = defaultNginxStorageClassName // create resource quota - createResourceQuota(client, namespace, rqLimit, storageClassName) + createResourceQuota(client, namespace, rqLimit, defaultNginxStorageClassName) } ginkgo.By("scale down CSI driver POD to 1 , so that it will" + @@ -717,7 +725,7 @@ var _ = ginkgo.Describe("statefulset", func() { statefulset := GetStatefulSetFromManifest(namespace) ginkgo.By("Creating statefulset") statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &storageClassName + Annotations["volume.beta.kubernetes.io/storage-class"] = storageClassName CreateStatefulSet(namespace, statefulset, client) replicas := *(statefulset.Spec.Replicas) // Waiting for pods status to be Ready @@ -749,29 +757,7 @@ var _ = ginkgo.Describe("statefulset", func() { time.Sleep(pollTimeoutShort) ginkgo.By("Validate ListVolume Response for all the volumes") - var logMessage string - if vanillaCluster { - logMessage = "List volume response: entries:" - nimbusGeneratedK8sVmPwd := GetAndExpectStringEnvVar(nimbusK8sVmPwd) - sshClientConfig = &ssh.ClientConfig{ - User: "root", - Auth: []ssh.AuthMethod{ - ssh.Password(nimbusGeneratedK8sVmPwd), - }, - HostKeyCallback: ssh.InsecureIgnoreHostKey(), - } - } - if supervisorCluster { - logMessage = "ListVolumes:" - svcMasterPswd = GetAndExpectStringEnvVar(svcMasterPassword) - sshClientConfig = &ssh.ClientConfig{ - User: "root", - Auth: []ssh.AuthMethod{ - ssh.Password(svcMasterPswd), - }, - HostKeyCallback: ssh.InsecureIgnoreHostKey(), - } - } + logMessage := "List volume response: entries:" _, _, err = getCSIPodWhereListVolumeResponseIsPresent(ctx, client, sshClientConfig, containerName, logMessage, volumesBeforeScaleUp) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -794,19 +780,17 @@ var _ = ginkgo.Describe("statefulset", func() { _, _, err = getCSIPodWhereListVolumeResponseIsPresent(ctx, client, sshClientConfig, containerName, logMessage, nil) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - if vanillaCluster { - ginkgo.By("Delete volume from CNS and verify the error message") - logMessage = "difference between number of K8s volumes and CNS volumes is greater than threshold" - _, err = e2eVSphere.deleteCNSvolume(volumesBeforeScaleUp[0], false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - _, err = e2eVSphere.deleteCNSvolume(volumesBeforeScaleUp[1], false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - //List volume responses will show up in the interval of every 1 minute. - //To see the error, It is required to wait for 1 min after deleteting few Volumes - time.Sleep(pollTimeoutShort) - _, _, err = getCSIPodWhereListVolumeResponseIsPresent(ctx, client, sshClientConfig, containerName, logMessage, nil) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } + ginkgo.By("Delete volume from CNS and verify the error message") + logMessage = "difference between number of K8s volumes and CNS volumes is greater than threshold" + _, err = e2eVSphere.deleteCNSvolume(volumesBeforeScaleUp[0], false) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + _, err = e2eVSphere.deleteCNSvolume(volumesBeforeScaleUp[1], false) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + //List volume responses will show up in the interval of every 1 minute. + //To see the error, It is required to wait for 1 min after deleteting few Volumes + time.Sleep(pollTimeoutShort) + _, _, err = getCSIPodWhereListVolumeResponseIsPresent(ctx, client, sshClientConfig, containerName, logMessage, nil) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) replicas = 0 ginkgo.By(fmt.Sprintf("Scaling down statefulsets to number of Replica: %v", replicas)) diff --git a/tests/e2e/storage_policy_utils.go b/tests/e2e/storage_policy_utils.go index 76cb675022..27904a8ae0 100644 --- a/tests/e2e/storage_policy_utils.go +++ b/tests/e2e/storage_policy_utils.go @@ -24,11 +24,10 @@ import ( "strings" "time" - "github.com/davecgh/go-spew/spew" "github.com/onsi/gomega" "github.com/vmware/govmomi/object" "github.com/vmware/govmomi/pbm" - pbmtypes "github.com/vmware/govmomi/pbm/types" + "github.com/vmware/govmomi/pbm/types" "github.com/vmware/govmomi/vapi/tags" vim25types "github.com/vmware/govmomi/vim25/types" "k8s.io/kubernetes/test/e2e/framework" @@ -36,7 +35,7 @@ import ( // createVmfsStoragePolicy create a vmfs policy with given allocation type and category/tag map func createVmfsStoragePolicy(ctx context.Context, pbmClient *pbm.Client, allocationType string, - categoryTagMap map[string]string) (*pbmtypes.PbmProfileId, string) { + categoryTagMap map[string]string) (*types.PbmProfileId, string) { s1 := rand.NewSource(time.Now().UnixNano()) r1 := rand.New(s1) profileName := fmt.Sprintf("vmfs-policy-%v-%v", time.Now().UnixNano(), strconv.Itoa(r1.Intn(1000))) @@ -84,8 +83,8 @@ func createVmfsStoragePolicy(ctx context.Context, pbmClient *pbm.Client, allocat } // deleteStoragePolicy deletes the given storage policy -func deleteStoragePolicy(ctx context.Context, pbmClient *pbm.Client, profileID *pbmtypes.PbmProfileId) { - _, err := pbmClient.DeleteProfile(ctx, []pbmtypes.PbmProfileId{*profileID}) +func deleteStoragePolicy(ctx context.Context, pbmClient *pbm.Client, profileID *types.PbmProfileId) { + _, err := pbmClient.DeleteProfile(ctx, []types.PbmProfileId{*profileID}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } @@ -152,7 +151,7 @@ func getDsMoRefFromURL(ctx context.Context, dsURL string) vim25types.ManagedObje // createTagBasedPolicy creates a tag based storage policy with given tag and category map func createTagBasedPolicy(ctx context.Context, pbmClient *pbm.Client, - categoryTagMap map[string]string) (*pbmtypes.PbmProfileId, string) { + categoryTagMap map[string]string) (*types.PbmProfileId, string) { s1 := rand.NewSource(time.Now().UnixNano()) r1 := rand.New(s1) profileName := fmt.Sprintf("shared-ds-policy-%v-%v", time.Now().UnixNano(), strconv.Itoa(r1.Intn(1000))) @@ -187,237 +186,3 @@ func createTagBasedPolicy(ctx context.Context, pbmClient *pbm.Client, return profileID, profileName } - -// updateVmfsPolicyAlloctype updates the given policy's allocation type to given type -func updateVmfsPolicyAlloctype( - ctx context.Context, pbmClient *pbm.Client, allocationType string, policyName string, - policyId *pbmtypes.PbmProfileId) error { - - updateSpec := pbmtypes.PbmCapabilityProfileUpdateSpec{ - Name: policyName, - Constraints: &pbmtypes.PbmCapabilitySubProfileConstraints{ - SubProfiles: []pbmtypes.PbmCapabilitySubProfile{ - { - Capability: []pbmtypes.PbmCapabilityInstance{ - { - Id: pbmtypes.PbmCapabilityMetadataUniqueId{ - Id: "VolumeAllocationType", - Namespace: "com.vmware.storage.volumeallocation", - }, - Constraint: []pbmtypes.PbmCapabilityConstraintInstance{ - { - PropertyInstance: []pbmtypes.PbmCapabilityPropertyInstance{ - { - Id: "VolumeAllocationType", - Value: allocationType, - }, - }, - }, - }, - }, - }, - Name: "volumeallocation.capabilityobjectschema.namespaceInfo.info.label rules", - }, - }, - }, - } - err := pbmClient.UpdateProfile(ctx, *policyId, updateSpec) - if err != nil { - return err - } - policyContent, err := pbmClient.RetrieveContent(ctx, []pbmtypes.PbmProfileId{*policyId}) - if err != nil { - return err - } - framework.Logf("policy content after update", spew.Sdump(policyContent)) - return nil -} - -// createVsanDStoragePolicy create a vsand storage policy with given volume allocation type and category/tag map -func createVsanDStoragePolicy(ctx context.Context, pbmClient *pbm.Client, allocationType string, - categoryTagMap map[string]string) (*pbmtypes.PbmProfileId, string) { - s1 := rand.NewSource(time.Now().UnixNano()) - r1 := rand.New(s1) - profileName := fmt.Sprintf("vsand-policy-%v-%v", time.Now().UnixNano(), strconv.Itoa(r1.Intn(1000))) - pbmCreateSpec := pbm.CapabilityProfileCreateSpec{ - Name: profileName, - Description: "VSAND test policy", - Category: "REQUIREMENT", - CapabilityList: []pbm.Capability{ - { - ID: "vSANDirectVolumeAllocation", - Namespace: "vSANDirect", - PropertyList: []pbm.Property{ - { - ID: "vSANDirectVolumeAllocation", - Value: allocationType, - DataType: "string", - }, - }, - }, - { - ID: "vSANDirectType", - Namespace: "vSANDirect", - PropertyList: []pbm.Property{ - { - ID: "vSANDirectType", - Value: "vSANDirect", - DataType: "string", - }, - }, - }, - }, - } - for k, v := range categoryTagMap { - - pbmCreateSpec.CapabilityList = append(pbmCreateSpec.CapabilityList, pbm.Capability{ - ID: k, - Namespace: "http://www.vmware.com/storage/tag", - PropertyList: []pbm.Property{ - { - ID: "com.vmware.storage.tag." + k + ".property", - Value: v, - DataType: "set", - }, - }, - }) - } - createSpecVSAND, err := pbm.CreateCapabilityProfileSpec(pbmCreateSpec) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - profileID, err := pbmClient.CreateProfile(ctx, *createSpecVSAND) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - framework.Logf("VSAND profile with id: %v and name: '%v' created", profileID.UniqueId, profileName) - - return profileID, profileName -} - -// createStoragePolicyWithSharedVmfsNVsand create a storage policy with vmfs and vsand rules -// with given volume allocation type and category/tag map -func createStoragePolicyWithSharedVmfsNVsand(ctx context.Context, pbmClient *pbm.Client, - allocationType string, categoryTagMap map[string]string) (*pbmtypes.PbmProfileId, string) { - s1 := rand.NewSource(time.Now().UnixNano()) - r1 := rand.New(s1) - var category, tag string - profileName := fmt.Sprintf("storage-policy-%v-%v", time.Now().UnixNano(), strconv.Itoa(r1.Intn(1000))) - - for k, v := range categoryTagMap { - category = k - tag = v - } - framework.Logf("category: %v, tag: %v", category, tag) - - pbmCapabilityProfileSpec := pbmtypes.PbmCapabilityProfileCreateSpec{ - Name: profileName, - Description: "VSAND-VMFS test policy", - Category: "REQUIREMENT", - ResourceType: pbmtypes.PbmProfileResourceType{ - ResourceType: string(pbmtypes.PbmProfileResourceTypeEnumSTORAGE), - }, - Constraints: &pbmtypes.PbmCapabilitySubProfileConstraints{ - SubProfiles: []pbmtypes.PbmCapabilitySubProfile{ - { - Capability: []pbmtypes.PbmCapabilityInstance{ - { - Id: pbmtypes.PbmCapabilityMetadataUniqueId{ - Id: "vSANDirectVolumeAllocation", - Namespace: "vSANDirect", - }, - Constraint: []pbmtypes.PbmCapabilityConstraintInstance{ - { - PropertyInstance: []pbmtypes.PbmCapabilityPropertyInstance{ - { - Id: "vSANDirectVolumeAllocation", - Value: allocationType, - }, - }, - }, - }, - }, - { - Id: pbmtypes.PbmCapabilityMetadataUniqueId{ - Id: "vSANDirectType", - Namespace: "vSANDirect", - }, - Constraint: []pbmtypes.PbmCapabilityConstraintInstance{ - { - PropertyInstance: []pbmtypes.PbmCapabilityPropertyInstance{ - { - Id: "vSANDirectType", - Value: "vSANDirect", - }, - }, - }, - }, - }, - { - Id: pbmtypes.PbmCapabilityMetadataUniqueId{ - Id: category, - Namespace: "http://www.vmware.com/storage/tag", - }, - Constraint: []pbmtypes.PbmCapabilityConstraintInstance{ - { - PropertyInstance: []pbmtypes.PbmCapabilityPropertyInstance{ - { - Id: "com.vmware.storage.tag." + category + ".property", - Value: pbmtypes.PbmCapabilityDiscreteSet{ - Values: []vim25types.AnyType{tag}, - }, - }, - }, - }, - }, - }, - }, - Name: "vsandirect rules", - }, - { - Capability: []pbmtypes.PbmCapabilityInstance{ - { - Id: pbmtypes.PbmCapabilityMetadataUniqueId{ - Id: "VolumeAllocationType", - Namespace: "com.vmware.storage.volumeallocation", - }, - Constraint: []pbmtypes.PbmCapabilityConstraintInstance{ - { - PropertyInstance: []pbmtypes.PbmCapabilityPropertyInstance{ - { - Id: "VolumeAllocationType", - Value: allocationType, - }, - }, - }, - }, - }, - { - Id: pbmtypes.PbmCapabilityMetadataUniqueId{ - Id: category, - Namespace: "http://www.vmware.com/storage/tag", - }, - Constraint: []pbmtypes.PbmCapabilityConstraintInstance{ - { - PropertyInstance: []pbmtypes.PbmCapabilityPropertyInstance{ - { - Id: "com.vmware.storage.tag." + category + ".property", - Value: pbmtypes.PbmCapabilityDiscreteSet{ - Values: []vim25types.AnyType{tag}, - }, - }, - }, - }, - }, - }, - }, - Name: "vmfs rules", - }, - }, - }, - } - profileID, err := pbmClient.CreateProfile(ctx, pbmCapabilityProfileSpec) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - framework.Logf("VSAND and vmfs profile with id: %v and name: '%v' created", profileID.UniqueId, profileName) - - return profileID, profileName -} diff --git a/tests/e2e/storagepolicy.go b/tests/e2e/storagepolicy.go index 1347e86484..65c976e82e 100644 --- a/tests/e2e/storagepolicy.go +++ b/tests/e2e/storagepolicy.go @@ -78,12 +78,10 @@ var _ = ginkgo.Describe("[csi-block-vanilla] [csi-block-vanilla-parallelized] "+ ginkgo.AfterEach(func() { if supervisorCluster { deleteResourceQuota(client, namespace) - dumpSvcNsEventsOnTestFailure(client, namespace) } if guestCluster { svcClient, svNamespace := getSvcClientAndNamespace() setResourceQuota(svcClient, svNamespace, defaultrqLimit) - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) } }) diff --git a/tests/e2e/svmotion_detached_volume.go b/tests/e2e/svmotion_detached_volume.go new file mode 100644 index 0000000000..fb90ce8822 --- /dev/null +++ b/tests/e2e/svmotion_detached_volume.go @@ -0,0 +1,444 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + "math/rand" + "os" + "strings" + "time" + + ginkgo "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + "github.com/vmware/govmomi/find" + "github.com/vmware/govmomi/object" + pbmtypes "github.com/vmware/govmomi/pbm/types" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" + fnodes "k8s.io/kubernetes/test/e2e/framework/node" + fpod "k8s.io/kubernetes/test/e2e/framework/pod" + fpv "k8s.io/kubernetes/test/e2e/framework/pv" + admissionapi "k8s.io/pod-security-admission/api" +) + +/* + Test to verify sVmotion works fine for volumes in detached state + + Steps + 1. Create StorageClass. + 2. Create PVC. + 3. Expect PVC to pass and verified it is created correctly. + 4. Relocate detached volume + 5. Invoke CNS Query API and validate datastore URL value changed correctly +*/ + +var _ = ginkgo.Describe("[csi-block-vanilla] [csi-block-vanilla-parallelized] Relocate detached volume ", func() { + f := framework.NewDefaultFramework("svmotion-detached-disk") + f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged + var ( + client clientset.Interface + namespace string + scParameters map[string]string + datastoreURL string + sourceDatastore *object.Datastore + destDatastore *object.Datastore + datacenter *object.Datacenter + destDsURL string + pvclaims []*v1.PersistentVolumeClaim + fcdID string + ) + ginkgo.BeforeEach(func() { + bootstrap() + client = f.ClientSet + namespace = f.Namespace.Name + scParameters = make(map[string]string) + datastoreURL = GetAndExpectStringEnvVar(envSharedDatastoreURL) + destDsURL = GetAndExpectStringEnvVar(destinationDatastoreURL) + nodeList, err := fnodes.GetReadySchedulableNodes(f.ClientSet) + framework.ExpectNoError(err, "Unable to find ready and schedulable Node") + if !(len(nodeList.Items) > 0) { + framework.Failf("Unable to find ready and schedulable Node") + } + }) + + // Test for relocating volume being detached state + ginkgo.It("Verify relocating detached volume works fine", func() { + ginkgo.By("Invoking Test for relocating detached volume") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + finder := find.NewFinder(e2eVSphere.Client.Client, false) + cfg, err := getConfig() + var datacenters []string + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + dcList := strings.Split(cfg.Global.Datacenters, + ",") + for _, dc := range dcList { + dcName := strings.TrimSpace(dc) + if dcName != "" { + datacenters = append(datacenters, dcName) + } + } + + for _, dc := range datacenters { + datacenter, err = finder.Datacenter(ctx, dc) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + finder.SetDatacenter(datacenter) + sourceDatastore, err = getDatastoreByURL(ctx, datastoreURL, datacenter) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + destDatastore, err = getDatastoreByURL(ctx, destDsURL, datacenter) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } + + scParameters[scParamDatastoreURL] = datastoreURL + storageclass, pvclaim, err := createPVCAndStorageClass(client, + namespace, nil, scParameters, "", nil, "", false, "") + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + defer func() { + err = client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }() + + ginkgo.By("Expect claim to provision volume successfully") + err = fpv.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, + pvclaim.Namespace, pvclaim.Name, framework.Poll, time.Minute) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to provision volume") + + pvclaims = append(pvclaims, pvclaim) + + persistentvolumes, err := fpv.WaitForPVClaimBoundPhase(client, pvclaims, framework.ClaimProvisionTimeout) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + fcdID = persistentvolumes[0].Spec.CSI.VolumeHandle + + ginkgo.By(fmt.Sprintf("Invoking QueryCNSVolumeWithResult with VolumeID: %s", + persistentvolumes[0].Spec.CSI.VolumeHandle)) + queryResult, err := e2eVSphere.queryCNSVolumeWithResult(persistentvolumes[0].Spec.CSI.VolumeHandle) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + if len(queryResult.Volumes) == 0 { + err = fmt.Errorf("error: QueryCNSVolumeWithResult returned no volume") + } + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + ginkgo.By("Verifying disk is created on the specified datastore") + if queryResult.Volumes[0].DatastoreUrl != datastoreURL { + err = fmt.Errorf("disk is created on the wrong datastore") + } + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // Relocate volume + err = e2eVSphere.relocateFCD(ctx, fcdID, sourceDatastore.Reference(), destDatastore.Reference()) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to finish FCD relocation:%s to sync with pandora", + defaultPandoraSyncWaitTime, fcdID)) + time.Sleep(time.Duration(defaultPandoraSyncWaitTime) * time.Second) + + // verify disk is relocated to the specified destination datastore + ginkgo.By(fmt.Sprintf("Invoking QueryCNSVolumeWithResult with VolumeID: %s after relocating the disk", + persistentvolumes[0].Spec.CSI.VolumeHandle)) + queryResult, err = e2eVSphere.queryCNSVolumeWithResult(persistentvolumes[0].Spec.CSI.VolumeHandle) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + if len(queryResult.Volumes) == 0 { + err = fmt.Errorf("error: QueryCNSVolumeWithResult returned no volume") + } + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + ginkgo.By("Verifying disk is relocated to the specified datastore") + if queryResult.Volumes[0].DatastoreUrl != destDsURL { + err = fmt.Errorf("disk is relocated on the wrong datastore") + } + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + /* Online relocation of volume using cnsRelocate Volume API + STEPS: + 1. Create a tag based policy with 2 shared vmfs datastores(sharedVmfs-0 and sharedVmfs-1). + 2. Create SC with storage policy created in step 1. + 3. Create a PVC with sc created in step 2 and wait for it to come to bound state. + 4. Create a pod with the pvc created in step 3 and wait for it to come to Running state. + 5. Relocate volume from one shared datastore to another datastore using + CnsRelocateVolume API and verify the datastore of fcd after migration and volume compliance. + 6. Delete pod,pvc and sc. + */ + ginkgo.It("Online relocation of volume using cnsRelocate Volume API", func() { + ginkgo.By("Invoking Test for offline relocation") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sharedvmfsURL := os.Getenv(envSharedVMFSDatastoreURL) + if sharedvmfsURL == "" { + ginkgo.Skip(fmt.Sprintf("Env %v is missing", envSharedVMFSDatastoreURL)) + } + + sharedvmfs2URL := os.Getenv(envSharedVMFSDatastore2URL) + if sharedvmfs2URL == "" { + ginkgo.Skip(fmt.Sprintf("Env %v is missing", envSharedVMFSDatastore2URL)) + } + datastoreUrls := []string{sharedvmfsURL, sharedvmfs2URL} + + govmomiClient := newClient(ctx, &e2eVSphere) + pc := newPbmClient(ctx, govmomiClient) + scParameters := make(map[string]string) + pvcs := []*v1.PersistentVolumeClaim{} + + ginkgo.By("Creating tag and category to tag datastore") + + rand.New(rand.NewSource(time.Now().UnixNano())) + suffix := fmt.Sprintf("-%v-%v", time.Now().UnixNano(), rand.Intn(10000)) + categoryName := "category" + suffix + tagName := "tag" + suffix + + catID, tagID := createCategoryNTag(ctx, categoryName, tagName) + defer func() { + deleteCategoryNTag(ctx, catID, tagID) + }() + + ginkgo.By("Attaching tag to shared vmfs datastores") + + attachTagToDS(ctx, tagID, sharedvmfsURL) + defer func() { + detachTagFromDS(ctx, tagID, sharedvmfsURL) + }() + + attachTagToDS(ctx, tagID, sharedvmfs2URL) + defer func() { + detachTagFromDS(ctx, tagID, sharedvmfs2URL) + }() + + policyID, policyName := createTagBasedPolicy( + ctx, pc, map[string]string{categoryName: tagName}) + defer func() { + deleteStoragePolicy(ctx, pc, policyID) + }() + scParameters[scParamStoragePolicyName] = policyName + storageclass, pvclaim, err := createPVCAndStorageClass(client, + namespace, nil, scParameters, "", nil, "", false, "") + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + pvcs = append(pvcs, pvclaim) + + defer func() { + ginkgo.By("Delete the SCs created") + err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Deleted the SPBM polices created") + _, err = pc.DeleteProfile(ctx, []pbmtypes.PbmProfileId{*policyID}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }() + + ginkgo.By("Verify the PVCs created in step 3 are bound") + pvs, err := fpv.WaitForPVClaimBoundPhase(client, pvcs, framework.ClaimProvisionTimeout) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + volumeID := pvs[0].Spec.CSI.VolumeHandle + + defer func() { + err := fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = e2eVSphere.waitForCNSVolumeToBeDeleted(pvs[0].Spec.CSI.VolumeHandle) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }() + + ginkgo.By("Verify that the created CNS volumes are compliant and have correct policy id") + storagePolicyExists, err := e2eVSphere.VerifySpbmPolicyOfVolume(volumeID, policyName) + e2eVSphere.verifyVolumeCompliance(volumeID, true) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(storagePolicyExists).To(gomega.BeTrue(), "storage policy verification failed") + + ginkgo.By("Creating a pod") + pod, err := createPod(client, namespace, nil, pvcs, false, "") + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + filePath := "/mnt/volume1/file1.txt" + filePath2 := "/mnt/volume1/file2.txt" + + //Write data on file.txt on Pod + data := "This file is written by Pod" + ginkgo.By("write to a file in pod") + writeDataOnFileFromPod(namespace, pod.Name, filePath, data) + + defer func() { + ginkgo.By("Delete the pod created") + err := fpod.DeletePodWithWait(client, pod) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }() + + ginkgo.By("Verify if VolumeID is created on the given datastores") + dsUrlWhereVolumeIsPresent := fetchDsUrl4CnsVol(e2eVSphere, volumeID) + framework.Logf("Volume is present on %s", dsUrlWhereVolumeIsPresent) + e2eVSphere.verifyDatastoreMatch(volumeID, datastoreUrls) + + // Get the destination ds url where the volume will get relocated + destDsUrl := "" + for _, dsurl := range datastoreUrls { + if dsurl != dsUrlWhereVolumeIsPresent { + destDsUrl = dsurl + } + } + + ginkgo.By("Relocate volume from one shared datastore to another datastore using" + + "CnsRelocateVolume API") + dsRefDest := getDsMoRefFromURL(ctx, destDsUrl) + err = e2eVSphere.cnsRelocateVolume(e2eVSphere, ctx, volumeID, dsRefDest) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + e2eVSphere.verifyDatastoreMatch(volumeID, []string{destDsUrl}) + + ginkgo.By("Verify that the relocated CNS volumes are compliant and have correct policy id") + storagePolicyExists, err = e2eVSphere.VerifySpbmPolicyOfVolume(volumeID, policyName) + e2eVSphere.verifyVolumeCompliance(volumeID, true) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(storagePolicyExists).To(gomega.BeTrue(), "storage policy verification failed") + + //Read file1.txt created from Pod + ginkgo.By("Read file.txt from Pod created by Pod") + output := readFileFromPod(namespace, pod.Name, filePath) + ginkgo.By(fmt.Sprintf("File contents from file.txt are: %s", output)) + data = data + "\n" + gomega.Expect(output == data).To(gomega.BeTrue(), "data verification failed after relocation") + + data = "Writing some data to pod post relocation" + ginkgo.By("writing to a file in pod post relocation") + writeDataOnFileFromPod(namespace, pod.Name, filePath2, data) + + ginkgo.By("Read file.txt created by Pod") + output = readFileFromPod(namespace, pod.Name, filePath2) + ginkgo.By(fmt.Sprintf("File contents from file.txt are: %s", output)) + data = data + "\n" + gomega.Expect(output == data).To(gomega.BeTrue(), "data verification failed after relocation") + + }) + + /* Offline relocation of volume using cnsRelocate Volume API + STEPS: + 1. Create a tag based policy with 2 shared vmfs datastores(sharedVmfs-0 and sharedVmfs-1). + 2. Create SC with storage policy created in step 1. + 3. Create a PVC with sc created in step 2 and wait for it to come to bound state. + 4. Relocate volume from one shared datastore to another datastore + using CnsRelocateVolume API and verify the datastore of fcd after migration and volume compliance. + 5. Delete pvc and sc. + */ + ginkgo.It("Offline relocation of volume using cnsRelocate Volume API", func() { + ginkgo.By("Invoking Test for offline relocation") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sharedvmfsURL := os.Getenv(envSharedVMFSDatastoreURL) + if sharedvmfsURL == "" { + ginkgo.Skip(fmt.Sprintf("Env %v is missing", envSharedVMFSDatastoreURL)) + } + + sharedvmfs2URL := os.Getenv(envSharedVMFSDatastore2URL) + if sharedvmfs2URL == "" { + ginkgo.Skip(fmt.Sprintf("Env %v is missing", envSharedVMFSDatastore2URL)) + } + datastoreUrls := []string{sharedvmfsURL, sharedvmfs2URL} + + govmomiClient := newClient(ctx, &e2eVSphere) + pc := newPbmClient(ctx, govmomiClient) + scParameters := make(map[string]string) + pvcs := []*v1.PersistentVolumeClaim{} + + rand.New(rand.NewSource(time.Now().UnixNano())) + suffix := fmt.Sprintf("-%v-%v", time.Now().UnixNano(), rand.Intn(10000)) + categoryName := "category" + suffix + tagName := "tag" + suffix + + ginkgo.By("Creating tag and category to tag datastore") + + catID, tagID := createCategoryNTag(ctx, categoryName, tagName) + defer func() { + deleteCategoryNTag(ctx, catID, tagID) + }() + + ginkgo.By("Attaching tag to shared vmfs datastores") + + attachTagToDS(ctx, tagID, sharedvmfsURL) + defer func() { + detachTagFromDS(ctx, tagID, sharedvmfsURL) + }() + + attachTagToDS(ctx, tagID, sharedvmfs2URL) + defer func() { + detachTagFromDS(ctx, tagID, sharedvmfs2URL) + }() + + policyID, policyName := createTagBasedPolicy( + ctx, pc, map[string]string{categoryName: tagName}) + defer func() { + deleteStoragePolicy(ctx, pc, policyID) + }() + scParameters[scParamStoragePolicyName] = policyName + storageclass, pvclaim, err := createPVCAndStorageClass(client, + namespace, nil, scParameters, "", nil, "", false, "") + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + pvcs = append(pvcs, pvclaim) + + defer func() { + ginkgo.By("Delete the SCs created") + err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }() + + ginkgo.By("Verify the PVCs created in step 3 are bound") + pvs, err := fpv.WaitForPVClaimBoundPhase(client, pvcs, framework.ClaimProvisionTimeout) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + volumeID := pvs[0].Spec.CSI.VolumeHandle + + defer func() { + err := fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = e2eVSphere.waitForCNSVolumeToBeDeleted(pvs[0].Spec.CSI.VolumeHandle) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }() + + ginkgo.By("Verify that the created CNS volumes are compliant and have correct policy id") + storagePolicyExists, err := e2eVSphere.VerifySpbmPolicyOfVolume(volumeID, policyName) + e2eVSphere.verifyVolumeCompliance(volumeID, true) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(storagePolicyExists).To(gomega.BeTrue(), "storage policy verification failed") + + ginkgo.By("Verify if VolumeID is created on the given datastores") + dsUrlWhereVolumeIsPresent := fetchDsUrl4CnsVol(e2eVSphere, volumeID) + framework.Logf("Volume is present on %s", dsUrlWhereVolumeIsPresent) + e2eVSphere.verifyDatastoreMatch(volumeID, datastoreUrls) + + // Get the destination ds url where the volume will get relocated + destDsUrl := "" + for _, dsurl := range datastoreUrls { + if dsurl != dsUrlWhereVolumeIsPresent { + destDsUrl = dsurl + } + } + + dsRefDest := getDsMoRefFromURL(ctx, destDsUrl) + err = e2eVSphere.cnsRelocateVolume(e2eVSphere, ctx, volumeID, dsRefDest) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + e2eVSphere.verifyDatastoreMatch(volumeID, []string{destDsUrl}) + + ginkgo.By("Verify that the relocated CNS volumes are compliant and have correct policy id") + storagePolicyExists, err = e2eVSphere.VerifySpbmPolicyOfVolume(volumeID, policyName) + e2eVSphere.verifyVolumeCompliance(volumeID, true) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(storagePolicyExists).To(gomega.BeTrue(), "storage policy verification failed") + + }) +}) diff --git a/tests/e2e/svmotion_volumes.go b/tests/e2e/svmotion_volumes.go deleted file mode 100644 index 7526408fd5..0000000000 --- a/tests/e2e/svmotion_volumes.go +++ /dev/null @@ -1,1377 +0,0 @@ -/* -Copyright 2019-2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2e - -import ( - "context" - "fmt" - "math/rand" - "os" - "os/exec" - "strings" - "sync" - "time" - - snapV1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" - snapclient "github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned" - ginkgo "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" - cnstypes "github.com/vmware/govmomi/cns/types" - "github.com/vmware/govmomi/find" - "github.com/vmware/govmomi/object" - pbmtypes "github.com/vmware/govmomi/pbm/types" - v1 "k8s.io/api/core/v1" - storagev1 "k8s.io/api/storage/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/kubernetes/test/e2e/framework" - fnodes "k8s.io/kubernetes/test/e2e/framework/node" - fpod "k8s.io/kubernetes/test/e2e/framework/pod" - fpv "k8s.io/kubernetes/test/e2e/framework/pv" - admissionapi "k8s.io/pod-security-admission/api" -) - -/* - Test to verify sVmotion works fine for volumes in detached state - - Steps - 1. Create StorageClass. - 2. Create PVC. - 3. Expect PVC to pass and verified it is created correctly. - 4. Relocate detached volume - 5. Invoke CNS Query API and validate datastore URL value changed correctly -*/ - -var _ = ginkgo.Describe("[csi-block-vanilla] [csi-block-vanilla-parallelized] Relocate detached volume ", func() { - f := framework.NewDefaultFramework("svmotion-disk") - f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged - var ( - client clientset.Interface - namespace string - scParameters map[string]string - datastoreURL string - sourceDatastore *object.Datastore - destDatastore *object.Datastore - datacenter *object.Datacenter - destDsURL string - pvclaims []*v1.PersistentVolumeClaim - fcdID string - labelKey string - labelValue string - pvc10g string - ) - ginkgo.BeforeEach(func() { - bootstrap() - client = f.ClientSet - namespace = f.Namespace.Name - scParameters = make(map[string]string) - datastoreURL = GetAndExpectStringEnvVar(envSharedDatastoreURL) - destDsURL = GetAndExpectStringEnvVar(destinationDatastoreURL) - nodeList, err := fnodes.GetReadySchedulableNodes(f.ClientSet) - framework.ExpectNoError(err, "Unable to find ready and schedulable Node") - if !(len(nodeList.Items) > 0) { - framework.Failf("Unable to find ready and schedulable Node") - } - labelKey = "app" - labelValue = "e2e-labels" - pvc10g = "10Gi" - }) - - // Test for relocating volume being detached state - ginkgo.It("Verify relocating detached volume works fine", func() { - ginkgo.By("Invoking Test for relocating detached volume") - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - finder := find.NewFinder(e2eVSphere.Client.Client, false) - cfg, err := getConfig() - var datacenters []string - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - dcList := strings.Split(cfg.Global.Datacenters, - ",") - for _, dc := range dcList { - dcName := strings.TrimSpace(dc) - if dcName != "" { - datacenters = append(datacenters, dcName) - } - } - - for _, dc := range datacenters { - datacenter, err = finder.Datacenter(ctx, dc) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - finder.SetDatacenter(datacenter) - sourceDatastore, err = getDatastoreByURL(ctx, datastoreURL, datacenter) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - destDatastore, err = getDatastoreByURL(ctx, destDsURL, datacenter) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - ginkgo.By("Create storageclass and PVC from that storageclass") - scParameters[scParamDatastoreURL] = datastoreURL - storageclass, pvclaim, err := createPVCAndStorageClass(client, - namespace, nil, scParameters, "", nil, "", false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - defer func() { - ginkgo.By("Delete Storageclass and PVC") - err = client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Expect claim to provision volume successfully") - err = fpv.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, - pvclaim.Namespace, pvclaim.Name, framework.Poll, time.Minute) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to provision volume") - - pvclaims = append(pvclaims, pvclaim) - - persistentvolumes, err := fpv.WaitForPVClaimBoundPhase(client, pvclaims, framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - fcdID = persistentvolumes[0].Spec.CSI.VolumeHandle - - ginkgo.By(fmt.Sprintf("Invoking QueryCNSVolumeWithResult with VolumeID: %s", - persistentvolumes[0].Spec.CSI.VolumeHandle)) - queryResult, err := e2eVSphere.queryCNSVolumeWithResult(persistentvolumes[0].Spec.CSI.VolumeHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - if len(queryResult.Volumes) == 0 { - err = fmt.Errorf("error: QueryCNSVolumeWithResult returned no volume") - } - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By("Verifying disk is created on the specified datastore") - if queryResult.Volumes[0].DatastoreUrl != datastoreURL { - err = fmt.Errorf("disk is created on the wrong datastore") - } - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - // Relocate volume - ginkgo.By("Relocating FCD to different datastore") - err = e2eVSphere.relocateFCD(ctx, fcdID, sourceDatastore.Reference(), destDatastore.Reference()) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to finish FCD relocation:%s to sync with pandora", - defaultPandoraSyncWaitTime, fcdID)) - time.Sleep(time.Duration(defaultPandoraSyncWaitTime) * time.Second) - - // verify disk is relocated to the specified destination datastore - ginkgo.By(fmt.Sprintf("Invoking QueryCNSVolumeWithResult with VolumeID: %s after relocating the disk", - persistentvolumes[0].Spec.CSI.VolumeHandle)) - queryResult, err = e2eVSphere.queryCNSVolumeWithResult(persistentvolumes[0].Spec.CSI.VolumeHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - if len(queryResult.Volumes) == 0 { - err = fmt.Errorf("error: QueryCNSVolumeWithResult returned no volume") - } - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By("Verifying disk is relocated to the specified datastore") - if queryResult.Volumes[0].DatastoreUrl != destDsURL { - err = fmt.Errorf("disk is relocated on the wrong datastore") - } - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - - /* Online relocation of volume using cnsRelocate Volume API - STEPS: - 1. Create a tag based policy with 2 shared vmfs datastores(sharedVmfs-0 and sharedVmfs-1). - 2. Create SC with storage policy created in step 1. - 3. Create a PVC with sc created in step 2 and wait for it to come to bound state. - 4. Create a pod with the pvc created in step 3 and wait for it to come to Running state. - 5. Relocate volume from one shared datastore to another datastore using - CnsRelocateVolume API and verify the datastore of fcd after migration and volume compliance. - 6. Delete pod,pvc and sc. - */ - ginkgo.It("Online relocation of volume using cnsRelocate Volume API", func() { - ginkgo.By("Invoking Test for offline relocation") - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - sharedvmfsURL := os.Getenv(envSharedVMFSDatastoreURL) - if sharedvmfsURL == "" { - ginkgo.Skip(fmt.Sprintf("Env %v is missing", envSharedVMFSDatastoreURL)) - } - - sharedvmfs2URL := os.Getenv(envSharedVMFSDatastore2URL) - if sharedvmfs2URL == "" { - ginkgo.Skip(fmt.Sprintf("Env %v is missing", envSharedVMFSDatastore2URL)) - } - datastoreUrls := []string{sharedvmfsURL, sharedvmfs2URL} - - govmomiClient := newClient(ctx, &e2eVSphere) - pc := newPbmClient(ctx, govmomiClient) - scParameters := make(map[string]string) - pvcs := []*v1.PersistentVolumeClaim{} - - ginkgo.By("Creating tag and category to tag datastore") - - rand.New(rand.NewSource(time.Now().UnixNano())) - suffix := fmt.Sprintf("-%v-%v", time.Now().UnixNano(), rand.Intn(10000)) - categoryName := "category" + suffix - tagName := "tag" + suffix - - catID, tagID := createCategoryNTag(ctx, categoryName, tagName) - defer func() { - deleteCategoryNTag(ctx, catID, tagID) - }() - - ginkgo.By("Attaching tag to shared vmfs datastores") - - attachTagToDS(ctx, tagID, sharedvmfsURL) - defer func() { - detachTagFromDS(ctx, tagID, sharedvmfsURL) - }() - - attachTagToDS(ctx, tagID, sharedvmfs2URL) - defer func() { - detachTagFromDS(ctx, tagID, sharedvmfs2URL) - }() - - ginkgo.By("Create Tag Based policy with shared datstores") - policyID, policyName := createTagBasedPolicy( - ctx, pc, map[string]string{categoryName: tagName}) - defer func() { - deleteStoragePolicy(ctx, pc, policyID) - }() - - ginkgo.By("Create Storageclass and a PVC from storageclass created") - scParameters[scParamStoragePolicyName] = policyName - storageclass, pvclaim, err := createPVCAndStorageClass(client, - namespace, nil, scParameters, "", nil, "", false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pvcs = append(pvcs, pvclaim) - - defer func() { - ginkgo.By("Delete the SCs created") - err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Deleted the SPBM polices created") - _, err = pc.DeleteProfile(ctx, []pbmtypes.PbmProfileId{*policyID}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Verify the PVCs created in step 3 are bound") - pvs, err := fpv.WaitForPVClaimBoundPhase(client, pvcs, framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - volumeID := pvs[0].Spec.CSI.VolumeHandle - - defer func() { - ginkgo.By("Delete the PVCs created in test") - err := fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(pvs[0].Spec.CSI.VolumeHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Verify that the created CNS volumes are compliant and have correct policy id") - storagePolicyMatches, err := e2eVSphere.VerifySpbmPolicyOfVolume(volumeID, policyName) - e2eVSphere.verifyVolumeCompliance(volumeID, true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(storagePolicyMatches).To(gomega.BeTrue(), "storage policy verification failed") - - ginkgo.By("Creating a pod") - pod, err := createPod(client, namespace, nil, pvcs, false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - defer func() { - ginkgo.By("Delete the pod created") - err := fpod.DeletePodWithWait(client, pod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - filePath := "/mnt/volume1/file1.txt" - filePath2 := "/mnt/volume1/file2.txt" - - //Write data on file.txt on Pod - data := "This file is written by Pod" - ginkgo.By("write to a file in pod") - writeDataOnFileFromPod(namespace, pod.Name, filePath, data) - - ginkgo.By("Verify if VolumeID is created on the given datastores") - dsUrlWhereVolumeIsPresent := fetchDsUrl4CnsVol(e2eVSphere, volumeID) - framework.Logf("Volume: %s is present on %s", volumeID, dsUrlWhereVolumeIsPresent) - e2eVSphere.verifyDatastoreMatch(volumeID, datastoreUrls) - - // Get the destination ds url where the volume will get relocated - destDsUrl := "" - for _, dsurl := range datastoreUrls { - if dsurl != dsUrlWhereVolumeIsPresent { - destDsUrl = dsurl - break - } - } - - ginkgo.By("Relocate volume from one shared datastore to another datastore using" + - "CnsRelocateVolume API") - dsRefDest := getDsMoRefFromURL(ctx, destDsUrl) - _, err = e2eVSphere.cnsRelocateVolume(e2eVSphere, ctx, volumeID, dsRefDest) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - e2eVSphere.verifyDatastoreMatch(volumeID, []string{destDsUrl}) - - ginkgo.By("Verify that the relocated CNS volumes are compliant and have correct policy id") - storagePolicyMatches, err = e2eVSphere.VerifySpbmPolicyOfVolume(volumeID, policyName) - e2eVSphere.verifyVolumeCompliance(volumeID, true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(storagePolicyMatches).To(gomega.BeTrue(), "storage policy verification failed") - - //Read file1.txt created from Pod - ginkgo.By("Read file.txt from Pod created by Pod") - output := readFileFromPod(namespace, pod.Name, filePath) - ginkgo.By(fmt.Sprintf("File contents from file.txt are: %s", output)) - data = data + "\n" - gomega.Expect(output == data).To(gomega.BeTrue(), "data verification failed after relocation") - - data = "Writing some data to pod post relocation" - ginkgo.By("writing to a file in pod post relocation") - writeDataOnFileFromPod(namespace, pod.Name, filePath2, data) - - ginkgo.By("Read file.txt created by Pod") - output = readFileFromPod(namespace, pod.Name, filePath2) - ginkgo.By(fmt.Sprintf("File contents from file.txt are: %s", output)) - data = data + "\n" - gomega.Expect(output == data).To(gomega.BeTrue(), "data verification failed after relocation") - - }) - - /* Offline relocation of volume using cnsRelocate Volume API - STEPS: - 1. Create a tag based policy with 2 shared vmfs datastores(sharedVmfs-0 and sharedVmfs-1). - 2. Create SC with storage policy created in step 1. - 3. Create a PVC with sc created in step 2 and wait for it to come to bound state. - 4. Relocate volume from one shared datastore to another datastore - using CnsRelocateVolume API and verify the datastore of fcd after migration and volume compliance. - 5. Delete pvc and sc. - */ - ginkgo.It("Offline relocation of volume using cnsRelocate Volume API", func() { - ginkgo.By("Invoking Test for offline relocation") - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - sharedvmfsURL := os.Getenv(envSharedVMFSDatastoreURL) - if sharedvmfsURL == "" { - ginkgo.Skip(fmt.Sprintf("Env %v is missing", envSharedVMFSDatastoreURL)) - } - - sharedvmfs2URL := os.Getenv(envSharedVMFSDatastore2URL) - if sharedvmfs2URL == "" { - ginkgo.Skip(fmt.Sprintf("Env %v is missing", envSharedVMFSDatastore2URL)) - } - datastoreUrls := []string{sharedvmfsURL, sharedvmfs2URL} - - govmomiClient := newClient(ctx, &e2eVSphere) - pc := newPbmClient(ctx, govmomiClient) - scParameters := make(map[string]string) - pvcs := []*v1.PersistentVolumeClaim{} - - rand.New(rand.NewSource(time.Now().UnixNano())) - suffix := fmt.Sprintf("-%v-%v", time.Now().UnixNano(), rand.Intn(10000)) - categoryName := "category" + suffix - tagName := "tag" + suffix - - ginkgo.By("Creating tag and category to tag datastore") - - catID, tagID := createCategoryNTag(ctx, categoryName, tagName) - defer func() { - deleteCategoryNTag(ctx, catID, tagID) - }() - - ginkgo.By("Attaching tag to shared vmfs datastores") - - attachTagToDS(ctx, tagID, sharedvmfsURL) - defer func() { - detachTagFromDS(ctx, tagID, sharedvmfsURL) - }() - - attachTagToDS(ctx, tagID, sharedvmfs2URL) - defer func() { - detachTagFromDS(ctx, tagID, sharedvmfs2URL) - }() - - ginkgo.By("Create Tag Based policy with shared datstores") - policyID, policyName := createTagBasedPolicy( - ctx, pc, map[string]string{categoryName: tagName}) - defer func() { - deleteStoragePolicy(ctx, pc, policyID) - }() - - ginkgo.By("Create Storageclass and a PVC from storageclass created") - scParameters[scParamStoragePolicyName] = policyName - storageclass, pvclaim, err := createPVCAndStorageClass(client, - namespace, nil, scParameters, "", nil, "", false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pvcs = append(pvcs, pvclaim) - - defer func() { - ginkgo.By("Delete the SCs created") - err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Verify the PVCs created in step 3 are bound") - pvs, err := fpv.WaitForPVClaimBoundPhase(client, pvcs, framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - volumeID := pvs[0].Spec.CSI.VolumeHandle - - defer func() { - ginkgo.By("Delete the PVCs created in test") - err := fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(pvs[0].Spec.CSI.VolumeHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Verify that the created CNS volumes are compliant and have correct policy id") - storagePolicyMatches, err := e2eVSphere.VerifySpbmPolicyOfVolume(volumeID, policyName) - e2eVSphere.verifyVolumeCompliance(volumeID, true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(storagePolicyMatches).To(gomega.BeTrue(), "storage policy verification failed") - - ginkgo.By("Verify if VolumeID is created on the given datastores") - dsUrlWhereVolumeIsPresent := fetchDsUrl4CnsVol(e2eVSphere, volumeID) - framework.Logf("Volume: %s is present on %s", volumeID, dsUrlWhereVolumeIsPresent) - e2eVSphere.verifyDatastoreMatch(volumeID, datastoreUrls) - - // Get the destination ds url where the volume will get relocated - destDsUrl := "" - for _, dsurl := range datastoreUrls { - if dsurl != dsUrlWhereVolumeIsPresent { - destDsUrl = dsurl - break - } - } - - ginkgo.By("Relocate volume from one shared datastore to another datastore using" + - "CnsRelocateVolume API") - dsRefDest := getDsMoRefFromURL(ctx, destDsUrl) - _, err = e2eVSphere.cnsRelocateVolume(e2eVSphere, ctx, volumeID, dsRefDest) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - e2eVSphere.verifyDatastoreMatch(volumeID, []string{destDsUrl}) - - ginkgo.By("Verify that the relocated CNS volumes are compliant and have correct policy id") - storagePolicyMatches, err = e2eVSphere.VerifySpbmPolicyOfVolume(volumeID, policyName) - e2eVSphere.verifyVolumeCompliance(volumeID, true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(storagePolicyMatches).To(gomega.BeTrue(), "storage policy verification failed") - - }) - - /* - Start attached volume's relocation and then expand it - Steps: - 1. Create a SPBM policy with tag based rules. - 2. Create SC using policy created in step 1. - 3. Create PVC using SC created in step 2 and and wait for it to be bound. - 4. Create a pod with the pod created in step 3. - 5. Start writing some IO to pod which run in parallel to steps 6-7. - 6. Relocate CNS volume corresponding to pvc from step 3 to a different datastore. - 7. While relocation is running resize the volume. - 8. Verify the IO written so far. - 9. Verify relocation was successful. - 10. Verify online volume expansion is successful. - 11. Delete all the objects created during the test. - */ - ginkgo.It("[csi-block-vanilla][csi-block-vanilla-parallelized]"+ - " Start attached volume's relocation and then expand it", func() { - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - sharedvmfsURL, sharedVsanDatastoreURL := "", "" - var datastoreUrls []string - var policyName string - pc := newPbmClient(ctx, e2eVSphere.Client) - - sharedvmfsURL = os.Getenv(envSharedVMFSDatastoreURL) - if sharedvmfsURL == "" { - ginkgo.Skip(fmt.Sprintf("Env %v is missing", envSharedVMFSDatastoreURL)) - } - - sharedVsanDatastoreURL = os.Getenv(envSharedDatastoreURL) - if sharedVsanDatastoreURL == "" { - ginkgo.Skip(fmt.Sprintf("Env %v is missing", envSharedDatastoreURL)) - } - datastoreUrls = append(datastoreUrls, sharedvmfsURL, sharedVsanDatastoreURL) - - scParameters := make(map[string]string) - pvcs := []*v1.PersistentVolumeClaim{} - pvclaims2d := [][]*v1.PersistentVolumeClaim{} - - rand.New(rand.NewSource(time.Now().UnixNano())) - suffix := fmt.Sprintf("-%v-%v", time.Now().UnixNano(), rand.Intn(10000)) - categoryName := "category" + suffix - tagName := "tag" + suffix - - catID, tagID := createCategoryNTag(ctx, categoryName, tagName) - defer func() { - deleteCategoryNTag(ctx, catID, tagID) - }() - - attachTagToDS(ctx, tagID, sharedvmfsURL) - defer func() { - detachTagFromDS(ctx, tagID, sharedvmfsURL) - }() - - attachTagToDS(ctx, tagID, sharedVsanDatastoreURL) - defer func() { - detachTagFromDS(ctx, tagID, sharedVsanDatastoreURL) - }() - - ginkgo.By("create SPBM policy with tag based rules") - ginkgo.By("create a storage class with a SPBM policy created from step 1") - ginkgo.By("create a PVC each using the storage policy created from step 2") - var storageclass *storagev1.StorageClass - var pvclaim *v1.PersistentVolumeClaim - var err error - var policyID *pbmtypes.PbmProfileId - - policyID, policyName = createTagBasedPolicy( - ctx, pc, map[string]string{categoryName: tagName}) - defer func() { - deleteStoragePolicy(ctx, pc, policyID) - }() - - framework.Logf("CNS_TEST: Running for vanilla k8s setup") - scParameters[scParamStoragePolicyName] = policyName - storageclass, pvclaim, err = createPVCAndStorageClass(client, - namespace, nil, scParameters, pvc10g, nil, "", true, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - pvcs = append(pvcs, pvclaim) - pvclaims2d = append(pvclaims2d, []*v1.PersistentVolumeClaim{pvclaim}) - - defer func() { - ginkgo.By("Delete the SCs created in step 2") - err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - }() - - ginkgo.By("Verify the PVCs created in step 3 are bound") - pvs, err := fpv.WaitForPVClaimBoundPhase(client, pvcs, framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Verify that the created CNS volumes are compliant and have correct policy id") - volumeID := pvs[0].Spec.CSI.VolumeHandle - storagePolicyMatches, err := e2eVSphere.VerifySpbmPolicyOfVolume(volumeID, policyName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(storagePolicyMatches).To(gomega.BeTrue(), "storage policy verification failed") - e2eVSphere.verifyVolumeCompliance(volumeID, true) - - defer func() { - ginkgo.By("Delete the PVCs created in step 3") - err := fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volumeID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - }() - - ginkgo.By("Create pods with using the PVCs created in step 3 and wait for them to be ready") - ginkgo.By("verify we can read and write on the PVCs") - pods := createMultiplePods(ctx, client, pvclaims2d, true) - defer func() { - ginkgo.By("Delete the pod created") - deletePodsAndWaitForVolsToDetach(ctx, client, pods, true) - }() - - ginkgo.By("Verify if VolumeID is created on the given datastores") - dsUrlWhereVolumeIsPresent := fetchDsUrl4CnsVol(e2eVSphere, volumeID) - framework.Logf("Volume: %s is present on %s", volumeID, dsUrlWhereVolumeIsPresent) - e2eVSphere.verifyDatastoreMatch(volumeID, datastoreUrls) - - // Get the destination ds url where the volume will get relocated - destDsUrl := "" - for _, dsurl := range datastoreUrls { - if dsurl != dsUrlWhereVolumeIsPresent { - destDsUrl = dsurl - break - } - } - - framework.Logf("dest url: %s", destDsUrl) - dsRefDest := getDsMoRefFromURL(ctx, destDsUrl) - - rand.New(rand.NewSource(time.Now().Unix())) - testdataFile := fmt.Sprintf("/tmp/testdata_%v_%v", time.Now().Unix(), rand.Intn(1000)) - ginkgo.By(fmt.Sprintf("Creating a 100mb test data file %v", testdataFile)) - op, err := exec.Command("dd", "if=/dev/urandom", fmt.Sprintf("of=%v", testdataFile), - "bs=1M", "count=100").Output() - fmt.Println(op) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - op, err = exec.Command("rm", "-f", testdataFile).Output() - fmt.Println(op) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - currentPvcSize := pvcs[0].Spec.Resources.Requests[v1.ResourceStorage] - newSize := currentPvcSize.DeepCopy() - newSize.Add(resource.MustParse(diskSize)) - - originalSizeInMb, err := getFSSizeMb(f, pods[0]) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Start relocation of volume to a different datastore") - task, err := e2eVSphere.cnsRelocateVolume(e2eVSphere, ctx, volumeID, dsRefDest, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - framework.Logf("Waiting for a few seconds for relocation to be started properly on VC") - time.Sleep(time.Duration(10) * time.Second) - data := "This file is written by Pod" - - ginkgo.By("Resize volume and writing IO to pod while relocating volume") - var wg sync.WaitGroup - wg.Add(2) - go writeDataToMultipleFilesOnPodInParallel(namespace, pods[0].Name, data, &wg) - go resize(client, pvcs[0], pvcs[0].Spec.Resources.Requests[v1.ResourceStorage], newSize, &wg) - wg.Wait() - - ginkgo.By("Wait for relocation task to complete") - - cnsFault := waitForCNSTaskToComplete(ctx, task) - if cnsFault != nil { - err = fmt.Errorf("failed to relocate volume=%+v", cnsFault) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - ginkgo.By("Verify relocation of volume is successful") - e2eVSphere.verifyDatastoreMatch(volumeID, []string{destDsUrl}) - - ginkgo.By("Wait and verify the file system resize on pvcs") - framework.Logf("Waiting for file system resize to finish for pvc %v", pvcs[0].Name) - pvcs[0], err = waitForFSResize(pvcs[0], client) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - pvcConditions := pvcs[0].Status.Conditions - expectEqual(len(pvcConditions), 0, "pvc %v should not have status conditions", pvcs[0].Name) - - var fsSize int64 - framework.Logf("Verify filesystem size for mount point /mnt/volume1 for pod %v", pods[0].Name) - fsSize, err = getFSSizeMb(f, pods[0]) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - framework.Logf("File system size after expansion : %v", fsSize) - gomega.Expect(fsSize > originalSizeInMb).To(gomega.BeTrue(), - fmt.Sprintf( - "filesystem size %v is not > than before expansion %v for pvc %q", - fsSize, originalSizeInMb, pvcs[0].Name)) - - framework.Logf("File system resize finished successfully for pvc %v", pvcs[0].Name) - - ginkgo.By("Verify the data on the PVCs match what was written in step 7") - for i := 0; i < 10; i++ { - filePath := fmt.Sprintf("/mnt/volume1/file%v.txt", i) - output := readFileFromPod(namespace, pods[0].Name, filePath) - ginkgo.By(fmt.Sprintf("File contents from file%v.txt are: %s", i, output)) - dataToVerify := data + "\n" - gomega.Expect(output == dataToVerify).To(gomega.BeTrue(), "data verification failed after relocation") - } - - storagePolicyMatches, err = e2eVSphere.VerifySpbmPolicyOfVolume(volumeID, policyName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(storagePolicyMatches).To(gomega.BeTrue(), "storage policy verification failed") - e2eVSphere.verifyVolumeCompliance(volumeID, true) - - ginkgo.By("Delete the pod created") - deletePodsAndWaitForVolsToDetach(ctx, client, pods, true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - podsNew := createMultiplePods(ctx, client, pvclaims2d, true) - deletePodsAndWaitForVolsToDetach(ctx, client, podsNew, true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - }) - - /* - Start attached volume's expansion and then relocate it - Steps: - 1. Create a SPBM policy with tag based rules. - 2. Create SC using policy created in step 1. - 3. Create PVC using SC created in step 2 and and wait for it to be bound. - 4. Create a pod with the pod created in step 3. - 5. Start writing some IO to pod which run in parallel to steps 6-7. - 6. Resize the volume. - 7. While expansion is running relocate the volume allocation to different datastore. - 8. Verify the IO written so far. - 9. Verify relocation was successful. - 10. Verify online volume expansion is successful. - 11. Delete all the objects created during the test. - */ - ginkgo.It("[csi-block-vanilla][csi-block-vanilla-parallelized]"+ - " Start attached volume's expansion and then relocate it", func() { - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - sharedvmfsURL, sharedVsanDatastoreURL := "", "" - var datastoreUrls []string - var policyName string - pc := newPbmClient(ctx, e2eVSphere.Client) - - sharedvmfsURL = os.Getenv(envSharedVMFSDatastoreURL) - if sharedvmfsURL == "" { - ginkgo.Skip(fmt.Sprintf("Env %v is missing", envSharedVMFSDatastoreURL)) - } - - sharedVsanDatastoreURL = os.Getenv(envSharedDatastoreURL) - if sharedVsanDatastoreURL == "" { - ginkgo.Skip(fmt.Sprintf("Env %v is missing", envSharedDatastoreURL)) - } - datastoreUrls = append(datastoreUrls, sharedvmfsURL, sharedVsanDatastoreURL) - - scParameters := make(map[string]string) - pvcs := []*v1.PersistentVolumeClaim{} - pvclaims2d := [][]*v1.PersistentVolumeClaim{} - - rand.New(rand.NewSource(time.Now().UnixNano())) - suffix := fmt.Sprintf("-%v-%v", time.Now().UnixNano(), rand.Intn(10000)) - categoryName := "category" + suffix - tagName := "tag" + suffix - - catID, tagID := createCategoryNTag(ctx, categoryName, tagName) - defer func() { - deleteCategoryNTag(ctx, catID, tagID) - }() - - attachTagToDS(ctx, tagID, sharedvmfsURL) - defer func() { - detachTagFromDS(ctx, tagID, sharedvmfsURL) - }() - - attachTagToDS(ctx, tagID, sharedVsanDatastoreURL) - defer func() { - detachTagFromDS(ctx, tagID, sharedVsanDatastoreURL) - }() - - ginkgo.By("create SPBM policy with tag based rules") - ginkgo.By("create a storage class with a SPBM policy created from step 1") - ginkgo.By("create a PVC each using the storage policy created from step 2") - var storageclass *storagev1.StorageClass - var pvclaim *v1.PersistentVolumeClaim - var err error - var policyID *pbmtypes.PbmProfileId - - policyID, policyName = createTagBasedPolicy( - ctx, pc, map[string]string{categoryName: tagName}) - defer func() { - deleteStoragePolicy(ctx, pc, policyID) - }() - - framework.Logf("CNS_TEST: Running for vanilla k8s setup") - scParameters[scParamStoragePolicyName] = policyName - storageclass, pvclaim, err = createPVCAndStorageClass(client, - namespace, nil, scParameters, pvc10g, nil, "", true, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - pvcs = append(pvcs, pvclaim) - pvclaims2d = append(pvclaims2d, []*v1.PersistentVolumeClaim{pvclaim}) - - defer func() { - ginkgo.By("Delete the SCs created in step 2") - err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - }() - - ginkgo.By("Verify the PVCs created in step 3 are bound") - pvs, err := fpv.WaitForPVClaimBoundPhase(client, pvcs, framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Verify that the created CNS volumes are compliant and have correct policy id") - volumeID := pvs[0].Spec.CSI.VolumeHandle - storagePolicyMatches, err := e2eVSphere.VerifySpbmPolicyOfVolume(volumeID, policyName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(storagePolicyMatches).To(gomega.BeTrue(), "storage policy verification failed") - e2eVSphere.verifyVolumeCompliance(volumeID, true) - - defer func() { - ginkgo.By("Delete the PVCs created in step 3") - err := fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volumeID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - }() - - ginkgo.By("Create pods with using the PVCs created in step 3 and wait for them to be ready") - ginkgo.By("verify we can read and write on the PVCs") - pods := createMultiplePods(ctx, client, pvclaims2d, true) - defer func() { - ginkgo.By("Delete the pod created") - deletePodsAndWaitForVolsToDetach(ctx, client, pods, true) - }() - - ginkgo.By("Verify if VolumeID is created on the given datastores") - dsUrlWhereVolumeIsPresent := fetchDsUrl4CnsVol(e2eVSphere, volumeID) - framework.Logf("Volume: %s is present on %s", volumeID, dsUrlWhereVolumeIsPresent) - e2eVSphere.verifyDatastoreMatch(volumeID, datastoreUrls) - - // Get the destination ds url where the volume will get relocated - destDsUrl := "" - for _, dsurl := range datastoreUrls { - if dsurl != dsUrlWhereVolumeIsPresent { - destDsUrl = dsurl - break - } - } - - framework.Logf("dest url: %s", destDsUrl) - dsRefDest := getDsMoRefFromURL(ctx, destDsUrl) - - rand.New(rand.NewSource(time.Now().Unix())) - testdataFile := fmt.Sprintf("/tmp/testdata_%v_%v", time.Now().Unix(), rand.Intn(1000)) - ginkgo.By(fmt.Sprintf("Creating a 100mb test data file %v", testdataFile)) - op, err := exec.Command("dd", "if=/dev/urandom", fmt.Sprintf("of=%v", testdataFile), - "bs=1M", "count=100").Output() - fmt.Println(op) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - op, err = exec.Command("rm", "-f", testdataFile).Output() - fmt.Println(op) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - currentPvcSize := pvcs[0].Spec.Resources.Requests[v1.ResourceStorage] - newSize := currentPvcSize.DeepCopy() - newSize.Add(resource.MustParse(diskSize)) - - originalSizeInMb, err := getFSSizeMb(f, pods[0]) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Start online expansion of volume") - framework.Logf("currentPvcSize %v, newSize %v", currentPvcSize, newSize) - _, err = expandPVCSize(pvcs[0], newSize, client) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - data := "This file is written by Pod" - - ginkgo.By("Relocate volume and writing IO to pod while resizing volume") - var wg sync.WaitGroup - wg.Add(2) - go writeDataToMultipleFilesOnPodInParallel(namespace, pods[0].Name, data, &wg) - go cnsRelocateVolumeInParallel(e2eVSphere, ctx, volumeID, dsRefDest, true, &wg) - wg.Wait() - - ginkgo.By("Verify relocation of volume is successful") - e2eVSphere.verifyDatastoreMatch(volumeID, []string{destDsUrl}) - - ginkgo.By("Wait and verify the file system resize on pvcs") - framework.Logf("Waiting for file system resize to finish for pvc %v", pvcs[0].Name) - pvcs[0], err = waitForFSResize(pvcs[0], client) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - pvcConditions := pvcs[0].Status.Conditions - expectEqual(len(pvcConditions), 0, "pvc %v should not have status conditions", pvcs[0].Name) - - var fsSize int64 - framework.Logf("Verify filesystem size for mount point /mnt/volume1 for pod %v", pods[0].Name) - fsSize, err = getFSSizeMb(f, pods[0]) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - framework.Logf("File system size after expansion : %v", fsSize) - gomega.Expect(fsSize > originalSizeInMb).To(gomega.BeTrue(), - fmt.Sprintf( - "filesystem size %v is not > than before expansion %v for pvc %q", - fsSize, originalSizeInMb, pvcs[0].Name)) - - framework.Logf("File system resize finished successfully for pvc %v", pvcs[0].Name) - - ginkgo.By("Verify the data on the PVCs match what was written in step 7") - for i := 0; i < 10; i++ { - filePath := fmt.Sprintf("/mnt/volume1/file%v.txt", i) - output := readFileFromPod(namespace, pods[0].Name, filePath) - ginkgo.By(fmt.Sprintf("File contents from file%v.txt are: %s", i, output)) - dataToVerify := data + "\n" - gomega.Expect(output == dataToVerify).To(gomega.BeTrue(), "data verification failed after relocation") - } - - storagePolicyMatches, err = e2eVSphere.VerifySpbmPolicyOfVolume(volumeID, policyName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(storagePolicyMatches).To(gomega.BeTrue(), "storage policy verification failed") - e2eVSphere.verifyVolumeCompliance(volumeID, true) - - ginkgo.By("Delete the pod created") - deletePodsAndWaitForVolsToDetach(ctx, client, pods, true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - podsNew := createMultiplePods(ctx, client, pvclaims2d, true) - deletePodsAndWaitForVolsToDetach(ctx, client, podsNew, true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - }) - - /* - Start volume relocation and then update its metadata - Steps: - 1. Create a SPBM policy with tag based rules. - 2. Create SC using policy created in step 1. - 3. Create PVC using SC created in step 2 and and wait for it to be bound. - 4. Create a pod with the pod created in step 3. - 5. Start writing some IO to pod which run in parallel to steps 6-7. - 6. Relocate CNS volume corresponding to pvc from step 3 to a different datastore. - 7. While relocation is running add labels to PV and PVC. - 8. Verify the IO written so far. - 9. Verify relocation was successful. - 10. Verify the labels in cnsvolume metadata post relocation. - 11. Delete all the objects created during the test. - */ - ginkgo.It("[csi-block-vanilla][csi-block-vanilla-parallelized]"+ - " Start volume relocation and then update its metadata", func() { - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - sharedvmfsURL, sharedNfsURL := "", "" - var datastoreUrls []string - var policyName string - pc := newPbmClient(ctx, e2eVSphere.Client) - - sharedvmfsURL = os.Getenv(envSharedVMFSDatastoreURL) - if sharedvmfsURL == "" { - ginkgo.Skip(fmt.Sprintf("Env %v is missing", envSharedVMFSDatastoreURL)) - } - - sharedNfsURL = os.Getenv(envSharedNFSDatastoreURL) - if sharedNfsURL == "" { - ginkgo.Skip(fmt.Sprintf("Env %v is missing", sharedNfsURL)) - } - datastoreUrls = append(datastoreUrls, sharedvmfsURL, sharedNfsURL) - - scParameters := make(map[string]string) - policyNames := []string{} - pvcs := []*v1.PersistentVolumeClaim{} - pvclaims2d := [][]*v1.PersistentVolumeClaim{} - - rand.New(rand.NewSource(time.Now().UnixNano())) - suffix := fmt.Sprintf("-%v-%v", time.Now().UnixNano(), rand.Intn(10000)) - categoryName := "category" + suffix - tagName := "tag" + suffix - - catID, tagID := createCategoryNTag(ctx, categoryName, tagName) - defer func() { - deleteCategoryNTag(ctx, catID, tagID) - }() - - attachTagToDS(ctx, tagID, sharedvmfsURL) - defer func() { - detachTagFromDS(ctx, tagID, sharedvmfsURL) - }() - - attachTagToDS(ctx, tagID, sharedNfsURL) - defer func() { - detachTagFromDS(ctx, tagID, sharedNfsURL) - }() - - ginkgo.By("create SPBM policy with tag based rules") - ginkgo.By("create a storage class with a SPBM policy created from step 1") - ginkgo.By("create a PVC each using the storage policy created from step 2") - var storageclass *storagev1.StorageClass - var pvclaim *v1.PersistentVolumeClaim - var err error - var policyID *pbmtypes.PbmProfileId - - policyID, policyName = createTagBasedPolicy( - ctx, pc, map[string]string{categoryName: tagName}) - defer func() { - deleteStoragePolicy(ctx, pc, policyID) - }() - policyNames = append(policyNames, policyName) - - framework.Logf("CNS_TEST: Running for vanilla k8s setup") - scParameters[scParamStoragePolicyName] = policyName - storageclass, pvclaim, err = createPVCAndStorageClass(client, - namespace, nil, scParameters, pvc10g, nil, "", false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - pvcs = append(pvcs, pvclaim) - pvclaims2d = append(pvclaims2d, []*v1.PersistentVolumeClaim{pvclaim}) - - defer func() { - ginkgo.By("Delete the SCs created in step 2") - err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - }() - - ginkgo.By("Verify the PVCs created in step 3 are bound") - pvs, err := fpv.WaitForPVClaimBoundPhase(client, pvcs, framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Verify that the created CNS volumes are compliant and have correct policy id") - - volumeID := pvs[0].Spec.CSI.VolumeHandle - storagePolicyMatches, err := e2eVSphere.VerifySpbmPolicyOfVolume(volumeID, policyNames[0]) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(storagePolicyMatches).To(gomega.BeTrue(), "storage policy verification failed") - e2eVSphere.verifyVolumeCompliance(volumeID, true) - - defer func() { - ginkgo.By("Delete the PVCs created in step 3") - err := fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volumeID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - }() - - ginkgo.By("Create pods with using the PVCs created in step 3 and wait for them to be ready") - ginkgo.By("verify we can read and write on the PVCs") - pods := createMultiplePods(ctx, client, pvclaims2d, true) - defer func() { - ginkgo.By("Delete the pod created") - deletePodsAndWaitForVolsToDetach(ctx, client, pods, true) - }() - - ginkgo.By("Verify if VolumeID is created on the given datastores") - dsUrlWhereVolumeIsPresent := fetchDsUrl4CnsVol(e2eVSphere, volumeID) - framework.Logf("Volume: %s is present on %s", volumeID, dsUrlWhereVolumeIsPresent) - e2eVSphere.verifyDatastoreMatch(volumeID, datastoreUrls) - - // Get the destination ds url where the volume will get relocated - destDsUrl := "" - for _, dsurl := range datastoreUrls { - if dsurl != dsUrlWhereVolumeIsPresent { - destDsUrl = dsurl - break - } - } - - framework.Logf("dest url: %s", destDsUrl) - dsRefDest := getDsMoRefFromURL(ctx, destDsUrl) - - rand.New(rand.NewSource(time.Now().Unix())) - testdataFile := fmt.Sprintf("/tmp/testdata_%v_%v", time.Now().Unix(), rand.Intn(1000)) - ginkgo.By(fmt.Sprintf("Creating a 100mb test data file %v", testdataFile)) - op, err := exec.Command("dd", "if=/dev/urandom", fmt.Sprintf("of=%v", testdataFile), - "bs=1M", "count=100").Output() - fmt.Println(op) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - op, err = exec.Command("rm", "-f", testdataFile).Output() - fmt.Println(op) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - labels := make(map[string]string) - labels[labelKey] = labelValue - - ginkgo.By("Start relocation of volume to a different datastore") - task, err := e2eVSphere.cnsRelocateVolume(e2eVSphere, ctx, volumeID, dsRefDest, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - framework.Logf("Waiting for a few seconds for relocation to be started properly on VC") - time.Sleep(time.Duration(10) * time.Second) - - ginkgo.By("Adding labels to volume and writing IO to pod whle relocating volume") - var wg sync.WaitGroup - wg.Add(3) - go writeKnownData2PodInParallel(f, pods[0], testdataFile, &wg) - go updatePvcLabelsInParallel(ctx, client, namespace, labels, pvcs, &wg) - go updatePvLabelsInParallel(ctx, client, namespace, labels, pvs, &wg) - wg.Wait() - - ginkgo.By("Wait for relocation task to complete") - cnsFault := waitForCNSTaskToComplete(ctx, task) - if cnsFault != nil { - err = fmt.Errorf("failed to relocate volume=%+v", cnsFault) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - ginkgo.By("Verify relocation of volume is successful") - e2eVSphere.verifyDatastoreMatch(volumeID, []string{destDsUrl}) - - ginkgo.By(fmt.Sprintf("Waiting for labels %+v to be updated for pvc %s in namespace %s", - labels, pvclaim.Name, namespace)) - pv := getPvFromClaim(client, namespace, pvclaim.Name) - err = e2eVSphere.waitForLabelsToBeUpdated(pv.Spec.CSI.VolumeHandle, labels, - string(cnstypes.CnsKubernetesEntityTypePVC), pvclaim.Name, pvclaim.Namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By(fmt.Sprintf("Waiting for labels %+v to be updated for pv %s", - labels, pv.Name)) - err = e2eVSphere.waitForLabelsToBeUpdated(pv.Spec.CSI.VolumeHandle, labels, - string(cnstypes.CnsKubernetesEntityTypePV), pv.Name, pv.Namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Verify the data on the PVCs match what was written in step 7") - verifyKnownDataInPod(f, pods[0], testdataFile) - - storagePolicyMatches, err = e2eVSphere.VerifySpbmPolicyOfVolume(volumeID, policyNames[0]) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(storagePolicyMatches).To(gomega.BeTrue(), "storage policy verification failed") - e2eVSphere.verifyVolumeCompliance(volumeID, true) - - ginkgo.By("Delete the pod created") - deletePodsAndWaitForVolsToDetach(ctx, client, pods, true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - podsNew := createMultiplePods(ctx, client, pvclaims2d, true) - deletePodsAndWaitForVolsToDetach(ctx, client, podsNew, true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - }) - - /* - Start volume relocation and take a snapshot - Steps: - 1. Create a SPBM policy with tag based rules. - 2. Create SC using policy created in step 1. - 3. Create PVC using SC created in step 2 and and wait for it to be bound. - 4. Create a pod with the pod created in step 3. - 5. Start writing some IO to pod which run in parallel to steps 6-7. - 6. Relocate CNS volume corresponding to pvc from step 3 to a different datastore. - 7. While relocation is running create a CSI snapshot. - 8. Verify the IO written so far. - 9. Verify relocation was successful - 10. Verify snapshot creation was successful. - 11. Delete all the objects created during the test. - */ - ginkgo.It("[csi-block-vanilla][csi-block-vanilla-parallelized]"+ - " Start volume relocation and take a snapshot", func() { - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - sharedNfsUrl, sharedVsanDsurl := "", "" - var datastoreUrls []string - var policyName string - pc := newPbmClient(ctx, e2eVSphere.Client) - - sharedVsanDsurl = os.Getenv(envSharedDatastoreURL) - if sharedVsanDsurl == "" { - ginkgo.Skip(fmt.Sprintf("Env %v is missing", envSharedDatastoreURL)) - } - - sharedNfsUrl = os.Getenv(envSharedNFSDatastoreURL) - if sharedNfsUrl == "" { - ginkgo.Skip(fmt.Sprintf("Env %v is missing", envSharedNFSDatastoreURL)) - } - datastoreUrls = append(datastoreUrls, sharedNfsUrl, sharedVsanDsurl) - - scParameters := make(map[string]string) - policyNames := []string{} - pvcs := []*v1.PersistentVolumeClaim{} - pvclaims2d := [][]*v1.PersistentVolumeClaim{} - - rand.New(rand.NewSource(time.Now().UnixNano())) - suffix := fmt.Sprintf("-%v-%v", time.Now().UnixNano(), rand.Intn(10000)) - categoryName := "category" + suffix - tagName := "tag" + suffix - - catID, tagID := createCategoryNTag(ctx, categoryName, tagName) - defer func() { - deleteCategoryNTag(ctx, catID, tagID) - }() - - attachTagToDS(ctx, tagID, sharedVsanDsurl) - defer func() { - detachTagFromDS(ctx, tagID, sharedVsanDsurl) - }() - - attachTagToDS(ctx, tagID, sharedNfsUrl) - defer func() { - detachTagFromDS(ctx, tagID, sharedNfsUrl) - }() - - ginkgo.By("create SPBM policy with tag based rules") - ginkgo.By("create a storage class with a SPBM policy created from step 1") - ginkgo.By("create a PVC each using the storage policy created from step 2") - var storageclass *storagev1.StorageClass - var pvclaim *v1.PersistentVolumeClaim - var err error - var policyID *pbmtypes.PbmProfileId - - policyID, policyName = createTagBasedPolicy( - ctx, pc, map[string]string{categoryName: tagName}) - defer func() { - deleteStoragePolicy(ctx, pc, policyID) - }() - - policyNames = append(policyNames, policyName) - - framework.Logf("CNS_TEST: Running for vanilla k8s setup") - scParameters[scParamStoragePolicyName] = policyName - storageclass, pvclaim, err = createPVCAndStorageClass(client, - namespace, nil, scParameters, pvc10g, nil, "", false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pvcs = append(pvcs, pvclaim) - pvclaims2d = append(pvclaims2d, []*v1.PersistentVolumeClaim{pvclaim}) - - defer func() { - if vanillaCluster { - ginkgo.By("Delete the SCs created in step 2") - err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }() - - ginkgo.By("Verify the PVCs created in step 3 are bound") - pvs, err := fpv.WaitForPVClaimBoundPhase(client, pvcs, framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Verify that the created CNS volumes are compliant and have correct policy id") - volumeID := pvs[0].Spec.CSI.VolumeHandle - if guestCluster { - volumeID = getVolumeIDFromSupervisorCluster(pvs[0].Spec.CSI.VolumeHandle) - gomega.Expect(volumeID).NotTo(gomega.BeEmpty()) - } - storagePolicyMatches, err := e2eVSphere.VerifySpbmPolicyOfVolume(volumeID, policyNames[0]) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(storagePolicyMatches).To(gomega.BeTrue(), "storage policy verification failed") - e2eVSphere.verifyVolumeCompliance(volumeID, true) - - defer func() { - ginkgo.By("Delete the PVCs created in step 3") - err := fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volumeID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - }() - - ginkgo.By("Create pods with using the PVCs created in step 3 and wait for them to be ready") - ginkgo.By("verify we can read and write on the PVCs") - pods := createMultiplePods(ctx, client, pvclaims2d, true) - defer func() { - ginkgo.By("Delete the pod created") - deletePodsAndWaitForVolsToDetach(ctx, client, pods, true) - }() - - ginkgo.By("Verify if VolumeID is created on the given datastores") - dsUrlWhereVolumeIsPresent := fetchDsUrl4CnsVol(e2eVSphere, volumeID) - framework.Logf("Volume: %s is present on %s", volumeID, dsUrlWhereVolumeIsPresent) - e2eVSphere.verifyDatastoreMatch(volumeID, datastoreUrls) - - // Get the destination ds url where the volume will get relocated - destDsUrl := "" - for _, dsurl := range datastoreUrls { - if dsurl != dsUrlWhereVolumeIsPresent { - destDsUrl = dsurl - break - } - } - - framework.Logf("dest url: %s", destDsUrl) - dsRefDest := getDsMoRefFromURL(ctx, destDsUrl) - - rand.New(rand.NewSource(time.Now().Unix())) - testdataFile := fmt.Sprintf("/tmp/testdata_%v_%v", time.Now().Unix(), rand.Intn(1000)) - ginkgo.By(fmt.Sprintf("Creating a 100mb test data file %v", testdataFile)) - op, err := exec.Command("dd", "if=/dev/urandom", fmt.Sprintf("of=%v", testdataFile), - "bs=1M", "count=100").Output() - fmt.Println(op) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - op, err = exec.Command("rm", "-f", testdataFile).Output() - fmt.Println(op) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - snaps := []*snapV1.VolumeSnapshot{} - snapIDs := []string{} - - //Get snapshot client using the rest config - restConfig := getRestConfigClient() - snapc, err := snapclient.NewForConfig(restConfig) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Create volume snapshot class") - volumeSnapshotClass, err := snapc.SnapshotV1().VolumeSnapshotClasses().Create(ctx, - getVolumeSnapshotClassSpec(snapV1.DeletionPolicy("Delete"), nil), metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - framework.Logf("Volume snapshot class with name %q created", volumeSnapshotClass.Name) - - defer func() { - err := snapc.SnapshotV1().VolumeSnapshotClasses().Delete( - ctx, volumeSnapshotClass.Name, metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Start relocation of volume to a different datastore") - task, err := e2eVSphere.cnsRelocateVolume(e2eVSphere, ctx, volumeID, dsRefDest, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - framework.Logf("Waiting for a few seconds for relocation to be started properly on VC") - time.Sleep(time.Duration(10) * time.Second) - - ginkgo.By("Create a snapshot of volume and writing IO to pod while relocating volume") - ch := make(chan *snapV1.VolumeSnapshot) - lock := &sync.Mutex{} - var wg sync.WaitGroup - wg.Add(2) - go writeKnownData2PodInParallel(f, pods[0], testdataFile, &wg) - go createSnapshotInParallel(ctx, namespace, snapc, pvclaim.Name, volumeSnapshotClass.Name, - ch, lock, &wg) - go func() { - for v := range ch { - snaps = append(snaps, v) - } - }() - wg.Wait() - - ginkgo.By("Wait for relocation task to complete") - cnsFault := waitForCNSTaskToComplete(ctx, task) - if cnsFault != nil { - err = fmt.Errorf("failed to relocate volume=%+v", cnsFault) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - ginkgo.By("Verify relocation of volume is successful") - e2eVSphere.verifyDatastoreMatch(volumeID, []string{destDsUrl}) - - volumeSnapshot := snaps[0] - ginkgo.By("Verify volume snapshot is created") - volumeSnapshot, err = waitForVolumeSnapshotReadyToUse(*snapc, ctx, namespace, volumeSnapshot.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - framework.Logf("snapshot restore size is : %s", volumeSnapshot.Status.RestoreSize.String()) - gomega.Expect(volumeSnapshot.Status.RestoreSize.Cmp(pvclaim.Spec.Resources.Requests[v1.ResourceStorage])).To( - gomega.BeZero()) - - ginkgo.By("Verify volume snapshot content is created") - snapshotContent, err := snapc.SnapshotV1().VolumeSnapshotContents().Get(ctx, - *volumeSnapshot.Status.BoundVolumeSnapshotContentName, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(*snapshotContent.Status.ReadyToUse).To(gomega.BeTrue()) - - framework.Logf("Get volume snapshot ID from snapshot handle") - snapshothandle := *snapshotContent.Status.SnapshotHandle - snapshotId := strings.Split(snapshothandle, "+")[1] - snapIDs = append(snapIDs, snapshotId) - - defer func() { - if len(snaps) > 0 { - framework.Logf("Delete volume snapshot %v", volumeSnapshot.Name) - err = snapc.SnapshotV1().VolumeSnapshots(namespace).Delete( - ctx, volumeSnapshot.Name, metav1.DeleteOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - framework.Logf("Verify snapshot entry %v is deleted from CNS for volume %v", snapIDs[0], volumeID) - err = waitForCNSSnapshotToBeDeleted(volumeID, snapIDs[0]) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - } - }() - - ginkgo.By("Query CNS and check the volume snapshot entry") - err = verifySnapshotIsCreatedInCNS(volumeID, snapshotId, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Verify the data on the PVCs match what was written in step 7") - verifyKnownDataInPod(f, pods[0], testdataFile) - - storagePolicyMatches, err = e2eVSphere.VerifySpbmPolicyOfVolume(volumeID, policyNames[0]) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(storagePolicyMatches).To(gomega.BeTrue(), "storage policy verification failed") - e2eVSphere.verifyVolumeCompliance(volumeID, true) - - ginkgo.By("Delete the pod created") - deletePodsAndWaitForVolsToDetach(ctx, client, pods, true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - podsNew := createMultiplePods(ctx, client, pvclaims2d, true) - deletePodsAndWaitForVolsToDetach(ctx, client, podsNew, true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - }) -}) diff --git a/tests/e2e/testing-manifests/statefulset/nginx/statefulset.yaml b/tests/e2e/testing-manifests/statefulset/nginx/statefulset.yaml index d6041d047f..79bef22a2c 100644 --- a/tests/e2e/testing-manifests/statefulset/nginx/statefulset.yaml +++ b/tests/e2e/testing-manifests/statefulset/nginx/statefulset.yaml @@ -25,9 +25,10 @@ spec: volumeClaimTemplates: - metadata: name: www + annotations: + volume.beta.kubernetes.io/storage-class: nginx-sc spec: accessModes: [ "ReadWriteOnce" ] - storageClassName: nginx-sc resources: requests: storage: 1Gi \ No newline at end of file diff --git a/tests/e2e/testing-manifests/tkg/tkg.yaml b/tests/e2e/testing-manifests/tkg/tkg.yaml index 8a1f9f1328..374fc16562 100644 --- a/tests/e2e/testing-manifests/tkg/tkg.yaml +++ b/tests/e2e/testing-manifests/tkg/tkg.yaml @@ -1,4 +1,4 @@ -apiVersion: run.tanzu.vmware.com/v1alpha3 +apiVersion: run.tanzu.vmware.com/v1alpha1 kind: TanzuKubernetesCluster metadata: name: test-cluster-e2e-script @@ -6,25 +6,22 @@ metadata: spec: topology: controlPlane: - tkr: - reference: - name: replaceImage # this will be replaced with the actual image name by pipeline - replicas: 3 - vmClass: best-effort-xsmall # vmclass to be used for master(s) + count: 3 + class: best-effort-xsmall # vmclass to be used for master(s) storageClass: gc-storage-profile - nodePools: - - replicas: 2 - name: np-2 - vmClass: best-effort-xsmall # vmclass to be used for workers(s) + workers: + count: 2 + class: best-effort-xsmall # vmclass to be used for workers(s) storageClass: gc-storage-profile + distribution: + version: replaceImage # this will be replaced with the actual image name by pipeline settings: network: cni: - name: antrea + name: calico services: - cidrBlocks: - - 198.51.100.0/12 + cidrBlocks: ["198.51.100.0/12"] pods: - cidrBlocks: - - 192.0.2.0/16 - serviceDomain: cluster.local + cidrBlocks: ["192.0.2.0/16"] + serviceDomain: "managedcluster.local" + diff --git a/tests/e2e/tkgs_ha.go b/tests/e2e/tkgs_ha.go index 471e15c467..2f3fd18721 100644 --- a/tests/e2e/tkgs_ha.go +++ b/tests/e2e/tkgs_ha.go @@ -19,7 +19,6 @@ package e2e import ( "context" "fmt" - "os" "strconv" "strings" "time" @@ -60,7 +59,6 @@ var _ = ginkgo.Describe("[csi-tkgs-ha] Tkgs-HA-SanityTests", func() { isSPSServiceStopped bool sshWcpConfig *ssh.ClientConfig svcMasterIp string - clientNewGc clientset.Interface ) ginkgo.BeforeEach(func() { client = f.ClientSet @@ -103,16 +101,7 @@ var _ = ginkgo.Describe("[csi-tkgs-ha] Tkgs-HA-SanityTests", func() { svcClient, svNamespace := getSvcClientAndNamespace() setResourceQuota(svcClient, svNamespace, rqLimit) } - }) - ginkgo.AfterEach(func() { - if supervisorCluster { - dumpSvcNsEventsOnTestFailure(client, namespace) - } - if guestCluster { - svcClient, svNamespace := getSvcClientAndNamespace() - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) - } }) /* @@ -251,7 +240,7 @@ var _ = ginkgo.Describe("[csi-tkgs-ha] Tkgs-HA-SanityTests", func() { ginkgo.By("Creating statefulset") statefulset.Spec.PodManagementPolicy = appsv1.ParallelPodManagement statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &storageclass.Name + Annotations["volume.beta.kubernetes.io/storage-class"] = storageclass.Name *statefulset.Spec.Replicas = 3 CreateStatefulSet(namespace, statefulset, client) replicas := *(statefulset.Spec.Replicas) @@ -797,7 +786,7 @@ var _ = ginkgo.Describe("[csi-tkgs-ha] Tkgs-HA-SanityTests", func() { framework.Logf("allowedTopo: %v", allowedTopologies) statefulset.Spec.PodManagementPolicy = appsv1.ParallelPodManagement statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &storageclass.Name + Annotations["volume.beta.kubernetes.io/storage-class"] = storageclass.Name statefulset.Spec.Template.Spec.Affinity = new(v1.Affinity) statefulset.Spec.Template.Spec.Affinity.NodeAffinity = new(v1.NodeAffinity) statefulset.Spec.Template.Spec.Affinity.NodeAffinity. @@ -897,7 +886,7 @@ var _ = ginkgo.Describe("[csi-tkgs-ha] Tkgs-HA-SanityTests", func() { statefulset := GetStatefulSetFromManifest(namespace) ginkgo.By("Creating statefulset") statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &storageclass.Name + Annotations["volume.beta.kubernetes.io/storage-class"] = storageclass.Name *statefulset.Spec.Replicas = 1 replicas := *(statefulset.Spec.Replicas) @@ -1096,7 +1085,7 @@ var _ = ginkgo.Describe("[csi-tkgs-ha] Tkgs-HA-SanityTests", func() { ginkgo.By("Creating statefulset") statefulset.Spec.PodManagementPolicy = appsv1.ParallelPodManagement statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &storageclass.Name + Annotations["volume.beta.kubernetes.io/storage-class"] = storageclass.Name *statefulset.Spec.Replicas = 3 CreateStatefulSet(namespace, statefulset, client) @@ -1205,7 +1194,7 @@ var _ = ginkgo.Describe("[csi-tkgs-ha] Tkgs-HA-SanityTests", func() { statefulset.Spec.Template.Labels["app"] = statefulset.Name statefulset.Spec.Selector.MatchLabels["app"] = statefulset.Name statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &storageclass.Name + Annotations["volume.beta.kubernetes.io/storage-class"] = storageclass.Name *statefulset.Spec.Replicas = statefulSetReplicaCount CreateStatefulSet(namespace, statefulset, client) stsList = append(stsList, statefulset) @@ -1317,7 +1306,7 @@ var _ = ginkgo.Describe("[csi-tkgs-ha] Tkgs-HA-SanityTests", func() { statefulset.Spec.Template.Labels["app"] = statefulset.Name statefulset.Spec.Selector.MatchLabels["app"] = statefulset.Name statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &storageclass.Name + Annotations["volume.beta.kubernetes.io/storage-class"] = storageclass.Name *statefulset.Spec.Replicas = statefulSetReplicaCount CreateStatefulSet(namespace, statefulset, client) stsList = append(stsList, statefulset) @@ -1520,7 +1509,7 @@ var _ = ginkgo.Describe("[csi-tkgs-ha] Tkgs-HA-SanityTests", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) } - pvclaimsList := createMultiplePVCsInParallel(ctx, client, namespace, storageclass, volumeOpsScale, nil) + pvclaimsList := createMultiplePVCsInParallel(ctx, client, namespace, storageclass, volumeOpsScale) _, err = fpv.WaitForPVClaimBoundPhase(client, pvclaimsList, framework.ClaimProvisionTimeout) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -1722,7 +1711,7 @@ var _ = ginkgo.Describe("[csi-tkgs-ha] Tkgs-HA-SanityTests", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) } - pvclaimsList := createMultiplePVCsInParallel(ctx, client, namespace, storageclass, volumeOpsScale, nil) + pvclaimsList := createMultiplePVCsInParallel(ctx, client, namespace, storageclass, volumeOpsScale) _, err = fpv.WaitForPVClaimBoundPhase(client, pvclaimsList, framework.ClaimProvisionTimeout) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -2170,7 +2159,7 @@ var _ = ginkgo.Describe("[csi-tkgs-ha] Tkgs-HA-SanityTests", func() { statefulset.Spec.Template.Labels["app"] = statefulset.Name statefulset.Spec.Selector.MatchLabels["app"] = statefulset.Name statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &storageclass.Name + Annotations["volume.beta.kubernetes.io/storage-class"] = storageclass.Name *statefulset.Spec.Replicas = replicas _, err := client.AppsV1().StatefulSets(namespace).Create(ctx, statefulset, metav1.CreateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -2270,7 +2259,7 @@ var _ = ginkgo.Describe("[csi-tkgs-ha] Tkgs-HA-SanityTests", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) } - pvclaimsList := createMultiplePVCsInParallel(ctx, client, namespace, storageclass, volumeOpsScale, nil) + pvclaimsList := createMultiplePVCsInParallel(ctx, client, namespace, storageclass, volumeOpsScale) pvs, err := fpv.WaitForPVClaimBoundPhase(client, pvclaimsList, framework.ClaimProvisionTimeout) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -2406,641 +2395,4 @@ var _ = ginkgo.Describe("[csi-tkgs-ha] Tkgs-HA-SanityTests", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }) - - /* - Statefulset - storage class with Zonal storage and - Wffc and with default pod management policy with PodAffinity - 1. Create a zonal storage policy, on the datastore that is shared only to specific cluster - 2. Use the Zonal storage class and Wait for first consumer binding mode - and create statefulset - with parallel pod management policy with replica 3 and PodAntiAffinity - 3. wait for all the gc-PVC to bound - Make sure corresponding SVC-PVC will - have "csi.vsphere.volume-accessible-topology" annotation - csi.vsphere.requested.cluster-topology= - [{"topology.kubernetes.io/zone":"zone1"},{"topology.kubernetes.io/zone":"zone2"}, - {"topology.kubernetes.io/zone":"zone2"}] - 4. storageClassName: should point to gcStorageclass - 5. Wait for the PODs to reach running state - make sure Pod scheduled on appropriate nodes - preset in the availability zone - 6. Describe SVC-PV , and GC-PV and verify node affinity, make sure appropriate node affinity gets added - 7. Delete the above statefulset - 8. Create New statefulset with PODAffinity rules set - 9. Wait for all the PVC's and POD's to come up - 7. Scale up the statefulset replica to 5 , and validate the node affinity on - the newly create PV's and annotations on PVC's - 8. Validate the CNS metadata - 9. Scale down the sts to 0 - 10.Delete Statefulset,PVC,POD,SC - */ - ginkgo.It("Validate statefulset creation with POD affinity and POD Anti affinity", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - ginkgo.By("CNS_TEST: Running for GC setup") - - cleanupsts := false - nodeList, _ := fnodes.GetReadySchedulableNodes(client) - - ginkgo.By("Create statefulset with parallel pod management policy with replica 3") - createResourceQuota(client, namespace, rqLimit, zonalWffcPolicy) - scParameters[svStorageClassName] = zonalWffcPolicy - storageclass, err := client.StorageV1().StorageClasses().Get(ctx, zonalWffcPolicy, metav1.GetOptions{}) - if !apierrors.IsNotFound(err) { - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - // Creating StatefulSet service - ginkgo.By("Creating service") - service := CreateService(namespace, client) - defer func() { - deleteService(namespace, client, service) - }() - - statefulset := GetStatefulSetFromManifest(namespace) - ginkgo.By("Creating statefulset with POD Anti affinity") - allowedTopologies := getTopologySelector(allowedTopologyHAMap, categories, - tkgshaTopologyLevels) - framework.Logf("allowedTopo: %v", allowedTopologies) - statefulset.Spec.PodManagementPolicy = appsv1.ParallelPodManagement - statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &storageclass.Name - statefulset.Spec.Template.Spec.Affinity = new(v1.Affinity) - statefulset.Spec.Template.Spec.Affinity.NodeAffinity = new(v1.NodeAffinity) - statefulset.Spec.Template.Spec.Affinity.NodeAffinity. - RequiredDuringSchedulingIgnoredDuringExecution = new(v1.NodeSelector) - statefulset.Spec.Template.Spec.Affinity.NodeAffinity. - RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = getNodeSelectorTerms(allowedTopologies) - - statefulset.Spec.Template.Spec.Affinity.PodAntiAffinity = new(v1.PodAntiAffinity) - statefulset.Spec.Template.Spec.Affinity.PodAntiAffinity. - RequiredDuringSchedulingIgnoredDuringExecution = getPodAffinityTerm(allowedTopologyHAMap) - - *statefulset.Spec.Replicas = 3 - framework.Logf("Statefulset spec: %v", statefulset) - ginkgo.By("Create Statefulset with PodAntiAffinity") - cleanupsts = true - CreateStatefulSet(namespace, statefulset, client) - replicas := *(statefulset.Spec.Replicas) - - defer func() { - if cleanupsts { - framework.Logf("cleaning up statefulset with podAtiAffinity") - cleaupStatefulset(client, ctx, namespace, statefulset) - } - }() - - ginkgo.By("Verify annotations on SVC PV and required node affinity details on SVC PV and GC PV") - ginkgo.By("Verify pod gets scheduled on appropriate nodes preset in the availability zone") - verifyStsVolumeMetadata(client, ctx, namespace, statefulset, replicas, - allowedTopologyHAMap, categories, zonalPolicy, nodeList, f) - - ginkgo.By("Delete Statefulset with PodAntiAffinity") - cleaupStatefulset(client, ctx, namespace, statefulset) - cleanupsts = false - - ginkgo.By("Creating statefulset with POD-affinity") - statefulset = GetStatefulSetFromManifest(namespace) - allowedTopologies = getTopologySelector(allowedTopologyHAMap, categories, - tkgshaTopologyLevels) - framework.Logf("allowedTopo: %v", allowedTopologies) - statefulset.Spec.PodManagementPolicy = appsv1.ParallelPodManagement - statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &storageclass.Name - statefulset.Spec.Template.Spec.Affinity = new(v1.Affinity) - statefulset.Spec.Template.Spec.Affinity.NodeAffinity = new(v1.NodeAffinity) - statefulset.Spec.Template.Spec.Affinity.NodeAffinity. - RequiredDuringSchedulingIgnoredDuringExecution = new(v1.NodeSelector) - statefulset.Spec.Template.Spec.Affinity.NodeAffinity. - RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = getNodeSelectorTerms(allowedTopologies) - - statefulset.Spec.Template.Spec.Affinity.PodAffinity = new(v1.PodAffinity) - statefulset.Spec.Template.Spec.Affinity.PodAffinity. - RequiredDuringSchedulingIgnoredDuringExecution = getPodAffinityTerm(allowedTopologyHAMap) - - *statefulset.Spec.Replicas = 3 - framework.Logf("Statefulset spec: %v", statefulset) - ginkgo.By("Create Statefulset with PodAffinity") - CreateStatefulSet(namespace, statefulset, client) - replicas = *(statefulset.Spec.Replicas) - - defer func() { - framework.Logf("cleaning up statefulset with podAffinity") - cleaupStatefulset(client, ctx, namespace, statefulset) - }() - - framework.Logf("Verify statefulset volume metadata, node affinities and pod's availability on appropriate zone") - verifyStsVolumeMetadata(client, ctx, namespace, statefulset, replicas, - allowedTopologyHAMap, categories, zonalPolicy, nodeList, f) - - replicas = 5 - framework.Logf(fmt.Sprintf("Scaling up statefulset: %v to number of Replica: %v", - statefulset.Name, replicas)) - _, scaleupErr := fss.Scale(client, statefulset, replicas) - gomega.Expect(scaleupErr).NotTo(gomega.HaveOccurred()) - - fss.WaitForStatusReplicas(client, statefulset, replicas) - fss.WaitForStatusReadyReplicas(client, statefulset, replicas) - ssPodsAfterScaleUp := fss.GetPodList(client, statefulset) - gomega.Expect(ssPodsAfterScaleUp.Items).NotTo(gomega.BeEmpty(), - fmt.Sprintf("Unable to get list of Pods from the Statefulset: %v", statefulset.Name)) - gomega.Expect(len(ssPodsAfterScaleUp.Items) == int(replicas)).To(gomega.BeTrue(), - "Number of Pods in the statefulset %s, %v, should match with number of replicas %v", - statefulset.Name, ssPodsAfterScaleUp.Size(), replicas, - ) - - verifyStsVolumeMetadata(client, ctx, namespace, statefulset, replicas, - allowedTopologyHAMap, categories, zonalPolicy, nodeList, f) - - }) - - /* - Verify volume provisioning after VC reboot using zonal storage - 1. Create few statefulsets , PVC's, deployment POD's using zonal SC's and note the details - 2. Re-boot VC and wait till all the services up and running - 3. Validate the Pre-data, sts's, PVC's and PODs's should be in up and running state - 4. Use the existing SC's and create stateful set with 3 replica. Make sure PVC's - reach bound state, POd's reach running state - 5. validate node affinity details on the gc-PV's and svc-pv's - 7. Create PVC using the zonal sc - 8. Wait for PVC to reach bound state and PV should have appropriate node affinity - 9. Create POD using the PVC created in step 9 , POD should come up on appropriate zone - 10. trigger online and offline volume expansion and validate - 11. delete all sts's , PVC's, SC and POD's - */ - ginkgo.It("Verify volume provisioning after VC reboot using zonal storage", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - ginkgo.By("CNS_TEST: Running for GC setup") - nodeList, err := fnodes.GetReadySchedulableNodes(client) - framework.ExpectNoError(err, "Unable to find ready and schedulable Node") - if !(len(nodeList.Items) > 0) { - framework.Failf("Unable to find ready and schedulable Node") - } - - ginkgo.By("Create 3 statefulsets with parallel pod management policy with replica 3") - createResourceQuota(client, namespace, rqLimit, zonalWffcPolicy) - scParameters[svStorageClassName] = zonalWffcPolicy - storageclass, err := client.StorageV1().StorageClasses().Get(ctx, zonalWffcPolicy, metav1.GetOptions{}) - if !apierrors.IsNotFound(err) { - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - createResourceQuota(client, namespace, rqLimit, zonalPolicy) - scParameters[svStorageClassName] = zonalPolicy - storageclassImmediate, err := client.StorageV1().StorageClasses().Get(ctx, zonalPolicy, metav1.GetOptions{}) - if !apierrors.IsNotFound(err) { - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - var stsList []*appsv1.StatefulSet - var deploymentList []*appsv1.Deployment - var replicas int32 - var pvclaims, pvcs, svcPVCs []*v1.PersistentVolumeClaim - var volumeHandles, svcPVCNames []string - var pods []*v1.Pod - volumeOpsScale := 3 - - // Creating StatefulSet service - ginkgo.By("Creating service") - service := CreateService(namespace, client) - defer func() { - deleteService(namespace, client, service) - }() - - ginkgo.By("Creating 3 statefulsets with parallel pod management policy and 3 replicas") - - for i := 0; i < volumeOpsScale; i++ { - statefulset := GetStatefulSetFromManifest(namespace) - ginkgo.By("Creating statefulset") - statefulset.Spec.PodManagementPolicy = appsv1.ParallelPodManagement - statefulset.Name = "sts-" + strconv.Itoa(i) + "-" + statefulset.Name - statefulset.Spec.Template.Labels["app"] = statefulset.Name - statefulset.Spec.Selector.MatchLabels["app"] = statefulset.Name - statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &storageclass.Name - *statefulset.Spec.Replicas = 3 - CreateStatefulSet(namespace, statefulset, client) - stsList = append(stsList, statefulset) - } - replicas = 3 - - ginkgo.By("Creating 3 PVCs") - for i := 0; i < volumeOpsScale; i++ { - framework.Logf("Creating pvc%v", i) - - pvclaim, err := createPVC(client, namespace, nil, "", storageclassImmediate, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pvclaims = append(pvclaims, pvclaim) - } - - ginkgo.By("Expect all pvcs to provision volume successfully") - _, err = fpv.WaitForPVClaimBoundPhase(client, pvclaims, framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - labelsMap := make(map[string]string) - labelsMap["app"] = "test" - - ginkgo.By("Creating 3 deployment with each PVC created earlier") - - for i := 0; i < volumeOpsScale; i++ { - deployment, err := createDeployment( - ctx, client, 1, labelsMap, nil, namespace, []*v1.PersistentVolumeClaim{pvclaims[i]}, - "", false, busyBoxImageOnGcr) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - deploymentList = append(deploymentList, deployment) - - } - - ginkgo.By("Rebooting VC") - vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + sshdPort - err = invokeVCenterReboot(vcAddress) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = waitForHostToBeUp(e2eVSphere.Config.Global.VCenterHostname) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By("Done with reboot") - essentialServices := []string{spsServiceName, vsanhealthServiceName, vpxdServiceName, wcpServiceName} - checkVcenterServicesRunning(ctx, vcAddress, essentialServices, healthStatusPollTimeout) - - // After reboot. - bootstrap() - - defer func() { - scaleDownNDeleteStsDeploymentsInNamespace(ctx, client, namespace) - pvcs, err := client.CoreV1().PersistentVolumeClaims(namespace).List(ctx, metav1.ListOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - for _, claim := range pvcs.Items { - pv := getPvFromClaim(client, namespace, claim.Name) - err := fpv.DeletePersistentVolumeClaim(client, claim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By("Verify it's PV and corresponding volumes are deleted from CNS") - err = fpv.WaitForPersistentVolumeDeleted(client, pv.Name, poll, - pollTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - volumeHandle := pv.Spec.CSI.VolumeHandle - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volumeHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), - fmt.Sprintf("Volume: %s should not be present in the CNS after it is deleted from "+ - "kubernetes", volumeHandle)) - } - }() - - framework.Logf("After the VC reboot, Wait for all the PVC's to reach bound state") - _, err = fpv.WaitForPVClaimBoundPhase(client, pvclaims, framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - framework.Logf("After the VC reboot, Verify all the pre-created deployment pod's, its status and metadata") - for _, deployment := range deploymentList { - pods, err := fdep.GetPodsForDeployment(client, deployment) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pod := pods.Items[0] - err = fpod.WaitForPodNameRunningInNamespace(client, pod.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - verifyVolumeMetadataOnDeployments(ctx, client, deployment, namespace, allowedTopologyHAMap, - categories, nodeList, zonalPolicy) - - } - - framework.Logf("After the VC reboot, Verify all the pre-created stateful set metadata") - for _, sts := range stsList { - verifyStsVolumeMetadata(client, ctx, namespace, sts, replicas, - allowedTopologyHAMap, categories, zonalPolicy, nodeList, f) - } - - replicas = 5 - framework.Logf(fmt.Sprintf("Increase statefulset %v to number of Replica: %v", - stsList[0].Name, replicas)) - time.Sleep(60 * time.Second) - _, scaleupErr := fss.Scale(client, stsList[0], replicas) - gomega.Expect(scaleupErr).NotTo(gomega.HaveOccurred()) - fss.WaitForStatusReplicas(client, stsList[0], replicas) - fss.WaitForStatusReadyReplicas(client, stsList[0], replicas) - ssPodsAfterScaleUp := fss.GetPodList(client, stsList[0]) - gomega.Expect(ssPodsAfterScaleUp.Items).NotTo(gomega.BeEmpty(), - fmt.Sprintf("Unable to get list of Pods from the Statefulset: %v", stsList[0].Name)) - gomega.Expect(len(ssPodsAfterScaleUp.Items) == int(replicas)).To(gomega.BeTrue(), - "Number of Pods in the statefulset %s, %v, should match with number of replicas %v", - stsList[0].Name, ssPodsAfterScaleUp.Size(), replicas, - ) - - ginkgo.By("Creating Pvc with Immediate topology storageclass") - ginkgo.By("Creating 3 PVCs for volume expansion") - for i := 0; i < volumeOpsScale; i++ { - framework.Logf("Creating pvc%v", i) - - pvc, err := createPVC(client, namespace, nil, "", storageclassImmediate, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pvcs = append(pvclaims, pvc) - } - - ginkgo.By("Wait for GC PVC to come to bound state") - pvs, err := fpv.WaitForPVClaimBoundPhase(client, pvcs, - framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - for _, pv := range pvs { - volHandle := getVolumeIDFromSupervisorCluster(pv.Spec.CSI.VolumeHandle) - volumeHandles = append(volumeHandles, volHandle) - gomega.Expect(volHandle).NotTo(gomega.BeEmpty()) - svcPVCName := pv.Spec.CSI.VolumeHandle - svcPVCNames = append(svcPVCNames, svcPVCName) - svcPVC := getPVCFromSupervisorCluster(svcPVCName) - svcPVCs = append(svcPVCs, svcPVC) - } - - ginkgo.By("Create a pod and wait for it to come to Running state") - for _, pvc := range pvcs { - pod, err := createPod(client, namespace, nil, []*v1.PersistentVolumeClaim{pvc}, false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pods = append(pods, pod) - } - - defer func() { - ginkgo.By("Delete pods") - for _, pod := range pods { - err = fpod.DeletePodWithWait(client, pod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - }() - - ginkgo.By("Verify annotations on SVC PV and required node affinity details on SVC PV and GC PV") - ginkgo.By("Verify pod gets scheduled on appropriate nodes preset in the availability zone") - for i, pv := range pvs { - verifyAnnotationsAndNodeAffinity(allowedTopologyHAMap, categories, pods[i], - nodeList, svcPVCs[i], pv, svcPVCNames[i]) - } - - ginkgo.By("Triggering online volume expansion on PVCs") - for i := range pods { - verifyOnlineVolumeExpansionOnGc(client, namespace, svcPVCNames[i], - volumeHandles[i], pvcs[i], pods[i], f) - } - - ginkgo.By("Triggering offline volume expansion on PVCs") - for i := range pods { - verifyOfflineVolumeExpansionOnGc(client, pvcs[i], svcPVCNames[i], namespace, - volumeHandles[i], pods[i], pvs[i], f) - } - }) - - /* - Static volume provisioning using zonal storage - 1. Create a zonal storage policy, on the datastore that is shared only to specific cluster - 2. Use the Zonal storage class and Immediate binding mode - 3. Create svcpvc and wait for it to bound - 4. switch to gc1 and statically create PV and PVC pointing to svc-pvc - 5. Verify topology details on PV - 6. Delete GC1 PVC - 7. switch to GC2 - 8. Create static pvc on gc2PVC point to svc-pvc - 9. Verify the node affinity of gc1-pv and svc-pv - 9. Create POD, verify the status. - 10. Wait for the PODs to reach running state - make sure Pod scheduled on - appropriate nodes preset in the availability zone - 10. Delete pod, gc1-pv and gc1-pvc and svc pvc. - */ - ginkgo.It("tkgs-ha Verify static provisioning across Guest Clusters", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - newGcKubconfigPath := os.Getenv("NEW_GUEST_CLUSTER_KUBE_CONFIG") - if newGcKubconfigPath == "" { - ginkgo.Skip("Env NEW_GUEST_CLUSTER_KUBE_CONFIG is missing") - } - - svClient, svNamespace := getSvcClientAndNamespace() - pvcAnnotations := make(map[string]string) - annotationVal := "[" - var topoList []string - - for key, val := range allowedTopologyHAMap { - for _, topoVal := range val { - str := `{"` + key + `":"` + topoVal + `"}` - topoList = append(topoList, str) - } - } - framework.Logf("topoList: %v", topoList) - annotationVal += strings.Join(topoList, ",") + "]" - pvcAnnotations[tkgHARequestedAnnotationKey] = annotationVal - framework.Logf("annotationVal :%s, pvcAnnotations: %v", annotationVal, pvcAnnotations) - - ginkgo.By("Creating Pvc with Immediate topology storageclass") - createResourceQuota(client, namespace, rqLimit, zonalPolicy) - scParameters[svStorageClassName] = zonalPolicy - storageclass, err := client.StorageV1().StorageClasses().Get(ctx, zonalPolicy, metav1.GetOptions{}) - if !apierrors.IsNotFound(err) { - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - pvcSpec := getPersistentVolumeClaimSpecWithStorageClass(svNamespace, "", storageclass, nil, "") - pvcSpec.Annotations = pvcAnnotations - svPvclaim, err := svClient.CoreV1().PersistentVolumeClaims(svNamespace).Create(context.TODO(), - pvcSpec, metav1.CreateOptions{}) - svcPVCName := svPvclaim.Name - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - isSVCPvcCreated := true - - ginkgo.By("Wait for SV PVC to come to bound state") - svcPv, err := fpv.WaitForPVClaimBoundPhase(svClient, []*v1.PersistentVolumeClaim{svPvclaim}, - framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - volumeID := svPvclaim.Name - staticPVLabels := make(map[string]string) - staticPVLabels["fcd-id"] = volumeID - - framework.Logf("PVC name in SV " + svcPVCName) - pvcUID := string(svPvclaim.GetUID()) - framework.Logf("PVC UUID in GC " + pvcUID) - gcClusterID := strings.Replace(svcPVCName, pvcUID, "", -1) - - framework.Logf("gcClusterId " + gcClusterID) - pv := getPvFromClaim(svClient, svPvclaim.Namespace, svPvclaim.Name) - pvUID := string(pv.UID) - framework.Logf("PV uuid " + pvUID) - - defer func() { - if isSVCPvcCreated { - err := fpv.DeletePersistentVolumeClaim(svClient, svcPVCName, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - err = fpv.DeletePersistentVolume(svClient, pv.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - isSVCPvcCreated = false - }() - - // Get allowed topologies for zonal storage - allowedTopologies := getTopologySelector(allowedTopologyHAMap, categories, - tkgshaTopologyLevels) - - ginkgo.By("Creating the PV") - staticPv := getPersistentVolumeSpecWithStorageClassFCDNodeSelector(volumeID, - v1.PersistentVolumeReclaimRetain, storageclass.Name, staticPVLabels, - diskSize, allowedTopologies) - staticPv, err = client.CoreV1().PersistentVolumes().Create(ctx, staticPv, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Creating the PVC") - staticPvc := getPersistentVolumeClaimSpec(namespace, staticPVLabels, staticPv.Name) - staticPvc.Spec.StorageClassName = &storageclass.Name - staticPvc, err = client.CoreV1().PersistentVolumeClaims(namespace).Create(ctx, staticPvc, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - isGC1PvcCreated := true - - // Wait for PV and PVC to Bind. - framework.ExpectNoError(fpv.WaitOnPVandPVC(client, framework.NewTimeoutContextWithDefaults(), - namespace, staticPv, staticPvc)) - - defer func() { - if isGC1PvcCreated { - err := fpv.DeletePersistentVolumeClaim(client, staticPvc.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - err = fpv.DeletePersistentVolume(client, staticPv.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Verify PVs, volumes are deleted from CNS") - err = fpv.WaitForPersistentVolumeDeleted(client, staticPv.Name, framework.Poll, - framework.PodDeleteTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volumeID) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), - fmt.Sprintf("Volume: %s should not be present in the CNS after it is deleted from "+ - "kubernetes", volumeID)) - ginkgo.By("Verify volume is deleted in Supervisor Cluster") - volumeExists := verifyVolumeExistInSupervisorCluster(svcPv[0].Spec.CSI.VolumeHandle) - gomega.Expect(volumeExists).To(gomega.BeFalse()) - } - isGC1PvcCreated = false - - }() - - ginkgo.By("Verify SV storageclass points to GC storageclass") - gomega.Expect(*svPvclaim.Spec.StorageClassName == storageclass.Name).To( - gomega.BeTrue(), "SV storageclass does not match with gc storageclass") - framework.Logf("GC PVC's storageclass matches SVC PVC's storageclass") - - ginkgo.By("Verify GV PV has has required PV node affinity details") - _, err = verifyVolumeTopologyForLevel5(staticPv, allowedTopologyHAMap) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - framework.Logf("GC PV: %s has required Pv node affinity details", staticPv.Name) - - ginkgo.By("Verify SV PV has has required PV node affinity details") - _, err = verifyVolumeTopologyForLevel5(svcPv[0], allowedTopologyHAMap) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - framework.Logf("SVC PV: %s has required PV node affinity details", svcPv[0].Name) - time.Sleep(time.Duration(60) * time.Second) - - ginkgo.By("Delete PVC in GC1") - err = fpv.DeletePersistentVolumeClaim(client, staticPvc.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - isGC1PvcCreated = false - - err = fpv.DeletePersistentVolume(client, staticPv.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Verifying if volume still exists in the Supervisor Cluster") - // svcPVCName refers to PVC Name in the supervisor cluster. - volumeID = getVolumeIDFromSupervisorCluster(svPvclaim.Name) - gomega.Expect(volumeID).NotTo(gomega.BeEmpty()) - pvAnnotations := svcPv[0].Annotations - pvSpec := svcPv[0].Spec.CSI - pvStorageClass := svcPv[0].Spec.StorageClassName - - newGcKubconfigPath = os.Getenv("NEW_GUEST_CLUSTER_KUBE_CONFIG") - if newGcKubconfigPath == "" { - ginkgo.Skip("Env NEW_GUEST_CLUSTER_KUBE_CONFIG is missing") - } - clientNewGc, err = createKubernetesClientFromConfig(newGcKubconfigPath) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), - fmt.Sprintf("Error creating k8s client with %v: %v", newGcKubconfigPath, err)) - ginkgo.By("Creating namespace on second GC") - ns, err := framework.CreateTestingNS(f.BaseName, clientNewGc, map[string]string{ - "e2e-framework": f.BaseName, - }) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Error creating namespace on second GC") - - namespaceNewGC := ns.Name - framework.Logf("Created namespace on second GC %v", namespaceNewGC) - defer func() { - err := clientNewGc.CoreV1().Namespaces().Delete(ctx, namespaceNewGC, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Getting ready nodes on GC 2") - nodeList, err := fnodes.GetReadySchedulableNodes(clientNewGc) - framework.ExpectNoError(err, "Unable to find ready and schedulable Node") - gomega.Expect(len(nodeList.Items)).NotTo(gomega.BeZero(), "Unable to find ready and schedulable Node") - - ginkgo.By("Creating PVC in New GC with the vol handle from SVC") - scParameters = make(map[string]string) - scParameters[scParamFsType] = ext4FSType - scParameters[svStorageClassName] = storageclass.Name - storageclassNewGC, err := clientNewGc.StorageV1().StorageClasses().Get(ctx, zonalPolicy, metav1.GetOptions{}) - if !apierrors.IsNotFound(err) { - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - pvcNew, err := createPVC(clientNewGc, namespaceNewGC, nil, "", storageclassNewGC, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - var pvcs []*v1.PersistentVolumeClaim - pvcs = append(pvcs, pvcNew) - ginkgo.By("Waiting for all claims to be in bound state") - _, err = fpv.WaitForPVClaimBoundPhase(clientNewGc, pvcs, framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - pvNewGC := getPvFromClaim(clientNewGc, pvcNew.Namespace, pvcNew.Name) - volumeIDNewGC := pvNewGC.Spec.CSI.VolumeHandle - svcNewPVCName := volumeIDNewGC - volumeIDNewGC = getVolumeIDFromSupervisorCluster(svcNewPVCName) - gomega.Expect(volumeIDNewGC).NotTo(gomega.BeEmpty()) - - framework.Logf("PVC name in SV " + svcNewPVCName) - pvcNewUID := string(pvcNew.GetUID()) - framework.Logf("pvcNewUID in GC " + pvcNewUID) - gcNewClusterID := strings.Replace(svcNewPVCName, pvcNewUID, "", -1) - framework.Logf("pvNew uuid " + gcNewClusterID) - - ginkgo.By("Creating PV in new guest cluster with volume handle from SVC") - pvNew := getPersistentVolumeSpec(svPvclaim.Name, v1.PersistentVolumeReclaimDelete, nil, ext4FSType) - pvNew.Annotations = pvAnnotations - pvNew.Spec.StorageClassName = pvStorageClass - pvNew.Spec.CSI = pvSpec - pvNew.Spec.CSI.VolumeHandle = svPvclaim.Name - pvNew, err = clientNewGc.CoreV1().PersistentVolumes().Create(ctx, pvNew, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - pvNewUID := string(pvNew.UID) - framework.Logf("pvNew uuid " + pvNewUID) - - defer func() { - ginkgo.By("Delete PVC in GC2") - err = fpv.DeletePersistentVolumeClaim(clientNewGc, pvcNew.Name, namespaceNewGC) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - err = fpv.DeletePersistentVolume(clientNewGc, pvNew.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Create a pod and verify pod gets scheduled on appropriate " + - "nodes preset in the availability zone") - pod, err := createPod(clientNewGc, namespaceNewGC, nil, []*v1.PersistentVolumeClaim{pvcNew}, false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - _, err = verifyPodLocationLevel5(pod, nodeList, allowedTopologyHAMap) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - defer func() { - ginkgo.By("Delete pod") - err = fpod.DeletePodWithWait(clientNewGc, pod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Verify volume is detached from the node") - isDiskDetached, err := e2eVSphere.waitForVolumeDetachedFromNode(clientNewGc, - staticPv.Spec.CSI.VolumeHandle, pod.Spec.NodeName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(isDiskDetached).To(gomega.BeTrue(), - fmt.Sprintf("Volume %q is not detached from the node %q", - staticPv.Spec.CSI.VolumeHandle, pod.Spec.NodeName)) - }() - - }) - }) diff --git a/tests/e2e/tkgs_ha_site_down.go b/tests/e2e/tkgs_ha_site_down.go index 83e2ea0c18..8073dbeed5 100644 --- a/tests/e2e/tkgs_ha_site_down.go +++ b/tests/e2e/tkgs_ha_site_down.go @@ -95,16 +95,6 @@ var _ = ginkgo.Describe("[csi-tkgs-ha] Tkgs-HA-SiteDownTests", func() { }) - ginkgo.AfterEach(func() { - if supervisorCluster { - dumpSvcNsEventsOnTestFailure(client, namespace) - } - if guestCluster { - svcClient, svNamespace := getSvcClientAndNamespace() - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) - } - }) - /* Bring down ESX in AZ1 1. Use Zonal storage class of AZ1 with immediate binding @@ -167,7 +157,7 @@ var _ = ginkgo.Describe("[csi-tkgs-ha] Tkgs-HA-SiteDownTests", func() { statefulset.Spec.Template.Labels["app"] = statefulset.Name statefulset.Spec.Selector.MatchLabels["app"] = statefulset.Name statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &storageclass.Name + Annotations["volume.beta.kubernetes.io/storage-class"] = storageclass.Name *statefulset.Spec.Replicas = replicas CreateStatefulSet(namespace, statefulset, client) stsList = append(stsList, statefulset) @@ -311,7 +301,7 @@ var _ = ginkgo.Describe("[csi-tkgs-ha] Tkgs-HA-SiteDownTests", func() { statefulset.Spec.Template.Labels["app"] = statefulset.Name statefulset.Spec.Selector.MatchLabels["app"] = statefulset.Name statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &storageclass.Name + Annotations["volume.beta.kubernetes.io/storage-class"] = storageclass.Name *statefulset.Spec.Replicas = replicas CreateStatefulSet(namespace, statefulset, client) stsList = append(stsList, statefulset) @@ -482,7 +472,7 @@ var _ = ginkgo.Describe("[csi-tkgs-ha] Tkgs-HA-SiteDownTests", func() { statefulset.Spec.Template.Labels["app"] = statefulset.Name statefulset.Spec.Selector.MatchLabels["app"] = statefulset.Name statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &storageclass.Name + Annotations["volume.beta.kubernetes.io/storage-class"] = storageclass.Name *statefulset.Spec.Replicas = replicas CreateStatefulSet(namespace, statefulset, client) stsList = append(stsList, statefulset) @@ -611,7 +601,7 @@ var _ = ginkgo.Describe("[csi-tkgs-ha] Tkgs-HA-SiteDownTests", func() { statefulset.Spec.Template.Labels["app"] = statefulset.Name statefulset.Spec.Selector.MatchLabels["app"] = statefulset.Name statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &storageclass.Name + Annotations["volume.beta.kubernetes.io/storage-class"] = storageclass.Name *statefulset.Spec.Replicas = replicas CreateStatefulSet(namespace, statefulset, client) stsList = append(stsList, statefulset) @@ -732,7 +722,7 @@ var _ = ginkgo.Describe("[csi-tkgs-ha] Tkgs-HA-SiteDownTests", func() { statefulset.Spec.Template.Labels["app"] = statefulset.Name statefulset.Spec.Selector.MatchLabels["app"] = statefulset.Name statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &storageclass.Name + Annotations["volume.beta.kubernetes.io/storage-class"] = storageclass.Name *statefulset.Spec.Replicas = replicas CreateStatefulSet(namespace, statefulset, client) stsList = append(stsList, statefulset) @@ -861,7 +851,7 @@ var _ = ginkgo.Describe("[csi-tkgs-ha] Tkgs-HA-SiteDownTests", func() { statefulset.Spec.Template.Labels["app"] = statefulset.Name statefulset.Spec.Selector.MatchLabels["app"] = statefulset.Name statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &storageclass.Name + Annotations["volume.beta.kubernetes.io/storage-class"] = storageclass.Name *statefulset.Spec.Replicas = replicas CreateStatefulSet(namespace, statefulset, client) stsList = append(stsList, statefulset) diff --git a/tests/e2e/tkgs_ha_utils.go b/tests/e2e/tkgs_ha_utils.go index 07f814dbae..69e79899a9 100644 --- a/tests/e2e/tkgs_ha_utils.go +++ b/tests/e2e/tkgs_ha_utils.go @@ -182,7 +182,7 @@ func verifyVolumeProvisioningWithServiceDown(serviceName string, namespace strin statefulset := GetStatefulSetFromManifest(namespace) ginkgo.By("Creating statefulset") statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &storageclass.Name + Annotations["volume.beta.kubernetes.io/storage-class"] = storageclass.Name *statefulset.Spec.Replicas = 3 _, err = client.AppsV1().StatefulSets(namespace).Create(ctx, statefulset, metav1.CreateOptions{}) framework.ExpectNoError(err) @@ -282,10 +282,32 @@ func verifyOnlineVolumeExpansionOnGc(client clientset.Interface, namespace strin // verifyOfflineVolumeExpansionOnGc is a util method which helps in verifying offline volume expansion on gc func verifyOfflineVolumeExpansionOnGc(client clientset.Interface, pvclaim *v1.PersistentVolumeClaim, svcPVCName string, namespace string, volHandle string, pod *v1.Pod, pv *v1.PersistentVolume, f *framework.Framework) { + cmd := []string{"exec", "", "--namespace=" + namespace, "--", "/bin/sh", "-c", "df -Tkm | grep /mnt/volume1"} + ginkgo.By("Verify the volume is accessible and filesystem type is as expected") + cmd[1] = pod.Name + lastOutput := framework.RunKubectlOrDie(namespace, cmd...) + gomega.Expect(strings.Contains(lastOutput, ext4FSType)).NotTo(gomega.BeFalse()) + ginkgo.By("Check filesystem size for mount point /mnt/volume1 before expansion") originalFsSize, err := getFSSizeMb(f, pod) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + rand.New(rand.NewSource(time.Now().Unix())) + testdataFile := fmt.Sprintf("/tmp/testdata_%v_%v", time.Now().Unix(), rand.Intn(1000)) + ginkgo.By(fmt.Sprintf("Creating a 512mb test data file %v", testdataFile)) + op, err := exec.Command("dd", "if=/dev/urandom", fmt.Sprintf("of=%v", testdataFile), + "bs=64k", "count=8000").Output() + fmt.Println(op) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer func() { + op, err = exec.Command("rm", "-f", testdataFile).Output() + fmt.Println(op) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }() + + _ = framework.RunKubectlOrDie(namespace, "cp", testdataFile, + fmt.Sprintf("%v/%v:/mnt/volume1/testdata", namespace, pod.Name)) + // Delete POD. ginkgo.By(fmt.Sprintf("Deleting the pod %s in namespace %s before expansion", pod.Name, namespace)) err = fpod.DeletePodWithWait(client, pod) @@ -369,6 +391,11 @@ func verifyOfflineVolumeExpansionOnGc(client clientset.Interface, pvclaim *v1.Pe gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Expect(isDiskAttached).To(gomega.BeTrue(), "Volume is not attached to the node") + ginkgo.By("Verify after expansion the filesystem type is as expected") + cmd[1] = pod.Name + lastOutput = framework.RunKubectlOrDie(namespace, cmd...) + gomega.Expect(strings.Contains(lastOutput, ext4FSType)).NotTo(gomega.BeFalse()) + ginkgo.By("Waiting for file system resize to finish") pvclaim, err = waitForFSResize(pvclaim, client) framework.ExpectNoError(err, "while waiting for fs resize to finish") @@ -386,6 +413,20 @@ func verifyOfflineVolumeExpansionOnGc(client clientset.Interface, pvclaim *v1.Pe framework.Failf("error updating filesystem size for %q. Resulting filesystem size is %d", pvclaim.Name, fsSize) } + ginkgo.By("Checking data consistency after PVC resize") + _ = framework.RunKubectlOrDie(namespace, "cp", + fmt.Sprintf("%v/%v:/mnt/volume1/testdata", namespace, pod.Name), testdataFile+"_pod") + defer func() { + op, err = exec.Command("rm", "-f", testdataFile+"_pod").Output() + fmt.Println("rm: ", op) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }() + ginkgo.By("Running diff...") + op, err = exec.Command("diff", testdataFile, testdataFile+"_pod").Output() + fmt.Println("diff: ", op) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(len(op)).To(gomega.BeZero()) + ginkgo.By("File system resize finished successfully in GC") ginkgo.By("Checking for PVC resize completion on SVC PVC") _, err = waitForFSResizeInSvc(svcPVCName) @@ -522,28 +563,22 @@ func verifyVolumeMetadataOnDeployments(ctx context.Context, pvcName, metav1.GetOptions{}) gomega.Expect(pvclaim).NotTo(gomega.BeNil()) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + volHandle := getVolumeIDFromSupervisorCluster(pv.Spec.CSI.VolumeHandle) + gomega.Expect(volHandle).NotTo(gomega.BeEmpty()) + svcPVCName := pv.Spec.CSI.VolumeHandle - if guestCluster { - volHandle := getVolumeIDFromSupervisorCluster(pv.Spec.CSI.VolumeHandle) - gomega.Expect(volHandle).NotTo(gomega.BeEmpty()) - svcPVCName := pv.Spec.CSI.VolumeHandle - - svcPVC := getPVCFromSupervisorCluster(svcPVCName) - gomega.Expect(*svcPVC.Spec.StorageClassName == storagePolicyName).To( - gomega.BeTrue(), "SV Pvc storageclass does not match with SV storageclass") - framework.Logf("GC PVC's storageclass matches SVC PVC's storageclass") - - verifyAnnotationsAndNodeAffinity(allowedTopologyHAMap, categories, pod, - nodeList, svcPVC, pv, svcPVCName) - - // Verify the attached volume match the one in CNS cache - err = waitAndVerifyCnsVolumeMetadata4GCVol(volHandle, svcPVCName, pvclaim, - pv, pod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } else if vanillaCluster { - err = waitAndVerifyCnsVolumeMetadata(pv.Spec.CSI.VolumeHandle, pvclaim, pv, pod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } + svcPVC := getPVCFromSupervisorCluster(svcPVCName) + gomega.Expect(*svcPVC.Spec.StorageClassName == storagePolicyName).To( + gomega.BeTrue(), "SV Pvc storageclass does not match with SV storageclass") + framework.Logf("GC PVC's storageclass matches SVC PVC's storageclass") + + verifyAnnotationsAndNodeAffinity(allowedTopologyHAMap, categories, pod, + nodeList, svcPVC, pv, svcPVCName) + + // Verify the attached volume match the one in CNS cache + err = waitAndVerifyCnsVolumeMetadata4GCVol(volHandle, svcPVCName, pvclaim, + pv, pod) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) } } } @@ -804,93 +839,3 @@ func exitHostMM(ctx context.Context, host *object.HostSystem, timeout int32) { framework.Logf("Host: %v exited from maintenance mode", host) } - -// PodAffinity values are set in this method -func getPodAffinityTerm(allowedTopologyHAMap map[string][]string) []v1.PodAffinityTerm { - var podAffinityTerm v1.PodAffinityTerm - var podAffinityTerms []v1.PodAffinityTerm - var labelSelector *metav1.LabelSelector - var labelSelectorRequirements []metav1.LabelSelectorRequirement - var labelSelectorRequirement metav1.LabelSelectorRequirement - - labelSelectorRequirement.Key = "app" - labelSelectorRequirement.Operator = "In" - labelSelectorRequirement.Values = []string{"nginx"} - labelSelectorRequirements = append(labelSelectorRequirements, labelSelectorRequirement) - labelSelector = new(metav1.LabelSelector) - labelSelector.MatchExpressions = labelSelectorRequirements - podAffinityTerm.LabelSelector = labelSelector - for key := range allowedTopologyHAMap { - podAffinityTerm.TopologyKey = key - } - podAffinityTerms = append(podAffinityTerms, podAffinityTerm) - return podAffinityTerms -} - -// verifyStsVolumeMetadata verifies sts pod replicas and tkg annotations and -// node affinities on svc pvc and verify cns volume meetadata -func verifyStsVolumeMetadata(client clientset.Interface, ctx context.Context, namespace string, - statefulset *appsv1.StatefulSet, replicas int32, allowedTopologyHAMap map[string][]string, - categories []string, storagePolicyName string, nodeList *v1.NodeList, f *framework.Framework) { - // Waiting for pods status to be Ready - fss.WaitForStatusReadyReplicas(client, statefulset, replicas) - gomega.Expect(fss.CheckMount(client, statefulset, mountPath)).NotTo(gomega.HaveOccurred()) - ssPodsBeforeScaleDown := fss.GetPodList(client, statefulset) - gomega.Expect(ssPodsBeforeScaleDown.Items).NotTo(gomega.BeEmpty(), - fmt.Sprintf("Unable to get list of Pods from the Statefulset: %v", statefulset.Name)) - gomega.Expect(len(ssPodsBeforeScaleDown.Items) == int(replicas)).To(gomega.BeTrue(), - "Number of Pods in the statefulset should match with number of replicas") - - ginkgo.By("Verify GV PV and SV PV has has required PV node affinity details") - ginkgo.By("Verify SV PVC has TKG HA annotations set") - // Get the list of Volumes attached to Pods before scale down - for _, sspod := range ssPodsBeforeScaleDown.Items { - pod, err := client.CoreV1().Pods(namespace).Get(ctx, sspod.Name, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - for _, volumespec := range sspod.Spec.Volumes { - if volumespec.PersistentVolumeClaim != nil { - pvcName := volumespec.PersistentVolumeClaim.ClaimName - pv := getPvFromClaim(client, statefulset.Namespace, pvcName) - pvclaim, err := client.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, - pvcName, metav1.GetOptions{}) - gomega.Expect(pvclaim).NotTo(gomega.BeNil()) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - volHandle := getVolumeIDFromSupervisorCluster(pv.Spec.CSI.VolumeHandle) - gomega.Expect(volHandle).NotTo(gomega.BeEmpty()) - svcPVCName := pv.Spec.CSI.VolumeHandle - - svcPVC := getPVCFromSupervisorCluster(svcPVCName) - gomega.Expect(*svcPVC.Spec.StorageClassName == storagePolicyName).To( - gomega.BeTrue(), "SV Pvc storageclass does not match with SV storageclass") - framework.Logf("GC PVC's storageclass matches SVC PVC's storageclass") - - verifyAnnotationsAndNodeAffinity(allowedTopologyHAMap, categories, pod, - nodeList, svcPVC, pv, svcPVCName) - - // Verify the attached volume match the one in CNS cache - err = waitAndVerifyCnsVolumeMetadata4GCVol(volHandle, svcPVCName, pvclaim, - pv, pod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - framework.Logf(fmt.Sprintf("Verify volume: %s is attached to the node: %s", - pv.Spec.CSI.VolumeHandle, sspod.Spec.NodeName)) - var vmUUID string - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - vmUUID, err = getVMUUIDFromNodeName(pod.Spec.NodeName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - verifyCRDInSupervisorWithWait(ctx, f, pod.Spec.NodeName+"-"+svcPVCName, - crdCNSNodeVMAttachment, crdVersion, crdGroup, true) - - isDiskAttached, err := e2eVSphere.isVolumeAttachedToVM(client, volHandle, vmUUID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(isDiskAttached).To(gomega.BeTrue(), "Disk is not attached to the node") - framework.Logf("verify the attached volumes match those in CNS Cache") - err = waitAndVerifyCnsVolumeMetadata4GCVol(volHandle, svcPVCName, pvclaim, - pv, pod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - } - } - -} diff --git a/tests/e2e/topology_aware_node_poweroff.go b/tests/e2e/topology_aware_node_poweroff.go index 55e58b4272..28bd40fee5 100644 --- a/tests/e2e/topology_aware_node_poweroff.go +++ b/tests/e2e/topology_aware_node_poweroff.go @@ -223,12 +223,6 @@ var _ = ginkgo.Describe("[csi-topology-vanilla] Topology-Aware-Provisioning-With gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() - ginkgo.By("Creating service") - service := CreateService(namespace, client) - defer func() { - deleteService(namespace, client, service) - }() - ginkgo.By("Creating statefulset with single replica") statefulset, service := createStatefulSetWithOneReplica(client, manifestPath, namespace) defer func() { diff --git a/tests/e2e/topology_multi_replica.go b/tests/e2e/topology_multi_replica.go index 8c61815781..1af9005d3f 100644 --- a/tests/e2e/topology_multi_replica.go +++ b/tests/e2e/topology_multi_replica.go @@ -81,7 +81,6 @@ var _ = ginkgo.Describe("[csi-topology-multireplica-level5] Topology-Aware-Provi k8sVersion string nimbusGeneratedVcPwd string nimbusGeneratedK8sVmPwd string - clientIndex int ) ginkgo.BeforeEach(func() { var cancel context.CancelFunc @@ -109,7 +108,6 @@ var _ = ginkgo.Describe("[csi-topology-multireplica-level5] Topology-Aware-Provi topologyLength = 5 isSPSServiceStopped = false isVsanHealthServiceStopped = false - clientIndex = 0 topologyMap := GetAndExpectStringEnvVar(topologyMap) topologyAffinityDetails, topologyCategories = createTopologyMapLevel5(topologyMap, topologyLength) @@ -309,9 +307,8 @@ var _ = ginkgo.Describe("[csi-topology-multireplica-level5] Topology-Aware-Provi for each StatefulSet pod */ ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") for i := 0; i < len(statefulSets); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, - statefulSets[i], namespace, allowedTopologies, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, + statefulSets[i], namespace, allowedTopologies, true) } /* Get current leader Csi-Controller-Pod where CSI Attacher is running" + @@ -331,8 +328,7 @@ var _ = ginkgo.Describe("[csi-topology-multireplica-level5] Topology-Aware-Provi statefulSetReplicaCount = 2 ginkgo.By("Scale down statefulset replica and verify the replica count") for i := 0; i < len(statefulSets); i++ { - err = scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true) ssPodsAfterScaleDown := GetListOfPodsInSts(client, statefulSets[i]) gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(statefulSetReplicaCount)).To(gomega.BeTrue(), "Number of Pods in the statefulset should match with number of replicas") @@ -361,8 +357,7 @@ var _ = ginkgo.Describe("[csi-topology-multireplica-level5] Topology-Aware-Provi statefulSetReplicaCount = 0 ginkgo.By("Scale down statefulset replica count to 0") for i := 0; i < len(statefulSets); i++ { - err = scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true) ssPodsAfterScaleDown := GetListOfPodsInSts(client, statefulSets[i]) gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(statefulSetReplicaCount)).To(gomega.BeTrue(), "Number of Pods in the statefulset should match with number of replicas") @@ -510,17 +505,15 @@ var _ = ginkgo.Describe("[csi-topology-multireplica-level5] Topology-Aware-Provi for each StatefulSet pod */ ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") for i := 0; i < len(statefulSets); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, - statefulSets[i], namespace, allowedTopologies, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, + statefulSets[i], namespace, allowedTopologies, true) } // Scale down StatefulSets replicas count statefulSetReplicaCount = 2 ginkgo.By("Scale down statefulset replica count") for i := 0; i < len(statefulSets); i++ { - err = scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true) ssPodsAfterScaleDown := GetListOfPodsInSts(client, statefulSets[i]) gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(statefulSetReplicaCount)).To(gomega.BeTrue(), "Number of Pods in the statefulset should match with number of replicas") @@ -546,8 +539,7 @@ var _ = ginkgo.Describe("[csi-topology-multireplica-level5] Topology-Aware-Provi statefulSetReplicaCount = 0 ginkgo.By("Scale down statefulset replica count to 0") for i := 0; i < len(statefulSets); i++ { - err = scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true) ssPodsAfterScaleDown := GetListOfPodsInSts(client, statefulSets[i]) gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(statefulSetReplicaCount)).To(gomega.BeTrue(), "Number of Pods in the statefulset should match with number of replicas") @@ -691,9 +683,8 @@ var _ = ginkgo.Describe("[csi-topology-multireplica-level5] Topology-Aware-Provi for each StatefulSet pod */ ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") for i := 0; i < len(statefulSets); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, - statefulSets[i], namespace, allowedTopologies, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, + statefulSets[i], namespace, allowedTopologies, true) } // Fetch the number of CSI pods running before restart @@ -721,25 +712,22 @@ var _ = ginkgo.Describe("[csi-topology-multireplica-level5] Topology-Aware-Provi ginkgo.By("Scale up StaefulSets replicas in parallel") statefulSetReplicaCount = 5 for i := 0; i < len(statefulSets); i++ { - err = scaleUpStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleUpStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true) } /* Verify PV nde affinity and that the pods are running on appropriate nodes for each StatefulSet pod */ ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") for i := 0; i < len(statefulSets); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, - statefulSets[i], namespace, allowedTopologies, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, + statefulSets[i], namespace, allowedTopologies, true) } // Scale down statefulset to 0 replicas statefulSetReplicaCount = 0 ginkgo.By("Scale down statefulset replica count to 0") for i := 0; i < len(statefulSets); i++ { - err = scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true) } }) @@ -802,7 +790,7 @@ var _ = ginkgo.Describe("[csi-topology-multireplica-level5] Topology-Aware-Provi // Creating multiple PVCs ginkgo.By("Trigger multiple PVCs") - pvclaimsList := createMultiplePVCsInParallel(ctx, client, namespace, storageclass, pvcCount, nil) + pvclaimsList := createMultiplePVCsInParallel(ctx, client, namespace, storageclass, pvcCount) // Verify PVC claim to be in bound phase and create POD for each PVC ginkgo.By("Verify PVC claim to be in bound phase and create POD for each PVC") @@ -946,9 +934,8 @@ var _ = ginkgo.Describe("[csi-topology-multireplica-level5] Topology-Aware-Provi for each StatefulSet pod */ ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") for i := 0; i < len(podList); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStandalonePodLevel5(ctx, client, podList[i], - namespace, allowedTopologies, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsFoStandalonePodLevel5(ctx, client, podList[i], + namespace, allowedTopologies) } }) @@ -1003,7 +990,7 @@ var _ = ginkgo.Describe("[csi-topology-multireplica-level5] Topology-Aware-Provi // Creating multiple PVCs ginkgo.By("Trigger multiple PVCs") - pvclaimsList := createMultiplePVCsInParallel(ctx, client, namespace, storageclass, pvcCount, nil) + pvclaimsList := createMultiplePVCsInParallel(ctx, client, namespace, storageclass, pvcCount) // Verify PVC claim to be in bound phase and create POD for each PVC ginkgo.By("Verify PVC claim to be in bound phase and create POD for each PVC") @@ -1152,9 +1139,8 @@ var _ = ginkgo.Describe("[csi-topology-multireplica-level5] Topology-Aware-Provi for each StatefulSet pod */ ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") for i := 0; i < len(podList); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStandalonePodLevel5(ctx, client, podList[i], - namespace, allowedTopologies, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsFoStandalonePodLevel5(ctx, client, podList[i], + namespace, allowedTopologies) } }) @@ -1213,7 +1199,7 @@ var _ = ginkgo.Describe("[csi-topology-multireplica-level5] Topology-Aware-Provi // Creating multiple PVCs ginkgo.By("Trigger multiple PVCs") - pvclaimsList := createMultiplePVCsInParallel(ctx, client, namespace, storageclass, pvcCount, nil) + pvclaimsList := createMultiplePVCsInParallel(ctx, client, namespace, storageclass, pvcCount) /* Verifying if all PVCs are in Bound phase and trigger Deployment Pods for each created PVC. @@ -1237,10 +1223,10 @@ var _ = ginkgo.Describe("[csi-topology-multireplica-level5] Topology-Aware-Provi gomega.Expect(err).NotTo(gomega.HaveOccurred()) /* Verify PV nde affinity and that the pods are running on appropriate nodes for each StatefulSet pod */ - err = verifyPVnodeAffinityAndPODnodedetailsForDeploymentSetsLevel5(ctx, client, deployment, - namespace, allowedTopologies, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForDeploymentSetsLevel5(ctx, client, deployment, + namespace, allowedTopologies, true) deploymentList = append(deploymentList, deployment) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) // Delete elected leader Csi-Controller-Pod where CSi-Attacher is running if i == 2 { ginkgo.By("Delete elected leader Csi-Controller-Pod where CSi-Attacher is running") @@ -1255,7 +1241,7 @@ var _ = ginkgo.Describe("[csi-topology-multireplica-level5] Topology-Aware-Provi pv := getPvFromClaim(client, pvclaimsList[i].Namespace, pvclaimsList[i].Name) err = fpv.DeletePersistentVolumeClaim(client, pvclaimsList[i].Name, namespace) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - framework.ExpectNoError(fpv.WaitForPersistentVolumeDeleted(client, pv.Name, poll, framework.ClaimProvisionTimeout)) + framework.ExpectNoError(fpv.WaitForPersistentVolumeDeleted(client, pv.Name, poll, pollTimeoutShort)) err = e2eVSphere.waitForCNSVolumeToBeDeleted(pv.Spec.CSI.VolumeHandle) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } @@ -1427,9 +1413,8 @@ var _ = ginkgo.Describe("[csi-topology-multireplica-level5] Topology-Aware-Provi for each StatefulSet pod */ ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") for i := 0; i < len(statefulSets); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, - statefulSets[i], namespace, allowedTopologies, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, + statefulSets[i], namespace, allowedTopologies, true) } /* Get elected current leader Csi-Controller-Pod where CSI Attacher is running" + @@ -1447,8 +1432,7 @@ var _ = ginkgo.Describe("[csi-topology-multireplica-level5] Topology-Aware-Provi statefulSetReplicaCount = 2 ginkgo.By("Scale down statefulset replica count") for i := 0; i < len(statefulSets); i++ { - err = scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true) if i == 1 { /* Delete newly elected leader CSi-Controller-Pod where CSI-Attacher is running */ ginkgo.By("Delete elected leader CSi-Controller-Pod where CSI-Attacher is running") @@ -1471,8 +1455,7 @@ var _ = ginkgo.Describe("[csi-topology-multireplica-level5] Topology-Aware-Provi statefulSetReplicaCount = 0 ginkgo.By("Scale down statefulset replica count to 0") for i := 0; i < len(statefulSets); i++ { - err = scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true) } // Verify that the StatefulSet Pods, PVC's are deleted successfully @@ -1654,9 +1637,8 @@ var _ = ginkgo.Describe("[csi-topology-multireplica-level5] Topology-Aware-Provi for each StatefulSet pod */ ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") for i := 0; i < len(podList); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStandalonePodLevel5(ctx, client, podList[i], - namespace, allowedTopologies, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsFoStandalonePodLevel5(ctx, client, podList[i], + namespace, allowedTopologies) } }) @@ -1736,8 +1718,7 @@ var _ = ginkgo.Describe("[csi-topology-multireplica-level5] Topology-Aware-Provi vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + sshdPort username := vsphereCfg.Global.User newPassword := e2eTestPassword - err = invokeVCenterChangePassword(username, nimbusGeneratedVcPwd, newPassword, vcAddress, - false, clientIndex) + err = invokeVCenterChangePassword(username, nimbusGeneratedVcPwd, newPassword, vcAddress) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Modifying the password in the secret") @@ -1759,17 +1740,17 @@ var _ = ginkgo.Describe("[csi-topology-multireplica-level5] Topology-Aware-Provi csiReplicaCount := *deployment.Spec.Replicas ginkgo.By("Stopping CSI driver") - isServiceStopped, err := stopCSIPods(ctx, c, csiSystemNamespace) + isServiceStopped, err := stopCSIPods(ctx, c) gomega.Expect(err).NotTo(gomega.HaveOccurred()) defer func() { if isServiceStopped { framework.Logf("Starting CSI driver") - isServiceStopped, err = startCSIPods(ctx, c, csiReplicaCount, csiSystemNamespace) + isServiceStopped, err = startCSIPods(ctx, c, csiReplicaCount) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() framework.Logf("Starting CSI driver") - _, err = startCSIPods(ctx, c, csiReplicaCount, csiSystemNamespace) + _, err = startCSIPods(ctx, c, csiReplicaCount) gomega.Expect(err).NotTo(gomega.HaveOccurred()) // As we are in the same vCenter session, deletion of PVC should go through @@ -1778,8 +1759,7 @@ var _ = ginkgo.Describe("[csi-topology-multireplica-level5] Topology-Aware-Provi gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Reverting the password change") - err = invokeVCenterChangePassword(username, newPassword, nimbusGeneratedVcPwd, vcAddress, false, - clientIndex) + err = invokeVCenterChangePassword(username, newPassword, nimbusGeneratedVcPwd, vcAddress) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Reverting the secret change back to reflect the original password") @@ -1894,9 +1874,8 @@ var _ = ginkgo.Describe("[csi-topology-multireplica-level5] Topology-Aware-Provi for each StatefulSet pod */ ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") for i := 0; i < len(podList); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStandalonePodLevel5(ctx, client, podList[i], - namespace, allowedTopologies, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsFoStandalonePodLevel5(ctx, client, podList[i], + namespace, allowedTopologies) } }) // TESTCASE-6 @@ -2061,9 +2040,8 @@ var _ = ginkgo.Describe("[csi-topology-multireplica-level5] Topology-Aware-Provi ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") for i := 0; i < len(podList); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStandalonePodLevel5(ctx, client, podList[i], namespace, - allowedTopologies, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsFoStandalonePodLevel5(ctx, client, podList[i], namespace, + allowedTopologies) } // Deleting Pod's @@ -2238,7 +2216,7 @@ var _ = ginkgo.Describe("[csi-topology-multireplica-level5] Topology-Aware-Provi // Creating multiple PVCs ginkgo.By("Trigger multiple PVCs") - pvclaimsList := createMultiplePVCsInParallel(ctx, client, namespace, storageclass, pvcCount, nil) + pvclaimsList := createMultiplePVCsInParallel(ctx, client, namespace, storageclass, pvcCount) defer func() { // cleanup code for deleting PVC ginkgo.By("Deleting PVC's and PV's") diff --git a/tests/e2e/topology_operation_strom_cases.go b/tests/e2e/topology_operation_strom_cases.go index f9378d664d..58c48a2366 100644 --- a/tests/e2e/topology_operation_strom_cases.go +++ b/tests/e2e/topology_operation_strom_cases.go @@ -204,9 +204,8 @@ var _ = ginkgo.Describe("[csi-topology-operation-strom-level5] "+ for each StatefulSet pod */ ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") for i := 0; i < len(statefulSets); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, - statefulSets[i], namespace, allowedTopologies, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, + statefulSets[i], namespace, allowedTopologies, true) } // Bring down few esxi hosts that belongs to zone3 @@ -237,9 +236,7 @@ var _ = ginkgo.Describe("[csi-topology-operation-strom-level5] "+ // Scale up statefulSets replicas count ginkgo.By("Scale up statefulset replica and verify the replica count") statefulSetReplicaCount = 35 - err = scaleUpStatefulSetPod(ctx, client, statefulSets[0], namespace, statefulSetReplicaCount, - true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleUpStatefulSetPod(ctx, client, statefulSets[0], namespace, statefulSetReplicaCount, true) ssPodsAfterScaleDown := GetListOfPodsInSts(client, statefulSets[0]) gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(statefulSetReplicaCount)).To(gomega.BeTrue(), "Number of Pods in the statefulset should match with number of replicas") @@ -247,8 +244,7 @@ var _ = ginkgo.Describe("[csi-topology-operation-strom-level5] "+ // Scale down statefulSets replica count statefulSetReplicaCount = 5 ginkgo.By("Scale down statefulset replica and verify the replica count") - err = scaleDownStatefulSetPod(ctx, client, statefulSets[1], namespace, statefulSetReplicaCount, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleDownStatefulSetPod(ctx, client, statefulSets[1], namespace, statefulSetReplicaCount, true) ssPodsAfterScaleDown = GetListOfPodsInSts(client, statefulSets[1]) gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(statefulSetReplicaCount)).To(gomega.BeTrue(), "Number of Pods in the statefulset should match with number of replicas") @@ -285,16 +281,10 @@ var _ = ginkgo.Describe("[csi-topology-operation-strom-level5] "+ } }() - ginkgo.By("Wait for k8s cluster to be healthy") - wait4AllK8sNodesToBeUp(ctx, client, nodeList) - err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - // Scale down statefulSets replicas count statefulSetReplicaCount = 10 ginkgo.By("Scale down statefulset replica and verify the replica count") - err = scaleDownStatefulSetPod(ctx, client, statefulSets[0], namespace, statefulSetReplicaCount, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleDownStatefulSetPod(ctx, client, statefulSets[0], namespace, statefulSetReplicaCount, true) ssPodsAfterScaleDown = GetListOfPodsInSts(client, statefulSets[0]) gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(statefulSetReplicaCount)).To(gomega.BeTrue(), "Number of Pods in the statefulset should match with number of replicas") @@ -302,9 +292,7 @@ var _ = ginkgo.Describe("[csi-topology-operation-strom-level5] "+ // Scale up statefulSets replica count statefulSetReplicaCount = 35 ginkgo.By("Scale up statefulset replica and verify the replica count") - err = scaleUpStatefulSetPod(ctx, client, statefulSets[1], namespace, statefulSetReplicaCount, - true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleUpStatefulSetPod(ctx, client, statefulSets[1], namespace, statefulSetReplicaCount, true) ssPodsAfterScaleDown = GetListOfPodsInSts(client, statefulSets[1]) gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(statefulSetReplicaCount)).To(gomega.BeTrue(), "Number of Pods in the statefulset should match with number of replicas") @@ -313,9 +301,8 @@ var _ = ginkgo.Describe("[csi-topology-operation-strom-level5] "+ for each StatefulSet pod */ ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") for i := 0; i < len(statefulSets); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, - statefulSets[i], namespace, allowedTopologies, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, + statefulSets[i], namespace, allowedTopologies, true) } // Bring up all ESXi host which were powered off in zone2 @@ -354,12 +341,16 @@ var _ = ginkgo.Describe("[csi-topology-operation-strom-level5] "+ } } + ginkgo.By("Verify k8s cluster is healthy") + wait4AllK8sNodesToBeUp(ctx, client, nodeList) + err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + // Scale down statefulSets replica count statefulSetReplicaCount = 0 ginkgo.By("Scale down statefulset replica count") for i := 0; i < len(statefulSets); i++ { - err = scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true) ssPodsAfterScaleDown := GetListOfPodsInSts(client, statefulSets[i]) gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(statefulSetReplicaCount)).To(gomega.BeTrue(), "Number of Pods in the statefulset should match with number of replicas") @@ -553,11 +544,6 @@ var _ = ginkgo.Describe("[csi-topology-operation-strom-level5] "+ } } - ginkgo.By("Wait for k8s cluster to be healthy") - wait4AllK8sNodesToBeUp(ctx, client, nodeList) - err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - /* Get newly elected leader Csi-Controller-Pod where CSI Provisioner is running" + find new master node IP where this Csi-Controller-Pod is running */ ginkgo.By("Get newly Leader Csi-Controller-Pod where CSI Provisioner is running and " + @@ -579,9 +565,8 @@ var _ = ginkgo.Describe("[csi-topology-operation-strom-level5] "+ for each StatefulSet pod */ ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") for i := 0; i < len(statefulSets); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, - statefulSets[i], namespace, allowedTopologies, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, + statefulSets[i], namespace, allowedTopologies, true) } /* Get current leader Csi-Controller-Pod where CSI Attacher is running and " + @@ -595,6 +580,11 @@ var _ = ginkgo.Describe("[csi-topology-operation-strom-level5] "+ "which is running on master node %s", controller_name, k8sMasterIP) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + ginkgo.By("Wait for k8s cluster to be healthy") + wait4AllK8sNodesToBeUp(ctx, client, nodeList) + err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + // Verify all the workload Pods are in up and running state ginkgo.By("Verify all the workload Pods are in up and running state") ssPods = fss.GetPodList(client, statefulSets[1]) @@ -609,8 +599,7 @@ var _ = ginkgo.Describe("[csi-topology-operation-strom-level5] "+ statefulSetReplicaCount = 5 ginkgo.By("Scale down statefulset replica and verify the replica count") for i := 0; i < len(statefulSets); i++ { - err = scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true) ssPodsAfterScaleDown := GetListOfPodsInSts(client, statefulSets[i]) gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(statefulSetReplicaCount)).To(gomega.BeTrue(), "Number of Pods in the statefulset should match with number of replicas") @@ -636,9 +625,7 @@ var _ = ginkgo.Describe("[csi-topology-operation-strom-level5] "+ // Scale up statefulSets replicas count statefulSetReplicaCount = 20 ginkgo.By("Scale up statefulset replica and verify the replica count") - err = scaleUpStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, - true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleUpStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true) ssPodsAfterScaleDown := GetListOfPodsInSts(client, statefulSets[i]) gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(statefulSetReplicaCount)).To(gomega.BeTrue(), "Number of Pods in the statefulset should match with number of replicas") @@ -648,17 +635,15 @@ var _ = ginkgo.Describe("[csi-topology-operation-strom-level5] "+ for each StatefulSet pod */ ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") for i := 0; i < len(statefulSets); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, - statefulSets[i], namespace, allowedTopologies, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, + statefulSets[i], namespace, allowedTopologies, true) } // Scale down statefulSets replica count statefulSetReplicaCount = 0 ginkgo.By("Scale down statefulset replica count") for i := 0; i < len(statefulSets); i++ { - err = scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true) ssPodsAfterScaleDown := GetListOfPodsInSts(client, statefulSets[i]) gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(statefulSetReplicaCount)).To(gomega.BeTrue(), "Number of Pods in the statefulset should match with number of replicas") diff --git a/tests/e2e/topology_site_down_cases.go b/tests/e2e/topology_site_down_cases.go index e3e0f7734e..a4d491f82d 100644 --- a/tests/e2e/topology_site_down_cases.go +++ b/tests/e2e/topology_site_down_cases.go @@ -188,9 +188,8 @@ var _ = ginkgo.Describe("[csi-topology-sitedown-level5] Topology-Aware-Provision for each StatefulSet pod */ ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") for i := 0; i < len(statefulSets); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, - statefulSets[i], namespace, allowedTopologies, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, + statefulSets[i], namespace, allowedTopologies, true) } // Bring down 1 ESXi's that belongs to zone1 and Bring down 1 ESXi's that belongs to zone2 @@ -230,18 +229,15 @@ var _ = ginkgo.Describe("[csi-topology-sitedown-level5] Topology-Aware-Provision for each StatefulSet pod */ ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") for i := 0; i < len(statefulSets); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, - statefulSets[i], namespace, allowedTopologies, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, + statefulSets[i], namespace, allowedTopologies, true) } // Scale up statefulSets replicas count ginkgo.By("Scaleup any one StatefulSets replica") statefulSetReplicaCount = 10 ginkgo.By("Scale down statefulset replica and verify the replica count") - err = scaleUpStatefulSetPod(ctx, client, statefulSets[0], namespace, statefulSetReplicaCount, - true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleUpStatefulSetPod(ctx, client, statefulSets[0], namespace, statefulSetReplicaCount, true) ssPodsAfterScaleDown := GetListOfPodsInSts(client, statefulSets[0]) gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(statefulSetReplicaCount)).To(gomega.BeTrue(), "Number of Pods in the statefulset should match with number of replicas") @@ -250,8 +246,7 @@ var _ = ginkgo.Describe("[csi-topology-sitedown-level5] Topology-Aware-Provision statefulSetReplicaCount = 5 ginkgo.By("Scale down statefulset replica count") for i := 0; i < len(statefulSets); i++ { - err = scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true) ssPodsAfterScaleDown := GetListOfPodsInSts(client, statefulSets[i]) gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(statefulSetReplicaCount)).To(gomega.BeTrue(), "Number of Pods in the statefulset should match with number of replicas") @@ -280,9 +275,8 @@ var _ = ginkgo.Describe("[csi-topology-sitedown-level5] Topology-Aware-Provision for each StatefulSet pod */ ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") for i := 0; i < len(statefulSets); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, - statefulSets[i], namespace, allowedTopologies, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, + statefulSets[i], namespace, allowedTopologies, true) } // verifyVolumeMetadataInCNS @@ -301,13 +295,16 @@ var _ = ginkgo.Describe("[csi-topology-sitedown-level5] Topology-Aware-Provision } } } + ginkgo.By("Verify k8s cluster is healthy") + wait4AllK8sNodesToBeUp(ctx, client, nodeList) + err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) // Scale down statefulSets replica count statefulSetReplicaCount = 0 ginkgo.By("Scale down statefulset replica count") for i := 0; i < len(statefulSets); i++ { - err = scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true) ssPodsAfterScaleDown := GetListOfPodsInSts(client, statefulSets[i]) gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(statefulSetReplicaCount)).To(gomega.BeTrue(), "Number of Pods in the statefulset should match with number of replicas") @@ -396,9 +393,8 @@ var _ = ginkgo.Describe("[csi-topology-sitedown-level5] Topology-Aware-Provision for each StatefulSet pod */ ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") for i := 0; i < len(statefulSets); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, - statefulSets[i], namespace, allowedTopologies, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, + statefulSets[i], namespace, allowedTopologies, true) } // Bring down 1 ESXi's that belongs to Cluster2 and Bring down 1 ESXi's that belongs to Cluster3 @@ -437,18 +433,15 @@ var _ = ginkgo.Describe("[csi-topology-sitedown-level5] Topology-Aware-Provision for each StatefulSet pod */ ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") for i := 0; i < len(statefulSets); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, - statefulSets[i], namespace, allowedTopologies, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, + statefulSets[i], namespace, allowedTopologies, true) } // Scale up statefulSets replicas count ginkgo.By("Scaleup any one StatefulSets replica") statefulSetReplicaCount = 10 ginkgo.By("Scale down statefulset replica and verify the replica count") - err = scaleUpStatefulSetPod(ctx, client, statefulSets[0], namespace, statefulSetReplicaCount, - true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleUpStatefulSetPod(ctx, client, statefulSets[0], namespace, statefulSetReplicaCount, true) ssPodsAfterScaleDown := GetListOfPodsInSts(client, statefulSets[0]) gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(statefulSetReplicaCount)).To(gomega.BeTrue(), "Number of Pods in the statefulset should match with number of replicas") @@ -457,8 +450,7 @@ var _ = ginkgo.Describe("[csi-topology-sitedown-level5] Topology-Aware-Provision statefulSetReplicaCount = 5 ginkgo.By("Scale down statefulset replica count") for i := 0; i < len(statefulSets); i++ { - err = scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true) ssPodsAfterScaleDown := GetListOfPodsInSts(client, statefulSets[i]) gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(statefulSetReplicaCount)).To(gomega.BeTrue(), "Number of Pods in the statefulset should match with number of replicas") @@ -479,9 +471,8 @@ var _ = ginkgo.Describe("[csi-topology-sitedown-level5] Topology-Aware-Provision for each StatefulSet pod */ ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") for i := 0; i < len(statefulSets); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, - statefulSets[i], namespace, allowedTopologies, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, + statefulSets[i], namespace, allowedTopologies, true) } // verifyVolumeMetadataInCNS @@ -501,12 +492,16 @@ var _ = ginkgo.Describe("[csi-topology-sitedown-level5] Topology-Aware-Provision } } + ginkgo.By("Verify k8s cluster is healthy") + wait4AllK8sNodesToBeUp(ctx, client, nodeList) + err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + // Scale down statefulSets replica count statefulSetReplicaCount = 0 ginkgo.By("Scale down statefulset replica count") for i := 0; i < len(statefulSets); i++ { - err = scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true) ssPodsAfterScaleDown := GetListOfPodsInSts(client, statefulSets[i]) gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(statefulSetReplicaCount)).To(gomega.BeTrue(), "Number of Pods in the statefulset should match with number of replicas") @@ -595,9 +590,8 @@ var _ = ginkgo.Describe("[csi-topology-sitedown-level5] Topology-Aware-Provision for each StatefulSet pod */ ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") for i := 0; i < len(statefulSets); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, - statefulSets[i], namespace, allowedTopologies, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, + statefulSets[i], namespace, allowedTopologies, true) } // Bring down ESXi host that belongs to zone1, zone2 and zone3 @@ -622,19 +616,12 @@ var _ = ginkgo.Describe("[csi-topology-sitedown-level5] Topology-Aware-Provision }() powerOffHostsList = append(append(powerOffHostsList, powerOffHostsList2...), powerOffHostsList3...) - // Wait for k8s cluster to be healthy - ginkgo.By("Wait for k8s cluster to be healthy") - wait4AllK8sNodesToBeUp(ctx, client, nodeList) - err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - /* Verify PV nde affinity and that the pods are running on appropriate nodes for each StatefulSet pod */ ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") for i := 0; i < len(statefulSets); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, - statefulSets[i], namespace, allowedTopologies, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, + statefulSets[i], namespace, allowedTopologies, true) } // Bring up @@ -643,19 +630,11 @@ var _ = ginkgo.Describe("[csi-topology-sitedown-level5] Topology-Aware-Provision powerOnEsxiHostByCluster(powerOffHostsList[i]) } - // Wait for k8s cluster to be healthy - ginkgo.By("Wait for k8s cluster to be healthy") - wait4AllK8sNodesToBeUp(ctx, client, nodeList) - err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - // Scale up statefulSets replicas count ginkgo.By("Scaleup any one StatefulSets replica") statefulSetReplicaCount = 10 ginkgo.By("Scale down statefulset replica and verify the replica count") - err = scaleUpStatefulSetPod(ctx, client, statefulSets[0], namespace, statefulSetReplicaCount, - true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleUpStatefulSetPod(ctx, client, statefulSets[0], namespace, statefulSetReplicaCount, true) ssPodsAfterScaleDown := GetListOfPodsInSts(client, statefulSets[0]) gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(statefulSetReplicaCount)).To(gomega.BeTrue(), "Number of Pods in the statefulset should match with number of replicas") @@ -664,20 +643,23 @@ var _ = ginkgo.Describe("[csi-topology-sitedown-level5] Topology-Aware-Provision statefulSetReplicaCount = 5 ginkgo.By("Scale down statefulset replica count") for i := 0; i < len(statefulSets); i++ { - err = scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true) ssPodsAfterScaleDown := GetListOfPodsInSts(client, statefulSets[i]) gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(statefulSetReplicaCount)).To(gomega.BeTrue(), "Number of Pods in the statefulset should match with number of replicas") } + ginkgo.By("Wait for k8s cluster to be healthy") + wait4AllK8sNodesToBeUp(ctx, client, nodeList) + err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + /* Verify PV nde affinity and that the pods are running on appropriate nodes for each StatefulSet pod */ ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") for i := 0; i < len(statefulSets); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, - statefulSets[i], namespace, allowedTopologies, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, + statefulSets[i], namespace, allowedTopologies, true) } // verifyVolumeMetadataInCNS @@ -697,12 +679,16 @@ var _ = ginkgo.Describe("[csi-topology-sitedown-level5] Topology-Aware-Provision } } + ginkgo.By("Verify k8s cluster is healthy") + wait4AllK8sNodesToBeUp(ctx, client, nodeList) + err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + // Scale down statefulSets replica count statefulSetReplicaCount = 0 ginkgo.By("Scale down statefulset replica count") for i := 0; i < len(statefulSets); i++ { - err = scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true) ssPodsAfterScaleDown := GetListOfPodsInSts(client, statefulSets[i]) gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(statefulSetReplicaCount)).To(gomega.BeTrue(), "Number of Pods in the statefulset should match with number of replicas") @@ -794,9 +780,8 @@ var _ = ginkgo.Describe("[csi-topology-sitedown-level5] Topology-Aware-Provision for each StatefulSet pod */ ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") for i := 0; i < len(statefulSets); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, - statefulSets[i], namespace, allowedTopologies, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, + statefulSets[i], namespace, allowedTopologies, true) } // Bring down ESXi hosts that belongs to zone2 @@ -826,19 +811,15 @@ var _ = ginkgo.Describe("[csi-topology-sitedown-level5] Topology-Aware-Provision for each StatefulSet pod */ ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") for i := 0; i < len(statefulSets); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, - statefulSets[i], namespace, allowedTopologies, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, + statefulSets[i], namespace, allowedTopologies, true) } // Scale up statefulSets replicas count ginkgo.By("Scaleup any one StatefulSets replica") statefulSetReplicaCount = 10 ginkgo.By("Scale down statefulset replica and verify the replica count") - err = scaleUpStatefulSetPod(ctx, client, statefulSets[0], namespace, statefulSetReplicaCount, - true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleUpStatefulSetPod(ctx, client, statefulSets[0], namespace, statefulSetReplicaCount, true) ssPodsAfterScaleDown := GetListOfPodsInSts(client, statefulSets[0]) gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(statefulSetReplicaCount)).To(gomega.BeTrue(), "Number of Pods in the statefulset should match with number of replicas") @@ -847,8 +828,7 @@ var _ = ginkgo.Describe("[csi-topology-sitedown-level5] Topology-Aware-Provision statefulSetReplicaCount = 5 ginkgo.By("Scale down statefulset replica count") for i := 0; i < len(statefulSets); i++ { - err = scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true) ssPodsAfterScaleDown := GetListOfPodsInSts(client, statefulSets[i]) gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(statefulSetReplicaCount)).To(gomega.BeTrue(), "Number of Pods in the statefulset should match with number of replicas") @@ -869,10 +849,8 @@ var _ = ginkgo.Describe("[csi-topology-sitedown-level5] Topology-Aware-Provision for each StatefulSet pod */ ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") for i := 0; i < len(statefulSets); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, - statefulSets[i], namespace, allowedTopologies, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, + statefulSets[i], namespace, allowedTopologies, true) } // verifyVolumeMetadataInCNS @@ -892,12 +870,16 @@ var _ = ginkgo.Describe("[csi-topology-sitedown-level5] Topology-Aware-Provision } } + ginkgo.By("Verify k8s cluster is healthy") + wait4AllK8sNodesToBeUp(ctx, client, nodeList) + err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + // Scale down statefulSets replica count statefulSetReplicaCount = 0 ginkgo.By("Scale down statefulset replica count") for i := 0; i < len(statefulSets); i++ { - err = scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true) ssPodsAfterScaleDown := GetListOfPodsInSts(client, statefulSets[i]) gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(statefulSetReplicaCount)).To(gomega.BeTrue(), "Number of Pods in the statefulset should match with number of replicas") @@ -989,9 +971,8 @@ var _ = ginkgo.Describe("[csi-topology-sitedown-level5] Topology-Aware-Provision for each StatefulSet pod */ ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") for i := 0; i < len(statefulSets); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, - statefulSets[i], namespace, allowedTopologies, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, + statefulSets[i], namespace, allowedTopologies, true) } // Bring down ESXi hosts that belongs to zone3 @@ -1006,12 +987,6 @@ var _ = ginkgo.Describe("[csi-topology-sitedown-level5] Topology-Aware-Provision } }() - // Wait for k8s cluster to be healthy - ginkgo.By("Wait for k8s cluster to be healthy") - wait4AllK8sNodesToBeUp(ctx, client, nodeList) - err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - // Verify all the workload Pods are in up and running state ginkgo.By("Verify all the workload Pods are in up and running state") ssPods = fss.GetPodList(client, statefulSets[1]) @@ -1025,17 +1000,15 @@ var _ = ginkgo.Describe("[csi-topology-sitedown-level5] Topology-Aware-Provision for each StatefulSet pod */ ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") for i := 0; i < len(statefulSets); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, - statefulSets[i], namespace, allowedTopologies, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, + statefulSets[i], namespace, allowedTopologies, true) } // Scale up statefulSets replicas count ginkgo.By("Scaleup any one StatefulSets replica") statefulSetReplicaCount = 5 ginkgo.By("Scale down statefulset replica and verify the replica count") - err = scaleUpStatefulSetPod(ctx, client, statefulSets[0], namespace, statefulSetReplicaCount, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleUpStatefulSetPod(ctx, client, statefulSets[0], namespace, statefulSetReplicaCount, true) ssPodsAfterScaleDown := GetListOfPodsInSts(client, statefulSets[0]) gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(statefulSetReplicaCount)).To(gomega.BeTrue(), "Number of Pods in the statefulset should match with number of replicas") @@ -1044,8 +1017,7 @@ var _ = ginkgo.Describe("[csi-topology-sitedown-level5] Topology-Aware-Provision statefulSetReplicaCount = 10 ginkgo.By("Scale down statefulset replica count") for i := 0; i < len(statefulSets); i++ { - err = scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true) ssPodsAfterScaleDown := GetListOfPodsInSts(client, statefulSets[i]) gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(statefulSetReplicaCount)).To(gomega.BeTrue(), "Number of Pods in the statefulset should match with number of replicas") @@ -1066,9 +1038,8 @@ var _ = ginkgo.Describe("[csi-topology-sitedown-level5] Topology-Aware-Provision for each StatefulSet pod */ ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") for i := 0; i < len(statefulSets); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, - statefulSets[i], namespace, allowedTopologies, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, + statefulSets[i], namespace, allowedTopologies, true) } // verifyVolumeMetadataInCNS @@ -1088,12 +1059,16 @@ var _ = ginkgo.Describe("[csi-topology-sitedown-level5] Topology-Aware-Provision } } + ginkgo.By("Verify k8s cluster is healthy") + wait4AllK8sNodesToBeUp(ctx, client, nodeList) + err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + // Scale down statefulSets replica count statefulSetReplicaCount = 0 ginkgo.By("Scale down statefulset replica count") for i := 0; i < len(statefulSets); i++ { - err = scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true) ssPodsAfterScaleDown := GetListOfPodsInSts(client, statefulSets[i]) gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(statefulSetReplicaCount)).To(gomega.BeTrue(), "Number of Pods in the statefulset should match with number of replicas") @@ -1183,9 +1158,8 @@ var _ = ginkgo.Describe("[csi-topology-sitedown-level5] Topology-Aware-Provision for each StatefulSet pod */ ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") for i := 0; i < len(statefulSets); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, - statefulSets[i], namespace, allowedTopologies, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, + statefulSets[i], namespace, allowedTopologies, true) } // Bring down ESXi hosts that belongs to zone2 @@ -1280,9 +1254,7 @@ var _ = ginkgo.Describe("[csi-topology-sitedown-level5] Topology-Aware-Provision // Scale down statefulSets replica count statefulSetReplicaCount = 5 ginkgo.By("Scale down statefulset replica count") - err = scaleDownStatefulSetPod(ctx, client, statefulSets[0], namespace, statefulSetReplicaCount, - true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleDownStatefulSetPod(ctx, client, statefulSets[0], namespace, statefulSetReplicaCount, true) ssPodsAfterScaleDown := GetListOfPodsInSts(client, statefulSets[0]) gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(statefulSetReplicaCount)).To(gomega.BeTrue(), "Number of Pods in the statefulset should match with number of replicas") @@ -1299,9 +1271,8 @@ var _ = ginkgo.Describe("[csi-topology-sitedown-level5] Topology-Aware-Provision for each StatefulSet pod */ ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") for i := 0; i < len(statefulSets); i++ { - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, - statefulSets[i], namespace, allowedTopologies, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, + statefulSets[i], namespace, allowedTopologies, true) } // verifyVolumeMetadataInCNS @@ -1320,13 +1291,16 @@ var _ = ginkgo.Describe("[csi-topology-sitedown-level5] Topology-Aware-Provision } } } + ginkgo.By("Verify k8s cluster is healthy") + wait4AllK8sNodesToBeUp(ctx, client, nodeList) + err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) // Scale down statefulSets replica count statefulSetReplicaCount = 0 ginkgo.By("Scale down statefulset replica count") for i := 0; i < len(statefulSets); i++ { - err = scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleDownStatefulSetPod(ctx, client, statefulSets[i], namespace, statefulSetReplicaCount, true) ssPodsAfterScaleDown := GetListOfPodsInSts(client, statefulSets[i]) gomega.Expect(len(ssPodsAfterScaleDown.Items) == int(statefulSetReplicaCount)).To(gomega.BeTrue(), "Number of Pods in the statefulset should match with number of replicas") diff --git a/tests/e2e/topology_snapshot.go b/tests/e2e/topology_snapshot.go index 51f1b52105..97722ed42c 100644 --- a/tests/e2e/topology_snapshot.go +++ b/tests/e2e/topology_snapshot.go @@ -16,8 +16,6 @@ package e2e import ( "context" "fmt" - "os" - "strconv" "strings" "time" @@ -28,6 +26,7 @@ import ( v1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" @@ -39,8 +38,8 @@ import ( fss "k8s.io/kubernetes/test/e2e/framework/statefulset" admissionapi "k8s.io/pod-security-admission/api" - snapV1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" - snapclient "github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned" + snapV1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" + snapclient "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned" ) var _ = ginkgo.Describe("[topology-snapshot] Topology Volume Snapshot tests", func() { @@ -61,7 +60,6 @@ var _ = ginkgo.Describe("[topology-snapshot] Topology Volume Snapshot tests", fu leafNode int leafNodeTag1 int leafNodeTag2 int - pandoraSyncWaitTime int ) ginkgo.BeforeEach(func() { @@ -111,12 +109,6 @@ var _ = ginkgo.Describe("[topology-snapshot] Topology Volume Snapshot tests", fu gomega.Expect(err).NotTo(gomega.HaveOccurred()) } - if os.Getenv(envPandoraSyncWaitTime) != "" { - pandoraSyncWaitTime, err = strconv.Atoi(os.Getenv(envPandoraSyncWaitTime)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } else { - pandoraSyncWaitTime = defaultPandoraSyncWaitTime - } }) /* @@ -204,17 +196,15 @@ var _ = ginkgo.Describe("[topology-snapshot] Topology Volume Snapshot tests", fu defer func() { if snapshotContentCreated { framework.Logf("Deleting volume snapshot content") - deleteVolumeSnapshotContentWithPandoraWait(ctx, snapc, - *volumeSnapshot.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, *volumeSnapshot.Status.BoundVolumeSnapshotContentName) + err := snapc.SnapshotV1().VolumeSnapshotContents().Delete(ctx, + *volumeSnapshot.Status.BoundVolumeSnapshotContentName, metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } if snapshotCreated { framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) + err := snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, volumeSnapshot.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() @@ -235,7 +225,7 @@ var _ = ginkgo.Describe("[topology-snapshot] Topology Volume Snapshot tests", fu snapshotId := strings.Split(snapshothandle, "+")[1] ginkgo.By("Query CNS and check the volume snapshot entry") - err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId, false) + err = verifySnapshotIsCreatedInCNS(volHandle, snapshotId) gomega.Expect(err).NotTo(gomega.HaveOccurred()) _, err = snapc.SnapshotV1().VolumeSnapshotContents().Get(ctx, @@ -272,9 +262,8 @@ var _ = ginkgo.Describe("[topology-snapshot] Topology Volume Snapshot tests", fu specified in the allowed topologies of SC */ ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStandalonePodLevel5(ctx, client, pod, namespace, - allowedTopologies, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsFoStandalonePodLevel5(ctx, client, pod, namespace, + allowedTopologies) defer func() { ginkgo.By(fmt.Sprintf("Deleting the pod %s in namespace %s", pod.Name, namespace)) @@ -291,7 +280,8 @@ var _ = ginkgo.Describe("[topology-snapshot] Topology Volume Snapshot tests", fu }() ginkgo.By("Delete volume snapshot and verify the snapshot content is deleted") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) + err = snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, volumeSnapshot.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) snapshotCreated = false framework.Logf("Wait till the volume snapshot is deleted") @@ -300,11 +290,15 @@ var _ = ginkgo.Describe("[topology-snapshot] Topology Volume Snapshot tests", fu snapshotContentCreated = false ginkgo.By("Verify snapshot entry is deleted from CNS") - err = verifySnapshotIsDeletedInCNS(volHandle, snapshotId, false) + err = verifySnapshotIsDeletedInCNS(volHandle, snapshotId) gomega.Expect(err).NotTo(gomega.HaveOccurred()) framework.Logf("Deleting volume snapshot Again to check Not found error") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime) + err = snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, volumeSnapshot.Name, metav1.DeleteOptions{}) + if !apierrors.IsNotFound(err) { + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } + time.Sleep(40 * time.Second) }) /* @@ -369,9 +363,8 @@ var _ = ginkgo.Describe("[topology-snapshot] Topology Volume Snapshot tests", fu // Verify PV node affinity and that the PODS are running on appropriate nodes ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, - namespace, allowedTopologies, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, + namespace, allowedTopologies, false) framework.Logf("Fetching pod 3, pvc3 and pv3 details") pod3, err := client.CoreV1().Pods(namespace).Get(ctx, @@ -413,17 +406,16 @@ var _ = ginkgo.Describe("[topology-snapshot] Topology Volume Snapshot tests", fu defer func() { if snapshotContentCreated { framework.Logf("Deleting volume snapshot content") - deleteVolumeSnapshotContentWithPandoraWait(ctx, snapc, - *volumeSnapshot3.Status.BoundVolumeSnapshotContentName, pandoraSyncWaitTime) - - framework.Logf("Wait till the volume snapshot is deleted") - err = waitForVolumeSnapshotContentToBeDeleted(*snapc, ctx, *volumeSnapshot3.Status.BoundVolumeSnapshotContentName) + err := snapc.SnapshotV1().VolumeSnapshotContents().Delete(ctx, + *volumeSnapshot3.Status.BoundVolumeSnapshotContentName, metav1.DeleteOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } if snapshotCreated { framework.Logf("Deleting volume snapshot") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot3.Name, pandoraSyncWaitTime) + err := snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, volumeSnapshot3.Name, + metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) } }() @@ -445,7 +437,7 @@ var _ = ginkgo.Describe("[topology-snapshot] Topology Volume Snapshot tests", fu snapshotId2 := strings.Split(snapshothandle3, "+")[1] ginkgo.By("Query CNS and check the volume snapshot entry") - err = verifySnapshotIsCreatedInCNS(volHandle3, snapshotId2, false) + err = verifySnapshotIsCreatedInCNS(volHandle3, snapshotId2) gomega.Expect(err).NotTo(gomega.HaveOccurred()) pvcSpec := getPersistentVolumeClaimSpecWithDatasource(namespace, "1Gi", storageclass, nil, @@ -474,12 +466,13 @@ var _ = ginkgo.Describe("[topology-snapshot] Topology Volume Snapshot tests", fu appropriate node as specified in the allowed topologies of SC */ ginkgo.By("Verify newly created PV node affinity and that the new PODS are running " + "on appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, - namespace, allowedTopologies, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, + namespace, allowedTopologies, false) ginkgo.By("Delete volume snapshot and verify the snapshot content is deleted") - deleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot3.Name, pandoraSyncWaitTime) + err = snapc.SnapshotV1().VolumeSnapshots(namespace).Delete(ctx, + volumeSnapshot3.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) snapshotCreated = false framework.Logf("Wait till the volume snapshot content is deleted") @@ -489,7 +482,7 @@ var _ = ginkgo.Describe("[topology-snapshot] Topology Volume Snapshot tests", fu snapshotContentCreated = false ginkgo.By("Verify snapshot entry is deleted from CNS") - err = verifySnapshotIsDeletedInCNS(volHandle3, snapshotId2, false) + err = verifySnapshotIsDeletedInCNS(volHandle3, snapshotId2) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) diff --git a/tests/e2e/util.go b/tests/e2e/util.go index 22bb45a2ee..4f2c2c6ac3 100644 --- a/tests/e2e/util.go +++ b/tests/e2e/util.go @@ -78,6 +78,8 @@ import ( fssh "k8s.io/kubernetes/test/e2e/framework/ssh" fss "k8s.io/kubernetes/test/e2e/framework/statefulset" + snapc "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" + snapclient "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned" "sigs.k8s.io/controller-runtime/pkg/client" cnsoperatorv1alpha1 "sigs.k8s.io/vsphere-csi-driver/v3/pkg/apis/cnsoperator" cnsfileaccessconfigv1alpha1 "sigs.k8s.io/vsphere-csi-driver/v3/pkg/apis/cnsoperator/cnsfileaccessconfig/v1alpha1" @@ -481,50 +483,6 @@ type GetTaskTstatus struct { OrgID string `json:"org_id"` } -// This Struct is used for Creating tanzu cluster -type TanzuCluster struct { - APIVersion string `yaml:"apiVersion"` - Kind string `yaml:"kind"` - Metadata struct { - Name string `yaml:"name"` - Namespace string `yaml:"namespace"` - } `yaml:"metadata"` - Spec struct { - Topology struct { - ControlPlane struct { - TKR struct { - Reference struct { - Name string `yaml:"name"` - } `yaml:"reference"` - } `yaml:"tkr"` - Replicas int `yaml:"replicas"` - VMClass string `yaml:"vmClass"` - StorageClass string `yaml:"storageClass"` - } `yaml:"controlPlane"` - NodePools []struct { - Replicas int `yaml:"replicas"` - Name string `yaml:"name"` - VMClass string `yaml:"vmClass"` - StorageClass string `yaml:"storageClass"` - } `yaml:"nodePools"` - } `yaml:"topology"` - Settings struct { - Network struct { - CNI struct { - Name string `yaml:"name"` - } `yaml:"cni"` - Services struct { - CIDRBlocks []string `yaml:"cidrBlocks"` - } `yaml:"services"` - Pods struct { - CIDRBlocks []string `yaml:"cidrBlocks"` - } `yaml:"pods"` - ServiceDomain string `yaml:"serviceDomain"` - } `yaml:"network"` - } `yaml:"settings"` - } `yaml:"spec"` -} - // getVSphereStorageClassSpec returns Storage Class Spec with supplied storage // class parameters. func getVSphereStorageClassSpec(scName string, scParameters map[string]string, @@ -588,16 +546,23 @@ func getPvFromClaim(client clientset.Interface, namespace string, claimName stri // getNodeUUID returns Node VM UUID for requested node. func getNodeUUID(ctx context.Context, client clientset.Interface, nodeName string) string { vmUUID := "" - csiNode, err := client.StorageV1().CSINodes().Get(ctx, nodeName, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - csiDriverFound := false - for _, driver := range csiNode.Spec.Drivers { - if driver.Name == e2evSphereCSIDriverName { - csiDriverFound = true - vmUUID = driver.NodeID + if isCsiFssEnabled(ctx, client, GetAndExpectStringEnvVar(envCSINamespace), useCsiNodeID) { + csiNode, err := client.StorageV1().CSINodes().Get(ctx, nodeName, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + csiDriverFound := false + for _, driver := range csiNode.Spec.Drivers { + if driver.Name == e2evSphereCSIDriverName { + csiDriverFound = true + vmUUID = driver.NodeID + } } + gomega.Expect(csiDriverFound).To(gomega.BeTrue(), "CSI driver not found in CSI node %s", nodeName) + } else { + node, err := client.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + vmUUID = strings.TrimPrefix(node.Spec.ProviderID, providerPrefix) + gomega.Expect(vmUUID).NotTo(gomega.BeEmpty()) } - gomega.Expect(csiDriverFound).To(gomega.BeTrue(), "CSI driver not found in CSI node %s", nodeName) ginkgo.By(fmt.Sprintf("VM UUID is: %s for node: %s", vmUUID, nodeName)) return vmUUID } @@ -893,7 +858,6 @@ func createPVC(client clientset.Interface, pvcnamespace string, pvclaimlabels ma // storage class. func scaleCreatePVC(client clientset.Interface, pvcnamespace string, pvclaimlabels map[string]string, ds string, storageclass *storagev1.StorageClass, accessMode v1.PersistentVolumeAccessMode, wg *sync.WaitGroup) { - defer ginkgo.GinkgoRecover() defer wg.Done() pvcspec := getPersistentVolumeClaimSpecWithStorageClass(pvcnamespace, ds, storageclass, pvclaimlabels, accessMode) @@ -912,7 +876,6 @@ func scaleCreatePVC(client clientset.Interface, pvcnamespace string, pvclaimlabe func scaleCreateDeletePVC(client clientset.Interface, pvcnamespace string, pvclaimlabels map[string]string, ds string, storageclass *storagev1.StorageClass, accessMode v1.PersistentVolumeAccessMode, wg *sync.WaitGroup, lock *sync.Mutex, worker int) { - defer ginkgo.GinkgoRecover() ctx, cancel := context.WithCancel(context.Background()) var totalPVCDeleted int = 0 defer cancel() @@ -985,12 +948,9 @@ func updateDeploymentReplicawithWait(client clientset.Interface, count int32, na var err error waitErr := wait.Poll(healthStatusPollInterval, healthStatusPollTimeout, func() (bool, error) { deployment, err = client.AppsV1().Deployments(namespace).Get(ctx, name, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) if err != nil { - if count == 0 && apierrors.IsNotFound(err) { - return true, nil - } else { - return false, err - } + return false, nil } *deployment.Spec.Replicas = count ginkgo.By("Waiting for update operation on deployment to take effect") @@ -1150,7 +1110,7 @@ func updateCSIDeploymentProvisionerTimeout(client clientset.Interface, namespace framework.Logf("Waiting for a min for update operation on deployment to take effect...") time.Sleep(1 * time.Minute) err = fpod.WaitForPodsRunningReady(client, csiSystemNamespace, int32(num_csi_pods), 0, - 2*pollTimeout, ignoreLabels) + pollTimeout, ignoreLabels) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } @@ -1450,7 +1410,7 @@ func checkVcenterServicesRunning( var pollTime time.Duration if len(timeout) == 0 { - pollTime = pollTimeout * 6 + pollTime = pollTimeout * 2 } else { pollTime = timeout[0] } @@ -1636,7 +1596,7 @@ func upgradeTKG(wcpHost string, wcpToken string, tkgCluster string, tkgImage str } // createGC method creates GC and takes WCP host and bearer token as input param -func createGC(wcpHost string, wcpToken string, tkgImageName string, clusterName string) { +func createGC(wcpHost string, wcpToken string) { transCfg := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // ignore expired SSL certificates @@ -1651,26 +1611,7 @@ func createGC(wcpHost string, wcpToken string, tkgImageName string, clusterName gcBytes, err := os.ReadFile(tkg_yaml) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - var tkg TanzuCluster - err = yaml.Unmarshal([]byte(gcBytes), &tkg) - if err != nil { - framework.Logf("Error: %v", err) - } - - // Change the value of the replaceImage field - tkg.Spec.Topology.ControlPlane.TKR.Reference.Name = tkgImageName - tkg.Metadata.Name = clusterName - - // Marshal the updated struct back to YAML - updatedYAML, err := yaml.Marshal(&tkg) - if err != nil { - framework.Logf("Error: %v", err) - } - - // Convert the marshalled YAML to []byte - updatedYAMLBytes := []byte(updatedYAML) - - req, err := http.NewRequest("POST", createGCURL, bytes.NewBuffer(updatedYAMLBytes)) + req, err := http.NewRequest("POST", createGCURL, bytes.NewBuffer(gcBytes)) gomega.Expect(err).NotTo(gomega.HaveOccurred()) req.Header.Add("Authorization", "Bearer "+wcpToken) req.Header.Add("Accept", "application/yaml") @@ -2010,10 +1951,7 @@ func writeToFile(filePath, data string) error { // invokeVCenterChangePassword invokes `dir-cli password reset` command on the // given vCenter host over SSH, thereby resetting the currentPassword of the // `user` to the `newPassword`. -func invokeVCenterChangePassword(user, adminPassword, newPassword, host string, - isMultiVcSetup bool, clientIndex int) error { - var copyCmd string - var removeCmd string +func invokeVCenterChangePassword(user, adminPassword, newPassword, host string) error { // Create an input file and write passwords into it. path := "input.txt" data := fmt.Sprintf("%s\n%s\n", adminPassword, newPassword) @@ -2026,27 +1964,16 @@ func invokeVCenterChangePassword(user, adminPassword, newPassword, host string, gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() // Remote copy this input file to VC. - if !isMultiVcSetup { - copyCmd = fmt.Sprintf("/bin/cat %s | /usr/bin/ssh root@%s '/usr/bin/cat >> input_copy.txt'", - path, e2eVSphere.Config.Global.VCenterHostname) - } else { - vCenter := strings.Split(multiVCe2eVSphere.multivcConfig.Global.VCenterHostname, ",")[clientIndex] - copyCmd = fmt.Sprintf("/bin/cat %s | /usr/bin/ssh root@%s '/usr/bin/cat >> input_copy.txt'", - path, vCenter) - } + copyCmd := fmt.Sprintf("/bin/cat %s | /usr/bin/ssh root@%s '/usr/bin/cat >> input_copy.txt'", + path, e2eVSphere.Config.Global.VCenterHostname) fmt.Printf("Executing the command: %s\n", copyCmd) _, err = exec.Command("/bin/sh", "-c", copyCmd).Output() gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer func() { // Remove the input_copy.txt file from VC. - if !isMultiVcSetup { - removeCmd = fmt.Sprintf("/usr/bin/ssh root@%s '/usr/bin/rm input_copy.txt'", - e2eVSphere.Config.Global.VCenterHostname) - } else { - vCenter := strings.Split(multiVCe2eVSphere.multivcConfig.Global.VCenterHostname, ",")[clientIndex] - removeCmd = fmt.Sprintf("/usr/bin/ssh root@%s '/usr/bin/rm input_copy.txt'", - vCenter) - } + removeCmd := fmt.Sprintf("/usr/bin/ssh root@%s '/usr/bin/rm input_copy.txt'", + e2eVSphere.Config.Global.VCenterHostname) _, err = exec.Command("/bin/sh", "-c", removeCmd).Output() gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() @@ -2343,7 +2270,6 @@ func getVolumeIDFromSupervisorCluster(pvcName string) string { svcPV := getPvFromClaim(svcClient, svNamespace, pvcName) volumeHandle := svcPV.Spec.CSI.VolumeHandle ginkgo.By(fmt.Sprintf("Found volume in Supervisor cluster with VolumeID: %s", volumeHandle)) - return volumeHandle } @@ -2556,7 +2482,6 @@ func verifyIsDetachedInSupervisor(ctx context.Context, f *framework.Framework, // namespace. It takes client, namespace, pvc, pv as input. func verifyPodCreation(f *framework.Framework, client clientset.Interface, namespace string, pvc *v1.PersistentVolumeClaim, pv *v1.PersistentVolume) { - defer ginkgo.GinkgoRecover() ginkgo.By("Create pod and wait for this to be in running phase") pod, err := createPod(client, namespace, nil, []*v1.PersistentVolumeClaim{pvc}, false, "") gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -3422,9 +3347,6 @@ func VsanObjIndentities(ctx context.Context, vs *vSphere, pvName string) string for _, cluster := range clusterComputeResource { if strings.Contains(cluster.Name(), computeCluster) { - // Fix for NotAuthenticated issue - bootstrap() - clusterConfig, err := vsanHealthClient.VsanQueryObjectIdentities(ctx, cluster.Reference()) framework.Logf("clusterconfig %v", clusterConfig) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -4008,9 +3930,14 @@ func createPod(client clientset.Interface, namespace string, nodeSelector map[st return pod, nil } -func getDeploymentSpec(ctx context.Context, client clientset.Interface, replicas int32, +// createDeployment create a deployment with 1 replica for given pvcs and node +// selector. +func createDeployment(ctx context.Context, client clientset.Interface, replicas int32, podLabels map[string]string, nodeSelector map[string]string, namespace string, - pvclaims []*v1.PersistentVolumeClaim, command string, isPrivileged bool, image string) *appsv1.Deployment { + pvclaims []*v1.PersistentVolumeClaim, command string, isPrivileged bool, image string) (*appsv1.Deployment, error) { + if len(command) == 0 { + command = "trap exit TERM; while true; do sleep 1; done" + } zero := int64(0) deploymentName := "deployment-" + string(uuid.NewUUID()) deploymentSpec := &appsv1.Deployment{ @@ -4058,19 +3985,6 @@ func getDeploymentSpec(ctx context.Context, client clientset.Interface, replicas if nodeSelector != nil { deploymentSpec.Spec.Template.Spec.NodeSelector = nodeSelector } - return deploymentSpec -} - -// createDeployment create a deployment with 1 replica for given pvcs and node -// selector. -func createDeployment(ctx context.Context, client clientset.Interface, replicas int32, - podLabels map[string]string, nodeSelector map[string]string, namespace string, - pvclaims []*v1.PersistentVolumeClaim, command string, isPrivileged bool, image string) (*appsv1.Deployment, error) { - if len(command) == 0 { - command = "trap exit TERM; while true; do sleep 1; done" - } - deploymentSpec := getDeploymentSpec(ctx, client, replicas, podLabels, nodeSelector, namespace, - pvclaims, command, isPrivileged, image) deployment, err := client.AppsV1().Deployments(namespace).Create(ctx, deploymentSpec, metav1.CreateOptions{}) if err != nil { return nil, fmt.Errorf("deployment %q Create API error: %v", deploymentSpec.Name, err) @@ -4719,24 +4633,17 @@ which PV is provisioned. */ func verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx context.Context, client clientset.Interface, statefulset *appsv1.StatefulSet, namespace string, - allowedTopologies []v1.TopologySelectorLabelRequirement, - parallelStatefulSetCreation bool, isMultiVcSetup bool) error { + allowedTopologies []v1.TopologySelectorLabelRequirement, parallelStatefulSetCreation bool) { allowedTopologiesMap := createAllowedTopologiesMap(allowedTopologies) var ssPodsBeforeScaleDown *v1.PodList - var err error - if parallelStatefulSetCreation { ssPodsBeforeScaleDown = GetListOfPodsInSts(client, statefulset) } else { ssPodsBeforeScaleDown = fss.GetPodList(client, statefulset) } - for _, sspod := range ssPodsBeforeScaleDown.Items { - _, err = client.CoreV1().Pods(namespace).Get(ctx, sspod.Name, metav1.GetOptions{}) - if err != nil { - return err - } - + _, err := client.CoreV1().Pods(namespace).Get(ctx, sspod.Name, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) for _, volumespec := range sspod.Spec.Volumes { if volumespec.PersistentVolumeClaim != nil { // get pv details @@ -4746,60 +4653,36 @@ func verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx context.Cont ginkgo.By("Verifying PV node affinity details") res, err := verifyVolumeTopologyForLevel5(pv, allowedTopologiesMap) if res { - framework.Logf("PV %s node affinity details lie in the specified allowed "+ - "topologies of Storage Class", pv.Name) - } - if !res { - return fmt.Errorf("PV %s node affinity details are not in the specified allowed "+ - "topologies of Storage Class", pv.Name) - } - if err != nil { - return err + framework.Logf("PV %s node affinity details lies in the specified allowed topologies of Storage Class", pv.Name) } + gomega.Expect(res).To(gomega.BeTrue(), "PV %s node affinity details is not in the "+ + "specified allowed topologies of Storage Class", pv.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) // fetch node details nodeList, err := fnodes.GetReadySchedulableNodes(client) - if err != nil { - return err - } - if len(nodeList.Items) <= 0 { - return fmt.Errorf("unable to find ready and schedulable Node") + framework.ExpectNoError(err, "Unable to find ready and schedulable Node") + if !(len(nodeList.Items) > 0) { + framework.Failf("Unable to find ready and schedulable Node") } - // verify pod is running on appropriate nodes ginkgo.By("Verifying If Pods are running on appropriate nodes as mentioned in SC") res, err = verifyPodLocationLevel5(&sspod, nodeList, allowedTopologiesMap) if res { - framework.Logf("Pod %v is running on an appropriate node as specified in the "+ - "allowed topologies of Storage Class", sspod.Name) - } - if !res { - return fmt.Errorf("pod %v is not running on an appropriate node as specified "+ - "in the allowed topologies of Storage Class", sspod.Name) - } - if err != nil { - return err + framework.Logf("Pod %v is running on appropriate node as specified "+ + "in the allowed topolgies of Storage Class", sspod.Name) } + gomega.Expect(res).To(gomega.BeTrue(), "Pod %v is not running on appropriate node "+ + "as specified in allowed topolgies of Storage Class", sspod.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) // Verify the attached volume match the one in CNS cache - if !isMultiVcSetup { - err := verifyVolumeMetadataInCNS(&e2eVSphere, pv.Spec.CSI.VolumeHandle, - volumespec.PersistentVolumeClaim.ClaimName, pv.ObjectMeta.Name, sspod.Name) - if err != nil { - return err - } - } else { - err := verifyVolumeMetadataInCNSForMultiVC(&multiVCe2eVSphere, pv.Spec.CSI.VolumeHandle, - volumespec.PersistentVolumeClaim.ClaimName, pv.ObjectMeta.Name, sspod.Name) - if err != nil { - return err - } - } + error := verifyVolumeMetadataInCNS(&e2eVSphere, pv.Spec.CSI.VolumeHandle, + volumespec.PersistentVolumeClaim.ClaimName, pv.ObjectMeta.Name, sspod.Name) + gomega.Expect(error).NotTo(gomega.HaveOccurred()) } } } - - return nil } /* @@ -4877,29 +4760,20 @@ func scaleStatefulSetPods(c clientset.Interface, ss *appsv1.StatefulSet, count i } /* -scaleDownStatefulSetPod util is used to perform scale down operation on StatefulSert Pods -and later verifies that after scale down operation volume gets detached from the node -and returns nil if no error found +scaleDownStatefulSetPod is a utility method which is used to scale down the count of StatefulSet replicas. */ func scaleDownStatefulSetPod(ctx context.Context, client clientset.Interface, - statefulset *appsv1.StatefulSet, namespace string, replicas int32, parallelStatefulSetCreation bool, - isMultiVcSetup bool) error { + statefulset *appsv1.StatefulSet, namespace string, replicas int32, parallelStatefulSetCreation bool) { ginkgo.By(fmt.Sprintf("Scaling down statefulsets to number of Replica: %v", replicas)) var ssPodsAfterScaleDown *v1.PodList - var err error - if parallelStatefulSetCreation { _, scaledownErr := scaleStatefulSetPods(client, statefulset, replicas) - if scaledownErr != nil { - return scaledownErr - } + gomega.Expect(scaledownErr).NotTo(gomega.HaveOccurred()) fss.WaitForStatusReadyReplicas(client, statefulset, replicas) ssPodsAfterScaleDown = GetListOfPodsInSts(client, statefulset) } else { _, scaledownErr := fss.Scale(client, statefulset, replicas) - if scaledownErr != nil { - return scaledownErr - } + gomega.Expect(scaledownErr).NotTo(gomega.HaveOccurred()) fss.WaitForStatusReadyReplicas(client, statefulset, replicas) ssPodsAfterScaleDown = fss.GetPodList(client, statefulset) } @@ -4907,124 +4781,75 @@ func scaleDownStatefulSetPod(ctx context.Context, client clientset.Interface, // After scale down, verify vSphere volumes are detached from deleted pods ginkgo.By("Verify Volumes are detached from Nodes after Statefulsets is scaled down") for _, sspod := range ssPodsAfterScaleDown.Items { - _, err = client.CoreV1().Pods(namespace).Get(ctx, sspod.Name, metav1.GetOptions{}) + _, err := client.CoreV1().Pods(namespace).Get(ctx, sspod.Name, metav1.GetOptions{}) if err != nil { - if apierrors.IsNotFound(err) { - for _, volumespec := range sspod.Spec.Volumes { - if volumespec.PersistentVolumeClaim != nil { - pv := getPvFromClaim(client, statefulset.Namespace, volumespec.PersistentVolumeClaim.ClaimName) - if !isMultiVcSetup { - isDiskDetached, detachErr := e2eVSphere.waitForVolumeDetachedFromNode( - client, pv.Spec.CSI.VolumeHandle, sspod.Spec.NodeName) - if detachErr != nil { - return detachErr - } - if !isDiskDetached { - return fmt.Errorf("volume %q is not detached from the node %q", - pv.Spec.CSI.VolumeHandle, sspod.Spec.NodeName) - } - } else { - isDiskDetached, detachErr := multiVCe2eVSphere.waitForVolumeDetachedFromNodeInMultiVC( - client, pv.Spec.CSI.VolumeHandle, sspod.Spec.NodeName) - if detachErr != nil { - return detachErr - } - if !isDiskDetached { - return fmt.Errorf("volume %q is not detached from the node %q", - pv.Spec.CSI.VolumeHandle, sspod.Spec.NodeName) - } - } - } + gomega.Expect(apierrors.IsNotFound(err), gomega.BeTrue()) + for _, volumespec := range sspod.Spec.Volumes { + if volumespec.PersistentVolumeClaim != nil { + pv := getPvFromClaim(client, statefulset.Namespace, volumespec.PersistentVolumeClaim.ClaimName) + isDiskDetached, err := e2eVSphere.waitForVolumeDetachedFromNode( + client, pv.Spec.CSI.VolumeHandle, sspod.Spec.NodeName) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(isDiskDetached).To(gomega.BeTrue(), + fmt.Sprintf("Volume %q is not detached from the node %q", + pv.Spec.CSI.VolumeHandle, sspod.Spec.NodeName)) } - } else { - return err } } } // After scale down, verify the attached volumes match those in CNS Cache for _, sspod := range ssPodsAfterScaleDown.Items { - _, err = client.CoreV1().Pods(namespace).Get(ctx, sspod.Name, metav1.GetOptions{}) - if err != nil { - return err - } + _, err := client.CoreV1().Pods(namespace).Get(ctx, sspod.Name, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) for _, volumespec := range sspod.Spec.Volumes { if volumespec.PersistentVolumeClaim != nil { - if !isMultiVcSetup { - pv := getPvFromClaim(client, statefulset.Namespace, volumespec.PersistentVolumeClaim.ClaimName) - err := verifyVolumeMetadataInCNS(&e2eVSphere, pv.Spec.CSI.VolumeHandle, - volumespec.PersistentVolumeClaim.ClaimName, pv.ObjectMeta.Name, sspod.Name) - if err != nil { - return err - } - } else { - pv := getPvFromClaim(client, statefulset.Namespace, volumespec.PersistentVolumeClaim.ClaimName) - err := verifyVolumeMetadataInCNSForMultiVC(&multiVCe2eVSphere, pv.Spec.CSI.VolumeHandle, - volumespec.PersistentVolumeClaim.ClaimName, pv.ObjectMeta.Name, sspod.Name) - if err != nil { - return err - } - } + pv := getPvFromClaim(client, statefulset.Namespace, volumespec.PersistentVolumeClaim.ClaimName) + err := verifyVolumeMetadataInCNS(&e2eVSphere, pv.Spec.CSI.VolumeHandle, + volumespec.PersistentVolumeClaim.ClaimName, pv.ObjectMeta.Name, sspod.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) } } } - return nil } /* -scaleUpStatefulSetPod util is used to perform scale up operation on StatefulSert Pods -and later verifies that after scale up operation volume get successfully attached to the node -and returns nil if no error found +scaleUpStatefulSetPod is a utility method which is used to scale up the count of StatefulSet replicas. */ func scaleUpStatefulSetPod(ctx context.Context, client clientset.Interface, - statefulset *appsv1.StatefulSet, namespace string, replicas int32, - parallelStatefulSetCreation bool, isMultiVcSetup bool) error { + statefulset *appsv1.StatefulSet, namespace string, replicas int32, parallelStatefulSetCreation bool) { ginkgo.By(fmt.Sprintf("Scaling up statefulsets to number of Replica: %v", replicas)) var ssPodsAfterScaleUp *v1.PodList - var err error - if parallelStatefulSetCreation { _, scaleupErr := scaleStatefulSetPods(client, statefulset, replicas) - if scaleupErr != nil { - return scaleupErr - } + gomega.Expect(scaleupErr).NotTo(gomega.HaveOccurred()) fss.WaitForStatusReplicas(client, statefulset, replicas) fss.WaitForStatusReadyReplicas(client, statefulset, replicas) ssPodsAfterScaleUp = GetListOfPodsInSts(client, statefulset) - if len(ssPodsAfterScaleUp.Items) == 0 { - return fmt.Errorf("unable to get list of Pods from the Statefulset: %v", statefulset.Name) - } - if len(ssPodsAfterScaleUp.Items) != int(replicas) { - return fmt.Errorf("number of Pods in the statefulset should match with number of replicas") - } + gomega.Expect(ssPodsAfterScaleUp.Items).NotTo(gomega.BeEmpty(), + fmt.Sprintf("Unable to get list of Pods from the Statefulset: %v", statefulset.Name)) + gomega.Expect(len(ssPodsAfterScaleUp.Items) == int(replicas)).To(gomega.BeTrue(), + "Number of Pods in the statefulset should match with number of replicas") } else { _, scaleupErr := fss.Scale(client, statefulset, replicas) - if scaleupErr != nil { - return scaleupErr - } + gomega.Expect(scaleupErr).NotTo(gomega.HaveOccurred()) fss.WaitForStatusReplicas(client, statefulset, replicas) fss.WaitForStatusReadyReplicas(client, statefulset, replicas) ssPodsAfterScaleUp = fss.GetPodList(client, statefulset) - if len(ssPodsAfterScaleUp.Items) == 0 { - return fmt.Errorf("unable to get list of Pods from the Statefulset: %v", statefulset.Name) - } - if len(ssPodsAfterScaleUp.Items) != int(replicas) { - return fmt.Errorf("number of Pods in the statefulset should match with number of replicas") - } + gomega.Expect(ssPodsAfterScaleUp.Items).NotTo(gomega.BeEmpty(), + fmt.Sprintf("Unable to get list of Pods from the Statefulset: %v", statefulset.Name)) + gomega.Expect(len(ssPodsAfterScaleUp.Items) == int(replicas)).To(gomega.BeTrue(), + "Number of Pods in the statefulset should match with number of replicas") } // After scale up, verify all vSphere volumes are attached to node VMs. ginkgo.By("Verify all volumes are attached to Nodes after Statefulsets is scaled up") for _, sspod := range ssPodsAfterScaleUp.Items { - err = fpod.WaitTimeoutForPodReadyInNamespace(client, sspod.Name, statefulset.Namespace, pollTimeout) - if err != nil { - return err - } + err := fpod.WaitTimeoutForPodReadyInNamespace(client, sspod.Name, statefulset.Namespace, pollTimeout) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) pod, err := client.CoreV1().Pods(namespace).Get(ctx, sspod.Name, metav1.GetOptions{}) - if err != nil { - return err - } + gomega.Expect(err).NotTo(gomega.HaveOccurred()) for _, volumespec := range pod.Spec.Volumes { if volumespec.PersistentVolumeClaim != nil { pv := getPvFromClaim(client, statefulset.Namespace, volumespec.PersistentVolumeClaim.ClaimName) @@ -5039,53 +4864,20 @@ func scaleUpStatefulSetPod(ctx context.Context, client clientset.Interface, } else { annotations := pod.Annotations vmUUID, exists = annotations[vmUUIDLabel] - if !exists { - return fmt.Errorf("pod doesn't have %s annotation", vmUUIDLabel) - } - if !isMultiVcSetup { - _, err := e2eVSphere.getVMByUUID(ctx, vmUUID) - if err != nil { - return err - } - } else { - _, err := multiVCe2eVSphere.getVMByUUIDForMultiVC(ctx, vmUUID) - if err != nil { - return err - } - } - } - if !isMultiVcSetup { - isDiskAttached, err := e2eVSphere.isVolumeAttachedToVM(client, pv.Spec.CSI.VolumeHandle, vmUUID) - if err != nil { - return err - } - if !isDiskAttached { - return fmt.Errorf("disk is not attached to the node") - } - err = verifyVolumeMetadataInCNS(&e2eVSphere, pv.Spec.CSI.VolumeHandle, - volumespec.PersistentVolumeClaim.ClaimName, pv.ObjectMeta.Name, sspod.Name) - if err != nil { - return err - } - } else { - isDiskAttached, err := multiVCe2eVSphere.verifyVolumeIsAttachedToVMInMultiVC(client, - pv.Spec.CSI.VolumeHandle, vmUUID) - if err != nil { - return err - } - if !isDiskAttached { - return fmt.Errorf("disk is not attached to the node") - } - err = verifyVolumeMetadataInCNSForMultiVC(&multiVCe2eVSphere, pv.Spec.CSI.VolumeHandle, - volumespec.PersistentVolumeClaim.ClaimName, pv.ObjectMeta.Name, sspod.Name) - if err != nil { - return err - } + gomega.Expect(exists).To(gomega.BeTrue(), fmt.Sprintf("Pod doesn't have %s annotation", vmUUIDLabel)) + _, err := e2eVSphere.getVMByUUID(ctx, vmUUID) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) } + isDiskAttached, err := e2eVSphere.isVolumeAttachedToVM(client, pv.Spec.CSI.VolumeHandle, vmUUID) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Disk is not attached to the node") + gomega.Expect(isDiskAttached).To(gomega.BeTrue(), "Disk is not attached") + ginkgo.By("After scale up, verify the attached volumes match those in CNS Cache") + err = verifyVolumeMetadataInCNS(&e2eVSphere, pv.Spec.CSI.VolumeHandle, + volumespec.PersistentVolumeClaim.ClaimName, pv.ObjectMeta.Name, sspod.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) } } } - return nil } /* @@ -5190,30 +4982,20 @@ PV is provisioned. */ func verifyPVnodeAffinityAndPODnodedetailsForDeploymentSetsLevel5(ctx context.Context, client clientset.Interface, deployment *appsv1.Deployment, namespace string, - allowedTopologies []v1.TopologySelectorLabelRequirement, - parallelDeplCreation bool, isMultiVcSetup bool) error { + allowedTopologies []v1.TopologySelectorLabelRequirement, parallelDeplCreation bool) { allowedTopologiesMap := createAllowedTopologiesMap(allowedTopologies) var pods *v1.PodList var err error - if parallelDeplCreation { pods, err = GetPodsForMultipleDeployment(client, deployment) - if err != nil { - return err - } + gomega.Expect(err).NotTo(gomega.HaveOccurred()) } else { pods, err = fdep.GetPodsForDeployment(client, deployment) - if err != nil { - return err - } + gomega.Expect(err).NotTo(gomega.HaveOccurred()) } - for _, sspod := range pods.Items { _, err := client.CoreV1().Pods(namespace).Get(ctx, sspod.Name, metav1.GetOptions{}) - if err != nil { - return err - } - + gomega.Expect(err).NotTo(gomega.HaveOccurred()) for _, volumespec := range sspod.Spec.Volumes { if volumespec.PersistentVolumeClaim != nil { // get pv details @@ -5223,130 +5005,87 @@ func verifyPVnodeAffinityAndPODnodedetailsForDeploymentSetsLevel5(ctx context.Co ginkgo.By("Verifying PV node affinity details") res, err := verifyVolumeTopologyForLevel5(pv, allowedTopologiesMap) if res { - framework.Logf("PV %s node affinity details lie in the specified allowed "+ - "topologies of Storage Class", pv.Name) - } - if !res { - return fmt.Errorf("PV %s node affinity details are not in the specified allowed "+ - "topologies of Storage Class", pv.Name) - } - if err != nil { - return err + framework.Logf("PV %s node affinity details lies in the specified allowed topologies of Storage Class", pv.Name) } + gomega.Expect(res).To(gomega.BeTrue(), "PV %s node affinity details is not in the "+ + "specified allowed topologies of Storage Class", pv.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) // fetch node details nodeList, err := fnodes.GetReadySchedulableNodes(client) - if err != nil { - return err - } - if len(nodeList.Items) <= 0 { - return fmt.Errorf("unable to find ready and schedulable Node") + framework.ExpectNoError(err, "Unable to find ready and schedulable Node") + if !(len(nodeList.Items) > 0) { + framework.Failf("Unable to find ready and schedulable Node") } - // verify pod is running on appropriate nodes ginkgo.By("Verifying If Pods are running on appropriate nodes as mentioned in SC") res, err = verifyPodLocationLevel5(&sspod, nodeList, allowedTopologiesMap) if res { - framework.Logf("Pod %v is running on an appropriate node as specified in the "+ - "allowed topologies of Storage Class", sspod.Name) - } - if !res { - return fmt.Errorf("pod %v is not running on an appropriate node as specified in the "+ - "allowed topologies of Storage Class", sspod.Name) - } - if err != nil { - return err + framework.Logf("Pod %v is running on appropriate node as specified in the "+ + "allowed topolgies of Storage Class", sspod.Name) } + gomega.Expect(res).To(gomega.BeTrue(), "Pod %v is not running on appropriate node "+ + "as specified in allowed topolgies of Storage Class", sspod.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) // Verify the attached volume match the one in CNS cache - if !isMultiVcSetup { - err := verifyVolumeMetadataInCNS(&e2eVSphere, pv.Spec.CSI.VolumeHandle, - volumespec.PersistentVolumeClaim.ClaimName, pv.ObjectMeta.Name, sspod.Name) - if err != nil { - return err - } - } else { - err := verifyVolumeMetadataInCNSForMultiVC(&multiVCe2eVSphere, pv.Spec.CSI.VolumeHandle, - volumespec.PersistentVolumeClaim.ClaimName, pv.ObjectMeta.Name, sspod.Name) - if err != nil { - return err - } - } + error := verifyVolumeMetadataInCNS(&e2eVSphere, pv.Spec.CSI.VolumeHandle, + volumespec.PersistentVolumeClaim.ClaimName, pv.ObjectMeta.Name, sspod.Name) + gomega.Expect(error).NotTo(gomega.HaveOccurred()) } } } - - return nil } /* For Standalone Pod -verifyPVnodeAffinityAndPODnodedetailsForStandalonePodLevel5 -verifyPVnodeAffinityAndPODnodedetailsFoStandalonePodLevel5 for Standalone Pod verifies that PV +verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5 for Standalone Pod verifies that PV node Affinity rules should match the topology constraints specified in the storage class. Also it verifies that a pod is scheduled on a node that belongs to the topology on which PV is provisioned. */ -func verifyPVnodeAffinityAndPODnodedetailsForStandalonePodLevel5(ctx context.Context, +func verifyPVnodeAffinityAndPODnodedetailsFoStandalonePodLevel5(ctx context.Context, client clientset.Interface, pod *v1.Pod, namespace string, - allowedTopologies []v1.TopologySelectorLabelRequirement, isMultiVcSetup bool) error { + allowedTopologies []v1.TopologySelectorLabelRequirement) { allowedTopologiesMap := createAllowedTopologiesMap(allowedTopologies) for _, volumespec := range pod.Spec.Volumes { if volumespec.PersistentVolumeClaim != nil { // get pv details pv := getPvFromClaim(client, pod.Namespace, volumespec.PersistentVolumeClaim.ClaimName) - if pv == nil { - return fmt.Errorf("failed to get PV for claim: %s", volumespec.PersistentVolumeClaim.ClaimName) - } // verify pv node affinity details as specified on SC ginkgo.By("Verifying PV node affinity details") res, err := verifyVolumeTopologyForLevel5(pv, allowedTopologiesMap) - if err != nil { - return fmt.Errorf("error verifying PV node affinity: %v", err) - } - if !res { - return fmt.Errorf("PV %s node affinity details are not in the specified allowed "+ - "topologies of Storage Class", pv.Name) + if res { + framework.Logf("PV %s node affinity details lies in the specified allowed topologies of Storage Class", pv.Name) } + gomega.Expect(res).To(gomega.BeTrue(), "PV %s node affinity details is not in the specified "+ + "allowed topologies of Storage Class", pv.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) // fetch node details nodeList, err := fnodes.GetReadySchedulableNodes(client) - if err != nil { - return fmt.Errorf("error getting ready and schedulable nodes: %v", err) - } + framework.ExpectNoError(err, "Unable to find ready and schedulable Node") if !(len(nodeList.Items) > 0) { - return errors.New("no ready and schedulable nodes found") + framework.Failf("Unable to find ready and schedulable Node") } - // verify pod is running on appropriate nodes ginkgo.By("Verifying If Pods are running on appropriate nodes as mentioned in SC") res, err = verifyPodLocationLevel5(pod, nodeList, allowedTopologiesMap) - if err != nil { - return fmt.Errorf("error verifying pod location: %v", err) - } - if !res { - return fmt.Errorf("pod %v is not running on appropriate node as specified in allowed "+ - "topologies of Storage Class", pod.Name) + if res { + framework.Logf("Pod %v is running on appropriate node as specified in the allowed "+ + "topolgies of Storage Class", pod.Name) } + gomega.Expect(res).To(gomega.BeTrue(), "Pod %v is not running on appropriate node as "+ + "specified in allowed topolgies of Storage Class", pod.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) - // Verify the attached volume matches the one in CNS cache - if !isMultiVcSetup { - err := verifyVolumeMetadataInCNS(&e2eVSphere, pv.Spec.CSI.VolumeHandle, - volumespec.PersistentVolumeClaim.ClaimName, pv.ObjectMeta.Name, pod.Name) - if err != nil { - return fmt.Errorf("error verifying volume metadata in CNS: %v", err) - } - } else { - err := verifyVolumeMetadataInCNSForMultiVC(&multiVCe2eVSphere, pv.Spec.CSI.VolumeHandle, - volumespec.PersistentVolumeClaim.ClaimName, pv.ObjectMeta.Name, pod.Name) - if err != nil { - return fmt.Errorf("error verifying volume metadata in CNS for multi-VC: %v", err) - } - } + // Verify the attached volume match the one in CNS cache + error := verifyVolumeMetadataInCNS(&e2eVSphere, pv.Spec.CSI.VolumeHandle, + volumespec.PersistentVolumeClaim.ClaimName, pv.ObjectMeta.Name, pod.Name) + gomega.Expect(error).NotTo(gomega.HaveOccurred()) } } - return nil } func getPersistentVolumeSpecWithStorageClassFCDNodeSelector(volumeHandle string, @@ -5406,7 +5145,6 @@ func getPersistentVolumeSpecWithStorageClassFCDNodeSelector(volumeHandle string, func getNodeSelectorTerms(allowedTopologies []v1.TopologySelectorLabelRequirement) []v1.NodeSelectorTerm { var nodeSelectorRequirements []v1.NodeSelectorRequirement var nodeSelectorTerms []v1.NodeSelectorTerm - rackTopology := allowedTopologies[len(allowedTopologies)-1] for i := 0; i < len(allowedTopologies)-1; i++ { topologySelector := allowedTopologies[i] @@ -5416,7 +5154,7 @@ func getNodeSelectorTerms(allowedTopologies []v1.TopologySelectorLabelRequiremen nodeSelectorRequirement.Values = topologySelector.Values nodeSelectorRequirements = append(nodeSelectorRequirements, nodeSelectorRequirement) } - + rackTopology := allowedTopologies[len(allowedTopologies)-1] for i := 0; i < len(rackTopology.Values); i++ { var nodeSelectorTerm v1.NodeSelectorTerm var nodeSelectorRequirement v1.NodeSelectorRequirement @@ -5426,7 +5164,6 @@ func getNodeSelectorTerms(allowedTopologies []v1.TopologySelectorLabelRequiremen nodeSelectorTerm.MatchExpressions = append(nodeSelectorRequirements, nodeSelectorRequirement) nodeSelectorTerms = append(nodeSelectorTerms, nodeSelectorTerm) } - return nodeSelectorTerms } @@ -5458,6 +5195,80 @@ func createKubernetesClientFromConfig(kubeConfigPath string) (clientset.Interfac return client, nil } +// getVolumeSnapshotClassSpec returns a spec for the volume snapshot class +func getVolumeSnapshotClassSpec(deletionPolicy snapc.DeletionPolicy, + parameters map[string]string) *snapc.VolumeSnapshotClass { + var volumesnapshotclass = &snapc.VolumeSnapshotClass{ + TypeMeta: metav1.TypeMeta{ + Kind: "VolumeSnapshotClass", + }, + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "volumesnapshot-", + }, + Driver: e2evSphereCSIDriverName, + DeletionPolicy: deletionPolicy, + } + + volumesnapshotclass.Parameters = parameters + return volumesnapshotclass +} + +// getVolumeSnapshotSpec returns a spec for the volume snapshot +func getVolumeSnapshotSpec(namespace string, snapshotclassname string, pvcName string) *snapc.VolumeSnapshot { + var volumesnapshotSpec = &snapc.VolumeSnapshot{ + TypeMeta: metav1.TypeMeta{ + Kind: "VolumeSnapshot", + }, + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "snapshot-", + Namespace: namespace, + }, + Spec: snapc.VolumeSnapshotSpec{ + VolumeSnapshotClassName: &snapshotclassname, + Source: snapc.VolumeSnapshotSource{ + PersistentVolumeClaimName: &pvcName, + }, + }, + } + return volumesnapshotSpec +} + +// waitForVolumeSnapshotReadyToUse waits for the volume's snapshot to be in ReadyToUse +func waitForVolumeSnapshotReadyToUse(client snapclient.Clientset, ctx context.Context, namespace string, + name string) (*snapc.VolumeSnapshot, error) { + var volumeSnapshot *snapc.VolumeSnapshot + var err error + waitErr := wait.PollImmediate(poll, pollTimeout, func() (bool, error) { + volumeSnapshot, err = client.SnapshotV1().VolumeSnapshots(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return false, fmt.Errorf("error fetching volumesnapshot details : %v", err) + } + if volumeSnapshot.Status != nil && *volumeSnapshot.Status.ReadyToUse { + return true, nil + } + return false, nil + }) + return volumeSnapshot, waitErr +} + +// waitForVolumeSnapshotContentToBeDeleted wait till the volume snapshot content is deleted +func waitForVolumeSnapshotContentToBeDeleted(client snapclient.Clientset, ctx context.Context, + name string) error { + var err error + waitErr := wait.PollImmediate(poll, pollTimeout, func() (bool, error) { + _, err = client.SnapshotV1().VolumeSnapshotContents().Get(ctx, name, metav1.GetOptions{}) + if err != nil { + if strings.Contains(err.Error(), "not found") { + return true, nil + } else { + return false, fmt.Errorf("error fetching volumesnapshotcontent details : %v", err) + } + } + return false, nil + }) + return waitErr +} + // getK8sMasterNodeIPWhereControllerLeaderIsRunning fetches the master node IP // where controller is running func getK8sMasterNodeIPWhereContainerLeaderIsRunning(ctx context.Context, @@ -5620,9 +5431,53 @@ func getMasterIpFromMasterNodeName(ctx context.Context, client clientset.Interfa } } +// getVolumeSnapshotContentSpec returns a spec for the volume snapshot content +func getVolumeSnapshotContentSpec(deletionPolicy snapc.DeletionPolicy, snapshotHandle string, + futureSnapshotName string, namespace string) *snapc.VolumeSnapshotContent { + var volumesnapshotContentSpec = &snapc.VolumeSnapshotContent{ + TypeMeta: metav1.TypeMeta{ + Kind: "VolumeSnapshotContent", + }, + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "snapshotcontent-", + }, + Spec: snapc.VolumeSnapshotContentSpec{ + DeletionPolicy: deletionPolicy, + Driver: e2evSphereCSIDriverName, + Source: snapc.VolumeSnapshotContentSource{ + SnapshotHandle: &snapshotHandle, + }, + VolumeSnapshotRef: v1.ObjectReference{ + Name: futureSnapshotName, + Namespace: namespace, + }, + }, + } + return volumesnapshotContentSpec +} + +// getVolumeSnapshotSpecByName returns a spec for the volume snapshot by name +func getVolumeSnapshotSpecByName(namespace string, snapshotName string, + snapshotcontentname string) *snapc.VolumeSnapshot { + var volumesnapshotSpec = &snapc.VolumeSnapshot{ + TypeMeta: metav1.TypeMeta{ + Kind: "VolumeSnapshot", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: snapshotName, + Namespace: namespace, + }, + Spec: snapc.VolumeSnapshotSpec{ + Source: snapc.VolumeSnapshotSource{ + VolumeSnapshotContentName: &snapshotcontentname, + }, + }, + } + return volumesnapshotSpec +} + func createParallelStatefulSets(client clientset.Interface, namespace string, statefulset *appsv1.StatefulSet, replicas int32, wg *sync.WaitGroup) { - defer ginkgo.GinkgoRecover() defer wg.Done() ginkgo.By("Creating statefulset") ctx, cancel := context.WithCancel(context.Background()) @@ -5638,12 +5493,11 @@ func createParallelStatefulSetSpec(namespace string, no_of_sts int, replicas int var statefulset *appsv1.StatefulSet for i := 0; i < no_of_sts; i++ { - scName := defaultNginxStorageClassName statefulset = GetStatefulSetFromManifest(namespace) statefulset.Name = "thread-" + strconv.Itoa(i) + "-" + statefulset.Name statefulset.Spec.PodManagementPolicy = appsv1.ParallelPodManagement statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &scName + Annotations["volume.beta.kubernetes.io/storage-class"] = defaultNginxStorageClassName statefulset.Spec.Replicas = &replicas stss = append(stss, statefulset) } @@ -5651,7 +5505,7 @@ func createParallelStatefulSetSpec(namespace string, no_of_sts int, replicas int } func createMultiplePVCsInParallel(ctx context.Context, client clientset.Interface, namespace string, - storageclass *storagev1.StorageClass, count int, pvclaimlabels map[string]string) []*v1.PersistentVolumeClaim { + storageclass *storagev1.StorageClass, count int) []*v1.PersistentVolumeClaim { var pvclaims []*v1.PersistentVolumeClaim for i := 0; i < count; i++ { pvclaim, err := createPVC(client, namespace, nil, "", storageclass, "") @@ -5722,6 +5576,49 @@ func deleteCsiControllerPodWhereLeaderIsRunning(ctx context.Context, return nil } +// getPersistentVolumeClaimSpecWithDatasource return the PersistentVolumeClaim +// spec with specified storage class. +func getPersistentVolumeClaimSpecWithDatasource(namespace string, ds string, storageclass *storagev1.StorageClass, + pvclaimlabels map[string]string, accessMode v1.PersistentVolumeAccessMode, + datasourceName string, snapshotapigroup string) *v1.PersistentVolumeClaim { + disksize := diskSize + if ds != "" { + disksize = ds + } + if accessMode == "" { + // If accessMode is not specified, set the default accessMode. + accessMode = v1.ReadWriteOnce + } + claim := &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "pvc-", + Namespace: namespace, + }, + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{ + accessMode, + }, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceName(v1.ResourceStorage): resource.MustParse(disksize), + }, + }, + StorageClassName: &(storageclass.Name), + DataSource: &v1.TypedLocalObjectReference{ + APIGroup: &snapshotapigroup, + Kind: "VolumeSnapshot", + Name: datasourceName, + }, + }, + } + + if pvclaimlabels != nil { + claim.Labels = pvclaimlabels + } + + return claim +} + // get topology cluster lists func ListTopologyClusterNames(topologyCluster string) []string { topologyClusterList := strings.Split(topologyCluster, ",") @@ -5788,21 +5685,41 @@ func powerOffEsxiHostByCluster(ctx context.Context, vs *vSphere, clusterName str return powerOffHostsList } +// getVolumeSnapshotSpecWithoutSC returns a spec for the volume snapshot +func getVolumeSnapshotSpecWithoutSC(namespace string, pvcName string) *snapc.VolumeSnapshot { + var volumesnapshotSpec = &snapc.VolumeSnapshot{ + TypeMeta: metav1.TypeMeta{ + Kind: "VolumeSnapshot", + }, + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "snapshot-", + Namespace: namespace, + }, + Spec: snapc.VolumeSnapshotSpec{ + Source: snapc.VolumeSnapshotSource{ + PersistentVolumeClaimName: &pvcName, + }, + }, + } + return volumesnapshotSpec +} + // waitForPvcToBeDeleted waits by polling for a particular pvc to be deleted in a namespace func waitForPvcToBeDeleted(ctx context.Context, client clientset.Interface, pvcName string, namespace string) error { + var pvc *v1.PersistentVolumeClaim waitErr := wait.PollImmediate(poll, pollTimeout, func() (bool, error) { - _, err := client.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{}) + pvc, err := client.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{}) if err != nil { if strings.Contains(err.Error(), "not found") { - framework.Logf("PVC is deleted: %v", pvcName) return true, nil } else { return false, fmt.Errorf("pvc %s is still not deleted in"+ - "namespace %s with err: %v", pvcName, namespace, err) + "namespace %s with err: %v", pvc.Name, namespace, err) } } return false, nil }) + framework.Logf("Status of pvc is: %v", pvc.Status.Phase) return waitErr } @@ -5834,33 +5751,29 @@ func waitForEventWithReason(client clientset.Interface, namespace string, } // stopCSIPods function stops all the running csi pods -func stopCSIPods(ctx context.Context, client clientset.Interface, namespace string) (bool, error) { +func stopCSIPods(ctx context.Context, client clientset.Interface) (bool, error) { collectPodLogs(ctx, client, csiSystemNamespace) isServiceStopped := false err := updateDeploymentReplicawithWait(client, 0, vSphereCSIControllerPodNamePrefix, - namespace) + csiSystemNamespace) gomega.Expect(err).NotTo(gomega.HaveOccurred()) isServiceStopped = true return isServiceStopped, err } // startCSIPods function starts the csi pods and waits till all the pods comes up -func startCSIPods(ctx context.Context, client clientset.Interface, csiReplicas int32, - namespace string) (bool, error) { +func startCSIPods(ctx context.Context, client clientset.Interface, csiReplicas int32) (bool, error) { ignoreLabels := make(map[string]string) err := updateDeploymentReplicawithWait(client, csiReplicas, vSphereCSIControllerPodNamePrefix, - namespace) - if err != nil { - return true, err - } + csiSystemNamespace) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) // Wait for the CSI Pods to be up and Running - list_of_pods, err := fpod.GetPodsInNamespace(client, namespace, ignoreLabels) - if err != nil { - return true, err - } + list_of_pods, err := fpod.GetPodsInNamespace(client, csiSystemNamespace, ignoreLabels) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) num_csi_pods := len(list_of_pods) - err = fpod.WaitForPodsRunningReady(client, namespace, int32(num_csi_pods), 0, + err = fpod.WaitForPodsRunningReady(client, csiSystemNamespace, int32(num_csi_pods), 0, pollTimeout, ignoreLabels) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) isServiceStopped := false return isServiceStopped, err } @@ -6112,13 +6025,8 @@ func getCSIPodWhereListVolumeResponseIsPresent(ctx context.Context, csiPods, err := fpod.GetPodsInNamespace(client, csiSystemNamespace, ignoreLabels) gomega.Expect(err).NotTo(gomega.HaveOccurred()) var k8sMasterIP string - if vanillaCluster { - k8sMasterIPs := getK8sMasterIPs(ctx, client) - k8sMasterIP = k8sMasterIPs[0] - } else { - k8sMasterIP = GetAndExpectStringEnvVar(svcMasterIP) - } - + k8sMasterIPs := getK8sMasterIPs(ctx, client) + k8sMasterIP = k8sMasterIPs[0] for _, csiPod := range csiPods { if strings.Contains(csiPod.Name, vSphereCSIControllerPodNamePrefix) { // Putting the grepped logs for leader of container of different CSI pods @@ -6176,342 +6084,3 @@ func getAllPVCFromNamespace(client clientset.Interface, namespace string) *v1.Pe gomega.Expect(pvcList).NotTo(gomega.BeNil()) return pvcList } - -// CheckDevice helps verify the raw block device inside pod is accessible correctly -func CheckDevice(client clientset.Interface, sts *appsv1.StatefulSet, devicePath string) error { - for _, cmd := range []string{ - fmt.Sprintf("ls -idlh %v", devicePath), - fmt.Sprintf("find %v", devicePath), - fmt.Sprintf("dd if=/dev/zero of=%v bs=1024 count=1 seek=0", devicePath), - } { - if err := fss.ExecInStatefulPods(client, sts, cmd); err != nil { - return fmt.Errorf("failed to check device in command %v, err %v", cmd, err) - } - } - return nil -} - -// verifyIOOnRawBlockVolume helps check data integrity for raw block volumes -func verifyIOOnRawBlockVolume(ns string, podName string, devicePath string, testdataFile string, - startSizeInMB, dataSizeInMB int64) { - //Write some data to file first and then to raw block device - writeDataOnRawBlockVolume(ns, podName, devicePath, testdataFile, startSizeInMB, dataSizeInMB) - // Read the data to verify that is it same as what written - verifyDataFromRawBlockVolume(ns, podName, devicePath, testdataFile, startSizeInMB, dataSizeInMB) -} - -// writeDataOnRawBlockVolume writes test data to raw block device -func writeDataOnRawBlockVolume(ns string, podName string, devicePath string, testdataFile string, - startSizeInMB, dataSizeInMB int64) { - cmd := []string{"exec", podName, "--namespace=" + ns, "--", "/bin/sh", "-c", - fmt.Sprintf("/bin/ls %v", devicePath)} - _, err := framework.RunKubectl(ns, cmd...) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - _ = framework.RunKubectlOrDie(ns, "cp", testdataFile, fmt.Sprintf( - "%v/%v:/tmp/data_to_write", ns, podName)) - framework.ExpectNoError(err, fmt.Sprintf("failed to write testdata inside the pod: %q", podName)) - - // If startSizeInMB is given, fill 1M with testData from offset=startSizeInMB given, upto dataSizeInMB. - // Otherwise write the testData given from offset=0. - gomega.Expect(dataSizeInMB).NotTo(gomega.BeZero()) - for i := int64(0); i < dataSizeInMB; i = i + 1 { - seek := fmt.Sprintf("%v", startSizeInMB+i) - cmd = []string{"exec", podName, "--namespace=" + ns, "--", "/bin/sh", "-c", - fmt.Sprintf("/bin/dd if=/tmp/data_to_write of=%v bs=1M count=1 conv=fsync seek=%v", - devicePath, seek)} - _, err = framework.RunKubectl(ns, cmd...) - framework.ExpectNoError(err, fmt.Sprintf("failed to write device: %q inside the pod: %q", devicePath, podName)) - } - cmd = []string{"--namespace=" + ns, "exec", podName, "--", "/bin/sh", "-c", "rm /tmp/data_to_write"} - _ = framework.RunKubectlOrDie(ns, cmd...) -} - -// verifyDataFromRawBlockVolume reads data from raw block device and verifies it against given input -func verifyDataFromRawBlockVolume(ns string, podName string, devicePath string, testdataFile string, - startSizeInMB, dataSizeInMB int64) { - cmd := []string{"exec", podName, "--namespace=" + ns, "--", "/bin/sh", "-c", - fmt.Sprintf("/bin/ls %v", devicePath)} - _, err := framework.RunKubectl(ns, cmd...) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - // Verify testData written on 1MB upto dataSizeInMB from the specified offset=startSizeInMB - // Otherwise verify the testdata specified from offset=0 - gomega.Expect(dataSizeInMB).NotTo(gomega.BeZero()) - for i := int64(0); i < dataSizeInMB; i = i + 1 { - skip := fmt.Sprintf("%v", startSizeInMB+i) - cmd = []string{"exec", podName, "--namespace=" + ns, "--", "/bin/sh", "-c", - fmt.Sprintf("/bin/dd if=%v of=/tmp/data_to_read bs=1M count=1 skip=%v", devicePath, skip)} - _, err = framework.RunKubectl(ns, cmd...) - framework.ExpectNoError(err, fmt.Sprintf("failed to read device: %q inside the pod: %q", devicePath, podName)) - _ = framework.RunKubectlOrDie(ns, "cp", - fmt.Sprintf("%v/%v:/tmp/data_to_read", ns, podName), testdataFile+podName) - - framework.Logf("Running diff with source file and file from pod %v for 1M starting %vM", podName, skip) - op, err := exec.Command("diff", testdataFile, testdataFile+podName).Output() - framework.Logf("diff: ", op) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(len(op)).To(gomega.BeZero()) - } - -} - -// getBlockDevSizeInBytes returns size of block device at given path -func getBlockDevSizeInBytes(f *framework.Framework, ns string, pod *v1.Pod, devicePath string) (int64, error) { - cmd := []string{"exec", pod.Name, "--namespace=" + ns, "--", "/bin/sh", "-c", - fmt.Sprintf("/bin/blockdev --getsize64 %v", devicePath)} - output, err := framework.RunKubectl(ns, cmd...) - if err != nil { - return -1, fmt.Errorf("failed to get size of raw device %v inside pod", devicePath) - } - output = strings.TrimSuffix(output, "\n") - return strconv.ParseInt(output, 10, 64) -} - -// checkClusterIdValueOnWorkloads checks clusterId value by querying cns metadata -// for all k8s workloads in a particular namespace -func checkClusterIdValueOnWorkloads(vs *vSphere, client clientset.Interface, - ctx context.Context, namespace string, clusterID string) error { - podList, err := client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - for _, pod := range podList.Items { - pvcName := pod.Spec.Volumes[0].PersistentVolumeClaim.ClaimName - pvc, err := client.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pvName := pvc.Spec.VolumeName - pv, err := client.CoreV1().PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - volumeID := pv.Spec.CSI.VolumeHandle - queryResult, err := vs.queryCNSVolumeWithResult(volumeID) - if err != nil { - return err - } - gomega.Expect(queryResult.Volumes).ShouldNot(gomega.BeEmpty()) - if len(queryResult.Volumes) != 1 || queryResult.Volumes[0].VolumeId.Id != volumeID { - return fmt.Errorf("failed to query cns volume %s", volumeID) - } - for _, metadata := range queryResult.Volumes[0].Metadata.EntityMetadata { - kubernetesMetadata := metadata.(*cnstypes.CnsKubernetesEntityMetadata) - if kubernetesMetadata.EntityType == "POD" && kubernetesMetadata.ClusterID != clusterID { - return fmt.Errorf("clusterID %s is not matching with %s ", clusterID, kubernetesMetadata.ClusterID) - } else if kubernetesMetadata.EntityType == "PERSISTENT_VOLUME" && - kubernetesMetadata.ClusterID != clusterID { - return fmt.Errorf("clusterID %s is not matching with %s ", clusterID, kubernetesMetadata.ClusterID) - } else if kubernetesMetadata.EntityType == "PERSISTENT_VOLUME_CLAIM" && - kubernetesMetadata.ClusterID != clusterID { - return fmt.Errorf("clusterID %s is not matching with %s ", clusterID, kubernetesMetadata.ClusterID) - } - } - framework.Logf("successfully verified clusterID of the volume %q", volumeID) - } - return nil -} - -// Clean up statefulset and make sure no volume is left in CNS after it is deleted from k8s -// TODO: Code improvements is needed in case if the function is called from snapshot test. -// add a logic to delete the snapshots for the volumes and then delete volumes -func cleaupStatefulset(client clientset.Interface, ctx context.Context, namespace string, - statefulset *appsv1.StatefulSet) { - scaleDownNDeleteStsDeploymentsInNamespace(ctx, client, namespace) - pvcs, err := client.CoreV1().PersistentVolumeClaims(namespace).List(ctx, metav1.ListOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - for _, claim := range pvcs.Items { - pv := getPvFromClaim(client, namespace, claim.Name) - err := fpv.DeletePersistentVolumeClaim(client, claim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By("Verify it's PV and corresponding volumes are deleted from CNS") - err = fpv.WaitForPersistentVolumeDeleted(client, pv.Name, poll, - pollTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - volumeHandle := pv.Spec.CSI.VolumeHandle - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volumeHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), - fmt.Sprintf("Volume: %s should not be present in the CNS after it is deleted from "+ - "kubernetes", volumeHandle)) - } -} - -// getVsanDPersistentVolumeClaimSpecWithStorageClass return the PersistentVolumeClaim -// spec for vsanDirect datastore with specified storage class. -func getVsanDPersistentVolumeClaimSpecWithStorageClass(namespace string, ds string, - storageclass *storagev1.StorageClass, pvcName string, podName string, - accessMode v1.PersistentVolumeAccessMode) *v1.PersistentVolumeClaim { - pvcAnnotations := make(map[string]string) - pvcAnnotations["volume.beta.kubernetes.io/storage-class"] = storageclass.Name - pvcAnnotations["placement.beta.vmware.com/storagepool_antiAffinityRequired"] = podName - - pvclaimlabels := make(map[string]string) - pvclaimlabels["supervisor"] = "true" - - disksize := diskSize - if ds != "" { - disksize = ds - } - if accessMode == "" { - // If accessMode is not specified, set the default accessMode. - accessMode = v1.ReadWriteOnce - } - - claim := &v1.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: pvcName, - Namespace: namespace, - }, - Spec: v1.PersistentVolumeClaimSpec{ - AccessModes: []v1.PersistentVolumeAccessMode{ - accessMode, - }, - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceName(v1.ResourceStorage): resource.MustParse(disksize), - }, - }, - StorageClassName: &(storageclass.Name), - }, - } - claim.Labels = pvclaimlabels - claim.Annotations = pvcAnnotations - framework.Logf("pvc spec: %v", claim) - return claim -} - -// getVsanDPodSpec returns pod spec for vsan direct datastore for a given persistentVolumeClaim -func getVsanDPodSpec(ns string, nodeSelector map[string]string, pvclaims []*v1.PersistentVolumeClaim, - isPrivileged bool, command string, podName string) *v1.Pod { - - podlabels := make(map[string]string) - podlabels["psp.vmware.com/pod-placement-opt-in"] = "true" - podSpec := &v1.Pod{ - TypeMeta: metav1.TypeMeta{ - Kind: "Pod", - APIVersion: "v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: podName, - Namespace: ns, - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "write-pod", - Image: nginxImage, - SecurityContext: fpod.GenerateContainerSecurityContext(isPrivileged), - }, - }, - RestartPolicy: v1.RestartPolicyOnFailure, - }, - } - var volumeMounts = make([]v1.VolumeMount, len(pvclaims)) - var volumes = make([]v1.Volume, len(pvclaims)) - for index, pvclaim := range pvclaims { - volumename := "data0" - volumeMounts[index] = v1.VolumeMount{Name: volumename, MountPath: "/data0"} - volumes[index] = v1.Volume{Name: volumename, VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ClaimName: pvclaim.Name, ReadOnly: false}}} - } - podSpec.Spec.Containers[0].VolumeMounts = volumeMounts - podSpec.Spec.Volumes = volumes - if nodeSelector != nil { - podSpec.Spec.NodeSelector = nodeSelector - } - - podSpec.Labels = podlabels - podSpec.Spec.ServiceAccountName = "default" - podSpec.Spec.RestartPolicy = v1.RestartPolicyAlways - taintKeys := []string{v1.TaintNodeNotReady, v1.TaintNodeUnreachable} - podSpec.Spec.Subdomain = "sample-pe-svc" - var x int64 = 300 - var tolerations []v1.Toleration - for _, taintkey := range taintKeys { - var toleration v1.Toleration - toleration.Key = taintkey - toleration.Operator = v1.TolerationOpExists - toleration.TolerationSeconds = &x - toleration.Effect = v1.TaintEffectNoExecute - framework.Logf("toleration: %v", toleration) - tolerations = append(tolerations, toleration) - } - podSpec.Spec.Tolerations = tolerations - framework.Logf("pod spec: %v", podSpec) - return podSpec -} - -// createVsanDPvcAndPod creates pvc and pod for vsan direct as per wffc policy. -// It ensures that pvc and pod is in in healthy state. -func createVsanDPvcAndPod(client clientset.Interface, ctx context.Context, - namespace string, sc *storagev1.StorageClass, pvcName string, - podName string) (*v1.PersistentVolumeClaim, *v1.Pod) { - framework.Logf("Creating pvc %s with storage class %s", pvcName, sc.Name) - pvclaim, err := fpv.CreatePVC(client, namespace, - getVsanDPersistentVolumeClaimSpecWithStorageClass(namespace, - diskSize, sc, pvcName, podName, "")) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Expect claim status to be in Pending state") - err = fpv.WaitForPersistentVolumeClaimPhase(v1.ClaimPending, client, - namespace, pvclaim.Name, framework.Poll, time.Minute) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), - "Failed to find the volume: %s in pending state with err: %v", pvcName, err) - - ginkgo.By("Creating a pod") - podSpec := getVsanDPodSpec(namespace, nil, []*v1.PersistentVolumeClaim{pvclaim}, false, execCommand, podName) - pod, err := client.CoreV1().Pods(namespace).Create(ctx, podSpec, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - err = fpod.WaitForPodNameRunningInNamespace(client, pod.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Expect claim to be in Bound state and provisioning volume passes") - err = fpv.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, - namespace, pvclaim.Name, framework.Poll, time.Minute) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to provision volume with err: %v", err) - - return pvclaim, pod -} - -// writeDataToMultipleFilesOnPodInParallel writes data to multiple files -// on a given pod in parallel -func writeDataToMultipleFilesOnPodInParallel(namespace string, podName string, data string, - wg *sync.WaitGroup) { - defer ginkgo.GinkgoRecover() - defer wg.Done() - for i := 0; i < 10; i++ { - ginkgo.By("write to a file in pod") - filePath := fmt.Sprintf("/mnt/volume1/file%v.txt", i) - writeDataOnFileFromPod(namespace, podName, filePath, data) - } - -} - -// byFirstTimeStamp sorts a slice of events by first timestamp, using their involvedObject's name as a tie breaker. -type byFirstTimeStamp []v1.Event - -func (o byFirstTimeStamp) Len() int { return len(o) } -func (o byFirstTimeStamp) Swap(i, j int) { o[i], o[j] = o[j], o[i] } - -func (o byFirstTimeStamp) Less(i, j int) bool { - if o[i].FirstTimestamp.Equal(&o[j].FirstTimestamp) { - return o[i].InvolvedObject.Name < o[j].InvolvedObject.Name - } - return o[i].FirstTimestamp.Before(&o[j].FirstTimestamp) -} - -// dumpSvcNsEventsOnTestFailure dumps the events from the given namespace in case of test failure -func dumpSvcNsEventsOnTestFailure(client clientset.Interface, namespace string) { - if !ginkgo.CurrentSpecReport().Failed() { - return - } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - events, err := client.CoreV1().Events(namespace).List(ctx, metav1.ListOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By(fmt.Sprintf("Found %d events in svc ns %s.", len(events.Items), namespace)) - sortedEvents := events.Items - if len(sortedEvents) > 1 { - sort.Sort(byFirstTimeStamp(sortedEvents)) - } - for _, e := range sortedEvents { - framework.Logf( - "At %v - event for %v: %v %v: %v", e.FirstTimestamp, e.InvolvedObject.Name, e.Source, e.Reason, e.Message) - } -} diff --git a/tests/e2e/vc_reboot_volume_lifecycle.go b/tests/e2e/vc_reboot_volume_lifecycle.go index 789b274acb..a7c09895be 100644 --- a/tests/e2e/vc_reboot_volume_lifecycle.go +++ b/tests/e2e/vc_reboot_volume_lifecycle.go @@ -65,12 +65,10 @@ var _ bool = ginkgo.Describe("Verify volume life_cycle operations works fine aft ginkgo.AfterEach(func() { if supervisorCluster || guestCluster { deleteResourceQuota(client, namespace) - dumpSvcNsEventsOnTestFailure(client, namespace) } if guestCluster { svcClient, svNamespace := getSvcClientAndNamespace() setResourceQuota(svcClient, svNamespace, defaultrqLimit) - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) } }) @@ -127,6 +125,11 @@ var _ bool = ginkgo.Describe("Verify volume life_cycle operations works fine aft } }() + defer func() { + err = fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }() + ginkgo.By("Waiting for claim to be in bound phase") pvc, err := fpv.WaitForPVClaimBoundPhase(client, []*v1.PersistentVolumeClaim{pvclaim}, framework.ClaimProvisionTimeout) @@ -135,13 +138,6 @@ var _ bool = ginkgo.Describe("Verify volume life_cycle operations works fine aft pv := getPvFromClaim(client, pvclaim.Namespace, pvclaim.Name) volumeID := pv.Spec.CSI.VolumeHandle - defer func() { - err = fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volumeID) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - if guestCluster { // svcPVCName refers to PVC Name in the supervisor cluster svcPVCName = volumeID diff --git a/tests/e2e/vcp_to_csi_syncer.go b/tests/e2e/vcp_to_csi_syncer.go index f597a65b54..04f1fdcfaa 100644 --- a/tests/e2e/vcp_to_csi_syncer.go +++ b/tests/e2e/vcp_to_csi_syncer.go @@ -21,7 +21,6 @@ import ( "fmt" "math/rand" "os" - "os/exec" "strconv" "strings" "time" @@ -76,7 +75,6 @@ var _ = ginkgo.Describe("[csi-vcp-mig] VCP to CSI migration syncer tests", func( fullSyncWaitTime int podsToDelete []*v1.Pod migrationEnabledByDefault bool - rawBlockVolumeMode = v1.PersistentVolumeBlock ) ginkgo.BeforeEach(func() { @@ -741,7 +739,7 @@ var _ = ginkgo.Describe("[csi-vcp-mig] VCP to CSI migration syncer tests", func( statefulset := GetStatefulSetFromManifest(namespace) temp := statefulset.Spec.VolumeClaimTemplates - temp[0].Spec.StorageClassName = &vcpSc.Name + temp[0].Annotations[scAnnotation4Statefulset] = vcpSc.Name statefulset.Spec.PodManagementPolicy = appsv1.ParallelPodManagement ginkgo.By("Creating statefulset and waiting for the replicas to be ready") CreateStatefulSet(namespace, statefulset, client) @@ -833,190 +831,6 @@ var _ = ginkgo.Describe("[csi-vcp-mig] VCP to CSI migration syncer tests", func( toggleCSIMigrationFeatureGatesOnK8snodes(ctx, client, false, namespace) }) - // TC to verify VCP to CSI migration workflow when xfs filesystem is used in VCP - // Steps: - // 1. Create SC1 StorageClass in VCP and use xfs as fstype. - // 2. Create PVC pvc1 using this SC - // 3. Create deployment using SC1 with 1 replica. - // 4. Wait for replica to come up. - // 5. Verify that filesystem used to mount volume inside pod is xfs. - // 6. Create file file1.txt at mountpath. - // 7. Enable CSIMigration and CSIMigrationvSphere feature gates on - // kube-controller-manager (& restart). - // 8. Verify PV/PVCs used by deployment have the following annotation - - // "pv.kubernetes.io/migrated-to": "csi.vsphere.vmware.com". - // 9. Verify cnsvspherevolumemigrations crd is created for PV/PVCs used - // by deployment. - // 10. Repeat the following steps for all the nodes in the k8s cluster. - // a. Drain and Cordon off the node. - // b. Enable CSIMigration and CSIMigrationvSphere feature gates on the - // kubelet and Restart kubelet. - // c. Verify CSI node for the corresponding K8s node has the following - // annotation - storage.alpha.kubernetes.io/migrated-plugins. - // d. Enable scheduling on the node. - // 11. Verify that filesystem used is xfs inside pod even after migration - // 12. Write new data at mountpath and verify that write is successful. - // 13. Create a new PVC post migration. - // 14. Verify "pv.kubernetes.io/provisioned-by": "csi.vsphere.vmware.com" - // annotation on new pvc created post migration. - // 15. Verify cnsvspherevolumemigrations crd is created for newly created PVC. - // 16. Scale down deployment replicas to 0. - // All cleanup will be done as part of AfterEach() function: - // 17. Delete deployment. - // 18. Delete all PVCs. - // 19. Wait for PVs and respective vmdks to get deleted. - // 20. Verify cnsvspherevolumemigrations crds are removed for all PV/PVCs. - // 21. Verify CNS entries are removed for all PVCs. - // 22. Delete SC1. - ginkgo.It("TC to verify VCP to CSI migration workflow when xfs filesystem is used in VCP", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - ginkgo.By("Creating VCP SC with fstype as xfs") - scParams := make(map[string]string) - scParams[vcpScParamDatastoreName] = GetAndExpectStringEnvVar(envSharedDatastoreName) - scParams[vcpScParamFstype] = "xfs" - vcpSc, err := createVcpStorageClass(client, scParams, nil, "", "", false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - vcpScs = append(vcpScs, vcpSc) - - ginkgo.By("Creating VCP PVC pvc1 before migration") - pvc1, err := createPVC(client, namespace, nil, "", vcpSc, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - vcpPvcsPreMig = append(vcpPvcsPreMig, pvc1) - - ginkgo.By("Waiting for all claims created before migration to be in bound state") - vcpPvsPreMig, err = fpv.WaitForPVClaimBoundPhase(client, vcpPvcsPreMig, framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Creating a Deployment using pvc1") - labelsMap := make(map[string]string) - labelsMap["dep-lkey"] = "lval" - dep1, err := createDeployment(ctx, client, 1, labelsMap, nil, - namespace, []*v1.PersistentVolumeClaim{pvc1}, execCommand, false, busyBoxImageOnGcr) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - pods, err := fdep.GetPodsForDeployment(client, dep1) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pod := pods.Items[0] - err = fpod.WaitForPodNameRunningInNamespace(client, pod.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - // Check filesystem used to mount volume inside pod is xfs as expeted - ginkgo.By("Verify if filesystem used to mount volume is xfs as expected") - _, err = framework.LookForStringInPodExec(namespace, pod.Name, []string{"/bin/cat", "/mnt/volume1/fstype"}, - xfsFSType, time.Minute) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - // Create file1.txt at mountpath inside pod. - ginkgo.By(fmt.Sprintf("Creating file file1.txt at mountpath inside pod: %v", pod.Name)) - data1 := "This file file1.txt is written before migration" - filePath1 := "/mnt/volume1/file1.txt" - writeDataOnFileFromPod(namespace, pod.Name, filePath1, data1) - - ginkgo.By("Enabling CSIMigration and CSIMigrationvSphere feature gates on kube-controller-manager") - err = toggleCSIMigrationFeatureGatesOnKubeControllerManager(ctx, client, true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - kcmMigEnabled = true - - ginkgo.By("Waiting for migration related annotations on PV/PVCs created before migration") - waitForMigAnnotationsPvcPvLists(ctx, client, vcpPvcsPreMig, vcpPvsPreMig, true, migrationEnabledByDefault) - - ginkgo.By("Verify CnsVSphereVolumeMigration crds and CNS volume metadata on pvc created before migration") - verifyCnsVolumeMetadataAndCnsVSphereVolumeMigrationCrdForPvcs(ctx, client, vcpPvcsPreMig) - - ginkgo.By("Enable CSI migration feature gates on kublets on k8s nodes") - toggleCSIMigrationFeatureGatesOnK8snodes(ctx, client, true, namespace) - kubectlMigEnabled = true - - // Verify that fstype used is still xfs after migration - // verify that data can be read successfully that was written before migration - // Verify that new write is successful post migration - dep1, err = client.AppsV1().Deployments(namespace).Get(ctx, dep1.Name, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - podsAfterMig, err := fdep.GetPodsForDeployment(client, dep1) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - podAfterMig := podsAfterMig.Items[0] - ginkgo.By("Verify if filesystem used to mount volume is xfs post migration") - _, err = framework.LookForStringInPodExec(namespace, podAfterMig.Name, []string{"/bin/cat", "/mnt/volume1/fstype"}, - xfsFSType, time.Minute) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Verify that data can be successfully read from file1.txt which was written before migration") - output := readFileFromPod(namespace, podAfterMig.Name, filePath1) - gomega.Expect(output == data1+"\n").To(gomega.BeTrue(), "Pod is not able to read file1.txt post migration") - - // Create new file file2.txt at mountpath inside pod. - ginkgo.By(fmt.Sprintf("Creating file file2.txt at mountpath inside pod: %v", podAfterMig.Name)) - data2 := "This file file2.txt is written post migration" - filePath2 := "/mnt/volume1/file2.txt" - writeDataOnFileFromPod(namespace, podAfterMig.Name, filePath2, data2) - - ginkgo.By("Verify that data written post migration can be successfully read from file2.txt") - output = readFileFromPod(namespace, podAfterMig.Name, filePath2) - gomega.Expect(output == data2+"\n").To(gomega.BeTrue(), "Pod is not able to read file2.txt") - - ginkgo.By("Creating VCP PVC pvc2 post migration") - pvc2, err := createPVC(client, namespace, nil, "", vcpSc, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - vcpPvcsPostMig = append(vcpPvcsPostMig, pvc2) - - ginkgo.By("Waiting for all claims created post migration to be in bound state") - vcpPvsPostMig, err = fpv.WaitForPVClaimBoundPhase(client, vcpPvcsPostMig, framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Verify annotations on PV/PVCs created post migration") - waitForMigAnnotationsPvcPvLists(ctx, client, vcpPvcsPostMig, vcpPvsPostMig, false, migrationEnabledByDefault) - - ginkgo.By("Wait and verify CNS entries for all CNS volumes created post migration " + - "along with their respective CnsVSphereVolumeMigration CRDs") - verifyCnsVolumeMetadataAndCnsVSphereVolumeMigrationCrdForPvcs(ctx, client, vcpPvcsPostMig) - - ginkgo.By("Creating a new deployment using pvc2") - dep2, err := createDeployment(ctx, client, 1, labelsMap, nil, - namespace, []*v1.PersistentVolumeClaim{pvc2}, execCommand, false, busyBoxImageOnGcr) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - podsDep2, err := fdep.GetPodsForDeployment(client, dep2) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - podDep2 := podsDep2.Items[0] - err = fpod.WaitForPodNameRunningInNamespace(client, podDep2.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - // Check filesystem used to mount volume inside pod is xfs as expeted - ginkgo.By("Verify if filesystem used to mount volume is xfs as expected") - _, err = framework.LookForStringInPodExec(namespace, podDep2.Name, []string{"/bin/cat", "/mnt/volume1/fstype"}, - xfsFSType, time.Minute) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Scale down deployment1 to 0 replica") - dep1, err = client.AppsV1().Deployments(namespace).Get(ctx, dep1.Name, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pods, err = fdep.GetPodsForDeployment(client, dep1) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pod = pods.Items[0] - rep := dep1.Spec.Replicas - *rep = 0 - dep1.Spec.Replicas = rep - _, err = client.AppsV1().Deployments(namespace).Update(ctx, dep1, metav1.UpdateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fpod.WaitForPodNotFoundInNamespace(client, pod.Name, namespace, pollTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Scale down deployment2 to 0 replica") - dep2, err = client.AppsV1().Deployments(namespace).Get(ctx, dep2.Name, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pods, err = fdep.GetPodsForDeployment(client, dep2) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pod = pods.Items[0] - rep2 := dep2.Spec.Replicas - *rep2 = 0 - dep2.Spec.Replicas = rep2 - _, err = client.AppsV1().Deployments(namespace).Update(ctx, dep2, metav1.UpdateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fpod.WaitForPodNotFoundInNamespace(client, pod.Name, namespace, pollTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - // Verify label and pod name updates with Deployment. // Steps: // 1. Create SC1 VCP SC. @@ -1500,7 +1314,7 @@ var _ = ginkgo.Describe("[csi-vcp-mig] VCP to CSI migration syncer tests", func( for i := 0; i < 10; i++ { statefulset := GetStatefulSetFromManifest(ns.Name) temp := statefulset.Spec.VolumeClaimTemplates - temp[0].Spec.StorageClassName = &vcpSc.Name + temp[0].Annotations[scAnnotation4Statefulset] = vcpSc.Name statefulset.Name = "pre-sts" + strconv.Itoa(i) statefulset.Spec.Template.Labels["app"] = statefulset.Name statefulset.Spec.Selector.MatchLabels["app"] = statefulset.Name @@ -1543,7 +1357,7 @@ var _ = ginkgo.Describe("[csi-vcp-mig] VCP to CSI migration syncer tests", func( for i := 0; i < 10; i++ { statefulset := GetStatefulSetFromManifest(ns.Name) temp := statefulset.Spec.VolumeClaimTemplates - temp[0].Spec.StorageClassName = &vcpSc.Name + temp[0].Annotations[scAnnotation4Statefulset] = vcpSc.Name statefulset.Name = "post-sts" + strconv.Itoa(i) statefulset.Spec.Template.Labels["app"] = statefulset.Name statefulset.Spec.Selector.MatchLabels["app"] = statefulset.Name @@ -1615,248 +1429,6 @@ var _ = ginkgo.Describe("[csi-vcp-mig] VCP to CSI migration syncer tests", func( }) - /* - Test VCP-to-CSI migration with raw block volume using deployment - Steps: - 1. Create SC1 VCP SC. - 2. Create PVC1 using SC1 with volumeMode=Block and wait for binding with PV (say PV1). - 3. Create nginx deployment DEP1 using PVC1 with 1 replica. - 4. Wait for all the replicas to come up. - 5. Write and read some data on raw block PVC PVC1 inside deployment pod. - 6. Enable CSIMigration and CSIMigrationvSphere feature gates on - kube-controller-manager (& restart). - 7. Repeat the following steps for all the nodes in the k8s cluster. - a. Drain and Cordon off the node. - b. Enable CSIMigration and CSIMigrationvSphere feature gates on the - kubelet and Restart kubelet. - c. Verify CSI node for the corresponding K8s node has the following - annotation - storage.alpha.kubernetes.io/migrated-plugins. - d. Enable scheduling on the node. - 8. Verify all PVC1 and PV1 and have the following annotation - - "pv.kubernetes.io/migrated-to": "csi.vsphere.vmware.com". - 9. Verify cnsvspherevolumemigrations crd is created for PVC1 and PV1. - 10. Verify CNS entries are present for all PVC1 and PV1 and all PVCs has - correct pod names. - 11. Verify data written before migration on PVC1. - 12. Create PVC2 using SC1 with volumeMode=Block and wait for binding with PV (say PV2). - 13. Verify cnsvspherevolumemigrations crd is created for PVC2 and PV2. - 14. Patch DEP1 to use PVC2 as well. - 15. Verify CNS entries are present for present for PV2 and PVC2. - 16. Verify CNS entries for PVC1 and PVC2 have correct pod names. - 17. Write and read some data on raw block PVC PVC2 inside deployment pod. - 18. Scale down DEP1 replicas to 0 replicas and wait for PVC1 and PVC2 - to detach. - 19. Verify CNS entries for PVC1 and PVC2 have pod names removed. - 20. Delete DEP1. - 21. Wait for PV1 and PV2 and respective vmdks to get deleted. - 22. Verify cnsvspherevolumemigrations crds are removed for all PV1, PV2, - PVC1 and PVC2. - 23. Verify CNS entries are removed for PV1, PV2, PVC1 and PVC2. - Following steps will be done as a part of AfterEach() call. - 24. Delete SC1. - 25. Repeat the following steps for all the nodes in the k8s cluster. - a. Drain and Cordon off the node. - b. Disable CSIMigration and CSIMigrationvSphere feature gates on the - kubelet and Restart kubelet. - c. Verify CSI node for the corresponding K8s node does not have the - following annotation - storage.alpha.kubernetes.io/migrated-plugins. - d. Enable scheduling on the node. - 26. Disable CSIMigration and CSIMigrationvSphere feature gates on - kube-controller-manager (& restart). - */ - ginkgo.It("Test VCP-to-CSI migration with raw block volume", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - ginkgo.By("Creating VCP SC") - scParameters := make(map[string]string) - scParameters[vcpScParamDatastoreName] = GetAndExpectStringEnvVar(envSharedDatastoreName) - vcpSc, err := createVcpStorageClass(client, scParameters, nil, "", "", false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err := client.StorageV1().StorageClasses().Delete(ctx, vcpSc.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Creating VCP PVC pvc1 with raw block volumemode before migration") - vcpPvcSpec := getPersistentVolumeClaimSpecWithStorageClass(namespace, "", vcpSc, nil, "") - vcpPvcSpec.Spec.VolumeMode = &rawBlockVolumeMode - pvc1, err := fpv.CreatePVC(client, namespace, vcpPvcSpec) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), fmt.Sprintf("Failed to create pvc with err: %v", err)) - vcpPvcsPreMig = append(vcpPvcsPreMig, pvc1) - - ginkgo.By("Waiting for all claims created before migration to be in bound state") - vcpPvsPreMig, err = fpv.WaitForPVClaimBoundPhase(client, vcpPvcsPreMig, framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - labelsMap := make(map[string]string) - labelsMap["dep-lkey"] = "lval" - ginkgo.By("Creating a Deployment using raw block PVC pvc1") - volumeIndex := 1 - volumeName := fmt.Sprintf("volume%v", volumeIndex) - pvc1_devicePath := fmt.Sprintf("%v%v", pod_devicePathPrefix, volumeIndex) - depSpec := getDeploymentSpec(ctx, client, 1, labelsMap, nil, - namespace, []*v1.PersistentVolumeClaim{pvc1}, - "trap exit TERM; while true; do sleep 1; done", false, busyBoxImageOnGcr) - depSpec.Spec.Template.Spec.Containers[len(depSpec.Spec.Template.Spec.Containers)-1].VolumeMounts = nil - depSpec.Spec.Template.Spec.Containers[len(depSpec.Spec.Template.Spec.Containers)-1]. - VolumeDevices = []v1.VolumeDevice{ - { - Name: volumeName, - DevicePath: pvc1_devicePath, - }, - } - dep1, err := client.AppsV1().Deployments(namespace).Create(ctx, depSpec, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), fmt.Errorf("deployment %q Create API error: %v", depSpec.Name, err)) - - ginkgo.By("Waiting deployment to complete") - err = fdep.WaitForDeploymentComplete(client, dep1) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), fmt.Errorf("deployment %q failed to complete: %v", depSpec.Name, err)) - - pods, err := fdep.GetPodsForDeployment(client, dep1) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pod := pods.Items[0] - err = fpod.WaitForPodNameRunningInNamespace(client, pod.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - // Write and read some data on raw block volume inside the deployment pod. - // Use same devicePath for raw block volume here as used inside podSpec by getDeploymentSpec(). - rand.New(rand.NewSource(time.Now().Unix())) - testdataFile := fmt.Sprintf("/tmp/testdata_%v_%v", time.Now().Unix(), rand.Intn(1000)) - ginkgo.By(fmt.Sprintf("Creating a 1mb test data file %v", testdataFile)) - op, err := exec.Command("dd", "if=/dev/urandom", fmt.Sprintf("of=%v", testdataFile), - "bs=1M", "count=1").Output() - fmt.Println(op) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - op, err = exec.Command("rm", "-f", testdataFile).Output() - fmt.Println(op) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - ginkgo.By(fmt.Sprintf("Write and read data on raw volume attached to: %v at path %v "+ - "before enabling migration", pod.Name, pod.Spec.Containers[0].VolumeDevices[0].DevicePath)) - verifyIOOnRawBlockVolume(namespace, pod.Name, pvc1_devicePath, testdataFile, 0, 1) - - ginkgo.By("Enabling CSIMigration and CSIMigrationvSphere feature gates on kube-controller-manager") - err = toggleCSIMigrationFeatureGatesOnKubeControllerManager(ctx, client, true) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - kcmMigEnabled = true - - ginkgo.By("Waiting for migration related annotations on PV/PVCs created before migration") - waitForMigAnnotationsPvcPvLists(ctx, client, vcpPvcsPreMig, vcpPvsPreMig, true, migrationEnabledByDefault) - - ginkgo.By("Verify CnsVSphereVolumeMigration crds and CNS volume metadata on pvc created before migration") - verifyCnsVolumeMetadataAndCnsVSphereVolumeMigrationCrdForPvcs(ctx, client, vcpPvcsPreMig) - - ginkgo.By("Enable CSI migration feature gates on kublets on k8s nodes") - toggleCSIMigrationFeatureGatesOnK8snodes(ctx, client, true, namespace) - kubectlMigEnabled = true - - // After migration, verify the original data on VCP PVC pvc1 - ginkgo.By(fmt.Sprintf("Verify previously written data on migrated raw volume attached to: %v at path %v", - pod.Name, pod.Spec.Containers[0].VolumeDevices[0].DevicePath)) - verifyDataFromRawBlockVolume(namespace, pod.Name, pvc1_devicePath, testdataFile, 0, 1) - - ginkgo.By("Creating VCP PVC pvc2 with raw block volumemode post migration") - pvc2, err := fpv.CreatePVC(client, namespace, vcpPvcSpec) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), fmt.Sprintf("Failed to create pvc with err: %v", err)) - vcpPvcsPostMig = append(vcpPvcsPostMig, pvc2) - - ginkgo.By("Waiting for all claims created post migration to be in bound state") - vcpPvsPostMig, err = fpv.WaitForPVClaimBoundPhase(client, vcpPvcsPostMig, framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Verify annotations on PV/PVCs created post migration") - waitForMigAnnotationsPvcPvLists(ctx, client, vcpPvcsPostMig, vcpPvsPostMig, false, migrationEnabledByDefault) - - ginkgo.By("Wait and verify CNS entries for all CNS volumes created post migration " + - "along with their respective CnsVSphereVolumeMigration CRDs") - verifyCnsVolumeMetadataAndCnsVSphereVolumeMigrationCrdForPvcs(ctx, client, vcpPvcsPostMig) - - dep1, err = client.AppsV1().Deployments(namespace).Get(ctx, dep1.Name, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pods, err = fdep.GetPodsForDeployment(client, dep1) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pod = pods.Items[0] - rep := dep1.Spec.Replicas - *rep = 0 - dep1.Spec.Replicas = rep - ginkgo.By("Scale down deployment to 0 replica") - dep1, err = client.AppsV1().Deployments(namespace).Update(ctx, dep1, metav1.UpdateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fpod.WaitForPodNotFoundInNamespace(client, pod.Name, namespace, pollTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - pvclaims := []*v1.PersistentVolumeClaim{pvc1, pvc2} - var volumeDevices = make([]v1.VolumeDevice, len(pvclaims)) - var volumes = make([]v1.Volume, len(pvclaims)) - for index, pvclaim := range pvclaims { - volumename := fmt.Sprintf("volume%v", index+1) - volumeDevices[index] = v1.VolumeDevice{Name: volumename, DevicePath: "/mnt/" + volumename} - volumes[index] = v1.Volume{Name: volumename, VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ClaimName: pvclaim.Name, ReadOnly: false}}} - } - dep1, err = client.AppsV1().Deployments(namespace).Get(ctx, dep1.Name, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - dep1.Spec.Template.Spec.Containers[0].VolumeDevices = volumeDevices - dep1.Spec.Template.Spec.Volumes = volumes - *rep = 1 - dep1.Spec.Replicas = rep - ginkgo.By("Update deployment to use pvc1 and pvc2") - dep1, err = client.AppsV1().Deployments(namespace).Update(ctx, dep1, metav1.UpdateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fdep.WaitForDeploymentComplete(client, dep1) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - dep1, err = client.AppsV1().Deployments(namespace).Get(ctx, dep1.Name, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - pods, err = wait4DeploymentPodsCreation(client, dep1) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(len(pods.Items)).NotTo(gomega.BeZero()) - pod = pods.Items[0] - err = fpod.WaitTimeoutForPodReadyInNamespace(client, pod.Name, namespace, pollTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Wait and verify CNS entries for all CNS volumes created post migration " + - "along with their respective CnsVSphereVolumeMigration CRDs") - verifyCnsVolumeMetadataAndCnsVSphereVolumeMigrationCrdForPvcs(ctx, client, - []*v1.PersistentVolumeClaim{pvc1, pvc2}) - - // Write and read some data on VCP PVC pvc2 - volumeIndex++ - pvc2_devicePath := fmt.Sprintf("%v%v", pod_devicePathPrefix, volumeIndex) - rand.New(rand.NewSource(time.Now().Unix())) - testdataFile = fmt.Sprintf("/tmp/testdata_%v_%v", time.Now().Unix(), rand.Intn(1000)) - ginkgo.By(fmt.Sprintf("Creating a 1mb test data file %v", testdataFile)) - op, err = exec.Command("dd", "if=/dev/urandom", fmt.Sprintf("of=%v", testdataFile), - "bs=1M", "count=1").Output() - fmt.Println(op) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - op, err = exec.Command("rm", "-f", testdataFile).Output() - fmt.Println(op) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - verifyIOOnRawBlockVolume(namespace, pod.Name, pvc2_devicePath, testdataFile, 0, 1) - - ginkgo.By("Scale down deployment to 0 replica") - dep1, err = client.AppsV1().Deployments(namespace).Get(ctx, dep1.Name, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - *rep = 0 - dep1.Spec.Replicas = rep - _, err = client.AppsV1().Deployments(namespace).Update(ctx, dep1, metav1.UpdateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fpod.WaitForPodNotFoundInNamespace(client, pod.Name, namespace, pollTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - framework.Logf("Delete deployment set") - err := client.AppsV1().Deployments(namespace).Delete(ctx, dep1.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Wait and verify CNS entries for all CNS volumes created post migration " + - "along with their respective CnsVSphereVolumeMigration CRDs") - verifyCnsVolumeMetadataAndCnsVSphereVolumeMigrationCrdForPvcs(ctx, client, - []*v1.PersistentVolumeClaim{pvc1, pvc2}) - }) }) // waitForCnsVSphereVolumeMigrationCrd waits for CnsVSphereVolumeMigration crd to be created for the given volume path @@ -2222,13 +1794,7 @@ func scaleDownNDeleteStsDeploymentsInNamespace(ctx context.Context, c clientset. gomega.Expect(err).NotTo(gomega.HaveOccurred()) deletePolicy := metav1.DeletePropagationForeground err = c.AppsV1().Deployments(ns).Delete(ctx, dep.Name, metav1.DeleteOptions{PropagationPolicy: &deletePolicy}) - if err != nil { - if apierrors.IsNotFound(err) { - return - } else { - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - } + gomega.Expect(err).NotTo(gomega.HaveOccurred()) } } diff --git a/tests/e2e/vmc_create_gc.go b/tests/e2e/vmc_create_gc.go index a9c7014337..5b8bc29f5d 100644 --- a/tests/e2e/vmc_create_gc.go +++ b/tests/e2e/vmc_create_gc.go @@ -17,9 +17,6 @@ limitations under the License. package e2e import ( - "fmt" - "os" - ginkgo "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" "k8s.io/kubernetes/test/e2e/framework" @@ -45,11 +42,6 @@ var _ = ginkgo.Describe("Create GC", func() { ginkgo.It("[vmc] Create GC using devops user", func() { - tkgImageName := os.Getenv(envTKGImage) - if tkgImageName == "" { - ginkgo.Skip(fmt.Sprintf("Env %v is missing", envTKGImage)) - } - ginkgo.By("Get WCP session id") gomega.Expect((e2eVSphere.Config.Global.VmcDevopsUser)).NotTo(gomega.BeEmpty(), "Devops user is not set") wcpToken := getWCPSessionId(vmcWcpHost, e2eVSphere.Config.Global.VmcDevopsUser, @@ -57,7 +49,7 @@ var _ = ginkgo.Describe("Create GC", func() { framework.Logf("vmcWcpHost %s", vmcWcpHost) ginkgo.By("Creating Guest Cluster with Devops User") - createGC(vmcWcpHost, wcpToken, tkgImageName, devopsTKG) + createGC(vmcWcpHost, wcpToken) ginkgo.By("Validate the Guest Cluster is up and running") err := getGC(vmcWcpHost, wcpToken, devopsTKG) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -74,11 +66,6 @@ var _ = ginkgo.Describe("Create GC", func() { ginkgo.It("[vmc] Create GC using cloudadmin user", func() { - tkgImageName := os.Getenv(envTKGImage) - if tkgImageName == "" { - ginkgo.Skip(fmt.Sprintf("Env %v is missing", envTKGImage)) - } - ginkgo.By("Get WCP session id") gomega.Expect((e2eVSphere.Config.Global.VmcCloudUser)).NotTo(gomega.BeEmpty(), "VmcCloudUser is not set") wcpToken := getWCPSessionId(vmcWcpHost, e2eVSphere.Config.Global.VmcCloudUser, @@ -86,7 +73,7 @@ var _ = ginkgo.Describe("Create GC", func() { framework.Logf("vmcWcpHost %s", vmcWcpHost) ginkgo.By("Creating Guest Cluster with cloudadmin User") - createGC(vmcWcpHost, wcpToken, tkgImageName, cloudadminTKG) + createGC(vmcWcpHost, wcpToken) ginkgo.By("Validate the Guest Cluster is up and running") err := getGC(vmcWcpHost, wcpToken, cloudadminTKG) gomega.Expect(err).NotTo(gomega.HaveOccurred()) diff --git a/tests/e2e/vmc_csi_deployments.go b/tests/e2e/vmc_csi_deployments.go index 031c783459..9ca41d522a 100644 --- a/tests/e2e/vmc_csi_deployments.go +++ b/tests/e2e/vmc_csi_deployments.go @@ -92,7 +92,7 @@ var _ = ginkgo.Describe("[vmc-gc] Deploy, Update and Scale Deployments", func() 1. Create Storage Class and PVC 2. Deploy nginx pods with volume 3. Update the nginx deployment pods, update the nginx image from - registry.k8s.io/nginx-slim:0.8 to registry.k8s.io/nginx-slim:0.9 + k8s.gcr.io/nginx-slim:0.8 to k8s.gcr.io/nginx-slim:0.9 4. Wait for some time and verify the update is successful 5. Scale dowm the deployment to 0 replicas 6. Scale up the deployment to 1 replicas and verify all the pods should be up and running diff --git a/tests/e2e/volume_health_test.go b/tests/e2e/volume_health_test.go index 8b0000ceb5..ba6376ba63 100644 --- a/tests/e2e/volume_health_test.go +++ b/tests/e2e/volume_health_test.go @@ -83,7 +83,6 @@ var _ = ginkgo.Describe("Volume health check", func() { vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + sshdPort if supervisorCluster { deleteResourceQuota(client, namespace) - dumpSvcNsEventsOnTestFailure(client, namespace) } if pvc != nil { if hostIP != "" { @@ -113,7 +112,6 @@ var _ = ginkgo.Describe("Volume health check", func() { if guestCluster { svcClient, svNamespace := getSvcClientAndNamespace() setResourceQuota(svcClient, svNamespace, defaultrqLimit) - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) } waitForAllHostsToBeUp(ctx, &e2eVSphere) }) diff --git a/tests/e2e/volume_provisioning_with_level5_topology.go b/tests/e2e/volume_provisioning_with_level5_topology.go index bec1992c6f..6378ec271d 100644 --- a/tests/e2e/volume_provisioning_with_level5_topology.go +++ b/tests/e2e/volume_provisioning_with_level5_topology.go @@ -154,8 +154,7 @@ var _ = ginkgo.Describe("[csi-topology-for-level5] Topology-Provisioning-For-Sta */ ginkgo.It("Provisioning volume when no topology details specified in storage class "+ - "and using default pod management policy for statefulset", ginkgo.Label(p0, topology, block, - vanilla, level5), func() { + "and using default pod management policy for statefulset", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() // Creating StorageClass when no topology details are specified using WFC Binding mode @@ -191,15 +190,13 @@ var _ = ginkgo.Describe("[csi-topology-for-level5] Topology-Provisioning-For-Sta // Verify PV node affinity and that the PODS are running on appropriate nodes ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate node") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, - namespace, allowedTopologies, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, + namespace, allowedTopologies, false) // Scale down statefulset to 0 replicas replicas -= 3 ginkgo.By("Scale down statefulset replica count to 0") - err = scaleDownStatefulSetPod(ctx, client, statefulset, namespace, replicas, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleDownStatefulSetPod(ctx, client, statefulset, namespace, replicas, false) }) /* @@ -253,7 +250,7 @@ var _ = ginkgo.Describe("[csi-topology-for-level5] Topology-Provisioning-For-Sta *(statefulset.Spec.Replicas) = 3 statefulset.Spec.PodManagementPolicy = apps.ParallelPodManagement statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &sc.Name + Annotations["volume.beta.kubernetes.io/storage-class"] = sc.Name CreateStatefulSet(namespace, statefulset, client) replicas := *(statefulset.Spec.Replicas) @@ -268,33 +265,28 @@ var _ = ginkgo.Describe("[csi-topology-for-level5] Topology-Provisioning-For-Sta // Verify PV node affinity and that the PODS are running on appropriate node ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate nodes") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, - statefulset, namespace, allowedTopologies, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, + statefulset, namespace, allowedTopologies, false) // Scale up statefulset replica count to 5 replicas += 5 ginkgo.By("Scale up statefulset replica count to 5") - err = scaleUpStatefulSetPod(ctx, client, statefulset, namespace, replicas, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleUpStatefulSetPod(ctx, client, statefulset, namespace, replicas, false) // Scale down statefulset replica count to 1 replicas -= 1 ginkgo.By("Scale down statefulset replica count to 1") - err = scaleDownStatefulSetPod(ctx, client, statefulset, namespace, replicas, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleDownStatefulSetPod(ctx, client, statefulset, namespace, replicas, false) // Verify newly created PV node affinity details and that the new PODS are running on appropriate nodes ginkgo.By("Verify newly created PV node affinity details and that the new PODS are running on appropriate nodes") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, - namespace, allowedTopologies, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, + namespace, allowedTopologies, false) // Scale down statefulset replicas to 0 replicas = 0 ginkgo.By("Scale down statefulset replica count to 0") - err = scaleDownStatefulSetPod(ctx, client, statefulset, namespace, replicas, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleDownStatefulSetPod(ctx, client, statefulset, namespace, replicas, false) }) /* @@ -346,7 +338,7 @@ var _ = ginkgo.Describe("[csi-topology-for-level5] Topology-Provisioning-For-Sta *(statefulset.Spec.Replicas) = 3 statefulset.Spec.PodManagementPolicy = apps.ParallelPodManagement statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &sc.Name + Annotations["volume.beta.kubernetes.io/storage-class"] = sc.Name ginkgo.By("Creating statefulset") CreateStatefulSet(namespace, statefulset, client) replicas := *(statefulset.Spec.Replicas) @@ -364,29 +356,25 @@ var _ = ginkgo.Describe("[csi-topology-for-level5] Topology-Provisioning-For-Sta appropriate node as specified in the allowed topologies of SC */ ginkgo.By("Verify PV node affinity and that the PODS are running " + "on appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, - namespace, allowedTopologies, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, + namespace, allowedTopologies, false) // Scale up statefulset replicas to 5 replicas += 5 ginkgo.By("Scale up statefulset replica count to 5") - err = scaleUpStatefulSetPod(ctx, client, statefulset, namespace, replicas, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleUpStatefulSetPod(ctx, client, statefulset, namespace, replicas, false) /* Verify newly created PV node affinity and that the news PODS are running on appropriate node as specified in the allowed topologies of SC */ ginkgo.By("Verify newly created PV node affinity and that the news PODS " + "are running on appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, - namespace, allowedTopologies, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, + namespace, allowedTopologies, false) // Scale down statefulset replicas to 0 replicas = 0 ginkgo.By("Scale down statefulset replica count to 0") - err = scaleDownStatefulSetPod(ctx, client, statefulset, namespace, replicas, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleDownStatefulSetPod(ctx, client, statefulset, namespace, replicas, false) }) /* @@ -456,35 +444,30 @@ var _ = ginkgo.Describe("[csi-topology-for-level5] Topology-Provisioning-For-Sta node as specified in the allowed topologies of SC */ ginkgo.By("Verify PV node affinity and that the PODS are running " + "on appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, - namespace, allowedTopologies, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, + namespace, allowedTopologies, false) // Scale up statefulset replicas to 5 replicas += 5 ginkgo.By("Scale up statefulset replica count to 5") - err = scaleUpStatefulSetPod(ctx, client, statefulset, namespace, replicas, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleUpStatefulSetPod(ctx, client, statefulset, namespace, replicas, false) // Scale down statefulset replicas to 1 replicas -= 1 ginkgo.By("Scale down statefulset replica count to 1") - err = scaleDownStatefulSetPod(ctx, client, statefulset, namespace, replicas, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleDownStatefulSetPod(ctx, client, statefulset, namespace, replicas, false) /* "Verify newly created PV node affinity and that the new PODS are running on appropriate node as specified in the allowed topologies of SC */ ginkgo.By("Verify newly created PV node affinity and that the new PODS are running " + "on appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, - namespace, allowedTopologies, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, + namespace, allowedTopologies, false) // Scale down statefulset replicas to 0 replicas = 0 ginkgo.By("Scale down statefulset replica count to 0") - err = scaleDownStatefulSetPod(ctx, client, statefulset, namespace, replicas, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleDownStatefulSetPod(ctx, client, statefulset, namespace, replicas, false) }) /* @@ -545,7 +528,7 @@ var _ = ginkgo.Describe("[csi-topology-for-level5] Topology-Provisioning-For-Sta *(statefulset.Spec.Replicas) = 3 statefulset.Spec.PodManagementPolicy = apps.ParallelPodManagement statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &storageclass.Name + Annotations["volume.beta.kubernetes.io/storage-class"] = storageclass.Name ginkgo.By("Creating statefulset") CreateStatefulSet(namespace, statefulset, client) replicas := *(statefulset.Spec.Replicas) @@ -563,35 +546,30 @@ var _ = ginkgo.Describe("[csi-topology-for-level5] Topology-Provisioning-For-Sta node as specified in the allowed topologies of SC */ ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate " + "node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, - namespace, allowedTopologies, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, + namespace, allowedTopologies, false) // Scale up statefulset replicas to 5 replicas += 5 ginkgo.By("Scale up statefulset replica count to 5") - err = scaleUpStatefulSetPod(ctx, client, statefulset, namespace, replicas, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleUpStatefulSetPod(ctx, client, statefulset, namespace, replicas, false) // Scale down statefulset replicas to 1 replicas -= 1 ginkgo.By("Scale down statefulset replica count to 1") - err = scaleDownStatefulSetPod(ctx, client, statefulset, namespace, replicas, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleDownStatefulSetPod(ctx, client, statefulset, namespace, replicas, false) /* Verify newly created PV node affinity and that the new PODS are running on appropriate node as specified in the allowed topologies of SC */ ginkgo.By("Verify newly created PV node affinity and that the new PODS are running " + "on appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, - namespace, allowedTopologies, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, + namespace, allowedTopologies, false) // Scale down statefulset replicas to 0 replicas = 0 ginkgo.By("Scale down statefulset replica count to 0") - err = scaleDownStatefulSetPod(ctx, client, statefulset, namespace, replicas, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleDownStatefulSetPod(ctx, client, statefulset, namespace, replicas, false) }) /* @@ -668,9 +646,8 @@ var _ = ginkgo.Describe("[csi-topology-for-level5] Topology-Provisioning-For-Sta node as specified in the allowed topologies of SC */ ginkgo.By("Verify PV node affinity and that the PODS are running " + "on appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForDeploymentSetsLevel5(ctx, client, deployment, - namespace, allowedTopologyForSC, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForDeploymentSetsLevel5(ctx, client, deployment, + namespace, allowedTopologyForSC, false) }) /* @@ -694,7 +671,7 @@ var _ = ginkgo.Describe("[csi-topology-for-level5] Topology-Provisioning-For-Sta ginkgo.It("Provisioning volume when storage class specified with multiple labels "+ "without specifying datastore url and using default pod management policy "+ - "for statefulset", ginkgo.Label(p2, topology, block, vanilla, level5), func() { + "for statefulset", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() // Get allowed topologies for Storage Class rack > (rack1,rack2,rack3) @@ -736,15 +713,13 @@ var _ = ginkgo.Describe("[csi-topology-for-level5] Topology-Provisioning-For-Sta as specified in the allowed topologies of SC */ ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, - namespace, allowedTopologies, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, client, statefulset, + namespace, allowedTopologies, false) // Scale down statefulset to 0 replicas replicas -= 3 ginkgo.By("Scale down statefulset replica count to 0") - err = scaleDownStatefulSetPod(ctx, client, statefulset, namespace, replicas, false, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + scaleDownStatefulSetPod(ctx, client, statefulset, namespace, replicas, false) }) /* @@ -933,9 +908,8 @@ var _ = ginkgo.Describe("[csi-topology-for-level5] Topology-Provisioning-For-Sta specified in the allowed topologies of SC */ ginkgo.By("Verify PV node affinity and that the PODS are running on appropriate " + "node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStandalonePodLevel5(ctx, client, pod, namespace, - allowedTopologies, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsFoStandalonePodLevel5(ctx, client, pod, namespace, + allowedTopologies) }) /* @@ -1019,9 +993,8 @@ var _ = ginkgo.Describe("[csi-topology-for-level5] Topology-Provisioning-For-Sta specified in the allowed topologies of SC */ ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStandalonePodLevel5(ctx, client, pod, namespace, - allowedTopologies, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsFoStandalonePodLevel5(ctx, client, pod, namespace, + allowedTopologies) }) /* @@ -1147,9 +1120,8 @@ var _ = ginkgo.Describe("[csi-topology-for-level5] Topology-Provisioning-For-Sta as specified in the allowed topologies of SC */ ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStandalonePodLevel5(ctx, client, pod, namespace, - allowedTopologies, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsFoStandalonePodLevel5(ctx, client, pod, namespace, + allowedTopologies) }) @@ -1290,9 +1262,8 @@ var _ = ginkgo.Describe("[csi-topology-for-level5] Topology-Provisioning-For-Sta specified in the allowed topologies of SC */ ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStandalonePodLevel5(ctx, client, pod, namespace, - allowedTopologies, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsFoStandalonePodLevel5(ctx, client, pod, namespace, + allowedTopologies) // Deleting Pod ginkgo.By("Deleting the Pod") @@ -1357,9 +1328,8 @@ var _ = ginkgo.Describe("[csi-topology-for-level5] Topology-Provisioning-For-Sta specified in the allowed topologies of SC */ ginkgo.By("Verify PV node affinity and that the PODS are running on " + "appropriate node as specified in the allowed topologies of SC") - err = verifyPVnodeAffinityAndPODnodedetailsForStandalonePodLevel5(ctx, client, pod, namespace, - allowedTopologies, false) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + verifyPVnodeAffinityAndPODnodedetailsFoStandalonePodLevel5(ctx, client, pod, namespace, + allowedTopologies) // Verify volume metadata for POD, PVC and PV ginkgo.By("Verify volume metadata for POD, PVC and PV") diff --git a/tests/e2e/vsan_stretched_cluster.go b/tests/e2e/vsan_stretched_cluster.go index a149f92a8a..0b07e067aa 100644 --- a/tests/e2e/vsan_stretched_cluster.go +++ b/tests/e2e/vsan_stretched_cluster.go @@ -234,7 +234,7 @@ var _ = ginkgo.Describe("[vsan-stretch-vanilla] vsan stretched cluster tests", f }() ginkgo.By("Creating statefulset and deployment with volumes from the stretched datastore") statefulset, _, _ := createStsDeployment(ctx, client, namespace, sc, true, - false, 0, "", "") + false, 0, "") ssPodsBeforeScaleDown := fss.GetPodList(client, statefulset) replicas := *(statefulset.Spec.Replicas) csipods, err := client.CoreV1().Pods(csiNs).List(ctx, metav1.ListOptions{}) @@ -346,9 +346,9 @@ var _ = ginkgo.Describe("[vsan-stretch-vanilla] vsan stretched cluster tests", f ginkgo.By("Creating statefulsets sts1 with replica count 1 and sts2 with 5 and wait for all" + "the replicas to be running") - statefulset1, _, _ := createStsDeployment(ctx, client, namespace, sc, false, true, 1, "web", "") + statefulset1, _, _ := createStsDeployment(ctx, client, namespace, sc, false, true, 1, "web") replicas1 := *(statefulset1.Spec.Replicas) - statefulset2, _, _ := createStsDeployment(ctx, client, namespace, sc, false, true, 5, "web-nginx", "") + statefulset2, _, _ := createStsDeployment(ctx, client, namespace, sc, false, true, 5, "web-nginx") ss2PodsBeforeScaleDown := fss.GetPodList(client, statefulset2) replicas2 := *(statefulset2.Spec.Replicas) @@ -748,7 +748,7 @@ var _ = ginkgo.Describe("[vsan-stretch-vanilla] vsan stretched cluster tests", f }() ginkgo.By("Creating statefulset and deployment with volumes from the stretched datastore") statefulset, deployment, _ := createStsDeployment(ctx, client, namespace, sc, true, - false, 0, "", "") + false, 0, "") ssPodsBeforeScaleDown := fss.GetPodList(client, statefulset) replicas := *(statefulset.Spec.Replicas) @@ -1743,7 +1743,7 @@ var _ = ginkgo.Describe("[vsan-stretch-vanilla] vsan stretched cluster tests", f ginkgo.By("Creating statefulset and deployment with volumes from the stretched datastore") statefulset, _, _ := createStsDeployment(ctx, client, namespace, sc, true, - false, 0, "", "") + false, 0, "") ssPodsBeforeScaleDown := fss.GetPodList(client, statefulset) replicas := *(statefulset.Spec.Replicas) @@ -1855,9 +1855,9 @@ var _ = ginkgo.Describe("[vsan-stretch-vanilla] vsan stretched cluster tests", f ginkgo.By("Creating statefulsets sts1 with replica count 1 and sts2 with 5 and wait for all" + "the replicas to be running") - statefulset1, _, _ := createStsDeployment(ctx, client, namespace, sc, false, true, 1, "web", "") + statefulset1, _, _ := createStsDeployment(ctx, client, namespace, sc, false, true, 1, "web") replicas1 := *(statefulset1.Spec.Replicas) - statefulset2, _, _ := createStsDeployment(ctx, client, namespace, sc, false, true, 5, "web-nginx", "") + statefulset2, _, _ := createStsDeployment(ctx, client, namespace, sc, false, true, 5, "web-nginx") ss2PodsBeforeScaleDown := fss.GetPodList(client, statefulset2) replicas2 := *(statefulset2.Spec.Replicas) @@ -1986,7 +1986,7 @@ var _ = ginkgo.Describe("[vsan-stretch-vanilla] vsan stretched cluster tests", f gomega.Expect(err).NotTo(gomega.HaveOccurred()) }() - statefulset, _, _ := createStsDeployment(ctx, client, namespace, sc, true, false, 0, "", "") + statefulset, _, _ := createStsDeployment(ctx, client, namespace, sc, true, false, 0, "") replicas := *(statefulset.Spec.Replicas) ssPodsBeforeScaleDown := fss.GetPodList(client, statefulset) // Scale down replicas of statefulset and verify CNS entries for volumes @@ -2072,9 +2072,9 @@ var _ = ginkgo.Describe("[vsan-stretch-vanilla] vsan stretched cluster tests", f ginkgo.By("Creating statefulsets sts1 with replica count 1 and sts2 with 5 and wait for all" + "the replicas to be running") - statefulset1, _, _ := createStsDeployment(ctx, client, namespace, sc, false, true, 1, "web", "") + statefulset1, _, _ := createStsDeployment(ctx, client, namespace, sc, false, true, 1, "web") replicas1 := *(statefulset1.Spec.Replicas) - statefulset2, _, _ := createStsDeployment(ctx, client, namespace, sc, false, true, 5, "web-nginx", "") + statefulset2, _, _ := createStsDeployment(ctx, client, namespace, sc, false, true, 5, "web-nginx") ss2PodsBeforeScaleDown := fss.GetPodList(client, statefulset2) replicas2 := *(statefulset2.Spec.Replicas) @@ -2421,9 +2421,9 @@ var _ = ginkgo.Describe("[vsan-stretch-vanilla] vsan stretched cluster tests", f ginkgo.By("Creating statefulsets sts1 with replica count 1 and sts2 with 5 and wait for all" + "the replicas to be running") - statefulset1, _, _ := createStsDeployment(ctx, client, namespace, sc, false, true, 1, "web", "") + statefulset1, _, _ := createStsDeployment(ctx, client, namespace, sc, false, true, 1, "web") replicas1 := *(statefulset1.Spec.Replicas) - statefulset2, _, _ := createStsDeployment(ctx, client, namespace, sc, false, true, 5, "web-nginx", "") + statefulset2, _, _ := createStsDeployment(ctx, client, namespace, sc, false, true, 5, "web-nginx") ss2PodsBeforeScaleDown := fss.GetPodList(client, statefulset2) replicas2 := *(statefulset2.Spec.Replicas) @@ -2571,9 +2571,9 @@ var _ = ginkgo.Describe("[vsan-stretch-vanilla] vsan stretched cluster tests", f ginkgo.By("Creating statefulsets sts1 with replica count 1 and sts2 with 5 and wait for all" + "the replicas to be running") - statefulset1, _, _ := createStsDeployment(ctx, client, namespace, sc, false, true, 1, "web", "") + statefulset1, _, _ := createStsDeployment(ctx, client, namespace, sc, false, true, 1, "web") replicas1 := *(statefulset1.Spec.Replicas) - statefulset2, _, _ := createStsDeployment(ctx, client, namespace, sc, false, true, 5, "web-nginx", "") + statefulset2, _, _ := createStsDeployment(ctx, client, namespace, sc, false, true, 5, "web-nginx") ss2PodsBeforeScaleDown := fss.GetPodList(client, statefulset2) replicas2 := *(statefulset2.Spec.Replicas) @@ -2731,7 +2731,7 @@ var _ = ginkgo.Describe("[vsan-stretch-vanilla] vsan stretched cluster tests", f for i := 0; i < operationStormScale; i++ { statefulsetName := prefix1 + strconv.Itoa(i) framework.Logf("Creating statefulset: %s", statefulsetName) - statefulset, _, _ := createStsDeployment(ctx, client, namespace, sc, false, true, 1, statefulsetName, "") + statefulset, _, _ := createStsDeployment(ctx, client, namespace, sc, false, true, 1, statefulsetName) replicas1 = *(statefulset.Spec.Replicas) stsList = append(stsList, statefulset) } @@ -2740,7 +2740,7 @@ var _ = ginkgo.Describe("[vsan-stretch-vanilla] vsan stretched cluster tests", f for i := 0; i < operationStormScale; i++ { statefulsetName := prefix2 + strconv.Itoa(i) framework.Logf("Creating statefulset: %s", statefulsetName) - statefulset, _, _ := createStsDeployment(ctx, client, namespace, sc, false, true, 2, statefulsetName, "") + statefulset, _, _ := createStsDeployment(ctx, client, namespace, sc, false, true, 2, statefulsetName) replicas2 = *(statefulset.Spec.Replicas) stsList = append(stsList, statefulset) } @@ -2946,7 +2946,7 @@ var _ = ginkgo.Describe("[vsan-stretch-vanilla] vsan stretched cluster tests", f ginkgo.By("Creating statefulset and deployment with volumes from the stretched datastore") statefulset, _, _ := createStsDeployment(ctx, client, namespace, sc, true, - false, 0, "", "") + false, 0, "") ssPodsBeforeScaleDown := fss.GetPodList(client, statefulset) replicas := *(statefulset.Spec.Replicas) csipods, err := client.CoreV1().Pods(csiNs).List(ctx, metav1.ListOptions{}) diff --git a/tests/e2e/vsan_stretched_cluster_utils.go b/tests/e2e/vsan_stretched_cluster_utils.go index 46b249f623..cfcb37a3e5 100644 --- a/tests/e2e/vsan_stretched_cluster_utils.go +++ b/tests/e2e/vsan_stretched_cluster_utils.go @@ -27,7 +27,6 @@ import ( "sync" "time" - ginkgo "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" "github.com/vmware/govmomi/find" vsan "github.com/vmware/govmomi/vsan" @@ -84,7 +83,6 @@ func initialiseFdsVar(ctx context.Context) { // siteFailureInParallel causes site Failure in multiple hosts of the site in parallel func siteFailureInParallel(primarySite bool, wg *sync.WaitGroup) { - defer ginkgo.GinkgoRecover() defer wg.Done() siteFailover(primarySite) } @@ -268,7 +266,6 @@ func wait4AllK8sNodesToBeUp( // deletePodsInParallel deletes pods in a given namespace in parallel func deletePodsInParallel(client clientset.Interface, namespace string, pods []*v1.Pod, wg *sync.WaitGroup) { - defer ginkgo.GinkgoRecover() defer wg.Done() for _, pod := range pods { fpod.DeletePodOrFail(client, namespace, pod.Name) @@ -278,7 +275,6 @@ func deletePodsInParallel(client clientset.Interface, namespace string, pods []* // createPvcInParallel creates number of PVC in a given namespace in parallel func createPvcInParallel(client clientset.Interface, namespace string, diskSize string, sc *storagev1.StorageClass, ch chan *v1.PersistentVolumeClaim, lock *sync.Mutex, wg *sync.WaitGroup, volumeOpsScale int) { - defer ginkgo.GinkgoRecover() defer wg.Done() for i := 0; i < volumeOpsScale; i++ { pvc, err := createPVC(client, namespace, nil, diskSize, sc, "") @@ -328,7 +324,6 @@ func waitForPodsToBeInErrorOrRunning(c clientset.Interface, podName, namespace s // runCmdOnHostsInParallel runs command on multiple ESX in parallel func runCmdOnHostsInParallel(hostIP string, sshCmd string, wg *sync.WaitGroup) { - defer ginkgo.GinkgoRecover() defer wg.Done() op, err := runCommandOnESX("root", hostIP, sshCmd) framework.Logf(op) @@ -380,7 +375,6 @@ func toggleNetworkFailureParallel(hosts []string, causeNetworkFailure bool) { // deletePVCInParallel deletes PVC in a given namespace in parallel func deletePvcInParallel(client clientset.Interface, pvclaims []*v1.PersistentVolumeClaim, namespace string, wg *sync.WaitGroup) { - defer ginkgo.GinkgoRecover() defer wg.Done() for _, pvclaim := range pvclaims { err := fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) @@ -391,8 +385,6 @@ func deletePvcInParallel(client clientset.Interface, pvclaims []*v1.PersistentVo // createPodsInParallel creates Pods in a given namespace in parallel func createPodsInParallel(client clientset.Interface, namespace string, pvclaims []*v1.PersistentVolumeClaim, ctx context.Context, lock *sync.Mutex, ch chan *v1.Pod, wg *sync.WaitGroup, volumeOpsScale int) { - - defer ginkgo.GinkgoRecover() defer wg.Done() for i := 0; i < volumeOpsScale; i++ { @@ -409,7 +401,6 @@ func createPodsInParallel(client clientset.Interface, namespace string, pvclaims // updatePvcLabelsInParallel updates the labels of pvc in a namespace in parallel func updatePvcLabelsInParallel(ctx context.Context, client clientset.Interface, namespace string, labels map[string]string, pvclaims []*v1.PersistentVolumeClaim, wg *sync.WaitGroup) { - defer ginkgo.GinkgoRecover() defer wg.Done() for _, pvc := range pvclaims { framework.Logf(fmt.Sprintf("Updating labels %+v for pvc %s in namespace %s", @@ -427,15 +418,12 @@ func updatePvcLabelsInParallel(ctx context.Context, client clientset.Interface, // updatePvLabelsInParallel updates the labels of pv in parallel func updatePvLabelsInParallel(ctx context.Context, client clientset.Interface, namespace string, labels map[string]string, persistentVolumes []*v1.PersistentVolume, wg *sync.WaitGroup) { - defer ginkgo.GinkgoRecover() defer wg.Done() for _, pv := range persistentVolumes { - framework.Logf("Updating labels %+v for pv %s in namespace %s", - labels, pv.Name, namespace) - pv, err := client.CoreV1().PersistentVolumes().Get(ctx, pv.Name, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.Logf(fmt.Sprintf("Updating labels %+v for pv %s in namespace %s", + labels, pv.Name, namespace)) pv.Labels = labels - _, err = client.CoreV1().PersistentVolumes().Update(ctx, pv, metav1.UpdateOptions{}) + _, err := client.CoreV1().PersistentVolumes().Update(ctx, pv, metav1.UpdateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Error on updating pv labels is: %v", err) @@ -551,7 +539,6 @@ func changeLeaderOfContainerToComeUpOnMaster(ctx context.Context, client clients // the particular CSI container on the master node in parallel func invokeDockerPauseNKillOnContainerInParallel(sshClientConfig *ssh.ClientConfig, k8sMasterIp string, csiContainerName string, k8sVersion string, wg *sync.WaitGroup) { - defer ginkgo.GinkgoRecover() defer wg.Done() err := execDockerPauseNKillOnContainer(sshClientConfig, k8sMasterIp, csiContainerName, k8sVersion) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -601,19 +588,12 @@ func checkVmStorageCompliance(client clientset.Interface, storagePolicy string) // statefulset, deployment and volumes of statfulset created func createStsDeployment(ctx context.Context, client clientset.Interface, namespace string, sc *storagev1.StorageClass, isDeploymentRequired bool, modifyStsSpec bool, - replicaCount int32, stsName string, - accessMode v1.PersistentVolumeAccessMode) (*appsv1.StatefulSet, *appsv1.Deployment, []string) { + replicaCount int32, stsName string) (*appsv1.StatefulSet, *appsv1.Deployment, []string) { var pvclaims []*v1.PersistentVolumeClaim - if accessMode == "" { - // If accessMode is not specified, set the default accessMode. - accessMode = v1.ReadWriteOnce - } statefulset := GetStatefulSetFromManifest(namespace) framework.Logf("Creating statefulset") statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1]. - Spec.StorageClassName = &sc.Name - statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1].Spec.AccessModes[0] = - accessMode + Annotations["volume.beta.kubernetes.io/storage-class"] = sc.Name if modifyStsSpec { statefulset.Name = stsName statefulset.Spec.Template.Labels["app"] = statefulset.Name @@ -650,7 +630,7 @@ func createStsDeployment(ctx context.Context, client clientset.Interface, namesp } if isDeploymentRequired { framework.Logf("Creating PVC") - pvclaim, err := createPVC(client, namespace, nil, diskSize, sc, accessMode) + pvclaim, err := createPVC(client, namespace, nil, diskSize, sc, "") gomega.Expect(err).NotTo(gomega.HaveOccurred()) pvclaims = append(pvclaims, pvclaim) persistentvolumes, err := fpv.WaitForPVClaimBoundPhase(client, pvclaims, framework.ClaimProvisionTimeout) @@ -837,11 +817,9 @@ func scaleUpStsAndVerifyPodMetadata(ctx context.Context, client clientset.Interf _, err := e2eVSphere.getVMByUUID(ctx, vmUUID) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } - if !rwxAccessMode { - isDiskAttached, err := e2eVSphere.isVolumeAttachedToVM(client, pv.Spec.CSI.VolumeHandle, vmUUID) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Disk is not attached to the node") - gomega.Expect(isDiskAttached).To(gomega.BeTrue(), "Disk is not attached") - } + isDiskAttached, err := e2eVSphere.isVolumeAttachedToVM(client, pv.Spec.CSI.VolumeHandle, vmUUID) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Disk is not attached to the node") + gomega.Expect(isDiskAttached).To(gomega.BeTrue(), "Disk is not attached") framework.Logf("After scale up, verify the attached volumes match those in CNS Cache") err = verifyVolumeMetadataInCNS(&e2eVSphere, pv.Spec.CSI.VolumeHandle, volumespec.PersistentVolumeClaim.ClaimName, pv.ObjectMeta.Name, sspod.Name) @@ -854,7 +832,6 @@ func scaleUpStsAndVerifyPodMetadata(ctx context.Context, client clientset.Interf // deleteCsiPodInParallel deletes csi pod present in csi namespace in parallel func deleteCsiPodInParallel(client clientset.Interface, pod *v1.Pod, namespace string, wg *sync.WaitGroup) { - defer ginkgo.GinkgoRecover() defer wg.Done() framework.Logf("Deleting the pod: %s", pod.Name) err := fpod.DeletePodWithWait(client, pod) @@ -900,7 +877,6 @@ func hostFailure(esxHost string, hostDown bool) { // scaleStsReplicaInParallel scales statefulset's replica up/down in parallel func scaleStsReplicaInParallel(client clientset.Interface, stsList []*appsv1.StatefulSet, regex string, replicas int32, wg *sync.WaitGroup) { - defer ginkgo.GinkgoRecover() defer wg.Done() for _, statefulset := range stsList { if strings.Contains(statefulset.Name, regex) { @@ -912,7 +888,6 @@ func scaleStsReplicaInParallel(client clientset.Interface, stsList []*appsv1.Sta // deletePvInParallel deletes PVs in parallel from k8s cluster func deletePvInParallel(client clientset.Interface, persistentVolumes []*v1.PersistentVolume, wg *sync.WaitGroup) { - defer ginkgo.GinkgoRecover() defer wg.Done() for _, pv := range persistentVolumes { framework.Logf("Deleting pv %s", pv.Name) @@ -926,7 +901,6 @@ func deletePvInParallel(client clientset.Interface, persistentVolumes []*v1.Pers func createStaticPvAndPvcInParallel(client clientset.Interface, ctx context.Context, fcdIDs []string, ch chan *v1.PersistentVolumeClaim, namespace string, wg *sync.WaitGroup, volumeOpsScale int) { - defer ginkgo.GinkgoRecover() defer wg.Done() staticPVLabels := make(map[string]string) for i := 0; i < volumeOpsScale; i++ { @@ -954,7 +928,6 @@ func createStaticPvAndPvcInParallel(client clientset.Interface, ctx context.Cont // using triggerFullSync() here func triggerFullSyncInParallel(ctx context.Context, client clientset.Interface, cnsOperatorClient client.Client, wg *sync.WaitGroup) { - defer ginkgo.GinkgoRecover() defer wg.Done() err := waitForFullSyncToFinish(client, ctx, cnsOperatorClient) if err != nil { diff --git a/tests/e2e/vsphere.go b/tests/e2e/vsphere.go index 08668a2713..01af367db0 100644 --- a/tests/e2e/vsphere.go +++ b/tests/e2e/vsphere.go @@ -3,13 +3,11 @@ package e2e import ( "context" "encoding/json" - "errors" "fmt" "os" "reflect" "strconv" "strings" - "sync" "time" "github.com/davecgh/go-spew/spew" @@ -141,16 +139,10 @@ func (vs *vSphere) queryCNSVolumeSnapshotWithResult(fcdID string, } // verifySnapshotIsDeletedInCNS verifies the snapshotId's presence on CNS -func verifySnapshotIsDeletedInCNS(volumeId string, snapshotId string, isMultiVcSetup bool) error { +func verifySnapshotIsDeletedInCNS(volumeId string, snapshotId string) error { ginkgo.By(fmt.Sprintf("Invoking queryCNSVolumeSnapshotWithResult with VolumeID: %s and SnapshotID: %s", volumeId, snapshotId)) - var querySnapshotResult *cnstypes.CnsSnapshotQueryResult - var err error - if !isMultiVcSetup { - querySnapshotResult, err = e2eVSphere.queryCNSVolumeSnapshotWithResult(volumeId, snapshotId) - } else { - querySnapshotResult, err = multiVCe2eVSphere.queryCNSVolumeSnapshotWithResultInMultiVC(volumeId, snapshotId) - } + querySnapshotResult, err := e2eVSphere.queryCNSVolumeSnapshotWithResult(volumeId, snapshotId) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By(fmt.Sprintf("Task result is %+v", querySnapshotResult)) gomega.Expect(querySnapshotResult.Entries).ShouldNot(gomega.BeEmpty()) @@ -162,16 +154,10 @@ func verifySnapshotIsDeletedInCNS(volumeId string, snapshotId string, isMultiVcS } // verifySnapshotIsCreatedInCNS verifies the snapshotId's presence on CNS -func verifySnapshotIsCreatedInCNS(volumeId string, snapshotId string, isMultiVC bool) error { +func verifySnapshotIsCreatedInCNS(volumeId string, snapshotId string) error { ginkgo.By(fmt.Sprintf("Invoking queryCNSVolumeSnapshotWithResult with VolumeID: %s and SnapshotID: %s", volumeId, snapshotId)) - var querySnapshotResult *cnstypes.CnsSnapshotQueryResult - var err error - if !isMultiVC { - querySnapshotResult, err = e2eVSphere.queryCNSVolumeSnapshotWithResult(volumeId, snapshotId) - } else { - querySnapshotResult, err = multiVCe2eVSphere.queryCNSVolumeSnapshotWithResultInMultiVC(volumeId, snapshotId) - } + querySnapshotResult, err := e2eVSphere.queryCNSVolumeSnapshotWithResult(volumeId, snapshotId) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By(fmt.Sprintf("Task result is %+v", querySnapshotResult)) gomega.Expect(querySnapshotResult.Entries).ShouldNot(gomega.BeEmpty()) @@ -462,7 +448,7 @@ func (vs *vSphere) waitForMetadataToBeDeleted(volumeID string, entityType string // waitForCNSVolumeToBeDeleted executes QueryVolume API on vCenter and verifies // volume entries are deleted from vCenter Database func (vs *vSphere) waitForCNSVolumeToBeDeleted(volumeID string) error { - err := wait.Poll(poll, 2*pollTimeout, func() (bool, error) { + err := wait.Poll(poll, pollTimeout, func() (bool, error) { queryResult, err := vs.queryCNSVolumeWithResult(volumeID) if err != nil { return true, err @@ -782,14 +768,9 @@ func (vs *vSphere) getVsanClusterResource(ctx context.Context, forceRefresh ...b } // getAllHostsIP reads cluster, gets hosts in it and returns IP array -func getAllHostsIP(ctx context.Context, forceRefresh ...bool) []string { +func getAllHostsIP(ctx context.Context) []string { var result []string - refresh := false - if len(forceRefresh) > 0 { - refresh = forceRefresh[0] - } - - cluster := e2eVSphere.getVsanClusterResource(ctx, refresh) + cluster := e2eVSphere.getVsanClusterResource(ctx) hosts, err := cluster.Hosts(ctx) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -802,7 +783,7 @@ func getAllHostsIP(ctx context.Context, forceRefresh ...bool) []string { // getHostConnectionState reads cluster, gets hosts in it and returns connection state of host func getHostConnectionState(ctx context.Context, addr string) (string, error) { var state string - cluster := e2eVSphere.getVsanClusterResource(ctx, true) + cluster := e2eVSphere.getVsanClusterResource(ctx) hosts, err := cluster.Hosts(ctx) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -1096,14 +1077,9 @@ func (vs *vSphere) verifyDatastoreMatch(volumeID string, dsUrls []string) { // cnsRelocateVolume relocates volume from one datastore to another using CNS relocate volume API func (vs *vSphere) cnsRelocateVolume(e2eVSphere vSphere, ctx context.Context, fcdID string, - dsRefDest vim25types.ManagedObjectReference, - waitForRelocateTaskToComplete ...bool) (*object.Task, error) { + dsRefDest vim25types.ManagedObjectReference) error { var pandoraSyncWaitTime int var err error - waitForTaskTocomplete := true - if len(waitForRelocateTaskToComplete) > 0 { - waitForTaskTocomplete = waitForRelocateTaskToComplete[0] - } if os.Getenv(envPandoraSyncWaitTime) != "" { pandoraSyncWaitTime, err = strconv.Atoi(os.Getenv(envPandoraSyncWaitTime)) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -1125,32 +1101,31 @@ func (vs *vSphere) cnsRelocateVolume(e2eVSphere vSphere, ctx context.Context, fc res, err := cnsmethods.CnsRelocateVolume(ctx, cnsClient, &req) framework.Logf("error is: %v", err) if err != nil { - return nil, err + return err } - task := object.NewTask(e2eVSphere.Client.Client, res.Returnval) - if waitForTaskTocomplete { - taskInfo, err := task.WaitForResult(ctx, nil) - framework.Logf("taskInfo: %v", taskInfo) - framework.Logf("error: %v", err) - if err != nil { - return nil, err - } - taskResult, err := cns.GetTaskResult(ctx, taskInfo) - if err != nil { - return nil, err - } - framework.Logf("Sleeping for %v seconds to allow CNS to sync with pandora", pandoraSyncWaitTime) - time.Sleep(time.Duration(pandoraSyncWaitTime) * time.Second) + task, err := object.NewTask(e2eVSphere.Client.Client, res.Returnval), nil + taskInfo, err := task.WaitForResult(ctx, nil) + framework.Logf("taskInfo: %v", taskInfo) + framework.Logf("error: %v", err) + if err != nil { + return err + } + taskResult, err := cns.GetTaskResult(ctx, taskInfo) + if err != nil { + return err + } + + framework.Logf("Sleeping for %v seconds to allow CNS to sync with pandora", pandoraSyncWaitTime) + time.Sleep(time.Duration(pandoraSyncWaitTime) * time.Second) - cnsRelocateVolumeRes := taskResult.GetCnsVolumeOperationResult() + cnsRelocateVolumeRes := taskResult.GetCnsVolumeOperationResult() - if cnsRelocateVolumeRes.Fault != nil { - err = fmt.Errorf("failed to relocate volume=%+v", cnsRelocateVolumeRes.Fault) - return nil, err - } + if cnsRelocateVolumeRes.Fault != nil { + err = fmt.Errorf("failed to relocate volume=%+v", cnsRelocateVolumeRes.Fault) + return err } - return task, nil + return nil } // fetchDsUrl4CnsVol executes query CNS volume to get the datastore @@ -1204,87 +1179,3 @@ func (vs *vSphere) deleteCNSvolume(volumeID string, isDeleteDisk bool) (*cnstype } return res, nil } - -// reconfigPolicy reconfigures given policy on the given volume -func (vs *vSphere) reconfigPolicy(ctx context.Context, volumeID string, profileID string) error { - cnsClient, err := newCnsClient(ctx, vs.Client.Client) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - CnsVolumeManagerInstance := vim25types.ManagedObjectReference{ - Type: "CnsVolumeManager", - Value: "cns-volume-manager", - } - req := cnstypes.CnsReconfigVolumePolicy{ - This: CnsVolumeManagerInstance, - VolumePolicyReconfigSpecs: []cnstypes.CnsVolumePolicyReconfigSpec{ - { - VolumeId: cnstypes.CnsVolumeId{Id: volumeID}, - Profile: []vim25types.BaseVirtualMachineProfileSpec{ - &vim25types.VirtualMachineDefinedProfileSpec{ - ProfileId: profileID, - }, - }, - }, - }, - } - res, err := cnsmethods.CnsReconfigVolumePolicy(ctx, cnsClient, &req) - if err != nil { - return err - } - task := object.NewTask(vs.Client.Client, res.Returnval) - taskInfo, err := cns.GetTaskInfo(ctx, task) - if err != nil { - return err - } - taskResult, err := cns.GetTaskResult(ctx, taskInfo) - if err != nil { - return err - } - if taskResult == nil { - return errors.New("TaskInfo result is empty") - } - reconfigVolumeOperationRes := taskResult.GetCnsVolumeOperationResult() - if reconfigVolumeOperationRes == nil { - return errors.New("cnsreconfigpolicy operation result is empty") - } - if reconfigVolumeOperationRes.Fault != nil { - return errors.New("cnsreconfigpolicy operation fault: " + reconfigVolumeOperationRes.Fault.LocalizedMessage) - } - framework.Logf("reconfigpolicy on volume %v with policy %v is successful", volumeID, profileID) - return nil -} - -// cnsRelocateVolumeInParallel relocates volume in parallel from one datastore to another -// using CNS API -func cnsRelocateVolumeInParallel(e2eVSphere vSphere, ctx context.Context, fcdID string, - dsRefDest vim25types.ManagedObjectReference, waitForRelocateTaskToComplete bool, - wg *sync.WaitGroup) { - defer ginkgo.GinkgoRecover() - defer wg.Done() - _, err := e2eVSphere.cnsRelocateVolume(e2eVSphere, ctx, fcdID, dsRefDest, waitForRelocateTaskToComplete) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - -} - -// waitForCNSTaskToComplete wait for CNS task to complete -// and gets the result and checks if any fault has occurred -func waitForCNSTaskToComplete(ctx context.Context, task *object.Task) *vim25types.LocalizedMethodFault { - var pandoraSyncWaitTime int - var err error - if os.Getenv(envPandoraSyncWaitTime) != "" { - pandoraSyncWaitTime, err = strconv.Atoi(os.Getenv(envPandoraSyncWaitTime)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } else { - pandoraSyncWaitTime = defaultPandoraSyncWaitTime - } - - taskInfo, err := task.WaitForResult(ctx, nil) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - taskResult, err := cns.GetTaskResult(ctx, taskInfo) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - framework.Logf("Sleeping for %v seconds to allow CNS to sync with pandora", pandoraSyncWaitTime) - time.Sleep(time.Duration(pandoraSyncWaitTime) * time.Second) - - cnsTaskRes := taskResult.GetCnsVolumeOperationResult() - return cnsTaskRes.Fault -} diff --git a/tests/e2e/vsphere_shared_datastore.go b/tests/e2e/vsphere_shared_datastore.go index 54b4b2547d..8f2c96a6c4 100644 --- a/tests/e2e/vsphere_shared_datastore.go +++ b/tests/e2e/vsphere_shared_datastore.go @@ -73,15 +73,6 @@ var _ = ginkgo.Describe("[csi-block-vanilla] [csi-block-vanilla-parallelized] "+ framework.Failf("Unable to find ready and schedulable Node") } }) - ginkgo.AfterEach(func() { - if supervisorCluster { - dumpSvcNsEventsOnTestFailure(client, namespace) - } - if guestCluster { - svcClient, svNamespace := getSvcClientAndNamespace() - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) - } - }) // Shared datastore should be provisioned successfully. ginkgo.It("Verify dynamic provisioning of PV passes with user specified shared datastore and "+ diff --git a/tests/e2e/vsphere_volume_disksize.go b/tests/e2e/vsphere_volume_disksize.go index e5d91b5729..88d1e2b701 100644 --- a/tests/e2e/vsphere_volume_disksize.go +++ b/tests/e2e/vsphere_volume_disksize.go @@ -75,12 +75,10 @@ var _ = ginkgo.Describe("[csi-block-vanilla] [csi-file-vanilla] [csi-supervisor] ginkgo.AfterEach(func() { if supervisorCluster { deleteResourceQuota(client, namespace) - dumpSvcNsEventsOnTestFailure(client, namespace) } if guestCluster { svcClient, svNamespace := getSvcClientAndNamespace() setResourceQuota(svcClient, svNamespace, defaultrqLimit) - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) } }) diff --git a/tests/e2e/vsphere_volume_expansion.go b/tests/e2e/vsphere_volume_expansion.go index 8cc368920f..e87d45dbe8 100644 --- a/tests/e2e/vsphere_volume_expansion.go +++ b/tests/e2e/vsphere_volume_expansion.go @@ -57,7 +57,6 @@ var _ = ginkgo.Describe("Volume Expansion Test", func() { f := framework.NewDefaultFramework("volume-expansion") f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged - framework.TestContext.DeleteNamespace = true var ( client clientset.Interface namespace string @@ -124,12 +123,10 @@ var _ = ginkgo.Describe("Volume Expansion Test", func() { if supervisorCluster { ginkgo.By("Delete Resource quota") deleteResourceQuota(client, namespace) - dumpSvcNsEventsOnTestFailure(client, namespace) } if guestCluster { svcClient, svNamespace := getSvcClientAndNamespace() setResourceQuota(svcClient, svNamespace, defaultrqLimit) - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) } }) @@ -176,29 +173,7 @@ var _ = ginkgo.Describe("Volume Expansion Test", func() { ginkgo.It("[csi-block-vanilla] [csi-guest] [csi-block-vanilla-parallelized] "+ "Verify volume expansion with initial filesystem before expansion", func() { - invokeTestForVolumeExpansionWithFilesystem(f, client, namespace, ext4FSType, "", storagePolicyName, profileID) - }) - - // Test to verify offline volume expansion workflow with xfs filesystem. - - // Steps - // 1. Create StorageClass with fstype set to xfs and allowVolumeExpansion set to true. - // 2. Create PVC which uses the StorageClass created in step 1. - // 3. Wait for PV to be provisioned. - // 4. Wait for PVC's status to become Bound. - // 5. Create pod using PVC on specific node. - // 6. Wait for Disk to be attached to the node. - // 7. Detach the volume. - // 8. Modify PVC's size to trigger offline volume expansion. - // 9. Create pod again using PVC on specific node. - // 10. Wait for Disk to be attached to the node. - // 11. Wait for file system resize to complete. - // 12. Delete pod and Wait for Volume Disk to be detached from the Node. - // 13. Delete PVC, PV and Storage Class. - - ginkgo.It("[csi-block-vanilla] [csi-guest] [csi-block-vanilla-parallelized] "+ - "Verify offline volume expansion workflow with xfs filesystem", func() { - invokeTestForVolumeExpansionWithFilesystem(f, client, namespace, xfsFSType, xfsFSType, storagePolicyName, profileID) + invokeTestForVolumeExpansionWithFilesystem(f, client, namespace, "", storagePolicyName, profileID) }) // Test to verify volume expansion is not supported if allowVolumeExpansion @@ -312,7 +287,7 @@ var _ = ginkgo.Describe("Volume Expansion Test", func() { ginkgo.By("Create StorageClass with allowVolumeExpansion set to true, Create PVC") sharedVSANDatastoreURL := GetAndExpectStringEnvVar(envSharedDatastoreURL) volHandle, pvclaim, pv, storageclass = createSCwithVolumeExpansionTrueAndDynamicPVC( - f, client, sharedVSANDatastoreURL, storagePolicyName, namespace, ext4FSType) + f, client, sharedVSANDatastoreURL, storagePolicyName, namespace) defer func() { if !supervisorCluster { @@ -326,7 +301,7 @@ var _ = ginkgo.Describe("Volume Expansion Test", func() { }() ginkgo.By("Create Pod using the above PVC") - pod, vmUUID := createPODandVerifyVolumeMount(ctx, f, client, namespace, pvclaim, volHandle, "") + pod, vmUUID := createPODandVerifyVolumeMount(ctx, f, client, namespace, pvclaim, volHandle) defer func() { // Delete Pod. @@ -354,64 +329,6 @@ var _ = ginkgo.Describe("Volume Expansion Test", func() { increaseSizeOfPvcAttachedToPod(f, client, namespace, pvclaim, pod) }) - /* - Test to verify online volume expansion workflow with xfs filesystem - - 1. Create StorageClass with fstype set to xfs and allowVolumeExpansion set to true. - 2. Create PVC which uses the StorageClass created in step 1. - 3. Wait for PV to be provisioned. - 4. Wait for PVC's status to become Bound and note down the size - 5. Create a Pod using the above created PVC - 6. Modify PVC's size to trigger online volume expansion - 7. verify the PVC status will change to "FilesystemResizePending". Wait till the status is removed - 8. Verify the resized PVC by doing CNS query - 9. Make sure data is intact on the PV mounted on the pod - 10. Make sure file system has increased - */ - ginkgo.It("[csi-block-vanilla] [csi-block-vanilla-parallelized] "+ - "Verify online volume expansion workflow with xfs filesystem", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - var storageclass *storagev1.StorageClass - var pvclaim *v1.PersistentVolumeClaim - var pv *v1.PersistentVolume - var volHandle string - - ginkgo.By("Create StorageClass with fstype set to xfs and allowVolumeExpansion set to true, Create PVC") - sharedVSANDatastoreURL := GetAndExpectStringEnvVar(envSharedDatastoreURL) - volHandle, pvclaim, pv, storageclass = createSCwithVolumeExpansionTrueAndDynamicPVC( - f, client, sharedVSANDatastoreURL, storagePolicyName, namespace, xfsFSType) - - defer func() { - err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - err = fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(pv.Spec.CSI.VolumeHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By("Create Pod using the above PVC") - pod, _ := createPODandVerifyVolumeMount(ctx, f, client, namespace, pvclaim, volHandle, xfsFSType) - - defer func() { - // Delete Pod. - ginkgo.By(fmt.Sprintf("Deleting the pod %s in namespace %s", pod.Name, namespace)) - err := fpod.DeletePodWithWait(client, pod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - isDiskDetached, err := e2eVSphere.waitForVolumeDetachedFromNode(client, - pv.Spec.CSI.VolumeHandle, pod.Spec.NodeName) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(isDiskDetached).To(gomega.BeTrue(), - fmt.Sprintf("Volume %q is not detached from the node %q", pv.Spec.CSI.VolumeHandle, pod.Spec.NodeName)) - }() - - ginkgo.By("Increase PVC size and verify online volume resize") - increaseSizeOfPvcAttachedToPod(f, client, namespace, pvclaim, pod) - }) - // Verify online volume expansion on static volume. // // 1. Create FCD and wait for fcd to allow syncing with pandora. @@ -445,7 +362,7 @@ var _ = ginkgo.Describe("Volume Expansion Test", func() { }() ginkgo.By("Create POD") - pod, _ := createPODandVerifyVolumeMount(ctx, f, client, namespace, pvclaim, volHandle, "") + pod, _ := createPODandVerifyVolumeMount(ctx, f, client, namespace, pvclaim, volHandle) defer func() { // Delete Pod. @@ -483,7 +400,7 @@ var _ = ginkgo.Describe("Volume Expansion Test", func() { ginkgo.By("Create StorageClass with allowVolumeExpansion set to true, Create PVC") volHandle, pvclaim, pv, storageclass := createSCwithVolumeExpansionTrueAndDynamicPVC( - f, client, "", storagePolicyName, namespace, ext4FSType) + f, client, "", storagePolicyName, namespace) defer func() { if !supervisorCluster { @@ -497,7 +414,7 @@ var _ = ginkgo.Describe("Volume Expansion Test", func() { }() ginkgo.By("Create POD") - pod, vmUUID := createPODandVerifyVolumeMount(ctx, f, client, namespace, pvclaim, volHandle, "") + pod, vmUUID := createPODandVerifyVolumeMount(ctx, f, client, namespace, pvclaim, volHandle) defer func() { // Delete POD @@ -553,7 +470,7 @@ var _ = ginkgo.Describe("Volume Expansion Test", func() { defer cancel() volHandle, pvclaim, pv, storageclass := createSCwithVolumeExpansionTrueAndDynamicPVC( - f, client, "", storagePolicyName, namespace, ext4FSType) + f, client, "", storagePolicyName, namespace) defer func() { if !supervisorCluster { err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) @@ -566,7 +483,7 @@ var _ = ginkgo.Describe("Volume Expansion Test", func() { }() ginkgo.By("Create POD") - pod, vmUUID := createPODandVerifyVolumeMount(ctx, f, client, namespace, pvclaim, volHandle, "") + pod, vmUUID := createPODandVerifyVolumeMount(ctx, f, client, namespace, pvclaim, volHandle) defer func() { // Delete POD @@ -621,7 +538,7 @@ var _ = ginkgo.Describe("Volume Expansion Test", func() { var expectedErrMsg string volHandle, pvclaim, pv, storageclass := createSCwithVolumeExpansionTrueAndDynamicPVC( - f, client, "", storagePolicyName, namespace, ext4FSType) + f, client, "", storagePolicyName, namespace) defer func() { if !supervisorCluster { err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) @@ -634,7 +551,7 @@ var _ = ginkgo.Describe("Volume Expansion Test", func() { }() ginkgo.By("Create POD") - pod, vmUUID := createPODandVerifyVolumeMount(ctx, f, client, namespace, pvclaim, volHandle, "") + pod, vmUUID := createPODandVerifyVolumeMount(ctx, f, client, namespace, pvclaim, volHandle) //Fetch original FileSystemSize ginkgo.By("Verify filesystem size for mount point /mnt/volume1 before expansion") @@ -751,7 +668,7 @@ var _ = ginkgo.Describe("Volume Expansion Test", func() { //featureEnabled := isFssEnabled(vcAddress, cnsNewSyncFSS) volHandle, pvclaim, pv, storageclass := createSCwithVolumeExpansionTrueAndDynamicPVC( - f, client, "", storagePolicyName, namespace, ext4FSType) + f, client, "", storagePolicyName, namespace) defer func() { if !supervisorCluster { err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) @@ -764,7 +681,7 @@ var _ = ginkgo.Describe("Volume Expansion Test", func() { }() ginkgo.By("Create POD") - pod, vmUUID := createPODandVerifyVolumeMount(ctx, f, client, namespace, pvclaim, volHandle, "") + pod, vmUUID := createPODandVerifyVolumeMount(ctx, f, client, namespace, pvclaim, volHandle) //Fetch original FileSystemSize ginkgo.By("Verify filesystem size for mount point /mnt/volume1 before expansion") @@ -871,7 +788,7 @@ var _ = ginkgo.Describe("Volume Expansion Test", func() { ginkgo.By("Create StorageClass with allowVolumeExpansion set to true, Create PVC") volHandle, pvclaim, pv, storageclass := createSCwithVolumeExpansionTrueAndDynamicPVC( - f, client, "", storagePolicyName, namespace, ext4FSType) + f, client, "", storagePolicyName, namespace) defer func() { if !supervisorCluster { err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) @@ -884,7 +801,7 @@ var _ = ginkgo.Describe("Volume Expansion Test", func() { }() ginkgo.By("Create Pod using the above PVC") - pod, vmUUID := createPODandVerifyVolumeMount(ctx, f, client, namespace, pvclaim, volHandle, "") + pod, vmUUID := createPODandVerifyVolumeMount(ctx, f, client, namespace, pvclaim, volHandle) //Fetch original FileSystemSize ginkgo.By("Verify filesystem size for mount point /mnt/volume1 before expansion") @@ -1009,7 +926,7 @@ var _ = ginkgo.Describe("Volume Expansion Test", func() { ginkgo.By("Create StorageClass on shared VVOL datastore with allowVolumeExpansion set to true, Create PVC") volHandle, pvclaim, pv, storageclass := createSCwithVolumeExpansionTrueAndDynamicPVC( - f, client, sharedVVOLdatastoreURL, storagePolicyName, namespace, ext4FSType) + f, client, sharedVVOLdatastoreURL, storagePolicyName, namespace) defer func() { if !supervisorCluster { err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) @@ -1028,7 +945,7 @@ var _ = ginkgo.Describe("Volume Expansion Test", func() { if vanillaCluster || guestCluster { ginkgo.By("Create POD using the above PVC") - pod, vmUUID = createPODandVerifyVolumeMount(ctx, f, client, namespace, pvclaim, volHandle, "") + pod, vmUUID = createPODandVerifyVolumeMount(ctx, f, client, namespace, pvclaim, volHandle) } defer func() { @@ -1099,7 +1016,7 @@ var _ = ginkgo.Describe("Volume Expansion Test", func() { ginkgo.By("Create StorageClass on shared NFS datastore with allowVolumeExpansion set to true") volHandle, pvclaim, pv, storageclass := createSCwithVolumeExpansionTrueAndDynamicPVC( - f, client, sharedNFSdatastoreURL, storagePolicyName, namespace, ext4FSType) + f, client, sharedNFSdatastoreURL, storagePolicyName, namespace) defer func() { if !supervisorCluster { err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) @@ -1118,7 +1035,7 @@ var _ = ginkgo.Describe("Volume Expansion Test", func() { if vanillaCluster || guestCluster { ginkgo.By("Create POD using the above PVC") - pod, vmUUID = createPODandVerifyVolumeMount(ctx, f, client, namespace, pvclaim, volHandle, "") + pod, vmUUID = createPODandVerifyVolumeMount(ctx, f, client, namespace, pvclaim, volHandle) } defer func() { @@ -1190,7 +1107,7 @@ var _ = ginkgo.Describe("Volume Expansion Test", func() { ginkgo.By("Create StorageClass on shared VMFS datastore with allowVolumeExpansion set to true") volHandle, pvclaim, pv, storageclass := createSCwithVolumeExpansionTrueAndDynamicPVC( - f, client, sharedVMFSdatastoreURL, storagePolicyName, namespace, ext4FSType) + f, client, sharedVMFSdatastoreURL, storagePolicyName, namespace) defer func() { if !supervisorCluster { err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) @@ -1209,7 +1126,7 @@ var _ = ginkgo.Describe("Volume Expansion Test", func() { if vanillaCluster || guestCluster { ginkgo.By("Create POD using the above PVC") - pod, vmUUID = createPODandVerifyVolumeMount(ctx, f, client, namespace, pvclaim, volHandle, "") + pod, vmUUID = createPODandVerifyVolumeMount(ctx, f, client, namespace, pvclaim, volHandle) } defer func() { @@ -1444,7 +1361,7 @@ var _ = ginkgo.Describe("Volume Expansion Test", func() { ginkgo.By("Create StorageClass with allowVolumeExpansion set to true, Create PVC") sharedVSANDatastoreURL := GetAndExpectStringEnvVar(envSharedDatastoreURL) volHandle, pvclaim, pv, storageclass2 = createSCwithVolumeExpansionTrueAndDynamicPVC( - f, client, sharedVSANDatastoreURL, storagePolicyName2, namespace, ext4FSType) + f, client, sharedVSANDatastoreURL, storagePolicyName2, namespace) defer func() { err := client.StorageV1().StorageClasses().Delete(ctx, storageclass2.Name, *metav1.NewDeleteOptions(0)) @@ -1469,7 +1386,7 @@ var _ = ginkgo.Describe("Volume Expansion Test", func() { gomega.Expect(err).To(gomega.HaveOccurred()) ginkgo.By("Create Pod using the above PVC") - pod, vmUUID := createPODandVerifyVolumeMount(ctx, f, client, namespace, pvclaim, volHandle, "") + pod, vmUUID := createPODandVerifyVolumeMount(ctx, f, client, namespace, pvclaim, volHandle) defer func() { // Delete POD @@ -1535,7 +1452,7 @@ var _ = ginkgo.Describe("Volume Expansion Test", func() { vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + sshdPort volHandle, pvclaim, pv, storageclass := createSCwithVolumeExpansionTrueAndDynamicPVC( - f, client, "", storagePolicyName, namespace, ext4FSType) + f, client, "", storagePolicyName, namespace) defer func() { if !supervisorCluster { err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) @@ -1686,7 +1603,7 @@ var _ = ginkgo.Describe("Volume Expansion Test", func() { var err error volHandle, pvclaim, pv, storageclass := createSCwithVolumeExpansionTrueAndDynamicPVC( - f, client, "", storagePolicyName, namespace, ext4FSType) + f, client, "", storagePolicyName, namespace) defer func() { if !supervisorCluster { err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) @@ -1840,7 +1757,7 @@ var _ = ginkgo.Describe("Volume Expansion Test", func() { ginkgo.By("Create StorageClass with allowVolumeExpansion set to true, Create PVC") sharedVSANDatastoreURL := GetAndExpectStringEnvVar(envSharedDatastoreURL) volHandle, pvclaim, pv, storageclass = createSCwithVolumeExpansionTrueAndDynamicPVC( - f, client, sharedVSANDatastoreURL, storagePolicyName, namespace, ext4FSType) + f, client, sharedVSANDatastoreURL, storagePolicyName, namespace) defer func() { if !supervisorCluster { @@ -1854,7 +1771,7 @@ var _ = ginkgo.Describe("Volume Expansion Test", func() { }() ginkgo.By("Create Pod using the above PVC") - pod, vmUUID := createPODandVerifyVolumeMount(ctx, f, client, namespace, pvclaim, volHandle, "") + pod, vmUUID := createPODandVerifyVolumeMount(ctx, f, client, namespace, pvclaim, volHandle) defer func() { // Delete POD @@ -1913,7 +1830,7 @@ var _ = ginkgo.Describe("Volume Expansion Test", func() { } ginkgo.By("re-create Pod using the same PVC") - pod, vmUUID = createPODandVerifyVolumeMount(ctx, f, client, namespace, pvclaim, volHandle, "") + pod, vmUUID = createPODandVerifyVolumeMount(ctx, f, client, namespace, pvclaim, volHandle) ginkgo.By("Waiting for file system resize to finish") pvclaim, err = waitForFSResize(pvclaim, client) @@ -1962,7 +1879,7 @@ var _ = ginkgo.Describe("Volume Expansion Test", func() { ginkgo.By("Create StorageClass with allowVolumeExpansion set to true, Create PVC") sharedVSANDatastoreURL := GetAndExpectStringEnvVar(envSharedDatastoreURL) volHandle, pvclaim, pv, storageclass = createSCwithVolumeExpansionTrueAndDynamicPVC( - f, client, sharedVSANDatastoreURL, storagePolicyName, namespace, ext4FSType) + f, client, sharedVSANDatastoreURL, storagePolicyName, namespace) defer func() { if !supervisorCluster { @@ -1978,7 +1895,7 @@ var _ = ginkgo.Describe("Volume Expansion Test", func() { }() ginkgo.By("Create Pod using the above PVC") - pod, vmUUID := createPODandVerifyVolumeMount(ctx, f, client, namespace, pvclaim, volHandle, "") + pod, vmUUID := createPODandVerifyVolumeMount(ctx, f, client, namespace, pvclaim, volHandle) defer func() { // Delete POD @@ -2070,7 +1987,7 @@ var _ = ginkgo.Describe("Volume Expansion Test", func() { ginkgo.By("Create StorageClass on shared VSAN datastore with allowVolumeExpansion set to true") sharedVSANDatastoreURL := GetAndExpectStringEnvVar(envSharedDatastoreURL) volHandle, pvclaim, pv, storageclass := createSCwithVolumeExpansionTrueAndDynamicPVC( - f, client, sharedVSANDatastoreURL, storagePolicyName, namespace, ext4FSType) + f, client, sharedVSANDatastoreURL, storagePolicyName, namespace) defer func() { if !supervisorCluster { err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) @@ -2083,7 +2000,7 @@ var _ = ginkgo.Describe("Volume Expansion Test", func() { }() ginkgo.By("Create Pod using the above PVC") - pod, vmUUID = createPODandVerifyVolumeMount(ctx, f, client, namespace, pvclaim, volHandle, "") + pod, vmUUID = createPODandVerifyVolumeMount(ctx, f, client, namespace, pvclaim, volHandle) defer func() { // Delete POD ginkgo.By(fmt.Sprintf("Deleting the pod %s in namespace %s", pod.Name, namespace)) @@ -2193,7 +2110,7 @@ var _ = ginkgo.Describe("Volume Expansion Test", func() { ginkgo.By("Create StorageClass on shared VSAN datastore with allowVolumeExpansion set to true") sharedVSANDatastoreURL := GetAndExpectStringEnvVar(envSharedDatastoreURL) volHandle, pvclaim, pv, storageclass := createSCwithVolumeExpansionTrueAndDynamicPVC( - f, client, sharedVSANDatastoreURL, storagePolicyName, namespace, ext4FSType) + f, client, sharedVSANDatastoreURL, storagePolicyName, namespace) defer func() { if !supervisorCluster { err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) @@ -2233,7 +2150,7 @@ var _ = ginkgo.Describe("Volume Expansion Test", func() { svcCsiDeployment = updateDeploymentReplica(client, 1, vSphereCSIControllerPodNamePrefix, csiSystemNamespace) ginkgo.By("Create Pod using the above PVC") - pod, vmUUID = createPODandVerifyVolumeMount(ctx, f, client, namespace, pvclaim, volHandle, "") + pod, vmUUID = createPODandVerifyVolumeMount(ctx, f, client, namespace, pvclaim, volHandle) defer func() { // Delete POD @@ -2378,7 +2295,7 @@ var _ = ginkgo.Describe("Volume Expansion Test", func() { ginkgo.By("Create StorageClass with allowVolumeExpansion set to true, Create PVC") sharedVSANDatastoreURL := GetAndExpectStringEnvVar(envSharedDatastoreURL) _, pvclaim, pv, storageclass = createSCwithVolumeExpansionTrueAndDynamicPVC( - f, client, sharedVSANDatastoreURL, storagePolicyName, namespace, ext4FSType) + f, client, sharedVSANDatastoreURL, storagePolicyName, namespace) defer func() { if !supervisorCluster { @@ -2580,10 +2497,10 @@ func createStaticPVC(ctx context.Context, f *framework.Framework, // allowVolumeExpansion set to true and Creates PVC. Waits till PV, PVC // are in bound. func createSCwithVolumeExpansionTrueAndDynamicPVC(f *framework.Framework, - client clientset.Interface, dsurl string, storagePolicyName string, namespace string, - fstype string) (string, *v1.PersistentVolumeClaim, *v1.PersistentVolume, *storagev1.StorageClass) { + client clientset.Interface, dsurl string, storagePolicyName string, + namespace string) (string, *v1.PersistentVolumeClaim, *v1.PersistentVolume, *storagev1.StorageClass) { scParameters := make(map[string]string) - scParameters[scParamFsType] = fstype + scParameters[scParamFsType] = ext4FSType // Create Storage class and PVC ginkgo.By("Creating Storage Class and PVC with allowVolumeExpansion = true") @@ -2636,7 +2553,7 @@ func createSCwithVolumeExpansionTrueAndDynamicPVC(f *framework.Framework, // createPODandVerifyVolumeMount this method creates Pod and verifies VolumeMount func createPODandVerifyVolumeMount(ctx context.Context, f *framework.Framework, client clientset.Interface, - namespace string, pvclaim *v1.PersistentVolumeClaim, volHandle string, expectedContent string) (*v1.Pod, string) { + namespace string, pvclaim *v1.PersistentVolumeClaim, volHandle string) (*v1.Pod, string) { // Create a Pod to use this PVC, and verify volume has been attached ginkgo.By("Creating pod to attach PV to the node") pod, err := createPod(client, namespace, nil, []*v1.PersistentVolumeClaim{pvclaim}, false, execCommand) @@ -2663,7 +2580,7 @@ func createPODandVerifyVolumeMount(ctx context.Context, f *framework.Framework, ginkgo.By("Verify the volume is accessible and filesystem type is as expected") _, err = framework.LookForStringInPodExec(namespace, pod.Name, - []string{"/bin/cat", "/mnt/volume1/fstype"}, expectedContent, time.Minute) + []string{"/bin/cat", "/mnt/volume1/fstype"}, "", time.Minute) gomega.Expect(err).NotTo(gomega.HaveOccurred()) return pod, vmUUID @@ -2674,12 +2591,10 @@ func increaseSizeOfPvcAttachedToPod(f *framework.Framework, client clientset.Int namespace string, pvclaim *v1.PersistentVolumeClaim, pod *v1.Pod) { var originalSizeInMb int64 var err error - //Fetch original FileSystemSize if not raw block volume - if *pvclaim.Spec.VolumeMode != v1.PersistentVolumeBlock { - ginkgo.By("Verify filesystem size for mount point /mnt/volume1 before expansion") - originalSizeInMb, err = getFSSizeMb(f, pod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } + //Fetch original FileSystemSize + ginkgo.By("Verify filesystem size for mount point /mnt/volume1 before expansion") + originalSizeInMb, err = getFSSizeMb(f, pod) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) //resize PVC // Modify PVC spec to trigger volume expansion @@ -2699,22 +2614,18 @@ func increaseSizeOfPvcAttachedToPod(f *framework.Framework, client clientset.Int pvcConditions := pvclaim.Status.Conditions expectEqual(len(pvcConditions), 0, "pvc should not have conditions") - if *pvclaim.Spec.VolumeMode != v1.PersistentVolumeBlock { - var fsSize int64 - ginkgo.By("Verify filesystem size for mount point /mnt/volume1") - fsSize, err = getFSSizeMb(f, pod) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - framework.Logf("File system size after expansion : %v", fsSize) - // Filesystem size may be smaller than the size of the block volume - // so here we are checking if the new filesystem size is greater than - // the original volume size as the filesystem is formatted for the - // first time - gomega.Expect(fsSize).Should(gomega.BeNumerically(">", originalSizeInMb), - fmt.Sprintf("error updating filesystem size for %q. Resulting filesystem size is %d", pvclaim.Name, fsSize)) - ginkgo.By("File system resize finished successfully") - } else { - ginkgo.By("Volume resize finished successfully") - } + var fsSize int64 + ginkgo.By("Verify filesystem size for mount point /mnt/volume1") + fsSize, err = getFSSizeMb(f, pod) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.Logf("File system size after expansion : %v", fsSize) + // Filesystem size may be smaller than the size of the block volume + // so here we are checking if the new filesystem size is greater than + // the original volume size as the filesystem is formatted for the + // first time + gomega.Expect(fsSize).Should(gomega.BeNumerically(">", originalSizeInMb), + fmt.Sprintf("error updating filesystem size for %q. Resulting filesystem size is %d", pvclaim.Name, fsSize)) + ginkgo.By("File system resize finished successfully") } func invokeTestForVolumeExpansion(f *framework.Framework, client clientset.Interface, @@ -2927,15 +2838,14 @@ func invokeTestForVolumeExpansion(f *framework.Framework, client clientset.Inter } func invokeTestForVolumeExpansionWithFilesystem(f *framework.Framework, client clientset.Interface, - namespace string, fstype string, expectedContent string, storagePolicyName string, profileID string) { + namespace string, expectedContent string, storagePolicyName string, profileID string) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() ginkgo.By("Invoking Test for Volume Expansion 2") scParameters := make(map[string]string) - scParameters[scParamFsType] = fstype + scParameters[scParamFsType] = ext4FSType // Create Storage class and PVC - ginkgo.By(fmt.Sprintf("Creating Storage Class with %s filesystem and PVC with allowVolumeExpansion = true", - fstype)) + ginkgo.By("Creating Storage Class and PVC with allowVolumeExpansion = true") var storageclass *storagev1.StorageClass var pvclaim *v1.PersistentVolumeClaim var err error @@ -3872,7 +3782,7 @@ func offlineVolumeExpansionOnSupervisorPVC(client clientset.Interface, f *framew gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Create Pod using the above PVC") - pod, vmUUID := createPODandVerifyVolumeMount(ctx, f, client, namespace, pvclaim, volHandle, "") + pod, vmUUID := createPODandVerifyVolumeMount(ctx, f, client, namespace, pvclaim, volHandle) ginkgo.By("Waiting for file system resize to finish") pvclaim, err = waitForFSResize(pvclaim, client) diff --git a/tests/e2e/vsphere_volume_fsgroup.go b/tests/e2e/vsphere_volume_fsgroup.go index 1c7bd5c784..4e9ab8bd7e 100644 --- a/tests/e2e/vsphere_volume_fsgroup.go +++ b/tests/e2e/vsphere_volume_fsgroup.go @@ -75,15 +75,6 @@ var _ = ginkgo.Describe("[csi-block-vanilla] [csi-file-vanilla] [csi-guest] [csi framework.Failf("Unable to find ready and schedulable Node") } }) - ginkgo.AfterEach(func() { - if supervisorCluster { - dumpSvcNsEventsOnTestFailure(client, namespace) - } - if guestCluster { - svcClient, svNamespace := getSvcClientAndNamespace() - dumpSvcNsEventsOnTestFailure(svcClient, svNamespace) - } - }) // Test for Pod creation works when SecurityContext has FSGroup ginkgo.It("Verify Pod Creation works when SecurityContext has FSGroup", func() { diff --git a/tests/e2e/vsphere_volume_with_alpha_feature.go b/tests/e2e/vsphere_volume_with_alpha_feature.go deleted file mode 100644 index d97e1089a4..0000000000 --- a/tests/e2e/vsphere_volume_with_alpha_feature.go +++ /dev/null @@ -1,207 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2e - -import ( - "context" - "fmt" - "time" - - ginkgo "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" - cnstypes "github.com/vmware/govmomi/cns/types" - v1 "k8s.io/api/core/v1" - storagev1 "k8s.io/api/storage/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/kubernetes/test/e2e/framework" - fnodes "k8s.io/kubernetes/test/e2e/framework/node" - fpv "k8s.io/kubernetes/test/e2e/framework/pv" - admissionapi "k8s.io/pod-security-admission/api" -) - -var _ = ginkgo.Describe("Alpha feature check", func() { - - f := framework.NewDefaultFramework("alpha-features") - f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged - var ( - client clientset.Interface - namespace string - scParameters map[string]string - pvtoBackingdiskidAnnotation string = "cns.vmware.com/pv-to-backingdiskobjectid-mapping" - datastoreURL string - ) - ginkgo.BeforeEach(func() { - bootstrap() - client = f.ClientSet - namespace = getNamespaceToRunTests(f) - scParameters = make(map[string]string) - datastoreURL = GetAndExpectStringEnvVar(envSharedDatastoreURL) - nodeList, err := fnodes.GetReadySchedulableNodes(f.ClientSet) - framework.ExpectNoError(err, "Unable to find ready and schedulable Node") - if !(len(nodeList.Items) > 0) { - framework.Failf("Unable to find ready and schedulable Node") - } - }) - - // Verify pvc is annotated in block vanilla setup. - // Steps - // Create a Storage Class. - // Create a PVC using above SC. - // Wait for PVC to be in Bound phase. - // Verify annotation is not added on the PVC. - // Delete PVC. - // Verify PV entry is deleted from CNS. - // Delete the SC. - - ginkgo.It("[csi-block-vanilla-alpha-feature][pv-to-backingdiskobjectid-mapping] Verify pvc is annotated", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - featureEnabled := isCsiFssEnabled(ctx, client, GetAndExpectStringEnvVar(envCSINamespace), - "pv-to-backingdiskobjectid-mapping") - gomega.Expect(featureEnabled).To(gomega.BeTrue()) - ginkgo.By("Invoking Test volume status") - var storageclass *storagev1.StorageClass - var pvclaim *v1.PersistentVolumeClaim - var pvclaims []*v1.PersistentVolumeClaim - var err error - - ginkgo.By("CNS_TEST: Running for vanilla k8s setup") - scParameters[scParamDatastoreURL] = datastoreURL - storageclass, pvclaim, err = createPVCAndStorageClass(client, namespace, - nil, scParameters, diskSize, nil, "", false, "") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - defer func() { - err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - pvclaims = append(pvclaims, pvclaim) - - persistentvolumes, err := fpv.WaitForPVClaimBoundPhase(client, pvclaims, framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - volHandle := persistentvolumes[0].Spec.CSI.VolumeHandle - - defer func() { - err := fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By(fmt.Sprintf("Invoking QueryCNSVolumeWithResult with VolumeID: %s", volHandle)) - queryResult, err := e2eVSphere.queryCNSVolumeWithResult(volHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - if len(queryResult.Volumes) == 0 { - err = fmt.Errorf("QueryCNSVolumeWithResult returned no volume") - } - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - ginkgo.By("Expect annotation is added on the pvc") - waitErr := wait.PollImmediate(pollTimeoutShort, pollTimeoutSixMin*3, func() (bool, error) { - var err error - ginkgo.By(fmt.Sprintf("Sleeping for %v minutes", pollTimeoutShort)) - pvc, err := client.CoreV1().PersistentVolumeClaims(pvclaim.Namespace).Get(ctx, pvclaim.Name, metav1.GetOptions{}) - if err != nil { - return false, fmt.Errorf("error fetching pvc %q for checking pvtobackingdisk annotation: %v", pvc.Name, err) - } - pvbackingAnnotation := pvc.Annotations[pvtoBackingdiskidAnnotation] - if pvbackingAnnotation != "" { - return true, nil - } - return false, nil - }) - - gomega.Expect(waitErr).NotTo(gomega.HaveOccurred()) - }) - - // Verify pvc is not annotated in file vanilla setup. - // Steps - // Create a Storage Class. - // Create a PVC using above SC. - // Wait for PVC to be in Bound phase. - // Verify annotation is not added on the PVC. - // Delete PVC. - // Verify PV entry is deleted from CNS. - // Delete the SC. - - ginkgo.It("[csi-file-vanilla-alpha-feature][pv-to-backingdiskobjectid-mapping] "+ - "File Vanilla Verify pvc is not annotated", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - featureEnabled := isCsiFssEnabled(ctx, client, GetAndExpectStringEnvVar(envCSINamespace), - "pv-to-backingdiskobjectid-mapping") - gomega.Expect(featureEnabled).To(gomega.BeTrue()) - scParameters := make(map[string]string) - scParameters[scParamFsType] = nfs4FSType - accessMode := v1.ReadWriteMany - // Create Storage class and PVC. - ginkgo.By(fmt.Sprintf("Creating Storage Class with access mode %q and fstype %q", accessMode, nfs4FSType)) - var storageclass *storagev1.StorageClass - var pvclaim *v1.PersistentVolumeClaim - var err error - - storageclass, pvclaim, err = createPVCAndStorageClass(client, namespace, - nil, scParameters, "", nil, "", false, accessMode) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - defer func() { - err := client.StorageV1().StorageClasses().Delete(ctx, storageclass.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - // Waiting for PVC to be bound. - var pvclaims []*v1.PersistentVolumeClaim - pvclaims = append(pvclaims, pvclaim) - ginkgo.By("Waiting for all claims to be in bound state") - persistentvolumes, err := fpv.WaitForPVClaimBoundPhase(client, pvclaims, framework.ClaimProvisionTimeout) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - volHandle := persistentvolumes[0].Spec.CSI.VolumeHandle - defer func() { - err := fpv.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2eVSphere.waitForCNSVolumeToBeDeleted(volHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }() - - ginkgo.By(fmt.Sprintf("Invoking QueryCNSVolumeWithResult with VolumeID: %s", volHandle)) - queryResult, err := e2eVSphere.queryCNSVolumeWithResult(volHandle) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(queryResult.Volumes).ShouldNot(gomega.BeEmpty()) - - ginkgo.By(fmt.Sprintf("volume Name:%s , capacity:%d volumeType:%s", queryResult.Volumes[0].Name, - queryResult.Volumes[0].BackingObjectDetails.(*cnstypes.CnsVsanFileShareBackingDetails).CapacityInMb, - queryResult.Volumes[0].VolumeType)) - - ginkgo.By("Verifying volume type specified in PVC is honored") - if queryResult.Volumes[0].VolumeType != testVolumeType { - err = fmt.Errorf("volume type is not %q", testVolumeType) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - ginkgo.By(fmt.Sprintf("Sleeping for %v minutes", pollTimeoutSixMin)) - time.Sleep(pollTimeoutSixMin) - ginkgo.By("Expect annotation is not added on the pvc") - pvc, err := client.CoreV1().PersistentVolumeClaims(pvclaim.Namespace).Get(ctx, pvclaim.Name, metav1.GetOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - for describe := range pvc.Annotations { - gomega.Expect(pvc.Annotations[describe]).ShouldNot(gomega.BeEquivalentTo(pvtoBackingdiskidAnnotation)) - } - }) -}) diff --git a/vendor/github.com/MakeNowJust/heredoc/README.md b/vendor/github.com/MakeNowJust/heredoc/README.md index 289ba31d6a..e9924d2974 100644 --- a/vendor/github.com/MakeNowJust/heredoc/README.md +++ b/vendor/github.com/MakeNowJust/heredoc/README.md @@ -1,52 +1,52 @@ -# heredoc - -[![Build Status](https://circleci.com/gh/MakeNowJust/heredoc.svg?style=svg)](https://circleci.com/gh/MakeNowJust/heredoc) [![GoDoc](https://godoc.org/github.com/MakeNowJusti/heredoc?status.svg)](https://godoc.org/github.com/MakeNowJust/heredoc) - -## About - -Package heredoc provides the here-document with keeping indent. - -## Install - -```console -$ go get github.com/MakeNowJust/heredoc -``` - -## Import - -```go -// usual -import "github.com/MakeNowJust/heredoc" -``` - -## Example - -```go -package main - -import ( - "fmt" - "github.com/MakeNowJust/heredoc" -) - -func main() { - fmt.Println(heredoc.Doc(` - Lorem ipsum dolor sit amet, consectetur adipisicing elit, - sed do eiusmod tempor incididunt ut labore et dolore magna - aliqua. Ut enim ad minim veniam, ... - `)) - // Output: - // Lorem ipsum dolor sit amet, consectetur adipisicing elit, - // sed do eiusmod tempor incididunt ut labore et dolore magna - // aliqua. Ut enim ad minim veniam, ... - // -} -``` - -## API Document - - - [heredoc - GoDoc](https://godoc.org/github.com/MakeNowJust/heredoc) - -## License - -This software is released under the MIT License, see LICENSE. +# heredoc + +[![Build Status](https://circleci.com/gh/MakeNowJust/heredoc.svg?style=svg)](https://circleci.com/gh/MakeNowJust/heredoc) [![GoDoc](https://godoc.org/github.com/MakeNowJusti/heredoc?status.svg)](https://godoc.org/github.com/MakeNowJust/heredoc) + +## About + +Package heredoc provides the here-document with keeping indent. + +## Install + +```console +$ go get github.com/MakeNowJust/heredoc +``` + +## Import + +```go +// usual +import "github.com/MakeNowJust/heredoc" +``` + +## Example + +```go +package main + +import ( + "fmt" + "github.com/MakeNowJust/heredoc" +) + +func main() { + fmt.Println(heredoc.Doc(` + Lorem ipsum dolor sit amet, consectetur adipisicing elit, + sed do eiusmod tempor incididunt ut labore et dolore magna + aliqua. Ut enim ad minim veniam, ... + `)) + // Output: + // Lorem ipsum dolor sit amet, consectetur adipisicing elit, + // sed do eiusmod tempor incididunt ut labore et dolore magna + // aliqua. Ut enim ad minim veniam, ... + // +} +``` + +## API Document + + - [heredoc - GoDoc](https://godoc.org/github.com/MakeNowJust/heredoc) + +## License + +This software is released under the MIT License, see LICENSE. diff --git a/vendor/github.com/PuerkitoBio/purell/.gitignore b/vendor/github.com/PuerkitoBio/purell/.gitignore new file mode 100644 index 0000000000..748e4c8073 --- /dev/null +++ b/vendor/github.com/PuerkitoBio/purell/.gitignore @@ -0,0 +1,5 @@ +*.sublime-* +.DS_Store +*.swp +*.swo +tags diff --git a/vendor/github.com/PuerkitoBio/purell/.travis.yml b/vendor/github.com/PuerkitoBio/purell/.travis.yml new file mode 100644 index 0000000000..cf31e6af6d --- /dev/null +++ b/vendor/github.com/PuerkitoBio/purell/.travis.yml @@ -0,0 +1,12 @@ +language: go + +go: + - 1.4.x + - 1.5.x + - 1.6.x + - 1.7.x + - 1.8.x + - 1.9.x + - "1.10.x" + - "1.11.x" + - tip diff --git a/vendor/github.com/PuerkitoBio/purell/LICENSE b/vendor/github.com/PuerkitoBio/purell/LICENSE new file mode 100644 index 0000000000..4b9986dea7 --- /dev/null +++ b/vendor/github.com/PuerkitoBio/purell/LICENSE @@ -0,0 +1,12 @@ +Copyright (c) 2012, Martin Angers +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +* Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/PuerkitoBio/purell/README.md b/vendor/github.com/PuerkitoBio/purell/README.md new file mode 100644 index 0000000000..07de0c4986 --- /dev/null +++ b/vendor/github.com/PuerkitoBio/purell/README.md @@ -0,0 +1,188 @@ +# Purell + +Purell is a tiny Go library to normalize URLs. It returns a pure URL. Pure-ell. Sanitizer and all. Yeah, I know... + +Based on the [wikipedia paper][wiki] and the [RFC 3986 document][rfc]. + +[![build status](https://travis-ci.org/PuerkitoBio/purell.svg?branch=master)](http://travis-ci.org/PuerkitoBio/purell) + +## Install + +`go get github.com/PuerkitoBio/purell` + +## Changelog + +* **v1.1.1** : Fix failing test due to Go1.12 changes (thanks to @ianlancetaylor). +* **2016-11-14 (v1.1.0)** : IDN: Conform to RFC 5895: Fold character width (thanks to @beeker1121). +* **2016-07-27 (v1.0.0)** : Normalize IDN to ASCII (thanks to @zenovich). +* **2015-02-08** : Add fix for relative paths issue ([PR #5][pr5]) and add fix for unnecessary encoding of reserved characters ([see issue #7][iss7]). +* **v0.2.0** : Add benchmarks, Attempt IDN support. +* **v0.1.0** : Initial release. + +## Examples + +From `example_test.go` (note that in your code, you would import "github.com/PuerkitoBio/purell", and would prefix references to its methods and constants with "purell."): + +```go +package purell + +import ( + "fmt" + "net/url" +) + +func ExampleNormalizeURLString() { + if normalized, err := NormalizeURLString("hTTp://someWEBsite.com:80/Amazing%3f/url/", + FlagLowercaseScheme|FlagLowercaseHost|FlagUppercaseEscapes); err != nil { + panic(err) + } else { + fmt.Print(normalized) + } + // Output: http://somewebsite.com:80/Amazing%3F/url/ +} + +func ExampleMustNormalizeURLString() { + normalized := MustNormalizeURLString("hTTpS://someWEBsite.com:443/Amazing%fa/url/", + FlagsUnsafeGreedy) + fmt.Print(normalized) + + // Output: http://somewebsite.com/Amazing%FA/url +} + +func ExampleNormalizeURL() { + if u, err := url.Parse("Http://SomeUrl.com:8080/a/b/.././c///g?c=3&a=1&b=9&c=0#target"); err != nil { + panic(err) + } else { + normalized := NormalizeURL(u, FlagsUsuallySafeGreedy|FlagRemoveDuplicateSlashes|FlagRemoveFragment) + fmt.Print(normalized) + } + + // Output: http://someurl.com:8080/a/c/g?c=3&a=1&b=9&c=0 +} +``` + +## API + +As seen in the examples above, purell offers three methods, `NormalizeURLString(string, NormalizationFlags) (string, error)`, `MustNormalizeURLString(string, NormalizationFlags) (string)` and `NormalizeURL(*url.URL, NormalizationFlags) (string)`. They all normalize the provided URL based on the specified flags. Here are the available flags: + +```go +const ( + // Safe normalizations + FlagLowercaseScheme NormalizationFlags = 1 << iota // HTTP://host -> http://host, applied by default in Go1.1 + FlagLowercaseHost // http://HOST -> http://host + FlagUppercaseEscapes // http://host/t%ef -> http://host/t%EF + FlagDecodeUnnecessaryEscapes // http://host/t%41 -> http://host/tA + FlagEncodeNecessaryEscapes // http://host/!"#$ -> http://host/%21%22#$ + FlagRemoveDefaultPort // http://host:80 -> http://host + FlagRemoveEmptyQuerySeparator // http://host/path? -> http://host/path + + // Usually safe normalizations + FlagRemoveTrailingSlash // http://host/path/ -> http://host/path + FlagAddTrailingSlash // http://host/path -> http://host/path/ (should choose only one of these add/remove trailing slash flags) + FlagRemoveDotSegments // http://host/path/./a/b/../c -> http://host/path/a/c + + // Unsafe normalizations + FlagRemoveDirectoryIndex // http://host/path/index.html -> http://host/path/ + FlagRemoveFragment // http://host/path#fragment -> http://host/path + FlagForceHTTP // https://host -> http://host + FlagRemoveDuplicateSlashes // http://host/path//a///b -> http://host/path/a/b + FlagRemoveWWW // http://www.host/ -> http://host/ + FlagAddWWW // http://host/ -> http://www.host/ (should choose only one of these add/remove WWW flags) + FlagSortQuery // http://host/path?c=3&b=2&a=1&b=1 -> http://host/path?a=1&b=1&b=2&c=3 + + // Normalizations not in the wikipedia article, required to cover tests cases + // submitted by jehiah + FlagDecodeDWORDHost // http://1113982867 -> http://66.102.7.147 + FlagDecodeOctalHost // http://0102.0146.07.0223 -> http://66.102.7.147 + FlagDecodeHexHost // http://0x42660793 -> http://66.102.7.147 + FlagRemoveUnnecessaryHostDots // http://.host../path -> http://host/path + FlagRemoveEmptyPortSeparator // http://host:/path -> http://host/path + + // Convenience set of safe normalizations + FlagsSafe NormalizationFlags = FlagLowercaseHost | FlagLowercaseScheme | FlagUppercaseEscapes | FlagDecodeUnnecessaryEscapes | FlagEncodeNecessaryEscapes | FlagRemoveDefaultPort | FlagRemoveEmptyQuerySeparator + + // For convenience sets, "greedy" uses the "remove trailing slash" and "remove www. prefix" flags, + // while "non-greedy" uses the "add (or keep) the trailing slash" and "add www. prefix". + + // Convenience set of usually safe normalizations (includes FlagsSafe) + FlagsUsuallySafeGreedy NormalizationFlags = FlagsSafe | FlagRemoveTrailingSlash | FlagRemoveDotSegments + FlagsUsuallySafeNonGreedy NormalizationFlags = FlagsSafe | FlagAddTrailingSlash | FlagRemoveDotSegments + + // Convenience set of unsafe normalizations (includes FlagsUsuallySafe) + FlagsUnsafeGreedy NormalizationFlags = FlagsUsuallySafeGreedy | FlagRemoveDirectoryIndex | FlagRemoveFragment | FlagForceHTTP | FlagRemoveDuplicateSlashes | FlagRemoveWWW | FlagSortQuery + FlagsUnsafeNonGreedy NormalizationFlags = FlagsUsuallySafeNonGreedy | FlagRemoveDirectoryIndex | FlagRemoveFragment | FlagForceHTTP | FlagRemoveDuplicateSlashes | FlagAddWWW | FlagSortQuery + + // Convenience set of all available flags + FlagsAllGreedy = FlagsUnsafeGreedy | FlagDecodeDWORDHost | FlagDecodeOctalHost | FlagDecodeHexHost | FlagRemoveUnnecessaryHostDots | FlagRemoveEmptyPortSeparator + FlagsAllNonGreedy = FlagsUnsafeNonGreedy | FlagDecodeDWORDHost | FlagDecodeOctalHost | FlagDecodeHexHost | FlagRemoveUnnecessaryHostDots | FlagRemoveEmptyPortSeparator +) +``` + +For convenience, the set of flags `FlagsSafe`, `FlagsUsuallySafe[Greedy|NonGreedy]`, `FlagsUnsafe[Greedy|NonGreedy]` and `FlagsAll[Greedy|NonGreedy]` are provided for the similarly grouped normalizations on [wikipedia's URL normalization page][wiki]. You can add (using the bitwise OR `|` operator) or remove (using the bitwise AND NOT `&^` operator) individual flags from the sets if required, to build your own custom set. + +The [full godoc reference is available on gopkgdoc][godoc]. + +Some things to note: + +* `FlagDecodeUnnecessaryEscapes`, `FlagEncodeNecessaryEscapes`, `FlagUppercaseEscapes` and `FlagRemoveEmptyQuerySeparator` are always implicitly set, because internally, the URL string is parsed as an URL object, which automatically decodes unnecessary escapes, uppercases and encodes necessary ones, and removes empty query separators (an unnecessary `?` at the end of the url). So this operation cannot **not** be done. For this reason, `FlagRemoveEmptyQuerySeparator` (as well as the other three) has been included in the `FlagsSafe` convenience set, instead of `FlagsUnsafe`, where Wikipedia puts it. + +* The `FlagDecodeUnnecessaryEscapes` decodes the following escapes (*from -> to*): + - %24 -> $ + - %26 -> & + - %2B-%3B -> +,-./0123456789:; + - %3D -> = + - %40-%5A -> @ABCDEFGHIJKLMNOPQRSTUVWXYZ + - %5F -> _ + - %61-%7A -> abcdefghijklmnopqrstuvwxyz + - %7E -> ~ + + +* When the `NormalizeURL` function is used (passing an URL object), this source URL object is modified (that is, after the call, the URL object will be modified to reflect the normalization). + +* The *replace IP with domain name* normalization (`http://208.77.188.166/ → http://www.example.com/`) is obviously not possible for a library without making some network requests. This is not implemented in purell. + +* The *remove unused query string parameters* and *remove default query parameters* are also not implemented, since this is a very case-specific normalization, and it is quite trivial to do with an URL object. + +### Safe vs Usually Safe vs Unsafe + +Purell allows you to control the level of risk you take while normalizing an URL. You can aggressively normalize, play it totally safe, or anything in between. + +Consider the following URL: + +`HTTPS://www.RooT.com/toto/t%45%1f///a/./b/../c/?z=3&w=2&a=4&w=1#invalid` + +Normalizing with the `FlagsSafe` gives: + +`https://www.root.com/toto/tE%1F///a/./b/../c/?z=3&w=2&a=4&w=1#invalid` + +With the `FlagsUsuallySafeGreedy`: + +`https://www.root.com/toto/tE%1F///a/c?z=3&w=2&a=4&w=1#invalid` + +And with `FlagsUnsafeGreedy`: + +`http://root.com/toto/tE%1F/a/c?a=4&w=1&w=2&z=3` + +## TODOs + +* Add a class/default instance to allow specifying custom directory index names? At the moment, removing directory index removes `(^|/)((?:default|index)\.\w{1,4})$`. + +## Thanks / Contributions + +@rogpeppe +@jehiah +@opennota +@pchristopher1275 +@zenovich +@beeker1121 + +## License + +The [BSD 3-Clause license][bsd]. + +[bsd]: http://opensource.org/licenses/BSD-3-Clause +[wiki]: http://en.wikipedia.org/wiki/URL_normalization +[rfc]: http://tools.ietf.org/html/rfc3986#section-6 +[godoc]: http://go.pkgdoc.org/github.com/PuerkitoBio/purell +[pr5]: https://github.com/PuerkitoBio/purell/pull/5 +[iss7]: https://github.com/PuerkitoBio/purell/issues/7 diff --git a/vendor/github.com/PuerkitoBio/purell/purell.go b/vendor/github.com/PuerkitoBio/purell/purell.go new file mode 100644 index 0000000000..6d0fc190a1 --- /dev/null +++ b/vendor/github.com/PuerkitoBio/purell/purell.go @@ -0,0 +1,379 @@ +/* +Package purell offers URL normalization as described on the wikipedia page: +http://en.wikipedia.org/wiki/URL_normalization +*/ +package purell + +import ( + "bytes" + "fmt" + "net/url" + "regexp" + "sort" + "strconv" + "strings" + + "github.com/PuerkitoBio/urlesc" + "golang.org/x/net/idna" + "golang.org/x/text/unicode/norm" + "golang.org/x/text/width" +) + +// A set of normalization flags determines how a URL will +// be normalized. +type NormalizationFlags uint + +const ( + // Safe normalizations + FlagLowercaseScheme NormalizationFlags = 1 << iota // HTTP://host -> http://host, applied by default in Go1.1 + FlagLowercaseHost // http://HOST -> http://host + FlagUppercaseEscapes // http://host/t%ef -> http://host/t%EF + FlagDecodeUnnecessaryEscapes // http://host/t%41 -> http://host/tA + FlagEncodeNecessaryEscapes // http://host/!"#$ -> http://host/%21%22#$ + FlagRemoveDefaultPort // http://host:80 -> http://host + FlagRemoveEmptyQuerySeparator // http://host/path? -> http://host/path + + // Usually safe normalizations + FlagRemoveTrailingSlash // http://host/path/ -> http://host/path + FlagAddTrailingSlash // http://host/path -> http://host/path/ (should choose only one of these add/remove trailing slash flags) + FlagRemoveDotSegments // http://host/path/./a/b/../c -> http://host/path/a/c + + // Unsafe normalizations + FlagRemoveDirectoryIndex // http://host/path/index.html -> http://host/path/ + FlagRemoveFragment // http://host/path#fragment -> http://host/path + FlagForceHTTP // https://host -> http://host + FlagRemoveDuplicateSlashes // http://host/path//a///b -> http://host/path/a/b + FlagRemoveWWW // http://www.host/ -> http://host/ + FlagAddWWW // http://host/ -> http://www.host/ (should choose only one of these add/remove WWW flags) + FlagSortQuery // http://host/path?c=3&b=2&a=1&b=1 -> http://host/path?a=1&b=1&b=2&c=3 + + // Normalizations not in the wikipedia article, required to cover tests cases + // submitted by jehiah + FlagDecodeDWORDHost // http://1113982867 -> http://66.102.7.147 + FlagDecodeOctalHost // http://0102.0146.07.0223 -> http://66.102.7.147 + FlagDecodeHexHost // http://0x42660793 -> http://66.102.7.147 + FlagRemoveUnnecessaryHostDots // http://.host../path -> http://host/path + FlagRemoveEmptyPortSeparator // http://host:/path -> http://host/path + + // Convenience set of safe normalizations + FlagsSafe NormalizationFlags = FlagLowercaseHost | FlagLowercaseScheme | FlagUppercaseEscapes | FlagDecodeUnnecessaryEscapes | FlagEncodeNecessaryEscapes | FlagRemoveDefaultPort | FlagRemoveEmptyQuerySeparator + + // For convenience sets, "greedy" uses the "remove trailing slash" and "remove www. prefix" flags, + // while "non-greedy" uses the "add (or keep) the trailing slash" and "add www. prefix". + + // Convenience set of usually safe normalizations (includes FlagsSafe) + FlagsUsuallySafeGreedy NormalizationFlags = FlagsSafe | FlagRemoveTrailingSlash | FlagRemoveDotSegments + FlagsUsuallySafeNonGreedy NormalizationFlags = FlagsSafe | FlagAddTrailingSlash | FlagRemoveDotSegments + + // Convenience set of unsafe normalizations (includes FlagsUsuallySafe) + FlagsUnsafeGreedy NormalizationFlags = FlagsUsuallySafeGreedy | FlagRemoveDirectoryIndex | FlagRemoveFragment | FlagForceHTTP | FlagRemoveDuplicateSlashes | FlagRemoveWWW | FlagSortQuery + FlagsUnsafeNonGreedy NormalizationFlags = FlagsUsuallySafeNonGreedy | FlagRemoveDirectoryIndex | FlagRemoveFragment | FlagForceHTTP | FlagRemoveDuplicateSlashes | FlagAddWWW | FlagSortQuery + + // Convenience set of all available flags + FlagsAllGreedy = FlagsUnsafeGreedy | FlagDecodeDWORDHost | FlagDecodeOctalHost | FlagDecodeHexHost | FlagRemoveUnnecessaryHostDots | FlagRemoveEmptyPortSeparator + FlagsAllNonGreedy = FlagsUnsafeNonGreedy | FlagDecodeDWORDHost | FlagDecodeOctalHost | FlagDecodeHexHost | FlagRemoveUnnecessaryHostDots | FlagRemoveEmptyPortSeparator +) + +const ( + defaultHttpPort = ":80" + defaultHttpsPort = ":443" +) + +// Regular expressions used by the normalizations +var rxPort = regexp.MustCompile(`(:\d+)/?$`) +var rxDirIndex = regexp.MustCompile(`(^|/)((?:default|index)\.\w{1,4})$`) +var rxDupSlashes = regexp.MustCompile(`/{2,}`) +var rxDWORDHost = regexp.MustCompile(`^(\d+)((?:\.+)?(?:\:\d*)?)$`) +var rxOctalHost = regexp.MustCompile(`^(0\d*)\.(0\d*)\.(0\d*)\.(0\d*)((?:\.+)?(?:\:\d*)?)$`) +var rxHexHost = regexp.MustCompile(`^0x([0-9A-Fa-f]+)((?:\.+)?(?:\:\d*)?)$`) +var rxHostDots = regexp.MustCompile(`^(.+?)(:\d+)?$`) +var rxEmptyPort = regexp.MustCompile(`:+$`) + +// Map of flags to implementation function. +// FlagDecodeUnnecessaryEscapes has no action, since it is done automatically +// by parsing the string as an URL. Same for FlagUppercaseEscapes and FlagRemoveEmptyQuerySeparator. + +// Since maps have undefined traversing order, make a slice of ordered keys +var flagsOrder = []NormalizationFlags{ + FlagLowercaseScheme, + FlagLowercaseHost, + FlagRemoveDefaultPort, + FlagRemoveDirectoryIndex, + FlagRemoveDotSegments, + FlagRemoveFragment, + FlagForceHTTP, // Must be after remove default port (because https=443/http=80) + FlagRemoveDuplicateSlashes, + FlagRemoveWWW, + FlagAddWWW, + FlagSortQuery, + FlagDecodeDWORDHost, + FlagDecodeOctalHost, + FlagDecodeHexHost, + FlagRemoveUnnecessaryHostDots, + FlagRemoveEmptyPortSeparator, + FlagRemoveTrailingSlash, // These two (add/remove trailing slash) must be last + FlagAddTrailingSlash, +} + +// ... and then the map, where order is unimportant +var flags = map[NormalizationFlags]func(*url.URL){ + FlagLowercaseScheme: lowercaseScheme, + FlagLowercaseHost: lowercaseHost, + FlagRemoveDefaultPort: removeDefaultPort, + FlagRemoveDirectoryIndex: removeDirectoryIndex, + FlagRemoveDotSegments: removeDotSegments, + FlagRemoveFragment: removeFragment, + FlagForceHTTP: forceHTTP, + FlagRemoveDuplicateSlashes: removeDuplicateSlashes, + FlagRemoveWWW: removeWWW, + FlagAddWWW: addWWW, + FlagSortQuery: sortQuery, + FlagDecodeDWORDHost: decodeDWORDHost, + FlagDecodeOctalHost: decodeOctalHost, + FlagDecodeHexHost: decodeHexHost, + FlagRemoveUnnecessaryHostDots: removeUnncessaryHostDots, + FlagRemoveEmptyPortSeparator: removeEmptyPortSeparator, + FlagRemoveTrailingSlash: removeTrailingSlash, + FlagAddTrailingSlash: addTrailingSlash, +} + +// MustNormalizeURLString returns the normalized string, and panics if an error occurs. +// It takes an URL string as input, as well as the normalization flags. +func MustNormalizeURLString(u string, f NormalizationFlags) string { + result, e := NormalizeURLString(u, f) + if e != nil { + panic(e) + } + return result +} + +// NormalizeURLString returns the normalized string, or an error if it can't be parsed into an URL object. +// It takes an URL string as input, as well as the normalization flags. +func NormalizeURLString(u string, f NormalizationFlags) (string, error) { + parsed, err := url.Parse(u) + if err != nil { + return "", err + } + + if f&FlagLowercaseHost == FlagLowercaseHost { + parsed.Host = strings.ToLower(parsed.Host) + } + + // The idna package doesn't fully conform to RFC 5895 + // (https://tools.ietf.org/html/rfc5895), so we do it here. + // Taken from Go 1.8 cycle source, courtesy of bradfitz. + // TODO: Remove when (if?) idna package conforms to RFC 5895. + parsed.Host = width.Fold.String(parsed.Host) + parsed.Host = norm.NFC.String(parsed.Host) + if parsed.Host, err = idna.ToASCII(parsed.Host); err != nil { + return "", err + } + + return NormalizeURL(parsed, f), nil +} + +// NormalizeURL returns the normalized string. +// It takes a parsed URL object as input, as well as the normalization flags. +func NormalizeURL(u *url.URL, f NormalizationFlags) string { + for _, k := range flagsOrder { + if f&k == k { + flags[k](u) + } + } + return urlesc.Escape(u) +} + +func lowercaseScheme(u *url.URL) { + if len(u.Scheme) > 0 { + u.Scheme = strings.ToLower(u.Scheme) + } +} + +func lowercaseHost(u *url.URL) { + if len(u.Host) > 0 { + u.Host = strings.ToLower(u.Host) + } +} + +func removeDefaultPort(u *url.URL) { + if len(u.Host) > 0 { + scheme := strings.ToLower(u.Scheme) + u.Host = rxPort.ReplaceAllStringFunc(u.Host, func(val string) string { + if (scheme == "http" && val == defaultHttpPort) || (scheme == "https" && val == defaultHttpsPort) { + return "" + } + return val + }) + } +} + +func removeTrailingSlash(u *url.URL) { + if l := len(u.Path); l > 0 { + if strings.HasSuffix(u.Path, "/") { + u.Path = u.Path[:l-1] + } + } else if l = len(u.Host); l > 0 { + if strings.HasSuffix(u.Host, "/") { + u.Host = u.Host[:l-1] + } + } +} + +func addTrailingSlash(u *url.URL) { + if l := len(u.Path); l > 0 { + if !strings.HasSuffix(u.Path, "/") { + u.Path += "/" + } + } else if l = len(u.Host); l > 0 { + if !strings.HasSuffix(u.Host, "/") { + u.Host += "/" + } + } +} + +func removeDotSegments(u *url.URL) { + if len(u.Path) > 0 { + var dotFree []string + var lastIsDot bool + + sections := strings.Split(u.Path, "/") + for _, s := range sections { + if s == ".." { + if len(dotFree) > 0 { + dotFree = dotFree[:len(dotFree)-1] + } + } else if s != "." { + dotFree = append(dotFree, s) + } + lastIsDot = (s == "." || s == "..") + } + // Special case if host does not end with / and new path does not begin with / + u.Path = strings.Join(dotFree, "/") + if u.Host != "" && !strings.HasSuffix(u.Host, "/") && !strings.HasPrefix(u.Path, "/") { + u.Path = "/" + u.Path + } + // Special case if the last segment was a dot, make sure the path ends with a slash + if lastIsDot && !strings.HasSuffix(u.Path, "/") { + u.Path += "/" + } + } +} + +func removeDirectoryIndex(u *url.URL) { + if len(u.Path) > 0 { + u.Path = rxDirIndex.ReplaceAllString(u.Path, "$1") + } +} + +func removeFragment(u *url.URL) { + u.Fragment = "" +} + +func forceHTTP(u *url.URL) { + if strings.ToLower(u.Scheme) == "https" { + u.Scheme = "http" + } +} + +func removeDuplicateSlashes(u *url.URL) { + if len(u.Path) > 0 { + u.Path = rxDupSlashes.ReplaceAllString(u.Path, "/") + } +} + +func removeWWW(u *url.URL) { + if len(u.Host) > 0 && strings.HasPrefix(strings.ToLower(u.Host), "www.") { + u.Host = u.Host[4:] + } +} + +func addWWW(u *url.URL) { + if len(u.Host) > 0 && !strings.HasPrefix(strings.ToLower(u.Host), "www.") { + u.Host = "www." + u.Host + } +} + +func sortQuery(u *url.URL) { + q := u.Query() + + if len(q) > 0 { + arKeys := make([]string, len(q)) + i := 0 + for k := range q { + arKeys[i] = k + i++ + } + sort.Strings(arKeys) + buf := new(bytes.Buffer) + for _, k := range arKeys { + sort.Strings(q[k]) + for _, v := range q[k] { + if buf.Len() > 0 { + buf.WriteRune('&') + } + buf.WriteString(fmt.Sprintf("%s=%s", k, urlesc.QueryEscape(v))) + } + } + + // Rebuild the raw query string + u.RawQuery = buf.String() + } +} + +func decodeDWORDHost(u *url.URL) { + if len(u.Host) > 0 { + if matches := rxDWORDHost.FindStringSubmatch(u.Host); len(matches) > 2 { + var parts [4]int64 + + dword, _ := strconv.ParseInt(matches[1], 10, 0) + for i, shift := range []uint{24, 16, 8, 0} { + parts[i] = dword >> shift & 0xFF + } + u.Host = fmt.Sprintf("%d.%d.%d.%d%s", parts[0], parts[1], parts[2], parts[3], matches[2]) + } + } +} + +func decodeOctalHost(u *url.URL) { + if len(u.Host) > 0 { + if matches := rxOctalHost.FindStringSubmatch(u.Host); len(matches) > 5 { + var parts [4]int64 + + for i := 1; i <= 4; i++ { + parts[i-1], _ = strconv.ParseInt(matches[i], 8, 0) + } + u.Host = fmt.Sprintf("%d.%d.%d.%d%s", parts[0], parts[1], parts[2], parts[3], matches[5]) + } + } +} + +func decodeHexHost(u *url.URL) { + if len(u.Host) > 0 { + if matches := rxHexHost.FindStringSubmatch(u.Host); len(matches) > 2 { + // Conversion is safe because of regex validation + parsed, _ := strconv.ParseInt(matches[1], 16, 0) + // Set host as DWORD (base 10) encoded host + u.Host = fmt.Sprintf("%d%s", parsed, matches[2]) + // The rest is the same as decoding a DWORD host + decodeDWORDHost(u) + } + } +} + +func removeUnncessaryHostDots(u *url.URL) { + if len(u.Host) > 0 { + if matches := rxHostDots.FindStringSubmatch(u.Host); len(matches) > 1 { + // Trim the leading and trailing dots + u.Host = strings.Trim(matches[1], ".") + if len(matches) > 2 { + u.Host += matches[2] + } + } + } +} + +func removeEmptyPortSeparator(u *url.URL) { + if len(u.Host) > 0 { + u.Host = rxEmptyPort.ReplaceAllString(u.Host, "") + } +} diff --git a/vendor/github.com/PuerkitoBio/urlesc/.travis.yml b/vendor/github.com/PuerkitoBio/urlesc/.travis.yml new file mode 100644 index 0000000000..ba6b225f91 --- /dev/null +++ b/vendor/github.com/PuerkitoBio/urlesc/.travis.yml @@ -0,0 +1,15 @@ +language: go + +go: + - 1.4.x + - 1.5.x + - 1.6.x + - 1.7.x + - 1.8.x + - tip + +install: + - go build . + +script: + - go test -v diff --git a/vendor/github.com/PuerkitoBio/urlesc/LICENSE b/vendor/github.com/PuerkitoBio/urlesc/LICENSE new file mode 100644 index 0000000000..7448756763 --- /dev/null +++ b/vendor/github.com/PuerkitoBio/urlesc/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2012 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/PuerkitoBio/urlesc/README.md b/vendor/github.com/PuerkitoBio/urlesc/README.md new file mode 100644 index 0000000000..57aff0a539 --- /dev/null +++ b/vendor/github.com/PuerkitoBio/urlesc/README.md @@ -0,0 +1,16 @@ +urlesc [![Build Status](https://travis-ci.org/PuerkitoBio/urlesc.svg?branch=master)](https://travis-ci.org/PuerkitoBio/urlesc) [![GoDoc](http://godoc.org/github.com/PuerkitoBio/urlesc?status.svg)](http://godoc.org/github.com/PuerkitoBio/urlesc) +====== + +Package urlesc implements query escaping as per RFC 3986. + +It contains some parts of the net/url package, modified so as to allow +some reserved characters incorrectly escaped by net/url (see [issue 5684](https://github.com/golang/go/issues/5684)). + +## Install + + go get github.com/PuerkitoBio/urlesc + +## License + +Go license (BSD-3-Clause) + diff --git a/vendor/github.com/PuerkitoBio/urlesc/urlesc.go b/vendor/github.com/PuerkitoBio/urlesc/urlesc.go new file mode 100644 index 0000000000..1b84624594 --- /dev/null +++ b/vendor/github.com/PuerkitoBio/urlesc/urlesc.go @@ -0,0 +1,180 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package urlesc implements query escaping as per RFC 3986. +// It contains some parts of the net/url package, modified so as to allow +// some reserved characters incorrectly escaped by net/url. +// See https://github.com/golang/go/issues/5684 +package urlesc + +import ( + "bytes" + "net/url" + "strings" +) + +type encoding int + +const ( + encodePath encoding = 1 + iota + encodeUserPassword + encodeQueryComponent + encodeFragment +) + +// Return true if the specified character should be escaped when +// appearing in a URL string, according to RFC 3986. +func shouldEscape(c byte, mode encoding) bool { + // §2.3 Unreserved characters (alphanum) + if 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' { + return false + } + + switch c { + case '-', '.', '_', '~': // §2.3 Unreserved characters (mark) + return false + + // §2.2 Reserved characters (reserved) + case ':', '/', '?', '#', '[', ']', '@', // gen-delims + '!', '$', '&', '\'', '(', ')', '*', '+', ',', ';', '=': // sub-delims + // Different sections of the URL allow a few of + // the reserved characters to appear unescaped. + switch mode { + case encodePath: // §3.3 + // The RFC allows sub-delims and : @. + // '/', '[' and ']' can be used to assign meaning to individual path + // segments. This package only manipulates the path as a whole, + // so we allow those as well. That leaves only ? and # to escape. + return c == '?' || c == '#' + + case encodeUserPassword: // §3.2.1 + // The RFC allows : and sub-delims in + // userinfo. The parsing of userinfo treats ':' as special so we must escape + // all the gen-delims. + return c == ':' || c == '/' || c == '?' || c == '#' || c == '[' || c == ']' || c == '@' + + case encodeQueryComponent: // §3.4 + // The RFC allows / and ?. + return c != '/' && c != '?' + + case encodeFragment: // §4.1 + // The RFC text is silent but the grammar allows + // everything, so escape nothing but # + return c == '#' + } + } + + // Everything else must be escaped. + return true +} + +// QueryEscape escapes the string so it can be safely placed +// inside a URL query. +func QueryEscape(s string) string { + return escape(s, encodeQueryComponent) +} + +func escape(s string, mode encoding) string { + spaceCount, hexCount := 0, 0 + for i := 0; i < len(s); i++ { + c := s[i] + if shouldEscape(c, mode) { + if c == ' ' && mode == encodeQueryComponent { + spaceCount++ + } else { + hexCount++ + } + } + } + + if spaceCount == 0 && hexCount == 0 { + return s + } + + t := make([]byte, len(s)+2*hexCount) + j := 0 + for i := 0; i < len(s); i++ { + switch c := s[i]; { + case c == ' ' && mode == encodeQueryComponent: + t[j] = '+' + j++ + case shouldEscape(c, mode): + t[j] = '%' + t[j+1] = "0123456789ABCDEF"[c>>4] + t[j+2] = "0123456789ABCDEF"[c&15] + j += 3 + default: + t[j] = s[i] + j++ + } + } + return string(t) +} + +var uiReplacer = strings.NewReplacer( + "%21", "!", + "%27", "'", + "%28", "(", + "%29", ")", + "%2A", "*", +) + +// unescapeUserinfo unescapes some characters that need not to be escaped as per RFC3986. +func unescapeUserinfo(s string) string { + return uiReplacer.Replace(s) +} + +// Escape reassembles the URL into a valid URL string. +// The general form of the result is one of: +// +// scheme:opaque +// scheme://userinfo@host/path?query#fragment +// +// If u.Opaque is non-empty, String uses the first form; +// otherwise it uses the second form. +// +// In the second form, the following rules apply: +// - if u.Scheme is empty, scheme: is omitted. +// - if u.User is nil, userinfo@ is omitted. +// - if u.Host is empty, host/ is omitted. +// - if u.Scheme and u.Host are empty and u.User is nil, +// the entire scheme://userinfo@host/ is omitted. +// - if u.Host is non-empty and u.Path begins with a /, +// the form host/path does not add its own /. +// - if u.RawQuery is empty, ?query is omitted. +// - if u.Fragment is empty, #fragment is omitted. +func Escape(u *url.URL) string { + var buf bytes.Buffer + if u.Scheme != "" { + buf.WriteString(u.Scheme) + buf.WriteByte(':') + } + if u.Opaque != "" { + buf.WriteString(u.Opaque) + } else { + if u.Scheme != "" || u.Host != "" || u.User != nil { + buf.WriteString("//") + if ui := u.User; ui != nil { + buf.WriteString(unescapeUserinfo(ui.String())) + buf.WriteByte('@') + } + if h := u.Host; h != "" { + buf.WriteString(h) + } + } + if u.Path != "" && u.Path[0] != '/' && u.Host != "" { + buf.WriteByte('/') + } + buf.WriteString(escape(u.Path, encodePath)) + } + if u.RawQuery != "" { + buf.WriteByte('?') + buf.WriteString(u.RawQuery) + } + if u.Fragment != "" { + buf.WriteByte('#') + buf.WriteString(escape(u.Fragment, encodeFragment)) + } + return buf.String() +} diff --git a/vendor/github.com/go-openapi/jsonreference/internal/normalize_url.go b/vendor/github.com/go-openapi/jsonreference/internal/normalize_url.go deleted file mode 100644 index 8956c30884..0000000000 --- a/vendor/github.com/go-openapi/jsonreference/internal/normalize_url.go +++ /dev/null @@ -1,63 +0,0 @@ -package internal - -import ( - "net/url" - "regexp" - "strings" -) - -const ( - defaultHttpPort = ":80" - defaultHttpsPort = ":443" -) - -// Regular expressions used by the normalizations -var rxPort = regexp.MustCompile(`(:\d+)/?$`) -var rxDupSlashes = regexp.MustCompile(`/{2,}`) - -// NormalizeURL will normalize the specified URL -// This was added to replace a previous call to the no longer maintained purell library: -// The call that was used looked like the following: -// url.Parse(purell.NormalizeURL(parsed, purell.FlagsSafe|purell.FlagRemoveDuplicateSlashes)) -// -// To explain all that was included in the call above, purell.FlagsSafe was really just the following: -// - FlagLowercaseScheme -// - FlagLowercaseHost -// - FlagRemoveDefaultPort -// - FlagRemoveDuplicateSlashes (and this was mixed in with the |) -func NormalizeURL(u *url.URL) { - lowercaseScheme(u) - lowercaseHost(u) - removeDefaultPort(u) - removeDuplicateSlashes(u) -} - -func lowercaseScheme(u *url.URL) { - if len(u.Scheme) > 0 { - u.Scheme = strings.ToLower(u.Scheme) - } -} - -func lowercaseHost(u *url.URL) { - if len(u.Host) > 0 { - u.Host = strings.ToLower(u.Host) - } -} - -func removeDefaultPort(u *url.URL) { - if len(u.Host) > 0 { - scheme := strings.ToLower(u.Scheme) - u.Host = rxPort.ReplaceAllStringFunc(u.Host, func(val string) string { - if (scheme == "http" && val == defaultHttpPort) || (scheme == "https" && val == defaultHttpsPort) { - return "" - } - return val - }) - } -} - -func removeDuplicateSlashes(u *url.URL) { - if len(u.Path) > 0 { - u.Path = rxDupSlashes.ReplaceAllString(u.Path, "/") - } -} diff --git a/vendor/github.com/go-openapi/jsonreference/reference.go b/vendor/github.com/go-openapi/jsonreference/reference.go index cfdef03e5d..3bc0a6e26f 100644 --- a/vendor/github.com/go-openapi/jsonreference/reference.go +++ b/vendor/github.com/go-openapi/jsonreference/reference.go @@ -30,8 +30,8 @@ import ( "net/url" "strings" + "github.com/PuerkitoBio/purell" "github.com/go-openapi/jsonpointer" - "github.com/go-openapi/jsonreference/internal" ) const ( @@ -114,9 +114,7 @@ func (r *Ref) parse(jsonReferenceString string) error { return err } - internal.NormalizeURL(parsed) - - r.referenceURL = parsed + r.referenceURL, _ = url.Parse(purell.NormalizeURL(parsed, purell.FlagsSafe|purell.FlagRemoveDuplicateSlashes)) refURL := r.referenceURL if refURL.Scheme != "" && refURL.Host != "" { diff --git a/vendor/github.com/go-openapi/swag/.gitattributes b/vendor/github.com/go-openapi/swag/.gitattributes deleted file mode 100644 index 49ad52766a..0000000000 --- a/vendor/github.com/go-openapi/swag/.gitattributes +++ /dev/null @@ -1,2 +0,0 @@ -# gofmt always uses LF, whereas Git uses CRLF on Windows. -*.go text eol=lf diff --git a/vendor/github.com/go-openapi/swag/.golangci.yml b/vendor/github.com/go-openapi/swag/.golangci.yml index bf503e4000..813c47aa64 100644 --- a/vendor/github.com/go-openapi/swag/.golangci.yml +++ b/vendor/github.com/go-openapi/swag/.golangci.yml @@ -37,18 +37,3 @@ linters: - gci - gocognit - paralleltest - - thelper - - ifshort - - gomoddirectives - - cyclop - - forcetypeassert - - ireturn - - tagliatelle - - varnamelen - - goimports - - tenv - - golint - - exhaustruct - - nilnil - - nonamedreturns - - nosnakecase diff --git a/vendor/github.com/go-openapi/swag/.travis.yml b/vendor/github.com/go-openapi/swag/.travis.yml new file mode 100644 index 0000000000..fc25a88728 --- /dev/null +++ b/vendor/github.com/go-openapi/swag/.travis.yml @@ -0,0 +1,37 @@ +after_success: +- bash <(curl -s https://codecov.io/bash) +go: +- 1.14.x +- 1.x +arch: +- amd64 +jobs: + include: + # include arch ppc, but only for latest go version - skip testing for race + - go: 1.x + arch: ppc64le + install: ~ + script: + - go test -v + + #- go: 1.x + # arch: arm + # install: ~ + # script: + # - go test -v + + # include linting job, but only for latest go version and amd64 arch + - go: 1.x + arch: amd64 + install: + go get github.com/golangci/golangci-lint/cmd/golangci-lint + script: + - golangci-lint run --new-from-rev master +install: +- GO111MODULE=off go get -u gotest.tools/gotestsum +language: go +notifications: + slack: + secure: QUWvCkBBK09GF7YtEvHHVt70JOkdlNBG0nIKu/5qc4/nW5HP8I2w0SEf/XR2je0eED1Qe3L/AfMCWwrEj+IUZc3l4v+ju8X8R3Lomhme0Eb0jd1MTMCuPcBT47YCj0M7RON7vXtbFfm1hFJ/jLe5+9FXz0hpXsR24PJc5ZIi/ogNwkaPqG4BmndzecpSh0vc2FJPZUD9LT0I09REY/vXR0oQAalLkW0asGD5taHZTUZq/kBpsNxaAFrLM23i4mUcf33M5fjLpvx5LRICrX/57XpBrDh2TooBU6Qj3CgoY0uPRYUmSNxbVx1czNzl2JtEpb5yjoxfVPQeg0BvQM00G8LJINISR+ohrjhkZmAqchDupAX+yFrxTtORa78CtnIL6z/aTNlgwwVD8kvL/1pFA/JWYmKDmz93mV/+6wubGzNSQCstzjkFA4/iZEKewKUoRIAi/fxyscP6L/rCpmY/4llZZvrnyTqVbt6URWpopUpH4rwYqreXAtJxJsfBJIeSmUIiDIOMGkCTvyTEW3fWGmGoqWtSHLoaWDyAIGb7azb+KvfpWtEcoPFWfSWU+LGee0A/YsUhBl7ADB9A0CJEuR8q4BPpKpfLwPKSiKSAXL7zDkyjExyhtgqbSl2jS+rKIHOZNL8JkCcTP2MKMVd563C5rC5FMKqu3S9m2b6380E= +script: +- gotestsum -f short-verbose -- -race -coverprofile=coverage.txt -covermode=atomic ./... diff --git a/vendor/github.com/go-openapi/swag/doc.go b/vendor/github.com/go-openapi/swag/doc.go index 55094cb74c..8d2c8c5014 100644 --- a/vendor/github.com/go-openapi/swag/doc.go +++ b/vendor/github.com/go-openapi/swag/doc.go @@ -17,15 +17,16 @@ Package swag contains a bunch of helper functions for go-openapi and go-swagger You may also use it standalone for your projects. - - convert between value and pointers for builtin types - - convert from string to builtin types (wraps strconv) - - fast json concatenation - - search in path - - load from file or http - - name mangling + * convert between value and pointers for builtin types + * convert from string to builtin types (wraps strconv) + * fast json concatenation + * search in path + * load from file or http + * name mangling + This repo has only few dependencies outside of the standard library: - - YAML utilities depend on gopkg.in/yaml.v2 + * YAML utilities depend on gopkg.in/yaml.v2 */ package swag diff --git a/vendor/github.com/go-openapi/swag/file.go b/vendor/github.com/go-openapi/swag/file.go deleted file mode 100644 index 16accc55f8..0000000000 --- a/vendor/github.com/go-openapi/swag/file.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2015 go-swagger maintainers -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package swag - -import "mime/multipart" - -// File represents an uploaded file. -type File struct { - Data multipart.File - Header *multipart.FileHeader -} - -// Read bytes from the file -func (f *File) Read(p []byte) (n int, err error) { - return f.Data.Read(p) -} - -// Close the file -func (f *File) Close() error { - return f.Data.Close() -} diff --git a/vendor/github.com/go-openapi/swag/loading.go b/vendor/github.com/go-openapi/swag/loading.go index 00038c3773..9a60409725 100644 --- a/vendor/github.com/go-openapi/swag/loading.go +++ b/vendor/github.com/go-openapi/swag/loading.go @@ -16,11 +16,10 @@ package swag import ( "fmt" - "io" + "io/ioutil" "log" "net/http" "net/url" - "os" "path/filepath" "runtime" "strings" @@ -41,13 +40,13 @@ var LoadHTTPCustomHeaders = map[string]string{} // LoadFromFileOrHTTP loads the bytes from a file or a remote http server based on the path passed in func LoadFromFileOrHTTP(path string) ([]byte, error) { - return LoadStrategy(path, os.ReadFile, loadHTTPBytes(LoadHTTPTimeout))(path) + return LoadStrategy(path, ioutil.ReadFile, loadHTTPBytes(LoadHTTPTimeout))(path) } // LoadFromFileOrHTTPWithTimeout loads the bytes from a file or a remote http server based on the path passed in // timeout arg allows for per request overriding of the request timeout func LoadFromFileOrHTTPWithTimeout(path string, timeout time.Duration) ([]byte, error) { - return LoadStrategy(path, os.ReadFile, loadHTTPBytes(timeout))(path) + return LoadStrategy(path, ioutil.ReadFile, loadHTTPBytes(timeout))(path) } // LoadStrategy returns a loader function for a given path or uri @@ -87,7 +86,7 @@ func LoadStrategy(path string, local, remote func(string) ([]byte, error)) func( func loadHTTPBytes(timeout time.Duration) func(path string) ([]byte, error) { return func(path string) ([]byte, error) { client := &http.Client{Timeout: timeout} - req, err := http.NewRequest(http.MethodGet, path, nil) //nolint:noctx + req, err := http.NewRequest("GET", path, nil) // nolint: noctx if err != nil { return nil, err } @@ -116,6 +115,6 @@ func loadHTTPBytes(timeout time.Duration) func(path string) ([]byte, error) { return nil, fmt.Errorf("could not access document at %q [%s] ", path, resp.Status) } - return io.ReadAll(resp.Body) + return ioutil.ReadAll(resp.Body) } } diff --git a/vendor/github.com/go-openapi/swag/post_go18.go b/vendor/github.com/go-openapi/swag/post_go18.go index f5228b82c0..c2e686d313 100644 --- a/vendor/github.com/go-openapi/swag/post_go18.go +++ b/vendor/github.com/go-openapi/swag/post_go18.go @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build go1.8 // +build go1.8 package swag diff --git a/vendor/github.com/go-openapi/swag/post_go19.go b/vendor/github.com/go-openapi/swag/post_go19.go index 7c7da9c088..eb2f2d8bc7 100644 --- a/vendor/github.com/go-openapi/swag/post_go19.go +++ b/vendor/github.com/go-openapi/swag/post_go19.go @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build go1.9 // +build go1.9 package swag diff --git a/vendor/github.com/go-openapi/swag/pre_go18.go b/vendor/github.com/go-openapi/swag/pre_go18.go index 2757d9b95f..6607f33935 100644 --- a/vendor/github.com/go-openapi/swag/pre_go18.go +++ b/vendor/github.com/go-openapi/swag/pre_go18.go @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build !go1.8 // +build !go1.8 package swag diff --git a/vendor/github.com/go-openapi/swag/pre_go19.go b/vendor/github.com/go-openapi/swag/pre_go19.go index 0565db377b..4bae187d1e 100644 --- a/vendor/github.com/go-openapi/swag/pre_go19.go +++ b/vendor/github.com/go-openapi/swag/pre_go19.go @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build !go1.9 // +build !go1.9 package swag diff --git a/vendor/github.com/go-openapi/swag/util.go b/vendor/github.com/go-openapi/swag/util.go index f78ab684a0..193702f2ce 100644 --- a/vendor/github.com/go-openapi/swag/util.go +++ b/vendor/github.com/go-openapi/swag/util.go @@ -99,11 +99,10 @@ const ( ) // JoinByFormat joins a string array by a known format (e.g. swagger's collectionFormat attribute): -// -// ssv: space separated value -// tsv: tab separated value -// pipes: pipe (|) separated value -// csv: comma separated value (default) +// ssv: space separated value +// tsv: tab separated value +// pipes: pipe (|) separated value +// csv: comma separated value (default) func JoinByFormat(data []string, format string) []string { if len(data) == 0 { return data @@ -125,11 +124,11 @@ func JoinByFormat(data []string, format string) []string { } // SplitByFormat splits a string by a known format: +// ssv: space separated value +// tsv: tab separated value +// pipes: pipe (|) separated value +// csv: comma separated value (default) // -// ssv: space separated value -// tsv: tab separated value -// pipes: pipe (|) separated value -// csv: comma separated value (default) func SplitByFormat(data, format string) []string { if data == "" { return nil diff --git a/vendor/github.com/go-openapi/swag/yaml.go b/vendor/github.com/go-openapi/swag/yaml.go index f09ee609f3..ec96914405 100644 --- a/vendor/github.com/go-openapi/swag/yaml.go +++ b/vendor/github.com/go-openapi/swag/yaml.go @@ -22,7 +22,7 @@ import ( "github.com/mailru/easyjson/jlexer" "github.com/mailru/easyjson/jwriter" - yaml "gopkg.in/yaml.v3" + yaml "gopkg.in/yaml.v2" ) // YAMLMatcher matches yaml @@ -43,126 +43,16 @@ func YAMLToJSON(data interface{}) (json.RawMessage, error) { // BytesToYAMLDoc converts a byte slice into a YAML document func BytesToYAMLDoc(data []byte) (interface{}, error) { - var document yaml.Node // preserve order that is present in the document - if err := yaml.Unmarshal(data, &document); err != nil { + var canary map[interface{}]interface{} // validate this is an object and not a different type + if err := yaml.Unmarshal(data, &canary); err != nil { return nil, err } - if document.Kind != yaml.DocumentNode || len(document.Content) != 1 || document.Content[0].Kind != yaml.MappingNode { - return nil, fmt.Errorf("only YAML documents that are objects are supported") - } - return &document, nil -} - -func yamlNode(root *yaml.Node) (interface{}, error) { - switch root.Kind { - case yaml.DocumentNode: - return yamlDocument(root) - case yaml.SequenceNode: - return yamlSequence(root) - case yaml.MappingNode: - return yamlMapping(root) - case yaml.ScalarNode: - return yamlScalar(root) - case yaml.AliasNode: - return yamlNode(root.Alias) - default: - return nil, fmt.Errorf("unsupported YAML node type: %v", root.Kind) - } -} - -func yamlDocument(node *yaml.Node) (interface{}, error) { - if len(node.Content) != 1 { - return nil, fmt.Errorf("unexpected YAML Document node content length: %d", len(node.Content)) - } - return yamlNode(node.Content[0]) -} - -func yamlMapping(node *yaml.Node) (interface{}, error) { - m := make(JSONMapSlice, len(node.Content)/2) - - var j int - for i := 0; i < len(node.Content); i += 2 { - var nmi JSONMapItem - k, err := yamlStringScalarC(node.Content[i]) - if err != nil { - return nil, fmt.Errorf("unable to decode YAML map key: %w", err) - } - nmi.Key = k - v, err := yamlNode(node.Content[i+1]) - if err != nil { - return nil, fmt.Errorf("unable to process YAML map value for key %q: %w", k, err) - } - nmi.Value = v - m[j] = nmi - j++ - } - return m, nil -} - -func yamlSequence(node *yaml.Node) (interface{}, error) { - s := make([]interface{}, 0) - - for i := 0; i < len(node.Content); i++ { - - v, err := yamlNode(node.Content[i]) - if err != nil { - return nil, fmt.Errorf("unable to decode YAML sequence value: %w", err) - } - s = append(s, v) - } - return s, nil -} -const ( // See https://yaml.org/type/ - yamlStringScalar = "tag:yaml.org,2002:str" - yamlIntScalar = "tag:yaml.org,2002:int" - yamlBoolScalar = "tag:yaml.org,2002:bool" - yamlFloatScalar = "tag:yaml.org,2002:float" - yamlTimestamp = "tag:yaml.org,2002:timestamp" - yamlNull = "tag:yaml.org,2002:null" -) - -func yamlScalar(node *yaml.Node) (interface{}, error) { - switch node.LongTag() { - case yamlStringScalar: - return node.Value, nil - case yamlBoolScalar: - b, err := strconv.ParseBool(node.Value) - if err != nil { - return nil, fmt.Errorf("unable to process scalar node. Got %q. Expecting bool content: %w", node.Value, err) - } - return b, nil - case yamlIntScalar: - i, err := strconv.ParseInt(node.Value, 10, 64) - if err != nil { - return nil, fmt.Errorf("unable to process scalar node. Got %q. Expecting integer content: %w", node.Value, err) - } - return i, nil - case yamlFloatScalar: - f, err := strconv.ParseFloat(node.Value, 64) - if err != nil { - return nil, fmt.Errorf("unable to process scalar node. Got %q. Expecting float content: %w", node.Value, err) - } - return f, nil - case yamlTimestamp: - return node.Value, nil - case yamlNull: - return nil, nil - default: - return nil, fmt.Errorf("YAML tag %q is not supported", node.LongTag()) - } -} - -func yamlStringScalarC(node *yaml.Node) (string, error) { - if node.Kind != yaml.ScalarNode { - return "", fmt.Errorf("expecting a string scalar but got %q", node.Kind) - } - switch node.LongTag() { - case yamlStringScalar, yamlIntScalar, yamlFloatScalar: - return node.Value, nil - default: - return "", fmt.Errorf("YAML tag %q is not supported as map key", node.LongTag()) + var document yaml.MapSlice // preserve order that is present in the document + if err := yaml.Unmarshal(data, &document); err != nil { + return nil, err } + return document, nil } // JSONMapSlice represent a JSON object, with the order of keys maintained @@ -215,113 +105,6 @@ func (s *JSONMapSlice) UnmarshalEasyJSON(in *jlexer.Lexer) { *s = result } -func (s JSONMapSlice) MarshalYAML() (interface{}, error) { - var n yaml.Node - n.Kind = yaml.DocumentNode - var nodes []*yaml.Node - for _, item := range s { - nn, err := json2yaml(item.Value) - if err != nil { - return nil, err - } - ns := []*yaml.Node{ - { - Kind: yaml.ScalarNode, - Tag: yamlStringScalar, - Value: item.Key, - }, - nn, - } - nodes = append(nodes, ns...) - } - - n.Content = []*yaml.Node{ - { - Kind: yaml.MappingNode, - Content: nodes, - }, - } - - return yaml.Marshal(&n) -} - -func json2yaml(item interface{}) (*yaml.Node, error) { - switch val := item.(type) { - case JSONMapSlice: - var n yaml.Node - n.Kind = yaml.MappingNode - for i := range val { - childNode, err := json2yaml(&val[i].Value) - if err != nil { - return nil, err - } - n.Content = append(n.Content, &yaml.Node{ - Kind: yaml.ScalarNode, - Tag: yamlStringScalar, - Value: val[i].Key, - }, childNode) - } - return &n, nil - case map[string]interface{}: - var n yaml.Node - n.Kind = yaml.MappingNode - for k, v := range val { - childNode, err := json2yaml(v) - if err != nil { - return nil, err - } - n.Content = append(n.Content, &yaml.Node{ - Kind: yaml.ScalarNode, - Tag: yamlStringScalar, - Value: k, - }, childNode) - } - return &n, nil - case []interface{}: - var n yaml.Node - n.Kind = yaml.SequenceNode - for i := range val { - childNode, err := json2yaml(val[i]) - if err != nil { - return nil, err - } - n.Content = append(n.Content, childNode) - } - return &n, nil - case string: - return &yaml.Node{ - Kind: yaml.ScalarNode, - Tag: yamlStringScalar, - Value: val, - }, nil - case float64: - return &yaml.Node{ - Kind: yaml.ScalarNode, - Tag: yamlFloatScalar, - Value: strconv.FormatFloat(val, 'f', -1, 64), - }, nil - case int64: - return &yaml.Node{ - Kind: yaml.ScalarNode, - Tag: yamlIntScalar, - Value: strconv.FormatInt(val, 10), - }, nil - case uint64: - return &yaml.Node{ - Kind: yaml.ScalarNode, - Tag: yamlIntScalar, - Value: strconv.FormatUint(val, 10), - }, nil - case bool: - return &yaml.Node{ - Kind: yaml.ScalarNode, - Tag: yamlBoolScalar, - Value: strconv.FormatBool(val), - }, nil - } - return nil, nil -} - // JSONMapItem represents the value of a key in a JSON object held by JSONMapSlice type JSONMapItem struct { Key string @@ -390,10 +173,23 @@ func transformData(input interface{}) (out interface{}, err error) { } switch in := input.(type) { - case yaml.Node: - return yamlNode(&in) - case *yaml.Node: - return yamlNode(in) + case yaml.MapSlice: + + o := make(JSONMapSlice, len(in)) + for i, mi := range in { + var nmi JSONMapItem + if nmi.Key, err = format(mi.Key); err != nil { + return nil, err + } + + v, ert := transformData(mi.Value) + if ert != nil { + return nil, ert + } + nmi.Value = v + o[i] = nmi + } + return o, nil case map[interface{}]interface{}: o := make(JSONMapSlice, 0, len(in)) for ke, va := range in { diff --git a/vendor/github.com/google/gnostic/jsonschema/display.go b/vendor/github.com/google/gnostic/jsonschema/display.go index 8677ed49a0..028a760a91 100644 --- a/vendor/github.com/google/gnostic/jsonschema/display.go +++ b/vendor/github.com/google/gnostic/jsonschema/display.go @@ -46,23 +46,8 @@ func (schema *Schema) describeSchema(indent string) string { if schema.Schema != nil { result += indent + "$schema: " + *(schema.Schema) + "\n" } - if schema.ReadOnly != nil && *schema.ReadOnly { - result += indent + fmt.Sprintf("readOnly: %+v\n", *(schema.ReadOnly)) - } - if schema.WriteOnly != nil && *schema.WriteOnly { - result += indent + fmt.Sprintf("writeOnly: %+v\n", *(schema.WriteOnly)) - } if schema.ID != nil { - switch strings.TrimSuffix(*schema.Schema, "#") { - case "http://json-schema.org/draft-04/schema#": - fallthrough - case "#": - fallthrough - case "": - result += indent + "id: " + *(schema.ID) + "\n" - default: - result += indent + "$id: " + *(schema.ID) + "\n" - } + result += indent + "id: " + *(schema.ID) + "\n" } if schema.MultipleOf != nil { result += indent + fmt.Sprintf("multipleOf: %+v\n", *(schema.MultipleOf)) diff --git a/vendor/github.com/google/gnostic/jsonschema/models.go b/vendor/github.com/google/gnostic/jsonschema/models.go index 0d877249ab..4781bdc5f5 100644 --- a/vendor/github.com/google/gnostic/jsonschema/models.go +++ b/vendor/github.com/google/gnostic/jsonschema/models.go @@ -23,11 +23,9 @@ import "gopkg.in/yaml.v3" // All fields are pointers and are nil if the associated values // are not specified. type Schema struct { - Schema *string // $schema - ID *string // id keyword used for $ref resolution scope - Ref *string // $ref, i.e. JSON Pointers - ReadOnly *bool - WriteOnly *bool + Schema *string // $schema + ID *string // id keyword used for $ref resolution scope + Ref *string // $ref, i.e. JSON Pointers // http://json-schema.org/latest/json-schema-validation.html // 5.1. Validation keywords for numeric instances (number and integer) diff --git a/vendor/github.com/google/gnostic/jsonschema/reader.go b/vendor/github.com/google/gnostic/jsonschema/reader.go index a909a34128..b8583d4660 100644 --- a/vendor/github.com/google/gnostic/jsonschema/reader.go +++ b/vendor/github.com/google/gnostic/jsonschema/reader.go @@ -165,6 +165,7 @@ func NewSchemaFromObject(jsonData *yaml.Node) *Schema { default: fmt.Printf("schemaValue: unexpected node %+v\n", jsonData) + return nil } return nil diff --git a/vendor/github.com/google/gnostic/jsonschema/writer.go b/vendor/github.com/google/gnostic/jsonschema/writer.go index 15b1f90506..340dc5f933 100644 --- a/vendor/github.com/google/gnostic/jsonschema/writer.go +++ b/vendor/github.com/google/gnostic/jsonschema/writer.go @@ -16,7 +16,6 @@ package jsonschema import ( "fmt" - "strings" "gopkg.in/yaml.v3" ) @@ -34,11 +33,7 @@ func renderMappingNode(node *yaml.Node, indent string) (result string) { value := node.Content[i+1] switch value.Kind { case yaml.ScalarNode: - if value.Tag == "!!bool" { - result += value.Value - } else { - result += "\"" + value.Value + "\"" - } + result += "\"" + value.Value + "\"" case yaml.MappingNode: result += renderMappingNode(value, innerIndent) case yaml.SequenceNode: @@ -63,11 +58,7 @@ func renderSequenceNode(node *yaml.Node, indent string) (result string) { item := node.Content[i] switch item.Kind { case yaml.ScalarNode: - if item.Tag == "!!bool" { - result += innerIndent + item.Value - } else { - result += innerIndent + "\"" + item.Value + "\"" - } + result += innerIndent + "\"" + item.Value + "\"" case yaml.MappingNode: result += innerIndent + renderMappingNode(item, innerIndent) + "" default: @@ -269,26 +260,11 @@ func (schema *Schema) nodeValue() *yaml.Node { content = appendPair(content, "title", nodeForString(*schema.Title)) } if schema.ID != nil { - switch strings.TrimSuffix(*schema.Schema, "#") { - case "http://json-schema.org/draft-04/schema": - fallthrough - case "#": - fallthrough - case "": - content = appendPair(content, "id", nodeForString(*schema.ID)) - default: - content = appendPair(content, "$id", nodeForString(*schema.ID)) - } + content = appendPair(content, "id", nodeForString(*schema.ID)) } if schema.Schema != nil { content = appendPair(content, "$schema", nodeForString(*schema.Schema)) } - if schema.ReadOnly != nil && *schema.ReadOnly { - content = appendPair(content, "readOnly", nodeForBoolean(*schema.ReadOnly)) - } - if schema.WriteOnly != nil && *schema.WriteOnly { - content = appendPair(content, "writeOnly", nodeForBoolean(*schema.WriteOnly)) - } if schema.Type != nil { content = appendPair(content, "type", schema.Type.nodeValue()) } diff --git a/vendor/github.com/google/gnostic/openapiv2/OpenAPIv2.go b/vendor/github.com/google/gnostic/openapiv2/OpenAPIv2.go index 28c2777d51..0f17907667 100644 --- a/vendor/github.com/google/gnostic/openapiv2/OpenAPIv2.go +++ b/vendor/github.com/google/gnostic/openapiv2/OpenAPIv2.go @@ -7887,12 +7887,7 @@ func (m *Oauth2Scopes) ToRawInfo() *yaml.Node { if m == nil { return info } - if m.AdditionalProperties != nil { - for _, item := range m.AdditionalProperties { - info.Content = append(info.Content, compiler.NewScalarNodeForString(item.Name)) - info.Content = append(info.Content, compiler.NewScalarNodeForString(item.Value)) - } - } + // &{Name:additionalProperties Type:NamedString StringEnumValues:[] MapType:string Repeated:true Pattern: Implicit:true Description:} return info } diff --git a/vendor/github.com/google/gnostic/openapiv3/OpenAPIv3.go b/vendor/github.com/google/gnostic/openapiv3/OpenAPIv3.go index d54a84db7c..5f4a7025ea 100644 --- a/vendor/github.com/google/gnostic/openapiv3/OpenAPIv3.go +++ b/vendor/github.com/google/gnostic/openapiv3/OpenAPIv3.go @@ -8560,12 +8560,7 @@ func (m *Strings) ToRawInfo() *yaml.Node { if m == nil { return info } - if m.AdditionalProperties != nil { - for _, item := range m.AdditionalProperties { - info.Content = append(info.Content, compiler.NewScalarNodeForString(item.Name)) - info.Content = append(info.Content, compiler.NewScalarNodeForString(item.Value)) - } - } + // &{Name:additionalProperties Type:NamedString StringEnumValues:[] MapType:string Repeated:true Pattern: Implicit:true Description:} return info } diff --git a/vendor/github.com/google/gnostic/openapiv3/OpenAPIv3.pb.go b/vendor/github.com/google/gnostic/openapiv3/OpenAPIv3.pb.go index 90a56f5526..499e7f932d 100644 --- a/vendor/github.com/google/gnostic/openapiv3/OpenAPIv3.pb.go +++ b/vendor/github.com/google/gnostic/openapiv3/OpenAPIv3.pb.go @@ -16,8 +16,8 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.28.0 -// protoc v3.19.4 +// protoc-gen-go v1.26.0 +// protoc v3.18.1 // source: openapiv3/OpenAPIv3.proto package openapi_v3 @@ -6760,13 +6760,12 @@ var file_openapiv3_OpenAPIv3_proto_rawDesc = []byte{ 0x5f, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x33, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x64, 0x41, 0x6e, 0x79, 0x52, 0x16, 0x73, 0x70, 0x65, 0x63, 0x69, 0x66, 0x69, 0x63, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x42, 0x56, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x42, 0x3e, 0x0a, 0x0e, 0x6f, 0x72, 0x67, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x61, 0x70, 0x69, 0x5f, 0x76, 0x33, 0x42, 0x0c, 0x4f, 0x70, 0x65, 0x6e, 0x41, 0x50, 0x49, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, - 0x5a, 0x2e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2f, 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x2f, 0x6f, 0x70, 0x65, 0x6e, - 0x61, 0x70, 0x69, 0x76, 0x33, 0x3b, 0x6f, 0x70, 0x65, 0x6e, 0x61, 0x70, 0x69, 0x5f, 0x76, 0x33, - 0xa2, 0x02, 0x03, 0x4f, 0x41, 0x53, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x5a, 0x16, 0x2e, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x61, 0x70, 0x69, 0x76, 0x33, 0x3b, 0x6f, 0x70, + 0x65, 0x6e, 0x61, 0x70, 0x69, 0x5f, 0x76, 0x33, 0xa2, 0x02, 0x03, 0x4f, 0x41, 0x53, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/vendor/github.com/google/gnostic/openapiv3/OpenAPIv3.proto b/vendor/github.com/google/gnostic/openapiv3/OpenAPIv3.proto index 7aede5ed90..1be335b89b 100644 --- a/vendor/github.com/google/gnostic/openapiv3/OpenAPIv3.proto +++ b/vendor/github.com/google/gnostic/openapiv3/OpenAPIv3.proto @@ -42,7 +42,7 @@ option java_package = "org.openapi_v3"; option objc_class_prefix = "OAS"; // The Go package name. -option go_package = "github.com/google/gnostic/openapiv3;openapi_v3"; +option go_package = "./openapiv3;openapi_v3"; message AdditionalPropertiesItem { oneof oneof { diff --git a/vendor/github.com/google/gnostic/openapiv3/README.md b/vendor/github.com/google/gnostic/openapiv3/README.md index 83603b82aa..5ee12d92e2 100644 --- a/vendor/github.com/google/gnostic/openapiv3/README.md +++ b/vendor/github.com/google/gnostic/openapiv3/README.md @@ -19,7 +19,3 @@ for OpenAPI. The schema-generator directory contains support code which generates openapi-3.1.json from the OpenAPI 3.1 specification document (Markdown). - -### How to rebuild - -`protoc -I=. -I=third_party --go_out=. --go_opt=paths=source_relative openapiv3/*.proto` \ No newline at end of file diff --git a/vendor/github.com/google/gnostic/openapiv3/annotations.pb.go b/vendor/github.com/google/gnostic/openapiv3/annotations.pb.go deleted file mode 100644 index ae242f3043..0000000000 --- a/vendor/github.com/google/gnostic/openapiv3/annotations.pb.go +++ /dev/null @@ -1,183 +0,0 @@ -// Copyright 2022 Google LLC. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Code generated by protoc-gen-go. DO NOT EDIT. -// versions: -// protoc-gen-go v1.28.0 -// protoc v3.19.4 -// source: openapiv3/annotations.proto - -package openapi_v3 - -import ( - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" - descriptorpb "google.golang.org/protobuf/types/descriptorpb" - reflect "reflect" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -var file_openapiv3_annotations_proto_extTypes = []protoimpl.ExtensionInfo{ - { - ExtendedType: (*descriptorpb.FileOptions)(nil), - ExtensionType: (*Document)(nil), - Field: 1143, - Name: "openapi.v3.document", - Tag: "bytes,1143,opt,name=document", - Filename: "openapiv3/annotations.proto", - }, - { - ExtendedType: (*descriptorpb.MethodOptions)(nil), - ExtensionType: (*Operation)(nil), - Field: 1143, - Name: "openapi.v3.operation", - Tag: "bytes,1143,opt,name=operation", - Filename: "openapiv3/annotations.proto", - }, - { - ExtendedType: (*descriptorpb.MessageOptions)(nil), - ExtensionType: (*Schema)(nil), - Field: 1143, - Name: "openapi.v3.schema", - Tag: "bytes,1143,opt,name=schema", - Filename: "openapiv3/annotations.proto", - }, - { - ExtendedType: (*descriptorpb.FieldOptions)(nil), - ExtensionType: (*Schema)(nil), - Field: 1143, - Name: "openapi.v3.property", - Tag: "bytes,1143,opt,name=property", - Filename: "openapiv3/annotations.proto", - }, -} - -// Extension fields to descriptorpb.FileOptions. -var ( - // optional openapi.v3.Document document = 1143; - E_Document = &file_openapiv3_annotations_proto_extTypes[0] -) - -// Extension fields to descriptorpb.MethodOptions. -var ( - // optional openapi.v3.Operation operation = 1143; - E_Operation = &file_openapiv3_annotations_proto_extTypes[1] -) - -// Extension fields to descriptorpb.MessageOptions. -var ( - // optional openapi.v3.Schema schema = 1143; - E_Schema = &file_openapiv3_annotations_proto_extTypes[2] -) - -// Extension fields to descriptorpb.FieldOptions. -var ( - // optional openapi.v3.Schema property = 1143; - E_Property = &file_openapiv3_annotations_proto_extTypes[3] -) - -var File_openapiv3_annotations_proto protoreflect.FileDescriptor - -var file_openapiv3_annotations_proto_rawDesc = []byte{ - 0x0a, 0x1b, 0x6f, 0x70, 0x65, 0x6e, 0x61, 0x70, 0x69, 0x76, 0x33, 0x2f, 0x61, 0x6e, 0x6e, 0x6f, - 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x6f, - 0x70, 0x65, 0x6e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x33, 0x1a, 0x19, 0x6f, 0x70, 0x65, 0x6e, 0x61, - 0x70, 0x69, 0x76, 0x33, 0x2f, 0x4f, 0x70, 0x65, 0x6e, 0x41, 0x50, 0x49, 0x76, 0x33, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x20, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x3a, 0x4f, 0x0a, 0x08, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, - 0x6e, 0x74, 0x12, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x18, 0xf7, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x61, 0x70, - 0x69, 0x2e, 0x76, 0x33, 0x2e, 0x44, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x08, 0x64, - 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x3a, 0x54, 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x4f, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x18, 0xf7, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6f, 0x70, - 0x65, 0x6e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x33, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x52, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x4c, 0x0a, - 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, 0x1f, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, - 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0xf7, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x12, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x33, 0x2e, 0x53, 0x63, 0x68, - 0x65, 0x6d, 0x61, 0x52, 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x3a, 0x4e, 0x0a, 0x08, 0x70, - 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x12, 0x1d, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x4f, - 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0xf7, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, - 0x6f, 0x70, 0x65, 0x6e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x33, 0x2e, 0x53, 0x63, 0x68, 0x65, 0x6d, - 0x61, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x42, 0x5a, 0x0a, 0x0e, 0x6f, - 0x72, 0x67, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x61, 0x70, 0x69, 0x5f, 0x76, 0x33, 0x42, 0x10, 0x41, - 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, - 0x01, 0x5a, 0x2e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x2f, 0x6f, 0x70, 0x65, - 0x6e, 0x61, 0x70, 0x69, 0x76, 0x33, 0x3b, 0x6f, 0x70, 0x65, 0x6e, 0x61, 0x70, 0x69, 0x5f, 0x76, - 0x33, 0xa2, 0x02, 0x03, 0x4f, 0x41, 0x53, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} - -var file_openapiv3_annotations_proto_goTypes = []interface{}{ - (*descriptorpb.FileOptions)(nil), // 0: google.protobuf.FileOptions - (*descriptorpb.MethodOptions)(nil), // 1: google.protobuf.MethodOptions - (*descriptorpb.MessageOptions)(nil), // 2: google.protobuf.MessageOptions - (*descriptorpb.FieldOptions)(nil), // 3: google.protobuf.FieldOptions - (*Document)(nil), // 4: openapi.v3.Document - (*Operation)(nil), // 5: openapi.v3.Operation - (*Schema)(nil), // 6: openapi.v3.Schema -} -var file_openapiv3_annotations_proto_depIdxs = []int32{ - 0, // 0: openapi.v3.document:extendee -> google.protobuf.FileOptions - 1, // 1: openapi.v3.operation:extendee -> google.protobuf.MethodOptions - 2, // 2: openapi.v3.schema:extendee -> google.protobuf.MessageOptions - 3, // 3: openapi.v3.property:extendee -> google.protobuf.FieldOptions - 4, // 4: openapi.v3.document:type_name -> openapi.v3.Document - 5, // 5: openapi.v3.operation:type_name -> openapi.v3.Operation - 6, // 6: openapi.v3.schema:type_name -> openapi.v3.Schema - 6, // 7: openapi.v3.property:type_name -> openapi.v3.Schema - 8, // [8:8] is the sub-list for method output_type - 8, // [8:8] is the sub-list for method input_type - 4, // [4:8] is the sub-list for extension type_name - 0, // [0:4] is the sub-list for extension extendee - 0, // [0:0] is the sub-list for field type_name -} - -func init() { file_openapiv3_annotations_proto_init() } -func file_openapiv3_annotations_proto_init() { - if File_openapiv3_annotations_proto != nil { - return - } - file_openapiv3_OpenAPIv3_proto_init() - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_openapiv3_annotations_proto_rawDesc, - NumEnums: 0, - NumMessages: 0, - NumExtensions: 4, - NumServices: 0, - }, - GoTypes: file_openapiv3_annotations_proto_goTypes, - DependencyIndexes: file_openapiv3_annotations_proto_depIdxs, - ExtensionInfos: file_openapiv3_annotations_proto_extTypes, - }.Build() - File_openapiv3_annotations_proto = out.File - file_openapiv3_annotations_proto_rawDesc = nil - file_openapiv3_annotations_proto_goTypes = nil - file_openapiv3_annotations_proto_depIdxs = nil -} diff --git a/vendor/github.com/google/gnostic/openapiv3/annotations.proto b/vendor/github.com/google/gnostic/openapiv3/annotations.proto deleted file mode 100644 index 0bd87810db..0000000000 --- a/vendor/github.com/google/gnostic/openapiv3/annotations.proto +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2022 Google LLC. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package openapi.v3; - -import "openapiv3/OpenAPIv3.proto"; -import "google/protobuf/descriptor.proto"; - -// This option lets the proto compiler generate Java code inside the package -// name (see below) instead of inside an outer class. It creates a simpler -// developer experience by reducing one-level of name nesting and be -// consistent with most programming languages that don't support outer classes. -option java_multiple_files = true; - -// The Java outer classname should be the filename in UpperCamelCase. This -// class is only used to hold proto descriptor, so developers don't need to -// work with it directly. -option java_outer_classname = "AnnotationsProto"; - -// The Java package name must be proto package name with proper prefix. -option java_package = "org.openapi_v3"; - -// A reasonable prefix for the Objective-C symbols generated from the package. -// It should at a minimum be 3 characters long, all uppercase, and convention -// is to use an abbreviation of the package name. Something short, but -// hopefully unique enough to not conflict with things that may come along in -// the future. 'GPB' is reserved for the protocol buffer implementation itself. -option objc_class_prefix = "OAS"; - -// The Go package name. -option go_package = "github.com/google/gnostic/openapiv3;openapi_v3"; - -extend google.protobuf.FileOptions { - Document document = 1143; -} - -extend google.protobuf.MethodOptions { - Operation operation = 1143; -} - -extend google.protobuf.MessageOptions { - Schema schema = 1143; -} - -extend google.protobuf.FieldOptions { - Schema property = 1143; -} \ No newline at end of file diff --git a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/LICENSE b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/LICENSE similarity index 100% rename from vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/LICENSE rename to vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/LICENSE diff --git a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1/doc.go b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1/doc.go similarity index 100% rename from vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1/doc.go rename to vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1/doc.go diff --git a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1/register.go b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1/register.go similarity index 100% rename from vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1/register.go rename to vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1/register.go diff --git a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1/types.go b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1/types.go similarity index 93% rename from vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1/types.go rename to vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1/types.go index 51ed543b0d..b9745df215 100644 --- a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1/types.go +++ b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1/types.go @@ -29,7 +29,7 @@ import ( // VolumeSnapshot is a user's request for either creating a point-in-time // snapshot of a persistent volume, or binding to a pre-existing snapshot. // +kubebuilder:object:root=true -// +kubebuilder:resource:scope=Namespaced,shortName=vs +// +kubebuilder:resource:scope=Namespaced // +kubebuilder:subresource:status // +kubebuilder:printcolumn:name="ReadyToUse",type=boolean,JSONPath=`.status.readyToUse`,description="Indicates if the snapshot is ready to be used to restore a volume." // +kubebuilder:printcolumn:name="SourcePVC",type=string,JSONPath=`.spec.source.persistentVolumeClaimName`,description="If a new snapshot needs to be created, this contains the name of the source PVC from which this snapshot was (or will be) created." @@ -42,7 +42,7 @@ import ( type VolumeSnapshot struct { metav1.TypeMeta `json:",inline"` // Standard object's metadata. - // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + // More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata // +optional metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` @@ -123,11 +123,11 @@ type VolumeSnapshotSource struct { // VolumeSnapshotStatus and VolumeSnapshotContentStatus. Fields in VolumeSnapshotStatus // are updated based on fields in VolumeSnapshotContentStatus. They are eventual // consistency. These fields are duplicate in both objects due to the following reasons: -// - Fields in VolumeSnapshotContentStatus can be used for filtering when importing a -// volumesnapshot. -// - VolumsnapshotStatus is used by end users because they cannot see VolumeSnapshotContent. -// - CSI snapshotter sidecar is light weight as it only watches VolumeSnapshotContent -// object, not VolumeSnapshot object. +// - Fields in VolumeSnapshotContentStatus can be used for filtering when importing a +// volumesnapshot. +// - VolumsnapshotStatus is used by end users because they cannot see VolumeSnapshotContent. +// - CSI snapshotter sidecar is light weight as it only watches VolumeSnapshotContent +// object, not VolumeSnapshot object. type VolumeSnapshotStatus struct { // boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent // object to which this VolumeSnapshot object intends to bind to. @@ -179,7 +179,7 @@ type VolumeSnapshotStatus struct { // This field could be helpful to upper level controllers(i.e., application controller) // to decide whether they should continue on waiting for the snapshot to be created // based on the type of error reported. - // The snapshot controller will keep retrying when an error occurs during the + // The snapshot controller will keep retrying when an error occurrs during the // snapshot creation. Upon success, this error field will be cleared. // +optional Error *VolumeSnapshotError `json:"error,omitempty" protobuf:"bytes,5,opt,name=error,casttype=VolumeSnapshotError"` @@ -194,14 +194,14 @@ type VolumeSnapshotStatus struct { // name in a VolumeSnapshot object. // VolumeSnapshotClasses are non-namespaced // +kubebuilder:object:root=true -// +kubebuilder:resource:scope=Cluster,shortName=vsclass;vsclasses +// +kubebuilder:resource:scope=Cluster // +kubebuilder:printcolumn:name="Driver",type=string,JSONPath=`.driver` // +kubebuilder:printcolumn:name="DeletionPolicy",type=string,JSONPath=`.deletionPolicy`,description="Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted." // +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` type VolumeSnapshotClass struct { metav1.TypeMeta `json:",inline"` // Standard object's metadata. - // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + // More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata // +optional metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` @@ -230,7 +230,7 @@ type VolumeSnapshotClass struct { type VolumeSnapshotClassList struct { metav1.TypeMeta `json:",inline"` // Standard list metadata - // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + // More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata // +optional metav1.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` @@ -245,7 +245,7 @@ type VolumeSnapshotClassList struct { // VolumeSnapshotContent represents the actual "on-disk" snapshot object in the // underlying storage system // +kubebuilder:object:root=true -// +kubebuilder:resource:scope=Cluster,shortName=vsc;vscs +// +kubebuilder:resource:scope=Cluster // +kubebuilder:subresource:status // +kubebuilder:printcolumn:name="ReadyToUse",type=boolean,JSONPath=`.status.readyToUse`,description="Indicates if the snapshot is ready to be used to restore a volume." // +kubebuilder:printcolumn:name="RestoreSize",type=integer,JSONPath=`.status.restoreSize`,description="Represents the complete size of the snapshot in bytes" @@ -257,7 +257,7 @@ type VolumeSnapshotClassList struct { type VolumeSnapshotContent struct { metav1.TypeMeta `json:",inline"` // Standard object's metadata. - // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + // More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata // +optional metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` @@ -328,19 +328,13 @@ type VolumeSnapshotContentSpec struct { // This field is immutable after creation. // Required. Source VolumeSnapshotContentSource `json:"source" protobuf:"bytes,5,opt,name=source"` - - // SourceVolumeMode is the mode of the volume whose snapshot is taken. - // Can be either “Filesystem” or “Block”. - // If not specified, it indicates the source volume's mode is unknown. - // This field is immutable. - // This field is an alpha field. - // +optional - SourceVolumeMode *core_v1.PersistentVolumeMode `json:"sourceVolumeMode" protobuf:"bytes,6,opt,name=sourceVolumeMode"` } // VolumeSnapshotContentSource represents the CSI source of a snapshot. // Exactly one of its members must be set. // Members in VolumeSnapshotContentSource are immutable. +// TODO(xiangqian): Add a webhook to ensure that VolumeSnapshotContentSource members +// will be immutable once specified. type VolumeSnapshotContentSource struct { // volumeHandle specifies the CSI "volume_id" of the volume from which a snapshot // should be dynamically taken from. @@ -361,11 +355,11 @@ type VolumeSnapshotContentSource struct { // VolumeSnapshotStatus and VolumeSnapshotContentStatus. Fields in VolumeSnapshotStatus // are updated based on fields in VolumeSnapshotContentStatus. They are eventual // consistency. These fields are duplicate in both objects due to the following reasons: -// - Fields in VolumeSnapshotContentStatus can be used for filtering when importing a -// volumesnapshot. -// - VolumsnapshotStatus is used by end users because they cannot see VolumeSnapshotContent. -// - CSI snapshotter sidecar is light weight as it only watches VolumeSnapshotContent -// object, not VolumeSnapshot object. +// - Fields in VolumeSnapshotContentStatus can be used for filtering when importing a +// volumesnapshot. +// - VolumsnapshotStatus is used by end users because they cannot see VolumeSnapshotContent. +// - CSI snapshotter sidecar is light weight as it only watches VolumeSnapshotContent +// object, not VolumeSnapshot object. type VolumeSnapshotContentStatus struct { // snapshotHandle is the CSI "snapshot_id" of a snapshot on the underlying storage system. // If not specified, it indicates that dynamic snapshot creation has either failed diff --git a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1/zz_generated.deepcopy.go b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1/zz_generated.deepcopy.go similarity index 98% rename from vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1/zz_generated.deepcopy.go rename to vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1/zz_generated.deepcopy.go index 901b2b080c..febab0cd40 100644 --- a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1/zz_generated.deepcopy.go +++ b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1/zz_generated.deepcopy.go @@ -1,8 +1,7 @@ -//go:build !ignore_autogenerated // +build !ignore_autogenerated /* -Copyright 2022 The Kubernetes Authors. +Copyright 2020 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -22,7 +21,6 @@ limitations under the License. package v1 import ( - corev1 "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -225,11 +223,6 @@ func (in *VolumeSnapshotContentSpec) DeepCopyInto(out *VolumeSnapshotContentSpec **out = **in } in.Source.DeepCopyInto(&out.Source) - if in.SourceVolumeMode != nil { - in, out := &in.SourceVolumeMode, &out.SourceVolumeMode - *out = new(corev1.PersistentVolumeMode) - **out = **in - } return } diff --git a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1beta1/doc.go b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1beta1/doc.go new file mode 100644 index 0000000000..d8a1f5dad2 --- /dev/null +++ b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1beta1/doc.go @@ -0,0 +1,20 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +k8s:deepcopy-gen=package +// +groupName=snapshot.storage.k8s.io + +package v1beta1 diff --git a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1beta1/register.go b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1beta1/register.go new file mode 100644 index 0000000000..91f4514242 --- /dev/null +++ b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1beta1/register.go @@ -0,0 +1,58 @@ +/* +Copyright 2018 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// GroupName is the group name use in this package. +const GroupName = "snapshot.storage.k8s.io" + +var ( + // SchemeBuilder is the new scheme builder + SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) + // AddToScheme adds to scheme + AddToScheme = SchemeBuilder.AddToScheme + // SchemeGroupVersion is the group version used to register these objects. + SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1beta1"} +) + +// Resource takes an unqualified resource and returns a Group-qualified GroupResource. +func Resource(resource string) schema.GroupResource { + return SchemeGroupVersion.WithResource(resource).GroupResource() +} + +func init() { + // We only register manually written functions here. The registration of the + // generated functions takes place in the generated files. The separation + // makes the code compile even when the generated files are missing. + SchemeBuilder.Register(addKnownTypes) +} + +// addKnownTypes adds the set of types defined in this package to the supplied scheme. +func addKnownTypes(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(SchemeGroupVersion, + &VolumeSnapshotClass{}, + &VolumeSnapshotClassList{}, + &VolumeSnapshot{}, + &VolumeSnapshotList{}, + &VolumeSnapshotContent{}, + &VolumeSnapshotContentList{}, + ) + metav1.AddToGroupVersion(scheme, SchemeGroupVersion) + return nil +} diff --git a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1beta1/types.go b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1beta1/types.go new file mode 100644 index 0000000000..1299d3655e --- /dev/null +++ b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1beta1/types.go @@ -0,0 +1,395 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +kubebuilder:object:generate=true +package v1beta1 + +import ( + core_v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// VolumeSnapshot is a user's request for either creating a point-in-time +// snapshot of a persistent volume, or binding to a pre-existing snapshot. +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Namespaced +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="ReadyToUse",type=boolean,JSONPath=`.status.readyToUse`,description="Indicates if a snapshot is ready to be used to restore a volume." +// +kubebuilder:printcolumn:name="SourcePVC",type=string,JSONPath=`.spec.source.persistentVolumeClaimName`,description="Name of the source PVC from where a dynamically taken snapshot will be created." +// +kubebuilder:printcolumn:name="SourceSnapshotContent",type=string,JSONPath=`.spec.source.volumeSnapshotContentName`,description="Name of the VolumeSnapshotContent which represents a pre-provisioned snapshot." +// +kubebuilder:printcolumn:name="RestoreSize",type=string,JSONPath=`.status.restoreSize`,description="Represents the complete size of the snapshot." +// +kubebuilder:printcolumn:name="SnapshotClass",type=string,JSONPath=`.spec.volumeSnapshotClassName`,description="The name of the VolumeSnapshotClass requested by the VolumeSnapshot." +// +kubebuilder:printcolumn:name="SnapshotContent",type=string,JSONPath=`.status.boundVolumeSnapshotContentName`,description="The name of the VolumeSnapshotContent to which this VolumeSnapshot is bound." +// +kubebuilder:printcolumn:name="CreationTime",type=date,JSONPath=`.status.creationTime`,description="Timestamp when the point-in-time snapshot is taken by the underlying storage system." +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` +type VolumeSnapshot struct { + metav1.TypeMeta `json:",inline"` + // Standard object's metadata. + // More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata + // +optional + metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` + + // spec defines the desired characteristics of a snapshot requested by a user. + // More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots + // Required. + Spec VolumeSnapshotSpec `json:"spec" protobuf:"bytes,2,opt,name=spec"` + + // status represents the current information of a snapshot. + // NOTE: status can be modified by sources other than system controllers, + // and must not be depended upon for accuracy. + // Controllers should only use information from the VolumeSnapshotContent object + // after verifying that the binding is accurate and complete. + // +optional + Status *VolumeSnapshotStatus `json:"status,omitempty" protobuf:"bytes,3,opt,name=status"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// VolumeSnapshotList is a list of VolumeSnapshot objects +type VolumeSnapshotList struct { + metav1.TypeMeta `json:",inline"` + // +optional + metav1.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` + + // List of VolumeSnapshots + Items []VolumeSnapshot `json:"items" protobuf:"bytes,2,rep,name=items"` +} + +// VolumeSnapshotSpec describes the common attributes of a volume snapshot. +type VolumeSnapshotSpec struct { + // source specifies where a snapshot will be created from. + // This field is immutable after creation. + // Required. + Source VolumeSnapshotSource `json:"source" protobuf:"bytes,1,opt,name=source"` + + // volumeSnapshotClassName is the name of the VolumeSnapshotClass requested by the VolumeSnapshot. + // If not specified, the default snapshot class will be used if one exists. + // If not specified, and there is no default snapshot class, dynamic snapshot creation will fail. + // Empty string is not allowed for this field. + // TODO(xiangqian): a webhook validation on empty string. + // More info: https://kubernetes.io/docs/concepts/storage/volume-snapshot-classes + // +optional + VolumeSnapshotClassName *string `json:"volumeSnapshotClassName,omitempty" protobuf:"bytes,2,opt,name=volumeSnapshotClassName"` +} + +// VolumeSnapshotSource specifies whether the underlying snapshot should be +// dynamically taken upon creation or if a pre-existing VolumeSnapshotContent +// object should be used. +// Exactly one of its members must be set. +// Members in VolumeSnapshotSource are immutable. +// TODO(xiangqian): Add a webhook to ensure that VolumeSnapshotSource members +// will not be updated once specified. +type VolumeSnapshotSource struct { + // persistentVolumeClaimName specifies the name of the PersistentVolumeClaim + // object in the same namespace as the VolumeSnapshot object where the + // snapshot should be dynamically taken from. + // This field is immutable. + // +optional + PersistentVolumeClaimName *string `json:"persistentVolumeClaimName,omitempty" protobuf:"bytes,1,opt,name=persistentVolumeClaimName"` + + // volumeSnapshotContentName specifies the name of a pre-existing VolumeSnapshotContent + // object. + // This field is immutable. + // +optional + VolumeSnapshotContentName *string `json:"volumeSnapshotContentName,omitempty" protobuf:"bytes,2,opt,name=volumeSnapshotContentName"` +} + +// VolumeSnapshotStatus is the status of the VolumeSnapshot +type VolumeSnapshotStatus struct { + // boundVolumeSnapshotContentName represents the name of the VolumeSnapshotContent + // object to which the VolumeSnapshot object is bound. + // If not specified, it indicates that the VolumeSnapshot object has not been + // successfully bound to a VolumeSnapshotContent object yet. + // NOTE: Specified boundVolumeSnapshotContentName alone does not mean binding + // is valid. Controllers MUST always verify bidirectional binding between + // VolumeSnapshot and VolumeSnapshotContent to avoid possible security issues. + // +optional + BoundVolumeSnapshotContentName *string `json:"boundVolumeSnapshotContentName,omitempty" protobuf:"bytes,1,opt,name=boundVolumeSnapshotContentName"` + + // creationTime is the timestamp when the point-in-time snapshot is taken + // by the underlying storage system. + // In dynamic snapshot creation case, this field will be filled in with the + // "creation_time" value returned from CSI "CreateSnapshotRequest" gRPC call. + // For a pre-existing snapshot, this field will be filled with the "creation_time" + // value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. + // If not specified, it indicates that the creation time of the snapshot is unknown. + // +optional + CreationTime *metav1.Time `json:"creationTime,omitempty" protobuf:"bytes,2,opt,name=creationTime"` + + // readyToUse indicates if a snapshot is ready to be used to restore a volume. + // In dynamic snapshot creation case, this field will be filled in with the + // "ready_to_use" value returned from CSI "CreateSnapshotRequest" gRPC call. + // For a pre-existing snapshot, this field will be filled with the "ready_to_use" + // value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, + // otherwise, this field will be set to "True". + // If not specified, it means the readiness of a snapshot is unknown. + // +optional + ReadyToUse *bool `json:"readyToUse,omitempty" protobuf:"varint,3,opt,name=readyToUse"` + + // restoreSize represents the complete size of the snapshot in bytes. + // In dynamic snapshot creation case, this field will be filled in with the + // "size_bytes" value returned from CSI "CreateSnapshotRequest" gRPC call. + // For a pre-existing snapshot, this field will be filled with the "size_bytes" + // value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. + // When restoring a volume from this snapshot, the size of the volume MUST NOT + // be smaller than the restoreSize if it is specified, otherwise the restoration will fail. + // If not specified, it indicates that the size is unknown. + // +optional + RestoreSize *resource.Quantity `json:"restoreSize,omitempty" protobuf:"bytes,4,opt,name=restoreSize"` + + // error is the last observed error during snapshot creation, if any. + // This field could be helpful to upper level controllers(i.e., application controller) + // to decide whether they should continue on waiting for the snapshot to be created + // based on the type of error reported. + // +optional + Error *VolumeSnapshotError `json:"error,omitempty" protobuf:"bytes,5,opt,name=error,casttype=VolumeSnapshotError"` +} + +// +genclient +// +genclient:nonNamespaced +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// VolumeSnapshotClass specifies parameters that a underlying storage system uses when +// creating a volume snapshot. A specific VolumeSnapshotClass is used by specifying its +// name in a VolumeSnapshot object. +// VolumeSnapshotClasses are non-namespaced +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Cluster +// +kubebuilder:printcolumn:name="Driver",type=string,JSONPath=`.driver` +// +kubebuilder:printcolumn:name="DeletionPolicy",type=string,JSONPath=`.deletionPolicy`,description="Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted." +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` +type VolumeSnapshotClass struct { + metav1.TypeMeta `json:",inline"` + // Standard object's metadata. + // More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata + // +optional + metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` + + // driver is the name of the storage driver that handles this VolumeSnapshotClass. + // Required. + Driver string `json:"driver" protobuf:"bytes,2,opt,name=driver"` + + // parameters is a key-value map with storage driver specific parameters for creating snapshots. + // These values are opaque to Kubernetes. + // +optional + Parameters map[string]string `json:"parameters,omitempty" protobuf:"bytes,3,rep,name=parameters"` + + // deletionPolicy determines whether a VolumeSnapshotContent created through + // the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. + // Supported values are "Retain" and "Delete". + // "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. + // "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. + // Required. + DeletionPolicy DeletionPolicy `json:"deletionPolicy" protobuf:"bytes,4,opt,name=deletionPolicy"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// VolumeSnapshotClassList is a collection of VolumeSnapshotClasses. +// +kubebuilder:object:root=true +type VolumeSnapshotClassList struct { + metav1.TypeMeta `json:",inline"` + // Standard list metadata + // More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata + // +optional + metav1.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` + + // items is the list of VolumeSnapshotClasses + Items []VolumeSnapshotClass `json:"items" protobuf:"bytes,2,rep,name=items"` +} + +// +genclient +// +genclient:nonNamespaced +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// VolumeSnapshotContent represents the actual "on-disk" snapshot object in the +// underlying storage system +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Cluster +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="ReadyToUse",type=boolean,JSONPath=`.status.readyToUse`,description="Indicates if a snapshot is ready to be used to restore a volume." +// +kubebuilder:printcolumn:name="RestoreSize",type=integer,JSONPath=`.status.restoreSize`,description="Represents the complete size of the snapshot in bytes" +// +kubebuilder:printcolumn:name="DeletionPolicy",type=string,JSONPath=`.spec.deletionPolicy`,description="Determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted." +// +kubebuilder:printcolumn:name="Driver",type=string,JSONPath=`.spec.driver`,description="Name of the CSI driver used to create the physical snapshot on the underlying storage system." +// +kubebuilder:printcolumn:name="VolumeSnapshotClass",type=string,JSONPath=`.spec.volumeSnapshotClassName`,description="Name of the VolumeSnapshotClass to which this snapshot belongs." +// +kubebuilder:printcolumn:name="VolumeSnapshot",type=string,JSONPath=`.spec.volumeSnapshotRef.name`,description="Name of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound." +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` +type VolumeSnapshotContent struct { + metav1.TypeMeta `json:",inline"` + // Standard object's metadata. + // More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata + // +optional + metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` + + // spec defines properties of a VolumeSnapshotContent created by the underlying storage system. + // Required. + Spec VolumeSnapshotContentSpec `json:"spec" protobuf:"bytes,2,opt,name=spec"` + + // status represents the current information of a snapshot. + // +optional + Status *VolumeSnapshotContentStatus `json:"status,omitempty" protobuf:"bytes,3,opt,name=status"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// VolumeSnapshotContentList is a list of VolumeSnapshotContent objects +// +kubebuilder:object:root=true +type VolumeSnapshotContentList struct { + metav1.TypeMeta `json:",inline"` + // +optional + metav1.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` + + // items is the list of VolumeSnapshotContents + Items []VolumeSnapshotContent `json:"items" protobuf:"bytes,2,rep,name=items"` +} + +// VolumeSnapshotContentSpec is the specification of a VolumeSnapshotContent +type VolumeSnapshotContentSpec struct { + // volumeSnapshotRef specifies the VolumeSnapshot object to which this + // VolumeSnapshotContent object is bound. + // VolumeSnapshot.Spec.VolumeSnapshotContentName field must reference to + // this VolumeSnapshotContent's name for the bidirectional binding to be valid. + // For a pre-existing VolumeSnapshotContent object, name and namespace of the + // VolumeSnapshot object MUST be provided for binding to happen. + // This field is immutable after creation. + // Required. + VolumeSnapshotRef core_v1.ObjectReference `json:"volumeSnapshotRef" protobuf:"bytes,1,opt,name=volumeSnapshotRef"` + + // deletionPolicy determines whether this VolumeSnapshotContent and its physical snapshot on + // the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. + // Supported values are "Retain" and "Delete". + // "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. + // "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. + // In dynamic snapshot creation case, this field will be filled in with the "DeletionPolicy" field defined in the + // VolumeSnapshotClass the VolumeSnapshot refers to. + // For pre-existing snapshots, users MUST specify this field when creating the VolumeSnapshotContent object. + // Required. + DeletionPolicy DeletionPolicy `json:"deletionPolicy" protobuf:"bytes,2,opt,name=deletionPolicy"` + + // driver is the name of the CSI driver used to create the physical snapshot on + // the underlying storage system. + // This MUST be the same as the name returned by the CSI GetPluginName() call for + // that driver. + // Required. + Driver string `json:"driver" protobuf:"bytes,3,opt,name=driver"` + + // name of the VolumeSnapshotClass to which this snapshot belongs. + // +optional + VolumeSnapshotClassName *string `json:"volumeSnapshotClassName,omitempty" protobuf:"bytes,4,opt,name=volumeSnapshotClassName"` + + // source specifies from where a snapshot will be created. + // This field is immutable after creation. + // Required. + Source VolumeSnapshotContentSource `json:"source" protobuf:"bytes,5,opt,name=source"` +} + +// VolumeSnapshotContentSource represents the CSI source of a snapshot. +// Exactly one of its members must be set. +// Members in VolumeSnapshotContentSource are immutable. +// TODO(xiangqian): Add a webhook to ensure that VolumeSnapshotContentSource members +// will be immutable once specified. +type VolumeSnapshotContentSource struct { + // volumeHandle specifies the CSI "volume_id" of the volume from which a snapshot + // should be dynamically taken from. + // This field is immutable. + // +optional + VolumeHandle *string `json:"volumeHandle,omitempty" protobuf:"bytes,1,opt,name=volumeHandle"` + + // snapshotHandle specifies the CSI "snapshot_id" of a pre-existing snapshot on + // the underlying storage system. + // This field is immutable. + // +optional + SnapshotHandle *string `json:"snapshotHandle,omitempty" protobuf:"bytes,2,opt,name=snapshotHandle"` +} + +// VolumeSnapshotContentStatus is the status of a VolumeSnapshotContent object +type VolumeSnapshotContentStatus struct { + // snapshotHandle is the CSI "snapshot_id" of a snapshot on the underlying storage system. + // If not specified, it indicates that dynamic snapshot creation has either failed + // or it is still in progress. + // +optional + SnapshotHandle *string `json:"snapshotHandle,omitempty" protobuf:"bytes,1,opt,name=snapshotHandle"` + + // creationTime is the timestamp when the point-in-time snapshot is taken + // by the underlying storage system. + // In dynamic snapshot creation case, this field will be filled in with the + // "creation_time" value returned from CSI "CreateSnapshotRequest" gRPC call. + // For a pre-existing snapshot, this field will be filled with the "creation_time" + // value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. + // If not specified, it indicates the creation time is unknown. + // The format of this field is a Unix nanoseconds time encoded as an int64. + // On Unix, the command `date +%s%N` returns the current time in nanoseconds + // since 1970-01-01 00:00:00 UTC. + // +optional + CreationTime *int64 `json:"creationTime,omitempty" protobuf:"varint,2,opt,name=creationTime"` + + // restoreSize represents the complete size of the snapshot in bytes. + // In dynamic snapshot creation case, this field will be filled in with the + // "size_bytes" value returned from CSI "CreateSnapshotRequest" gRPC call. + // For a pre-existing snapshot, this field will be filled with the "size_bytes" + // value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. + // When restoring a volume from this snapshot, the size of the volume MUST NOT + // be smaller than the restoreSize if it is specified, otherwise the restoration will fail. + // If not specified, it indicates that the size is unknown. + // +kubebuilder:validation:Minimum=0 + // +optional + RestoreSize *int64 `json:"restoreSize,omitempty" protobuf:"bytes,3,opt,name=restoreSize"` + + // readyToUse indicates if a snapshot is ready to be used to restore a volume. + // In dynamic snapshot creation case, this field will be filled in with the + // "ready_to_use" value returned from CSI "CreateSnapshotRequest" gRPC call. + // For a pre-existing snapshot, this field will be filled with the "ready_to_use" + // value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, + // otherwise, this field will be set to "True". + // If not specified, it means the readiness of a snapshot is unknown. + // +optional. + ReadyToUse *bool `json:"readyToUse,omitempty" protobuf:"varint,4,opt,name=readyToUse"` + + // error is the latest observed error during snapshot creation, if any. + // +optional + Error *VolumeSnapshotError `json:"error,omitempty" protobuf:"bytes,5,opt,name=error,casttype=VolumeSnapshotError"` +} + +// DeletionPolicy describes a policy for end-of-life maintenance of volume snapshot contents +// +kubebuilder:validation:Enum=Delete;Retain +type DeletionPolicy string + +const ( + // volumeSnapshotContentDelete means the snapshot will be deleted from the + // underlying storage system on release from its volume snapshot. + VolumeSnapshotContentDelete DeletionPolicy = "Delete" + + // volumeSnapshotContentRetain means the snapshot will be left in its current + // state on release from its volume snapshot. + VolumeSnapshotContentRetain DeletionPolicy = "Retain" +) + +// VolumeSnapshotError describes an error encountered during snapshot creation. +type VolumeSnapshotError struct { + // time is the timestamp when the error was encountered. + // +optional + Time *metav1.Time `json:"time,omitempty" protobuf:"bytes,1,opt,name=time"` + + // message is a string detailing the encountered error during snapshot + // creation if specified. + // NOTE: message may be logged, and it should not contain sensitive + // information. + // +optional + Message *string `json:"message,omitempty" protobuf:"bytes,2,opt,name=message"` +} diff --git a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1beta1/zz_generated.deepcopy.go b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1beta1/zz_generated.deepcopy.go new file mode 100644 index 0000000000..550ff57f8a --- /dev/null +++ b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1beta1/zz_generated.deepcopy.go @@ -0,0 +1,424 @@ +// +build !ignore_autogenerated + +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by deepcopy-gen. DO NOT EDIT. + +package v1beta1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolumeSnapshot) DeepCopyInto(out *VolumeSnapshot) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + if in.Status != nil { + in, out := &in.Status, &out.Status + *out = new(VolumeSnapshotStatus) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeSnapshot. +func (in *VolumeSnapshot) DeepCopy() *VolumeSnapshot { + if in == nil { + return nil + } + out := new(VolumeSnapshot) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *VolumeSnapshot) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolumeSnapshotClass) DeepCopyInto(out *VolumeSnapshotClass) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + if in.Parameters != nil { + in, out := &in.Parameters, &out.Parameters + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeSnapshotClass. +func (in *VolumeSnapshotClass) DeepCopy() *VolumeSnapshotClass { + if in == nil { + return nil + } + out := new(VolumeSnapshotClass) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *VolumeSnapshotClass) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolumeSnapshotClassList) DeepCopyInto(out *VolumeSnapshotClassList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]VolumeSnapshotClass, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeSnapshotClassList. +func (in *VolumeSnapshotClassList) DeepCopy() *VolumeSnapshotClassList { + if in == nil { + return nil + } + out := new(VolumeSnapshotClassList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *VolumeSnapshotClassList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolumeSnapshotContent) DeepCopyInto(out *VolumeSnapshotContent) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + if in.Status != nil { + in, out := &in.Status, &out.Status + *out = new(VolumeSnapshotContentStatus) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeSnapshotContent. +func (in *VolumeSnapshotContent) DeepCopy() *VolumeSnapshotContent { + if in == nil { + return nil + } + out := new(VolumeSnapshotContent) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *VolumeSnapshotContent) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolumeSnapshotContentList) DeepCopyInto(out *VolumeSnapshotContentList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]VolumeSnapshotContent, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeSnapshotContentList. +func (in *VolumeSnapshotContentList) DeepCopy() *VolumeSnapshotContentList { + if in == nil { + return nil + } + out := new(VolumeSnapshotContentList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *VolumeSnapshotContentList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolumeSnapshotContentSource) DeepCopyInto(out *VolumeSnapshotContentSource) { + *out = *in + if in.VolumeHandle != nil { + in, out := &in.VolumeHandle, &out.VolumeHandle + *out = new(string) + **out = **in + } + if in.SnapshotHandle != nil { + in, out := &in.SnapshotHandle, &out.SnapshotHandle + *out = new(string) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeSnapshotContentSource. +func (in *VolumeSnapshotContentSource) DeepCopy() *VolumeSnapshotContentSource { + if in == nil { + return nil + } + out := new(VolumeSnapshotContentSource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolumeSnapshotContentSpec) DeepCopyInto(out *VolumeSnapshotContentSpec) { + *out = *in + out.VolumeSnapshotRef = in.VolumeSnapshotRef + if in.VolumeSnapshotClassName != nil { + in, out := &in.VolumeSnapshotClassName, &out.VolumeSnapshotClassName + *out = new(string) + **out = **in + } + in.Source.DeepCopyInto(&out.Source) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeSnapshotContentSpec. +func (in *VolumeSnapshotContentSpec) DeepCopy() *VolumeSnapshotContentSpec { + if in == nil { + return nil + } + out := new(VolumeSnapshotContentSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolumeSnapshotContentStatus) DeepCopyInto(out *VolumeSnapshotContentStatus) { + *out = *in + if in.SnapshotHandle != nil { + in, out := &in.SnapshotHandle, &out.SnapshotHandle + *out = new(string) + **out = **in + } + if in.CreationTime != nil { + in, out := &in.CreationTime, &out.CreationTime + *out = new(int64) + **out = **in + } + if in.RestoreSize != nil { + in, out := &in.RestoreSize, &out.RestoreSize + *out = new(int64) + **out = **in + } + if in.ReadyToUse != nil { + in, out := &in.ReadyToUse, &out.ReadyToUse + *out = new(bool) + **out = **in + } + if in.Error != nil { + in, out := &in.Error, &out.Error + *out = new(VolumeSnapshotError) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeSnapshotContentStatus. +func (in *VolumeSnapshotContentStatus) DeepCopy() *VolumeSnapshotContentStatus { + if in == nil { + return nil + } + out := new(VolumeSnapshotContentStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolumeSnapshotError) DeepCopyInto(out *VolumeSnapshotError) { + *out = *in + if in.Time != nil { + in, out := &in.Time, &out.Time + *out = (*in).DeepCopy() + } + if in.Message != nil { + in, out := &in.Message, &out.Message + *out = new(string) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeSnapshotError. +func (in *VolumeSnapshotError) DeepCopy() *VolumeSnapshotError { + if in == nil { + return nil + } + out := new(VolumeSnapshotError) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolumeSnapshotList) DeepCopyInto(out *VolumeSnapshotList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]VolumeSnapshot, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeSnapshotList. +func (in *VolumeSnapshotList) DeepCopy() *VolumeSnapshotList { + if in == nil { + return nil + } + out := new(VolumeSnapshotList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *VolumeSnapshotList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolumeSnapshotSource) DeepCopyInto(out *VolumeSnapshotSource) { + *out = *in + if in.PersistentVolumeClaimName != nil { + in, out := &in.PersistentVolumeClaimName, &out.PersistentVolumeClaimName + *out = new(string) + **out = **in + } + if in.VolumeSnapshotContentName != nil { + in, out := &in.VolumeSnapshotContentName, &out.VolumeSnapshotContentName + *out = new(string) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeSnapshotSource. +func (in *VolumeSnapshotSource) DeepCopy() *VolumeSnapshotSource { + if in == nil { + return nil + } + out := new(VolumeSnapshotSource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolumeSnapshotSpec) DeepCopyInto(out *VolumeSnapshotSpec) { + *out = *in + in.Source.DeepCopyInto(&out.Source) + if in.VolumeSnapshotClassName != nil { + in, out := &in.VolumeSnapshotClassName, &out.VolumeSnapshotClassName + *out = new(string) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeSnapshotSpec. +func (in *VolumeSnapshotSpec) DeepCopy() *VolumeSnapshotSpec { + if in == nil { + return nil + } + out := new(VolumeSnapshotSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolumeSnapshotStatus) DeepCopyInto(out *VolumeSnapshotStatus) { + *out = *in + if in.BoundVolumeSnapshotContentName != nil { + in, out := &in.BoundVolumeSnapshotContentName, &out.BoundVolumeSnapshotContentName + *out = new(string) + **out = **in + } + if in.CreationTime != nil { + in, out := &in.CreationTime, &out.CreationTime + *out = (*in).DeepCopy() + } + if in.ReadyToUse != nil { + in, out := &in.ReadyToUse, &out.ReadyToUse + *out = new(bool) + **out = **in + } + if in.RestoreSize != nil { + in, out := &in.RestoreSize, &out.RestoreSize + x := (*in).DeepCopy() + *out = &x + } + if in.Error != nil { + in, out := &in.Error, &out.Error + *out = new(VolumeSnapshotError) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeSnapshotStatus. +func (in *VolumeSnapshotStatus) DeepCopy() *VolumeSnapshotStatus { + if in == nil { + return nil + } + out := new(VolumeSnapshotStatus) + in.DeepCopyInto(out) + return out +} diff --git a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/clientset.go b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/clientset.go similarity index 69% rename from vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/clientset.go rename to vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/clientset.go index 77d96ce98c..2e01b2327b 100644 --- a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/clientset.go +++ b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/clientset.go @@ -1,5 +1,5 @@ /* -Copyright 2022 The Kubernetes Authors. +Copyright 2020 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -20,9 +20,9 @@ package versioned import ( "fmt" - "net/http" - snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1" + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1" + snapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1" discovery "k8s.io/client-go/discovery" rest "k8s.io/client-go/rest" flowcontrol "k8s.io/client-go/util/flowcontrol" @@ -30,6 +30,7 @@ import ( type Interface interface { Discovery() discovery.DiscoveryInterface + SnapshotV1beta1() snapshotv1beta1.SnapshotV1beta1Interface SnapshotV1() snapshotv1.SnapshotV1Interface } @@ -37,7 +38,13 @@ type Interface interface { // version included in a Clientset. type Clientset struct { *discovery.DiscoveryClient - snapshotV1 *snapshotv1.SnapshotV1Client + snapshotV1beta1 *snapshotv1beta1.SnapshotV1beta1Client + snapshotV1 *snapshotv1.SnapshotV1Client +} + +// SnapshotV1beta1 retrieves the SnapshotV1beta1Client +func (c *Clientset) SnapshotV1beta1() snapshotv1beta1.SnapshotV1beta1Interface { + return c.snapshotV1beta1 } // SnapshotV1 retrieves the SnapshotV1Client @@ -56,45 +63,26 @@ func (c *Clientset) Discovery() discovery.DiscoveryInterface { // NewForConfig creates a new Clientset for the given config. // If config's RateLimiter is not set and QPS and Burst are acceptable, // NewForConfig will generate a rate-limiter in configShallowCopy. -// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), -// where httpClient was generated with rest.HTTPClientFor(c). func NewForConfig(c *rest.Config) (*Clientset, error) { configShallowCopy := *c - - if configShallowCopy.UserAgent == "" { - configShallowCopy.UserAgent = rest.DefaultKubernetesUserAgent() - } - - // share the transport between all clients - httpClient, err := rest.HTTPClientFor(&configShallowCopy) - if err != nil { - return nil, err - } - - return NewForConfigAndClient(&configShallowCopy, httpClient) -} - -// NewForConfigAndClient creates a new Clientset for the given config and http client. -// Note the http client provided takes precedence over the configured transport values. -// If config's RateLimiter is not set and QPS and Burst are acceptable, -// NewForConfigAndClient will generate a rate-limiter in configShallowCopy. -func NewForConfigAndClient(c *rest.Config, httpClient *http.Client) (*Clientset, error) { - configShallowCopy := *c if configShallowCopy.RateLimiter == nil && configShallowCopy.QPS > 0 { if configShallowCopy.Burst <= 0 { return nil, fmt.Errorf("burst is required to be greater than 0 when RateLimiter is not set and QPS is set to greater than 0") } configShallowCopy.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(configShallowCopy.QPS, configShallowCopy.Burst) } - var cs Clientset var err error - cs.snapshotV1, err = snapshotv1.NewForConfigAndClient(&configShallowCopy, httpClient) + cs.snapshotV1beta1, err = snapshotv1beta1.NewForConfig(&configShallowCopy) + if err != nil { + return nil, err + } + cs.snapshotV1, err = snapshotv1.NewForConfig(&configShallowCopy) if err != nil { return nil, err } - cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfigAndClient(&configShallowCopy, httpClient) + cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfig(&configShallowCopy) if err != nil { return nil, err } @@ -104,16 +92,18 @@ func NewForConfigAndClient(c *rest.Config, httpClient *http.Client) (*Clientset, // NewForConfigOrDie creates a new Clientset for the given config and // panics if there is an error in the config. func NewForConfigOrDie(c *rest.Config) *Clientset { - cs, err := NewForConfig(c) - if err != nil { - panic(err) - } - return cs + var cs Clientset + cs.snapshotV1beta1 = snapshotv1beta1.NewForConfigOrDie(c) + cs.snapshotV1 = snapshotv1.NewForConfigOrDie(c) + + cs.DiscoveryClient = discovery.NewDiscoveryClientForConfigOrDie(c) + return &cs } // New creates a new Clientset for the given RESTClient. func New(c rest.Interface) *Clientset { var cs Clientset + cs.snapshotV1beta1 = snapshotv1beta1.New(c) cs.snapshotV1 = snapshotv1.New(c) cs.DiscoveryClient = discovery.NewDiscoveryClient(c) diff --git a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/doc.go b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/doc.go similarity index 94% rename from vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/doc.go rename to vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/doc.go index ab7539cb9f..6ee812fc57 100644 --- a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/doc.go +++ b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2022 The Kubernetes Authors. +Copyright 2020 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/fake/clientset_generated.go b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/fake/clientset_generated.go similarity index 79% rename from vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/fake/clientset_generated.go rename to vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/fake/clientset_generated.go index b3ab82a6e8..bf0ed24396 100644 --- a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/fake/clientset_generated.go +++ b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/fake/clientset_generated.go @@ -1,5 +1,5 @@ /* -Copyright 2022 The Kubernetes Authors. +Copyright 2020 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -19,9 +19,11 @@ limitations under the License. package fake import ( - clientset "github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned" - snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1" - fakesnapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1/fake" + clientset "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned" + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1" + fakesnapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1/fake" + snapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1" + fakesnapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/fake" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/discovery" @@ -74,10 +76,12 @@ func (c *Clientset) Tracker() testing.ObjectTracker { return c.tracker } -var ( - _ clientset.Interface = &Clientset{} - _ testing.FakeClient = &Clientset{} -) +var _ clientset.Interface = &Clientset{} + +// SnapshotV1beta1 retrieves the SnapshotV1beta1Client +func (c *Clientset) SnapshotV1beta1() snapshotv1beta1.SnapshotV1beta1Interface { + return &fakesnapshotv1beta1.FakeSnapshotV1beta1{Fake: &c.Fake} +} // SnapshotV1 retrieves the SnapshotV1Client func (c *Clientset) SnapshotV1() snapshotv1.SnapshotV1Interface { diff --git a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/fake/doc.go b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/fake/doc.go similarity index 94% rename from vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/fake/doc.go rename to vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/fake/doc.go index 7d98eabcc8..d6baf01acb 100644 --- a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/fake/doc.go +++ b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/fake/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2022 The Kubernetes Authors. +Copyright 2020 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/fake/register.go b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/fake/register.go similarity index 72% rename from vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/fake/register.go rename to vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/fake/register.go index 5ae20f2237..eae2547754 100644 --- a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/fake/register.go +++ b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/fake/register.go @@ -1,5 +1,5 @@ /* -Copyright 2022 The Kubernetes Authors. +Copyright 2020 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -19,7 +19,8 @@ limitations under the License. package fake import ( - snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" + snapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" @@ -31,20 +32,21 @@ var scheme = runtime.NewScheme() var codecs = serializer.NewCodecFactory(scheme) var localSchemeBuilder = runtime.SchemeBuilder{ + snapshotv1beta1.AddToScheme, snapshotv1.AddToScheme, } // AddToScheme adds all types of this clientset into the given scheme. This allows composition // of clientsets, like in: // -// import ( -// "k8s.io/client-go/kubernetes" -// clientsetscheme "k8s.io/client-go/kubernetes/scheme" -// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" -// ) +// import ( +// "k8s.io/client-go/kubernetes" +// clientsetscheme "k8s.io/client-go/kubernetes/scheme" +// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" +// ) // -// kclientset, _ := kubernetes.NewForConfig(c) -// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) +// kclientset, _ := kubernetes.NewForConfig(c) +// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) // // After this, RawExtensions in Kubernetes types will serialize kube-aggregator types // correctly. diff --git a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/scheme/doc.go b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/scheme/doc.go similarity index 94% rename from vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/scheme/doc.go rename to vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/scheme/doc.go index 288d3794dc..7d06c9402d 100644 --- a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/scheme/doc.go +++ b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/scheme/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2022 The Kubernetes Authors. +Copyright 2020 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/scheme/register.go b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/scheme/register.go similarity index 73% rename from vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/scheme/register.go rename to vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/scheme/register.go index df04449fce..46e6628fdb 100644 --- a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/scheme/register.go +++ b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/scheme/register.go @@ -1,5 +1,5 @@ /* -Copyright 2022 The Kubernetes Authors. +Copyright 2020 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -19,7 +19,8 @@ limitations under the License. package scheme import ( - snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" + snapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" @@ -31,20 +32,21 @@ var Scheme = runtime.NewScheme() var Codecs = serializer.NewCodecFactory(Scheme) var ParameterCodec = runtime.NewParameterCodec(Scheme) var localSchemeBuilder = runtime.SchemeBuilder{ + snapshotv1beta1.AddToScheme, snapshotv1.AddToScheme, } // AddToScheme adds all types of this clientset into the given scheme. This allows composition // of clientsets, like in: // -// import ( -// "k8s.io/client-go/kubernetes" -// clientsetscheme "k8s.io/client-go/kubernetes/scheme" -// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" -// ) +// import ( +// "k8s.io/client-go/kubernetes" +// clientsetscheme "k8s.io/client-go/kubernetes/scheme" +// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" +// ) // -// kclientset, _ := kubernetes.NewForConfig(c) -// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) +// kclientset, _ := kubernetes.NewForConfig(c) +// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) // // After this, RawExtensions in Kubernetes types will serialize kube-aggregator types // correctly. diff --git a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1/doc.go b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1/doc.go similarity index 94% rename from vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1/doc.go rename to vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1/doc.go index 01fa5fd655..1917a62947 100644 --- a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1/doc.go +++ b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2022 The Kubernetes Authors. +Copyright 2020 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1/fake/doc.go b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1/fake/doc.go similarity index 94% rename from vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1/fake/doc.go rename to vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1/fake/doc.go index dd9e9e4c8f..0243e68ff4 100644 --- a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1/fake/doc.go +++ b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1/fake/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2022 The Kubernetes Authors. +Copyright 2020 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1/fake/fake_volumesnapshot.go b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1/fake/fake_volumesnapshot.go similarity index 96% rename from vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1/fake/fake_volumesnapshot.go rename to vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1/fake/fake_volumesnapshot.go index f6cdb4669d..286104b031 100644 --- a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1/fake/fake_volumesnapshot.go +++ b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1/fake/fake_volumesnapshot.go @@ -1,5 +1,5 @@ /* -Copyright 2022 The Kubernetes Authors. +Copyright 2020 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -21,7 +21,7 @@ package fake import ( "context" - volumesnapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" + volumesnapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" labels "k8s.io/apimachinery/pkg/labels" schema "k8s.io/apimachinery/pkg/runtime/schema" @@ -117,7 +117,7 @@ func (c *FakeVolumeSnapshots) UpdateStatus(ctx context.Context, volumeSnapshot * // Delete takes name of the volumeSnapshot and deletes it. Returns an error if one occurs. func (c *FakeVolumeSnapshots) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { _, err := c.Fake. - Invokes(testing.NewDeleteActionWithOptions(volumesnapshotsResource, c.ns, name, opts), &volumesnapshotv1.VolumeSnapshot{}) + Invokes(testing.NewDeleteAction(volumesnapshotsResource, c.ns, name), &volumesnapshotv1.VolumeSnapshot{}) return err } diff --git a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1/fake/fake_volumesnapshot_client.go b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1/fake/fake_volumesnapshot_client.go similarity index 92% rename from vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1/fake/fake_volumesnapshot_client.go rename to vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1/fake/fake_volumesnapshot_client.go index 866dce2f41..31fd4be64b 100644 --- a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1/fake/fake_volumesnapshot_client.go +++ b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1/fake/fake_volumesnapshot_client.go @@ -1,5 +1,5 @@ /* -Copyright 2022 The Kubernetes Authors. +Copyright 2020 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -19,7 +19,7 @@ limitations under the License. package fake import ( - v1 "github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1" + v1 "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1" rest "k8s.io/client-go/rest" testing "k8s.io/client-go/testing" ) diff --git a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1/fake/fake_volumesnapshotclass.go b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1/fake/fake_volumesnapshotclass.go similarity index 96% rename from vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1/fake/fake_volumesnapshotclass.go rename to vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1/fake/fake_volumesnapshotclass.go index e471b6b24d..eed85bd85e 100644 --- a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1/fake/fake_volumesnapshotclass.go +++ b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1/fake/fake_volumesnapshotclass.go @@ -1,5 +1,5 @@ /* -Copyright 2022 The Kubernetes Authors. +Copyright 2020 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -21,7 +21,7 @@ package fake import ( "context" - volumesnapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" + volumesnapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" labels "k8s.io/apimachinery/pkg/labels" schema "k8s.io/apimachinery/pkg/runtime/schema" @@ -99,7 +99,7 @@ func (c *FakeVolumeSnapshotClasses) Update(ctx context.Context, volumeSnapshotCl // Delete takes name of the volumeSnapshotClass and deletes it. Returns an error if one occurs. func (c *FakeVolumeSnapshotClasses) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { _, err := c.Fake. - Invokes(testing.NewRootDeleteActionWithOptions(volumesnapshotclassesResource, name, opts), &volumesnapshotv1.VolumeSnapshotClass{}) + Invokes(testing.NewRootDeleteAction(volumesnapshotclassesResource, name), &volumesnapshotv1.VolumeSnapshotClass{}) return err } diff --git a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1/fake/fake_volumesnapshotcontent.go b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1/fake/fake_volumesnapshotcontent.go similarity index 96% rename from vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1/fake/fake_volumesnapshotcontent.go rename to vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1/fake/fake_volumesnapshotcontent.go index 32b9f191dc..176c760a28 100644 --- a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1/fake/fake_volumesnapshotcontent.go +++ b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1/fake/fake_volumesnapshotcontent.go @@ -1,5 +1,5 @@ /* -Copyright 2022 The Kubernetes Authors. +Copyright 2020 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -21,7 +21,7 @@ package fake import ( "context" - volumesnapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" + volumesnapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" labels "k8s.io/apimachinery/pkg/labels" schema "k8s.io/apimachinery/pkg/runtime/schema" @@ -110,7 +110,7 @@ func (c *FakeVolumeSnapshotContents) UpdateStatus(ctx context.Context, volumeSna // Delete takes name of the volumeSnapshotContent and deletes it. Returns an error if one occurs. func (c *FakeVolumeSnapshotContents) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { _, err := c.Fake. - Invokes(testing.NewRootDeleteActionWithOptions(volumesnapshotcontentsResource, name, opts), &volumesnapshotv1.VolumeSnapshotContent{}) + Invokes(testing.NewRootDeleteAction(volumesnapshotcontentsResource, name), &volumesnapshotv1.VolumeSnapshotContent{}) return err } diff --git a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1/generated_expansion.go b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1/generated_expansion.go similarity index 94% rename from vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1/generated_expansion.go rename to vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1/generated_expansion.go index cae7c1cace..88df72cf7d 100644 --- a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1/generated_expansion.go +++ b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1/generated_expansion.go @@ -1,5 +1,5 @@ /* -Copyright 2022 The Kubernetes Authors. +Copyright 2020 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1/volumesnapshot.go b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1/volumesnapshot.go similarity index 98% rename from vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1/volumesnapshot.go rename to vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1/volumesnapshot.go index 6160690f0e..36a02388c8 100644 --- a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1/volumesnapshot.go +++ b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1/volumesnapshot.go @@ -1,5 +1,5 @@ /* -Copyright 2022 The Kubernetes Authors. +Copyright 2020 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -22,8 +22,8 @@ import ( "context" "time" - v1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" - scheme "github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/scheme" + v1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" + scheme "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/scheme" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" diff --git a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1/volumesnapshot_client.go b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1/volumesnapshot_client.go similarity index 74% rename from vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1/volumesnapshot_client.go rename to vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1/volumesnapshot_client.go index 938dbdb75b..996f1e16bd 100644 --- a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1/volumesnapshot_client.go +++ b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1/volumesnapshot_client.go @@ -1,5 +1,5 @@ /* -Copyright 2022 The Kubernetes Authors. +Copyright 2020 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -19,10 +19,8 @@ limitations under the License. package v1 import ( - "net/http" - - v1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" - "github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/scheme" + v1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" + "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/scheme" rest "k8s.io/client-go/rest" ) @@ -51,28 +49,12 @@ func (c *SnapshotV1Client) VolumeSnapshotContents() VolumeSnapshotContentInterfa } // NewForConfig creates a new SnapshotV1Client for the given config. -// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), -// where httpClient was generated with rest.HTTPClientFor(c). func NewForConfig(c *rest.Config) (*SnapshotV1Client, error) { config := *c if err := setConfigDefaults(&config); err != nil { return nil, err } - httpClient, err := rest.HTTPClientFor(&config) - if err != nil { - return nil, err - } - return NewForConfigAndClient(&config, httpClient) -} - -// NewForConfigAndClient creates a new SnapshotV1Client for the given config and http client. -// Note the http client provided takes precedence over the configured transport values. -func NewForConfigAndClient(c *rest.Config, h *http.Client) (*SnapshotV1Client, error) { - config := *c - if err := setConfigDefaults(&config); err != nil { - return nil, err - } - client, err := rest.RESTClientForConfigAndClient(&config, h) + client, err := rest.RESTClientFor(&config) if err != nil { return nil, err } diff --git a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1/volumesnapshotclass.go b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1/volumesnapshotclass.go similarity index 97% rename from vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1/volumesnapshotclass.go rename to vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1/volumesnapshotclass.go index 71ef507f33..7463fb40ea 100644 --- a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1/volumesnapshotclass.go +++ b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1/volumesnapshotclass.go @@ -1,5 +1,5 @@ /* -Copyright 2022 The Kubernetes Authors. +Copyright 2020 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -22,8 +22,8 @@ import ( "context" "time" - v1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" - scheme "github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/scheme" + v1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" + scheme "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/scheme" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" diff --git a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1/volumesnapshotcontent.go b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1/volumesnapshotcontent.go similarity index 98% rename from vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1/volumesnapshotcontent.go rename to vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1/volumesnapshotcontent.go index 7f634e68c2..3e0504cb01 100644 --- a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1/volumesnapshotcontent.go +++ b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1/volumesnapshotcontent.go @@ -1,5 +1,5 @@ /* -Copyright 2022 The Kubernetes Authors. +Copyright 2020 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -22,8 +22,8 @@ import ( "context" "time" - v1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" - scheme "github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/scheme" + v1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" + scheme "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/scheme" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" diff --git a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/doc.go b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/doc.go new file mode 100644 index 0000000000..68f9a55b51 --- /dev/null +++ b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/doc.go @@ -0,0 +1,20 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +// This package has the automatically generated typed clients. +package v1beta1 diff --git a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/fake/doc.go b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/fake/doc.go new file mode 100644 index 0000000000..0243e68ff4 --- /dev/null +++ b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/fake/doc.go @@ -0,0 +1,20 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +// Package fake has the automatically generated clients. +package fake diff --git a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshot.go b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshot.go new file mode 100644 index 0000000000..532f4d08af --- /dev/null +++ b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshot.go @@ -0,0 +1,142 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + v1beta1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1beta1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + schema "k8s.io/apimachinery/pkg/runtime/schema" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeVolumeSnapshots implements VolumeSnapshotInterface +type FakeVolumeSnapshots struct { + Fake *FakeSnapshotV1beta1 + ns string +} + +var volumesnapshotsResource = schema.GroupVersionResource{Group: "snapshot.storage.k8s.io", Version: "v1beta1", Resource: "volumesnapshots"} + +var volumesnapshotsKind = schema.GroupVersionKind{Group: "snapshot.storage.k8s.io", Version: "v1beta1", Kind: "VolumeSnapshot"} + +// Get takes name of the volumeSnapshot, and returns the corresponding volumeSnapshot object, and an error if there is any. +func (c *FakeVolumeSnapshots) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1beta1.VolumeSnapshot, err error) { + obj, err := c.Fake. + Invokes(testing.NewGetAction(volumesnapshotsResource, c.ns, name), &v1beta1.VolumeSnapshot{}) + + if obj == nil { + return nil, err + } + return obj.(*v1beta1.VolumeSnapshot), err +} + +// List takes label and field selectors, and returns the list of VolumeSnapshots that match those selectors. +func (c *FakeVolumeSnapshots) List(ctx context.Context, opts v1.ListOptions) (result *v1beta1.VolumeSnapshotList, err error) { + obj, err := c.Fake. + Invokes(testing.NewListAction(volumesnapshotsResource, volumesnapshotsKind, c.ns, opts), &v1beta1.VolumeSnapshotList{}) + + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1beta1.VolumeSnapshotList{ListMeta: obj.(*v1beta1.VolumeSnapshotList).ListMeta} + for _, item := range obj.(*v1beta1.VolumeSnapshotList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested volumeSnapshots. +func (c *FakeVolumeSnapshots) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchAction(volumesnapshotsResource, c.ns, opts)) + +} + +// Create takes the representation of a volumeSnapshot and creates it. Returns the server's representation of the volumeSnapshot, and an error, if there is any. +func (c *FakeVolumeSnapshots) Create(ctx context.Context, volumeSnapshot *v1beta1.VolumeSnapshot, opts v1.CreateOptions) (result *v1beta1.VolumeSnapshot, err error) { + obj, err := c.Fake. + Invokes(testing.NewCreateAction(volumesnapshotsResource, c.ns, volumeSnapshot), &v1beta1.VolumeSnapshot{}) + + if obj == nil { + return nil, err + } + return obj.(*v1beta1.VolumeSnapshot), err +} + +// Update takes the representation of a volumeSnapshot and updates it. Returns the server's representation of the volumeSnapshot, and an error, if there is any. +func (c *FakeVolumeSnapshots) Update(ctx context.Context, volumeSnapshot *v1beta1.VolumeSnapshot, opts v1.UpdateOptions) (result *v1beta1.VolumeSnapshot, err error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateAction(volumesnapshotsResource, c.ns, volumeSnapshot), &v1beta1.VolumeSnapshot{}) + + if obj == nil { + return nil, err + } + return obj.(*v1beta1.VolumeSnapshot), err +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeVolumeSnapshots) UpdateStatus(ctx context.Context, volumeSnapshot *v1beta1.VolumeSnapshot, opts v1.UpdateOptions) (*v1beta1.VolumeSnapshot, error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateSubresourceAction(volumesnapshotsResource, "status", c.ns, volumeSnapshot), &v1beta1.VolumeSnapshot{}) + + if obj == nil { + return nil, err + } + return obj.(*v1beta1.VolumeSnapshot), err +} + +// Delete takes name of the volumeSnapshot and deletes it. Returns an error if one occurs. +func (c *FakeVolumeSnapshots) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteAction(volumesnapshotsResource, c.ns, name), &v1beta1.VolumeSnapshot{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeVolumeSnapshots) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewDeleteCollectionAction(volumesnapshotsResource, c.ns, listOpts) + + _, err := c.Fake.Invokes(action, &v1beta1.VolumeSnapshotList{}) + return err +} + +// Patch applies the patch and returns the patched volumeSnapshot. +func (c *FakeVolumeSnapshots) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1beta1.VolumeSnapshot, err error) { + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceAction(volumesnapshotsResource, c.ns, name, pt, data, subresources...), &v1beta1.VolumeSnapshot{}) + + if obj == nil { + return nil, err + } + return obj.(*v1beta1.VolumeSnapshot), err +} diff --git a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshot_client.go b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshot_client.go new file mode 100644 index 0000000000..dde6a3fb52 --- /dev/null +++ b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshot_client.go @@ -0,0 +1,48 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + v1beta1 "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1" + rest "k8s.io/client-go/rest" + testing "k8s.io/client-go/testing" +) + +type FakeSnapshotV1beta1 struct { + *testing.Fake +} + +func (c *FakeSnapshotV1beta1) VolumeSnapshots(namespace string) v1beta1.VolumeSnapshotInterface { + return &FakeVolumeSnapshots{c, namespace} +} + +func (c *FakeSnapshotV1beta1) VolumeSnapshotClasses() v1beta1.VolumeSnapshotClassInterface { + return &FakeVolumeSnapshotClasses{c} +} + +func (c *FakeSnapshotV1beta1) VolumeSnapshotContents() v1beta1.VolumeSnapshotContentInterface { + return &FakeVolumeSnapshotContents{c} +} + +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *FakeSnapshotV1beta1) RESTClient() rest.Interface { + var ret *rest.RESTClient + return ret +} diff --git a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshotclass.go b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshotclass.go new file mode 100644 index 0000000000..2decfe30a6 --- /dev/null +++ b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshotclass.go @@ -0,0 +1,122 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + v1beta1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1beta1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + schema "k8s.io/apimachinery/pkg/runtime/schema" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeVolumeSnapshotClasses implements VolumeSnapshotClassInterface +type FakeVolumeSnapshotClasses struct { + Fake *FakeSnapshotV1beta1 +} + +var volumesnapshotclassesResource = schema.GroupVersionResource{Group: "snapshot.storage.k8s.io", Version: "v1beta1", Resource: "volumesnapshotclasses"} + +var volumesnapshotclassesKind = schema.GroupVersionKind{Group: "snapshot.storage.k8s.io", Version: "v1beta1", Kind: "VolumeSnapshotClass"} + +// Get takes name of the volumeSnapshotClass, and returns the corresponding volumeSnapshotClass object, and an error if there is any. +func (c *FakeVolumeSnapshotClasses) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1beta1.VolumeSnapshotClass, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootGetAction(volumesnapshotclassesResource, name), &v1beta1.VolumeSnapshotClass{}) + if obj == nil { + return nil, err + } + return obj.(*v1beta1.VolumeSnapshotClass), err +} + +// List takes label and field selectors, and returns the list of VolumeSnapshotClasses that match those selectors. +func (c *FakeVolumeSnapshotClasses) List(ctx context.Context, opts v1.ListOptions) (result *v1beta1.VolumeSnapshotClassList, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootListAction(volumesnapshotclassesResource, volumesnapshotclassesKind, opts), &v1beta1.VolumeSnapshotClassList{}) + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1beta1.VolumeSnapshotClassList{ListMeta: obj.(*v1beta1.VolumeSnapshotClassList).ListMeta} + for _, item := range obj.(*v1beta1.VolumeSnapshotClassList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested volumeSnapshotClasses. +func (c *FakeVolumeSnapshotClasses) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewRootWatchAction(volumesnapshotclassesResource, opts)) +} + +// Create takes the representation of a volumeSnapshotClass and creates it. Returns the server's representation of the volumeSnapshotClass, and an error, if there is any. +func (c *FakeVolumeSnapshotClasses) Create(ctx context.Context, volumeSnapshotClass *v1beta1.VolumeSnapshotClass, opts v1.CreateOptions) (result *v1beta1.VolumeSnapshotClass, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootCreateAction(volumesnapshotclassesResource, volumeSnapshotClass), &v1beta1.VolumeSnapshotClass{}) + if obj == nil { + return nil, err + } + return obj.(*v1beta1.VolumeSnapshotClass), err +} + +// Update takes the representation of a volumeSnapshotClass and updates it. Returns the server's representation of the volumeSnapshotClass, and an error, if there is any. +func (c *FakeVolumeSnapshotClasses) Update(ctx context.Context, volumeSnapshotClass *v1beta1.VolumeSnapshotClass, opts v1.UpdateOptions) (result *v1beta1.VolumeSnapshotClass, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootUpdateAction(volumesnapshotclassesResource, volumeSnapshotClass), &v1beta1.VolumeSnapshotClass{}) + if obj == nil { + return nil, err + } + return obj.(*v1beta1.VolumeSnapshotClass), err +} + +// Delete takes name of the volumeSnapshotClass and deletes it. Returns an error if one occurs. +func (c *FakeVolumeSnapshotClasses) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewRootDeleteAction(volumesnapshotclassesResource, name), &v1beta1.VolumeSnapshotClass{}) + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeVolumeSnapshotClasses) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewRootDeleteCollectionAction(volumesnapshotclassesResource, listOpts) + + _, err := c.Fake.Invokes(action, &v1beta1.VolumeSnapshotClassList{}) + return err +} + +// Patch applies the patch and returns the patched volumeSnapshotClass. +func (c *FakeVolumeSnapshotClasses) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1beta1.VolumeSnapshotClass, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootPatchSubresourceAction(volumesnapshotclassesResource, name, pt, data, subresources...), &v1beta1.VolumeSnapshotClass{}) + if obj == nil { + return nil, err + } + return obj.(*v1beta1.VolumeSnapshotClass), err +} diff --git a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshotcontent.go b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshotcontent.go new file mode 100644 index 0000000000..71fda210b0 --- /dev/null +++ b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/fake/fake_volumesnapshotcontent.go @@ -0,0 +1,133 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + v1beta1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1beta1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + schema "k8s.io/apimachinery/pkg/runtime/schema" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeVolumeSnapshotContents implements VolumeSnapshotContentInterface +type FakeVolumeSnapshotContents struct { + Fake *FakeSnapshotV1beta1 +} + +var volumesnapshotcontentsResource = schema.GroupVersionResource{Group: "snapshot.storage.k8s.io", Version: "v1beta1", Resource: "volumesnapshotcontents"} + +var volumesnapshotcontentsKind = schema.GroupVersionKind{Group: "snapshot.storage.k8s.io", Version: "v1beta1", Kind: "VolumeSnapshotContent"} + +// Get takes name of the volumeSnapshotContent, and returns the corresponding volumeSnapshotContent object, and an error if there is any. +func (c *FakeVolumeSnapshotContents) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1beta1.VolumeSnapshotContent, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootGetAction(volumesnapshotcontentsResource, name), &v1beta1.VolumeSnapshotContent{}) + if obj == nil { + return nil, err + } + return obj.(*v1beta1.VolumeSnapshotContent), err +} + +// List takes label and field selectors, and returns the list of VolumeSnapshotContents that match those selectors. +func (c *FakeVolumeSnapshotContents) List(ctx context.Context, opts v1.ListOptions) (result *v1beta1.VolumeSnapshotContentList, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootListAction(volumesnapshotcontentsResource, volumesnapshotcontentsKind, opts), &v1beta1.VolumeSnapshotContentList{}) + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1beta1.VolumeSnapshotContentList{ListMeta: obj.(*v1beta1.VolumeSnapshotContentList).ListMeta} + for _, item := range obj.(*v1beta1.VolumeSnapshotContentList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested volumeSnapshotContents. +func (c *FakeVolumeSnapshotContents) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewRootWatchAction(volumesnapshotcontentsResource, opts)) +} + +// Create takes the representation of a volumeSnapshotContent and creates it. Returns the server's representation of the volumeSnapshotContent, and an error, if there is any. +func (c *FakeVolumeSnapshotContents) Create(ctx context.Context, volumeSnapshotContent *v1beta1.VolumeSnapshotContent, opts v1.CreateOptions) (result *v1beta1.VolumeSnapshotContent, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootCreateAction(volumesnapshotcontentsResource, volumeSnapshotContent), &v1beta1.VolumeSnapshotContent{}) + if obj == nil { + return nil, err + } + return obj.(*v1beta1.VolumeSnapshotContent), err +} + +// Update takes the representation of a volumeSnapshotContent and updates it. Returns the server's representation of the volumeSnapshotContent, and an error, if there is any. +func (c *FakeVolumeSnapshotContents) Update(ctx context.Context, volumeSnapshotContent *v1beta1.VolumeSnapshotContent, opts v1.UpdateOptions) (result *v1beta1.VolumeSnapshotContent, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootUpdateAction(volumesnapshotcontentsResource, volumeSnapshotContent), &v1beta1.VolumeSnapshotContent{}) + if obj == nil { + return nil, err + } + return obj.(*v1beta1.VolumeSnapshotContent), err +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeVolumeSnapshotContents) UpdateStatus(ctx context.Context, volumeSnapshotContent *v1beta1.VolumeSnapshotContent, opts v1.UpdateOptions) (*v1beta1.VolumeSnapshotContent, error) { + obj, err := c.Fake. + Invokes(testing.NewRootUpdateSubresourceAction(volumesnapshotcontentsResource, "status", volumeSnapshotContent), &v1beta1.VolumeSnapshotContent{}) + if obj == nil { + return nil, err + } + return obj.(*v1beta1.VolumeSnapshotContent), err +} + +// Delete takes name of the volumeSnapshotContent and deletes it. Returns an error if one occurs. +func (c *FakeVolumeSnapshotContents) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewRootDeleteAction(volumesnapshotcontentsResource, name), &v1beta1.VolumeSnapshotContent{}) + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeVolumeSnapshotContents) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewRootDeleteCollectionAction(volumesnapshotcontentsResource, listOpts) + + _, err := c.Fake.Invokes(action, &v1beta1.VolumeSnapshotContentList{}) + return err +} + +// Patch applies the patch and returns the patched volumeSnapshotContent. +func (c *FakeVolumeSnapshotContents) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1beta1.VolumeSnapshotContent, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootPatchSubresourceAction(volumesnapshotcontentsResource, name, pt, data, subresources...), &v1beta1.VolumeSnapshotContent{}) + if obj == nil { + return nil, err + } + return obj.(*v1beta1.VolumeSnapshotContent), err +} diff --git a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/generated_expansion.go b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/generated_expansion.go new file mode 100644 index 0000000000..181fe9947e --- /dev/null +++ b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/generated_expansion.go @@ -0,0 +1,25 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1beta1 + +type VolumeSnapshotExpansion interface{} + +type VolumeSnapshotClassExpansion interface{} + +type VolumeSnapshotContentExpansion interface{} diff --git a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshot.go b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshot.go new file mode 100644 index 0000000000..37e6c5e2ea --- /dev/null +++ b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshot.go @@ -0,0 +1,195 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1beta1 + +import ( + "context" + "time" + + v1beta1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1beta1" + scheme "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + rest "k8s.io/client-go/rest" +) + +// VolumeSnapshotsGetter has a method to return a VolumeSnapshotInterface. +// A group's client should implement this interface. +type VolumeSnapshotsGetter interface { + VolumeSnapshots(namespace string) VolumeSnapshotInterface +} + +// VolumeSnapshotInterface has methods to work with VolumeSnapshot resources. +type VolumeSnapshotInterface interface { + Create(ctx context.Context, volumeSnapshot *v1beta1.VolumeSnapshot, opts v1.CreateOptions) (*v1beta1.VolumeSnapshot, error) + Update(ctx context.Context, volumeSnapshot *v1beta1.VolumeSnapshot, opts v1.UpdateOptions) (*v1beta1.VolumeSnapshot, error) + UpdateStatus(ctx context.Context, volumeSnapshot *v1beta1.VolumeSnapshot, opts v1.UpdateOptions) (*v1beta1.VolumeSnapshot, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*v1beta1.VolumeSnapshot, error) + List(ctx context.Context, opts v1.ListOptions) (*v1beta1.VolumeSnapshotList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1beta1.VolumeSnapshot, err error) + VolumeSnapshotExpansion +} + +// volumeSnapshots implements VolumeSnapshotInterface +type volumeSnapshots struct { + client rest.Interface + ns string +} + +// newVolumeSnapshots returns a VolumeSnapshots +func newVolumeSnapshots(c *SnapshotV1beta1Client, namespace string) *volumeSnapshots { + return &volumeSnapshots{ + client: c.RESTClient(), + ns: namespace, + } +} + +// Get takes name of the volumeSnapshot, and returns the corresponding volumeSnapshot object, and an error if there is any. +func (c *volumeSnapshots) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1beta1.VolumeSnapshot, err error) { + result = &v1beta1.VolumeSnapshot{} + err = c.client.Get(). + Namespace(c.ns). + Resource("volumesnapshots"). + Name(name). + VersionedParams(&options, scheme.ParameterCodec). + Do(ctx). + Into(result) + return +} + +// List takes label and field selectors, and returns the list of VolumeSnapshots that match those selectors. +func (c *volumeSnapshots) List(ctx context.Context, opts v1.ListOptions) (result *v1beta1.VolumeSnapshotList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + result = &v1beta1.VolumeSnapshotList{} + err = c.client.Get(). + Namespace(c.ns). + Resource("volumesnapshots"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Do(ctx). + Into(result) + return +} + +// Watch returns a watch.Interface that watches the requested volumeSnapshots. +func (c *volumeSnapshots) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + opts.Watch = true + return c.client.Get(). + Namespace(c.ns). + Resource("volumesnapshots"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Watch(ctx) +} + +// Create takes the representation of a volumeSnapshot and creates it. Returns the server's representation of the volumeSnapshot, and an error, if there is any. +func (c *volumeSnapshots) Create(ctx context.Context, volumeSnapshot *v1beta1.VolumeSnapshot, opts v1.CreateOptions) (result *v1beta1.VolumeSnapshot, err error) { + result = &v1beta1.VolumeSnapshot{} + err = c.client.Post(). + Namespace(c.ns). + Resource("volumesnapshots"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(volumeSnapshot). + Do(ctx). + Into(result) + return +} + +// Update takes the representation of a volumeSnapshot and updates it. Returns the server's representation of the volumeSnapshot, and an error, if there is any. +func (c *volumeSnapshots) Update(ctx context.Context, volumeSnapshot *v1beta1.VolumeSnapshot, opts v1.UpdateOptions) (result *v1beta1.VolumeSnapshot, err error) { + result = &v1beta1.VolumeSnapshot{} + err = c.client.Put(). + Namespace(c.ns). + Resource("volumesnapshots"). + Name(volumeSnapshot.Name). + VersionedParams(&opts, scheme.ParameterCodec). + Body(volumeSnapshot). + Do(ctx). + Into(result) + return +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *volumeSnapshots) UpdateStatus(ctx context.Context, volumeSnapshot *v1beta1.VolumeSnapshot, opts v1.UpdateOptions) (result *v1beta1.VolumeSnapshot, err error) { + result = &v1beta1.VolumeSnapshot{} + err = c.client.Put(). + Namespace(c.ns). + Resource("volumesnapshots"). + Name(volumeSnapshot.Name). + SubResource("status"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(volumeSnapshot). + Do(ctx). + Into(result) + return +} + +// Delete takes name of the volumeSnapshot and deletes it. Returns an error if one occurs. +func (c *volumeSnapshots) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + return c.client.Delete(). + Namespace(c.ns). + Resource("volumesnapshots"). + Name(name). + Body(&opts). + Do(ctx). + Error() +} + +// DeleteCollection deletes a collection of objects. +func (c *volumeSnapshots) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + var timeout time.Duration + if listOpts.TimeoutSeconds != nil { + timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second + } + return c.client.Delete(). + Namespace(c.ns). + Resource("volumesnapshots"). + VersionedParams(&listOpts, scheme.ParameterCodec). + Timeout(timeout). + Body(&opts). + Do(ctx). + Error() +} + +// Patch applies the patch and returns the patched volumeSnapshot. +func (c *volumeSnapshots) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1beta1.VolumeSnapshot, err error) { + result = &v1beta1.VolumeSnapshot{} + err = c.client.Patch(pt). + Namespace(c.ns). + Resource("volumesnapshots"). + Name(name). + SubResource(subresources...). + VersionedParams(&opts, scheme.ParameterCodec). + Body(data). + Do(ctx). + Into(result) + return +} diff --git a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshot_client.go b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshot_client.go new file mode 100644 index 0000000000..347e1042a7 --- /dev/null +++ b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshot_client.go @@ -0,0 +1,99 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1beta1 + +import ( + v1beta1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1beta1" + "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/scheme" + rest "k8s.io/client-go/rest" +) + +type SnapshotV1beta1Interface interface { + RESTClient() rest.Interface + VolumeSnapshotsGetter + VolumeSnapshotClassesGetter + VolumeSnapshotContentsGetter +} + +// SnapshotV1beta1Client is used to interact with features provided by the snapshot.storage.k8s.io group. +type SnapshotV1beta1Client struct { + restClient rest.Interface +} + +func (c *SnapshotV1beta1Client) VolumeSnapshots(namespace string) VolumeSnapshotInterface { + return newVolumeSnapshots(c, namespace) +} + +func (c *SnapshotV1beta1Client) VolumeSnapshotClasses() VolumeSnapshotClassInterface { + return newVolumeSnapshotClasses(c) +} + +func (c *SnapshotV1beta1Client) VolumeSnapshotContents() VolumeSnapshotContentInterface { + return newVolumeSnapshotContents(c) +} + +// NewForConfig creates a new SnapshotV1beta1Client for the given config. +func NewForConfig(c *rest.Config) (*SnapshotV1beta1Client, error) { + config := *c + if err := setConfigDefaults(&config); err != nil { + return nil, err + } + client, err := rest.RESTClientFor(&config) + if err != nil { + return nil, err + } + return &SnapshotV1beta1Client{client}, nil +} + +// NewForConfigOrDie creates a new SnapshotV1beta1Client for the given config and +// panics if there is an error in the config. +func NewForConfigOrDie(c *rest.Config) *SnapshotV1beta1Client { + client, err := NewForConfig(c) + if err != nil { + panic(err) + } + return client +} + +// New creates a new SnapshotV1beta1Client for the given RESTClient. +func New(c rest.Interface) *SnapshotV1beta1Client { + return &SnapshotV1beta1Client{c} +} + +func setConfigDefaults(config *rest.Config) error { + gv := v1beta1.SchemeGroupVersion + config.GroupVersion = &gv + config.APIPath = "/apis" + config.NegotiatedSerializer = scheme.Codecs.WithoutConversion() + + if config.UserAgent == "" { + config.UserAgent = rest.DefaultKubernetesUserAgent() + } + + return nil +} + +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *SnapshotV1beta1Client) RESTClient() rest.Interface { + if c == nil { + return nil + } + return c.restClient +} diff --git a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshotclass.go b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshotclass.go new file mode 100644 index 0000000000..056b970516 --- /dev/null +++ b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshotclass.go @@ -0,0 +1,168 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1beta1 + +import ( + "context" + "time" + + v1beta1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1beta1" + scheme "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + rest "k8s.io/client-go/rest" +) + +// VolumeSnapshotClassesGetter has a method to return a VolumeSnapshotClassInterface. +// A group's client should implement this interface. +type VolumeSnapshotClassesGetter interface { + VolumeSnapshotClasses() VolumeSnapshotClassInterface +} + +// VolumeSnapshotClassInterface has methods to work with VolumeSnapshotClass resources. +type VolumeSnapshotClassInterface interface { + Create(ctx context.Context, volumeSnapshotClass *v1beta1.VolumeSnapshotClass, opts v1.CreateOptions) (*v1beta1.VolumeSnapshotClass, error) + Update(ctx context.Context, volumeSnapshotClass *v1beta1.VolumeSnapshotClass, opts v1.UpdateOptions) (*v1beta1.VolumeSnapshotClass, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*v1beta1.VolumeSnapshotClass, error) + List(ctx context.Context, opts v1.ListOptions) (*v1beta1.VolumeSnapshotClassList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1beta1.VolumeSnapshotClass, err error) + VolumeSnapshotClassExpansion +} + +// volumeSnapshotClasses implements VolumeSnapshotClassInterface +type volumeSnapshotClasses struct { + client rest.Interface +} + +// newVolumeSnapshotClasses returns a VolumeSnapshotClasses +func newVolumeSnapshotClasses(c *SnapshotV1beta1Client) *volumeSnapshotClasses { + return &volumeSnapshotClasses{ + client: c.RESTClient(), + } +} + +// Get takes name of the volumeSnapshotClass, and returns the corresponding volumeSnapshotClass object, and an error if there is any. +func (c *volumeSnapshotClasses) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1beta1.VolumeSnapshotClass, err error) { + result = &v1beta1.VolumeSnapshotClass{} + err = c.client.Get(). + Resource("volumesnapshotclasses"). + Name(name). + VersionedParams(&options, scheme.ParameterCodec). + Do(ctx). + Into(result) + return +} + +// List takes label and field selectors, and returns the list of VolumeSnapshotClasses that match those selectors. +func (c *volumeSnapshotClasses) List(ctx context.Context, opts v1.ListOptions) (result *v1beta1.VolumeSnapshotClassList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + result = &v1beta1.VolumeSnapshotClassList{} + err = c.client.Get(). + Resource("volumesnapshotclasses"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Do(ctx). + Into(result) + return +} + +// Watch returns a watch.Interface that watches the requested volumeSnapshotClasses. +func (c *volumeSnapshotClasses) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + opts.Watch = true + return c.client.Get(). + Resource("volumesnapshotclasses"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Watch(ctx) +} + +// Create takes the representation of a volumeSnapshotClass and creates it. Returns the server's representation of the volumeSnapshotClass, and an error, if there is any. +func (c *volumeSnapshotClasses) Create(ctx context.Context, volumeSnapshotClass *v1beta1.VolumeSnapshotClass, opts v1.CreateOptions) (result *v1beta1.VolumeSnapshotClass, err error) { + result = &v1beta1.VolumeSnapshotClass{} + err = c.client.Post(). + Resource("volumesnapshotclasses"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(volumeSnapshotClass). + Do(ctx). + Into(result) + return +} + +// Update takes the representation of a volumeSnapshotClass and updates it. Returns the server's representation of the volumeSnapshotClass, and an error, if there is any. +func (c *volumeSnapshotClasses) Update(ctx context.Context, volumeSnapshotClass *v1beta1.VolumeSnapshotClass, opts v1.UpdateOptions) (result *v1beta1.VolumeSnapshotClass, err error) { + result = &v1beta1.VolumeSnapshotClass{} + err = c.client.Put(). + Resource("volumesnapshotclasses"). + Name(volumeSnapshotClass.Name). + VersionedParams(&opts, scheme.ParameterCodec). + Body(volumeSnapshotClass). + Do(ctx). + Into(result) + return +} + +// Delete takes name of the volumeSnapshotClass and deletes it. Returns an error if one occurs. +func (c *volumeSnapshotClasses) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + return c.client.Delete(). + Resource("volumesnapshotclasses"). + Name(name). + Body(&opts). + Do(ctx). + Error() +} + +// DeleteCollection deletes a collection of objects. +func (c *volumeSnapshotClasses) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + var timeout time.Duration + if listOpts.TimeoutSeconds != nil { + timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second + } + return c.client.Delete(). + Resource("volumesnapshotclasses"). + VersionedParams(&listOpts, scheme.ParameterCodec). + Timeout(timeout). + Body(&opts). + Do(ctx). + Error() +} + +// Patch applies the patch and returns the patched volumeSnapshotClass. +func (c *volumeSnapshotClasses) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1beta1.VolumeSnapshotClass, err error) { + result = &v1beta1.VolumeSnapshotClass{} + err = c.client.Patch(pt). + Resource("volumesnapshotclasses"). + Name(name). + SubResource(subresources...). + VersionedParams(&opts, scheme.ParameterCodec). + Body(data). + Do(ctx). + Into(result) + return +} diff --git a/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshotcontent.go b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshotcontent.go new file mode 100644 index 0000000000..eab9ce2a58 --- /dev/null +++ b/vendor/github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/volumesnapshotcontent.go @@ -0,0 +1,184 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1beta1 + +import ( + "context" + "time" + + v1beta1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1beta1" + scheme "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + rest "k8s.io/client-go/rest" +) + +// VolumeSnapshotContentsGetter has a method to return a VolumeSnapshotContentInterface. +// A group's client should implement this interface. +type VolumeSnapshotContentsGetter interface { + VolumeSnapshotContents() VolumeSnapshotContentInterface +} + +// VolumeSnapshotContentInterface has methods to work with VolumeSnapshotContent resources. +type VolumeSnapshotContentInterface interface { + Create(ctx context.Context, volumeSnapshotContent *v1beta1.VolumeSnapshotContent, opts v1.CreateOptions) (*v1beta1.VolumeSnapshotContent, error) + Update(ctx context.Context, volumeSnapshotContent *v1beta1.VolumeSnapshotContent, opts v1.UpdateOptions) (*v1beta1.VolumeSnapshotContent, error) + UpdateStatus(ctx context.Context, volumeSnapshotContent *v1beta1.VolumeSnapshotContent, opts v1.UpdateOptions) (*v1beta1.VolumeSnapshotContent, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*v1beta1.VolumeSnapshotContent, error) + List(ctx context.Context, opts v1.ListOptions) (*v1beta1.VolumeSnapshotContentList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1beta1.VolumeSnapshotContent, err error) + VolumeSnapshotContentExpansion +} + +// volumeSnapshotContents implements VolumeSnapshotContentInterface +type volumeSnapshotContents struct { + client rest.Interface +} + +// newVolumeSnapshotContents returns a VolumeSnapshotContents +func newVolumeSnapshotContents(c *SnapshotV1beta1Client) *volumeSnapshotContents { + return &volumeSnapshotContents{ + client: c.RESTClient(), + } +} + +// Get takes name of the volumeSnapshotContent, and returns the corresponding volumeSnapshotContent object, and an error if there is any. +func (c *volumeSnapshotContents) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1beta1.VolumeSnapshotContent, err error) { + result = &v1beta1.VolumeSnapshotContent{} + err = c.client.Get(). + Resource("volumesnapshotcontents"). + Name(name). + VersionedParams(&options, scheme.ParameterCodec). + Do(ctx). + Into(result) + return +} + +// List takes label and field selectors, and returns the list of VolumeSnapshotContents that match those selectors. +func (c *volumeSnapshotContents) List(ctx context.Context, opts v1.ListOptions) (result *v1beta1.VolumeSnapshotContentList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + result = &v1beta1.VolumeSnapshotContentList{} + err = c.client.Get(). + Resource("volumesnapshotcontents"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Do(ctx). + Into(result) + return +} + +// Watch returns a watch.Interface that watches the requested volumeSnapshotContents. +func (c *volumeSnapshotContents) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + opts.Watch = true + return c.client.Get(). + Resource("volumesnapshotcontents"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Watch(ctx) +} + +// Create takes the representation of a volumeSnapshotContent and creates it. Returns the server's representation of the volumeSnapshotContent, and an error, if there is any. +func (c *volumeSnapshotContents) Create(ctx context.Context, volumeSnapshotContent *v1beta1.VolumeSnapshotContent, opts v1.CreateOptions) (result *v1beta1.VolumeSnapshotContent, err error) { + result = &v1beta1.VolumeSnapshotContent{} + err = c.client.Post(). + Resource("volumesnapshotcontents"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(volumeSnapshotContent). + Do(ctx). + Into(result) + return +} + +// Update takes the representation of a volumeSnapshotContent and updates it. Returns the server's representation of the volumeSnapshotContent, and an error, if there is any. +func (c *volumeSnapshotContents) Update(ctx context.Context, volumeSnapshotContent *v1beta1.VolumeSnapshotContent, opts v1.UpdateOptions) (result *v1beta1.VolumeSnapshotContent, err error) { + result = &v1beta1.VolumeSnapshotContent{} + err = c.client.Put(). + Resource("volumesnapshotcontents"). + Name(volumeSnapshotContent.Name). + VersionedParams(&opts, scheme.ParameterCodec). + Body(volumeSnapshotContent). + Do(ctx). + Into(result) + return +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *volumeSnapshotContents) UpdateStatus(ctx context.Context, volumeSnapshotContent *v1beta1.VolumeSnapshotContent, opts v1.UpdateOptions) (result *v1beta1.VolumeSnapshotContent, err error) { + result = &v1beta1.VolumeSnapshotContent{} + err = c.client.Put(). + Resource("volumesnapshotcontents"). + Name(volumeSnapshotContent.Name). + SubResource("status"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(volumeSnapshotContent). + Do(ctx). + Into(result) + return +} + +// Delete takes name of the volumeSnapshotContent and deletes it. Returns an error if one occurs. +func (c *volumeSnapshotContents) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + return c.client.Delete(). + Resource("volumesnapshotcontents"). + Name(name). + Body(&opts). + Do(ctx). + Error() +} + +// DeleteCollection deletes a collection of objects. +func (c *volumeSnapshotContents) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + var timeout time.Duration + if listOpts.TimeoutSeconds != nil { + timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second + } + return c.client.Delete(). + Resource("volumesnapshotcontents"). + VersionedParams(&listOpts, scheme.ParameterCodec). + Timeout(timeout). + Body(&opts). + Do(ctx). + Error() +} + +// Patch applies the patch and returns the patched volumeSnapshotContent. +func (c *volumeSnapshotContents) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1beta1.VolumeSnapshotContent, err error) { + result = &v1beta1.VolumeSnapshotContent{} + err = c.client.Patch(pt). + Resource("volumesnapshotcontents"). + Name(name). + SubResource(subresources...). + VersionedParams(&opts, scheme.ParameterCodec). + Body(data). + Do(ctx). + Into(result) + return +} diff --git a/vendor/github.com/mailru/easyjson/jlexer/lexer.go b/vendor/github.com/mailru/easyjson/jlexer/lexer.go index b5f5e26132..a42e9d65ad 100644 --- a/vendor/github.com/mailru/easyjson/jlexer/lexer.go +++ b/vendor/github.com/mailru/easyjson/jlexer/lexer.go @@ -401,7 +401,6 @@ func (r *Lexer) scanToken() { // consume resets the current token to allow scanning the next one. func (r *Lexer) consume() { r.token.kind = tokenUndef - r.token.byteValueCloned = false r.token.delimValue = 0 } @@ -529,7 +528,6 @@ func (r *Lexer) Skip() { func (r *Lexer) SkipRecursive() { r.scanToken() var start, end byte - startPos := r.start switch r.token.delimValue { case '{': @@ -555,14 +553,6 @@ func (r *Lexer) SkipRecursive() { level-- if level == 0 { r.pos += i + 1 - if !json.Valid(r.Data[startPos:r.pos]) { - r.pos = len(r.Data) - r.fatalError = &LexerError{ - Reason: "skipped array/object json value is invalid", - Offset: r.pos, - Data: string(r.Data[r.pos:]), - } - } return } case c == '\\' && inQuotes: @@ -712,10 +702,6 @@ func (r *Lexer) Bytes() []byte { r.errInvalidToken("string") return nil } - if err := r.unescapeStringToken(); err != nil { - r.errInvalidToken("string") - return nil - } ret := make([]byte, base64.StdEncoding.DecodedLen(len(r.token.byteValue))) n, err := base64.StdEncoding.Decode(ret, r.token.byteValue) if err != nil { diff --git a/vendor/github.com/vmware/govmomi/find/finder.go b/vendor/github.com/vmware/govmomi/find/finder.go index 4830fc26eb..61ac780c45 100644 --- a/vendor/github.com/vmware/govmomi/find/finder.go +++ b/vendor/github.com/vmware/govmomi/find/finder.go @@ -1,5 +1,5 @@ /* -Copyright (c) 2014-2023 VMware, Inc. All Rights Reserved. +Copyright (c) 2014-2020 VMware, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -784,11 +784,6 @@ func (f *Finder) NetworkList(ctx context.Context, path string) ([]object.Network } if len(ns) == 0 { - net, nerr := f.networkByID(ctx, path) - if nerr == nil { - return []object.NetworkReference{net}, nil - } - return nil, &NotFoundError{"network", path} } @@ -803,13 +798,18 @@ func (f *Finder) NetworkList(ctx context.Context, path string) ([]object.Network // Examples: // - Name: "dvpg-1" // - Inventory Path: "vds-1/dvpg-1" -// - Cluster Path: "/dc-1/host/cluster-1/dvpg-1" // - ManagedObject ID: "DistributedVirtualPortgroup:dvportgroup-53" // - Logical Switch UUID: "da2a59b8-2450-4cb2-b5cc-79c4c1d2144c" // - Segment ID: "/infra/segments/vnet_ce50e69b-1784-4a14-9206-ffd7f1f146f7" func (f *Finder) Network(ctx context.Context, path string) (object.NetworkReference, error) { networks, err := f.NetworkList(ctx, path) if err != nil { + if _, ok := err.(*NotFoundError); ok { + net, nerr := f.networkByID(ctx, path) + if nerr == nil { + return net, nil + } + } return nil, err } diff --git a/vendor/github.com/vmware/govmomi/list/lister.go b/vendor/github.com/vmware/govmomi/list/lister.go index 92a40e8ba8..9a4caed686 100644 --- a/vendor/github.com/vmware/govmomi/list/lister.go +++ b/vendor/github.com/vmware/govmomi/list/lister.go @@ -1,5 +1,5 @@ /* -Copyright (c) 2014-2023 VMware, Inc. All Rights Reserved. +Copyright (c) 2014-2016 VMware, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -312,7 +312,6 @@ func (l Lister) ListComputeResource(ctx context.Context) ([]Element, error) { fields := []string{ "host", - "network", "resourcePool", } @@ -328,7 +327,6 @@ func (l Lister) ListComputeResource(ctx context.Context) ([]Element, error) { childTypes := []string{ "HostSystem", - "Network", "ResourcePool", } diff --git a/vendor/github.com/vmware/govmomi/lookup/client.go b/vendor/github.com/vmware/govmomi/lookup/client.go index 4cc73e0d33..b3c19846a1 100644 --- a/vendor/github.com/vmware/govmomi/lookup/client.go +++ b/vendor/github.com/vmware/govmomi/lookup/client.go @@ -125,9 +125,7 @@ func EndpointURL(ctx context.Context, c *vim25.Client, path string, filter *type path = endpoint.Url if u, err := url.Parse(path); err == nil { - // Set thumbprint only for endpoints on hosts outside this vCenter. - // Platform Services may live on multiple hosts. - if c.URL().Host != u.Host && c.Thumbprint(u.Host) == "" { + if c.Thumbprint(u.Host) == "" { c.SetThumbprint(u.Host, endpointThumbprint(endpoint)) } } diff --git a/vendor/github.com/vmware/govmomi/simulator/container.go b/vendor/github.com/vmware/govmomi/simulator/container.go index fec1c0f486..2fc8515f68 100644 --- a/vendor/github.com/vmware/govmomi/simulator/container.go +++ b/vendor/github.com/vmware/govmomi/simulator/container.go @@ -114,20 +114,6 @@ func (c *container) inspect(vm *VirtualMachine) error { net := &vm.Guest.Net[0] net.IpAddress = []string{s.IPAddress} net.MacAddress = s.MacAddress - net.IpConfig = &types.NetIpConfigInfo{ - IpAddress: []types.NetIpConfigInfoIpAddress{{ - IpAddress: s.IPAddress, - PrefixLength: int32(s.IPPrefixLen), - State: string(types.NetIpConfigInfoIpAddressStatusPreferred), - }}, - } - } - - for _, d := range vm.Config.Hardware.Device { - if eth, ok := d.(types.BaseVirtualEthernetCard); ok { - eth.GetVirtualEthernetCard().MacAddress = s.MacAddress - break - } } } diff --git a/vendor/golang.org/x/text/width/kind_string.go b/vendor/golang.org/x/text/width/kind_string.go new file mode 100644 index 0000000000..dd3febd43b --- /dev/null +++ b/vendor/golang.org/x/text/width/kind_string.go @@ -0,0 +1,28 @@ +// Code generated by "stringer -type=Kind"; DO NOT EDIT. + +package width + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[Neutral-0] + _ = x[EastAsianAmbiguous-1] + _ = x[EastAsianWide-2] + _ = x[EastAsianNarrow-3] + _ = x[EastAsianFullwidth-4] + _ = x[EastAsianHalfwidth-5] +} + +const _Kind_name = "NeutralEastAsianAmbiguousEastAsianWideEastAsianNarrowEastAsianFullwidthEastAsianHalfwidth" + +var _Kind_index = [...]uint8{0, 7, 25, 38, 53, 71, 89} + +func (i Kind) String() string { + if i < 0 || i >= Kind(len(_Kind_index)-1) { + return "Kind(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _Kind_name[_Kind_index[i]:_Kind_index[i+1]] +} diff --git a/vendor/golang.org/x/text/width/tables10.0.0.go b/vendor/golang.org/x/text/width/tables10.0.0.go new file mode 100644 index 0000000000..cd9d91cafb --- /dev/null +++ b/vendor/golang.org/x/text/width/tables10.0.0.go @@ -0,0 +1,1329 @@ +// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. + +//go:build go1.10 && !go1.13 +// +build go1.10,!go1.13 + +package width + +// UnicodeVersion is the Unicode version from which the tables in this package are derived. +const UnicodeVersion = "10.0.0" + +// lookup returns the trie value for the first UTF-8 encoding in s and +// the width in bytes of this encoding. The size will be 0 if s does not +// hold enough bytes to complete the encoding. len(s) must be greater than 0. +func (t *widthTrie) lookup(s []byte) (v uint16, sz int) { + c0 := s[0] + switch { + case c0 < 0x80: // is ASCII + return widthValues[c0], 1 + case c0 < 0xC2: + return 0, 1 // Illegal UTF-8: not a starter, not ASCII. + case c0 < 0xE0: // 2-byte UTF-8 + if len(s) < 2 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c1), 2 + case c0 < 0xF0: // 3-byte UTF-8 + if len(s) < 3 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = widthIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c2), 3 + case c0 < 0xF8: // 4-byte UTF-8 + if len(s) < 4 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = widthIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + o = uint32(i)<<6 + uint32(c2) + i = widthIndex[o] + c3 := s[3] + if c3 < 0x80 || 0xC0 <= c3 { + return 0, 3 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c3), 4 + } + // Illegal rune + return 0, 1 +} + +// lookupUnsafe returns the trie value for the first UTF-8 encoding in s. +// s must start with a full and valid UTF-8 encoded rune. +func (t *widthTrie) lookupUnsafe(s []byte) uint16 { + c0 := s[0] + if c0 < 0x80 { // is ASCII + return widthValues[c0] + } + i := widthIndex[c0] + if c0 < 0xE0 { // 2-byte UTF-8 + return t.lookupValue(uint32(i), s[1]) + } + i = widthIndex[uint32(i)<<6+uint32(s[1])] + if c0 < 0xF0 { // 3-byte UTF-8 + return t.lookupValue(uint32(i), s[2]) + } + i = widthIndex[uint32(i)<<6+uint32(s[2])] + if c0 < 0xF8 { // 4-byte UTF-8 + return t.lookupValue(uint32(i), s[3]) + } + return 0 +} + +// lookupString returns the trie value for the first UTF-8 encoding in s and +// the width in bytes of this encoding. The size will be 0 if s does not +// hold enough bytes to complete the encoding. len(s) must be greater than 0. +func (t *widthTrie) lookupString(s string) (v uint16, sz int) { + c0 := s[0] + switch { + case c0 < 0x80: // is ASCII + return widthValues[c0], 1 + case c0 < 0xC2: + return 0, 1 // Illegal UTF-8: not a starter, not ASCII. + case c0 < 0xE0: // 2-byte UTF-8 + if len(s) < 2 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c1), 2 + case c0 < 0xF0: // 3-byte UTF-8 + if len(s) < 3 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = widthIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c2), 3 + case c0 < 0xF8: // 4-byte UTF-8 + if len(s) < 4 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = widthIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + o = uint32(i)<<6 + uint32(c2) + i = widthIndex[o] + c3 := s[3] + if c3 < 0x80 || 0xC0 <= c3 { + return 0, 3 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c3), 4 + } + // Illegal rune + return 0, 1 +} + +// lookupStringUnsafe returns the trie value for the first UTF-8 encoding in s. +// s must start with a full and valid UTF-8 encoded rune. +func (t *widthTrie) lookupStringUnsafe(s string) uint16 { + c0 := s[0] + if c0 < 0x80 { // is ASCII + return widthValues[c0] + } + i := widthIndex[c0] + if c0 < 0xE0 { // 2-byte UTF-8 + return t.lookupValue(uint32(i), s[1]) + } + i = widthIndex[uint32(i)<<6+uint32(s[1])] + if c0 < 0xF0 { // 3-byte UTF-8 + return t.lookupValue(uint32(i), s[2]) + } + i = widthIndex[uint32(i)<<6+uint32(s[2])] + if c0 < 0xF8 { // 4-byte UTF-8 + return t.lookupValue(uint32(i), s[3]) + } + return 0 +} + +// widthTrie. Total size: 14336 bytes (14.00 KiB). Checksum: c59df54630d3dc4a. +type widthTrie struct{} + +func newWidthTrie(i int) *widthTrie { + return &widthTrie{} +} + +// lookupValue determines the type of block n and looks up the value for b. +func (t *widthTrie) lookupValue(n uint32, b byte) uint16 { + switch { + default: + return uint16(widthValues[n<<6+uint32(b)]) + } +} + +// widthValues: 101 blocks, 6464 entries, 12928 bytes +// The third block is the zero block. +var widthValues = [6464]uint16{ + // Block 0x0, offset 0x0 + 0x20: 0x6001, 0x21: 0x6002, 0x22: 0x6002, 0x23: 0x6002, + 0x24: 0x6002, 0x25: 0x6002, 0x26: 0x6002, 0x27: 0x6002, 0x28: 0x6002, 0x29: 0x6002, + 0x2a: 0x6002, 0x2b: 0x6002, 0x2c: 0x6002, 0x2d: 0x6002, 0x2e: 0x6002, 0x2f: 0x6002, + 0x30: 0x6002, 0x31: 0x6002, 0x32: 0x6002, 0x33: 0x6002, 0x34: 0x6002, 0x35: 0x6002, + 0x36: 0x6002, 0x37: 0x6002, 0x38: 0x6002, 0x39: 0x6002, 0x3a: 0x6002, 0x3b: 0x6002, + 0x3c: 0x6002, 0x3d: 0x6002, 0x3e: 0x6002, 0x3f: 0x6002, + // Block 0x1, offset 0x40 + 0x40: 0x6003, 0x41: 0x6003, 0x42: 0x6003, 0x43: 0x6003, 0x44: 0x6003, 0x45: 0x6003, + 0x46: 0x6003, 0x47: 0x6003, 0x48: 0x6003, 0x49: 0x6003, 0x4a: 0x6003, 0x4b: 0x6003, + 0x4c: 0x6003, 0x4d: 0x6003, 0x4e: 0x6003, 0x4f: 0x6003, 0x50: 0x6003, 0x51: 0x6003, + 0x52: 0x6003, 0x53: 0x6003, 0x54: 0x6003, 0x55: 0x6003, 0x56: 0x6003, 0x57: 0x6003, + 0x58: 0x6003, 0x59: 0x6003, 0x5a: 0x6003, 0x5b: 0x6003, 0x5c: 0x6003, 0x5d: 0x6003, + 0x5e: 0x6003, 0x5f: 0x6003, 0x60: 0x6004, 0x61: 0x6004, 0x62: 0x6004, 0x63: 0x6004, + 0x64: 0x6004, 0x65: 0x6004, 0x66: 0x6004, 0x67: 0x6004, 0x68: 0x6004, 0x69: 0x6004, + 0x6a: 0x6004, 0x6b: 0x6004, 0x6c: 0x6004, 0x6d: 0x6004, 0x6e: 0x6004, 0x6f: 0x6004, + 0x70: 0x6004, 0x71: 0x6004, 0x72: 0x6004, 0x73: 0x6004, 0x74: 0x6004, 0x75: 0x6004, + 0x76: 0x6004, 0x77: 0x6004, 0x78: 0x6004, 0x79: 0x6004, 0x7a: 0x6004, 0x7b: 0x6004, + 0x7c: 0x6004, 0x7d: 0x6004, 0x7e: 0x6004, + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0xe1: 0x2000, 0xe2: 0x6005, 0xe3: 0x6005, + 0xe4: 0x2000, 0xe5: 0x6006, 0xe6: 0x6005, 0xe7: 0x2000, 0xe8: 0x2000, + 0xea: 0x2000, 0xec: 0x6007, 0xed: 0x2000, 0xee: 0x2000, 0xef: 0x6008, + 0xf0: 0x2000, 0xf1: 0x2000, 0xf2: 0x2000, 0xf3: 0x2000, 0xf4: 0x2000, + 0xf6: 0x2000, 0xf7: 0x2000, 0xf8: 0x2000, 0xf9: 0x2000, 0xfa: 0x2000, + 0xfc: 0x2000, 0xfd: 0x2000, 0xfe: 0x2000, 0xff: 0x2000, + // Block 0x4, offset 0x100 + 0x106: 0x2000, + 0x110: 0x2000, + 0x117: 0x2000, + 0x118: 0x2000, + 0x11e: 0x2000, 0x11f: 0x2000, 0x120: 0x2000, 0x121: 0x2000, + 0x126: 0x2000, 0x128: 0x2000, 0x129: 0x2000, + 0x12a: 0x2000, 0x12c: 0x2000, 0x12d: 0x2000, + 0x130: 0x2000, 0x132: 0x2000, 0x133: 0x2000, + 0x137: 0x2000, 0x138: 0x2000, 0x139: 0x2000, 0x13a: 0x2000, + 0x13c: 0x2000, 0x13e: 0x2000, + // Block 0x5, offset 0x140 + 0x141: 0x2000, + 0x151: 0x2000, + 0x153: 0x2000, + 0x15b: 0x2000, + 0x166: 0x2000, 0x167: 0x2000, + 0x16b: 0x2000, + 0x171: 0x2000, 0x172: 0x2000, 0x173: 0x2000, + 0x178: 0x2000, + 0x17f: 0x2000, + // Block 0x6, offset 0x180 + 0x180: 0x2000, 0x181: 0x2000, 0x182: 0x2000, 0x184: 0x2000, + 0x188: 0x2000, 0x189: 0x2000, 0x18a: 0x2000, 0x18b: 0x2000, + 0x18d: 0x2000, + 0x192: 0x2000, 0x193: 0x2000, + 0x1a6: 0x2000, 0x1a7: 0x2000, + 0x1ab: 0x2000, + // Block 0x7, offset 0x1c0 + 0x1ce: 0x2000, 0x1d0: 0x2000, + 0x1d2: 0x2000, 0x1d4: 0x2000, 0x1d6: 0x2000, + 0x1d8: 0x2000, 0x1da: 0x2000, 0x1dc: 0x2000, + // Block 0x8, offset 0x200 + 0x211: 0x2000, + 0x221: 0x2000, + // Block 0x9, offset 0x240 + 0x244: 0x2000, + 0x247: 0x2000, 0x249: 0x2000, 0x24a: 0x2000, 0x24b: 0x2000, + 0x24d: 0x2000, 0x250: 0x2000, + 0x258: 0x2000, 0x259: 0x2000, 0x25a: 0x2000, 0x25b: 0x2000, 0x25d: 0x2000, + 0x25f: 0x2000, + // Block 0xa, offset 0x280 + 0x280: 0x2000, 0x281: 0x2000, 0x282: 0x2000, 0x283: 0x2000, 0x284: 0x2000, 0x285: 0x2000, + 0x286: 0x2000, 0x287: 0x2000, 0x288: 0x2000, 0x289: 0x2000, 0x28a: 0x2000, 0x28b: 0x2000, + 0x28c: 0x2000, 0x28d: 0x2000, 0x28e: 0x2000, 0x28f: 0x2000, 0x290: 0x2000, 0x291: 0x2000, + 0x292: 0x2000, 0x293: 0x2000, 0x294: 0x2000, 0x295: 0x2000, 0x296: 0x2000, 0x297: 0x2000, + 0x298: 0x2000, 0x299: 0x2000, 0x29a: 0x2000, 0x29b: 0x2000, 0x29c: 0x2000, 0x29d: 0x2000, + 0x29e: 0x2000, 0x29f: 0x2000, 0x2a0: 0x2000, 0x2a1: 0x2000, 0x2a2: 0x2000, 0x2a3: 0x2000, + 0x2a4: 0x2000, 0x2a5: 0x2000, 0x2a6: 0x2000, 0x2a7: 0x2000, 0x2a8: 0x2000, 0x2a9: 0x2000, + 0x2aa: 0x2000, 0x2ab: 0x2000, 0x2ac: 0x2000, 0x2ad: 0x2000, 0x2ae: 0x2000, 0x2af: 0x2000, + 0x2b0: 0x2000, 0x2b1: 0x2000, 0x2b2: 0x2000, 0x2b3: 0x2000, 0x2b4: 0x2000, 0x2b5: 0x2000, + 0x2b6: 0x2000, 0x2b7: 0x2000, 0x2b8: 0x2000, 0x2b9: 0x2000, 0x2ba: 0x2000, 0x2bb: 0x2000, + 0x2bc: 0x2000, 0x2bd: 0x2000, 0x2be: 0x2000, 0x2bf: 0x2000, + // Block 0xb, offset 0x2c0 + 0x2c0: 0x2000, 0x2c1: 0x2000, 0x2c2: 0x2000, 0x2c3: 0x2000, 0x2c4: 0x2000, 0x2c5: 0x2000, + 0x2c6: 0x2000, 0x2c7: 0x2000, 0x2c8: 0x2000, 0x2c9: 0x2000, 0x2ca: 0x2000, 0x2cb: 0x2000, + 0x2cc: 0x2000, 0x2cd: 0x2000, 0x2ce: 0x2000, 0x2cf: 0x2000, 0x2d0: 0x2000, 0x2d1: 0x2000, + 0x2d2: 0x2000, 0x2d3: 0x2000, 0x2d4: 0x2000, 0x2d5: 0x2000, 0x2d6: 0x2000, 0x2d7: 0x2000, + 0x2d8: 0x2000, 0x2d9: 0x2000, 0x2da: 0x2000, 0x2db: 0x2000, 0x2dc: 0x2000, 0x2dd: 0x2000, + 0x2de: 0x2000, 0x2df: 0x2000, 0x2e0: 0x2000, 0x2e1: 0x2000, 0x2e2: 0x2000, 0x2e3: 0x2000, + 0x2e4: 0x2000, 0x2e5: 0x2000, 0x2e6: 0x2000, 0x2e7: 0x2000, 0x2e8: 0x2000, 0x2e9: 0x2000, + 0x2ea: 0x2000, 0x2eb: 0x2000, 0x2ec: 0x2000, 0x2ed: 0x2000, 0x2ee: 0x2000, 0x2ef: 0x2000, + // Block 0xc, offset 0x300 + 0x311: 0x2000, + 0x312: 0x2000, 0x313: 0x2000, 0x314: 0x2000, 0x315: 0x2000, 0x316: 0x2000, 0x317: 0x2000, + 0x318: 0x2000, 0x319: 0x2000, 0x31a: 0x2000, 0x31b: 0x2000, 0x31c: 0x2000, 0x31d: 0x2000, + 0x31e: 0x2000, 0x31f: 0x2000, 0x320: 0x2000, 0x321: 0x2000, 0x323: 0x2000, + 0x324: 0x2000, 0x325: 0x2000, 0x326: 0x2000, 0x327: 0x2000, 0x328: 0x2000, 0x329: 0x2000, + 0x331: 0x2000, 0x332: 0x2000, 0x333: 0x2000, 0x334: 0x2000, 0x335: 0x2000, + 0x336: 0x2000, 0x337: 0x2000, 0x338: 0x2000, 0x339: 0x2000, 0x33a: 0x2000, 0x33b: 0x2000, + 0x33c: 0x2000, 0x33d: 0x2000, 0x33e: 0x2000, 0x33f: 0x2000, + // Block 0xd, offset 0x340 + 0x340: 0x2000, 0x341: 0x2000, 0x343: 0x2000, 0x344: 0x2000, 0x345: 0x2000, + 0x346: 0x2000, 0x347: 0x2000, 0x348: 0x2000, 0x349: 0x2000, + // Block 0xe, offset 0x380 + 0x381: 0x2000, + 0x390: 0x2000, 0x391: 0x2000, + 0x392: 0x2000, 0x393: 0x2000, 0x394: 0x2000, 0x395: 0x2000, 0x396: 0x2000, 0x397: 0x2000, + 0x398: 0x2000, 0x399: 0x2000, 0x39a: 0x2000, 0x39b: 0x2000, 0x39c: 0x2000, 0x39d: 0x2000, + 0x39e: 0x2000, 0x39f: 0x2000, 0x3a0: 0x2000, 0x3a1: 0x2000, 0x3a2: 0x2000, 0x3a3: 0x2000, + 0x3a4: 0x2000, 0x3a5: 0x2000, 0x3a6: 0x2000, 0x3a7: 0x2000, 0x3a8: 0x2000, 0x3a9: 0x2000, + 0x3aa: 0x2000, 0x3ab: 0x2000, 0x3ac: 0x2000, 0x3ad: 0x2000, 0x3ae: 0x2000, 0x3af: 0x2000, + 0x3b0: 0x2000, 0x3b1: 0x2000, 0x3b2: 0x2000, 0x3b3: 0x2000, 0x3b4: 0x2000, 0x3b5: 0x2000, + 0x3b6: 0x2000, 0x3b7: 0x2000, 0x3b8: 0x2000, 0x3b9: 0x2000, 0x3ba: 0x2000, 0x3bb: 0x2000, + 0x3bc: 0x2000, 0x3bd: 0x2000, 0x3be: 0x2000, 0x3bf: 0x2000, + // Block 0xf, offset 0x3c0 + 0x3c0: 0x2000, 0x3c1: 0x2000, 0x3c2: 0x2000, 0x3c3: 0x2000, 0x3c4: 0x2000, 0x3c5: 0x2000, + 0x3c6: 0x2000, 0x3c7: 0x2000, 0x3c8: 0x2000, 0x3c9: 0x2000, 0x3ca: 0x2000, 0x3cb: 0x2000, + 0x3cc: 0x2000, 0x3cd: 0x2000, 0x3ce: 0x2000, 0x3cf: 0x2000, 0x3d1: 0x2000, + // Block 0x10, offset 0x400 + 0x400: 0x4000, 0x401: 0x4000, 0x402: 0x4000, 0x403: 0x4000, 0x404: 0x4000, 0x405: 0x4000, + 0x406: 0x4000, 0x407: 0x4000, 0x408: 0x4000, 0x409: 0x4000, 0x40a: 0x4000, 0x40b: 0x4000, + 0x40c: 0x4000, 0x40d: 0x4000, 0x40e: 0x4000, 0x40f: 0x4000, 0x410: 0x4000, 0x411: 0x4000, + 0x412: 0x4000, 0x413: 0x4000, 0x414: 0x4000, 0x415: 0x4000, 0x416: 0x4000, 0x417: 0x4000, + 0x418: 0x4000, 0x419: 0x4000, 0x41a: 0x4000, 0x41b: 0x4000, 0x41c: 0x4000, 0x41d: 0x4000, + 0x41e: 0x4000, 0x41f: 0x4000, 0x420: 0x4000, 0x421: 0x4000, 0x422: 0x4000, 0x423: 0x4000, + 0x424: 0x4000, 0x425: 0x4000, 0x426: 0x4000, 0x427: 0x4000, 0x428: 0x4000, 0x429: 0x4000, + 0x42a: 0x4000, 0x42b: 0x4000, 0x42c: 0x4000, 0x42d: 0x4000, 0x42e: 0x4000, 0x42f: 0x4000, + 0x430: 0x4000, 0x431: 0x4000, 0x432: 0x4000, 0x433: 0x4000, 0x434: 0x4000, 0x435: 0x4000, + 0x436: 0x4000, 0x437: 0x4000, 0x438: 0x4000, 0x439: 0x4000, 0x43a: 0x4000, 0x43b: 0x4000, + 0x43c: 0x4000, 0x43d: 0x4000, 0x43e: 0x4000, 0x43f: 0x4000, + // Block 0x11, offset 0x440 + 0x440: 0x4000, 0x441: 0x4000, 0x442: 0x4000, 0x443: 0x4000, 0x444: 0x4000, 0x445: 0x4000, + 0x446: 0x4000, 0x447: 0x4000, 0x448: 0x4000, 0x449: 0x4000, 0x44a: 0x4000, 0x44b: 0x4000, + 0x44c: 0x4000, 0x44d: 0x4000, 0x44e: 0x4000, 0x44f: 0x4000, 0x450: 0x4000, 0x451: 0x4000, + 0x452: 0x4000, 0x453: 0x4000, 0x454: 0x4000, 0x455: 0x4000, 0x456: 0x4000, 0x457: 0x4000, + 0x458: 0x4000, 0x459: 0x4000, 0x45a: 0x4000, 0x45b: 0x4000, 0x45c: 0x4000, 0x45d: 0x4000, + 0x45e: 0x4000, 0x45f: 0x4000, + // Block 0x12, offset 0x480 + 0x490: 0x2000, + 0x493: 0x2000, 0x494: 0x2000, 0x495: 0x2000, 0x496: 0x2000, + 0x498: 0x2000, 0x499: 0x2000, 0x49c: 0x2000, 0x49d: 0x2000, + 0x4a0: 0x2000, 0x4a1: 0x2000, 0x4a2: 0x2000, + 0x4a4: 0x2000, 0x4a5: 0x2000, 0x4a6: 0x2000, 0x4a7: 0x2000, + 0x4b0: 0x2000, 0x4b2: 0x2000, 0x4b3: 0x2000, 0x4b5: 0x2000, + 0x4bb: 0x2000, + 0x4be: 0x2000, + // Block 0x13, offset 0x4c0 + 0x4f4: 0x2000, + 0x4ff: 0x2000, + // Block 0x14, offset 0x500 + 0x501: 0x2000, 0x502: 0x2000, 0x503: 0x2000, 0x504: 0x2000, + 0x529: 0xa009, + 0x52c: 0x2000, + // Block 0x15, offset 0x540 + 0x543: 0x2000, 0x545: 0x2000, + 0x549: 0x2000, + 0x553: 0x2000, 0x556: 0x2000, + 0x561: 0x2000, 0x562: 0x2000, + 0x566: 0x2000, + 0x56b: 0x2000, + // Block 0x16, offset 0x580 + 0x593: 0x2000, 0x594: 0x2000, + 0x59b: 0x2000, 0x59c: 0x2000, 0x59d: 0x2000, + 0x59e: 0x2000, 0x5a0: 0x2000, 0x5a1: 0x2000, 0x5a2: 0x2000, 0x5a3: 0x2000, + 0x5a4: 0x2000, 0x5a5: 0x2000, 0x5a6: 0x2000, 0x5a7: 0x2000, 0x5a8: 0x2000, 0x5a9: 0x2000, + 0x5aa: 0x2000, 0x5ab: 0x2000, + 0x5b0: 0x2000, 0x5b1: 0x2000, 0x5b2: 0x2000, 0x5b3: 0x2000, 0x5b4: 0x2000, 0x5b5: 0x2000, + 0x5b6: 0x2000, 0x5b7: 0x2000, 0x5b8: 0x2000, 0x5b9: 0x2000, + // Block 0x17, offset 0x5c0 + 0x5c9: 0x2000, + 0x5d0: 0x200a, 0x5d1: 0x200b, + 0x5d2: 0x200a, 0x5d3: 0x200c, 0x5d4: 0x2000, 0x5d5: 0x2000, 0x5d6: 0x2000, 0x5d7: 0x2000, + 0x5d8: 0x2000, 0x5d9: 0x2000, + 0x5f8: 0x2000, 0x5f9: 0x2000, + // Block 0x18, offset 0x600 + 0x612: 0x2000, 0x614: 0x2000, + 0x627: 0x2000, + // Block 0x19, offset 0x640 + 0x640: 0x2000, 0x642: 0x2000, 0x643: 0x2000, + 0x647: 0x2000, 0x648: 0x2000, 0x64b: 0x2000, + 0x64f: 0x2000, 0x651: 0x2000, + 0x655: 0x2000, + 0x65a: 0x2000, 0x65d: 0x2000, + 0x65e: 0x2000, 0x65f: 0x2000, 0x660: 0x2000, 0x663: 0x2000, + 0x665: 0x2000, 0x667: 0x2000, 0x668: 0x2000, 0x669: 0x2000, + 0x66a: 0x2000, 0x66b: 0x2000, 0x66c: 0x2000, 0x66e: 0x2000, + 0x674: 0x2000, 0x675: 0x2000, + 0x676: 0x2000, 0x677: 0x2000, + 0x67c: 0x2000, 0x67d: 0x2000, + // Block 0x1a, offset 0x680 + 0x688: 0x2000, + 0x68c: 0x2000, + 0x692: 0x2000, + 0x6a0: 0x2000, 0x6a1: 0x2000, + 0x6a4: 0x2000, 0x6a5: 0x2000, 0x6a6: 0x2000, 0x6a7: 0x2000, + 0x6aa: 0x2000, 0x6ab: 0x2000, 0x6ae: 0x2000, 0x6af: 0x2000, + // Block 0x1b, offset 0x6c0 + 0x6c2: 0x2000, 0x6c3: 0x2000, + 0x6c6: 0x2000, 0x6c7: 0x2000, + 0x6d5: 0x2000, + 0x6d9: 0x2000, + 0x6e5: 0x2000, + 0x6ff: 0x2000, + // Block 0x1c, offset 0x700 + 0x712: 0x2000, + 0x71a: 0x4000, 0x71b: 0x4000, + 0x729: 0x4000, + 0x72a: 0x4000, + // Block 0x1d, offset 0x740 + 0x769: 0x4000, + 0x76a: 0x4000, 0x76b: 0x4000, 0x76c: 0x4000, + 0x770: 0x4000, 0x773: 0x4000, + // Block 0x1e, offset 0x780 + 0x7a0: 0x2000, 0x7a1: 0x2000, 0x7a2: 0x2000, 0x7a3: 0x2000, + 0x7a4: 0x2000, 0x7a5: 0x2000, 0x7a6: 0x2000, 0x7a7: 0x2000, 0x7a8: 0x2000, 0x7a9: 0x2000, + 0x7aa: 0x2000, 0x7ab: 0x2000, 0x7ac: 0x2000, 0x7ad: 0x2000, 0x7ae: 0x2000, 0x7af: 0x2000, + 0x7b0: 0x2000, 0x7b1: 0x2000, 0x7b2: 0x2000, 0x7b3: 0x2000, 0x7b4: 0x2000, 0x7b5: 0x2000, + 0x7b6: 0x2000, 0x7b7: 0x2000, 0x7b8: 0x2000, 0x7b9: 0x2000, 0x7ba: 0x2000, 0x7bb: 0x2000, + 0x7bc: 0x2000, 0x7bd: 0x2000, 0x7be: 0x2000, 0x7bf: 0x2000, + // Block 0x1f, offset 0x7c0 + 0x7c0: 0x2000, 0x7c1: 0x2000, 0x7c2: 0x2000, 0x7c3: 0x2000, 0x7c4: 0x2000, 0x7c5: 0x2000, + 0x7c6: 0x2000, 0x7c7: 0x2000, 0x7c8: 0x2000, 0x7c9: 0x2000, 0x7ca: 0x2000, 0x7cb: 0x2000, + 0x7cc: 0x2000, 0x7cd: 0x2000, 0x7ce: 0x2000, 0x7cf: 0x2000, 0x7d0: 0x2000, 0x7d1: 0x2000, + 0x7d2: 0x2000, 0x7d3: 0x2000, 0x7d4: 0x2000, 0x7d5: 0x2000, 0x7d6: 0x2000, 0x7d7: 0x2000, + 0x7d8: 0x2000, 0x7d9: 0x2000, 0x7da: 0x2000, 0x7db: 0x2000, 0x7dc: 0x2000, 0x7dd: 0x2000, + 0x7de: 0x2000, 0x7df: 0x2000, 0x7e0: 0x2000, 0x7e1: 0x2000, 0x7e2: 0x2000, 0x7e3: 0x2000, + 0x7e4: 0x2000, 0x7e5: 0x2000, 0x7e6: 0x2000, 0x7e7: 0x2000, 0x7e8: 0x2000, 0x7e9: 0x2000, + 0x7eb: 0x2000, 0x7ec: 0x2000, 0x7ed: 0x2000, 0x7ee: 0x2000, 0x7ef: 0x2000, + 0x7f0: 0x2000, 0x7f1: 0x2000, 0x7f2: 0x2000, 0x7f3: 0x2000, 0x7f4: 0x2000, 0x7f5: 0x2000, + 0x7f6: 0x2000, 0x7f7: 0x2000, 0x7f8: 0x2000, 0x7f9: 0x2000, 0x7fa: 0x2000, 0x7fb: 0x2000, + 0x7fc: 0x2000, 0x7fd: 0x2000, 0x7fe: 0x2000, 0x7ff: 0x2000, + // Block 0x20, offset 0x800 + 0x800: 0x2000, 0x801: 0x2000, 0x802: 0x200d, 0x803: 0x2000, 0x804: 0x2000, 0x805: 0x2000, + 0x806: 0x2000, 0x807: 0x2000, 0x808: 0x2000, 0x809: 0x2000, 0x80a: 0x2000, 0x80b: 0x2000, + 0x80c: 0x2000, 0x80d: 0x2000, 0x80e: 0x2000, 0x80f: 0x2000, 0x810: 0x2000, 0x811: 0x2000, + 0x812: 0x2000, 0x813: 0x2000, 0x814: 0x2000, 0x815: 0x2000, 0x816: 0x2000, 0x817: 0x2000, + 0x818: 0x2000, 0x819: 0x2000, 0x81a: 0x2000, 0x81b: 0x2000, 0x81c: 0x2000, 0x81d: 0x2000, + 0x81e: 0x2000, 0x81f: 0x2000, 0x820: 0x2000, 0x821: 0x2000, 0x822: 0x2000, 0x823: 0x2000, + 0x824: 0x2000, 0x825: 0x2000, 0x826: 0x2000, 0x827: 0x2000, 0x828: 0x2000, 0x829: 0x2000, + 0x82a: 0x2000, 0x82b: 0x2000, 0x82c: 0x2000, 0x82d: 0x2000, 0x82e: 0x2000, 0x82f: 0x2000, + 0x830: 0x2000, 0x831: 0x2000, 0x832: 0x2000, 0x833: 0x2000, 0x834: 0x2000, 0x835: 0x2000, + 0x836: 0x2000, 0x837: 0x2000, 0x838: 0x2000, 0x839: 0x2000, 0x83a: 0x2000, 0x83b: 0x2000, + 0x83c: 0x2000, 0x83d: 0x2000, 0x83e: 0x2000, 0x83f: 0x2000, + // Block 0x21, offset 0x840 + 0x840: 0x2000, 0x841: 0x2000, 0x842: 0x2000, 0x843: 0x2000, 0x844: 0x2000, 0x845: 0x2000, + 0x846: 0x2000, 0x847: 0x2000, 0x848: 0x2000, 0x849: 0x2000, 0x84a: 0x2000, 0x84b: 0x2000, + 0x850: 0x2000, 0x851: 0x2000, + 0x852: 0x2000, 0x853: 0x2000, 0x854: 0x2000, 0x855: 0x2000, 0x856: 0x2000, 0x857: 0x2000, + 0x858: 0x2000, 0x859: 0x2000, 0x85a: 0x2000, 0x85b: 0x2000, 0x85c: 0x2000, 0x85d: 0x2000, + 0x85e: 0x2000, 0x85f: 0x2000, 0x860: 0x2000, 0x861: 0x2000, 0x862: 0x2000, 0x863: 0x2000, + 0x864: 0x2000, 0x865: 0x2000, 0x866: 0x2000, 0x867: 0x2000, 0x868: 0x2000, 0x869: 0x2000, + 0x86a: 0x2000, 0x86b: 0x2000, 0x86c: 0x2000, 0x86d: 0x2000, 0x86e: 0x2000, 0x86f: 0x2000, + 0x870: 0x2000, 0x871: 0x2000, 0x872: 0x2000, 0x873: 0x2000, + // Block 0x22, offset 0x880 + 0x880: 0x2000, 0x881: 0x2000, 0x882: 0x2000, 0x883: 0x2000, 0x884: 0x2000, 0x885: 0x2000, + 0x886: 0x2000, 0x887: 0x2000, 0x888: 0x2000, 0x889: 0x2000, 0x88a: 0x2000, 0x88b: 0x2000, + 0x88c: 0x2000, 0x88d: 0x2000, 0x88e: 0x2000, 0x88f: 0x2000, + 0x892: 0x2000, 0x893: 0x2000, 0x894: 0x2000, 0x895: 0x2000, + 0x8a0: 0x200e, 0x8a1: 0x2000, 0x8a3: 0x2000, + 0x8a4: 0x2000, 0x8a5: 0x2000, 0x8a6: 0x2000, 0x8a7: 0x2000, 0x8a8: 0x2000, 0x8a9: 0x2000, + 0x8b2: 0x2000, 0x8b3: 0x2000, + 0x8b6: 0x2000, 0x8b7: 0x2000, + 0x8bc: 0x2000, 0x8bd: 0x2000, + // Block 0x23, offset 0x8c0 + 0x8c0: 0x2000, 0x8c1: 0x2000, + 0x8c6: 0x2000, 0x8c7: 0x2000, 0x8c8: 0x2000, 0x8cb: 0x200f, + 0x8ce: 0x2000, 0x8cf: 0x2000, 0x8d0: 0x2000, 0x8d1: 0x2000, + 0x8e2: 0x2000, 0x8e3: 0x2000, + 0x8e4: 0x2000, 0x8e5: 0x2000, + 0x8ef: 0x2000, + 0x8fd: 0x4000, 0x8fe: 0x4000, + // Block 0x24, offset 0x900 + 0x905: 0x2000, + 0x906: 0x2000, 0x909: 0x2000, + 0x90e: 0x2000, 0x90f: 0x2000, + 0x914: 0x4000, 0x915: 0x4000, + 0x91c: 0x2000, + 0x91e: 0x2000, + // Block 0x25, offset 0x940 + 0x940: 0x2000, 0x942: 0x2000, + 0x948: 0x4000, 0x949: 0x4000, 0x94a: 0x4000, 0x94b: 0x4000, + 0x94c: 0x4000, 0x94d: 0x4000, 0x94e: 0x4000, 0x94f: 0x4000, 0x950: 0x4000, 0x951: 0x4000, + 0x952: 0x4000, 0x953: 0x4000, + 0x960: 0x2000, 0x961: 0x2000, 0x963: 0x2000, + 0x964: 0x2000, 0x965: 0x2000, 0x967: 0x2000, 0x968: 0x2000, 0x969: 0x2000, + 0x96a: 0x2000, 0x96c: 0x2000, 0x96d: 0x2000, 0x96f: 0x2000, + 0x97f: 0x4000, + // Block 0x26, offset 0x980 + 0x993: 0x4000, + 0x99e: 0x2000, 0x99f: 0x2000, 0x9a1: 0x4000, + 0x9aa: 0x4000, 0x9ab: 0x4000, + 0x9bd: 0x4000, 0x9be: 0x4000, 0x9bf: 0x2000, + // Block 0x27, offset 0x9c0 + 0x9c4: 0x4000, 0x9c5: 0x4000, + 0x9c6: 0x2000, 0x9c7: 0x2000, 0x9c8: 0x2000, 0x9c9: 0x2000, 0x9ca: 0x2000, 0x9cb: 0x2000, + 0x9cc: 0x2000, 0x9cd: 0x2000, 0x9ce: 0x4000, 0x9cf: 0x2000, 0x9d0: 0x2000, 0x9d1: 0x2000, + 0x9d2: 0x2000, 0x9d3: 0x2000, 0x9d4: 0x4000, 0x9d5: 0x2000, 0x9d6: 0x2000, 0x9d7: 0x2000, + 0x9d8: 0x2000, 0x9d9: 0x2000, 0x9da: 0x2000, 0x9db: 0x2000, 0x9dc: 0x2000, 0x9dd: 0x2000, + 0x9de: 0x2000, 0x9df: 0x2000, 0x9e0: 0x2000, 0x9e1: 0x2000, 0x9e3: 0x2000, + 0x9e8: 0x2000, 0x9e9: 0x2000, + 0x9ea: 0x4000, 0x9eb: 0x2000, 0x9ec: 0x2000, 0x9ed: 0x2000, 0x9ee: 0x2000, 0x9ef: 0x2000, + 0x9f0: 0x2000, 0x9f1: 0x2000, 0x9f2: 0x4000, 0x9f3: 0x4000, 0x9f4: 0x2000, 0x9f5: 0x4000, + 0x9f6: 0x2000, 0x9f7: 0x2000, 0x9f8: 0x2000, 0x9f9: 0x2000, 0x9fa: 0x4000, 0x9fb: 0x2000, + 0x9fc: 0x2000, 0x9fd: 0x4000, 0x9fe: 0x2000, 0x9ff: 0x2000, + // Block 0x28, offset 0xa00 + 0xa05: 0x4000, + 0xa0a: 0x4000, 0xa0b: 0x4000, + 0xa28: 0x4000, + 0xa3d: 0x2000, + // Block 0x29, offset 0xa40 + 0xa4c: 0x4000, 0xa4e: 0x4000, + 0xa53: 0x4000, 0xa54: 0x4000, 0xa55: 0x4000, 0xa57: 0x4000, + 0xa76: 0x2000, 0xa77: 0x2000, 0xa78: 0x2000, 0xa79: 0x2000, 0xa7a: 0x2000, 0xa7b: 0x2000, + 0xa7c: 0x2000, 0xa7d: 0x2000, 0xa7e: 0x2000, 0xa7f: 0x2000, + // Block 0x2a, offset 0xa80 + 0xa95: 0x4000, 0xa96: 0x4000, 0xa97: 0x4000, + 0xab0: 0x4000, + 0xabf: 0x4000, + // Block 0x2b, offset 0xac0 + 0xae6: 0x6000, 0xae7: 0x6000, 0xae8: 0x6000, 0xae9: 0x6000, + 0xaea: 0x6000, 0xaeb: 0x6000, 0xaec: 0x6000, 0xaed: 0x6000, + // Block 0x2c, offset 0xb00 + 0xb05: 0x6010, + 0xb06: 0x6011, + // Block 0x2d, offset 0xb40 + 0xb5b: 0x4000, 0xb5c: 0x4000, + // Block 0x2e, offset 0xb80 + 0xb90: 0x4000, + 0xb95: 0x4000, 0xb96: 0x2000, 0xb97: 0x2000, + 0xb98: 0x2000, 0xb99: 0x2000, + // Block 0x2f, offset 0xbc0 + 0xbc0: 0x4000, 0xbc1: 0x4000, 0xbc2: 0x4000, 0xbc3: 0x4000, 0xbc4: 0x4000, 0xbc5: 0x4000, + 0xbc6: 0x4000, 0xbc7: 0x4000, 0xbc8: 0x4000, 0xbc9: 0x4000, 0xbca: 0x4000, 0xbcb: 0x4000, + 0xbcc: 0x4000, 0xbcd: 0x4000, 0xbce: 0x4000, 0xbcf: 0x4000, 0xbd0: 0x4000, 0xbd1: 0x4000, + 0xbd2: 0x4000, 0xbd3: 0x4000, 0xbd4: 0x4000, 0xbd5: 0x4000, 0xbd6: 0x4000, 0xbd7: 0x4000, + 0xbd8: 0x4000, 0xbd9: 0x4000, 0xbdb: 0x4000, 0xbdc: 0x4000, 0xbdd: 0x4000, + 0xbde: 0x4000, 0xbdf: 0x4000, 0xbe0: 0x4000, 0xbe1: 0x4000, 0xbe2: 0x4000, 0xbe3: 0x4000, + 0xbe4: 0x4000, 0xbe5: 0x4000, 0xbe6: 0x4000, 0xbe7: 0x4000, 0xbe8: 0x4000, 0xbe9: 0x4000, + 0xbea: 0x4000, 0xbeb: 0x4000, 0xbec: 0x4000, 0xbed: 0x4000, 0xbee: 0x4000, 0xbef: 0x4000, + 0xbf0: 0x4000, 0xbf1: 0x4000, 0xbf2: 0x4000, 0xbf3: 0x4000, 0xbf4: 0x4000, 0xbf5: 0x4000, + 0xbf6: 0x4000, 0xbf7: 0x4000, 0xbf8: 0x4000, 0xbf9: 0x4000, 0xbfa: 0x4000, 0xbfb: 0x4000, + 0xbfc: 0x4000, 0xbfd: 0x4000, 0xbfe: 0x4000, 0xbff: 0x4000, + // Block 0x30, offset 0xc00 + 0xc00: 0x4000, 0xc01: 0x4000, 0xc02: 0x4000, 0xc03: 0x4000, 0xc04: 0x4000, 0xc05: 0x4000, + 0xc06: 0x4000, 0xc07: 0x4000, 0xc08: 0x4000, 0xc09: 0x4000, 0xc0a: 0x4000, 0xc0b: 0x4000, + 0xc0c: 0x4000, 0xc0d: 0x4000, 0xc0e: 0x4000, 0xc0f: 0x4000, 0xc10: 0x4000, 0xc11: 0x4000, + 0xc12: 0x4000, 0xc13: 0x4000, 0xc14: 0x4000, 0xc15: 0x4000, 0xc16: 0x4000, 0xc17: 0x4000, + 0xc18: 0x4000, 0xc19: 0x4000, 0xc1a: 0x4000, 0xc1b: 0x4000, 0xc1c: 0x4000, 0xc1d: 0x4000, + 0xc1e: 0x4000, 0xc1f: 0x4000, 0xc20: 0x4000, 0xc21: 0x4000, 0xc22: 0x4000, 0xc23: 0x4000, + 0xc24: 0x4000, 0xc25: 0x4000, 0xc26: 0x4000, 0xc27: 0x4000, 0xc28: 0x4000, 0xc29: 0x4000, + 0xc2a: 0x4000, 0xc2b: 0x4000, 0xc2c: 0x4000, 0xc2d: 0x4000, 0xc2e: 0x4000, 0xc2f: 0x4000, + 0xc30: 0x4000, 0xc31: 0x4000, 0xc32: 0x4000, 0xc33: 0x4000, + // Block 0x31, offset 0xc40 + 0xc40: 0x4000, 0xc41: 0x4000, 0xc42: 0x4000, 0xc43: 0x4000, 0xc44: 0x4000, 0xc45: 0x4000, + 0xc46: 0x4000, 0xc47: 0x4000, 0xc48: 0x4000, 0xc49: 0x4000, 0xc4a: 0x4000, 0xc4b: 0x4000, + 0xc4c: 0x4000, 0xc4d: 0x4000, 0xc4e: 0x4000, 0xc4f: 0x4000, 0xc50: 0x4000, 0xc51: 0x4000, + 0xc52: 0x4000, 0xc53: 0x4000, 0xc54: 0x4000, 0xc55: 0x4000, + 0xc70: 0x4000, 0xc71: 0x4000, 0xc72: 0x4000, 0xc73: 0x4000, 0xc74: 0x4000, 0xc75: 0x4000, + 0xc76: 0x4000, 0xc77: 0x4000, 0xc78: 0x4000, 0xc79: 0x4000, 0xc7a: 0x4000, 0xc7b: 0x4000, + // Block 0x32, offset 0xc80 + 0xc80: 0x9012, 0xc81: 0x4013, 0xc82: 0x4014, 0xc83: 0x4000, 0xc84: 0x4000, 0xc85: 0x4000, + 0xc86: 0x4000, 0xc87: 0x4000, 0xc88: 0x4000, 0xc89: 0x4000, 0xc8a: 0x4000, 0xc8b: 0x4000, + 0xc8c: 0x4015, 0xc8d: 0x4015, 0xc8e: 0x4000, 0xc8f: 0x4000, 0xc90: 0x4000, 0xc91: 0x4000, + 0xc92: 0x4000, 0xc93: 0x4000, 0xc94: 0x4000, 0xc95: 0x4000, 0xc96: 0x4000, 0xc97: 0x4000, + 0xc98: 0x4000, 0xc99: 0x4000, 0xc9a: 0x4000, 0xc9b: 0x4000, 0xc9c: 0x4000, 0xc9d: 0x4000, + 0xc9e: 0x4000, 0xc9f: 0x4000, 0xca0: 0x4000, 0xca1: 0x4000, 0xca2: 0x4000, 0xca3: 0x4000, + 0xca4: 0x4000, 0xca5: 0x4000, 0xca6: 0x4000, 0xca7: 0x4000, 0xca8: 0x4000, 0xca9: 0x4000, + 0xcaa: 0x4000, 0xcab: 0x4000, 0xcac: 0x4000, 0xcad: 0x4000, 0xcae: 0x4000, 0xcaf: 0x4000, + 0xcb0: 0x4000, 0xcb1: 0x4000, 0xcb2: 0x4000, 0xcb3: 0x4000, 0xcb4: 0x4000, 0xcb5: 0x4000, + 0xcb6: 0x4000, 0xcb7: 0x4000, 0xcb8: 0x4000, 0xcb9: 0x4000, 0xcba: 0x4000, 0xcbb: 0x4000, + 0xcbc: 0x4000, 0xcbd: 0x4000, 0xcbe: 0x4000, + // Block 0x33, offset 0xcc0 + 0xcc1: 0x4000, 0xcc2: 0x4000, 0xcc3: 0x4000, 0xcc4: 0x4000, 0xcc5: 0x4000, + 0xcc6: 0x4000, 0xcc7: 0x4000, 0xcc8: 0x4000, 0xcc9: 0x4000, 0xcca: 0x4000, 0xccb: 0x4000, + 0xccc: 0x4000, 0xccd: 0x4000, 0xcce: 0x4000, 0xccf: 0x4000, 0xcd0: 0x4000, 0xcd1: 0x4000, + 0xcd2: 0x4000, 0xcd3: 0x4000, 0xcd4: 0x4000, 0xcd5: 0x4000, 0xcd6: 0x4000, 0xcd7: 0x4000, + 0xcd8: 0x4000, 0xcd9: 0x4000, 0xcda: 0x4000, 0xcdb: 0x4000, 0xcdc: 0x4000, 0xcdd: 0x4000, + 0xcde: 0x4000, 0xcdf: 0x4000, 0xce0: 0x4000, 0xce1: 0x4000, 0xce2: 0x4000, 0xce3: 0x4000, + 0xce4: 0x4000, 0xce5: 0x4000, 0xce6: 0x4000, 0xce7: 0x4000, 0xce8: 0x4000, 0xce9: 0x4000, + 0xcea: 0x4000, 0xceb: 0x4000, 0xcec: 0x4000, 0xced: 0x4000, 0xcee: 0x4000, 0xcef: 0x4000, + 0xcf0: 0x4000, 0xcf1: 0x4000, 0xcf2: 0x4000, 0xcf3: 0x4000, 0xcf4: 0x4000, 0xcf5: 0x4000, + 0xcf6: 0x4000, 0xcf7: 0x4000, 0xcf8: 0x4000, 0xcf9: 0x4000, 0xcfa: 0x4000, 0xcfb: 0x4000, + 0xcfc: 0x4000, 0xcfd: 0x4000, 0xcfe: 0x4000, 0xcff: 0x4000, + // Block 0x34, offset 0xd00 + 0xd00: 0x4000, 0xd01: 0x4000, 0xd02: 0x4000, 0xd03: 0x4000, 0xd04: 0x4000, 0xd05: 0x4000, + 0xd06: 0x4000, 0xd07: 0x4000, 0xd08: 0x4000, 0xd09: 0x4000, 0xd0a: 0x4000, 0xd0b: 0x4000, + 0xd0c: 0x4000, 0xd0d: 0x4000, 0xd0e: 0x4000, 0xd0f: 0x4000, 0xd10: 0x4000, 0xd11: 0x4000, + 0xd12: 0x4000, 0xd13: 0x4000, 0xd14: 0x4000, 0xd15: 0x4000, 0xd16: 0x4000, + 0xd19: 0x4016, 0xd1a: 0x4017, 0xd1b: 0x4000, 0xd1c: 0x4000, 0xd1d: 0x4000, + 0xd1e: 0x4000, 0xd1f: 0x4000, 0xd20: 0x4000, 0xd21: 0x4018, 0xd22: 0x4019, 0xd23: 0x401a, + 0xd24: 0x401b, 0xd25: 0x401c, 0xd26: 0x401d, 0xd27: 0x401e, 0xd28: 0x401f, 0xd29: 0x4020, + 0xd2a: 0x4021, 0xd2b: 0x4022, 0xd2c: 0x4000, 0xd2d: 0x4010, 0xd2e: 0x4000, 0xd2f: 0x4023, + 0xd30: 0x4000, 0xd31: 0x4024, 0xd32: 0x4000, 0xd33: 0x4025, 0xd34: 0x4000, 0xd35: 0x4026, + 0xd36: 0x4000, 0xd37: 0x401a, 0xd38: 0x4000, 0xd39: 0x4027, 0xd3a: 0x4000, 0xd3b: 0x4028, + 0xd3c: 0x4000, 0xd3d: 0x4020, 0xd3e: 0x4000, 0xd3f: 0x4029, + // Block 0x35, offset 0xd40 + 0xd40: 0x4000, 0xd41: 0x402a, 0xd42: 0x4000, 0xd43: 0x402b, 0xd44: 0x402c, 0xd45: 0x4000, + 0xd46: 0x4017, 0xd47: 0x4000, 0xd48: 0x402d, 0xd49: 0x4000, 0xd4a: 0x402e, 0xd4b: 0x402f, + 0xd4c: 0x4030, 0xd4d: 0x4017, 0xd4e: 0x4016, 0xd4f: 0x4017, 0xd50: 0x4000, 0xd51: 0x4000, + 0xd52: 0x4031, 0xd53: 0x4000, 0xd54: 0x4000, 0xd55: 0x4031, 0xd56: 0x4000, 0xd57: 0x4000, + 0xd58: 0x4032, 0xd59: 0x4000, 0xd5a: 0x4000, 0xd5b: 0x4032, 0xd5c: 0x4000, 0xd5d: 0x4000, + 0xd5e: 0x4033, 0xd5f: 0x402e, 0xd60: 0x4034, 0xd61: 0x4035, 0xd62: 0x4034, 0xd63: 0x4036, + 0xd64: 0x4037, 0xd65: 0x4024, 0xd66: 0x4035, 0xd67: 0x4025, 0xd68: 0x4038, 0xd69: 0x4038, + 0xd6a: 0x4039, 0xd6b: 0x4039, 0xd6c: 0x403a, 0xd6d: 0x403a, 0xd6e: 0x4000, 0xd6f: 0x4035, + 0xd70: 0x4000, 0xd71: 0x4000, 0xd72: 0x403b, 0xd73: 0x403c, 0xd74: 0x4000, 0xd75: 0x4000, + 0xd76: 0x4000, 0xd77: 0x4000, 0xd78: 0x4000, 0xd79: 0x4000, 0xd7a: 0x4000, 0xd7b: 0x403d, + 0xd7c: 0x401c, 0xd7d: 0x4000, 0xd7e: 0x4000, 0xd7f: 0x4000, + // Block 0x36, offset 0xd80 + 0xd85: 0x4000, + 0xd86: 0x4000, 0xd87: 0x4000, 0xd88: 0x4000, 0xd89: 0x4000, 0xd8a: 0x4000, 0xd8b: 0x4000, + 0xd8c: 0x4000, 0xd8d: 0x4000, 0xd8e: 0x4000, 0xd8f: 0x4000, 0xd90: 0x4000, 0xd91: 0x4000, + 0xd92: 0x4000, 0xd93: 0x4000, 0xd94: 0x4000, 0xd95: 0x4000, 0xd96: 0x4000, 0xd97: 0x4000, + 0xd98: 0x4000, 0xd99: 0x4000, 0xd9a: 0x4000, 0xd9b: 0x4000, 0xd9c: 0x4000, 0xd9d: 0x4000, + 0xd9e: 0x4000, 0xd9f: 0x4000, 0xda0: 0x4000, 0xda1: 0x4000, 0xda2: 0x4000, 0xda3: 0x4000, + 0xda4: 0x4000, 0xda5: 0x4000, 0xda6: 0x4000, 0xda7: 0x4000, 0xda8: 0x4000, 0xda9: 0x4000, + 0xdaa: 0x4000, 0xdab: 0x4000, 0xdac: 0x4000, 0xdad: 0x4000, 0xdae: 0x4000, + 0xdb1: 0x403e, 0xdb2: 0x403e, 0xdb3: 0x403e, 0xdb4: 0x403e, 0xdb5: 0x403e, + 0xdb6: 0x403e, 0xdb7: 0x403e, 0xdb8: 0x403e, 0xdb9: 0x403e, 0xdba: 0x403e, 0xdbb: 0x403e, + 0xdbc: 0x403e, 0xdbd: 0x403e, 0xdbe: 0x403e, 0xdbf: 0x403e, + // Block 0x37, offset 0xdc0 + 0xdc0: 0x4037, 0xdc1: 0x4037, 0xdc2: 0x4037, 0xdc3: 0x4037, 0xdc4: 0x4037, 0xdc5: 0x4037, + 0xdc6: 0x4037, 0xdc7: 0x4037, 0xdc8: 0x4037, 0xdc9: 0x4037, 0xdca: 0x4037, 0xdcb: 0x4037, + 0xdcc: 0x4037, 0xdcd: 0x4037, 0xdce: 0x4037, 0xdcf: 0x400e, 0xdd0: 0x403f, 0xdd1: 0x4040, + 0xdd2: 0x4041, 0xdd3: 0x4040, 0xdd4: 0x403f, 0xdd5: 0x4042, 0xdd6: 0x4043, 0xdd7: 0x4044, + 0xdd8: 0x4040, 0xdd9: 0x4041, 0xdda: 0x4040, 0xddb: 0x4045, 0xddc: 0x4009, 0xddd: 0x4045, + 0xdde: 0x4046, 0xddf: 0x4045, 0xde0: 0x4047, 0xde1: 0x400b, 0xde2: 0x400a, 0xde3: 0x400c, + 0xde4: 0x4048, 0xde5: 0x4000, 0xde6: 0x4000, 0xde7: 0x4000, 0xde8: 0x4000, 0xde9: 0x4000, + 0xdea: 0x4000, 0xdeb: 0x4000, 0xdec: 0x4000, 0xded: 0x4000, 0xdee: 0x4000, 0xdef: 0x4000, + 0xdf0: 0x4000, 0xdf1: 0x4000, 0xdf2: 0x4000, 0xdf3: 0x4000, 0xdf4: 0x4000, 0xdf5: 0x4000, + 0xdf6: 0x4000, 0xdf7: 0x4000, 0xdf8: 0x4000, 0xdf9: 0x4000, 0xdfa: 0x4000, 0xdfb: 0x4000, + 0xdfc: 0x4000, 0xdfd: 0x4000, 0xdfe: 0x4000, 0xdff: 0x4000, + // Block 0x38, offset 0xe00 + 0xe00: 0x4000, 0xe01: 0x4000, 0xe02: 0x4000, 0xe03: 0x4000, 0xe04: 0x4000, 0xe05: 0x4000, + 0xe06: 0x4000, 0xe07: 0x4000, 0xe08: 0x4000, 0xe09: 0x4000, 0xe0a: 0x4000, 0xe0b: 0x4000, + 0xe0c: 0x4000, 0xe0d: 0x4000, 0xe0e: 0x4000, 0xe10: 0x4000, 0xe11: 0x4000, + 0xe12: 0x4000, 0xe13: 0x4000, 0xe14: 0x4000, 0xe15: 0x4000, 0xe16: 0x4000, 0xe17: 0x4000, + 0xe18: 0x4000, 0xe19: 0x4000, 0xe1a: 0x4000, 0xe1b: 0x4000, 0xe1c: 0x4000, 0xe1d: 0x4000, + 0xe1e: 0x4000, 0xe1f: 0x4000, 0xe20: 0x4000, 0xe21: 0x4000, 0xe22: 0x4000, 0xe23: 0x4000, + 0xe24: 0x4000, 0xe25: 0x4000, 0xe26: 0x4000, 0xe27: 0x4000, 0xe28: 0x4000, 0xe29: 0x4000, + 0xe2a: 0x4000, 0xe2b: 0x4000, 0xe2c: 0x4000, 0xe2d: 0x4000, 0xe2e: 0x4000, 0xe2f: 0x4000, + 0xe30: 0x4000, 0xe31: 0x4000, 0xe32: 0x4000, 0xe33: 0x4000, 0xe34: 0x4000, 0xe35: 0x4000, + 0xe36: 0x4000, 0xe37: 0x4000, 0xe38: 0x4000, 0xe39: 0x4000, 0xe3a: 0x4000, + // Block 0x39, offset 0xe40 + 0xe40: 0x4000, 0xe41: 0x4000, 0xe42: 0x4000, 0xe43: 0x4000, 0xe44: 0x4000, 0xe45: 0x4000, + 0xe46: 0x4000, 0xe47: 0x4000, 0xe48: 0x4000, 0xe49: 0x4000, 0xe4a: 0x4000, 0xe4b: 0x4000, + 0xe4c: 0x4000, 0xe4d: 0x4000, 0xe4e: 0x4000, 0xe4f: 0x4000, 0xe50: 0x4000, 0xe51: 0x4000, + 0xe52: 0x4000, 0xe53: 0x4000, 0xe54: 0x4000, 0xe55: 0x4000, 0xe56: 0x4000, 0xe57: 0x4000, + 0xe58: 0x4000, 0xe59: 0x4000, 0xe5a: 0x4000, 0xe5b: 0x4000, 0xe5c: 0x4000, 0xe5d: 0x4000, + 0xe5e: 0x4000, 0xe5f: 0x4000, 0xe60: 0x4000, 0xe61: 0x4000, 0xe62: 0x4000, 0xe63: 0x4000, + 0xe70: 0x4000, 0xe71: 0x4000, 0xe72: 0x4000, 0xe73: 0x4000, 0xe74: 0x4000, 0xe75: 0x4000, + 0xe76: 0x4000, 0xe77: 0x4000, 0xe78: 0x4000, 0xe79: 0x4000, 0xe7a: 0x4000, 0xe7b: 0x4000, + 0xe7c: 0x4000, 0xe7d: 0x4000, 0xe7e: 0x4000, 0xe7f: 0x4000, + // Block 0x3a, offset 0xe80 + 0xe80: 0x4000, 0xe81: 0x4000, 0xe82: 0x4000, 0xe83: 0x4000, 0xe84: 0x4000, 0xe85: 0x4000, + 0xe86: 0x4000, 0xe87: 0x4000, 0xe88: 0x4000, 0xe89: 0x4000, 0xe8a: 0x4000, 0xe8b: 0x4000, + 0xe8c: 0x4000, 0xe8d: 0x4000, 0xe8e: 0x4000, 0xe8f: 0x4000, 0xe90: 0x4000, 0xe91: 0x4000, + 0xe92: 0x4000, 0xe93: 0x4000, 0xe94: 0x4000, 0xe95: 0x4000, 0xe96: 0x4000, 0xe97: 0x4000, + 0xe98: 0x4000, 0xe99: 0x4000, 0xe9a: 0x4000, 0xe9b: 0x4000, 0xe9c: 0x4000, 0xe9d: 0x4000, + 0xe9e: 0x4000, 0xea0: 0x4000, 0xea1: 0x4000, 0xea2: 0x4000, 0xea3: 0x4000, + 0xea4: 0x4000, 0xea5: 0x4000, 0xea6: 0x4000, 0xea7: 0x4000, 0xea8: 0x4000, 0xea9: 0x4000, + 0xeaa: 0x4000, 0xeab: 0x4000, 0xeac: 0x4000, 0xead: 0x4000, 0xeae: 0x4000, 0xeaf: 0x4000, + 0xeb0: 0x4000, 0xeb1: 0x4000, 0xeb2: 0x4000, 0xeb3: 0x4000, 0xeb4: 0x4000, 0xeb5: 0x4000, + 0xeb6: 0x4000, 0xeb7: 0x4000, 0xeb8: 0x4000, 0xeb9: 0x4000, 0xeba: 0x4000, 0xebb: 0x4000, + 0xebc: 0x4000, 0xebd: 0x4000, 0xebe: 0x4000, 0xebf: 0x4000, + // Block 0x3b, offset 0xec0 + 0xec0: 0x4000, 0xec1: 0x4000, 0xec2: 0x4000, 0xec3: 0x4000, 0xec4: 0x4000, 0xec5: 0x4000, + 0xec6: 0x4000, 0xec7: 0x4000, 0xec8: 0x2000, 0xec9: 0x2000, 0xeca: 0x2000, 0xecb: 0x2000, + 0xecc: 0x2000, 0xecd: 0x2000, 0xece: 0x2000, 0xecf: 0x2000, 0xed0: 0x4000, 0xed1: 0x4000, + 0xed2: 0x4000, 0xed3: 0x4000, 0xed4: 0x4000, 0xed5: 0x4000, 0xed6: 0x4000, 0xed7: 0x4000, + 0xed8: 0x4000, 0xed9: 0x4000, 0xeda: 0x4000, 0xedb: 0x4000, 0xedc: 0x4000, 0xedd: 0x4000, + 0xede: 0x4000, 0xedf: 0x4000, 0xee0: 0x4000, 0xee1: 0x4000, 0xee2: 0x4000, 0xee3: 0x4000, + 0xee4: 0x4000, 0xee5: 0x4000, 0xee6: 0x4000, 0xee7: 0x4000, 0xee8: 0x4000, 0xee9: 0x4000, + 0xeea: 0x4000, 0xeeb: 0x4000, 0xeec: 0x4000, 0xeed: 0x4000, 0xeee: 0x4000, 0xeef: 0x4000, + 0xef0: 0x4000, 0xef1: 0x4000, 0xef2: 0x4000, 0xef3: 0x4000, 0xef4: 0x4000, 0xef5: 0x4000, + 0xef6: 0x4000, 0xef7: 0x4000, 0xef8: 0x4000, 0xef9: 0x4000, 0xefa: 0x4000, 0xefb: 0x4000, + 0xefc: 0x4000, 0xefd: 0x4000, 0xefe: 0x4000, 0xeff: 0x4000, + // Block 0x3c, offset 0xf00 + 0xf00: 0x4000, 0xf01: 0x4000, 0xf02: 0x4000, 0xf03: 0x4000, 0xf04: 0x4000, 0xf05: 0x4000, + 0xf06: 0x4000, 0xf07: 0x4000, 0xf08: 0x4000, 0xf09: 0x4000, 0xf0a: 0x4000, 0xf0b: 0x4000, + 0xf0c: 0x4000, 0xf0d: 0x4000, 0xf0e: 0x4000, 0xf0f: 0x4000, 0xf10: 0x4000, 0xf11: 0x4000, + 0xf12: 0x4000, 0xf13: 0x4000, 0xf14: 0x4000, 0xf15: 0x4000, 0xf16: 0x4000, 0xf17: 0x4000, + 0xf18: 0x4000, 0xf19: 0x4000, 0xf1a: 0x4000, 0xf1b: 0x4000, 0xf1c: 0x4000, 0xf1d: 0x4000, + 0xf1e: 0x4000, 0xf1f: 0x4000, 0xf20: 0x4000, 0xf21: 0x4000, 0xf22: 0x4000, 0xf23: 0x4000, + 0xf24: 0x4000, 0xf25: 0x4000, 0xf26: 0x4000, 0xf27: 0x4000, 0xf28: 0x4000, 0xf29: 0x4000, + 0xf2a: 0x4000, 0xf2b: 0x4000, 0xf2c: 0x4000, 0xf2d: 0x4000, 0xf2e: 0x4000, 0xf2f: 0x4000, + 0xf30: 0x4000, 0xf31: 0x4000, 0xf32: 0x4000, 0xf33: 0x4000, 0xf34: 0x4000, 0xf35: 0x4000, + 0xf36: 0x4000, 0xf37: 0x4000, 0xf38: 0x4000, 0xf39: 0x4000, 0xf3a: 0x4000, 0xf3b: 0x4000, + 0xf3c: 0x4000, 0xf3d: 0x4000, 0xf3e: 0x4000, + // Block 0x3d, offset 0xf40 + 0xf40: 0x4000, 0xf41: 0x4000, 0xf42: 0x4000, 0xf43: 0x4000, 0xf44: 0x4000, 0xf45: 0x4000, + 0xf46: 0x4000, 0xf47: 0x4000, 0xf48: 0x4000, 0xf49: 0x4000, 0xf4a: 0x4000, 0xf4b: 0x4000, + 0xf4c: 0x4000, 0xf50: 0x4000, 0xf51: 0x4000, + 0xf52: 0x4000, 0xf53: 0x4000, 0xf54: 0x4000, 0xf55: 0x4000, 0xf56: 0x4000, 0xf57: 0x4000, + 0xf58: 0x4000, 0xf59: 0x4000, 0xf5a: 0x4000, 0xf5b: 0x4000, 0xf5c: 0x4000, 0xf5d: 0x4000, + 0xf5e: 0x4000, 0xf5f: 0x4000, 0xf60: 0x4000, 0xf61: 0x4000, 0xf62: 0x4000, 0xf63: 0x4000, + 0xf64: 0x4000, 0xf65: 0x4000, 0xf66: 0x4000, 0xf67: 0x4000, 0xf68: 0x4000, 0xf69: 0x4000, + 0xf6a: 0x4000, 0xf6b: 0x4000, 0xf6c: 0x4000, 0xf6d: 0x4000, 0xf6e: 0x4000, 0xf6f: 0x4000, + 0xf70: 0x4000, 0xf71: 0x4000, 0xf72: 0x4000, 0xf73: 0x4000, 0xf74: 0x4000, 0xf75: 0x4000, + 0xf76: 0x4000, 0xf77: 0x4000, 0xf78: 0x4000, 0xf79: 0x4000, 0xf7a: 0x4000, 0xf7b: 0x4000, + 0xf7c: 0x4000, 0xf7d: 0x4000, 0xf7e: 0x4000, 0xf7f: 0x4000, + // Block 0x3e, offset 0xf80 + 0xf80: 0x4000, 0xf81: 0x4000, 0xf82: 0x4000, 0xf83: 0x4000, 0xf84: 0x4000, 0xf85: 0x4000, + 0xf86: 0x4000, + // Block 0x3f, offset 0xfc0 + 0xfe0: 0x4000, 0xfe1: 0x4000, 0xfe2: 0x4000, 0xfe3: 0x4000, + 0xfe4: 0x4000, 0xfe5: 0x4000, 0xfe6: 0x4000, 0xfe7: 0x4000, 0xfe8: 0x4000, 0xfe9: 0x4000, + 0xfea: 0x4000, 0xfeb: 0x4000, 0xfec: 0x4000, 0xfed: 0x4000, 0xfee: 0x4000, 0xfef: 0x4000, + 0xff0: 0x4000, 0xff1: 0x4000, 0xff2: 0x4000, 0xff3: 0x4000, 0xff4: 0x4000, 0xff5: 0x4000, + 0xff6: 0x4000, 0xff7: 0x4000, 0xff8: 0x4000, 0xff9: 0x4000, 0xffa: 0x4000, 0xffb: 0x4000, + 0xffc: 0x4000, + // Block 0x40, offset 0x1000 + 0x1000: 0x4000, 0x1001: 0x4000, 0x1002: 0x4000, 0x1003: 0x4000, 0x1004: 0x4000, 0x1005: 0x4000, + 0x1006: 0x4000, 0x1007: 0x4000, 0x1008: 0x4000, 0x1009: 0x4000, 0x100a: 0x4000, 0x100b: 0x4000, + 0x100c: 0x4000, 0x100d: 0x4000, 0x100e: 0x4000, 0x100f: 0x4000, 0x1010: 0x4000, 0x1011: 0x4000, + 0x1012: 0x4000, 0x1013: 0x4000, 0x1014: 0x4000, 0x1015: 0x4000, 0x1016: 0x4000, 0x1017: 0x4000, + 0x1018: 0x4000, 0x1019: 0x4000, 0x101a: 0x4000, 0x101b: 0x4000, 0x101c: 0x4000, 0x101d: 0x4000, + 0x101e: 0x4000, 0x101f: 0x4000, 0x1020: 0x4000, 0x1021: 0x4000, 0x1022: 0x4000, 0x1023: 0x4000, + // Block 0x41, offset 0x1040 + 0x1040: 0x2000, 0x1041: 0x2000, 0x1042: 0x2000, 0x1043: 0x2000, 0x1044: 0x2000, 0x1045: 0x2000, + 0x1046: 0x2000, 0x1047: 0x2000, 0x1048: 0x2000, 0x1049: 0x2000, 0x104a: 0x2000, 0x104b: 0x2000, + 0x104c: 0x2000, 0x104d: 0x2000, 0x104e: 0x2000, 0x104f: 0x2000, 0x1050: 0x4000, 0x1051: 0x4000, + 0x1052: 0x4000, 0x1053: 0x4000, 0x1054: 0x4000, 0x1055: 0x4000, 0x1056: 0x4000, 0x1057: 0x4000, + 0x1058: 0x4000, 0x1059: 0x4000, + 0x1070: 0x4000, 0x1071: 0x4000, 0x1072: 0x4000, 0x1073: 0x4000, 0x1074: 0x4000, 0x1075: 0x4000, + 0x1076: 0x4000, 0x1077: 0x4000, 0x1078: 0x4000, 0x1079: 0x4000, 0x107a: 0x4000, 0x107b: 0x4000, + 0x107c: 0x4000, 0x107d: 0x4000, 0x107e: 0x4000, 0x107f: 0x4000, + // Block 0x42, offset 0x1080 + 0x1080: 0x4000, 0x1081: 0x4000, 0x1082: 0x4000, 0x1083: 0x4000, 0x1084: 0x4000, 0x1085: 0x4000, + 0x1086: 0x4000, 0x1087: 0x4000, 0x1088: 0x4000, 0x1089: 0x4000, 0x108a: 0x4000, 0x108b: 0x4000, + 0x108c: 0x4000, 0x108d: 0x4000, 0x108e: 0x4000, 0x108f: 0x4000, 0x1090: 0x4000, 0x1091: 0x4000, + 0x1092: 0x4000, 0x1094: 0x4000, 0x1095: 0x4000, 0x1096: 0x4000, 0x1097: 0x4000, + 0x1098: 0x4000, 0x1099: 0x4000, 0x109a: 0x4000, 0x109b: 0x4000, 0x109c: 0x4000, 0x109d: 0x4000, + 0x109e: 0x4000, 0x109f: 0x4000, 0x10a0: 0x4000, 0x10a1: 0x4000, 0x10a2: 0x4000, 0x10a3: 0x4000, + 0x10a4: 0x4000, 0x10a5: 0x4000, 0x10a6: 0x4000, 0x10a8: 0x4000, 0x10a9: 0x4000, + 0x10aa: 0x4000, 0x10ab: 0x4000, + // Block 0x43, offset 0x10c0 + 0x10c1: 0x9012, 0x10c2: 0x9012, 0x10c3: 0x9012, 0x10c4: 0x9012, 0x10c5: 0x9012, + 0x10c6: 0x9012, 0x10c7: 0x9012, 0x10c8: 0x9012, 0x10c9: 0x9012, 0x10ca: 0x9012, 0x10cb: 0x9012, + 0x10cc: 0x9012, 0x10cd: 0x9012, 0x10ce: 0x9012, 0x10cf: 0x9012, 0x10d0: 0x9012, 0x10d1: 0x9012, + 0x10d2: 0x9012, 0x10d3: 0x9012, 0x10d4: 0x9012, 0x10d5: 0x9012, 0x10d6: 0x9012, 0x10d7: 0x9012, + 0x10d8: 0x9012, 0x10d9: 0x9012, 0x10da: 0x9012, 0x10db: 0x9012, 0x10dc: 0x9012, 0x10dd: 0x9012, + 0x10de: 0x9012, 0x10df: 0x9012, 0x10e0: 0x9049, 0x10e1: 0x9049, 0x10e2: 0x9049, 0x10e3: 0x9049, + 0x10e4: 0x9049, 0x10e5: 0x9049, 0x10e6: 0x9049, 0x10e7: 0x9049, 0x10e8: 0x9049, 0x10e9: 0x9049, + 0x10ea: 0x9049, 0x10eb: 0x9049, 0x10ec: 0x9049, 0x10ed: 0x9049, 0x10ee: 0x9049, 0x10ef: 0x9049, + 0x10f0: 0x9049, 0x10f1: 0x9049, 0x10f2: 0x9049, 0x10f3: 0x9049, 0x10f4: 0x9049, 0x10f5: 0x9049, + 0x10f6: 0x9049, 0x10f7: 0x9049, 0x10f8: 0x9049, 0x10f9: 0x9049, 0x10fa: 0x9049, 0x10fb: 0x9049, + 0x10fc: 0x9049, 0x10fd: 0x9049, 0x10fe: 0x9049, 0x10ff: 0x9049, + // Block 0x44, offset 0x1100 + 0x1100: 0x9049, 0x1101: 0x9049, 0x1102: 0x9049, 0x1103: 0x9049, 0x1104: 0x9049, 0x1105: 0x9049, + 0x1106: 0x9049, 0x1107: 0x9049, 0x1108: 0x9049, 0x1109: 0x9049, 0x110a: 0x9049, 0x110b: 0x9049, + 0x110c: 0x9049, 0x110d: 0x9049, 0x110e: 0x9049, 0x110f: 0x9049, 0x1110: 0x9049, 0x1111: 0x9049, + 0x1112: 0x9049, 0x1113: 0x9049, 0x1114: 0x9049, 0x1115: 0x9049, 0x1116: 0x9049, 0x1117: 0x9049, + 0x1118: 0x9049, 0x1119: 0x9049, 0x111a: 0x9049, 0x111b: 0x9049, 0x111c: 0x9049, 0x111d: 0x9049, + 0x111e: 0x9049, 0x111f: 0x904a, 0x1120: 0x904b, 0x1121: 0xb04c, 0x1122: 0xb04d, 0x1123: 0xb04d, + 0x1124: 0xb04e, 0x1125: 0xb04f, 0x1126: 0xb050, 0x1127: 0xb051, 0x1128: 0xb052, 0x1129: 0xb053, + 0x112a: 0xb054, 0x112b: 0xb055, 0x112c: 0xb056, 0x112d: 0xb057, 0x112e: 0xb058, 0x112f: 0xb059, + 0x1130: 0xb05a, 0x1131: 0xb05b, 0x1132: 0xb05c, 0x1133: 0xb05d, 0x1134: 0xb05e, 0x1135: 0xb05f, + 0x1136: 0xb060, 0x1137: 0xb061, 0x1138: 0xb062, 0x1139: 0xb063, 0x113a: 0xb064, 0x113b: 0xb065, + 0x113c: 0xb052, 0x113d: 0xb066, 0x113e: 0xb067, 0x113f: 0xb055, + // Block 0x45, offset 0x1140 + 0x1140: 0xb068, 0x1141: 0xb069, 0x1142: 0xb06a, 0x1143: 0xb06b, 0x1144: 0xb05a, 0x1145: 0xb056, + 0x1146: 0xb06c, 0x1147: 0xb06d, 0x1148: 0xb06b, 0x1149: 0xb06e, 0x114a: 0xb06b, 0x114b: 0xb06f, + 0x114c: 0xb06f, 0x114d: 0xb070, 0x114e: 0xb070, 0x114f: 0xb071, 0x1150: 0xb056, 0x1151: 0xb072, + 0x1152: 0xb073, 0x1153: 0xb072, 0x1154: 0xb074, 0x1155: 0xb073, 0x1156: 0xb075, 0x1157: 0xb075, + 0x1158: 0xb076, 0x1159: 0xb076, 0x115a: 0xb077, 0x115b: 0xb077, 0x115c: 0xb073, 0x115d: 0xb078, + 0x115e: 0xb079, 0x115f: 0xb067, 0x1160: 0xb07a, 0x1161: 0xb07b, 0x1162: 0xb07b, 0x1163: 0xb07b, + 0x1164: 0xb07b, 0x1165: 0xb07b, 0x1166: 0xb07b, 0x1167: 0xb07b, 0x1168: 0xb07b, 0x1169: 0xb07b, + 0x116a: 0xb07b, 0x116b: 0xb07b, 0x116c: 0xb07b, 0x116d: 0xb07b, 0x116e: 0xb07b, 0x116f: 0xb07b, + 0x1170: 0xb07c, 0x1171: 0xb07c, 0x1172: 0xb07c, 0x1173: 0xb07c, 0x1174: 0xb07c, 0x1175: 0xb07c, + 0x1176: 0xb07c, 0x1177: 0xb07c, 0x1178: 0xb07c, 0x1179: 0xb07c, 0x117a: 0xb07c, 0x117b: 0xb07c, + 0x117c: 0xb07c, 0x117d: 0xb07c, 0x117e: 0xb07c, + // Block 0x46, offset 0x1180 + 0x1182: 0xb07d, 0x1183: 0xb07e, 0x1184: 0xb07f, 0x1185: 0xb080, + 0x1186: 0xb07f, 0x1187: 0xb07e, 0x118a: 0xb081, 0x118b: 0xb082, + 0x118c: 0xb083, 0x118d: 0xb07f, 0x118e: 0xb080, 0x118f: 0xb07f, + 0x1192: 0xb084, 0x1193: 0xb085, 0x1194: 0xb084, 0x1195: 0xb086, 0x1196: 0xb084, 0x1197: 0xb087, + 0x119a: 0xb088, 0x119b: 0xb089, 0x119c: 0xb08a, + 0x11a0: 0x908b, 0x11a1: 0x908b, 0x11a2: 0x908c, 0x11a3: 0x908d, + 0x11a4: 0x908b, 0x11a5: 0x908e, 0x11a6: 0x908f, 0x11a8: 0xb090, 0x11a9: 0xb091, + 0x11aa: 0xb092, 0x11ab: 0xb091, 0x11ac: 0xb093, 0x11ad: 0xb094, 0x11ae: 0xb095, + 0x11bd: 0x2000, + // Block 0x47, offset 0x11c0 + 0x11e0: 0x4000, 0x11e1: 0x4000, + // Block 0x48, offset 0x1200 + 0x1200: 0x4000, 0x1201: 0x4000, 0x1202: 0x4000, 0x1203: 0x4000, 0x1204: 0x4000, 0x1205: 0x4000, + 0x1206: 0x4000, 0x1207: 0x4000, 0x1208: 0x4000, 0x1209: 0x4000, 0x120a: 0x4000, 0x120b: 0x4000, + 0x120c: 0x4000, 0x120d: 0x4000, 0x120e: 0x4000, 0x120f: 0x4000, 0x1210: 0x4000, 0x1211: 0x4000, + 0x1212: 0x4000, 0x1213: 0x4000, 0x1214: 0x4000, 0x1215: 0x4000, 0x1216: 0x4000, 0x1217: 0x4000, + 0x1218: 0x4000, 0x1219: 0x4000, 0x121a: 0x4000, 0x121b: 0x4000, 0x121c: 0x4000, 0x121d: 0x4000, + 0x121e: 0x4000, 0x121f: 0x4000, 0x1220: 0x4000, 0x1221: 0x4000, 0x1222: 0x4000, 0x1223: 0x4000, + 0x1224: 0x4000, 0x1225: 0x4000, 0x1226: 0x4000, 0x1227: 0x4000, 0x1228: 0x4000, 0x1229: 0x4000, + 0x122a: 0x4000, 0x122b: 0x4000, 0x122c: 0x4000, + // Block 0x49, offset 0x1240 + 0x1240: 0x4000, 0x1241: 0x4000, 0x1242: 0x4000, 0x1243: 0x4000, 0x1244: 0x4000, 0x1245: 0x4000, + 0x1246: 0x4000, 0x1247: 0x4000, 0x1248: 0x4000, 0x1249: 0x4000, 0x124a: 0x4000, 0x124b: 0x4000, + 0x124c: 0x4000, 0x124d: 0x4000, 0x124e: 0x4000, 0x124f: 0x4000, 0x1250: 0x4000, 0x1251: 0x4000, + 0x1252: 0x4000, 0x1253: 0x4000, 0x1254: 0x4000, 0x1255: 0x4000, 0x1256: 0x4000, 0x1257: 0x4000, + 0x1258: 0x4000, 0x1259: 0x4000, 0x125a: 0x4000, 0x125b: 0x4000, 0x125c: 0x4000, 0x125d: 0x4000, + 0x125e: 0x4000, 0x125f: 0x4000, 0x1260: 0x4000, 0x1261: 0x4000, 0x1262: 0x4000, 0x1263: 0x4000, + 0x1264: 0x4000, 0x1265: 0x4000, 0x1266: 0x4000, 0x1267: 0x4000, 0x1268: 0x4000, 0x1269: 0x4000, + 0x126a: 0x4000, 0x126b: 0x4000, 0x126c: 0x4000, 0x126d: 0x4000, 0x126e: 0x4000, 0x126f: 0x4000, + 0x1270: 0x4000, 0x1271: 0x4000, 0x1272: 0x4000, + // Block 0x4a, offset 0x1280 + 0x1280: 0x4000, 0x1281: 0x4000, 0x1282: 0x4000, 0x1283: 0x4000, 0x1284: 0x4000, 0x1285: 0x4000, + 0x1286: 0x4000, 0x1287: 0x4000, 0x1288: 0x4000, 0x1289: 0x4000, 0x128a: 0x4000, 0x128b: 0x4000, + 0x128c: 0x4000, 0x128d: 0x4000, 0x128e: 0x4000, 0x128f: 0x4000, 0x1290: 0x4000, 0x1291: 0x4000, + 0x1292: 0x4000, 0x1293: 0x4000, 0x1294: 0x4000, 0x1295: 0x4000, 0x1296: 0x4000, 0x1297: 0x4000, + 0x1298: 0x4000, 0x1299: 0x4000, 0x129a: 0x4000, 0x129b: 0x4000, 0x129c: 0x4000, 0x129d: 0x4000, + 0x129e: 0x4000, + // Block 0x4b, offset 0x12c0 + 0x12f0: 0x4000, 0x12f1: 0x4000, 0x12f2: 0x4000, 0x12f3: 0x4000, 0x12f4: 0x4000, 0x12f5: 0x4000, + 0x12f6: 0x4000, 0x12f7: 0x4000, 0x12f8: 0x4000, 0x12f9: 0x4000, 0x12fa: 0x4000, 0x12fb: 0x4000, + 0x12fc: 0x4000, 0x12fd: 0x4000, 0x12fe: 0x4000, 0x12ff: 0x4000, + // Block 0x4c, offset 0x1300 + 0x1300: 0x4000, 0x1301: 0x4000, 0x1302: 0x4000, 0x1303: 0x4000, 0x1304: 0x4000, 0x1305: 0x4000, + 0x1306: 0x4000, 0x1307: 0x4000, 0x1308: 0x4000, 0x1309: 0x4000, 0x130a: 0x4000, 0x130b: 0x4000, + 0x130c: 0x4000, 0x130d: 0x4000, 0x130e: 0x4000, 0x130f: 0x4000, 0x1310: 0x4000, 0x1311: 0x4000, + 0x1312: 0x4000, 0x1313: 0x4000, 0x1314: 0x4000, 0x1315: 0x4000, 0x1316: 0x4000, 0x1317: 0x4000, + 0x1318: 0x4000, 0x1319: 0x4000, 0x131a: 0x4000, 0x131b: 0x4000, 0x131c: 0x4000, 0x131d: 0x4000, + 0x131e: 0x4000, 0x131f: 0x4000, 0x1320: 0x4000, 0x1321: 0x4000, 0x1322: 0x4000, 0x1323: 0x4000, + 0x1324: 0x4000, 0x1325: 0x4000, 0x1326: 0x4000, 0x1327: 0x4000, 0x1328: 0x4000, 0x1329: 0x4000, + 0x132a: 0x4000, 0x132b: 0x4000, 0x132c: 0x4000, 0x132d: 0x4000, 0x132e: 0x4000, 0x132f: 0x4000, + 0x1330: 0x4000, 0x1331: 0x4000, 0x1332: 0x4000, 0x1333: 0x4000, 0x1334: 0x4000, 0x1335: 0x4000, + 0x1336: 0x4000, 0x1337: 0x4000, 0x1338: 0x4000, 0x1339: 0x4000, 0x133a: 0x4000, 0x133b: 0x4000, + // Block 0x4d, offset 0x1340 + 0x1344: 0x4000, + // Block 0x4e, offset 0x1380 + 0x138f: 0x4000, + // Block 0x4f, offset 0x13c0 + 0x13c0: 0x2000, 0x13c1: 0x2000, 0x13c2: 0x2000, 0x13c3: 0x2000, 0x13c4: 0x2000, 0x13c5: 0x2000, + 0x13c6: 0x2000, 0x13c7: 0x2000, 0x13c8: 0x2000, 0x13c9: 0x2000, 0x13ca: 0x2000, + 0x13d0: 0x2000, 0x13d1: 0x2000, + 0x13d2: 0x2000, 0x13d3: 0x2000, 0x13d4: 0x2000, 0x13d5: 0x2000, 0x13d6: 0x2000, 0x13d7: 0x2000, + 0x13d8: 0x2000, 0x13d9: 0x2000, 0x13da: 0x2000, 0x13db: 0x2000, 0x13dc: 0x2000, 0x13dd: 0x2000, + 0x13de: 0x2000, 0x13df: 0x2000, 0x13e0: 0x2000, 0x13e1: 0x2000, 0x13e2: 0x2000, 0x13e3: 0x2000, + 0x13e4: 0x2000, 0x13e5: 0x2000, 0x13e6: 0x2000, 0x13e7: 0x2000, 0x13e8: 0x2000, 0x13e9: 0x2000, + 0x13ea: 0x2000, 0x13eb: 0x2000, 0x13ec: 0x2000, 0x13ed: 0x2000, + 0x13f0: 0x2000, 0x13f1: 0x2000, 0x13f2: 0x2000, 0x13f3: 0x2000, 0x13f4: 0x2000, 0x13f5: 0x2000, + 0x13f6: 0x2000, 0x13f7: 0x2000, 0x13f8: 0x2000, 0x13f9: 0x2000, 0x13fa: 0x2000, 0x13fb: 0x2000, + 0x13fc: 0x2000, 0x13fd: 0x2000, 0x13fe: 0x2000, 0x13ff: 0x2000, + // Block 0x50, offset 0x1400 + 0x1400: 0x2000, 0x1401: 0x2000, 0x1402: 0x2000, 0x1403: 0x2000, 0x1404: 0x2000, 0x1405: 0x2000, + 0x1406: 0x2000, 0x1407: 0x2000, 0x1408: 0x2000, 0x1409: 0x2000, 0x140a: 0x2000, 0x140b: 0x2000, + 0x140c: 0x2000, 0x140d: 0x2000, 0x140e: 0x2000, 0x140f: 0x2000, 0x1410: 0x2000, 0x1411: 0x2000, + 0x1412: 0x2000, 0x1413: 0x2000, 0x1414: 0x2000, 0x1415: 0x2000, 0x1416: 0x2000, 0x1417: 0x2000, + 0x1418: 0x2000, 0x1419: 0x2000, 0x141a: 0x2000, 0x141b: 0x2000, 0x141c: 0x2000, 0x141d: 0x2000, + 0x141e: 0x2000, 0x141f: 0x2000, 0x1420: 0x2000, 0x1421: 0x2000, 0x1422: 0x2000, 0x1423: 0x2000, + 0x1424: 0x2000, 0x1425: 0x2000, 0x1426: 0x2000, 0x1427: 0x2000, 0x1428: 0x2000, 0x1429: 0x2000, + 0x1430: 0x2000, 0x1431: 0x2000, 0x1432: 0x2000, 0x1433: 0x2000, 0x1434: 0x2000, 0x1435: 0x2000, + 0x1436: 0x2000, 0x1437: 0x2000, 0x1438: 0x2000, 0x1439: 0x2000, 0x143a: 0x2000, 0x143b: 0x2000, + 0x143c: 0x2000, 0x143d: 0x2000, 0x143e: 0x2000, 0x143f: 0x2000, + // Block 0x51, offset 0x1440 + 0x1440: 0x2000, 0x1441: 0x2000, 0x1442: 0x2000, 0x1443: 0x2000, 0x1444: 0x2000, 0x1445: 0x2000, + 0x1446: 0x2000, 0x1447: 0x2000, 0x1448: 0x2000, 0x1449: 0x2000, 0x144a: 0x2000, 0x144b: 0x2000, + 0x144c: 0x2000, 0x144d: 0x2000, 0x144e: 0x4000, 0x144f: 0x2000, 0x1450: 0x2000, 0x1451: 0x4000, + 0x1452: 0x4000, 0x1453: 0x4000, 0x1454: 0x4000, 0x1455: 0x4000, 0x1456: 0x4000, 0x1457: 0x4000, + 0x1458: 0x4000, 0x1459: 0x4000, 0x145a: 0x4000, 0x145b: 0x2000, 0x145c: 0x2000, 0x145d: 0x2000, + 0x145e: 0x2000, 0x145f: 0x2000, 0x1460: 0x2000, 0x1461: 0x2000, 0x1462: 0x2000, 0x1463: 0x2000, + 0x1464: 0x2000, 0x1465: 0x2000, 0x1466: 0x2000, 0x1467: 0x2000, 0x1468: 0x2000, 0x1469: 0x2000, + 0x146a: 0x2000, 0x146b: 0x2000, 0x146c: 0x2000, + // Block 0x52, offset 0x1480 + 0x1480: 0x4000, 0x1481: 0x4000, 0x1482: 0x4000, + 0x1490: 0x4000, 0x1491: 0x4000, + 0x1492: 0x4000, 0x1493: 0x4000, 0x1494: 0x4000, 0x1495: 0x4000, 0x1496: 0x4000, 0x1497: 0x4000, + 0x1498: 0x4000, 0x1499: 0x4000, 0x149a: 0x4000, 0x149b: 0x4000, 0x149c: 0x4000, 0x149d: 0x4000, + 0x149e: 0x4000, 0x149f: 0x4000, 0x14a0: 0x4000, 0x14a1: 0x4000, 0x14a2: 0x4000, 0x14a3: 0x4000, + 0x14a4: 0x4000, 0x14a5: 0x4000, 0x14a6: 0x4000, 0x14a7: 0x4000, 0x14a8: 0x4000, 0x14a9: 0x4000, + 0x14aa: 0x4000, 0x14ab: 0x4000, 0x14ac: 0x4000, 0x14ad: 0x4000, 0x14ae: 0x4000, 0x14af: 0x4000, + 0x14b0: 0x4000, 0x14b1: 0x4000, 0x14b2: 0x4000, 0x14b3: 0x4000, 0x14b4: 0x4000, 0x14b5: 0x4000, + 0x14b6: 0x4000, 0x14b7: 0x4000, 0x14b8: 0x4000, 0x14b9: 0x4000, 0x14ba: 0x4000, 0x14bb: 0x4000, + // Block 0x53, offset 0x14c0 + 0x14c0: 0x4000, 0x14c1: 0x4000, 0x14c2: 0x4000, 0x14c3: 0x4000, 0x14c4: 0x4000, 0x14c5: 0x4000, + 0x14c6: 0x4000, 0x14c7: 0x4000, 0x14c8: 0x4000, + 0x14d0: 0x4000, 0x14d1: 0x4000, + 0x14e0: 0x4000, 0x14e1: 0x4000, 0x14e2: 0x4000, 0x14e3: 0x4000, + 0x14e4: 0x4000, 0x14e5: 0x4000, + // Block 0x54, offset 0x1500 + 0x1500: 0x4000, 0x1501: 0x4000, 0x1502: 0x4000, 0x1503: 0x4000, 0x1504: 0x4000, 0x1505: 0x4000, + 0x1506: 0x4000, 0x1507: 0x4000, 0x1508: 0x4000, 0x1509: 0x4000, 0x150a: 0x4000, 0x150b: 0x4000, + 0x150c: 0x4000, 0x150d: 0x4000, 0x150e: 0x4000, 0x150f: 0x4000, 0x1510: 0x4000, 0x1511: 0x4000, + 0x1512: 0x4000, 0x1513: 0x4000, 0x1514: 0x4000, 0x1515: 0x4000, 0x1516: 0x4000, 0x1517: 0x4000, + 0x1518: 0x4000, 0x1519: 0x4000, 0x151a: 0x4000, 0x151b: 0x4000, 0x151c: 0x4000, 0x151d: 0x4000, + 0x151e: 0x4000, 0x151f: 0x4000, 0x1520: 0x4000, + 0x152d: 0x4000, 0x152e: 0x4000, 0x152f: 0x4000, + 0x1530: 0x4000, 0x1531: 0x4000, 0x1532: 0x4000, 0x1533: 0x4000, 0x1534: 0x4000, 0x1535: 0x4000, + 0x1537: 0x4000, 0x1538: 0x4000, 0x1539: 0x4000, 0x153a: 0x4000, 0x153b: 0x4000, + 0x153c: 0x4000, 0x153d: 0x4000, 0x153e: 0x4000, 0x153f: 0x4000, + // Block 0x55, offset 0x1540 + 0x1540: 0x4000, 0x1541: 0x4000, 0x1542: 0x4000, 0x1543: 0x4000, 0x1544: 0x4000, 0x1545: 0x4000, + 0x1546: 0x4000, 0x1547: 0x4000, 0x1548: 0x4000, 0x1549: 0x4000, 0x154a: 0x4000, 0x154b: 0x4000, + 0x154c: 0x4000, 0x154d: 0x4000, 0x154e: 0x4000, 0x154f: 0x4000, 0x1550: 0x4000, 0x1551: 0x4000, + 0x1552: 0x4000, 0x1553: 0x4000, 0x1554: 0x4000, 0x1555: 0x4000, 0x1556: 0x4000, 0x1557: 0x4000, + 0x1558: 0x4000, 0x1559: 0x4000, 0x155a: 0x4000, 0x155b: 0x4000, 0x155c: 0x4000, 0x155d: 0x4000, + 0x155e: 0x4000, 0x155f: 0x4000, 0x1560: 0x4000, 0x1561: 0x4000, 0x1562: 0x4000, 0x1563: 0x4000, + 0x1564: 0x4000, 0x1565: 0x4000, 0x1566: 0x4000, 0x1567: 0x4000, 0x1568: 0x4000, 0x1569: 0x4000, + 0x156a: 0x4000, 0x156b: 0x4000, 0x156c: 0x4000, 0x156d: 0x4000, 0x156e: 0x4000, 0x156f: 0x4000, + 0x1570: 0x4000, 0x1571: 0x4000, 0x1572: 0x4000, 0x1573: 0x4000, 0x1574: 0x4000, 0x1575: 0x4000, + 0x1576: 0x4000, 0x1577: 0x4000, 0x1578: 0x4000, 0x1579: 0x4000, 0x157a: 0x4000, 0x157b: 0x4000, + 0x157c: 0x4000, 0x157e: 0x4000, 0x157f: 0x4000, + // Block 0x56, offset 0x1580 + 0x1580: 0x4000, 0x1581: 0x4000, 0x1582: 0x4000, 0x1583: 0x4000, 0x1584: 0x4000, 0x1585: 0x4000, + 0x1586: 0x4000, 0x1587: 0x4000, 0x1588: 0x4000, 0x1589: 0x4000, 0x158a: 0x4000, 0x158b: 0x4000, + 0x158c: 0x4000, 0x158d: 0x4000, 0x158e: 0x4000, 0x158f: 0x4000, 0x1590: 0x4000, 0x1591: 0x4000, + 0x1592: 0x4000, 0x1593: 0x4000, + 0x15a0: 0x4000, 0x15a1: 0x4000, 0x15a2: 0x4000, 0x15a3: 0x4000, + 0x15a4: 0x4000, 0x15a5: 0x4000, 0x15a6: 0x4000, 0x15a7: 0x4000, 0x15a8: 0x4000, 0x15a9: 0x4000, + 0x15aa: 0x4000, 0x15ab: 0x4000, 0x15ac: 0x4000, 0x15ad: 0x4000, 0x15ae: 0x4000, 0x15af: 0x4000, + 0x15b0: 0x4000, 0x15b1: 0x4000, 0x15b2: 0x4000, 0x15b3: 0x4000, 0x15b4: 0x4000, 0x15b5: 0x4000, + 0x15b6: 0x4000, 0x15b7: 0x4000, 0x15b8: 0x4000, 0x15b9: 0x4000, 0x15ba: 0x4000, 0x15bb: 0x4000, + 0x15bc: 0x4000, 0x15bd: 0x4000, 0x15be: 0x4000, 0x15bf: 0x4000, + // Block 0x57, offset 0x15c0 + 0x15c0: 0x4000, 0x15c1: 0x4000, 0x15c2: 0x4000, 0x15c3: 0x4000, 0x15c4: 0x4000, 0x15c5: 0x4000, + 0x15c6: 0x4000, 0x15c7: 0x4000, 0x15c8: 0x4000, 0x15c9: 0x4000, 0x15ca: 0x4000, + 0x15cf: 0x4000, 0x15d0: 0x4000, 0x15d1: 0x4000, + 0x15d2: 0x4000, 0x15d3: 0x4000, + 0x15e0: 0x4000, 0x15e1: 0x4000, 0x15e2: 0x4000, 0x15e3: 0x4000, + 0x15e4: 0x4000, 0x15e5: 0x4000, 0x15e6: 0x4000, 0x15e7: 0x4000, 0x15e8: 0x4000, 0x15e9: 0x4000, + 0x15ea: 0x4000, 0x15eb: 0x4000, 0x15ec: 0x4000, 0x15ed: 0x4000, 0x15ee: 0x4000, 0x15ef: 0x4000, + 0x15f0: 0x4000, 0x15f4: 0x4000, + 0x15f8: 0x4000, 0x15f9: 0x4000, 0x15fa: 0x4000, 0x15fb: 0x4000, + 0x15fc: 0x4000, 0x15fd: 0x4000, 0x15fe: 0x4000, 0x15ff: 0x4000, + // Block 0x58, offset 0x1600 + 0x1600: 0x4000, 0x1602: 0x4000, 0x1603: 0x4000, 0x1604: 0x4000, 0x1605: 0x4000, + 0x1606: 0x4000, 0x1607: 0x4000, 0x1608: 0x4000, 0x1609: 0x4000, 0x160a: 0x4000, 0x160b: 0x4000, + 0x160c: 0x4000, 0x160d: 0x4000, 0x160e: 0x4000, 0x160f: 0x4000, 0x1610: 0x4000, 0x1611: 0x4000, + 0x1612: 0x4000, 0x1613: 0x4000, 0x1614: 0x4000, 0x1615: 0x4000, 0x1616: 0x4000, 0x1617: 0x4000, + 0x1618: 0x4000, 0x1619: 0x4000, 0x161a: 0x4000, 0x161b: 0x4000, 0x161c: 0x4000, 0x161d: 0x4000, + 0x161e: 0x4000, 0x161f: 0x4000, 0x1620: 0x4000, 0x1621: 0x4000, 0x1622: 0x4000, 0x1623: 0x4000, + 0x1624: 0x4000, 0x1625: 0x4000, 0x1626: 0x4000, 0x1627: 0x4000, 0x1628: 0x4000, 0x1629: 0x4000, + 0x162a: 0x4000, 0x162b: 0x4000, 0x162c: 0x4000, 0x162d: 0x4000, 0x162e: 0x4000, 0x162f: 0x4000, + 0x1630: 0x4000, 0x1631: 0x4000, 0x1632: 0x4000, 0x1633: 0x4000, 0x1634: 0x4000, 0x1635: 0x4000, + 0x1636: 0x4000, 0x1637: 0x4000, 0x1638: 0x4000, 0x1639: 0x4000, 0x163a: 0x4000, 0x163b: 0x4000, + 0x163c: 0x4000, 0x163d: 0x4000, 0x163e: 0x4000, 0x163f: 0x4000, + // Block 0x59, offset 0x1640 + 0x1640: 0x4000, 0x1641: 0x4000, 0x1642: 0x4000, 0x1643: 0x4000, 0x1644: 0x4000, 0x1645: 0x4000, + 0x1646: 0x4000, 0x1647: 0x4000, 0x1648: 0x4000, 0x1649: 0x4000, 0x164a: 0x4000, 0x164b: 0x4000, + 0x164c: 0x4000, 0x164d: 0x4000, 0x164e: 0x4000, 0x164f: 0x4000, 0x1650: 0x4000, 0x1651: 0x4000, + 0x1652: 0x4000, 0x1653: 0x4000, 0x1654: 0x4000, 0x1655: 0x4000, 0x1656: 0x4000, 0x1657: 0x4000, + 0x1658: 0x4000, 0x1659: 0x4000, 0x165a: 0x4000, 0x165b: 0x4000, 0x165c: 0x4000, 0x165d: 0x4000, + 0x165e: 0x4000, 0x165f: 0x4000, 0x1660: 0x4000, 0x1661: 0x4000, 0x1662: 0x4000, 0x1663: 0x4000, + 0x1664: 0x4000, 0x1665: 0x4000, 0x1666: 0x4000, 0x1667: 0x4000, 0x1668: 0x4000, 0x1669: 0x4000, + 0x166a: 0x4000, 0x166b: 0x4000, 0x166c: 0x4000, 0x166d: 0x4000, 0x166e: 0x4000, 0x166f: 0x4000, + 0x1670: 0x4000, 0x1671: 0x4000, 0x1672: 0x4000, 0x1673: 0x4000, 0x1674: 0x4000, 0x1675: 0x4000, + 0x1676: 0x4000, 0x1677: 0x4000, 0x1678: 0x4000, 0x1679: 0x4000, 0x167a: 0x4000, 0x167b: 0x4000, + 0x167c: 0x4000, 0x167f: 0x4000, + // Block 0x5a, offset 0x1680 + 0x1680: 0x4000, 0x1681: 0x4000, 0x1682: 0x4000, 0x1683: 0x4000, 0x1684: 0x4000, 0x1685: 0x4000, + 0x1686: 0x4000, 0x1687: 0x4000, 0x1688: 0x4000, 0x1689: 0x4000, 0x168a: 0x4000, 0x168b: 0x4000, + 0x168c: 0x4000, 0x168d: 0x4000, 0x168e: 0x4000, 0x168f: 0x4000, 0x1690: 0x4000, 0x1691: 0x4000, + 0x1692: 0x4000, 0x1693: 0x4000, 0x1694: 0x4000, 0x1695: 0x4000, 0x1696: 0x4000, 0x1697: 0x4000, + 0x1698: 0x4000, 0x1699: 0x4000, 0x169a: 0x4000, 0x169b: 0x4000, 0x169c: 0x4000, 0x169d: 0x4000, + 0x169e: 0x4000, 0x169f: 0x4000, 0x16a0: 0x4000, 0x16a1: 0x4000, 0x16a2: 0x4000, 0x16a3: 0x4000, + 0x16a4: 0x4000, 0x16a5: 0x4000, 0x16a6: 0x4000, 0x16a7: 0x4000, 0x16a8: 0x4000, 0x16a9: 0x4000, + 0x16aa: 0x4000, 0x16ab: 0x4000, 0x16ac: 0x4000, 0x16ad: 0x4000, 0x16ae: 0x4000, 0x16af: 0x4000, + 0x16b0: 0x4000, 0x16b1: 0x4000, 0x16b2: 0x4000, 0x16b3: 0x4000, 0x16b4: 0x4000, 0x16b5: 0x4000, + 0x16b6: 0x4000, 0x16b7: 0x4000, 0x16b8: 0x4000, 0x16b9: 0x4000, 0x16ba: 0x4000, 0x16bb: 0x4000, + 0x16bc: 0x4000, 0x16bd: 0x4000, + // Block 0x5b, offset 0x16c0 + 0x16cb: 0x4000, + 0x16cc: 0x4000, 0x16cd: 0x4000, 0x16ce: 0x4000, 0x16d0: 0x4000, 0x16d1: 0x4000, + 0x16d2: 0x4000, 0x16d3: 0x4000, 0x16d4: 0x4000, 0x16d5: 0x4000, 0x16d6: 0x4000, 0x16d7: 0x4000, + 0x16d8: 0x4000, 0x16d9: 0x4000, 0x16da: 0x4000, 0x16db: 0x4000, 0x16dc: 0x4000, 0x16dd: 0x4000, + 0x16de: 0x4000, 0x16df: 0x4000, 0x16e0: 0x4000, 0x16e1: 0x4000, 0x16e2: 0x4000, 0x16e3: 0x4000, + 0x16e4: 0x4000, 0x16e5: 0x4000, 0x16e6: 0x4000, 0x16e7: 0x4000, + 0x16fa: 0x4000, + // Block 0x5c, offset 0x1700 + 0x1715: 0x4000, 0x1716: 0x4000, + 0x1724: 0x4000, + // Block 0x5d, offset 0x1740 + 0x177b: 0x4000, + 0x177c: 0x4000, 0x177d: 0x4000, 0x177e: 0x4000, 0x177f: 0x4000, + // Block 0x5e, offset 0x1780 + 0x1780: 0x4000, 0x1781: 0x4000, 0x1782: 0x4000, 0x1783: 0x4000, 0x1784: 0x4000, 0x1785: 0x4000, + 0x1786: 0x4000, 0x1787: 0x4000, 0x1788: 0x4000, 0x1789: 0x4000, 0x178a: 0x4000, 0x178b: 0x4000, + 0x178c: 0x4000, 0x178d: 0x4000, 0x178e: 0x4000, 0x178f: 0x4000, + // Block 0x5f, offset 0x17c0 + 0x17c0: 0x4000, 0x17c1: 0x4000, 0x17c2: 0x4000, 0x17c3: 0x4000, 0x17c4: 0x4000, 0x17c5: 0x4000, + 0x17cc: 0x4000, 0x17d0: 0x4000, 0x17d1: 0x4000, + 0x17d2: 0x4000, + 0x17eb: 0x4000, 0x17ec: 0x4000, + 0x17f4: 0x4000, 0x17f5: 0x4000, + 0x17f6: 0x4000, 0x17f7: 0x4000, 0x17f8: 0x4000, + // Block 0x60, offset 0x1800 + 0x1810: 0x4000, 0x1811: 0x4000, + 0x1812: 0x4000, 0x1813: 0x4000, 0x1814: 0x4000, 0x1815: 0x4000, 0x1816: 0x4000, 0x1817: 0x4000, + 0x1818: 0x4000, 0x1819: 0x4000, 0x181a: 0x4000, 0x181b: 0x4000, 0x181c: 0x4000, 0x181d: 0x4000, + 0x181e: 0x4000, 0x181f: 0x4000, 0x1820: 0x4000, 0x1821: 0x4000, 0x1822: 0x4000, 0x1823: 0x4000, + 0x1824: 0x4000, 0x1825: 0x4000, 0x1826: 0x4000, 0x1827: 0x4000, 0x1828: 0x4000, 0x1829: 0x4000, + 0x182a: 0x4000, 0x182b: 0x4000, 0x182c: 0x4000, 0x182d: 0x4000, 0x182e: 0x4000, 0x182f: 0x4000, + 0x1830: 0x4000, 0x1831: 0x4000, 0x1832: 0x4000, 0x1833: 0x4000, 0x1834: 0x4000, 0x1835: 0x4000, + 0x1836: 0x4000, 0x1837: 0x4000, 0x1838: 0x4000, 0x1839: 0x4000, 0x183a: 0x4000, 0x183b: 0x4000, + 0x183c: 0x4000, 0x183d: 0x4000, 0x183e: 0x4000, + // Block 0x61, offset 0x1840 + 0x1840: 0x4000, 0x1841: 0x4000, 0x1842: 0x4000, 0x1843: 0x4000, 0x1844: 0x4000, 0x1845: 0x4000, + 0x1846: 0x4000, 0x1847: 0x4000, 0x1848: 0x4000, 0x1849: 0x4000, 0x184a: 0x4000, 0x184b: 0x4000, + 0x184c: 0x4000, 0x1850: 0x4000, 0x1851: 0x4000, + 0x1852: 0x4000, 0x1853: 0x4000, 0x1854: 0x4000, 0x1855: 0x4000, 0x1856: 0x4000, 0x1857: 0x4000, + 0x1858: 0x4000, 0x1859: 0x4000, 0x185a: 0x4000, 0x185b: 0x4000, 0x185c: 0x4000, 0x185d: 0x4000, + 0x185e: 0x4000, 0x185f: 0x4000, 0x1860: 0x4000, 0x1861: 0x4000, 0x1862: 0x4000, 0x1863: 0x4000, + 0x1864: 0x4000, 0x1865: 0x4000, 0x1866: 0x4000, 0x1867: 0x4000, 0x1868: 0x4000, 0x1869: 0x4000, + 0x186a: 0x4000, 0x186b: 0x4000, + // Block 0x62, offset 0x1880 + 0x1880: 0x4000, 0x1881: 0x4000, 0x1882: 0x4000, 0x1883: 0x4000, 0x1884: 0x4000, 0x1885: 0x4000, + 0x1886: 0x4000, 0x1887: 0x4000, 0x1888: 0x4000, 0x1889: 0x4000, 0x188a: 0x4000, 0x188b: 0x4000, + 0x188c: 0x4000, 0x188d: 0x4000, 0x188e: 0x4000, 0x188f: 0x4000, 0x1890: 0x4000, 0x1891: 0x4000, + 0x1892: 0x4000, 0x1893: 0x4000, 0x1894: 0x4000, 0x1895: 0x4000, 0x1896: 0x4000, 0x1897: 0x4000, + // Block 0x63, offset 0x18c0 + 0x18c0: 0x4000, + 0x18d0: 0x4000, 0x18d1: 0x4000, + 0x18d2: 0x4000, 0x18d3: 0x4000, 0x18d4: 0x4000, 0x18d5: 0x4000, 0x18d6: 0x4000, 0x18d7: 0x4000, + 0x18d8: 0x4000, 0x18d9: 0x4000, 0x18da: 0x4000, 0x18db: 0x4000, 0x18dc: 0x4000, 0x18dd: 0x4000, + 0x18de: 0x4000, 0x18df: 0x4000, 0x18e0: 0x4000, 0x18e1: 0x4000, 0x18e2: 0x4000, 0x18e3: 0x4000, + 0x18e4: 0x4000, 0x18e5: 0x4000, 0x18e6: 0x4000, + // Block 0x64, offset 0x1900 + 0x1900: 0x2000, 0x1901: 0x2000, 0x1902: 0x2000, 0x1903: 0x2000, 0x1904: 0x2000, 0x1905: 0x2000, + 0x1906: 0x2000, 0x1907: 0x2000, 0x1908: 0x2000, 0x1909: 0x2000, 0x190a: 0x2000, 0x190b: 0x2000, + 0x190c: 0x2000, 0x190d: 0x2000, 0x190e: 0x2000, 0x190f: 0x2000, 0x1910: 0x2000, 0x1911: 0x2000, + 0x1912: 0x2000, 0x1913: 0x2000, 0x1914: 0x2000, 0x1915: 0x2000, 0x1916: 0x2000, 0x1917: 0x2000, + 0x1918: 0x2000, 0x1919: 0x2000, 0x191a: 0x2000, 0x191b: 0x2000, 0x191c: 0x2000, 0x191d: 0x2000, + 0x191e: 0x2000, 0x191f: 0x2000, 0x1920: 0x2000, 0x1921: 0x2000, 0x1922: 0x2000, 0x1923: 0x2000, + 0x1924: 0x2000, 0x1925: 0x2000, 0x1926: 0x2000, 0x1927: 0x2000, 0x1928: 0x2000, 0x1929: 0x2000, + 0x192a: 0x2000, 0x192b: 0x2000, 0x192c: 0x2000, 0x192d: 0x2000, 0x192e: 0x2000, 0x192f: 0x2000, + 0x1930: 0x2000, 0x1931: 0x2000, 0x1932: 0x2000, 0x1933: 0x2000, 0x1934: 0x2000, 0x1935: 0x2000, + 0x1936: 0x2000, 0x1937: 0x2000, 0x1938: 0x2000, 0x1939: 0x2000, 0x193a: 0x2000, 0x193b: 0x2000, + 0x193c: 0x2000, 0x193d: 0x2000, +} + +// widthIndex: 22 blocks, 1408 entries, 1408 bytes +// Block 0 is the zero block. +var widthIndex = [1408]uint8{ + // Block 0x0, offset 0x0 + // Block 0x1, offset 0x40 + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0xc2: 0x01, 0xc3: 0x02, 0xc4: 0x03, 0xc5: 0x04, 0xc7: 0x05, + 0xc9: 0x06, 0xcb: 0x07, 0xcc: 0x08, 0xcd: 0x09, 0xce: 0x0a, 0xcf: 0x0b, + 0xd0: 0x0c, 0xd1: 0x0d, + 0xe1: 0x02, 0xe2: 0x03, 0xe3: 0x04, 0xe4: 0x05, 0xe5: 0x06, 0xe6: 0x06, 0xe7: 0x06, + 0xe8: 0x06, 0xe9: 0x06, 0xea: 0x07, 0xeb: 0x06, 0xec: 0x06, 0xed: 0x08, 0xee: 0x09, 0xef: 0x0a, + 0xf0: 0x0f, 0xf3: 0x12, 0xf4: 0x13, + // Block 0x4, offset 0x100 + 0x104: 0x0e, 0x105: 0x0f, + // Block 0x5, offset 0x140 + 0x140: 0x10, 0x141: 0x11, 0x142: 0x12, 0x144: 0x13, 0x145: 0x14, 0x146: 0x15, 0x147: 0x16, + 0x148: 0x17, 0x149: 0x18, 0x14a: 0x19, 0x14c: 0x1a, 0x14f: 0x1b, + 0x151: 0x1c, 0x152: 0x08, 0x153: 0x1d, 0x154: 0x1e, 0x155: 0x1f, 0x156: 0x20, 0x157: 0x21, + 0x158: 0x22, 0x159: 0x23, 0x15a: 0x24, 0x15b: 0x25, 0x15c: 0x26, 0x15d: 0x27, 0x15e: 0x28, 0x15f: 0x29, + 0x166: 0x2a, + 0x16c: 0x2b, 0x16d: 0x2c, + 0x17a: 0x2d, 0x17b: 0x2e, 0x17c: 0x0e, 0x17d: 0x0e, 0x17e: 0x0e, 0x17f: 0x2f, + // Block 0x6, offset 0x180 + 0x180: 0x30, 0x181: 0x31, 0x182: 0x32, 0x183: 0x33, 0x184: 0x34, 0x185: 0x35, 0x186: 0x36, 0x187: 0x37, + 0x188: 0x38, 0x189: 0x39, 0x18a: 0x0e, 0x18b: 0x3a, 0x18c: 0x0e, 0x18d: 0x0e, 0x18e: 0x0e, 0x18f: 0x0e, + 0x190: 0x0e, 0x191: 0x0e, 0x192: 0x0e, 0x193: 0x0e, 0x194: 0x0e, 0x195: 0x0e, 0x196: 0x0e, 0x197: 0x0e, + 0x198: 0x0e, 0x199: 0x0e, 0x19a: 0x0e, 0x19b: 0x0e, 0x19c: 0x0e, 0x19d: 0x0e, 0x19e: 0x0e, 0x19f: 0x0e, + 0x1a0: 0x0e, 0x1a1: 0x0e, 0x1a2: 0x0e, 0x1a3: 0x0e, 0x1a4: 0x0e, 0x1a5: 0x0e, 0x1a6: 0x0e, 0x1a7: 0x0e, + 0x1a8: 0x0e, 0x1a9: 0x0e, 0x1aa: 0x0e, 0x1ab: 0x0e, 0x1ac: 0x0e, 0x1ad: 0x0e, 0x1ae: 0x0e, 0x1af: 0x0e, + 0x1b0: 0x0e, 0x1b1: 0x0e, 0x1b2: 0x0e, 0x1b3: 0x0e, 0x1b4: 0x0e, 0x1b5: 0x0e, 0x1b6: 0x0e, 0x1b7: 0x0e, + 0x1b8: 0x0e, 0x1b9: 0x0e, 0x1ba: 0x0e, 0x1bb: 0x0e, 0x1bc: 0x0e, 0x1bd: 0x0e, 0x1be: 0x0e, 0x1bf: 0x0e, + // Block 0x7, offset 0x1c0 + 0x1c0: 0x0e, 0x1c1: 0x0e, 0x1c2: 0x0e, 0x1c3: 0x0e, 0x1c4: 0x0e, 0x1c5: 0x0e, 0x1c6: 0x0e, 0x1c7: 0x0e, + 0x1c8: 0x0e, 0x1c9: 0x0e, 0x1ca: 0x0e, 0x1cb: 0x0e, 0x1cc: 0x0e, 0x1cd: 0x0e, 0x1ce: 0x0e, 0x1cf: 0x0e, + 0x1d0: 0x0e, 0x1d1: 0x0e, 0x1d2: 0x0e, 0x1d3: 0x0e, 0x1d4: 0x0e, 0x1d5: 0x0e, 0x1d6: 0x0e, 0x1d7: 0x0e, + 0x1d8: 0x0e, 0x1d9: 0x0e, 0x1da: 0x0e, 0x1db: 0x0e, 0x1dc: 0x0e, 0x1dd: 0x0e, 0x1de: 0x0e, 0x1df: 0x0e, + 0x1e0: 0x0e, 0x1e1: 0x0e, 0x1e2: 0x0e, 0x1e3: 0x0e, 0x1e4: 0x0e, 0x1e5: 0x0e, 0x1e6: 0x0e, 0x1e7: 0x0e, + 0x1e8: 0x0e, 0x1e9: 0x0e, 0x1ea: 0x0e, 0x1eb: 0x0e, 0x1ec: 0x0e, 0x1ed: 0x0e, 0x1ee: 0x0e, 0x1ef: 0x0e, + 0x1f0: 0x0e, 0x1f1: 0x0e, 0x1f2: 0x0e, 0x1f3: 0x0e, 0x1f4: 0x0e, 0x1f5: 0x0e, 0x1f6: 0x0e, + 0x1f8: 0x0e, 0x1f9: 0x0e, 0x1fa: 0x0e, 0x1fb: 0x0e, 0x1fc: 0x0e, 0x1fd: 0x0e, 0x1fe: 0x0e, 0x1ff: 0x0e, + // Block 0x8, offset 0x200 + 0x200: 0x0e, 0x201: 0x0e, 0x202: 0x0e, 0x203: 0x0e, 0x204: 0x0e, 0x205: 0x0e, 0x206: 0x0e, 0x207: 0x0e, + 0x208: 0x0e, 0x209: 0x0e, 0x20a: 0x0e, 0x20b: 0x0e, 0x20c: 0x0e, 0x20d: 0x0e, 0x20e: 0x0e, 0x20f: 0x0e, + 0x210: 0x0e, 0x211: 0x0e, 0x212: 0x0e, 0x213: 0x0e, 0x214: 0x0e, 0x215: 0x0e, 0x216: 0x0e, 0x217: 0x0e, + 0x218: 0x0e, 0x219: 0x0e, 0x21a: 0x0e, 0x21b: 0x0e, 0x21c: 0x0e, 0x21d: 0x0e, 0x21e: 0x0e, 0x21f: 0x0e, + 0x220: 0x0e, 0x221: 0x0e, 0x222: 0x0e, 0x223: 0x0e, 0x224: 0x0e, 0x225: 0x0e, 0x226: 0x0e, 0x227: 0x0e, + 0x228: 0x0e, 0x229: 0x0e, 0x22a: 0x0e, 0x22b: 0x0e, 0x22c: 0x0e, 0x22d: 0x0e, 0x22e: 0x0e, 0x22f: 0x0e, + 0x230: 0x0e, 0x231: 0x0e, 0x232: 0x0e, 0x233: 0x0e, 0x234: 0x0e, 0x235: 0x0e, 0x236: 0x0e, 0x237: 0x0e, + 0x238: 0x0e, 0x239: 0x0e, 0x23a: 0x0e, 0x23b: 0x0e, 0x23c: 0x0e, 0x23d: 0x0e, 0x23e: 0x0e, 0x23f: 0x0e, + // Block 0x9, offset 0x240 + 0x240: 0x0e, 0x241: 0x0e, 0x242: 0x0e, 0x243: 0x0e, 0x244: 0x0e, 0x245: 0x0e, 0x246: 0x0e, 0x247: 0x0e, + 0x248: 0x0e, 0x249: 0x0e, 0x24a: 0x0e, 0x24b: 0x0e, 0x24c: 0x0e, 0x24d: 0x0e, 0x24e: 0x0e, 0x24f: 0x0e, + 0x250: 0x0e, 0x251: 0x0e, 0x252: 0x3b, 0x253: 0x3c, + 0x265: 0x3d, + 0x270: 0x0e, 0x271: 0x0e, 0x272: 0x0e, 0x273: 0x0e, 0x274: 0x0e, 0x275: 0x0e, 0x276: 0x0e, 0x277: 0x0e, + 0x278: 0x0e, 0x279: 0x0e, 0x27a: 0x0e, 0x27b: 0x0e, 0x27c: 0x0e, 0x27d: 0x0e, 0x27e: 0x0e, 0x27f: 0x0e, + // Block 0xa, offset 0x280 + 0x280: 0x0e, 0x281: 0x0e, 0x282: 0x0e, 0x283: 0x0e, 0x284: 0x0e, 0x285: 0x0e, 0x286: 0x0e, 0x287: 0x0e, + 0x288: 0x0e, 0x289: 0x0e, 0x28a: 0x0e, 0x28b: 0x0e, 0x28c: 0x0e, 0x28d: 0x0e, 0x28e: 0x0e, 0x28f: 0x0e, + 0x290: 0x0e, 0x291: 0x0e, 0x292: 0x0e, 0x293: 0x0e, 0x294: 0x0e, 0x295: 0x0e, 0x296: 0x0e, 0x297: 0x0e, + 0x298: 0x0e, 0x299: 0x0e, 0x29a: 0x0e, 0x29b: 0x0e, 0x29c: 0x0e, 0x29d: 0x0e, 0x29e: 0x3e, + // Block 0xb, offset 0x2c0 + 0x2c0: 0x08, 0x2c1: 0x08, 0x2c2: 0x08, 0x2c3: 0x08, 0x2c4: 0x08, 0x2c5: 0x08, 0x2c6: 0x08, 0x2c7: 0x08, + 0x2c8: 0x08, 0x2c9: 0x08, 0x2ca: 0x08, 0x2cb: 0x08, 0x2cc: 0x08, 0x2cd: 0x08, 0x2ce: 0x08, 0x2cf: 0x08, + 0x2d0: 0x08, 0x2d1: 0x08, 0x2d2: 0x08, 0x2d3: 0x08, 0x2d4: 0x08, 0x2d5: 0x08, 0x2d6: 0x08, 0x2d7: 0x08, + 0x2d8: 0x08, 0x2d9: 0x08, 0x2da: 0x08, 0x2db: 0x08, 0x2dc: 0x08, 0x2dd: 0x08, 0x2de: 0x08, 0x2df: 0x08, + 0x2e0: 0x08, 0x2e1: 0x08, 0x2e2: 0x08, 0x2e3: 0x08, 0x2e4: 0x08, 0x2e5: 0x08, 0x2e6: 0x08, 0x2e7: 0x08, + 0x2e8: 0x08, 0x2e9: 0x08, 0x2ea: 0x08, 0x2eb: 0x08, 0x2ec: 0x08, 0x2ed: 0x08, 0x2ee: 0x08, 0x2ef: 0x08, + 0x2f0: 0x08, 0x2f1: 0x08, 0x2f2: 0x08, 0x2f3: 0x08, 0x2f4: 0x08, 0x2f5: 0x08, 0x2f6: 0x08, 0x2f7: 0x08, + 0x2f8: 0x08, 0x2f9: 0x08, 0x2fa: 0x08, 0x2fb: 0x08, 0x2fc: 0x08, 0x2fd: 0x08, 0x2fe: 0x08, 0x2ff: 0x08, + // Block 0xc, offset 0x300 + 0x300: 0x08, 0x301: 0x08, 0x302: 0x08, 0x303: 0x08, 0x304: 0x08, 0x305: 0x08, 0x306: 0x08, 0x307: 0x08, + 0x308: 0x08, 0x309: 0x08, 0x30a: 0x08, 0x30b: 0x08, 0x30c: 0x08, 0x30d: 0x08, 0x30e: 0x08, 0x30f: 0x08, + 0x310: 0x08, 0x311: 0x08, 0x312: 0x08, 0x313: 0x08, 0x314: 0x08, 0x315: 0x08, 0x316: 0x08, 0x317: 0x08, + 0x318: 0x08, 0x319: 0x08, 0x31a: 0x08, 0x31b: 0x08, 0x31c: 0x08, 0x31d: 0x08, 0x31e: 0x08, 0x31f: 0x08, + 0x320: 0x08, 0x321: 0x08, 0x322: 0x08, 0x323: 0x08, 0x324: 0x0e, 0x325: 0x0e, 0x326: 0x0e, 0x327: 0x0e, + 0x328: 0x0e, 0x329: 0x0e, 0x32a: 0x0e, 0x32b: 0x0e, + 0x338: 0x3f, 0x339: 0x40, 0x33c: 0x41, 0x33d: 0x42, 0x33e: 0x43, 0x33f: 0x44, + // Block 0xd, offset 0x340 + 0x37f: 0x45, + // Block 0xe, offset 0x380 + 0x380: 0x0e, 0x381: 0x0e, 0x382: 0x0e, 0x383: 0x0e, 0x384: 0x0e, 0x385: 0x0e, 0x386: 0x0e, 0x387: 0x0e, + 0x388: 0x0e, 0x389: 0x0e, 0x38a: 0x0e, 0x38b: 0x0e, 0x38c: 0x0e, 0x38d: 0x0e, 0x38e: 0x0e, 0x38f: 0x0e, + 0x390: 0x0e, 0x391: 0x0e, 0x392: 0x0e, 0x393: 0x0e, 0x394: 0x0e, 0x395: 0x0e, 0x396: 0x0e, 0x397: 0x0e, + 0x398: 0x0e, 0x399: 0x0e, 0x39a: 0x0e, 0x39b: 0x0e, 0x39c: 0x0e, 0x39d: 0x0e, 0x39e: 0x0e, 0x39f: 0x46, + 0x3a0: 0x0e, 0x3a1: 0x0e, 0x3a2: 0x0e, 0x3a3: 0x0e, 0x3a4: 0x0e, 0x3a5: 0x0e, 0x3a6: 0x0e, 0x3a7: 0x0e, + 0x3a8: 0x0e, 0x3a9: 0x0e, 0x3aa: 0x0e, 0x3ab: 0x47, + // Block 0xf, offset 0x3c0 + 0x3c0: 0x0e, 0x3c1: 0x0e, 0x3c2: 0x0e, 0x3c3: 0x0e, 0x3c4: 0x48, 0x3c5: 0x49, 0x3c6: 0x0e, 0x3c7: 0x0e, + 0x3c8: 0x0e, 0x3c9: 0x0e, 0x3ca: 0x0e, 0x3cb: 0x4a, + // Block 0x10, offset 0x400 + 0x400: 0x4b, 0x403: 0x4c, 0x404: 0x4d, 0x405: 0x4e, 0x406: 0x4f, + 0x408: 0x50, 0x409: 0x51, 0x40c: 0x52, 0x40d: 0x53, 0x40e: 0x54, 0x40f: 0x55, + 0x410: 0x3a, 0x411: 0x56, 0x412: 0x0e, 0x413: 0x57, 0x414: 0x58, 0x415: 0x59, 0x416: 0x5a, 0x417: 0x5b, + 0x418: 0x0e, 0x419: 0x5c, 0x41a: 0x0e, 0x41b: 0x5d, + 0x424: 0x5e, 0x425: 0x5f, 0x426: 0x60, 0x427: 0x61, + // Block 0x11, offset 0x440 + 0x456: 0x0b, 0x457: 0x06, + 0x458: 0x0c, 0x45b: 0x0d, 0x45f: 0x0e, + 0x460: 0x06, 0x461: 0x06, 0x462: 0x06, 0x463: 0x06, 0x464: 0x06, 0x465: 0x06, 0x466: 0x06, 0x467: 0x06, + 0x468: 0x06, 0x469: 0x06, 0x46a: 0x06, 0x46b: 0x06, 0x46c: 0x06, 0x46d: 0x06, 0x46e: 0x06, 0x46f: 0x06, + 0x470: 0x06, 0x471: 0x06, 0x472: 0x06, 0x473: 0x06, 0x474: 0x06, 0x475: 0x06, 0x476: 0x06, 0x477: 0x06, + 0x478: 0x06, 0x479: 0x06, 0x47a: 0x06, 0x47b: 0x06, 0x47c: 0x06, 0x47d: 0x06, 0x47e: 0x06, 0x47f: 0x06, + // Block 0x12, offset 0x480 + 0x484: 0x08, 0x485: 0x08, 0x486: 0x08, 0x487: 0x09, + // Block 0x13, offset 0x4c0 + 0x4c0: 0x08, 0x4c1: 0x08, 0x4c2: 0x08, 0x4c3: 0x08, 0x4c4: 0x08, 0x4c5: 0x08, 0x4c6: 0x08, 0x4c7: 0x08, + 0x4c8: 0x08, 0x4c9: 0x08, 0x4ca: 0x08, 0x4cb: 0x08, 0x4cc: 0x08, 0x4cd: 0x08, 0x4ce: 0x08, 0x4cf: 0x08, + 0x4d0: 0x08, 0x4d1: 0x08, 0x4d2: 0x08, 0x4d3: 0x08, 0x4d4: 0x08, 0x4d5: 0x08, 0x4d6: 0x08, 0x4d7: 0x08, + 0x4d8: 0x08, 0x4d9: 0x08, 0x4da: 0x08, 0x4db: 0x08, 0x4dc: 0x08, 0x4dd: 0x08, 0x4de: 0x08, 0x4df: 0x08, + 0x4e0: 0x08, 0x4e1: 0x08, 0x4e2: 0x08, 0x4e3: 0x08, 0x4e4: 0x08, 0x4e5: 0x08, 0x4e6: 0x08, 0x4e7: 0x08, + 0x4e8: 0x08, 0x4e9: 0x08, 0x4ea: 0x08, 0x4eb: 0x08, 0x4ec: 0x08, 0x4ed: 0x08, 0x4ee: 0x08, 0x4ef: 0x08, + 0x4f0: 0x08, 0x4f1: 0x08, 0x4f2: 0x08, 0x4f3: 0x08, 0x4f4: 0x08, 0x4f5: 0x08, 0x4f6: 0x08, 0x4f7: 0x08, + 0x4f8: 0x08, 0x4f9: 0x08, 0x4fa: 0x08, 0x4fb: 0x08, 0x4fc: 0x08, 0x4fd: 0x08, 0x4fe: 0x08, 0x4ff: 0x62, + // Block 0x14, offset 0x500 + 0x520: 0x10, + 0x530: 0x09, 0x531: 0x09, 0x532: 0x09, 0x533: 0x09, 0x534: 0x09, 0x535: 0x09, 0x536: 0x09, 0x537: 0x09, + 0x538: 0x09, 0x539: 0x09, 0x53a: 0x09, 0x53b: 0x09, 0x53c: 0x09, 0x53d: 0x09, 0x53e: 0x09, 0x53f: 0x11, + // Block 0x15, offset 0x540 + 0x540: 0x09, 0x541: 0x09, 0x542: 0x09, 0x543: 0x09, 0x544: 0x09, 0x545: 0x09, 0x546: 0x09, 0x547: 0x09, + 0x548: 0x09, 0x549: 0x09, 0x54a: 0x09, 0x54b: 0x09, 0x54c: 0x09, 0x54d: 0x09, 0x54e: 0x09, 0x54f: 0x11, +} + +// inverseData contains 4-byte entries of the following format: +// +// <0 padding> +// +// The last byte of the UTF-8-encoded rune is xor-ed with the last byte of the +// UTF-8 encoding of the original rune. Mappings often have the following +// pattern: +// +// Ａ -> A (U+FF21 -> U+0041) +// Ｂ -> B (U+FF22 -> U+0042) +// ... +// +// By xor-ing the last byte the same entry can be shared by many mappings. This +// reduces the total number of distinct entries by about two thirds. +// The resulting entry for the aforementioned mappings is +// +// { 0x01, 0xE0, 0x00, 0x00 } +// +// Using this entry to map U+FF21 (UTF-8 [EF BC A1]), we get +// +// E0 ^ A1 = 41. +// +// Similarly, for U+FF22 (UTF-8 [EF BC A2]), we get +// +// E0 ^ A2 = 42. +// +// Note that because of the xor-ing, the byte sequence stored in the entry is +// not valid UTF-8. +var inverseData = [150][4]byte{ + {0x00, 0x00, 0x00, 0x00}, + {0x03, 0xe3, 0x80, 0xa0}, + {0x03, 0xef, 0xbc, 0xa0}, + {0x03, 0xef, 0xbc, 0xe0}, + {0x03, 0xef, 0xbd, 0xe0}, + {0x03, 0xef, 0xbf, 0x02}, + {0x03, 0xef, 0xbf, 0x00}, + {0x03, 0xef, 0xbf, 0x0e}, + {0x03, 0xef, 0xbf, 0x0c}, + {0x03, 0xef, 0xbf, 0x0f}, + {0x03, 0xef, 0xbf, 0x39}, + {0x03, 0xef, 0xbf, 0x3b}, + {0x03, 0xef, 0xbf, 0x3f}, + {0x03, 0xef, 0xbf, 0x2a}, + {0x03, 0xef, 0xbf, 0x0d}, + {0x03, 0xef, 0xbf, 0x25}, + {0x03, 0xef, 0xbd, 0x1a}, + {0x03, 0xef, 0xbd, 0x26}, + {0x01, 0xa0, 0x00, 0x00}, + {0x03, 0xef, 0xbd, 0x25}, + {0x03, 0xef, 0xbd, 0x23}, + {0x03, 0xef, 0xbd, 0x2e}, + {0x03, 0xef, 0xbe, 0x07}, + {0x03, 0xef, 0xbe, 0x05}, + {0x03, 0xef, 0xbd, 0x06}, + {0x03, 0xef, 0xbd, 0x13}, + {0x03, 0xef, 0xbd, 0x0b}, + {0x03, 0xef, 0xbd, 0x16}, + {0x03, 0xef, 0xbd, 0x0c}, + {0x03, 0xef, 0xbd, 0x15}, + {0x03, 0xef, 0xbd, 0x0d}, + {0x03, 0xef, 0xbd, 0x1c}, + {0x03, 0xef, 0xbd, 0x02}, + {0x03, 0xef, 0xbd, 0x1f}, + {0x03, 0xef, 0xbd, 0x1d}, + {0x03, 0xef, 0xbd, 0x17}, + {0x03, 0xef, 0xbd, 0x08}, + {0x03, 0xef, 0xbd, 0x09}, + {0x03, 0xef, 0xbd, 0x0e}, + {0x03, 0xef, 0xbd, 0x04}, + {0x03, 0xef, 0xbd, 0x05}, + {0x03, 0xef, 0xbe, 0x3f}, + {0x03, 0xef, 0xbe, 0x00}, + {0x03, 0xef, 0xbd, 0x2c}, + {0x03, 0xef, 0xbe, 0x06}, + {0x03, 0xef, 0xbe, 0x0c}, + {0x03, 0xef, 0xbe, 0x0f}, + {0x03, 0xef, 0xbe, 0x0d}, + {0x03, 0xef, 0xbe, 0x0b}, + {0x03, 0xef, 0xbe, 0x19}, + {0x03, 0xef, 0xbe, 0x15}, + {0x03, 0xef, 0xbe, 0x11}, + {0x03, 0xef, 0xbe, 0x31}, + {0x03, 0xef, 0xbe, 0x33}, + {0x03, 0xef, 0xbd, 0x0f}, + {0x03, 0xef, 0xbe, 0x30}, + {0x03, 0xef, 0xbe, 0x3e}, + {0x03, 0xef, 0xbe, 0x32}, + {0x03, 0xef, 0xbe, 0x36}, + {0x03, 0xef, 0xbd, 0x14}, + {0x03, 0xef, 0xbe, 0x2e}, + {0x03, 0xef, 0xbd, 0x1e}, + {0x03, 0xef, 0xbe, 0x10}, + {0x03, 0xef, 0xbf, 0x13}, + {0x03, 0xef, 0xbf, 0x15}, + {0x03, 0xef, 0xbf, 0x17}, + {0x03, 0xef, 0xbf, 0x1f}, + {0x03, 0xef, 0xbf, 0x1d}, + {0x03, 0xef, 0xbf, 0x1b}, + {0x03, 0xef, 0xbf, 0x09}, + {0x03, 0xef, 0xbf, 0x0b}, + {0x03, 0xef, 0xbf, 0x37}, + {0x03, 0xef, 0xbe, 0x04}, + {0x01, 0xe0, 0x00, 0x00}, + {0x03, 0xe2, 0xa6, 0x1a}, + {0x03, 0xe2, 0xa6, 0x26}, + {0x03, 0xe3, 0x80, 0x23}, + {0x03, 0xe3, 0x80, 0x2e}, + {0x03, 0xe3, 0x80, 0x25}, + {0x03, 0xe3, 0x83, 0x1e}, + {0x03, 0xe3, 0x83, 0x14}, + {0x03, 0xe3, 0x82, 0x06}, + {0x03, 0xe3, 0x82, 0x0b}, + {0x03, 0xe3, 0x82, 0x0c}, + {0x03, 0xe3, 0x82, 0x0d}, + {0x03, 0xe3, 0x82, 0x02}, + {0x03, 0xe3, 0x83, 0x0f}, + {0x03, 0xe3, 0x83, 0x08}, + {0x03, 0xe3, 0x83, 0x09}, + {0x03, 0xe3, 0x83, 0x2c}, + {0x03, 0xe3, 0x83, 0x0c}, + {0x03, 0xe3, 0x82, 0x13}, + {0x03, 0xe3, 0x82, 0x16}, + {0x03, 0xe3, 0x82, 0x15}, + {0x03, 0xe3, 0x82, 0x1c}, + {0x03, 0xe3, 0x82, 0x1f}, + {0x03, 0xe3, 0x82, 0x1d}, + {0x03, 0xe3, 0x82, 0x1a}, + {0x03, 0xe3, 0x82, 0x17}, + {0x03, 0xe3, 0x82, 0x08}, + {0x03, 0xe3, 0x82, 0x09}, + {0x03, 0xe3, 0x82, 0x0e}, + {0x03, 0xe3, 0x82, 0x04}, + {0x03, 0xe3, 0x82, 0x05}, + {0x03, 0xe3, 0x82, 0x3f}, + {0x03, 0xe3, 0x83, 0x00}, + {0x03, 0xe3, 0x83, 0x06}, + {0x03, 0xe3, 0x83, 0x05}, + {0x03, 0xe3, 0x83, 0x0d}, + {0x03, 0xe3, 0x83, 0x0b}, + {0x03, 0xe3, 0x83, 0x07}, + {0x03, 0xe3, 0x83, 0x19}, + {0x03, 0xe3, 0x83, 0x15}, + {0x03, 0xe3, 0x83, 0x11}, + {0x03, 0xe3, 0x83, 0x31}, + {0x03, 0xe3, 0x83, 0x33}, + {0x03, 0xe3, 0x83, 0x30}, + {0x03, 0xe3, 0x83, 0x3e}, + {0x03, 0xe3, 0x83, 0x32}, + {0x03, 0xe3, 0x83, 0x36}, + {0x03, 0xe3, 0x83, 0x2e}, + {0x03, 0xe3, 0x82, 0x07}, + {0x03, 0xe3, 0x85, 0x04}, + {0x03, 0xe3, 0x84, 0x10}, + {0x03, 0xe3, 0x85, 0x30}, + {0x03, 0xe3, 0x85, 0x0d}, + {0x03, 0xe3, 0x85, 0x13}, + {0x03, 0xe3, 0x85, 0x15}, + {0x03, 0xe3, 0x85, 0x17}, + {0x03, 0xe3, 0x85, 0x1f}, + {0x03, 0xe3, 0x85, 0x1d}, + {0x03, 0xe3, 0x85, 0x1b}, + {0x03, 0xe3, 0x85, 0x09}, + {0x03, 0xe3, 0x85, 0x0f}, + {0x03, 0xe3, 0x85, 0x0b}, + {0x03, 0xe3, 0x85, 0x37}, + {0x03, 0xe3, 0x85, 0x3b}, + {0x03, 0xe3, 0x85, 0x39}, + {0x03, 0xe3, 0x85, 0x3f}, + {0x02, 0xc2, 0x02, 0x00}, + {0x02, 0xc2, 0x0e, 0x00}, + {0x02, 0xc2, 0x0c, 0x00}, + {0x02, 0xc2, 0x00, 0x00}, + {0x03, 0xe2, 0x82, 0x0f}, + {0x03, 0xe2, 0x94, 0x2a}, + {0x03, 0xe2, 0x86, 0x39}, + {0x03, 0xe2, 0x86, 0x3b}, + {0x03, 0xe2, 0x86, 0x3f}, + {0x03, 0xe2, 0x96, 0x0d}, + {0x03, 0xe2, 0x97, 0x25}, +} + +// Total table size 14936 bytes (14KiB) diff --git a/vendor/golang.org/x/text/width/tables11.0.0.go b/vendor/golang.org/x/text/width/tables11.0.0.go new file mode 100644 index 0000000000..327eaef9b7 --- /dev/null +++ b/vendor/golang.org/x/text/width/tables11.0.0.go @@ -0,0 +1,1341 @@ +// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. + +//go:build go1.13 && !go1.14 +// +build go1.13,!go1.14 + +package width + +// UnicodeVersion is the Unicode version from which the tables in this package are derived. +const UnicodeVersion = "11.0.0" + +// lookup returns the trie value for the first UTF-8 encoding in s and +// the width in bytes of this encoding. The size will be 0 if s does not +// hold enough bytes to complete the encoding. len(s) must be greater than 0. +func (t *widthTrie) lookup(s []byte) (v uint16, sz int) { + c0 := s[0] + switch { + case c0 < 0x80: // is ASCII + return widthValues[c0], 1 + case c0 < 0xC2: + return 0, 1 // Illegal UTF-8: not a starter, not ASCII. + case c0 < 0xE0: // 2-byte UTF-8 + if len(s) < 2 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c1), 2 + case c0 < 0xF0: // 3-byte UTF-8 + if len(s) < 3 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = widthIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c2), 3 + case c0 < 0xF8: // 4-byte UTF-8 + if len(s) < 4 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = widthIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + o = uint32(i)<<6 + uint32(c2) + i = widthIndex[o] + c3 := s[3] + if c3 < 0x80 || 0xC0 <= c3 { + return 0, 3 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c3), 4 + } + // Illegal rune + return 0, 1 +} + +// lookupUnsafe returns the trie value for the first UTF-8 encoding in s. +// s must start with a full and valid UTF-8 encoded rune. +func (t *widthTrie) lookupUnsafe(s []byte) uint16 { + c0 := s[0] + if c0 < 0x80 { // is ASCII + return widthValues[c0] + } + i := widthIndex[c0] + if c0 < 0xE0 { // 2-byte UTF-8 + return t.lookupValue(uint32(i), s[1]) + } + i = widthIndex[uint32(i)<<6+uint32(s[1])] + if c0 < 0xF0 { // 3-byte UTF-8 + return t.lookupValue(uint32(i), s[2]) + } + i = widthIndex[uint32(i)<<6+uint32(s[2])] + if c0 < 0xF8 { // 4-byte UTF-8 + return t.lookupValue(uint32(i), s[3]) + } + return 0 +} + +// lookupString returns the trie value for the first UTF-8 encoding in s and +// the width in bytes of this encoding. The size will be 0 if s does not +// hold enough bytes to complete the encoding. len(s) must be greater than 0. +func (t *widthTrie) lookupString(s string) (v uint16, sz int) { + c0 := s[0] + switch { + case c0 < 0x80: // is ASCII + return widthValues[c0], 1 + case c0 < 0xC2: + return 0, 1 // Illegal UTF-8: not a starter, not ASCII. + case c0 < 0xE0: // 2-byte UTF-8 + if len(s) < 2 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c1), 2 + case c0 < 0xF0: // 3-byte UTF-8 + if len(s) < 3 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = widthIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c2), 3 + case c0 < 0xF8: // 4-byte UTF-8 + if len(s) < 4 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = widthIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + o = uint32(i)<<6 + uint32(c2) + i = widthIndex[o] + c3 := s[3] + if c3 < 0x80 || 0xC0 <= c3 { + return 0, 3 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c3), 4 + } + // Illegal rune + return 0, 1 +} + +// lookupStringUnsafe returns the trie value for the first UTF-8 encoding in s. +// s must start with a full and valid UTF-8 encoded rune. +func (t *widthTrie) lookupStringUnsafe(s string) uint16 { + c0 := s[0] + if c0 < 0x80 { // is ASCII + return widthValues[c0] + } + i := widthIndex[c0] + if c0 < 0xE0 { // 2-byte UTF-8 + return t.lookupValue(uint32(i), s[1]) + } + i = widthIndex[uint32(i)<<6+uint32(s[1])] + if c0 < 0xF0 { // 3-byte UTF-8 + return t.lookupValue(uint32(i), s[2]) + } + i = widthIndex[uint32(i)<<6+uint32(s[2])] + if c0 < 0xF8 { // 4-byte UTF-8 + return t.lookupValue(uint32(i), s[3]) + } + return 0 +} + +// widthTrie. Total size: 14336 bytes (14.00 KiB). Checksum: c0f7712776e71cd4. +type widthTrie struct{} + +func newWidthTrie(i int) *widthTrie { + return &widthTrie{} +} + +// lookupValue determines the type of block n and looks up the value for b. +func (t *widthTrie) lookupValue(n uint32, b byte) uint16 { + switch { + default: + return uint16(widthValues[n<<6+uint32(b)]) + } +} + +// widthValues: 101 blocks, 6464 entries, 12928 bytes +// The third block is the zero block. +var widthValues = [6464]uint16{ + // Block 0x0, offset 0x0 + 0x20: 0x6001, 0x21: 0x6002, 0x22: 0x6002, 0x23: 0x6002, + 0x24: 0x6002, 0x25: 0x6002, 0x26: 0x6002, 0x27: 0x6002, 0x28: 0x6002, 0x29: 0x6002, + 0x2a: 0x6002, 0x2b: 0x6002, 0x2c: 0x6002, 0x2d: 0x6002, 0x2e: 0x6002, 0x2f: 0x6002, + 0x30: 0x6002, 0x31: 0x6002, 0x32: 0x6002, 0x33: 0x6002, 0x34: 0x6002, 0x35: 0x6002, + 0x36: 0x6002, 0x37: 0x6002, 0x38: 0x6002, 0x39: 0x6002, 0x3a: 0x6002, 0x3b: 0x6002, + 0x3c: 0x6002, 0x3d: 0x6002, 0x3e: 0x6002, 0x3f: 0x6002, + // Block 0x1, offset 0x40 + 0x40: 0x6003, 0x41: 0x6003, 0x42: 0x6003, 0x43: 0x6003, 0x44: 0x6003, 0x45: 0x6003, + 0x46: 0x6003, 0x47: 0x6003, 0x48: 0x6003, 0x49: 0x6003, 0x4a: 0x6003, 0x4b: 0x6003, + 0x4c: 0x6003, 0x4d: 0x6003, 0x4e: 0x6003, 0x4f: 0x6003, 0x50: 0x6003, 0x51: 0x6003, + 0x52: 0x6003, 0x53: 0x6003, 0x54: 0x6003, 0x55: 0x6003, 0x56: 0x6003, 0x57: 0x6003, + 0x58: 0x6003, 0x59: 0x6003, 0x5a: 0x6003, 0x5b: 0x6003, 0x5c: 0x6003, 0x5d: 0x6003, + 0x5e: 0x6003, 0x5f: 0x6003, 0x60: 0x6004, 0x61: 0x6004, 0x62: 0x6004, 0x63: 0x6004, + 0x64: 0x6004, 0x65: 0x6004, 0x66: 0x6004, 0x67: 0x6004, 0x68: 0x6004, 0x69: 0x6004, + 0x6a: 0x6004, 0x6b: 0x6004, 0x6c: 0x6004, 0x6d: 0x6004, 0x6e: 0x6004, 0x6f: 0x6004, + 0x70: 0x6004, 0x71: 0x6004, 0x72: 0x6004, 0x73: 0x6004, 0x74: 0x6004, 0x75: 0x6004, + 0x76: 0x6004, 0x77: 0x6004, 0x78: 0x6004, 0x79: 0x6004, 0x7a: 0x6004, 0x7b: 0x6004, + 0x7c: 0x6004, 0x7d: 0x6004, 0x7e: 0x6004, + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0xe1: 0x2000, 0xe2: 0x6005, 0xe3: 0x6005, + 0xe4: 0x2000, 0xe5: 0x6006, 0xe6: 0x6005, 0xe7: 0x2000, 0xe8: 0x2000, + 0xea: 0x2000, 0xec: 0x6007, 0xed: 0x2000, 0xee: 0x2000, 0xef: 0x6008, + 0xf0: 0x2000, 0xf1: 0x2000, 0xf2: 0x2000, 0xf3: 0x2000, 0xf4: 0x2000, + 0xf6: 0x2000, 0xf7: 0x2000, 0xf8: 0x2000, 0xf9: 0x2000, 0xfa: 0x2000, + 0xfc: 0x2000, 0xfd: 0x2000, 0xfe: 0x2000, 0xff: 0x2000, + // Block 0x4, offset 0x100 + 0x106: 0x2000, + 0x110: 0x2000, + 0x117: 0x2000, + 0x118: 0x2000, + 0x11e: 0x2000, 0x11f: 0x2000, 0x120: 0x2000, 0x121: 0x2000, + 0x126: 0x2000, 0x128: 0x2000, 0x129: 0x2000, + 0x12a: 0x2000, 0x12c: 0x2000, 0x12d: 0x2000, + 0x130: 0x2000, 0x132: 0x2000, 0x133: 0x2000, + 0x137: 0x2000, 0x138: 0x2000, 0x139: 0x2000, 0x13a: 0x2000, + 0x13c: 0x2000, 0x13e: 0x2000, + // Block 0x5, offset 0x140 + 0x141: 0x2000, + 0x151: 0x2000, + 0x153: 0x2000, + 0x15b: 0x2000, + 0x166: 0x2000, 0x167: 0x2000, + 0x16b: 0x2000, + 0x171: 0x2000, 0x172: 0x2000, 0x173: 0x2000, + 0x178: 0x2000, + 0x17f: 0x2000, + // Block 0x6, offset 0x180 + 0x180: 0x2000, 0x181: 0x2000, 0x182: 0x2000, 0x184: 0x2000, + 0x188: 0x2000, 0x189: 0x2000, 0x18a: 0x2000, 0x18b: 0x2000, + 0x18d: 0x2000, + 0x192: 0x2000, 0x193: 0x2000, + 0x1a6: 0x2000, 0x1a7: 0x2000, + 0x1ab: 0x2000, + // Block 0x7, offset 0x1c0 + 0x1ce: 0x2000, 0x1d0: 0x2000, + 0x1d2: 0x2000, 0x1d4: 0x2000, 0x1d6: 0x2000, + 0x1d8: 0x2000, 0x1da: 0x2000, 0x1dc: 0x2000, + // Block 0x8, offset 0x200 + 0x211: 0x2000, + 0x221: 0x2000, + // Block 0x9, offset 0x240 + 0x244: 0x2000, + 0x247: 0x2000, 0x249: 0x2000, 0x24a: 0x2000, 0x24b: 0x2000, + 0x24d: 0x2000, 0x250: 0x2000, + 0x258: 0x2000, 0x259: 0x2000, 0x25a: 0x2000, 0x25b: 0x2000, 0x25d: 0x2000, + 0x25f: 0x2000, + // Block 0xa, offset 0x280 + 0x280: 0x2000, 0x281: 0x2000, 0x282: 0x2000, 0x283: 0x2000, 0x284: 0x2000, 0x285: 0x2000, + 0x286: 0x2000, 0x287: 0x2000, 0x288: 0x2000, 0x289: 0x2000, 0x28a: 0x2000, 0x28b: 0x2000, + 0x28c: 0x2000, 0x28d: 0x2000, 0x28e: 0x2000, 0x28f: 0x2000, 0x290: 0x2000, 0x291: 0x2000, + 0x292: 0x2000, 0x293: 0x2000, 0x294: 0x2000, 0x295: 0x2000, 0x296: 0x2000, 0x297: 0x2000, + 0x298: 0x2000, 0x299: 0x2000, 0x29a: 0x2000, 0x29b: 0x2000, 0x29c: 0x2000, 0x29d: 0x2000, + 0x29e: 0x2000, 0x29f: 0x2000, 0x2a0: 0x2000, 0x2a1: 0x2000, 0x2a2: 0x2000, 0x2a3: 0x2000, + 0x2a4: 0x2000, 0x2a5: 0x2000, 0x2a6: 0x2000, 0x2a7: 0x2000, 0x2a8: 0x2000, 0x2a9: 0x2000, + 0x2aa: 0x2000, 0x2ab: 0x2000, 0x2ac: 0x2000, 0x2ad: 0x2000, 0x2ae: 0x2000, 0x2af: 0x2000, + 0x2b0: 0x2000, 0x2b1: 0x2000, 0x2b2: 0x2000, 0x2b3: 0x2000, 0x2b4: 0x2000, 0x2b5: 0x2000, + 0x2b6: 0x2000, 0x2b7: 0x2000, 0x2b8: 0x2000, 0x2b9: 0x2000, 0x2ba: 0x2000, 0x2bb: 0x2000, + 0x2bc: 0x2000, 0x2bd: 0x2000, 0x2be: 0x2000, 0x2bf: 0x2000, + // Block 0xb, offset 0x2c0 + 0x2c0: 0x2000, 0x2c1: 0x2000, 0x2c2: 0x2000, 0x2c3: 0x2000, 0x2c4: 0x2000, 0x2c5: 0x2000, + 0x2c6: 0x2000, 0x2c7: 0x2000, 0x2c8: 0x2000, 0x2c9: 0x2000, 0x2ca: 0x2000, 0x2cb: 0x2000, + 0x2cc: 0x2000, 0x2cd: 0x2000, 0x2ce: 0x2000, 0x2cf: 0x2000, 0x2d0: 0x2000, 0x2d1: 0x2000, + 0x2d2: 0x2000, 0x2d3: 0x2000, 0x2d4: 0x2000, 0x2d5: 0x2000, 0x2d6: 0x2000, 0x2d7: 0x2000, + 0x2d8: 0x2000, 0x2d9: 0x2000, 0x2da: 0x2000, 0x2db: 0x2000, 0x2dc: 0x2000, 0x2dd: 0x2000, + 0x2de: 0x2000, 0x2df: 0x2000, 0x2e0: 0x2000, 0x2e1: 0x2000, 0x2e2: 0x2000, 0x2e3: 0x2000, + 0x2e4: 0x2000, 0x2e5: 0x2000, 0x2e6: 0x2000, 0x2e7: 0x2000, 0x2e8: 0x2000, 0x2e9: 0x2000, + 0x2ea: 0x2000, 0x2eb: 0x2000, 0x2ec: 0x2000, 0x2ed: 0x2000, 0x2ee: 0x2000, 0x2ef: 0x2000, + // Block 0xc, offset 0x300 + 0x311: 0x2000, + 0x312: 0x2000, 0x313: 0x2000, 0x314: 0x2000, 0x315: 0x2000, 0x316: 0x2000, 0x317: 0x2000, + 0x318: 0x2000, 0x319: 0x2000, 0x31a: 0x2000, 0x31b: 0x2000, 0x31c: 0x2000, 0x31d: 0x2000, + 0x31e: 0x2000, 0x31f: 0x2000, 0x320: 0x2000, 0x321: 0x2000, 0x323: 0x2000, + 0x324: 0x2000, 0x325: 0x2000, 0x326: 0x2000, 0x327: 0x2000, 0x328: 0x2000, 0x329: 0x2000, + 0x331: 0x2000, 0x332: 0x2000, 0x333: 0x2000, 0x334: 0x2000, 0x335: 0x2000, + 0x336: 0x2000, 0x337: 0x2000, 0x338: 0x2000, 0x339: 0x2000, 0x33a: 0x2000, 0x33b: 0x2000, + 0x33c: 0x2000, 0x33d: 0x2000, 0x33e: 0x2000, 0x33f: 0x2000, + // Block 0xd, offset 0x340 + 0x340: 0x2000, 0x341: 0x2000, 0x343: 0x2000, 0x344: 0x2000, 0x345: 0x2000, + 0x346: 0x2000, 0x347: 0x2000, 0x348: 0x2000, 0x349: 0x2000, + // Block 0xe, offset 0x380 + 0x381: 0x2000, + 0x390: 0x2000, 0x391: 0x2000, + 0x392: 0x2000, 0x393: 0x2000, 0x394: 0x2000, 0x395: 0x2000, 0x396: 0x2000, 0x397: 0x2000, + 0x398: 0x2000, 0x399: 0x2000, 0x39a: 0x2000, 0x39b: 0x2000, 0x39c: 0x2000, 0x39d: 0x2000, + 0x39e: 0x2000, 0x39f: 0x2000, 0x3a0: 0x2000, 0x3a1: 0x2000, 0x3a2: 0x2000, 0x3a3: 0x2000, + 0x3a4: 0x2000, 0x3a5: 0x2000, 0x3a6: 0x2000, 0x3a7: 0x2000, 0x3a8: 0x2000, 0x3a9: 0x2000, + 0x3aa: 0x2000, 0x3ab: 0x2000, 0x3ac: 0x2000, 0x3ad: 0x2000, 0x3ae: 0x2000, 0x3af: 0x2000, + 0x3b0: 0x2000, 0x3b1: 0x2000, 0x3b2: 0x2000, 0x3b3: 0x2000, 0x3b4: 0x2000, 0x3b5: 0x2000, + 0x3b6: 0x2000, 0x3b7: 0x2000, 0x3b8: 0x2000, 0x3b9: 0x2000, 0x3ba: 0x2000, 0x3bb: 0x2000, + 0x3bc: 0x2000, 0x3bd: 0x2000, 0x3be: 0x2000, 0x3bf: 0x2000, + // Block 0xf, offset 0x3c0 + 0x3c0: 0x2000, 0x3c1: 0x2000, 0x3c2: 0x2000, 0x3c3: 0x2000, 0x3c4: 0x2000, 0x3c5: 0x2000, + 0x3c6: 0x2000, 0x3c7: 0x2000, 0x3c8: 0x2000, 0x3c9: 0x2000, 0x3ca: 0x2000, 0x3cb: 0x2000, + 0x3cc: 0x2000, 0x3cd: 0x2000, 0x3ce: 0x2000, 0x3cf: 0x2000, 0x3d1: 0x2000, + // Block 0x10, offset 0x400 + 0x400: 0x4000, 0x401: 0x4000, 0x402: 0x4000, 0x403: 0x4000, 0x404: 0x4000, 0x405: 0x4000, + 0x406: 0x4000, 0x407: 0x4000, 0x408: 0x4000, 0x409: 0x4000, 0x40a: 0x4000, 0x40b: 0x4000, + 0x40c: 0x4000, 0x40d: 0x4000, 0x40e: 0x4000, 0x40f: 0x4000, 0x410: 0x4000, 0x411: 0x4000, + 0x412: 0x4000, 0x413: 0x4000, 0x414: 0x4000, 0x415: 0x4000, 0x416: 0x4000, 0x417: 0x4000, + 0x418: 0x4000, 0x419: 0x4000, 0x41a: 0x4000, 0x41b: 0x4000, 0x41c: 0x4000, 0x41d: 0x4000, + 0x41e: 0x4000, 0x41f: 0x4000, 0x420: 0x4000, 0x421: 0x4000, 0x422: 0x4000, 0x423: 0x4000, + 0x424: 0x4000, 0x425: 0x4000, 0x426: 0x4000, 0x427: 0x4000, 0x428: 0x4000, 0x429: 0x4000, + 0x42a: 0x4000, 0x42b: 0x4000, 0x42c: 0x4000, 0x42d: 0x4000, 0x42e: 0x4000, 0x42f: 0x4000, + 0x430: 0x4000, 0x431: 0x4000, 0x432: 0x4000, 0x433: 0x4000, 0x434: 0x4000, 0x435: 0x4000, + 0x436: 0x4000, 0x437: 0x4000, 0x438: 0x4000, 0x439: 0x4000, 0x43a: 0x4000, 0x43b: 0x4000, + 0x43c: 0x4000, 0x43d: 0x4000, 0x43e: 0x4000, 0x43f: 0x4000, + // Block 0x11, offset 0x440 + 0x440: 0x4000, 0x441: 0x4000, 0x442: 0x4000, 0x443: 0x4000, 0x444: 0x4000, 0x445: 0x4000, + 0x446: 0x4000, 0x447: 0x4000, 0x448: 0x4000, 0x449: 0x4000, 0x44a: 0x4000, 0x44b: 0x4000, + 0x44c: 0x4000, 0x44d: 0x4000, 0x44e: 0x4000, 0x44f: 0x4000, 0x450: 0x4000, 0x451: 0x4000, + 0x452: 0x4000, 0x453: 0x4000, 0x454: 0x4000, 0x455: 0x4000, 0x456: 0x4000, 0x457: 0x4000, + 0x458: 0x4000, 0x459: 0x4000, 0x45a: 0x4000, 0x45b: 0x4000, 0x45c: 0x4000, 0x45d: 0x4000, + 0x45e: 0x4000, 0x45f: 0x4000, + // Block 0x12, offset 0x480 + 0x490: 0x2000, + 0x493: 0x2000, 0x494: 0x2000, 0x495: 0x2000, 0x496: 0x2000, + 0x498: 0x2000, 0x499: 0x2000, 0x49c: 0x2000, 0x49d: 0x2000, + 0x4a0: 0x2000, 0x4a1: 0x2000, 0x4a2: 0x2000, + 0x4a4: 0x2000, 0x4a5: 0x2000, 0x4a6: 0x2000, 0x4a7: 0x2000, + 0x4b0: 0x2000, 0x4b2: 0x2000, 0x4b3: 0x2000, 0x4b5: 0x2000, + 0x4bb: 0x2000, + 0x4be: 0x2000, + // Block 0x13, offset 0x4c0 + 0x4f4: 0x2000, + 0x4ff: 0x2000, + // Block 0x14, offset 0x500 + 0x501: 0x2000, 0x502: 0x2000, 0x503: 0x2000, 0x504: 0x2000, + 0x529: 0xa009, + 0x52c: 0x2000, + // Block 0x15, offset 0x540 + 0x543: 0x2000, 0x545: 0x2000, + 0x549: 0x2000, + 0x553: 0x2000, 0x556: 0x2000, + 0x561: 0x2000, 0x562: 0x2000, + 0x566: 0x2000, + 0x56b: 0x2000, + // Block 0x16, offset 0x580 + 0x593: 0x2000, 0x594: 0x2000, + 0x59b: 0x2000, 0x59c: 0x2000, 0x59d: 0x2000, + 0x59e: 0x2000, 0x5a0: 0x2000, 0x5a1: 0x2000, 0x5a2: 0x2000, 0x5a3: 0x2000, + 0x5a4: 0x2000, 0x5a5: 0x2000, 0x5a6: 0x2000, 0x5a7: 0x2000, 0x5a8: 0x2000, 0x5a9: 0x2000, + 0x5aa: 0x2000, 0x5ab: 0x2000, + 0x5b0: 0x2000, 0x5b1: 0x2000, 0x5b2: 0x2000, 0x5b3: 0x2000, 0x5b4: 0x2000, 0x5b5: 0x2000, + 0x5b6: 0x2000, 0x5b7: 0x2000, 0x5b8: 0x2000, 0x5b9: 0x2000, + // Block 0x17, offset 0x5c0 + 0x5c9: 0x2000, + 0x5d0: 0x200a, 0x5d1: 0x200b, + 0x5d2: 0x200a, 0x5d3: 0x200c, 0x5d4: 0x2000, 0x5d5: 0x2000, 0x5d6: 0x2000, 0x5d7: 0x2000, + 0x5d8: 0x2000, 0x5d9: 0x2000, + 0x5f8: 0x2000, 0x5f9: 0x2000, + // Block 0x18, offset 0x600 + 0x612: 0x2000, 0x614: 0x2000, + 0x627: 0x2000, + // Block 0x19, offset 0x640 + 0x640: 0x2000, 0x642: 0x2000, 0x643: 0x2000, + 0x647: 0x2000, 0x648: 0x2000, 0x64b: 0x2000, + 0x64f: 0x2000, 0x651: 0x2000, + 0x655: 0x2000, + 0x65a: 0x2000, 0x65d: 0x2000, + 0x65e: 0x2000, 0x65f: 0x2000, 0x660: 0x2000, 0x663: 0x2000, + 0x665: 0x2000, 0x667: 0x2000, 0x668: 0x2000, 0x669: 0x2000, + 0x66a: 0x2000, 0x66b: 0x2000, 0x66c: 0x2000, 0x66e: 0x2000, + 0x674: 0x2000, 0x675: 0x2000, + 0x676: 0x2000, 0x677: 0x2000, + 0x67c: 0x2000, 0x67d: 0x2000, + // Block 0x1a, offset 0x680 + 0x688: 0x2000, + 0x68c: 0x2000, + 0x692: 0x2000, + 0x6a0: 0x2000, 0x6a1: 0x2000, + 0x6a4: 0x2000, 0x6a5: 0x2000, 0x6a6: 0x2000, 0x6a7: 0x2000, + 0x6aa: 0x2000, 0x6ab: 0x2000, 0x6ae: 0x2000, 0x6af: 0x2000, + // Block 0x1b, offset 0x6c0 + 0x6c2: 0x2000, 0x6c3: 0x2000, + 0x6c6: 0x2000, 0x6c7: 0x2000, + 0x6d5: 0x2000, + 0x6d9: 0x2000, + 0x6e5: 0x2000, + 0x6ff: 0x2000, + // Block 0x1c, offset 0x700 + 0x712: 0x2000, + 0x71a: 0x4000, 0x71b: 0x4000, + 0x729: 0x4000, + 0x72a: 0x4000, + // Block 0x1d, offset 0x740 + 0x769: 0x4000, + 0x76a: 0x4000, 0x76b: 0x4000, 0x76c: 0x4000, + 0x770: 0x4000, 0x773: 0x4000, + // Block 0x1e, offset 0x780 + 0x7a0: 0x2000, 0x7a1: 0x2000, 0x7a2: 0x2000, 0x7a3: 0x2000, + 0x7a4: 0x2000, 0x7a5: 0x2000, 0x7a6: 0x2000, 0x7a7: 0x2000, 0x7a8: 0x2000, 0x7a9: 0x2000, + 0x7aa: 0x2000, 0x7ab: 0x2000, 0x7ac: 0x2000, 0x7ad: 0x2000, 0x7ae: 0x2000, 0x7af: 0x2000, + 0x7b0: 0x2000, 0x7b1: 0x2000, 0x7b2: 0x2000, 0x7b3: 0x2000, 0x7b4: 0x2000, 0x7b5: 0x2000, + 0x7b6: 0x2000, 0x7b7: 0x2000, 0x7b8: 0x2000, 0x7b9: 0x2000, 0x7ba: 0x2000, 0x7bb: 0x2000, + 0x7bc: 0x2000, 0x7bd: 0x2000, 0x7be: 0x2000, 0x7bf: 0x2000, + // Block 0x1f, offset 0x7c0 + 0x7c0: 0x2000, 0x7c1: 0x2000, 0x7c2: 0x2000, 0x7c3: 0x2000, 0x7c4: 0x2000, 0x7c5: 0x2000, + 0x7c6: 0x2000, 0x7c7: 0x2000, 0x7c8: 0x2000, 0x7c9: 0x2000, 0x7ca: 0x2000, 0x7cb: 0x2000, + 0x7cc: 0x2000, 0x7cd: 0x2000, 0x7ce: 0x2000, 0x7cf: 0x2000, 0x7d0: 0x2000, 0x7d1: 0x2000, + 0x7d2: 0x2000, 0x7d3: 0x2000, 0x7d4: 0x2000, 0x7d5: 0x2000, 0x7d6: 0x2000, 0x7d7: 0x2000, + 0x7d8: 0x2000, 0x7d9: 0x2000, 0x7da: 0x2000, 0x7db: 0x2000, 0x7dc: 0x2000, 0x7dd: 0x2000, + 0x7de: 0x2000, 0x7df: 0x2000, 0x7e0: 0x2000, 0x7e1: 0x2000, 0x7e2: 0x2000, 0x7e3: 0x2000, + 0x7e4: 0x2000, 0x7e5: 0x2000, 0x7e6: 0x2000, 0x7e7: 0x2000, 0x7e8: 0x2000, 0x7e9: 0x2000, + 0x7eb: 0x2000, 0x7ec: 0x2000, 0x7ed: 0x2000, 0x7ee: 0x2000, 0x7ef: 0x2000, + 0x7f0: 0x2000, 0x7f1: 0x2000, 0x7f2: 0x2000, 0x7f3: 0x2000, 0x7f4: 0x2000, 0x7f5: 0x2000, + 0x7f6: 0x2000, 0x7f7: 0x2000, 0x7f8: 0x2000, 0x7f9: 0x2000, 0x7fa: 0x2000, 0x7fb: 0x2000, + 0x7fc: 0x2000, 0x7fd: 0x2000, 0x7fe: 0x2000, 0x7ff: 0x2000, + // Block 0x20, offset 0x800 + 0x800: 0x2000, 0x801: 0x2000, 0x802: 0x200d, 0x803: 0x2000, 0x804: 0x2000, 0x805: 0x2000, + 0x806: 0x2000, 0x807: 0x2000, 0x808: 0x2000, 0x809: 0x2000, 0x80a: 0x2000, 0x80b: 0x2000, + 0x80c: 0x2000, 0x80d: 0x2000, 0x80e: 0x2000, 0x80f: 0x2000, 0x810: 0x2000, 0x811: 0x2000, + 0x812: 0x2000, 0x813: 0x2000, 0x814: 0x2000, 0x815: 0x2000, 0x816: 0x2000, 0x817: 0x2000, + 0x818: 0x2000, 0x819: 0x2000, 0x81a: 0x2000, 0x81b: 0x2000, 0x81c: 0x2000, 0x81d: 0x2000, + 0x81e: 0x2000, 0x81f: 0x2000, 0x820: 0x2000, 0x821: 0x2000, 0x822: 0x2000, 0x823: 0x2000, + 0x824: 0x2000, 0x825: 0x2000, 0x826: 0x2000, 0x827: 0x2000, 0x828: 0x2000, 0x829: 0x2000, + 0x82a: 0x2000, 0x82b: 0x2000, 0x82c: 0x2000, 0x82d: 0x2000, 0x82e: 0x2000, 0x82f: 0x2000, + 0x830: 0x2000, 0x831: 0x2000, 0x832: 0x2000, 0x833: 0x2000, 0x834: 0x2000, 0x835: 0x2000, + 0x836: 0x2000, 0x837: 0x2000, 0x838: 0x2000, 0x839: 0x2000, 0x83a: 0x2000, 0x83b: 0x2000, + 0x83c: 0x2000, 0x83d: 0x2000, 0x83e: 0x2000, 0x83f: 0x2000, + // Block 0x21, offset 0x840 + 0x840: 0x2000, 0x841: 0x2000, 0x842: 0x2000, 0x843: 0x2000, 0x844: 0x2000, 0x845: 0x2000, + 0x846: 0x2000, 0x847: 0x2000, 0x848: 0x2000, 0x849: 0x2000, 0x84a: 0x2000, 0x84b: 0x2000, + 0x850: 0x2000, 0x851: 0x2000, + 0x852: 0x2000, 0x853: 0x2000, 0x854: 0x2000, 0x855: 0x2000, 0x856: 0x2000, 0x857: 0x2000, + 0x858: 0x2000, 0x859: 0x2000, 0x85a: 0x2000, 0x85b: 0x2000, 0x85c: 0x2000, 0x85d: 0x2000, + 0x85e: 0x2000, 0x85f: 0x2000, 0x860: 0x2000, 0x861: 0x2000, 0x862: 0x2000, 0x863: 0x2000, + 0x864: 0x2000, 0x865: 0x2000, 0x866: 0x2000, 0x867: 0x2000, 0x868: 0x2000, 0x869: 0x2000, + 0x86a: 0x2000, 0x86b: 0x2000, 0x86c: 0x2000, 0x86d: 0x2000, 0x86e: 0x2000, 0x86f: 0x2000, + 0x870: 0x2000, 0x871: 0x2000, 0x872: 0x2000, 0x873: 0x2000, + // Block 0x22, offset 0x880 + 0x880: 0x2000, 0x881: 0x2000, 0x882: 0x2000, 0x883: 0x2000, 0x884: 0x2000, 0x885: 0x2000, + 0x886: 0x2000, 0x887: 0x2000, 0x888: 0x2000, 0x889: 0x2000, 0x88a: 0x2000, 0x88b: 0x2000, + 0x88c: 0x2000, 0x88d: 0x2000, 0x88e: 0x2000, 0x88f: 0x2000, + 0x892: 0x2000, 0x893: 0x2000, 0x894: 0x2000, 0x895: 0x2000, + 0x8a0: 0x200e, 0x8a1: 0x2000, 0x8a3: 0x2000, + 0x8a4: 0x2000, 0x8a5: 0x2000, 0x8a6: 0x2000, 0x8a7: 0x2000, 0x8a8: 0x2000, 0x8a9: 0x2000, + 0x8b2: 0x2000, 0x8b3: 0x2000, + 0x8b6: 0x2000, 0x8b7: 0x2000, + 0x8bc: 0x2000, 0x8bd: 0x2000, + // Block 0x23, offset 0x8c0 + 0x8c0: 0x2000, 0x8c1: 0x2000, + 0x8c6: 0x2000, 0x8c7: 0x2000, 0x8c8: 0x2000, 0x8cb: 0x200f, + 0x8ce: 0x2000, 0x8cf: 0x2000, 0x8d0: 0x2000, 0x8d1: 0x2000, + 0x8e2: 0x2000, 0x8e3: 0x2000, + 0x8e4: 0x2000, 0x8e5: 0x2000, + 0x8ef: 0x2000, + 0x8fd: 0x4000, 0x8fe: 0x4000, + // Block 0x24, offset 0x900 + 0x905: 0x2000, + 0x906: 0x2000, 0x909: 0x2000, + 0x90e: 0x2000, 0x90f: 0x2000, + 0x914: 0x4000, 0x915: 0x4000, + 0x91c: 0x2000, + 0x91e: 0x2000, + // Block 0x25, offset 0x940 + 0x940: 0x2000, 0x942: 0x2000, + 0x948: 0x4000, 0x949: 0x4000, 0x94a: 0x4000, 0x94b: 0x4000, + 0x94c: 0x4000, 0x94d: 0x4000, 0x94e: 0x4000, 0x94f: 0x4000, 0x950: 0x4000, 0x951: 0x4000, + 0x952: 0x4000, 0x953: 0x4000, + 0x960: 0x2000, 0x961: 0x2000, 0x963: 0x2000, + 0x964: 0x2000, 0x965: 0x2000, 0x967: 0x2000, 0x968: 0x2000, 0x969: 0x2000, + 0x96a: 0x2000, 0x96c: 0x2000, 0x96d: 0x2000, 0x96f: 0x2000, + 0x97f: 0x4000, + // Block 0x26, offset 0x980 + 0x993: 0x4000, + 0x99e: 0x2000, 0x99f: 0x2000, 0x9a1: 0x4000, + 0x9aa: 0x4000, 0x9ab: 0x4000, + 0x9bd: 0x4000, 0x9be: 0x4000, 0x9bf: 0x2000, + // Block 0x27, offset 0x9c0 + 0x9c4: 0x4000, 0x9c5: 0x4000, + 0x9c6: 0x2000, 0x9c7: 0x2000, 0x9c8: 0x2000, 0x9c9: 0x2000, 0x9ca: 0x2000, 0x9cb: 0x2000, + 0x9cc: 0x2000, 0x9cd: 0x2000, 0x9ce: 0x4000, 0x9cf: 0x2000, 0x9d0: 0x2000, 0x9d1: 0x2000, + 0x9d2: 0x2000, 0x9d3: 0x2000, 0x9d4: 0x4000, 0x9d5: 0x2000, 0x9d6: 0x2000, 0x9d7: 0x2000, + 0x9d8: 0x2000, 0x9d9: 0x2000, 0x9da: 0x2000, 0x9db: 0x2000, 0x9dc: 0x2000, 0x9dd: 0x2000, + 0x9de: 0x2000, 0x9df: 0x2000, 0x9e0: 0x2000, 0x9e1: 0x2000, 0x9e3: 0x2000, + 0x9e8: 0x2000, 0x9e9: 0x2000, + 0x9ea: 0x4000, 0x9eb: 0x2000, 0x9ec: 0x2000, 0x9ed: 0x2000, 0x9ee: 0x2000, 0x9ef: 0x2000, + 0x9f0: 0x2000, 0x9f1: 0x2000, 0x9f2: 0x4000, 0x9f3: 0x4000, 0x9f4: 0x2000, 0x9f5: 0x4000, + 0x9f6: 0x2000, 0x9f7: 0x2000, 0x9f8: 0x2000, 0x9f9: 0x2000, 0x9fa: 0x4000, 0x9fb: 0x2000, + 0x9fc: 0x2000, 0x9fd: 0x4000, 0x9fe: 0x2000, 0x9ff: 0x2000, + // Block 0x28, offset 0xa00 + 0xa05: 0x4000, + 0xa0a: 0x4000, 0xa0b: 0x4000, + 0xa28: 0x4000, + 0xa3d: 0x2000, + // Block 0x29, offset 0xa40 + 0xa4c: 0x4000, 0xa4e: 0x4000, + 0xa53: 0x4000, 0xa54: 0x4000, 0xa55: 0x4000, 0xa57: 0x4000, + 0xa76: 0x2000, 0xa77: 0x2000, 0xa78: 0x2000, 0xa79: 0x2000, 0xa7a: 0x2000, 0xa7b: 0x2000, + 0xa7c: 0x2000, 0xa7d: 0x2000, 0xa7e: 0x2000, 0xa7f: 0x2000, + // Block 0x2a, offset 0xa80 + 0xa95: 0x4000, 0xa96: 0x4000, 0xa97: 0x4000, + 0xab0: 0x4000, + 0xabf: 0x4000, + // Block 0x2b, offset 0xac0 + 0xae6: 0x6000, 0xae7: 0x6000, 0xae8: 0x6000, 0xae9: 0x6000, + 0xaea: 0x6000, 0xaeb: 0x6000, 0xaec: 0x6000, 0xaed: 0x6000, + // Block 0x2c, offset 0xb00 + 0xb05: 0x6010, + 0xb06: 0x6011, + // Block 0x2d, offset 0xb40 + 0xb5b: 0x4000, 0xb5c: 0x4000, + // Block 0x2e, offset 0xb80 + 0xb90: 0x4000, + 0xb95: 0x4000, 0xb96: 0x2000, 0xb97: 0x2000, + 0xb98: 0x2000, 0xb99: 0x2000, + // Block 0x2f, offset 0xbc0 + 0xbc0: 0x4000, 0xbc1: 0x4000, 0xbc2: 0x4000, 0xbc3: 0x4000, 0xbc4: 0x4000, 0xbc5: 0x4000, + 0xbc6: 0x4000, 0xbc7: 0x4000, 0xbc8: 0x4000, 0xbc9: 0x4000, 0xbca: 0x4000, 0xbcb: 0x4000, + 0xbcc: 0x4000, 0xbcd: 0x4000, 0xbce: 0x4000, 0xbcf: 0x4000, 0xbd0: 0x4000, 0xbd1: 0x4000, + 0xbd2: 0x4000, 0xbd3: 0x4000, 0xbd4: 0x4000, 0xbd5: 0x4000, 0xbd6: 0x4000, 0xbd7: 0x4000, + 0xbd8: 0x4000, 0xbd9: 0x4000, 0xbdb: 0x4000, 0xbdc: 0x4000, 0xbdd: 0x4000, + 0xbde: 0x4000, 0xbdf: 0x4000, 0xbe0: 0x4000, 0xbe1: 0x4000, 0xbe2: 0x4000, 0xbe3: 0x4000, + 0xbe4: 0x4000, 0xbe5: 0x4000, 0xbe6: 0x4000, 0xbe7: 0x4000, 0xbe8: 0x4000, 0xbe9: 0x4000, + 0xbea: 0x4000, 0xbeb: 0x4000, 0xbec: 0x4000, 0xbed: 0x4000, 0xbee: 0x4000, 0xbef: 0x4000, + 0xbf0: 0x4000, 0xbf1: 0x4000, 0xbf2: 0x4000, 0xbf3: 0x4000, 0xbf4: 0x4000, 0xbf5: 0x4000, + 0xbf6: 0x4000, 0xbf7: 0x4000, 0xbf8: 0x4000, 0xbf9: 0x4000, 0xbfa: 0x4000, 0xbfb: 0x4000, + 0xbfc: 0x4000, 0xbfd: 0x4000, 0xbfe: 0x4000, 0xbff: 0x4000, + // Block 0x30, offset 0xc00 + 0xc00: 0x4000, 0xc01: 0x4000, 0xc02: 0x4000, 0xc03: 0x4000, 0xc04: 0x4000, 0xc05: 0x4000, + 0xc06: 0x4000, 0xc07: 0x4000, 0xc08: 0x4000, 0xc09: 0x4000, 0xc0a: 0x4000, 0xc0b: 0x4000, + 0xc0c: 0x4000, 0xc0d: 0x4000, 0xc0e: 0x4000, 0xc0f: 0x4000, 0xc10: 0x4000, 0xc11: 0x4000, + 0xc12: 0x4000, 0xc13: 0x4000, 0xc14: 0x4000, 0xc15: 0x4000, 0xc16: 0x4000, 0xc17: 0x4000, + 0xc18: 0x4000, 0xc19: 0x4000, 0xc1a: 0x4000, 0xc1b: 0x4000, 0xc1c: 0x4000, 0xc1d: 0x4000, + 0xc1e: 0x4000, 0xc1f: 0x4000, 0xc20: 0x4000, 0xc21: 0x4000, 0xc22: 0x4000, 0xc23: 0x4000, + 0xc24: 0x4000, 0xc25: 0x4000, 0xc26: 0x4000, 0xc27: 0x4000, 0xc28: 0x4000, 0xc29: 0x4000, + 0xc2a: 0x4000, 0xc2b: 0x4000, 0xc2c: 0x4000, 0xc2d: 0x4000, 0xc2e: 0x4000, 0xc2f: 0x4000, + 0xc30: 0x4000, 0xc31: 0x4000, 0xc32: 0x4000, 0xc33: 0x4000, + // Block 0x31, offset 0xc40 + 0xc40: 0x4000, 0xc41: 0x4000, 0xc42: 0x4000, 0xc43: 0x4000, 0xc44: 0x4000, 0xc45: 0x4000, + 0xc46: 0x4000, 0xc47: 0x4000, 0xc48: 0x4000, 0xc49: 0x4000, 0xc4a: 0x4000, 0xc4b: 0x4000, + 0xc4c: 0x4000, 0xc4d: 0x4000, 0xc4e: 0x4000, 0xc4f: 0x4000, 0xc50: 0x4000, 0xc51: 0x4000, + 0xc52: 0x4000, 0xc53: 0x4000, 0xc54: 0x4000, 0xc55: 0x4000, + 0xc70: 0x4000, 0xc71: 0x4000, 0xc72: 0x4000, 0xc73: 0x4000, 0xc74: 0x4000, 0xc75: 0x4000, + 0xc76: 0x4000, 0xc77: 0x4000, 0xc78: 0x4000, 0xc79: 0x4000, 0xc7a: 0x4000, 0xc7b: 0x4000, + // Block 0x32, offset 0xc80 + 0xc80: 0x9012, 0xc81: 0x4013, 0xc82: 0x4014, 0xc83: 0x4000, 0xc84: 0x4000, 0xc85: 0x4000, + 0xc86: 0x4000, 0xc87: 0x4000, 0xc88: 0x4000, 0xc89: 0x4000, 0xc8a: 0x4000, 0xc8b: 0x4000, + 0xc8c: 0x4015, 0xc8d: 0x4015, 0xc8e: 0x4000, 0xc8f: 0x4000, 0xc90: 0x4000, 0xc91: 0x4000, + 0xc92: 0x4000, 0xc93: 0x4000, 0xc94: 0x4000, 0xc95: 0x4000, 0xc96: 0x4000, 0xc97: 0x4000, + 0xc98: 0x4000, 0xc99: 0x4000, 0xc9a: 0x4000, 0xc9b: 0x4000, 0xc9c: 0x4000, 0xc9d: 0x4000, + 0xc9e: 0x4000, 0xc9f: 0x4000, 0xca0: 0x4000, 0xca1: 0x4000, 0xca2: 0x4000, 0xca3: 0x4000, + 0xca4: 0x4000, 0xca5: 0x4000, 0xca6: 0x4000, 0xca7: 0x4000, 0xca8: 0x4000, 0xca9: 0x4000, + 0xcaa: 0x4000, 0xcab: 0x4000, 0xcac: 0x4000, 0xcad: 0x4000, 0xcae: 0x4000, 0xcaf: 0x4000, + 0xcb0: 0x4000, 0xcb1: 0x4000, 0xcb2: 0x4000, 0xcb3: 0x4000, 0xcb4: 0x4000, 0xcb5: 0x4000, + 0xcb6: 0x4000, 0xcb7: 0x4000, 0xcb8: 0x4000, 0xcb9: 0x4000, 0xcba: 0x4000, 0xcbb: 0x4000, + 0xcbc: 0x4000, 0xcbd: 0x4000, 0xcbe: 0x4000, + // Block 0x33, offset 0xcc0 + 0xcc1: 0x4000, 0xcc2: 0x4000, 0xcc3: 0x4000, 0xcc4: 0x4000, 0xcc5: 0x4000, + 0xcc6: 0x4000, 0xcc7: 0x4000, 0xcc8: 0x4000, 0xcc9: 0x4000, 0xcca: 0x4000, 0xccb: 0x4000, + 0xccc: 0x4000, 0xccd: 0x4000, 0xcce: 0x4000, 0xccf: 0x4000, 0xcd0: 0x4000, 0xcd1: 0x4000, + 0xcd2: 0x4000, 0xcd3: 0x4000, 0xcd4: 0x4000, 0xcd5: 0x4000, 0xcd6: 0x4000, 0xcd7: 0x4000, + 0xcd8: 0x4000, 0xcd9: 0x4000, 0xcda: 0x4000, 0xcdb: 0x4000, 0xcdc: 0x4000, 0xcdd: 0x4000, + 0xcde: 0x4000, 0xcdf: 0x4000, 0xce0: 0x4000, 0xce1: 0x4000, 0xce2: 0x4000, 0xce3: 0x4000, + 0xce4: 0x4000, 0xce5: 0x4000, 0xce6: 0x4000, 0xce7: 0x4000, 0xce8: 0x4000, 0xce9: 0x4000, + 0xcea: 0x4000, 0xceb: 0x4000, 0xcec: 0x4000, 0xced: 0x4000, 0xcee: 0x4000, 0xcef: 0x4000, + 0xcf0: 0x4000, 0xcf1: 0x4000, 0xcf2: 0x4000, 0xcf3: 0x4000, 0xcf4: 0x4000, 0xcf5: 0x4000, + 0xcf6: 0x4000, 0xcf7: 0x4000, 0xcf8: 0x4000, 0xcf9: 0x4000, 0xcfa: 0x4000, 0xcfb: 0x4000, + 0xcfc: 0x4000, 0xcfd: 0x4000, 0xcfe: 0x4000, 0xcff: 0x4000, + // Block 0x34, offset 0xd00 + 0xd00: 0x4000, 0xd01: 0x4000, 0xd02: 0x4000, 0xd03: 0x4000, 0xd04: 0x4000, 0xd05: 0x4000, + 0xd06: 0x4000, 0xd07: 0x4000, 0xd08: 0x4000, 0xd09: 0x4000, 0xd0a: 0x4000, 0xd0b: 0x4000, + 0xd0c: 0x4000, 0xd0d: 0x4000, 0xd0e: 0x4000, 0xd0f: 0x4000, 0xd10: 0x4000, 0xd11: 0x4000, + 0xd12: 0x4000, 0xd13: 0x4000, 0xd14: 0x4000, 0xd15: 0x4000, 0xd16: 0x4000, + 0xd19: 0x4016, 0xd1a: 0x4017, 0xd1b: 0x4000, 0xd1c: 0x4000, 0xd1d: 0x4000, + 0xd1e: 0x4000, 0xd1f: 0x4000, 0xd20: 0x4000, 0xd21: 0x4018, 0xd22: 0x4019, 0xd23: 0x401a, + 0xd24: 0x401b, 0xd25: 0x401c, 0xd26: 0x401d, 0xd27: 0x401e, 0xd28: 0x401f, 0xd29: 0x4020, + 0xd2a: 0x4021, 0xd2b: 0x4022, 0xd2c: 0x4000, 0xd2d: 0x4010, 0xd2e: 0x4000, 0xd2f: 0x4023, + 0xd30: 0x4000, 0xd31: 0x4024, 0xd32: 0x4000, 0xd33: 0x4025, 0xd34: 0x4000, 0xd35: 0x4026, + 0xd36: 0x4000, 0xd37: 0x401a, 0xd38: 0x4000, 0xd39: 0x4027, 0xd3a: 0x4000, 0xd3b: 0x4028, + 0xd3c: 0x4000, 0xd3d: 0x4020, 0xd3e: 0x4000, 0xd3f: 0x4029, + // Block 0x35, offset 0xd40 + 0xd40: 0x4000, 0xd41: 0x402a, 0xd42: 0x4000, 0xd43: 0x402b, 0xd44: 0x402c, 0xd45: 0x4000, + 0xd46: 0x4017, 0xd47: 0x4000, 0xd48: 0x402d, 0xd49: 0x4000, 0xd4a: 0x402e, 0xd4b: 0x402f, + 0xd4c: 0x4030, 0xd4d: 0x4017, 0xd4e: 0x4016, 0xd4f: 0x4017, 0xd50: 0x4000, 0xd51: 0x4000, + 0xd52: 0x4031, 0xd53: 0x4000, 0xd54: 0x4000, 0xd55: 0x4031, 0xd56: 0x4000, 0xd57: 0x4000, + 0xd58: 0x4032, 0xd59: 0x4000, 0xd5a: 0x4000, 0xd5b: 0x4032, 0xd5c: 0x4000, 0xd5d: 0x4000, + 0xd5e: 0x4033, 0xd5f: 0x402e, 0xd60: 0x4034, 0xd61: 0x4035, 0xd62: 0x4034, 0xd63: 0x4036, + 0xd64: 0x4037, 0xd65: 0x4024, 0xd66: 0x4035, 0xd67: 0x4025, 0xd68: 0x4038, 0xd69: 0x4038, + 0xd6a: 0x4039, 0xd6b: 0x4039, 0xd6c: 0x403a, 0xd6d: 0x403a, 0xd6e: 0x4000, 0xd6f: 0x4035, + 0xd70: 0x4000, 0xd71: 0x4000, 0xd72: 0x403b, 0xd73: 0x403c, 0xd74: 0x4000, 0xd75: 0x4000, + 0xd76: 0x4000, 0xd77: 0x4000, 0xd78: 0x4000, 0xd79: 0x4000, 0xd7a: 0x4000, 0xd7b: 0x403d, + 0xd7c: 0x401c, 0xd7d: 0x4000, 0xd7e: 0x4000, 0xd7f: 0x4000, + // Block 0x36, offset 0xd80 + 0xd85: 0x4000, + 0xd86: 0x4000, 0xd87: 0x4000, 0xd88: 0x4000, 0xd89: 0x4000, 0xd8a: 0x4000, 0xd8b: 0x4000, + 0xd8c: 0x4000, 0xd8d: 0x4000, 0xd8e: 0x4000, 0xd8f: 0x4000, 0xd90: 0x4000, 0xd91: 0x4000, + 0xd92: 0x4000, 0xd93: 0x4000, 0xd94: 0x4000, 0xd95: 0x4000, 0xd96: 0x4000, 0xd97: 0x4000, + 0xd98: 0x4000, 0xd99: 0x4000, 0xd9a: 0x4000, 0xd9b: 0x4000, 0xd9c: 0x4000, 0xd9d: 0x4000, + 0xd9e: 0x4000, 0xd9f: 0x4000, 0xda0: 0x4000, 0xda1: 0x4000, 0xda2: 0x4000, 0xda3: 0x4000, + 0xda4: 0x4000, 0xda5: 0x4000, 0xda6: 0x4000, 0xda7: 0x4000, 0xda8: 0x4000, 0xda9: 0x4000, + 0xdaa: 0x4000, 0xdab: 0x4000, 0xdac: 0x4000, 0xdad: 0x4000, 0xdae: 0x4000, 0xdaf: 0x4000, + 0xdb1: 0x403e, 0xdb2: 0x403e, 0xdb3: 0x403e, 0xdb4: 0x403e, 0xdb5: 0x403e, + 0xdb6: 0x403e, 0xdb7: 0x403e, 0xdb8: 0x403e, 0xdb9: 0x403e, 0xdba: 0x403e, 0xdbb: 0x403e, + 0xdbc: 0x403e, 0xdbd: 0x403e, 0xdbe: 0x403e, 0xdbf: 0x403e, + // Block 0x37, offset 0xdc0 + 0xdc0: 0x4037, 0xdc1: 0x4037, 0xdc2: 0x4037, 0xdc3: 0x4037, 0xdc4: 0x4037, 0xdc5: 0x4037, + 0xdc6: 0x4037, 0xdc7: 0x4037, 0xdc8: 0x4037, 0xdc9: 0x4037, 0xdca: 0x4037, 0xdcb: 0x4037, + 0xdcc: 0x4037, 0xdcd: 0x4037, 0xdce: 0x4037, 0xdcf: 0x400e, 0xdd0: 0x403f, 0xdd1: 0x4040, + 0xdd2: 0x4041, 0xdd3: 0x4040, 0xdd4: 0x403f, 0xdd5: 0x4042, 0xdd6: 0x4043, 0xdd7: 0x4044, + 0xdd8: 0x4040, 0xdd9: 0x4041, 0xdda: 0x4040, 0xddb: 0x4045, 0xddc: 0x4009, 0xddd: 0x4045, + 0xdde: 0x4046, 0xddf: 0x4045, 0xde0: 0x4047, 0xde1: 0x400b, 0xde2: 0x400a, 0xde3: 0x400c, + 0xde4: 0x4048, 0xde5: 0x4000, 0xde6: 0x4000, 0xde7: 0x4000, 0xde8: 0x4000, 0xde9: 0x4000, + 0xdea: 0x4000, 0xdeb: 0x4000, 0xdec: 0x4000, 0xded: 0x4000, 0xdee: 0x4000, 0xdef: 0x4000, + 0xdf0: 0x4000, 0xdf1: 0x4000, 0xdf2: 0x4000, 0xdf3: 0x4000, 0xdf4: 0x4000, 0xdf5: 0x4000, + 0xdf6: 0x4000, 0xdf7: 0x4000, 0xdf8: 0x4000, 0xdf9: 0x4000, 0xdfa: 0x4000, 0xdfb: 0x4000, + 0xdfc: 0x4000, 0xdfd: 0x4000, 0xdfe: 0x4000, 0xdff: 0x4000, + // Block 0x38, offset 0xe00 + 0xe00: 0x4000, 0xe01: 0x4000, 0xe02: 0x4000, 0xe03: 0x4000, 0xe04: 0x4000, 0xe05: 0x4000, + 0xe06: 0x4000, 0xe07: 0x4000, 0xe08: 0x4000, 0xe09: 0x4000, 0xe0a: 0x4000, 0xe0b: 0x4000, + 0xe0c: 0x4000, 0xe0d: 0x4000, 0xe0e: 0x4000, 0xe10: 0x4000, 0xe11: 0x4000, + 0xe12: 0x4000, 0xe13: 0x4000, 0xe14: 0x4000, 0xe15: 0x4000, 0xe16: 0x4000, 0xe17: 0x4000, + 0xe18: 0x4000, 0xe19: 0x4000, 0xe1a: 0x4000, 0xe1b: 0x4000, 0xe1c: 0x4000, 0xe1d: 0x4000, + 0xe1e: 0x4000, 0xe1f: 0x4000, 0xe20: 0x4000, 0xe21: 0x4000, 0xe22: 0x4000, 0xe23: 0x4000, + 0xe24: 0x4000, 0xe25: 0x4000, 0xe26: 0x4000, 0xe27: 0x4000, 0xe28: 0x4000, 0xe29: 0x4000, + 0xe2a: 0x4000, 0xe2b: 0x4000, 0xe2c: 0x4000, 0xe2d: 0x4000, 0xe2e: 0x4000, 0xe2f: 0x4000, + 0xe30: 0x4000, 0xe31: 0x4000, 0xe32: 0x4000, 0xe33: 0x4000, 0xe34: 0x4000, 0xe35: 0x4000, + 0xe36: 0x4000, 0xe37: 0x4000, 0xe38: 0x4000, 0xe39: 0x4000, 0xe3a: 0x4000, + // Block 0x39, offset 0xe40 + 0xe40: 0x4000, 0xe41: 0x4000, 0xe42: 0x4000, 0xe43: 0x4000, 0xe44: 0x4000, 0xe45: 0x4000, + 0xe46: 0x4000, 0xe47: 0x4000, 0xe48: 0x4000, 0xe49: 0x4000, 0xe4a: 0x4000, 0xe4b: 0x4000, + 0xe4c: 0x4000, 0xe4d: 0x4000, 0xe4e: 0x4000, 0xe4f: 0x4000, 0xe50: 0x4000, 0xe51: 0x4000, + 0xe52: 0x4000, 0xe53: 0x4000, 0xe54: 0x4000, 0xe55: 0x4000, 0xe56: 0x4000, 0xe57: 0x4000, + 0xe58: 0x4000, 0xe59: 0x4000, 0xe5a: 0x4000, 0xe5b: 0x4000, 0xe5c: 0x4000, 0xe5d: 0x4000, + 0xe5e: 0x4000, 0xe5f: 0x4000, 0xe60: 0x4000, 0xe61: 0x4000, 0xe62: 0x4000, 0xe63: 0x4000, + 0xe70: 0x4000, 0xe71: 0x4000, 0xe72: 0x4000, 0xe73: 0x4000, 0xe74: 0x4000, 0xe75: 0x4000, + 0xe76: 0x4000, 0xe77: 0x4000, 0xe78: 0x4000, 0xe79: 0x4000, 0xe7a: 0x4000, 0xe7b: 0x4000, + 0xe7c: 0x4000, 0xe7d: 0x4000, 0xe7e: 0x4000, 0xe7f: 0x4000, + // Block 0x3a, offset 0xe80 + 0xe80: 0x4000, 0xe81: 0x4000, 0xe82: 0x4000, 0xe83: 0x4000, 0xe84: 0x4000, 0xe85: 0x4000, + 0xe86: 0x4000, 0xe87: 0x4000, 0xe88: 0x4000, 0xe89: 0x4000, 0xe8a: 0x4000, 0xe8b: 0x4000, + 0xe8c: 0x4000, 0xe8d: 0x4000, 0xe8e: 0x4000, 0xe8f: 0x4000, 0xe90: 0x4000, 0xe91: 0x4000, + 0xe92: 0x4000, 0xe93: 0x4000, 0xe94: 0x4000, 0xe95: 0x4000, 0xe96: 0x4000, 0xe97: 0x4000, + 0xe98: 0x4000, 0xe99: 0x4000, 0xe9a: 0x4000, 0xe9b: 0x4000, 0xe9c: 0x4000, 0xe9d: 0x4000, + 0xe9e: 0x4000, 0xea0: 0x4000, 0xea1: 0x4000, 0xea2: 0x4000, 0xea3: 0x4000, + 0xea4: 0x4000, 0xea5: 0x4000, 0xea6: 0x4000, 0xea7: 0x4000, 0xea8: 0x4000, 0xea9: 0x4000, + 0xeaa: 0x4000, 0xeab: 0x4000, 0xeac: 0x4000, 0xead: 0x4000, 0xeae: 0x4000, 0xeaf: 0x4000, + 0xeb0: 0x4000, 0xeb1: 0x4000, 0xeb2: 0x4000, 0xeb3: 0x4000, 0xeb4: 0x4000, 0xeb5: 0x4000, + 0xeb6: 0x4000, 0xeb7: 0x4000, 0xeb8: 0x4000, 0xeb9: 0x4000, 0xeba: 0x4000, 0xebb: 0x4000, + 0xebc: 0x4000, 0xebd: 0x4000, 0xebe: 0x4000, 0xebf: 0x4000, + // Block 0x3b, offset 0xec0 + 0xec0: 0x4000, 0xec1: 0x4000, 0xec2: 0x4000, 0xec3: 0x4000, 0xec4: 0x4000, 0xec5: 0x4000, + 0xec6: 0x4000, 0xec7: 0x4000, 0xec8: 0x2000, 0xec9: 0x2000, 0xeca: 0x2000, 0xecb: 0x2000, + 0xecc: 0x2000, 0xecd: 0x2000, 0xece: 0x2000, 0xecf: 0x2000, 0xed0: 0x4000, 0xed1: 0x4000, + 0xed2: 0x4000, 0xed3: 0x4000, 0xed4: 0x4000, 0xed5: 0x4000, 0xed6: 0x4000, 0xed7: 0x4000, + 0xed8: 0x4000, 0xed9: 0x4000, 0xeda: 0x4000, 0xedb: 0x4000, 0xedc: 0x4000, 0xedd: 0x4000, + 0xede: 0x4000, 0xedf: 0x4000, 0xee0: 0x4000, 0xee1: 0x4000, 0xee2: 0x4000, 0xee3: 0x4000, + 0xee4: 0x4000, 0xee5: 0x4000, 0xee6: 0x4000, 0xee7: 0x4000, 0xee8: 0x4000, 0xee9: 0x4000, + 0xeea: 0x4000, 0xeeb: 0x4000, 0xeec: 0x4000, 0xeed: 0x4000, 0xeee: 0x4000, 0xeef: 0x4000, + 0xef0: 0x4000, 0xef1: 0x4000, 0xef2: 0x4000, 0xef3: 0x4000, 0xef4: 0x4000, 0xef5: 0x4000, + 0xef6: 0x4000, 0xef7: 0x4000, 0xef8: 0x4000, 0xef9: 0x4000, 0xefa: 0x4000, 0xefb: 0x4000, + 0xefc: 0x4000, 0xefd: 0x4000, 0xefe: 0x4000, 0xeff: 0x4000, + // Block 0x3c, offset 0xf00 + 0xf00: 0x4000, 0xf01: 0x4000, 0xf02: 0x4000, 0xf03: 0x4000, 0xf04: 0x4000, 0xf05: 0x4000, + 0xf06: 0x4000, 0xf07: 0x4000, 0xf08: 0x4000, 0xf09: 0x4000, 0xf0a: 0x4000, 0xf0b: 0x4000, + 0xf0c: 0x4000, 0xf0d: 0x4000, 0xf0e: 0x4000, 0xf0f: 0x4000, 0xf10: 0x4000, 0xf11: 0x4000, + 0xf12: 0x4000, 0xf13: 0x4000, 0xf14: 0x4000, 0xf15: 0x4000, 0xf16: 0x4000, 0xf17: 0x4000, + 0xf18: 0x4000, 0xf19: 0x4000, 0xf1a: 0x4000, 0xf1b: 0x4000, 0xf1c: 0x4000, 0xf1d: 0x4000, + 0xf1e: 0x4000, 0xf1f: 0x4000, 0xf20: 0x4000, 0xf21: 0x4000, 0xf22: 0x4000, 0xf23: 0x4000, + 0xf24: 0x4000, 0xf25: 0x4000, 0xf26: 0x4000, 0xf27: 0x4000, 0xf28: 0x4000, 0xf29: 0x4000, + 0xf2a: 0x4000, 0xf2b: 0x4000, 0xf2c: 0x4000, 0xf2d: 0x4000, 0xf2e: 0x4000, 0xf2f: 0x4000, + 0xf30: 0x4000, 0xf31: 0x4000, 0xf32: 0x4000, 0xf33: 0x4000, 0xf34: 0x4000, 0xf35: 0x4000, + 0xf36: 0x4000, 0xf37: 0x4000, 0xf38: 0x4000, 0xf39: 0x4000, 0xf3a: 0x4000, 0xf3b: 0x4000, + 0xf3c: 0x4000, 0xf3d: 0x4000, 0xf3e: 0x4000, + // Block 0x3d, offset 0xf40 + 0xf40: 0x4000, 0xf41: 0x4000, 0xf42: 0x4000, 0xf43: 0x4000, 0xf44: 0x4000, 0xf45: 0x4000, + 0xf46: 0x4000, 0xf47: 0x4000, 0xf48: 0x4000, 0xf49: 0x4000, 0xf4a: 0x4000, 0xf4b: 0x4000, + 0xf4c: 0x4000, 0xf50: 0x4000, 0xf51: 0x4000, + 0xf52: 0x4000, 0xf53: 0x4000, 0xf54: 0x4000, 0xf55: 0x4000, 0xf56: 0x4000, 0xf57: 0x4000, + 0xf58: 0x4000, 0xf59: 0x4000, 0xf5a: 0x4000, 0xf5b: 0x4000, 0xf5c: 0x4000, 0xf5d: 0x4000, + 0xf5e: 0x4000, 0xf5f: 0x4000, 0xf60: 0x4000, 0xf61: 0x4000, 0xf62: 0x4000, 0xf63: 0x4000, + 0xf64: 0x4000, 0xf65: 0x4000, 0xf66: 0x4000, 0xf67: 0x4000, 0xf68: 0x4000, 0xf69: 0x4000, + 0xf6a: 0x4000, 0xf6b: 0x4000, 0xf6c: 0x4000, 0xf6d: 0x4000, 0xf6e: 0x4000, 0xf6f: 0x4000, + 0xf70: 0x4000, 0xf71: 0x4000, 0xf72: 0x4000, 0xf73: 0x4000, 0xf74: 0x4000, 0xf75: 0x4000, + 0xf76: 0x4000, 0xf77: 0x4000, 0xf78: 0x4000, 0xf79: 0x4000, 0xf7a: 0x4000, 0xf7b: 0x4000, + 0xf7c: 0x4000, 0xf7d: 0x4000, 0xf7e: 0x4000, 0xf7f: 0x4000, + // Block 0x3e, offset 0xf80 + 0xf80: 0x4000, 0xf81: 0x4000, 0xf82: 0x4000, 0xf83: 0x4000, 0xf84: 0x4000, 0xf85: 0x4000, + 0xf86: 0x4000, + // Block 0x3f, offset 0xfc0 + 0xfe0: 0x4000, 0xfe1: 0x4000, 0xfe2: 0x4000, 0xfe3: 0x4000, + 0xfe4: 0x4000, 0xfe5: 0x4000, 0xfe6: 0x4000, 0xfe7: 0x4000, 0xfe8: 0x4000, 0xfe9: 0x4000, + 0xfea: 0x4000, 0xfeb: 0x4000, 0xfec: 0x4000, 0xfed: 0x4000, 0xfee: 0x4000, 0xfef: 0x4000, + 0xff0: 0x4000, 0xff1: 0x4000, 0xff2: 0x4000, 0xff3: 0x4000, 0xff4: 0x4000, 0xff5: 0x4000, + 0xff6: 0x4000, 0xff7: 0x4000, 0xff8: 0x4000, 0xff9: 0x4000, 0xffa: 0x4000, 0xffb: 0x4000, + 0xffc: 0x4000, + // Block 0x40, offset 0x1000 + 0x1000: 0x4000, 0x1001: 0x4000, 0x1002: 0x4000, 0x1003: 0x4000, 0x1004: 0x4000, 0x1005: 0x4000, + 0x1006: 0x4000, 0x1007: 0x4000, 0x1008: 0x4000, 0x1009: 0x4000, 0x100a: 0x4000, 0x100b: 0x4000, + 0x100c: 0x4000, 0x100d: 0x4000, 0x100e: 0x4000, 0x100f: 0x4000, 0x1010: 0x4000, 0x1011: 0x4000, + 0x1012: 0x4000, 0x1013: 0x4000, 0x1014: 0x4000, 0x1015: 0x4000, 0x1016: 0x4000, 0x1017: 0x4000, + 0x1018: 0x4000, 0x1019: 0x4000, 0x101a: 0x4000, 0x101b: 0x4000, 0x101c: 0x4000, 0x101d: 0x4000, + 0x101e: 0x4000, 0x101f: 0x4000, 0x1020: 0x4000, 0x1021: 0x4000, 0x1022: 0x4000, 0x1023: 0x4000, + // Block 0x41, offset 0x1040 + 0x1040: 0x2000, 0x1041: 0x2000, 0x1042: 0x2000, 0x1043: 0x2000, 0x1044: 0x2000, 0x1045: 0x2000, + 0x1046: 0x2000, 0x1047: 0x2000, 0x1048: 0x2000, 0x1049: 0x2000, 0x104a: 0x2000, 0x104b: 0x2000, + 0x104c: 0x2000, 0x104d: 0x2000, 0x104e: 0x2000, 0x104f: 0x2000, 0x1050: 0x4000, 0x1051: 0x4000, + 0x1052: 0x4000, 0x1053: 0x4000, 0x1054: 0x4000, 0x1055: 0x4000, 0x1056: 0x4000, 0x1057: 0x4000, + 0x1058: 0x4000, 0x1059: 0x4000, + 0x1070: 0x4000, 0x1071: 0x4000, 0x1072: 0x4000, 0x1073: 0x4000, 0x1074: 0x4000, 0x1075: 0x4000, + 0x1076: 0x4000, 0x1077: 0x4000, 0x1078: 0x4000, 0x1079: 0x4000, 0x107a: 0x4000, 0x107b: 0x4000, + 0x107c: 0x4000, 0x107d: 0x4000, 0x107e: 0x4000, 0x107f: 0x4000, + // Block 0x42, offset 0x1080 + 0x1080: 0x4000, 0x1081: 0x4000, 0x1082: 0x4000, 0x1083: 0x4000, 0x1084: 0x4000, 0x1085: 0x4000, + 0x1086: 0x4000, 0x1087: 0x4000, 0x1088: 0x4000, 0x1089: 0x4000, 0x108a: 0x4000, 0x108b: 0x4000, + 0x108c: 0x4000, 0x108d: 0x4000, 0x108e: 0x4000, 0x108f: 0x4000, 0x1090: 0x4000, 0x1091: 0x4000, + 0x1092: 0x4000, 0x1094: 0x4000, 0x1095: 0x4000, 0x1096: 0x4000, 0x1097: 0x4000, + 0x1098: 0x4000, 0x1099: 0x4000, 0x109a: 0x4000, 0x109b: 0x4000, 0x109c: 0x4000, 0x109d: 0x4000, + 0x109e: 0x4000, 0x109f: 0x4000, 0x10a0: 0x4000, 0x10a1: 0x4000, 0x10a2: 0x4000, 0x10a3: 0x4000, + 0x10a4: 0x4000, 0x10a5: 0x4000, 0x10a6: 0x4000, 0x10a8: 0x4000, 0x10a9: 0x4000, + 0x10aa: 0x4000, 0x10ab: 0x4000, + // Block 0x43, offset 0x10c0 + 0x10c1: 0x9012, 0x10c2: 0x9012, 0x10c3: 0x9012, 0x10c4: 0x9012, 0x10c5: 0x9012, + 0x10c6: 0x9012, 0x10c7: 0x9012, 0x10c8: 0x9012, 0x10c9: 0x9012, 0x10ca: 0x9012, 0x10cb: 0x9012, + 0x10cc: 0x9012, 0x10cd: 0x9012, 0x10ce: 0x9012, 0x10cf: 0x9012, 0x10d0: 0x9012, 0x10d1: 0x9012, + 0x10d2: 0x9012, 0x10d3: 0x9012, 0x10d4: 0x9012, 0x10d5: 0x9012, 0x10d6: 0x9012, 0x10d7: 0x9012, + 0x10d8: 0x9012, 0x10d9: 0x9012, 0x10da: 0x9012, 0x10db: 0x9012, 0x10dc: 0x9012, 0x10dd: 0x9012, + 0x10de: 0x9012, 0x10df: 0x9012, 0x10e0: 0x9049, 0x10e1: 0x9049, 0x10e2: 0x9049, 0x10e3: 0x9049, + 0x10e4: 0x9049, 0x10e5: 0x9049, 0x10e6: 0x9049, 0x10e7: 0x9049, 0x10e8: 0x9049, 0x10e9: 0x9049, + 0x10ea: 0x9049, 0x10eb: 0x9049, 0x10ec: 0x9049, 0x10ed: 0x9049, 0x10ee: 0x9049, 0x10ef: 0x9049, + 0x10f0: 0x9049, 0x10f1: 0x9049, 0x10f2: 0x9049, 0x10f3: 0x9049, 0x10f4: 0x9049, 0x10f5: 0x9049, + 0x10f6: 0x9049, 0x10f7: 0x9049, 0x10f8: 0x9049, 0x10f9: 0x9049, 0x10fa: 0x9049, 0x10fb: 0x9049, + 0x10fc: 0x9049, 0x10fd: 0x9049, 0x10fe: 0x9049, 0x10ff: 0x9049, + // Block 0x44, offset 0x1100 + 0x1100: 0x9049, 0x1101: 0x9049, 0x1102: 0x9049, 0x1103: 0x9049, 0x1104: 0x9049, 0x1105: 0x9049, + 0x1106: 0x9049, 0x1107: 0x9049, 0x1108: 0x9049, 0x1109: 0x9049, 0x110a: 0x9049, 0x110b: 0x9049, + 0x110c: 0x9049, 0x110d: 0x9049, 0x110e: 0x9049, 0x110f: 0x9049, 0x1110: 0x9049, 0x1111: 0x9049, + 0x1112: 0x9049, 0x1113: 0x9049, 0x1114: 0x9049, 0x1115: 0x9049, 0x1116: 0x9049, 0x1117: 0x9049, + 0x1118: 0x9049, 0x1119: 0x9049, 0x111a: 0x9049, 0x111b: 0x9049, 0x111c: 0x9049, 0x111d: 0x9049, + 0x111e: 0x9049, 0x111f: 0x904a, 0x1120: 0x904b, 0x1121: 0xb04c, 0x1122: 0xb04d, 0x1123: 0xb04d, + 0x1124: 0xb04e, 0x1125: 0xb04f, 0x1126: 0xb050, 0x1127: 0xb051, 0x1128: 0xb052, 0x1129: 0xb053, + 0x112a: 0xb054, 0x112b: 0xb055, 0x112c: 0xb056, 0x112d: 0xb057, 0x112e: 0xb058, 0x112f: 0xb059, + 0x1130: 0xb05a, 0x1131: 0xb05b, 0x1132: 0xb05c, 0x1133: 0xb05d, 0x1134: 0xb05e, 0x1135: 0xb05f, + 0x1136: 0xb060, 0x1137: 0xb061, 0x1138: 0xb062, 0x1139: 0xb063, 0x113a: 0xb064, 0x113b: 0xb065, + 0x113c: 0xb052, 0x113d: 0xb066, 0x113e: 0xb067, 0x113f: 0xb055, + // Block 0x45, offset 0x1140 + 0x1140: 0xb068, 0x1141: 0xb069, 0x1142: 0xb06a, 0x1143: 0xb06b, 0x1144: 0xb05a, 0x1145: 0xb056, + 0x1146: 0xb06c, 0x1147: 0xb06d, 0x1148: 0xb06b, 0x1149: 0xb06e, 0x114a: 0xb06b, 0x114b: 0xb06f, + 0x114c: 0xb06f, 0x114d: 0xb070, 0x114e: 0xb070, 0x114f: 0xb071, 0x1150: 0xb056, 0x1151: 0xb072, + 0x1152: 0xb073, 0x1153: 0xb072, 0x1154: 0xb074, 0x1155: 0xb073, 0x1156: 0xb075, 0x1157: 0xb075, + 0x1158: 0xb076, 0x1159: 0xb076, 0x115a: 0xb077, 0x115b: 0xb077, 0x115c: 0xb073, 0x115d: 0xb078, + 0x115e: 0xb079, 0x115f: 0xb067, 0x1160: 0xb07a, 0x1161: 0xb07b, 0x1162: 0xb07b, 0x1163: 0xb07b, + 0x1164: 0xb07b, 0x1165: 0xb07b, 0x1166: 0xb07b, 0x1167: 0xb07b, 0x1168: 0xb07b, 0x1169: 0xb07b, + 0x116a: 0xb07b, 0x116b: 0xb07b, 0x116c: 0xb07b, 0x116d: 0xb07b, 0x116e: 0xb07b, 0x116f: 0xb07b, + 0x1170: 0xb07c, 0x1171: 0xb07c, 0x1172: 0xb07c, 0x1173: 0xb07c, 0x1174: 0xb07c, 0x1175: 0xb07c, + 0x1176: 0xb07c, 0x1177: 0xb07c, 0x1178: 0xb07c, 0x1179: 0xb07c, 0x117a: 0xb07c, 0x117b: 0xb07c, + 0x117c: 0xb07c, 0x117d: 0xb07c, 0x117e: 0xb07c, + // Block 0x46, offset 0x1180 + 0x1182: 0xb07d, 0x1183: 0xb07e, 0x1184: 0xb07f, 0x1185: 0xb080, + 0x1186: 0xb07f, 0x1187: 0xb07e, 0x118a: 0xb081, 0x118b: 0xb082, + 0x118c: 0xb083, 0x118d: 0xb07f, 0x118e: 0xb080, 0x118f: 0xb07f, + 0x1192: 0xb084, 0x1193: 0xb085, 0x1194: 0xb084, 0x1195: 0xb086, 0x1196: 0xb084, 0x1197: 0xb087, + 0x119a: 0xb088, 0x119b: 0xb089, 0x119c: 0xb08a, + 0x11a0: 0x908b, 0x11a1: 0x908b, 0x11a2: 0x908c, 0x11a3: 0x908d, + 0x11a4: 0x908b, 0x11a5: 0x908e, 0x11a6: 0x908f, 0x11a8: 0xb090, 0x11a9: 0xb091, + 0x11aa: 0xb092, 0x11ab: 0xb091, 0x11ac: 0xb093, 0x11ad: 0xb094, 0x11ae: 0xb095, + 0x11bd: 0x2000, + // Block 0x47, offset 0x11c0 + 0x11e0: 0x4000, 0x11e1: 0x4000, + // Block 0x48, offset 0x1200 + 0x1200: 0x4000, 0x1201: 0x4000, 0x1202: 0x4000, 0x1203: 0x4000, 0x1204: 0x4000, 0x1205: 0x4000, + 0x1206: 0x4000, 0x1207: 0x4000, 0x1208: 0x4000, 0x1209: 0x4000, 0x120a: 0x4000, 0x120b: 0x4000, + 0x120c: 0x4000, 0x120d: 0x4000, 0x120e: 0x4000, 0x120f: 0x4000, 0x1210: 0x4000, 0x1211: 0x4000, + 0x1212: 0x4000, 0x1213: 0x4000, 0x1214: 0x4000, 0x1215: 0x4000, 0x1216: 0x4000, 0x1217: 0x4000, + 0x1218: 0x4000, 0x1219: 0x4000, 0x121a: 0x4000, 0x121b: 0x4000, 0x121c: 0x4000, 0x121d: 0x4000, + 0x121e: 0x4000, 0x121f: 0x4000, 0x1220: 0x4000, 0x1221: 0x4000, 0x1222: 0x4000, 0x1223: 0x4000, + 0x1224: 0x4000, 0x1225: 0x4000, 0x1226: 0x4000, 0x1227: 0x4000, 0x1228: 0x4000, 0x1229: 0x4000, + 0x122a: 0x4000, 0x122b: 0x4000, 0x122c: 0x4000, 0x122d: 0x4000, 0x122e: 0x4000, 0x122f: 0x4000, + 0x1230: 0x4000, 0x1231: 0x4000, + // Block 0x49, offset 0x1240 + 0x1240: 0x4000, 0x1241: 0x4000, 0x1242: 0x4000, 0x1243: 0x4000, 0x1244: 0x4000, 0x1245: 0x4000, + 0x1246: 0x4000, 0x1247: 0x4000, 0x1248: 0x4000, 0x1249: 0x4000, 0x124a: 0x4000, 0x124b: 0x4000, + 0x124c: 0x4000, 0x124d: 0x4000, 0x124e: 0x4000, 0x124f: 0x4000, 0x1250: 0x4000, 0x1251: 0x4000, + 0x1252: 0x4000, 0x1253: 0x4000, 0x1254: 0x4000, 0x1255: 0x4000, 0x1256: 0x4000, 0x1257: 0x4000, + 0x1258: 0x4000, 0x1259: 0x4000, 0x125a: 0x4000, 0x125b: 0x4000, 0x125c: 0x4000, 0x125d: 0x4000, + 0x125e: 0x4000, 0x125f: 0x4000, 0x1260: 0x4000, 0x1261: 0x4000, 0x1262: 0x4000, 0x1263: 0x4000, + 0x1264: 0x4000, 0x1265: 0x4000, 0x1266: 0x4000, 0x1267: 0x4000, 0x1268: 0x4000, 0x1269: 0x4000, + 0x126a: 0x4000, 0x126b: 0x4000, 0x126c: 0x4000, 0x126d: 0x4000, 0x126e: 0x4000, 0x126f: 0x4000, + 0x1270: 0x4000, 0x1271: 0x4000, 0x1272: 0x4000, + // Block 0x4a, offset 0x1280 + 0x1280: 0x4000, 0x1281: 0x4000, 0x1282: 0x4000, 0x1283: 0x4000, 0x1284: 0x4000, 0x1285: 0x4000, + 0x1286: 0x4000, 0x1287: 0x4000, 0x1288: 0x4000, 0x1289: 0x4000, 0x128a: 0x4000, 0x128b: 0x4000, + 0x128c: 0x4000, 0x128d: 0x4000, 0x128e: 0x4000, 0x128f: 0x4000, 0x1290: 0x4000, 0x1291: 0x4000, + 0x1292: 0x4000, 0x1293: 0x4000, 0x1294: 0x4000, 0x1295: 0x4000, 0x1296: 0x4000, 0x1297: 0x4000, + 0x1298: 0x4000, 0x1299: 0x4000, 0x129a: 0x4000, 0x129b: 0x4000, 0x129c: 0x4000, 0x129d: 0x4000, + 0x129e: 0x4000, + // Block 0x4b, offset 0x12c0 + 0x12f0: 0x4000, 0x12f1: 0x4000, 0x12f2: 0x4000, 0x12f3: 0x4000, 0x12f4: 0x4000, 0x12f5: 0x4000, + 0x12f6: 0x4000, 0x12f7: 0x4000, 0x12f8: 0x4000, 0x12f9: 0x4000, 0x12fa: 0x4000, 0x12fb: 0x4000, + 0x12fc: 0x4000, 0x12fd: 0x4000, 0x12fe: 0x4000, 0x12ff: 0x4000, + // Block 0x4c, offset 0x1300 + 0x1300: 0x4000, 0x1301: 0x4000, 0x1302: 0x4000, 0x1303: 0x4000, 0x1304: 0x4000, 0x1305: 0x4000, + 0x1306: 0x4000, 0x1307: 0x4000, 0x1308: 0x4000, 0x1309: 0x4000, 0x130a: 0x4000, 0x130b: 0x4000, + 0x130c: 0x4000, 0x130d: 0x4000, 0x130e: 0x4000, 0x130f: 0x4000, 0x1310: 0x4000, 0x1311: 0x4000, + 0x1312: 0x4000, 0x1313: 0x4000, 0x1314: 0x4000, 0x1315: 0x4000, 0x1316: 0x4000, 0x1317: 0x4000, + 0x1318: 0x4000, 0x1319: 0x4000, 0x131a: 0x4000, 0x131b: 0x4000, 0x131c: 0x4000, 0x131d: 0x4000, + 0x131e: 0x4000, 0x131f: 0x4000, 0x1320: 0x4000, 0x1321: 0x4000, 0x1322: 0x4000, 0x1323: 0x4000, + 0x1324: 0x4000, 0x1325: 0x4000, 0x1326: 0x4000, 0x1327: 0x4000, 0x1328: 0x4000, 0x1329: 0x4000, + 0x132a: 0x4000, 0x132b: 0x4000, 0x132c: 0x4000, 0x132d: 0x4000, 0x132e: 0x4000, 0x132f: 0x4000, + 0x1330: 0x4000, 0x1331: 0x4000, 0x1332: 0x4000, 0x1333: 0x4000, 0x1334: 0x4000, 0x1335: 0x4000, + 0x1336: 0x4000, 0x1337: 0x4000, 0x1338: 0x4000, 0x1339: 0x4000, 0x133a: 0x4000, 0x133b: 0x4000, + // Block 0x4d, offset 0x1340 + 0x1344: 0x4000, + // Block 0x4e, offset 0x1380 + 0x138f: 0x4000, + // Block 0x4f, offset 0x13c0 + 0x13c0: 0x2000, 0x13c1: 0x2000, 0x13c2: 0x2000, 0x13c3: 0x2000, 0x13c4: 0x2000, 0x13c5: 0x2000, + 0x13c6: 0x2000, 0x13c7: 0x2000, 0x13c8: 0x2000, 0x13c9: 0x2000, 0x13ca: 0x2000, + 0x13d0: 0x2000, 0x13d1: 0x2000, + 0x13d2: 0x2000, 0x13d3: 0x2000, 0x13d4: 0x2000, 0x13d5: 0x2000, 0x13d6: 0x2000, 0x13d7: 0x2000, + 0x13d8: 0x2000, 0x13d9: 0x2000, 0x13da: 0x2000, 0x13db: 0x2000, 0x13dc: 0x2000, 0x13dd: 0x2000, + 0x13de: 0x2000, 0x13df: 0x2000, 0x13e0: 0x2000, 0x13e1: 0x2000, 0x13e2: 0x2000, 0x13e3: 0x2000, + 0x13e4: 0x2000, 0x13e5: 0x2000, 0x13e6: 0x2000, 0x13e7: 0x2000, 0x13e8: 0x2000, 0x13e9: 0x2000, + 0x13ea: 0x2000, 0x13eb: 0x2000, 0x13ec: 0x2000, 0x13ed: 0x2000, + 0x13f0: 0x2000, 0x13f1: 0x2000, 0x13f2: 0x2000, 0x13f3: 0x2000, 0x13f4: 0x2000, 0x13f5: 0x2000, + 0x13f6: 0x2000, 0x13f7: 0x2000, 0x13f8: 0x2000, 0x13f9: 0x2000, 0x13fa: 0x2000, 0x13fb: 0x2000, + 0x13fc: 0x2000, 0x13fd: 0x2000, 0x13fe: 0x2000, 0x13ff: 0x2000, + // Block 0x50, offset 0x1400 + 0x1400: 0x2000, 0x1401: 0x2000, 0x1402: 0x2000, 0x1403: 0x2000, 0x1404: 0x2000, 0x1405: 0x2000, + 0x1406: 0x2000, 0x1407: 0x2000, 0x1408: 0x2000, 0x1409: 0x2000, 0x140a: 0x2000, 0x140b: 0x2000, + 0x140c: 0x2000, 0x140d: 0x2000, 0x140e: 0x2000, 0x140f: 0x2000, 0x1410: 0x2000, 0x1411: 0x2000, + 0x1412: 0x2000, 0x1413: 0x2000, 0x1414: 0x2000, 0x1415: 0x2000, 0x1416: 0x2000, 0x1417: 0x2000, + 0x1418: 0x2000, 0x1419: 0x2000, 0x141a: 0x2000, 0x141b: 0x2000, 0x141c: 0x2000, 0x141d: 0x2000, + 0x141e: 0x2000, 0x141f: 0x2000, 0x1420: 0x2000, 0x1421: 0x2000, 0x1422: 0x2000, 0x1423: 0x2000, + 0x1424: 0x2000, 0x1425: 0x2000, 0x1426: 0x2000, 0x1427: 0x2000, 0x1428: 0x2000, 0x1429: 0x2000, + 0x1430: 0x2000, 0x1431: 0x2000, 0x1432: 0x2000, 0x1433: 0x2000, 0x1434: 0x2000, 0x1435: 0x2000, + 0x1436: 0x2000, 0x1437: 0x2000, 0x1438: 0x2000, 0x1439: 0x2000, 0x143a: 0x2000, 0x143b: 0x2000, + 0x143c: 0x2000, 0x143d: 0x2000, 0x143e: 0x2000, 0x143f: 0x2000, + // Block 0x51, offset 0x1440 + 0x1440: 0x2000, 0x1441: 0x2000, 0x1442: 0x2000, 0x1443: 0x2000, 0x1444: 0x2000, 0x1445: 0x2000, + 0x1446: 0x2000, 0x1447: 0x2000, 0x1448: 0x2000, 0x1449: 0x2000, 0x144a: 0x2000, 0x144b: 0x2000, + 0x144c: 0x2000, 0x144d: 0x2000, 0x144e: 0x4000, 0x144f: 0x2000, 0x1450: 0x2000, 0x1451: 0x4000, + 0x1452: 0x4000, 0x1453: 0x4000, 0x1454: 0x4000, 0x1455: 0x4000, 0x1456: 0x4000, 0x1457: 0x4000, + 0x1458: 0x4000, 0x1459: 0x4000, 0x145a: 0x4000, 0x145b: 0x2000, 0x145c: 0x2000, 0x145d: 0x2000, + 0x145e: 0x2000, 0x145f: 0x2000, 0x1460: 0x2000, 0x1461: 0x2000, 0x1462: 0x2000, 0x1463: 0x2000, + 0x1464: 0x2000, 0x1465: 0x2000, 0x1466: 0x2000, 0x1467: 0x2000, 0x1468: 0x2000, 0x1469: 0x2000, + 0x146a: 0x2000, 0x146b: 0x2000, 0x146c: 0x2000, + // Block 0x52, offset 0x1480 + 0x1480: 0x4000, 0x1481: 0x4000, 0x1482: 0x4000, + 0x1490: 0x4000, 0x1491: 0x4000, + 0x1492: 0x4000, 0x1493: 0x4000, 0x1494: 0x4000, 0x1495: 0x4000, 0x1496: 0x4000, 0x1497: 0x4000, + 0x1498: 0x4000, 0x1499: 0x4000, 0x149a: 0x4000, 0x149b: 0x4000, 0x149c: 0x4000, 0x149d: 0x4000, + 0x149e: 0x4000, 0x149f: 0x4000, 0x14a0: 0x4000, 0x14a1: 0x4000, 0x14a2: 0x4000, 0x14a3: 0x4000, + 0x14a4: 0x4000, 0x14a5: 0x4000, 0x14a6: 0x4000, 0x14a7: 0x4000, 0x14a8: 0x4000, 0x14a9: 0x4000, + 0x14aa: 0x4000, 0x14ab: 0x4000, 0x14ac: 0x4000, 0x14ad: 0x4000, 0x14ae: 0x4000, 0x14af: 0x4000, + 0x14b0: 0x4000, 0x14b1: 0x4000, 0x14b2: 0x4000, 0x14b3: 0x4000, 0x14b4: 0x4000, 0x14b5: 0x4000, + 0x14b6: 0x4000, 0x14b7: 0x4000, 0x14b8: 0x4000, 0x14b9: 0x4000, 0x14ba: 0x4000, 0x14bb: 0x4000, + // Block 0x53, offset 0x14c0 + 0x14c0: 0x4000, 0x14c1: 0x4000, 0x14c2: 0x4000, 0x14c3: 0x4000, 0x14c4: 0x4000, 0x14c5: 0x4000, + 0x14c6: 0x4000, 0x14c7: 0x4000, 0x14c8: 0x4000, + 0x14d0: 0x4000, 0x14d1: 0x4000, + 0x14e0: 0x4000, 0x14e1: 0x4000, 0x14e2: 0x4000, 0x14e3: 0x4000, + 0x14e4: 0x4000, 0x14e5: 0x4000, + // Block 0x54, offset 0x1500 + 0x1500: 0x4000, 0x1501: 0x4000, 0x1502: 0x4000, 0x1503: 0x4000, 0x1504: 0x4000, 0x1505: 0x4000, + 0x1506: 0x4000, 0x1507: 0x4000, 0x1508: 0x4000, 0x1509: 0x4000, 0x150a: 0x4000, 0x150b: 0x4000, + 0x150c: 0x4000, 0x150d: 0x4000, 0x150e: 0x4000, 0x150f: 0x4000, 0x1510: 0x4000, 0x1511: 0x4000, + 0x1512: 0x4000, 0x1513: 0x4000, 0x1514: 0x4000, 0x1515: 0x4000, 0x1516: 0x4000, 0x1517: 0x4000, + 0x1518: 0x4000, 0x1519: 0x4000, 0x151a: 0x4000, 0x151b: 0x4000, 0x151c: 0x4000, 0x151d: 0x4000, + 0x151e: 0x4000, 0x151f: 0x4000, 0x1520: 0x4000, + 0x152d: 0x4000, 0x152e: 0x4000, 0x152f: 0x4000, + 0x1530: 0x4000, 0x1531: 0x4000, 0x1532: 0x4000, 0x1533: 0x4000, 0x1534: 0x4000, 0x1535: 0x4000, + 0x1537: 0x4000, 0x1538: 0x4000, 0x1539: 0x4000, 0x153a: 0x4000, 0x153b: 0x4000, + 0x153c: 0x4000, 0x153d: 0x4000, 0x153e: 0x4000, 0x153f: 0x4000, + // Block 0x55, offset 0x1540 + 0x1540: 0x4000, 0x1541: 0x4000, 0x1542: 0x4000, 0x1543: 0x4000, 0x1544: 0x4000, 0x1545: 0x4000, + 0x1546: 0x4000, 0x1547: 0x4000, 0x1548: 0x4000, 0x1549: 0x4000, 0x154a: 0x4000, 0x154b: 0x4000, + 0x154c: 0x4000, 0x154d: 0x4000, 0x154e: 0x4000, 0x154f: 0x4000, 0x1550: 0x4000, 0x1551: 0x4000, + 0x1552: 0x4000, 0x1553: 0x4000, 0x1554: 0x4000, 0x1555: 0x4000, 0x1556: 0x4000, 0x1557: 0x4000, + 0x1558: 0x4000, 0x1559: 0x4000, 0x155a: 0x4000, 0x155b: 0x4000, 0x155c: 0x4000, 0x155d: 0x4000, + 0x155e: 0x4000, 0x155f: 0x4000, 0x1560: 0x4000, 0x1561: 0x4000, 0x1562: 0x4000, 0x1563: 0x4000, + 0x1564: 0x4000, 0x1565: 0x4000, 0x1566: 0x4000, 0x1567: 0x4000, 0x1568: 0x4000, 0x1569: 0x4000, + 0x156a: 0x4000, 0x156b: 0x4000, 0x156c: 0x4000, 0x156d: 0x4000, 0x156e: 0x4000, 0x156f: 0x4000, + 0x1570: 0x4000, 0x1571: 0x4000, 0x1572: 0x4000, 0x1573: 0x4000, 0x1574: 0x4000, 0x1575: 0x4000, + 0x1576: 0x4000, 0x1577: 0x4000, 0x1578: 0x4000, 0x1579: 0x4000, 0x157a: 0x4000, 0x157b: 0x4000, + 0x157c: 0x4000, 0x157e: 0x4000, 0x157f: 0x4000, + // Block 0x56, offset 0x1580 + 0x1580: 0x4000, 0x1581: 0x4000, 0x1582: 0x4000, 0x1583: 0x4000, 0x1584: 0x4000, 0x1585: 0x4000, + 0x1586: 0x4000, 0x1587: 0x4000, 0x1588: 0x4000, 0x1589: 0x4000, 0x158a: 0x4000, 0x158b: 0x4000, + 0x158c: 0x4000, 0x158d: 0x4000, 0x158e: 0x4000, 0x158f: 0x4000, 0x1590: 0x4000, 0x1591: 0x4000, + 0x1592: 0x4000, 0x1593: 0x4000, + 0x15a0: 0x4000, 0x15a1: 0x4000, 0x15a2: 0x4000, 0x15a3: 0x4000, + 0x15a4: 0x4000, 0x15a5: 0x4000, 0x15a6: 0x4000, 0x15a7: 0x4000, 0x15a8: 0x4000, 0x15a9: 0x4000, + 0x15aa: 0x4000, 0x15ab: 0x4000, 0x15ac: 0x4000, 0x15ad: 0x4000, 0x15ae: 0x4000, 0x15af: 0x4000, + 0x15b0: 0x4000, 0x15b1: 0x4000, 0x15b2: 0x4000, 0x15b3: 0x4000, 0x15b4: 0x4000, 0x15b5: 0x4000, + 0x15b6: 0x4000, 0x15b7: 0x4000, 0x15b8: 0x4000, 0x15b9: 0x4000, 0x15ba: 0x4000, 0x15bb: 0x4000, + 0x15bc: 0x4000, 0x15bd: 0x4000, 0x15be: 0x4000, 0x15bf: 0x4000, + // Block 0x57, offset 0x15c0 + 0x15c0: 0x4000, 0x15c1: 0x4000, 0x15c2: 0x4000, 0x15c3: 0x4000, 0x15c4: 0x4000, 0x15c5: 0x4000, + 0x15c6: 0x4000, 0x15c7: 0x4000, 0x15c8: 0x4000, 0x15c9: 0x4000, 0x15ca: 0x4000, + 0x15cf: 0x4000, 0x15d0: 0x4000, 0x15d1: 0x4000, + 0x15d2: 0x4000, 0x15d3: 0x4000, + 0x15e0: 0x4000, 0x15e1: 0x4000, 0x15e2: 0x4000, 0x15e3: 0x4000, + 0x15e4: 0x4000, 0x15e5: 0x4000, 0x15e6: 0x4000, 0x15e7: 0x4000, 0x15e8: 0x4000, 0x15e9: 0x4000, + 0x15ea: 0x4000, 0x15eb: 0x4000, 0x15ec: 0x4000, 0x15ed: 0x4000, 0x15ee: 0x4000, 0x15ef: 0x4000, + 0x15f0: 0x4000, 0x15f4: 0x4000, + 0x15f8: 0x4000, 0x15f9: 0x4000, 0x15fa: 0x4000, 0x15fb: 0x4000, + 0x15fc: 0x4000, 0x15fd: 0x4000, 0x15fe: 0x4000, 0x15ff: 0x4000, + // Block 0x58, offset 0x1600 + 0x1600: 0x4000, 0x1602: 0x4000, 0x1603: 0x4000, 0x1604: 0x4000, 0x1605: 0x4000, + 0x1606: 0x4000, 0x1607: 0x4000, 0x1608: 0x4000, 0x1609: 0x4000, 0x160a: 0x4000, 0x160b: 0x4000, + 0x160c: 0x4000, 0x160d: 0x4000, 0x160e: 0x4000, 0x160f: 0x4000, 0x1610: 0x4000, 0x1611: 0x4000, + 0x1612: 0x4000, 0x1613: 0x4000, 0x1614: 0x4000, 0x1615: 0x4000, 0x1616: 0x4000, 0x1617: 0x4000, + 0x1618: 0x4000, 0x1619: 0x4000, 0x161a: 0x4000, 0x161b: 0x4000, 0x161c: 0x4000, 0x161d: 0x4000, + 0x161e: 0x4000, 0x161f: 0x4000, 0x1620: 0x4000, 0x1621: 0x4000, 0x1622: 0x4000, 0x1623: 0x4000, + 0x1624: 0x4000, 0x1625: 0x4000, 0x1626: 0x4000, 0x1627: 0x4000, 0x1628: 0x4000, 0x1629: 0x4000, + 0x162a: 0x4000, 0x162b: 0x4000, 0x162c: 0x4000, 0x162d: 0x4000, 0x162e: 0x4000, 0x162f: 0x4000, + 0x1630: 0x4000, 0x1631: 0x4000, 0x1632: 0x4000, 0x1633: 0x4000, 0x1634: 0x4000, 0x1635: 0x4000, + 0x1636: 0x4000, 0x1637: 0x4000, 0x1638: 0x4000, 0x1639: 0x4000, 0x163a: 0x4000, 0x163b: 0x4000, + 0x163c: 0x4000, 0x163d: 0x4000, 0x163e: 0x4000, 0x163f: 0x4000, + // Block 0x59, offset 0x1640 + 0x1640: 0x4000, 0x1641: 0x4000, 0x1642: 0x4000, 0x1643: 0x4000, 0x1644: 0x4000, 0x1645: 0x4000, + 0x1646: 0x4000, 0x1647: 0x4000, 0x1648: 0x4000, 0x1649: 0x4000, 0x164a: 0x4000, 0x164b: 0x4000, + 0x164c: 0x4000, 0x164d: 0x4000, 0x164e: 0x4000, 0x164f: 0x4000, 0x1650: 0x4000, 0x1651: 0x4000, + 0x1652: 0x4000, 0x1653: 0x4000, 0x1654: 0x4000, 0x1655: 0x4000, 0x1656: 0x4000, 0x1657: 0x4000, + 0x1658: 0x4000, 0x1659: 0x4000, 0x165a: 0x4000, 0x165b: 0x4000, 0x165c: 0x4000, 0x165d: 0x4000, + 0x165e: 0x4000, 0x165f: 0x4000, 0x1660: 0x4000, 0x1661: 0x4000, 0x1662: 0x4000, 0x1663: 0x4000, + 0x1664: 0x4000, 0x1665: 0x4000, 0x1666: 0x4000, 0x1667: 0x4000, 0x1668: 0x4000, 0x1669: 0x4000, + 0x166a: 0x4000, 0x166b: 0x4000, 0x166c: 0x4000, 0x166d: 0x4000, 0x166e: 0x4000, 0x166f: 0x4000, + 0x1670: 0x4000, 0x1671: 0x4000, 0x1672: 0x4000, 0x1673: 0x4000, 0x1674: 0x4000, 0x1675: 0x4000, + 0x1676: 0x4000, 0x1677: 0x4000, 0x1678: 0x4000, 0x1679: 0x4000, 0x167a: 0x4000, 0x167b: 0x4000, + 0x167c: 0x4000, 0x167f: 0x4000, + // Block 0x5a, offset 0x1680 + 0x1680: 0x4000, 0x1681: 0x4000, 0x1682: 0x4000, 0x1683: 0x4000, 0x1684: 0x4000, 0x1685: 0x4000, + 0x1686: 0x4000, 0x1687: 0x4000, 0x1688: 0x4000, 0x1689: 0x4000, 0x168a: 0x4000, 0x168b: 0x4000, + 0x168c: 0x4000, 0x168d: 0x4000, 0x168e: 0x4000, 0x168f: 0x4000, 0x1690: 0x4000, 0x1691: 0x4000, + 0x1692: 0x4000, 0x1693: 0x4000, 0x1694: 0x4000, 0x1695: 0x4000, 0x1696: 0x4000, 0x1697: 0x4000, + 0x1698: 0x4000, 0x1699: 0x4000, 0x169a: 0x4000, 0x169b: 0x4000, 0x169c: 0x4000, 0x169d: 0x4000, + 0x169e: 0x4000, 0x169f: 0x4000, 0x16a0: 0x4000, 0x16a1: 0x4000, 0x16a2: 0x4000, 0x16a3: 0x4000, + 0x16a4: 0x4000, 0x16a5: 0x4000, 0x16a6: 0x4000, 0x16a7: 0x4000, 0x16a8: 0x4000, 0x16a9: 0x4000, + 0x16aa: 0x4000, 0x16ab: 0x4000, 0x16ac: 0x4000, 0x16ad: 0x4000, 0x16ae: 0x4000, 0x16af: 0x4000, + 0x16b0: 0x4000, 0x16b1: 0x4000, 0x16b2: 0x4000, 0x16b3: 0x4000, 0x16b4: 0x4000, 0x16b5: 0x4000, + 0x16b6: 0x4000, 0x16b7: 0x4000, 0x16b8: 0x4000, 0x16b9: 0x4000, 0x16ba: 0x4000, 0x16bb: 0x4000, + 0x16bc: 0x4000, 0x16bd: 0x4000, + // Block 0x5b, offset 0x16c0 + 0x16cb: 0x4000, + 0x16cc: 0x4000, 0x16cd: 0x4000, 0x16ce: 0x4000, 0x16d0: 0x4000, 0x16d1: 0x4000, + 0x16d2: 0x4000, 0x16d3: 0x4000, 0x16d4: 0x4000, 0x16d5: 0x4000, 0x16d6: 0x4000, 0x16d7: 0x4000, + 0x16d8: 0x4000, 0x16d9: 0x4000, 0x16da: 0x4000, 0x16db: 0x4000, 0x16dc: 0x4000, 0x16dd: 0x4000, + 0x16de: 0x4000, 0x16df: 0x4000, 0x16e0: 0x4000, 0x16e1: 0x4000, 0x16e2: 0x4000, 0x16e3: 0x4000, + 0x16e4: 0x4000, 0x16e5: 0x4000, 0x16e6: 0x4000, 0x16e7: 0x4000, + 0x16fa: 0x4000, + // Block 0x5c, offset 0x1700 + 0x1715: 0x4000, 0x1716: 0x4000, + 0x1724: 0x4000, + // Block 0x5d, offset 0x1740 + 0x177b: 0x4000, + 0x177c: 0x4000, 0x177d: 0x4000, 0x177e: 0x4000, 0x177f: 0x4000, + // Block 0x5e, offset 0x1780 + 0x1780: 0x4000, 0x1781: 0x4000, 0x1782: 0x4000, 0x1783: 0x4000, 0x1784: 0x4000, 0x1785: 0x4000, + 0x1786: 0x4000, 0x1787: 0x4000, 0x1788: 0x4000, 0x1789: 0x4000, 0x178a: 0x4000, 0x178b: 0x4000, + 0x178c: 0x4000, 0x178d: 0x4000, 0x178e: 0x4000, 0x178f: 0x4000, + // Block 0x5f, offset 0x17c0 + 0x17c0: 0x4000, 0x17c1: 0x4000, 0x17c2: 0x4000, 0x17c3: 0x4000, 0x17c4: 0x4000, 0x17c5: 0x4000, + 0x17cc: 0x4000, 0x17d0: 0x4000, 0x17d1: 0x4000, + 0x17d2: 0x4000, + 0x17eb: 0x4000, 0x17ec: 0x4000, + 0x17f4: 0x4000, 0x17f5: 0x4000, + 0x17f6: 0x4000, 0x17f7: 0x4000, 0x17f8: 0x4000, 0x17f9: 0x4000, + // Block 0x60, offset 0x1800 + 0x1810: 0x4000, 0x1811: 0x4000, + 0x1812: 0x4000, 0x1813: 0x4000, 0x1814: 0x4000, 0x1815: 0x4000, 0x1816: 0x4000, 0x1817: 0x4000, + 0x1818: 0x4000, 0x1819: 0x4000, 0x181a: 0x4000, 0x181b: 0x4000, 0x181c: 0x4000, 0x181d: 0x4000, + 0x181e: 0x4000, 0x181f: 0x4000, 0x1820: 0x4000, 0x1821: 0x4000, 0x1822: 0x4000, 0x1823: 0x4000, + 0x1824: 0x4000, 0x1825: 0x4000, 0x1826: 0x4000, 0x1827: 0x4000, 0x1828: 0x4000, 0x1829: 0x4000, + 0x182a: 0x4000, 0x182b: 0x4000, 0x182c: 0x4000, 0x182d: 0x4000, 0x182e: 0x4000, 0x182f: 0x4000, + 0x1830: 0x4000, 0x1831: 0x4000, 0x1832: 0x4000, 0x1833: 0x4000, 0x1834: 0x4000, 0x1835: 0x4000, + 0x1836: 0x4000, 0x1837: 0x4000, 0x1838: 0x4000, 0x1839: 0x4000, 0x183a: 0x4000, 0x183b: 0x4000, + 0x183c: 0x4000, 0x183d: 0x4000, 0x183e: 0x4000, + // Block 0x61, offset 0x1840 + 0x1840: 0x4000, 0x1841: 0x4000, 0x1842: 0x4000, 0x1843: 0x4000, 0x1844: 0x4000, 0x1845: 0x4000, + 0x1846: 0x4000, 0x1847: 0x4000, 0x1848: 0x4000, 0x1849: 0x4000, 0x184a: 0x4000, 0x184b: 0x4000, + 0x184c: 0x4000, 0x184d: 0x4000, 0x184e: 0x4000, 0x184f: 0x4000, 0x1850: 0x4000, 0x1851: 0x4000, + 0x1852: 0x4000, 0x1853: 0x4000, 0x1854: 0x4000, 0x1855: 0x4000, 0x1856: 0x4000, 0x1857: 0x4000, + 0x1858: 0x4000, 0x1859: 0x4000, 0x185a: 0x4000, 0x185b: 0x4000, 0x185c: 0x4000, 0x185d: 0x4000, + 0x185e: 0x4000, 0x185f: 0x4000, 0x1860: 0x4000, 0x1861: 0x4000, 0x1862: 0x4000, 0x1863: 0x4000, + 0x1864: 0x4000, 0x1865: 0x4000, 0x1866: 0x4000, 0x1867: 0x4000, 0x1868: 0x4000, 0x1869: 0x4000, + 0x186a: 0x4000, 0x186b: 0x4000, 0x186c: 0x4000, 0x186d: 0x4000, 0x186e: 0x4000, 0x186f: 0x4000, + 0x1870: 0x4000, 0x1873: 0x4000, 0x1874: 0x4000, 0x1875: 0x4000, + 0x1876: 0x4000, 0x187a: 0x4000, + 0x187c: 0x4000, 0x187d: 0x4000, 0x187e: 0x4000, 0x187f: 0x4000, + // Block 0x62, offset 0x1880 + 0x1880: 0x4000, 0x1881: 0x4000, 0x1882: 0x4000, 0x1883: 0x4000, 0x1884: 0x4000, 0x1885: 0x4000, + 0x1886: 0x4000, 0x1887: 0x4000, 0x1888: 0x4000, 0x1889: 0x4000, 0x188a: 0x4000, 0x188b: 0x4000, + 0x188c: 0x4000, 0x188d: 0x4000, 0x188e: 0x4000, 0x188f: 0x4000, 0x1890: 0x4000, 0x1891: 0x4000, + 0x1892: 0x4000, 0x1893: 0x4000, 0x1894: 0x4000, 0x1895: 0x4000, 0x1896: 0x4000, 0x1897: 0x4000, + 0x1898: 0x4000, 0x1899: 0x4000, 0x189a: 0x4000, 0x189b: 0x4000, 0x189c: 0x4000, 0x189d: 0x4000, + 0x189e: 0x4000, 0x189f: 0x4000, 0x18a0: 0x4000, 0x18a1: 0x4000, 0x18a2: 0x4000, + 0x18b0: 0x4000, 0x18b1: 0x4000, 0x18b2: 0x4000, 0x18b3: 0x4000, 0x18b4: 0x4000, 0x18b5: 0x4000, + 0x18b6: 0x4000, 0x18b7: 0x4000, 0x18b8: 0x4000, 0x18b9: 0x4000, + // Block 0x63, offset 0x18c0 + 0x18c0: 0x4000, 0x18c1: 0x4000, 0x18c2: 0x4000, + 0x18d0: 0x4000, 0x18d1: 0x4000, + 0x18d2: 0x4000, 0x18d3: 0x4000, 0x18d4: 0x4000, 0x18d5: 0x4000, 0x18d6: 0x4000, 0x18d7: 0x4000, + 0x18d8: 0x4000, 0x18d9: 0x4000, 0x18da: 0x4000, 0x18db: 0x4000, 0x18dc: 0x4000, 0x18dd: 0x4000, + 0x18de: 0x4000, 0x18df: 0x4000, 0x18e0: 0x4000, 0x18e1: 0x4000, 0x18e2: 0x4000, 0x18e3: 0x4000, + 0x18e4: 0x4000, 0x18e5: 0x4000, 0x18e6: 0x4000, 0x18e7: 0x4000, 0x18e8: 0x4000, 0x18e9: 0x4000, + 0x18ea: 0x4000, 0x18eb: 0x4000, 0x18ec: 0x4000, 0x18ed: 0x4000, 0x18ee: 0x4000, 0x18ef: 0x4000, + 0x18f0: 0x4000, 0x18f1: 0x4000, 0x18f2: 0x4000, 0x18f3: 0x4000, 0x18f4: 0x4000, 0x18f5: 0x4000, + 0x18f6: 0x4000, 0x18f7: 0x4000, 0x18f8: 0x4000, 0x18f9: 0x4000, 0x18fa: 0x4000, 0x18fb: 0x4000, + 0x18fc: 0x4000, 0x18fd: 0x4000, 0x18fe: 0x4000, 0x18ff: 0x4000, + // Block 0x64, offset 0x1900 + 0x1900: 0x2000, 0x1901: 0x2000, 0x1902: 0x2000, 0x1903: 0x2000, 0x1904: 0x2000, 0x1905: 0x2000, + 0x1906: 0x2000, 0x1907: 0x2000, 0x1908: 0x2000, 0x1909: 0x2000, 0x190a: 0x2000, 0x190b: 0x2000, + 0x190c: 0x2000, 0x190d: 0x2000, 0x190e: 0x2000, 0x190f: 0x2000, 0x1910: 0x2000, 0x1911: 0x2000, + 0x1912: 0x2000, 0x1913: 0x2000, 0x1914: 0x2000, 0x1915: 0x2000, 0x1916: 0x2000, 0x1917: 0x2000, + 0x1918: 0x2000, 0x1919: 0x2000, 0x191a: 0x2000, 0x191b: 0x2000, 0x191c: 0x2000, 0x191d: 0x2000, + 0x191e: 0x2000, 0x191f: 0x2000, 0x1920: 0x2000, 0x1921: 0x2000, 0x1922: 0x2000, 0x1923: 0x2000, + 0x1924: 0x2000, 0x1925: 0x2000, 0x1926: 0x2000, 0x1927: 0x2000, 0x1928: 0x2000, 0x1929: 0x2000, + 0x192a: 0x2000, 0x192b: 0x2000, 0x192c: 0x2000, 0x192d: 0x2000, 0x192e: 0x2000, 0x192f: 0x2000, + 0x1930: 0x2000, 0x1931: 0x2000, 0x1932: 0x2000, 0x1933: 0x2000, 0x1934: 0x2000, 0x1935: 0x2000, + 0x1936: 0x2000, 0x1937: 0x2000, 0x1938: 0x2000, 0x1939: 0x2000, 0x193a: 0x2000, 0x193b: 0x2000, + 0x193c: 0x2000, 0x193d: 0x2000, +} + +// widthIndex: 22 blocks, 1408 entries, 1408 bytes +// Block 0 is the zero block. +var widthIndex = [1408]uint8{ + // Block 0x0, offset 0x0 + // Block 0x1, offset 0x40 + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0xc2: 0x01, 0xc3: 0x02, 0xc4: 0x03, 0xc5: 0x04, 0xc7: 0x05, + 0xc9: 0x06, 0xcb: 0x07, 0xcc: 0x08, 0xcd: 0x09, 0xce: 0x0a, 0xcf: 0x0b, + 0xd0: 0x0c, 0xd1: 0x0d, + 0xe1: 0x02, 0xe2: 0x03, 0xe3: 0x04, 0xe4: 0x05, 0xe5: 0x06, 0xe6: 0x06, 0xe7: 0x06, + 0xe8: 0x06, 0xe9: 0x06, 0xea: 0x07, 0xeb: 0x06, 0xec: 0x06, 0xed: 0x08, 0xee: 0x09, 0xef: 0x0a, + 0xf0: 0x0f, 0xf3: 0x12, 0xf4: 0x13, + // Block 0x4, offset 0x100 + 0x104: 0x0e, 0x105: 0x0f, + // Block 0x5, offset 0x140 + 0x140: 0x10, 0x141: 0x11, 0x142: 0x12, 0x144: 0x13, 0x145: 0x14, 0x146: 0x15, 0x147: 0x16, + 0x148: 0x17, 0x149: 0x18, 0x14a: 0x19, 0x14c: 0x1a, 0x14f: 0x1b, + 0x151: 0x1c, 0x152: 0x08, 0x153: 0x1d, 0x154: 0x1e, 0x155: 0x1f, 0x156: 0x20, 0x157: 0x21, + 0x158: 0x22, 0x159: 0x23, 0x15a: 0x24, 0x15b: 0x25, 0x15c: 0x26, 0x15d: 0x27, 0x15e: 0x28, 0x15f: 0x29, + 0x166: 0x2a, + 0x16c: 0x2b, 0x16d: 0x2c, + 0x17a: 0x2d, 0x17b: 0x2e, 0x17c: 0x0e, 0x17d: 0x0e, 0x17e: 0x0e, 0x17f: 0x2f, + // Block 0x6, offset 0x180 + 0x180: 0x30, 0x181: 0x31, 0x182: 0x32, 0x183: 0x33, 0x184: 0x34, 0x185: 0x35, 0x186: 0x36, 0x187: 0x37, + 0x188: 0x38, 0x189: 0x39, 0x18a: 0x0e, 0x18b: 0x3a, 0x18c: 0x0e, 0x18d: 0x0e, 0x18e: 0x0e, 0x18f: 0x0e, + 0x190: 0x0e, 0x191: 0x0e, 0x192: 0x0e, 0x193: 0x0e, 0x194: 0x0e, 0x195: 0x0e, 0x196: 0x0e, 0x197: 0x0e, + 0x198: 0x0e, 0x199: 0x0e, 0x19a: 0x0e, 0x19b: 0x0e, 0x19c: 0x0e, 0x19d: 0x0e, 0x19e: 0x0e, 0x19f: 0x0e, + 0x1a0: 0x0e, 0x1a1: 0x0e, 0x1a2: 0x0e, 0x1a3: 0x0e, 0x1a4: 0x0e, 0x1a5: 0x0e, 0x1a6: 0x0e, 0x1a7: 0x0e, + 0x1a8: 0x0e, 0x1a9: 0x0e, 0x1aa: 0x0e, 0x1ab: 0x0e, 0x1ac: 0x0e, 0x1ad: 0x0e, 0x1ae: 0x0e, 0x1af: 0x0e, + 0x1b0: 0x0e, 0x1b1: 0x0e, 0x1b2: 0x0e, 0x1b3: 0x0e, 0x1b4: 0x0e, 0x1b5: 0x0e, 0x1b6: 0x0e, 0x1b7: 0x0e, + 0x1b8: 0x0e, 0x1b9: 0x0e, 0x1ba: 0x0e, 0x1bb: 0x0e, 0x1bc: 0x0e, 0x1bd: 0x0e, 0x1be: 0x0e, 0x1bf: 0x0e, + // Block 0x7, offset 0x1c0 + 0x1c0: 0x0e, 0x1c1: 0x0e, 0x1c2: 0x0e, 0x1c3: 0x0e, 0x1c4: 0x0e, 0x1c5: 0x0e, 0x1c6: 0x0e, 0x1c7: 0x0e, + 0x1c8: 0x0e, 0x1c9: 0x0e, 0x1ca: 0x0e, 0x1cb: 0x0e, 0x1cc: 0x0e, 0x1cd: 0x0e, 0x1ce: 0x0e, 0x1cf: 0x0e, + 0x1d0: 0x0e, 0x1d1: 0x0e, 0x1d2: 0x0e, 0x1d3: 0x0e, 0x1d4: 0x0e, 0x1d5: 0x0e, 0x1d6: 0x0e, 0x1d7: 0x0e, + 0x1d8: 0x0e, 0x1d9: 0x0e, 0x1da: 0x0e, 0x1db: 0x0e, 0x1dc: 0x0e, 0x1dd: 0x0e, 0x1de: 0x0e, 0x1df: 0x0e, + 0x1e0: 0x0e, 0x1e1: 0x0e, 0x1e2: 0x0e, 0x1e3: 0x0e, 0x1e4: 0x0e, 0x1e5: 0x0e, 0x1e6: 0x0e, 0x1e7: 0x0e, + 0x1e8: 0x0e, 0x1e9: 0x0e, 0x1ea: 0x0e, 0x1eb: 0x0e, 0x1ec: 0x0e, 0x1ed: 0x0e, 0x1ee: 0x0e, 0x1ef: 0x0e, + 0x1f0: 0x0e, 0x1f1: 0x0e, 0x1f2: 0x0e, 0x1f3: 0x0e, 0x1f4: 0x0e, 0x1f5: 0x0e, 0x1f6: 0x0e, + 0x1f8: 0x0e, 0x1f9: 0x0e, 0x1fa: 0x0e, 0x1fb: 0x0e, 0x1fc: 0x0e, 0x1fd: 0x0e, 0x1fe: 0x0e, 0x1ff: 0x0e, + // Block 0x8, offset 0x200 + 0x200: 0x0e, 0x201: 0x0e, 0x202: 0x0e, 0x203: 0x0e, 0x204: 0x0e, 0x205: 0x0e, 0x206: 0x0e, 0x207: 0x0e, + 0x208: 0x0e, 0x209: 0x0e, 0x20a: 0x0e, 0x20b: 0x0e, 0x20c: 0x0e, 0x20d: 0x0e, 0x20e: 0x0e, 0x20f: 0x0e, + 0x210: 0x0e, 0x211: 0x0e, 0x212: 0x0e, 0x213: 0x0e, 0x214: 0x0e, 0x215: 0x0e, 0x216: 0x0e, 0x217: 0x0e, + 0x218: 0x0e, 0x219: 0x0e, 0x21a: 0x0e, 0x21b: 0x0e, 0x21c: 0x0e, 0x21d: 0x0e, 0x21e: 0x0e, 0x21f: 0x0e, + 0x220: 0x0e, 0x221: 0x0e, 0x222: 0x0e, 0x223: 0x0e, 0x224: 0x0e, 0x225: 0x0e, 0x226: 0x0e, 0x227: 0x0e, + 0x228: 0x0e, 0x229: 0x0e, 0x22a: 0x0e, 0x22b: 0x0e, 0x22c: 0x0e, 0x22d: 0x0e, 0x22e: 0x0e, 0x22f: 0x0e, + 0x230: 0x0e, 0x231: 0x0e, 0x232: 0x0e, 0x233: 0x0e, 0x234: 0x0e, 0x235: 0x0e, 0x236: 0x0e, 0x237: 0x0e, + 0x238: 0x0e, 0x239: 0x0e, 0x23a: 0x0e, 0x23b: 0x0e, 0x23c: 0x0e, 0x23d: 0x0e, 0x23e: 0x0e, 0x23f: 0x0e, + // Block 0x9, offset 0x240 + 0x240: 0x0e, 0x241: 0x0e, 0x242: 0x0e, 0x243: 0x0e, 0x244: 0x0e, 0x245: 0x0e, 0x246: 0x0e, 0x247: 0x0e, + 0x248: 0x0e, 0x249: 0x0e, 0x24a: 0x0e, 0x24b: 0x0e, 0x24c: 0x0e, 0x24d: 0x0e, 0x24e: 0x0e, 0x24f: 0x0e, + 0x250: 0x0e, 0x251: 0x0e, 0x252: 0x3b, 0x253: 0x3c, + 0x265: 0x3d, + 0x270: 0x0e, 0x271: 0x0e, 0x272: 0x0e, 0x273: 0x0e, 0x274: 0x0e, 0x275: 0x0e, 0x276: 0x0e, 0x277: 0x0e, + 0x278: 0x0e, 0x279: 0x0e, 0x27a: 0x0e, 0x27b: 0x0e, 0x27c: 0x0e, 0x27d: 0x0e, 0x27e: 0x0e, 0x27f: 0x0e, + // Block 0xa, offset 0x280 + 0x280: 0x0e, 0x281: 0x0e, 0x282: 0x0e, 0x283: 0x0e, 0x284: 0x0e, 0x285: 0x0e, 0x286: 0x0e, 0x287: 0x0e, + 0x288: 0x0e, 0x289: 0x0e, 0x28a: 0x0e, 0x28b: 0x0e, 0x28c: 0x0e, 0x28d: 0x0e, 0x28e: 0x0e, 0x28f: 0x0e, + 0x290: 0x0e, 0x291: 0x0e, 0x292: 0x0e, 0x293: 0x0e, 0x294: 0x0e, 0x295: 0x0e, 0x296: 0x0e, 0x297: 0x0e, + 0x298: 0x0e, 0x299: 0x0e, 0x29a: 0x0e, 0x29b: 0x0e, 0x29c: 0x0e, 0x29d: 0x0e, 0x29e: 0x3e, + // Block 0xb, offset 0x2c0 + 0x2c0: 0x08, 0x2c1: 0x08, 0x2c2: 0x08, 0x2c3: 0x08, 0x2c4: 0x08, 0x2c5: 0x08, 0x2c6: 0x08, 0x2c7: 0x08, + 0x2c8: 0x08, 0x2c9: 0x08, 0x2ca: 0x08, 0x2cb: 0x08, 0x2cc: 0x08, 0x2cd: 0x08, 0x2ce: 0x08, 0x2cf: 0x08, + 0x2d0: 0x08, 0x2d1: 0x08, 0x2d2: 0x08, 0x2d3: 0x08, 0x2d4: 0x08, 0x2d5: 0x08, 0x2d6: 0x08, 0x2d7: 0x08, + 0x2d8: 0x08, 0x2d9: 0x08, 0x2da: 0x08, 0x2db: 0x08, 0x2dc: 0x08, 0x2dd: 0x08, 0x2de: 0x08, 0x2df: 0x08, + 0x2e0: 0x08, 0x2e1: 0x08, 0x2e2: 0x08, 0x2e3: 0x08, 0x2e4: 0x08, 0x2e5: 0x08, 0x2e6: 0x08, 0x2e7: 0x08, + 0x2e8: 0x08, 0x2e9: 0x08, 0x2ea: 0x08, 0x2eb: 0x08, 0x2ec: 0x08, 0x2ed: 0x08, 0x2ee: 0x08, 0x2ef: 0x08, + 0x2f0: 0x08, 0x2f1: 0x08, 0x2f2: 0x08, 0x2f3: 0x08, 0x2f4: 0x08, 0x2f5: 0x08, 0x2f6: 0x08, 0x2f7: 0x08, + 0x2f8: 0x08, 0x2f9: 0x08, 0x2fa: 0x08, 0x2fb: 0x08, 0x2fc: 0x08, 0x2fd: 0x08, 0x2fe: 0x08, 0x2ff: 0x08, + // Block 0xc, offset 0x300 + 0x300: 0x08, 0x301: 0x08, 0x302: 0x08, 0x303: 0x08, 0x304: 0x08, 0x305: 0x08, 0x306: 0x08, 0x307: 0x08, + 0x308: 0x08, 0x309: 0x08, 0x30a: 0x08, 0x30b: 0x08, 0x30c: 0x08, 0x30d: 0x08, 0x30e: 0x08, 0x30f: 0x08, + 0x310: 0x08, 0x311: 0x08, 0x312: 0x08, 0x313: 0x08, 0x314: 0x08, 0x315: 0x08, 0x316: 0x08, 0x317: 0x08, + 0x318: 0x08, 0x319: 0x08, 0x31a: 0x08, 0x31b: 0x08, 0x31c: 0x08, 0x31d: 0x08, 0x31e: 0x08, 0x31f: 0x08, + 0x320: 0x08, 0x321: 0x08, 0x322: 0x08, 0x323: 0x08, 0x324: 0x0e, 0x325: 0x0e, 0x326: 0x0e, 0x327: 0x0e, + 0x328: 0x0e, 0x329: 0x0e, 0x32a: 0x0e, 0x32b: 0x0e, + 0x338: 0x3f, 0x339: 0x40, 0x33c: 0x41, 0x33d: 0x42, 0x33e: 0x43, 0x33f: 0x44, + // Block 0xd, offset 0x340 + 0x37f: 0x45, + // Block 0xe, offset 0x380 + 0x380: 0x0e, 0x381: 0x0e, 0x382: 0x0e, 0x383: 0x0e, 0x384: 0x0e, 0x385: 0x0e, 0x386: 0x0e, 0x387: 0x0e, + 0x388: 0x0e, 0x389: 0x0e, 0x38a: 0x0e, 0x38b: 0x0e, 0x38c: 0x0e, 0x38d: 0x0e, 0x38e: 0x0e, 0x38f: 0x0e, + 0x390: 0x0e, 0x391: 0x0e, 0x392: 0x0e, 0x393: 0x0e, 0x394: 0x0e, 0x395: 0x0e, 0x396: 0x0e, 0x397: 0x0e, + 0x398: 0x0e, 0x399: 0x0e, 0x39a: 0x0e, 0x39b: 0x0e, 0x39c: 0x0e, 0x39d: 0x0e, 0x39e: 0x0e, 0x39f: 0x46, + 0x3a0: 0x0e, 0x3a1: 0x0e, 0x3a2: 0x0e, 0x3a3: 0x0e, 0x3a4: 0x0e, 0x3a5: 0x0e, 0x3a6: 0x0e, 0x3a7: 0x0e, + 0x3a8: 0x0e, 0x3a9: 0x0e, 0x3aa: 0x0e, 0x3ab: 0x47, + // Block 0xf, offset 0x3c0 + 0x3c0: 0x0e, 0x3c1: 0x0e, 0x3c2: 0x0e, 0x3c3: 0x0e, 0x3c4: 0x48, 0x3c5: 0x49, 0x3c6: 0x0e, 0x3c7: 0x0e, + 0x3c8: 0x0e, 0x3c9: 0x0e, 0x3ca: 0x0e, 0x3cb: 0x4a, + // Block 0x10, offset 0x400 + 0x400: 0x4b, 0x403: 0x4c, 0x404: 0x4d, 0x405: 0x4e, 0x406: 0x4f, + 0x408: 0x50, 0x409: 0x51, 0x40c: 0x52, 0x40d: 0x53, 0x40e: 0x54, 0x40f: 0x55, + 0x410: 0x3a, 0x411: 0x56, 0x412: 0x0e, 0x413: 0x57, 0x414: 0x58, 0x415: 0x59, 0x416: 0x5a, 0x417: 0x5b, + 0x418: 0x0e, 0x419: 0x5c, 0x41a: 0x0e, 0x41b: 0x5d, + 0x424: 0x5e, 0x425: 0x5f, 0x426: 0x60, 0x427: 0x61, + // Block 0x11, offset 0x440 + 0x456: 0x0b, 0x457: 0x06, + 0x458: 0x0c, 0x45b: 0x0d, 0x45f: 0x0e, + 0x460: 0x06, 0x461: 0x06, 0x462: 0x06, 0x463: 0x06, 0x464: 0x06, 0x465: 0x06, 0x466: 0x06, 0x467: 0x06, + 0x468: 0x06, 0x469: 0x06, 0x46a: 0x06, 0x46b: 0x06, 0x46c: 0x06, 0x46d: 0x06, 0x46e: 0x06, 0x46f: 0x06, + 0x470: 0x06, 0x471: 0x06, 0x472: 0x06, 0x473: 0x06, 0x474: 0x06, 0x475: 0x06, 0x476: 0x06, 0x477: 0x06, + 0x478: 0x06, 0x479: 0x06, 0x47a: 0x06, 0x47b: 0x06, 0x47c: 0x06, 0x47d: 0x06, 0x47e: 0x06, 0x47f: 0x06, + // Block 0x12, offset 0x480 + 0x484: 0x08, 0x485: 0x08, 0x486: 0x08, 0x487: 0x09, + // Block 0x13, offset 0x4c0 + 0x4c0: 0x08, 0x4c1: 0x08, 0x4c2: 0x08, 0x4c3: 0x08, 0x4c4: 0x08, 0x4c5: 0x08, 0x4c6: 0x08, 0x4c7: 0x08, + 0x4c8: 0x08, 0x4c9: 0x08, 0x4ca: 0x08, 0x4cb: 0x08, 0x4cc: 0x08, 0x4cd: 0x08, 0x4ce: 0x08, 0x4cf: 0x08, + 0x4d0: 0x08, 0x4d1: 0x08, 0x4d2: 0x08, 0x4d3: 0x08, 0x4d4: 0x08, 0x4d5: 0x08, 0x4d6: 0x08, 0x4d7: 0x08, + 0x4d8: 0x08, 0x4d9: 0x08, 0x4da: 0x08, 0x4db: 0x08, 0x4dc: 0x08, 0x4dd: 0x08, 0x4de: 0x08, 0x4df: 0x08, + 0x4e0: 0x08, 0x4e1: 0x08, 0x4e2: 0x08, 0x4e3: 0x08, 0x4e4: 0x08, 0x4e5: 0x08, 0x4e6: 0x08, 0x4e7: 0x08, + 0x4e8: 0x08, 0x4e9: 0x08, 0x4ea: 0x08, 0x4eb: 0x08, 0x4ec: 0x08, 0x4ed: 0x08, 0x4ee: 0x08, 0x4ef: 0x08, + 0x4f0: 0x08, 0x4f1: 0x08, 0x4f2: 0x08, 0x4f3: 0x08, 0x4f4: 0x08, 0x4f5: 0x08, 0x4f6: 0x08, 0x4f7: 0x08, + 0x4f8: 0x08, 0x4f9: 0x08, 0x4fa: 0x08, 0x4fb: 0x08, 0x4fc: 0x08, 0x4fd: 0x08, 0x4fe: 0x08, 0x4ff: 0x62, + // Block 0x14, offset 0x500 + 0x520: 0x10, + 0x530: 0x09, 0x531: 0x09, 0x532: 0x09, 0x533: 0x09, 0x534: 0x09, 0x535: 0x09, 0x536: 0x09, 0x537: 0x09, + 0x538: 0x09, 0x539: 0x09, 0x53a: 0x09, 0x53b: 0x09, 0x53c: 0x09, 0x53d: 0x09, 0x53e: 0x09, 0x53f: 0x11, + // Block 0x15, offset 0x540 + 0x540: 0x09, 0x541: 0x09, 0x542: 0x09, 0x543: 0x09, 0x544: 0x09, 0x545: 0x09, 0x546: 0x09, 0x547: 0x09, + 0x548: 0x09, 0x549: 0x09, 0x54a: 0x09, 0x54b: 0x09, 0x54c: 0x09, 0x54d: 0x09, 0x54e: 0x09, 0x54f: 0x11, +} + +// inverseData contains 4-byte entries of the following format: +// +// <0 padding> +// +// The last byte of the UTF-8-encoded rune is xor-ed with the last byte of the +// UTF-8 encoding of the original rune. Mappings often have the following +// pattern: +// +// Ａ -> A (U+FF21 -> U+0041) +// Ｂ -> B (U+FF22 -> U+0042) +// ... +// +// By xor-ing the last byte the same entry can be shared by many mappings. This +// reduces the total number of distinct entries by about two thirds. +// The resulting entry for the aforementioned mappings is +// +// { 0x01, 0xE0, 0x00, 0x00 } +// +// Using this entry to map U+FF21 (UTF-8 [EF BC A1]), we get +// +// E0 ^ A1 = 41. +// +// Similarly, for U+FF22 (UTF-8 [EF BC A2]), we get +// +// E0 ^ A2 = 42. +// +// Note that because of the xor-ing, the byte sequence stored in the entry is +// not valid UTF-8. +var inverseData = [150][4]byte{ + {0x00, 0x00, 0x00, 0x00}, + {0x03, 0xe3, 0x80, 0xa0}, + {0x03, 0xef, 0xbc, 0xa0}, + {0x03, 0xef, 0xbc, 0xe0}, + {0x03, 0xef, 0xbd, 0xe0}, + {0x03, 0xef, 0xbf, 0x02}, + {0x03, 0xef, 0xbf, 0x00}, + {0x03, 0xef, 0xbf, 0x0e}, + {0x03, 0xef, 0xbf, 0x0c}, + {0x03, 0xef, 0xbf, 0x0f}, + {0x03, 0xef, 0xbf, 0x39}, + {0x03, 0xef, 0xbf, 0x3b}, + {0x03, 0xef, 0xbf, 0x3f}, + {0x03, 0xef, 0xbf, 0x2a}, + {0x03, 0xef, 0xbf, 0x0d}, + {0x03, 0xef, 0xbf, 0x25}, + {0x03, 0xef, 0xbd, 0x1a}, + {0x03, 0xef, 0xbd, 0x26}, + {0x01, 0xa0, 0x00, 0x00}, + {0x03, 0xef, 0xbd, 0x25}, + {0x03, 0xef, 0xbd, 0x23}, + {0x03, 0xef, 0xbd, 0x2e}, + {0x03, 0xef, 0xbe, 0x07}, + {0x03, 0xef, 0xbe, 0x05}, + {0x03, 0xef, 0xbd, 0x06}, + {0x03, 0xef, 0xbd, 0x13}, + {0x03, 0xef, 0xbd, 0x0b}, + {0x03, 0xef, 0xbd, 0x16}, + {0x03, 0xef, 0xbd, 0x0c}, + {0x03, 0xef, 0xbd, 0x15}, + {0x03, 0xef, 0xbd, 0x0d}, + {0x03, 0xef, 0xbd, 0x1c}, + {0x03, 0xef, 0xbd, 0x02}, + {0x03, 0xef, 0xbd, 0x1f}, + {0x03, 0xef, 0xbd, 0x1d}, + {0x03, 0xef, 0xbd, 0x17}, + {0x03, 0xef, 0xbd, 0x08}, + {0x03, 0xef, 0xbd, 0x09}, + {0x03, 0xef, 0xbd, 0x0e}, + {0x03, 0xef, 0xbd, 0x04}, + {0x03, 0xef, 0xbd, 0x05}, + {0x03, 0xef, 0xbe, 0x3f}, + {0x03, 0xef, 0xbe, 0x00}, + {0x03, 0xef, 0xbd, 0x2c}, + {0x03, 0xef, 0xbe, 0x06}, + {0x03, 0xef, 0xbe, 0x0c}, + {0x03, 0xef, 0xbe, 0x0f}, + {0x03, 0xef, 0xbe, 0x0d}, + {0x03, 0xef, 0xbe, 0x0b}, + {0x03, 0xef, 0xbe, 0x19}, + {0x03, 0xef, 0xbe, 0x15}, + {0x03, 0xef, 0xbe, 0x11}, + {0x03, 0xef, 0xbe, 0x31}, + {0x03, 0xef, 0xbe, 0x33}, + {0x03, 0xef, 0xbd, 0x0f}, + {0x03, 0xef, 0xbe, 0x30}, + {0x03, 0xef, 0xbe, 0x3e}, + {0x03, 0xef, 0xbe, 0x32}, + {0x03, 0xef, 0xbe, 0x36}, + {0x03, 0xef, 0xbd, 0x14}, + {0x03, 0xef, 0xbe, 0x2e}, + {0x03, 0xef, 0xbd, 0x1e}, + {0x03, 0xef, 0xbe, 0x10}, + {0x03, 0xef, 0xbf, 0x13}, + {0x03, 0xef, 0xbf, 0x15}, + {0x03, 0xef, 0xbf, 0x17}, + {0x03, 0xef, 0xbf, 0x1f}, + {0x03, 0xef, 0xbf, 0x1d}, + {0x03, 0xef, 0xbf, 0x1b}, + {0x03, 0xef, 0xbf, 0x09}, + {0x03, 0xef, 0xbf, 0x0b}, + {0x03, 0xef, 0xbf, 0x37}, + {0x03, 0xef, 0xbe, 0x04}, + {0x01, 0xe0, 0x00, 0x00}, + {0x03, 0xe2, 0xa6, 0x1a}, + {0x03, 0xe2, 0xa6, 0x26}, + {0x03, 0xe3, 0x80, 0x23}, + {0x03, 0xe3, 0x80, 0x2e}, + {0x03, 0xe3, 0x80, 0x25}, + {0x03, 0xe3, 0x83, 0x1e}, + {0x03, 0xe3, 0x83, 0x14}, + {0x03, 0xe3, 0x82, 0x06}, + {0x03, 0xe3, 0x82, 0x0b}, + {0x03, 0xe3, 0x82, 0x0c}, + {0x03, 0xe3, 0x82, 0x0d}, + {0x03, 0xe3, 0x82, 0x02}, + {0x03, 0xe3, 0x83, 0x0f}, + {0x03, 0xe3, 0x83, 0x08}, + {0x03, 0xe3, 0x83, 0x09}, + {0x03, 0xe3, 0x83, 0x2c}, + {0x03, 0xe3, 0x83, 0x0c}, + {0x03, 0xe3, 0x82, 0x13}, + {0x03, 0xe3, 0x82, 0x16}, + {0x03, 0xe3, 0x82, 0x15}, + {0x03, 0xe3, 0x82, 0x1c}, + {0x03, 0xe3, 0x82, 0x1f}, + {0x03, 0xe3, 0x82, 0x1d}, + {0x03, 0xe3, 0x82, 0x1a}, + {0x03, 0xe3, 0x82, 0x17}, + {0x03, 0xe3, 0x82, 0x08}, + {0x03, 0xe3, 0x82, 0x09}, + {0x03, 0xe3, 0x82, 0x0e}, + {0x03, 0xe3, 0x82, 0x04}, + {0x03, 0xe3, 0x82, 0x05}, + {0x03, 0xe3, 0x82, 0x3f}, + {0x03, 0xe3, 0x83, 0x00}, + {0x03, 0xe3, 0x83, 0x06}, + {0x03, 0xe3, 0x83, 0x05}, + {0x03, 0xe3, 0x83, 0x0d}, + {0x03, 0xe3, 0x83, 0x0b}, + {0x03, 0xe3, 0x83, 0x07}, + {0x03, 0xe3, 0x83, 0x19}, + {0x03, 0xe3, 0x83, 0x15}, + {0x03, 0xe3, 0x83, 0x11}, + {0x03, 0xe3, 0x83, 0x31}, + {0x03, 0xe3, 0x83, 0x33}, + {0x03, 0xe3, 0x83, 0x30}, + {0x03, 0xe3, 0x83, 0x3e}, + {0x03, 0xe3, 0x83, 0x32}, + {0x03, 0xe3, 0x83, 0x36}, + {0x03, 0xe3, 0x83, 0x2e}, + {0x03, 0xe3, 0x82, 0x07}, + {0x03, 0xe3, 0x85, 0x04}, + {0x03, 0xe3, 0x84, 0x10}, + {0x03, 0xe3, 0x85, 0x30}, + {0x03, 0xe3, 0x85, 0x0d}, + {0x03, 0xe3, 0x85, 0x13}, + {0x03, 0xe3, 0x85, 0x15}, + {0x03, 0xe3, 0x85, 0x17}, + {0x03, 0xe3, 0x85, 0x1f}, + {0x03, 0xe3, 0x85, 0x1d}, + {0x03, 0xe3, 0x85, 0x1b}, + {0x03, 0xe3, 0x85, 0x09}, + {0x03, 0xe3, 0x85, 0x0f}, + {0x03, 0xe3, 0x85, 0x0b}, + {0x03, 0xe3, 0x85, 0x37}, + {0x03, 0xe3, 0x85, 0x3b}, + {0x03, 0xe3, 0x85, 0x39}, + {0x03, 0xe3, 0x85, 0x3f}, + {0x02, 0xc2, 0x02, 0x00}, + {0x02, 0xc2, 0x0e, 0x00}, + {0x02, 0xc2, 0x0c, 0x00}, + {0x02, 0xc2, 0x00, 0x00}, + {0x03, 0xe2, 0x82, 0x0f}, + {0x03, 0xe2, 0x94, 0x2a}, + {0x03, 0xe2, 0x86, 0x39}, + {0x03, 0xe2, 0x86, 0x3b}, + {0x03, 0xe2, 0x86, 0x3f}, + {0x03, 0xe2, 0x96, 0x0d}, + {0x03, 0xe2, 0x97, 0x25}, +} + +// Total table size 14936 bytes (14KiB) diff --git a/vendor/golang.org/x/text/width/tables12.0.0.go b/vendor/golang.org/x/text/width/tables12.0.0.go new file mode 100644 index 0000000000..5c14ade6d9 --- /dev/null +++ b/vendor/golang.org/x/text/width/tables12.0.0.go @@ -0,0 +1,1361 @@ +// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. + +//go:build go1.14 && !go1.16 +// +build go1.14,!go1.16 + +package width + +// UnicodeVersion is the Unicode version from which the tables in this package are derived. +const UnicodeVersion = "12.0.0" + +// lookup returns the trie value for the first UTF-8 encoding in s and +// the width in bytes of this encoding. The size will be 0 if s does not +// hold enough bytes to complete the encoding. len(s) must be greater than 0. +func (t *widthTrie) lookup(s []byte) (v uint16, sz int) { + c0 := s[0] + switch { + case c0 < 0x80: // is ASCII + return widthValues[c0], 1 + case c0 < 0xC2: + return 0, 1 // Illegal UTF-8: not a starter, not ASCII. + case c0 < 0xE0: // 2-byte UTF-8 + if len(s) < 2 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c1), 2 + case c0 < 0xF0: // 3-byte UTF-8 + if len(s) < 3 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = widthIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c2), 3 + case c0 < 0xF8: // 4-byte UTF-8 + if len(s) < 4 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = widthIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + o = uint32(i)<<6 + uint32(c2) + i = widthIndex[o] + c3 := s[3] + if c3 < 0x80 || 0xC0 <= c3 { + return 0, 3 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c3), 4 + } + // Illegal rune + return 0, 1 +} + +// lookupUnsafe returns the trie value for the first UTF-8 encoding in s. +// s must start with a full and valid UTF-8 encoded rune. +func (t *widthTrie) lookupUnsafe(s []byte) uint16 { + c0 := s[0] + if c0 < 0x80 { // is ASCII + return widthValues[c0] + } + i := widthIndex[c0] + if c0 < 0xE0 { // 2-byte UTF-8 + return t.lookupValue(uint32(i), s[1]) + } + i = widthIndex[uint32(i)<<6+uint32(s[1])] + if c0 < 0xF0 { // 3-byte UTF-8 + return t.lookupValue(uint32(i), s[2]) + } + i = widthIndex[uint32(i)<<6+uint32(s[2])] + if c0 < 0xF8 { // 4-byte UTF-8 + return t.lookupValue(uint32(i), s[3]) + } + return 0 +} + +// lookupString returns the trie value for the first UTF-8 encoding in s and +// the width in bytes of this encoding. The size will be 0 if s does not +// hold enough bytes to complete the encoding. len(s) must be greater than 0. +func (t *widthTrie) lookupString(s string) (v uint16, sz int) { + c0 := s[0] + switch { + case c0 < 0x80: // is ASCII + return widthValues[c0], 1 + case c0 < 0xC2: + return 0, 1 // Illegal UTF-8: not a starter, not ASCII. + case c0 < 0xE0: // 2-byte UTF-8 + if len(s) < 2 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c1), 2 + case c0 < 0xF0: // 3-byte UTF-8 + if len(s) < 3 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = widthIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c2), 3 + case c0 < 0xF8: // 4-byte UTF-8 + if len(s) < 4 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = widthIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + o = uint32(i)<<6 + uint32(c2) + i = widthIndex[o] + c3 := s[3] + if c3 < 0x80 || 0xC0 <= c3 { + return 0, 3 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c3), 4 + } + // Illegal rune + return 0, 1 +} + +// lookupStringUnsafe returns the trie value for the first UTF-8 encoding in s. +// s must start with a full and valid UTF-8 encoded rune. +func (t *widthTrie) lookupStringUnsafe(s string) uint16 { + c0 := s[0] + if c0 < 0x80 { // is ASCII + return widthValues[c0] + } + i := widthIndex[c0] + if c0 < 0xE0 { // 2-byte UTF-8 + return t.lookupValue(uint32(i), s[1]) + } + i = widthIndex[uint32(i)<<6+uint32(s[1])] + if c0 < 0xF0 { // 3-byte UTF-8 + return t.lookupValue(uint32(i), s[2]) + } + i = widthIndex[uint32(i)<<6+uint32(s[2])] + if c0 < 0xF8 { // 4-byte UTF-8 + return t.lookupValue(uint32(i), s[3]) + } + return 0 +} + +// widthTrie. Total size: 14720 bytes (14.38 KiB). Checksum: 3f4f2516ded5489b. +type widthTrie struct{} + +func newWidthTrie(i int) *widthTrie { + return &widthTrie{} +} + +// lookupValue determines the type of block n and looks up the value for b. +func (t *widthTrie) lookupValue(n uint32, b byte) uint16 { + switch { + default: + return uint16(widthValues[n<<6+uint32(b)]) + } +} + +// widthValues: 104 blocks, 6656 entries, 13312 bytes +// The third block is the zero block. +var widthValues = [6656]uint16{ + // Block 0x0, offset 0x0 + 0x20: 0x6001, 0x21: 0x6002, 0x22: 0x6002, 0x23: 0x6002, + 0x24: 0x6002, 0x25: 0x6002, 0x26: 0x6002, 0x27: 0x6002, 0x28: 0x6002, 0x29: 0x6002, + 0x2a: 0x6002, 0x2b: 0x6002, 0x2c: 0x6002, 0x2d: 0x6002, 0x2e: 0x6002, 0x2f: 0x6002, + 0x30: 0x6002, 0x31: 0x6002, 0x32: 0x6002, 0x33: 0x6002, 0x34: 0x6002, 0x35: 0x6002, + 0x36: 0x6002, 0x37: 0x6002, 0x38: 0x6002, 0x39: 0x6002, 0x3a: 0x6002, 0x3b: 0x6002, + 0x3c: 0x6002, 0x3d: 0x6002, 0x3e: 0x6002, 0x3f: 0x6002, + // Block 0x1, offset 0x40 + 0x40: 0x6003, 0x41: 0x6003, 0x42: 0x6003, 0x43: 0x6003, 0x44: 0x6003, 0x45: 0x6003, + 0x46: 0x6003, 0x47: 0x6003, 0x48: 0x6003, 0x49: 0x6003, 0x4a: 0x6003, 0x4b: 0x6003, + 0x4c: 0x6003, 0x4d: 0x6003, 0x4e: 0x6003, 0x4f: 0x6003, 0x50: 0x6003, 0x51: 0x6003, + 0x52: 0x6003, 0x53: 0x6003, 0x54: 0x6003, 0x55: 0x6003, 0x56: 0x6003, 0x57: 0x6003, + 0x58: 0x6003, 0x59: 0x6003, 0x5a: 0x6003, 0x5b: 0x6003, 0x5c: 0x6003, 0x5d: 0x6003, + 0x5e: 0x6003, 0x5f: 0x6003, 0x60: 0x6004, 0x61: 0x6004, 0x62: 0x6004, 0x63: 0x6004, + 0x64: 0x6004, 0x65: 0x6004, 0x66: 0x6004, 0x67: 0x6004, 0x68: 0x6004, 0x69: 0x6004, + 0x6a: 0x6004, 0x6b: 0x6004, 0x6c: 0x6004, 0x6d: 0x6004, 0x6e: 0x6004, 0x6f: 0x6004, + 0x70: 0x6004, 0x71: 0x6004, 0x72: 0x6004, 0x73: 0x6004, 0x74: 0x6004, 0x75: 0x6004, + 0x76: 0x6004, 0x77: 0x6004, 0x78: 0x6004, 0x79: 0x6004, 0x7a: 0x6004, 0x7b: 0x6004, + 0x7c: 0x6004, 0x7d: 0x6004, 0x7e: 0x6004, + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0xe1: 0x2000, 0xe2: 0x6005, 0xe3: 0x6005, + 0xe4: 0x2000, 0xe5: 0x6006, 0xe6: 0x6005, 0xe7: 0x2000, 0xe8: 0x2000, + 0xea: 0x2000, 0xec: 0x6007, 0xed: 0x2000, 0xee: 0x2000, 0xef: 0x6008, + 0xf0: 0x2000, 0xf1: 0x2000, 0xf2: 0x2000, 0xf3: 0x2000, 0xf4: 0x2000, + 0xf6: 0x2000, 0xf7: 0x2000, 0xf8: 0x2000, 0xf9: 0x2000, 0xfa: 0x2000, + 0xfc: 0x2000, 0xfd: 0x2000, 0xfe: 0x2000, 0xff: 0x2000, + // Block 0x4, offset 0x100 + 0x106: 0x2000, + 0x110: 0x2000, + 0x117: 0x2000, + 0x118: 0x2000, + 0x11e: 0x2000, 0x11f: 0x2000, 0x120: 0x2000, 0x121: 0x2000, + 0x126: 0x2000, 0x128: 0x2000, 0x129: 0x2000, + 0x12a: 0x2000, 0x12c: 0x2000, 0x12d: 0x2000, + 0x130: 0x2000, 0x132: 0x2000, 0x133: 0x2000, + 0x137: 0x2000, 0x138: 0x2000, 0x139: 0x2000, 0x13a: 0x2000, + 0x13c: 0x2000, 0x13e: 0x2000, + // Block 0x5, offset 0x140 + 0x141: 0x2000, + 0x151: 0x2000, + 0x153: 0x2000, + 0x15b: 0x2000, + 0x166: 0x2000, 0x167: 0x2000, + 0x16b: 0x2000, + 0x171: 0x2000, 0x172: 0x2000, 0x173: 0x2000, + 0x178: 0x2000, + 0x17f: 0x2000, + // Block 0x6, offset 0x180 + 0x180: 0x2000, 0x181: 0x2000, 0x182: 0x2000, 0x184: 0x2000, + 0x188: 0x2000, 0x189: 0x2000, 0x18a: 0x2000, 0x18b: 0x2000, + 0x18d: 0x2000, + 0x192: 0x2000, 0x193: 0x2000, + 0x1a6: 0x2000, 0x1a7: 0x2000, + 0x1ab: 0x2000, + // Block 0x7, offset 0x1c0 + 0x1ce: 0x2000, 0x1d0: 0x2000, + 0x1d2: 0x2000, 0x1d4: 0x2000, 0x1d6: 0x2000, + 0x1d8: 0x2000, 0x1da: 0x2000, 0x1dc: 0x2000, + // Block 0x8, offset 0x200 + 0x211: 0x2000, + 0x221: 0x2000, + // Block 0x9, offset 0x240 + 0x244: 0x2000, + 0x247: 0x2000, 0x249: 0x2000, 0x24a: 0x2000, 0x24b: 0x2000, + 0x24d: 0x2000, 0x250: 0x2000, + 0x258: 0x2000, 0x259: 0x2000, 0x25a: 0x2000, 0x25b: 0x2000, 0x25d: 0x2000, + 0x25f: 0x2000, + // Block 0xa, offset 0x280 + 0x280: 0x2000, 0x281: 0x2000, 0x282: 0x2000, 0x283: 0x2000, 0x284: 0x2000, 0x285: 0x2000, + 0x286: 0x2000, 0x287: 0x2000, 0x288: 0x2000, 0x289: 0x2000, 0x28a: 0x2000, 0x28b: 0x2000, + 0x28c: 0x2000, 0x28d: 0x2000, 0x28e: 0x2000, 0x28f: 0x2000, 0x290: 0x2000, 0x291: 0x2000, + 0x292: 0x2000, 0x293: 0x2000, 0x294: 0x2000, 0x295: 0x2000, 0x296: 0x2000, 0x297: 0x2000, + 0x298: 0x2000, 0x299: 0x2000, 0x29a: 0x2000, 0x29b: 0x2000, 0x29c: 0x2000, 0x29d: 0x2000, + 0x29e: 0x2000, 0x29f: 0x2000, 0x2a0: 0x2000, 0x2a1: 0x2000, 0x2a2: 0x2000, 0x2a3: 0x2000, + 0x2a4: 0x2000, 0x2a5: 0x2000, 0x2a6: 0x2000, 0x2a7: 0x2000, 0x2a8: 0x2000, 0x2a9: 0x2000, + 0x2aa: 0x2000, 0x2ab: 0x2000, 0x2ac: 0x2000, 0x2ad: 0x2000, 0x2ae: 0x2000, 0x2af: 0x2000, + 0x2b0: 0x2000, 0x2b1: 0x2000, 0x2b2: 0x2000, 0x2b3: 0x2000, 0x2b4: 0x2000, 0x2b5: 0x2000, + 0x2b6: 0x2000, 0x2b7: 0x2000, 0x2b8: 0x2000, 0x2b9: 0x2000, 0x2ba: 0x2000, 0x2bb: 0x2000, + 0x2bc: 0x2000, 0x2bd: 0x2000, 0x2be: 0x2000, 0x2bf: 0x2000, + // Block 0xb, offset 0x2c0 + 0x2c0: 0x2000, 0x2c1: 0x2000, 0x2c2: 0x2000, 0x2c3: 0x2000, 0x2c4: 0x2000, 0x2c5: 0x2000, + 0x2c6: 0x2000, 0x2c7: 0x2000, 0x2c8: 0x2000, 0x2c9: 0x2000, 0x2ca: 0x2000, 0x2cb: 0x2000, + 0x2cc: 0x2000, 0x2cd: 0x2000, 0x2ce: 0x2000, 0x2cf: 0x2000, 0x2d0: 0x2000, 0x2d1: 0x2000, + 0x2d2: 0x2000, 0x2d3: 0x2000, 0x2d4: 0x2000, 0x2d5: 0x2000, 0x2d6: 0x2000, 0x2d7: 0x2000, + 0x2d8: 0x2000, 0x2d9: 0x2000, 0x2da: 0x2000, 0x2db: 0x2000, 0x2dc: 0x2000, 0x2dd: 0x2000, + 0x2de: 0x2000, 0x2df: 0x2000, 0x2e0: 0x2000, 0x2e1: 0x2000, 0x2e2: 0x2000, 0x2e3: 0x2000, + 0x2e4: 0x2000, 0x2e5: 0x2000, 0x2e6: 0x2000, 0x2e7: 0x2000, 0x2e8: 0x2000, 0x2e9: 0x2000, + 0x2ea: 0x2000, 0x2eb: 0x2000, 0x2ec: 0x2000, 0x2ed: 0x2000, 0x2ee: 0x2000, 0x2ef: 0x2000, + // Block 0xc, offset 0x300 + 0x311: 0x2000, + 0x312: 0x2000, 0x313: 0x2000, 0x314: 0x2000, 0x315: 0x2000, 0x316: 0x2000, 0x317: 0x2000, + 0x318: 0x2000, 0x319: 0x2000, 0x31a: 0x2000, 0x31b: 0x2000, 0x31c: 0x2000, 0x31d: 0x2000, + 0x31e: 0x2000, 0x31f: 0x2000, 0x320: 0x2000, 0x321: 0x2000, 0x323: 0x2000, + 0x324: 0x2000, 0x325: 0x2000, 0x326: 0x2000, 0x327: 0x2000, 0x328: 0x2000, 0x329: 0x2000, + 0x331: 0x2000, 0x332: 0x2000, 0x333: 0x2000, 0x334: 0x2000, 0x335: 0x2000, + 0x336: 0x2000, 0x337: 0x2000, 0x338: 0x2000, 0x339: 0x2000, 0x33a: 0x2000, 0x33b: 0x2000, + 0x33c: 0x2000, 0x33d: 0x2000, 0x33e: 0x2000, 0x33f: 0x2000, + // Block 0xd, offset 0x340 + 0x340: 0x2000, 0x341: 0x2000, 0x343: 0x2000, 0x344: 0x2000, 0x345: 0x2000, + 0x346: 0x2000, 0x347: 0x2000, 0x348: 0x2000, 0x349: 0x2000, + // Block 0xe, offset 0x380 + 0x381: 0x2000, + 0x390: 0x2000, 0x391: 0x2000, + 0x392: 0x2000, 0x393: 0x2000, 0x394: 0x2000, 0x395: 0x2000, 0x396: 0x2000, 0x397: 0x2000, + 0x398: 0x2000, 0x399: 0x2000, 0x39a: 0x2000, 0x39b: 0x2000, 0x39c: 0x2000, 0x39d: 0x2000, + 0x39e: 0x2000, 0x39f: 0x2000, 0x3a0: 0x2000, 0x3a1: 0x2000, 0x3a2: 0x2000, 0x3a3: 0x2000, + 0x3a4: 0x2000, 0x3a5: 0x2000, 0x3a6: 0x2000, 0x3a7: 0x2000, 0x3a8: 0x2000, 0x3a9: 0x2000, + 0x3aa: 0x2000, 0x3ab: 0x2000, 0x3ac: 0x2000, 0x3ad: 0x2000, 0x3ae: 0x2000, 0x3af: 0x2000, + 0x3b0: 0x2000, 0x3b1: 0x2000, 0x3b2: 0x2000, 0x3b3: 0x2000, 0x3b4: 0x2000, 0x3b5: 0x2000, + 0x3b6: 0x2000, 0x3b7: 0x2000, 0x3b8: 0x2000, 0x3b9: 0x2000, 0x3ba: 0x2000, 0x3bb: 0x2000, + 0x3bc: 0x2000, 0x3bd: 0x2000, 0x3be: 0x2000, 0x3bf: 0x2000, + // Block 0xf, offset 0x3c0 + 0x3c0: 0x2000, 0x3c1: 0x2000, 0x3c2: 0x2000, 0x3c3: 0x2000, 0x3c4: 0x2000, 0x3c5: 0x2000, + 0x3c6: 0x2000, 0x3c7: 0x2000, 0x3c8: 0x2000, 0x3c9: 0x2000, 0x3ca: 0x2000, 0x3cb: 0x2000, + 0x3cc: 0x2000, 0x3cd: 0x2000, 0x3ce: 0x2000, 0x3cf: 0x2000, 0x3d1: 0x2000, + // Block 0x10, offset 0x400 + 0x400: 0x4000, 0x401: 0x4000, 0x402: 0x4000, 0x403: 0x4000, 0x404: 0x4000, 0x405: 0x4000, + 0x406: 0x4000, 0x407: 0x4000, 0x408: 0x4000, 0x409: 0x4000, 0x40a: 0x4000, 0x40b: 0x4000, + 0x40c: 0x4000, 0x40d: 0x4000, 0x40e: 0x4000, 0x40f: 0x4000, 0x410: 0x4000, 0x411: 0x4000, + 0x412: 0x4000, 0x413: 0x4000, 0x414: 0x4000, 0x415: 0x4000, 0x416: 0x4000, 0x417: 0x4000, + 0x418: 0x4000, 0x419: 0x4000, 0x41a: 0x4000, 0x41b: 0x4000, 0x41c: 0x4000, 0x41d: 0x4000, + 0x41e: 0x4000, 0x41f: 0x4000, 0x420: 0x4000, 0x421: 0x4000, 0x422: 0x4000, 0x423: 0x4000, + 0x424: 0x4000, 0x425: 0x4000, 0x426: 0x4000, 0x427: 0x4000, 0x428: 0x4000, 0x429: 0x4000, + 0x42a: 0x4000, 0x42b: 0x4000, 0x42c: 0x4000, 0x42d: 0x4000, 0x42e: 0x4000, 0x42f: 0x4000, + 0x430: 0x4000, 0x431: 0x4000, 0x432: 0x4000, 0x433: 0x4000, 0x434: 0x4000, 0x435: 0x4000, + 0x436: 0x4000, 0x437: 0x4000, 0x438: 0x4000, 0x439: 0x4000, 0x43a: 0x4000, 0x43b: 0x4000, + 0x43c: 0x4000, 0x43d: 0x4000, 0x43e: 0x4000, 0x43f: 0x4000, + // Block 0x11, offset 0x440 + 0x440: 0x4000, 0x441: 0x4000, 0x442: 0x4000, 0x443: 0x4000, 0x444: 0x4000, 0x445: 0x4000, + 0x446: 0x4000, 0x447: 0x4000, 0x448: 0x4000, 0x449: 0x4000, 0x44a: 0x4000, 0x44b: 0x4000, + 0x44c: 0x4000, 0x44d: 0x4000, 0x44e: 0x4000, 0x44f: 0x4000, 0x450: 0x4000, 0x451: 0x4000, + 0x452: 0x4000, 0x453: 0x4000, 0x454: 0x4000, 0x455: 0x4000, 0x456: 0x4000, 0x457: 0x4000, + 0x458: 0x4000, 0x459: 0x4000, 0x45a: 0x4000, 0x45b: 0x4000, 0x45c: 0x4000, 0x45d: 0x4000, + 0x45e: 0x4000, 0x45f: 0x4000, + // Block 0x12, offset 0x480 + 0x490: 0x2000, + 0x493: 0x2000, 0x494: 0x2000, 0x495: 0x2000, 0x496: 0x2000, + 0x498: 0x2000, 0x499: 0x2000, 0x49c: 0x2000, 0x49d: 0x2000, + 0x4a0: 0x2000, 0x4a1: 0x2000, 0x4a2: 0x2000, + 0x4a4: 0x2000, 0x4a5: 0x2000, 0x4a6: 0x2000, 0x4a7: 0x2000, + 0x4b0: 0x2000, 0x4b2: 0x2000, 0x4b3: 0x2000, 0x4b5: 0x2000, + 0x4bb: 0x2000, + 0x4be: 0x2000, + // Block 0x13, offset 0x4c0 + 0x4f4: 0x2000, + 0x4ff: 0x2000, + // Block 0x14, offset 0x500 + 0x501: 0x2000, 0x502: 0x2000, 0x503: 0x2000, 0x504: 0x2000, + 0x529: 0xa009, + 0x52c: 0x2000, + // Block 0x15, offset 0x540 + 0x543: 0x2000, 0x545: 0x2000, + 0x549: 0x2000, + 0x553: 0x2000, 0x556: 0x2000, + 0x561: 0x2000, 0x562: 0x2000, + 0x566: 0x2000, + 0x56b: 0x2000, + // Block 0x16, offset 0x580 + 0x593: 0x2000, 0x594: 0x2000, + 0x59b: 0x2000, 0x59c: 0x2000, 0x59d: 0x2000, + 0x59e: 0x2000, 0x5a0: 0x2000, 0x5a1: 0x2000, 0x5a2: 0x2000, 0x5a3: 0x2000, + 0x5a4: 0x2000, 0x5a5: 0x2000, 0x5a6: 0x2000, 0x5a7: 0x2000, 0x5a8: 0x2000, 0x5a9: 0x2000, + 0x5aa: 0x2000, 0x5ab: 0x2000, + 0x5b0: 0x2000, 0x5b1: 0x2000, 0x5b2: 0x2000, 0x5b3: 0x2000, 0x5b4: 0x2000, 0x5b5: 0x2000, + 0x5b6: 0x2000, 0x5b7: 0x2000, 0x5b8: 0x2000, 0x5b9: 0x2000, + // Block 0x17, offset 0x5c0 + 0x5c9: 0x2000, + 0x5d0: 0x200a, 0x5d1: 0x200b, + 0x5d2: 0x200a, 0x5d3: 0x200c, 0x5d4: 0x2000, 0x5d5: 0x2000, 0x5d6: 0x2000, 0x5d7: 0x2000, + 0x5d8: 0x2000, 0x5d9: 0x2000, + 0x5f8: 0x2000, 0x5f9: 0x2000, + // Block 0x18, offset 0x600 + 0x612: 0x2000, 0x614: 0x2000, + 0x627: 0x2000, + // Block 0x19, offset 0x640 + 0x640: 0x2000, 0x642: 0x2000, 0x643: 0x2000, + 0x647: 0x2000, 0x648: 0x2000, 0x64b: 0x2000, + 0x64f: 0x2000, 0x651: 0x2000, + 0x655: 0x2000, + 0x65a: 0x2000, 0x65d: 0x2000, + 0x65e: 0x2000, 0x65f: 0x2000, 0x660: 0x2000, 0x663: 0x2000, + 0x665: 0x2000, 0x667: 0x2000, 0x668: 0x2000, 0x669: 0x2000, + 0x66a: 0x2000, 0x66b: 0x2000, 0x66c: 0x2000, 0x66e: 0x2000, + 0x674: 0x2000, 0x675: 0x2000, + 0x676: 0x2000, 0x677: 0x2000, + 0x67c: 0x2000, 0x67d: 0x2000, + // Block 0x1a, offset 0x680 + 0x688: 0x2000, + 0x68c: 0x2000, + 0x692: 0x2000, + 0x6a0: 0x2000, 0x6a1: 0x2000, + 0x6a4: 0x2000, 0x6a5: 0x2000, 0x6a6: 0x2000, 0x6a7: 0x2000, + 0x6aa: 0x2000, 0x6ab: 0x2000, 0x6ae: 0x2000, 0x6af: 0x2000, + // Block 0x1b, offset 0x6c0 + 0x6c2: 0x2000, 0x6c3: 0x2000, + 0x6c6: 0x2000, 0x6c7: 0x2000, + 0x6d5: 0x2000, + 0x6d9: 0x2000, + 0x6e5: 0x2000, + 0x6ff: 0x2000, + // Block 0x1c, offset 0x700 + 0x712: 0x2000, + 0x71a: 0x4000, 0x71b: 0x4000, + 0x729: 0x4000, + 0x72a: 0x4000, + // Block 0x1d, offset 0x740 + 0x769: 0x4000, + 0x76a: 0x4000, 0x76b: 0x4000, 0x76c: 0x4000, + 0x770: 0x4000, 0x773: 0x4000, + // Block 0x1e, offset 0x780 + 0x7a0: 0x2000, 0x7a1: 0x2000, 0x7a2: 0x2000, 0x7a3: 0x2000, + 0x7a4: 0x2000, 0x7a5: 0x2000, 0x7a6: 0x2000, 0x7a7: 0x2000, 0x7a8: 0x2000, 0x7a9: 0x2000, + 0x7aa: 0x2000, 0x7ab: 0x2000, 0x7ac: 0x2000, 0x7ad: 0x2000, 0x7ae: 0x2000, 0x7af: 0x2000, + 0x7b0: 0x2000, 0x7b1: 0x2000, 0x7b2: 0x2000, 0x7b3: 0x2000, 0x7b4: 0x2000, 0x7b5: 0x2000, + 0x7b6: 0x2000, 0x7b7: 0x2000, 0x7b8: 0x2000, 0x7b9: 0x2000, 0x7ba: 0x2000, 0x7bb: 0x2000, + 0x7bc: 0x2000, 0x7bd: 0x2000, 0x7be: 0x2000, 0x7bf: 0x2000, + // Block 0x1f, offset 0x7c0 + 0x7c0: 0x2000, 0x7c1: 0x2000, 0x7c2: 0x2000, 0x7c3: 0x2000, 0x7c4: 0x2000, 0x7c5: 0x2000, + 0x7c6: 0x2000, 0x7c7: 0x2000, 0x7c8: 0x2000, 0x7c9: 0x2000, 0x7ca: 0x2000, 0x7cb: 0x2000, + 0x7cc: 0x2000, 0x7cd: 0x2000, 0x7ce: 0x2000, 0x7cf: 0x2000, 0x7d0: 0x2000, 0x7d1: 0x2000, + 0x7d2: 0x2000, 0x7d3: 0x2000, 0x7d4: 0x2000, 0x7d5: 0x2000, 0x7d6: 0x2000, 0x7d7: 0x2000, + 0x7d8: 0x2000, 0x7d9: 0x2000, 0x7da: 0x2000, 0x7db: 0x2000, 0x7dc: 0x2000, 0x7dd: 0x2000, + 0x7de: 0x2000, 0x7df: 0x2000, 0x7e0: 0x2000, 0x7e1: 0x2000, 0x7e2: 0x2000, 0x7e3: 0x2000, + 0x7e4: 0x2000, 0x7e5: 0x2000, 0x7e6: 0x2000, 0x7e7: 0x2000, 0x7e8: 0x2000, 0x7e9: 0x2000, + 0x7eb: 0x2000, 0x7ec: 0x2000, 0x7ed: 0x2000, 0x7ee: 0x2000, 0x7ef: 0x2000, + 0x7f0: 0x2000, 0x7f1: 0x2000, 0x7f2: 0x2000, 0x7f3: 0x2000, 0x7f4: 0x2000, 0x7f5: 0x2000, + 0x7f6: 0x2000, 0x7f7: 0x2000, 0x7f8: 0x2000, 0x7f9: 0x2000, 0x7fa: 0x2000, 0x7fb: 0x2000, + 0x7fc: 0x2000, 0x7fd: 0x2000, 0x7fe: 0x2000, 0x7ff: 0x2000, + // Block 0x20, offset 0x800 + 0x800: 0x2000, 0x801: 0x2000, 0x802: 0x200d, 0x803: 0x2000, 0x804: 0x2000, 0x805: 0x2000, + 0x806: 0x2000, 0x807: 0x2000, 0x808: 0x2000, 0x809: 0x2000, 0x80a: 0x2000, 0x80b: 0x2000, + 0x80c: 0x2000, 0x80d: 0x2000, 0x80e: 0x2000, 0x80f: 0x2000, 0x810: 0x2000, 0x811: 0x2000, + 0x812: 0x2000, 0x813: 0x2000, 0x814: 0x2000, 0x815: 0x2000, 0x816: 0x2000, 0x817: 0x2000, + 0x818: 0x2000, 0x819: 0x2000, 0x81a: 0x2000, 0x81b: 0x2000, 0x81c: 0x2000, 0x81d: 0x2000, + 0x81e: 0x2000, 0x81f: 0x2000, 0x820: 0x2000, 0x821: 0x2000, 0x822: 0x2000, 0x823: 0x2000, + 0x824: 0x2000, 0x825: 0x2000, 0x826: 0x2000, 0x827: 0x2000, 0x828: 0x2000, 0x829: 0x2000, + 0x82a: 0x2000, 0x82b: 0x2000, 0x82c: 0x2000, 0x82d: 0x2000, 0x82e: 0x2000, 0x82f: 0x2000, + 0x830: 0x2000, 0x831: 0x2000, 0x832: 0x2000, 0x833: 0x2000, 0x834: 0x2000, 0x835: 0x2000, + 0x836: 0x2000, 0x837: 0x2000, 0x838: 0x2000, 0x839: 0x2000, 0x83a: 0x2000, 0x83b: 0x2000, + 0x83c: 0x2000, 0x83d: 0x2000, 0x83e: 0x2000, 0x83f: 0x2000, + // Block 0x21, offset 0x840 + 0x840: 0x2000, 0x841: 0x2000, 0x842: 0x2000, 0x843: 0x2000, 0x844: 0x2000, 0x845: 0x2000, + 0x846: 0x2000, 0x847: 0x2000, 0x848: 0x2000, 0x849: 0x2000, 0x84a: 0x2000, 0x84b: 0x2000, + 0x850: 0x2000, 0x851: 0x2000, + 0x852: 0x2000, 0x853: 0x2000, 0x854: 0x2000, 0x855: 0x2000, 0x856: 0x2000, 0x857: 0x2000, + 0x858: 0x2000, 0x859: 0x2000, 0x85a: 0x2000, 0x85b: 0x2000, 0x85c: 0x2000, 0x85d: 0x2000, + 0x85e: 0x2000, 0x85f: 0x2000, 0x860: 0x2000, 0x861: 0x2000, 0x862: 0x2000, 0x863: 0x2000, + 0x864: 0x2000, 0x865: 0x2000, 0x866: 0x2000, 0x867: 0x2000, 0x868: 0x2000, 0x869: 0x2000, + 0x86a: 0x2000, 0x86b: 0x2000, 0x86c: 0x2000, 0x86d: 0x2000, 0x86e: 0x2000, 0x86f: 0x2000, + 0x870: 0x2000, 0x871: 0x2000, 0x872: 0x2000, 0x873: 0x2000, + // Block 0x22, offset 0x880 + 0x880: 0x2000, 0x881: 0x2000, 0x882: 0x2000, 0x883: 0x2000, 0x884: 0x2000, 0x885: 0x2000, + 0x886: 0x2000, 0x887: 0x2000, 0x888: 0x2000, 0x889: 0x2000, 0x88a: 0x2000, 0x88b: 0x2000, + 0x88c: 0x2000, 0x88d: 0x2000, 0x88e: 0x2000, 0x88f: 0x2000, + 0x892: 0x2000, 0x893: 0x2000, 0x894: 0x2000, 0x895: 0x2000, + 0x8a0: 0x200e, 0x8a1: 0x2000, 0x8a3: 0x2000, + 0x8a4: 0x2000, 0x8a5: 0x2000, 0x8a6: 0x2000, 0x8a7: 0x2000, 0x8a8: 0x2000, 0x8a9: 0x2000, + 0x8b2: 0x2000, 0x8b3: 0x2000, + 0x8b6: 0x2000, 0x8b7: 0x2000, + 0x8bc: 0x2000, 0x8bd: 0x2000, + // Block 0x23, offset 0x8c0 + 0x8c0: 0x2000, 0x8c1: 0x2000, + 0x8c6: 0x2000, 0x8c7: 0x2000, 0x8c8: 0x2000, 0x8cb: 0x200f, + 0x8ce: 0x2000, 0x8cf: 0x2000, 0x8d0: 0x2000, 0x8d1: 0x2000, + 0x8e2: 0x2000, 0x8e3: 0x2000, + 0x8e4: 0x2000, 0x8e5: 0x2000, + 0x8ef: 0x2000, + 0x8fd: 0x4000, 0x8fe: 0x4000, + // Block 0x24, offset 0x900 + 0x905: 0x2000, + 0x906: 0x2000, 0x909: 0x2000, + 0x90e: 0x2000, 0x90f: 0x2000, + 0x914: 0x4000, 0x915: 0x4000, + 0x91c: 0x2000, + 0x91e: 0x2000, + // Block 0x25, offset 0x940 + 0x940: 0x2000, 0x942: 0x2000, + 0x948: 0x4000, 0x949: 0x4000, 0x94a: 0x4000, 0x94b: 0x4000, + 0x94c: 0x4000, 0x94d: 0x4000, 0x94e: 0x4000, 0x94f: 0x4000, 0x950: 0x4000, 0x951: 0x4000, + 0x952: 0x4000, 0x953: 0x4000, + 0x960: 0x2000, 0x961: 0x2000, 0x963: 0x2000, + 0x964: 0x2000, 0x965: 0x2000, 0x967: 0x2000, 0x968: 0x2000, 0x969: 0x2000, + 0x96a: 0x2000, 0x96c: 0x2000, 0x96d: 0x2000, 0x96f: 0x2000, + 0x97f: 0x4000, + // Block 0x26, offset 0x980 + 0x993: 0x4000, + 0x99e: 0x2000, 0x99f: 0x2000, 0x9a1: 0x4000, + 0x9aa: 0x4000, 0x9ab: 0x4000, + 0x9bd: 0x4000, 0x9be: 0x4000, 0x9bf: 0x2000, + // Block 0x27, offset 0x9c0 + 0x9c4: 0x4000, 0x9c5: 0x4000, + 0x9c6: 0x2000, 0x9c7: 0x2000, 0x9c8: 0x2000, 0x9c9: 0x2000, 0x9ca: 0x2000, 0x9cb: 0x2000, + 0x9cc: 0x2000, 0x9cd: 0x2000, 0x9ce: 0x4000, 0x9cf: 0x2000, 0x9d0: 0x2000, 0x9d1: 0x2000, + 0x9d2: 0x2000, 0x9d3: 0x2000, 0x9d4: 0x4000, 0x9d5: 0x2000, 0x9d6: 0x2000, 0x9d7: 0x2000, + 0x9d8: 0x2000, 0x9d9: 0x2000, 0x9da: 0x2000, 0x9db: 0x2000, 0x9dc: 0x2000, 0x9dd: 0x2000, + 0x9de: 0x2000, 0x9df: 0x2000, 0x9e0: 0x2000, 0x9e1: 0x2000, 0x9e3: 0x2000, + 0x9e8: 0x2000, 0x9e9: 0x2000, + 0x9ea: 0x4000, 0x9eb: 0x2000, 0x9ec: 0x2000, 0x9ed: 0x2000, 0x9ee: 0x2000, 0x9ef: 0x2000, + 0x9f0: 0x2000, 0x9f1: 0x2000, 0x9f2: 0x4000, 0x9f3: 0x4000, 0x9f4: 0x2000, 0x9f5: 0x4000, + 0x9f6: 0x2000, 0x9f7: 0x2000, 0x9f8: 0x2000, 0x9f9: 0x2000, 0x9fa: 0x4000, 0x9fb: 0x2000, + 0x9fc: 0x2000, 0x9fd: 0x4000, 0x9fe: 0x2000, 0x9ff: 0x2000, + // Block 0x28, offset 0xa00 + 0xa05: 0x4000, + 0xa0a: 0x4000, 0xa0b: 0x4000, + 0xa28: 0x4000, + 0xa3d: 0x2000, + // Block 0x29, offset 0xa40 + 0xa4c: 0x4000, 0xa4e: 0x4000, + 0xa53: 0x4000, 0xa54: 0x4000, 0xa55: 0x4000, 0xa57: 0x4000, + 0xa76: 0x2000, 0xa77: 0x2000, 0xa78: 0x2000, 0xa79: 0x2000, 0xa7a: 0x2000, 0xa7b: 0x2000, + 0xa7c: 0x2000, 0xa7d: 0x2000, 0xa7e: 0x2000, 0xa7f: 0x2000, + // Block 0x2a, offset 0xa80 + 0xa95: 0x4000, 0xa96: 0x4000, 0xa97: 0x4000, + 0xab0: 0x4000, + 0xabf: 0x4000, + // Block 0x2b, offset 0xac0 + 0xae6: 0x6000, 0xae7: 0x6000, 0xae8: 0x6000, 0xae9: 0x6000, + 0xaea: 0x6000, 0xaeb: 0x6000, 0xaec: 0x6000, 0xaed: 0x6000, + // Block 0x2c, offset 0xb00 + 0xb05: 0x6010, + 0xb06: 0x6011, + // Block 0x2d, offset 0xb40 + 0xb5b: 0x4000, 0xb5c: 0x4000, + // Block 0x2e, offset 0xb80 + 0xb90: 0x4000, + 0xb95: 0x4000, 0xb96: 0x2000, 0xb97: 0x2000, + 0xb98: 0x2000, 0xb99: 0x2000, + // Block 0x2f, offset 0xbc0 + 0xbc0: 0x4000, 0xbc1: 0x4000, 0xbc2: 0x4000, 0xbc3: 0x4000, 0xbc4: 0x4000, 0xbc5: 0x4000, + 0xbc6: 0x4000, 0xbc7: 0x4000, 0xbc8: 0x4000, 0xbc9: 0x4000, 0xbca: 0x4000, 0xbcb: 0x4000, + 0xbcc: 0x4000, 0xbcd: 0x4000, 0xbce: 0x4000, 0xbcf: 0x4000, 0xbd0: 0x4000, 0xbd1: 0x4000, + 0xbd2: 0x4000, 0xbd3: 0x4000, 0xbd4: 0x4000, 0xbd5: 0x4000, 0xbd6: 0x4000, 0xbd7: 0x4000, + 0xbd8: 0x4000, 0xbd9: 0x4000, 0xbdb: 0x4000, 0xbdc: 0x4000, 0xbdd: 0x4000, + 0xbde: 0x4000, 0xbdf: 0x4000, 0xbe0: 0x4000, 0xbe1: 0x4000, 0xbe2: 0x4000, 0xbe3: 0x4000, + 0xbe4: 0x4000, 0xbe5: 0x4000, 0xbe6: 0x4000, 0xbe7: 0x4000, 0xbe8: 0x4000, 0xbe9: 0x4000, + 0xbea: 0x4000, 0xbeb: 0x4000, 0xbec: 0x4000, 0xbed: 0x4000, 0xbee: 0x4000, 0xbef: 0x4000, + 0xbf0: 0x4000, 0xbf1: 0x4000, 0xbf2: 0x4000, 0xbf3: 0x4000, 0xbf4: 0x4000, 0xbf5: 0x4000, + 0xbf6: 0x4000, 0xbf7: 0x4000, 0xbf8: 0x4000, 0xbf9: 0x4000, 0xbfa: 0x4000, 0xbfb: 0x4000, + 0xbfc: 0x4000, 0xbfd: 0x4000, 0xbfe: 0x4000, 0xbff: 0x4000, + // Block 0x30, offset 0xc00 + 0xc00: 0x4000, 0xc01: 0x4000, 0xc02: 0x4000, 0xc03: 0x4000, 0xc04: 0x4000, 0xc05: 0x4000, + 0xc06: 0x4000, 0xc07: 0x4000, 0xc08: 0x4000, 0xc09: 0x4000, 0xc0a: 0x4000, 0xc0b: 0x4000, + 0xc0c: 0x4000, 0xc0d: 0x4000, 0xc0e: 0x4000, 0xc0f: 0x4000, 0xc10: 0x4000, 0xc11: 0x4000, + 0xc12: 0x4000, 0xc13: 0x4000, 0xc14: 0x4000, 0xc15: 0x4000, 0xc16: 0x4000, 0xc17: 0x4000, + 0xc18: 0x4000, 0xc19: 0x4000, 0xc1a: 0x4000, 0xc1b: 0x4000, 0xc1c: 0x4000, 0xc1d: 0x4000, + 0xc1e: 0x4000, 0xc1f: 0x4000, 0xc20: 0x4000, 0xc21: 0x4000, 0xc22: 0x4000, 0xc23: 0x4000, + 0xc24: 0x4000, 0xc25: 0x4000, 0xc26: 0x4000, 0xc27: 0x4000, 0xc28: 0x4000, 0xc29: 0x4000, + 0xc2a: 0x4000, 0xc2b: 0x4000, 0xc2c: 0x4000, 0xc2d: 0x4000, 0xc2e: 0x4000, 0xc2f: 0x4000, + 0xc30: 0x4000, 0xc31: 0x4000, 0xc32: 0x4000, 0xc33: 0x4000, + // Block 0x31, offset 0xc40 + 0xc40: 0x4000, 0xc41: 0x4000, 0xc42: 0x4000, 0xc43: 0x4000, 0xc44: 0x4000, 0xc45: 0x4000, + 0xc46: 0x4000, 0xc47: 0x4000, 0xc48: 0x4000, 0xc49: 0x4000, 0xc4a: 0x4000, 0xc4b: 0x4000, + 0xc4c: 0x4000, 0xc4d: 0x4000, 0xc4e: 0x4000, 0xc4f: 0x4000, 0xc50: 0x4000, 0xc51: 0x4000, + 0xc52: 0x4000, 0xc53: 0x4000, 0xc54: 0x4000, 0xc55: 0x4000, + 0xc70: 0x4000, 0xc71: 0x4000, 0xc72: 0x4000, 0xc73: 0x4000, 0xc74: 0x4000, 0xc75: 0x4000, + 0xc76: 0x4000, 0xc77: 0x4000, 0xc78: 0x4000, 0xc79: 0x4000, 0xc7a: 0x4000, 0xc7b: 0x4000, + // Block 0x32, offset 0xc80 + 0xc80: 0x9012, 0xc81: 0x4013, 0xc82: 0x4014, 0xc83: 0x4000, 0xc84: 0x4000, 0xc85: 0x4000, + 0xc86: 0x4000, 0xc87: 0x4000, 0xc88: 0x4000, 0xc89: 0x4000, 0xc8a: 0x4000, 0xc8b: 0x4000, + 0xc8c: 0x4015, 0xc8d: 0x4015, 0xc8e: 0x4000, 0xc8f: 0x4000, 0xc90: 0x4000, 0xc91: 0x4000, + 0xc92: 0x4000, 0xc93: 0x4000, 0xc94: 0x4000, 0xc95: 0x4000, 0xc96: 0x4000, 0xc97: 0x4000, + 0xc98: 0x4000, 0xc99: 0x4000, 0xc9a: 0x4000, 0xc9b: 0x4000, 0xc9c: 0x4000, 0xc9d: 0x4000, + 0xc9e: 0x4000, 0xc9f: 0x4000, 0xca0: 0x4000, 0xca1: 0x4000, 0xca2: 0x4000, 0xca3: 0x4000, + 0xca4: 0x4000, 0xca5: 0x4000, 0xca6: 0x4000, 0xca7: 0x4000, 0xca8: 0x4000, 0xca9: 0x4000, + 0xcaa: 0x4000, 0xcab: 0x4000, 0xcac: 0x4000, 0xcad: 0x4000, 0xcae: 0x4000, 0xcaf: 0x4000, + 0xcb0: 0x4000, 0xcb1: 0x4000, 0xcb2: 0x4000, 0xcb3: 0x4000, 0xcb4: 0x4000, 0xcb5: 0x4000, + 0xcb6: 0x4000, 0xcb7: 0x4000, 0xcb8: 0x4000, 0xcb9: 0x4000, 0xcba: 0x4000, 0xcbb: 0x4000, + 0xcbc: 0x4000, 0xcbd: 0x4000, 0xcbe: 0x4000, + // Block 0x33, offset 0xcc0 + 0xcc1: 0x4000, 0xcc2: 0x4000, 0xcc3: 0x4000, 0xcc4: 0x4000, 0xcc5: 0x4000, + 0xcc6: 0x4000, 0xcc7: 0x4000, 0xcc8: 0x4000, 0xcc9: 0x4000, 0xcca: 0x4000, 0xccb: 0x4000, + 0xccc: 0x4000, 0xccd: 0x4000, 0xcce: 0x4000, 0xccf: 0x4000, 0xcd0: 0x4000, 0xcd1: 0x4000, + 0xcd2: 0x4000, 0xcd3: 0x4000, 0xcd4: 0x4000, 0xcd5: 0x4000, 0xcd6: 0x4000, 0xcd7: 0x4000, + 0xcd8: 0x4000, 0xcd9: 0x4000, 0xcda: 0x4000, 0xcdb: 0x4000, 0xcdc: 0x4000, 0xcdd: 0x4000, + 0xcde: 0x4000, 0xcdf: 0x4000, 0xce0: 0x4000, 0xce1: 0x4000, 0xce2: 0x4000, 0xce3: 0x4000, + 0xce4: 0x4000, 0xce5: 0x4000, 0xce6: 0x4000, 0xce7: 0x4000, 0xce8: 0x4000, 0xce9: 0x4000, + 0xcea: 0x4000, 0xceb: 0x4000, 0xcec: 0x4000, 0xced: 0x4000, 0xcee: 0x4000, 0xcef: 0x4000, + 0xcf0: 0x4000, 0xcf1: 0x4000, 0xcf2: 0x4000, 0xcf3: 0x4000, 0xcf4: 0x4000, 0xcf5: 0x4000, + 0xcf6: 0x4000, 0xcf7: 0x4000, 0xcf8: 0x4000, 0xcf9: 0x4000, 0xcfa: 0x4000, 0xcfb: 0x4000, + 0xcfc: 0x4000, 0xcfd: 0x4000, 0xcfe: 0x4000, 0xcff: 0x4000, + // Block 0x34, offset 0xd00 + 0xd00: 0x4000, 0xd01: 0x4000, 0xd02: 0x4000, 0xd03: 0x4000, 0xd04: 0x4000, 0xd05: 0x4000, + 0xd06: 0x4000, 0xd07: 0x4000, 0xd08: 0x4000, 0xd09: 0x4000, 0xd0a: 0x4000, 0xd0b: 0x4000, + 0xd0c: 0x4000, 0xd0d: 0x4000, 0xd0e: 0x4000, 0xd0f: 0x4000, 0xd10: 0x4000, 0xd11: 0x4000, + 0xd12: 0x4000, 0xd13: 0x4000, 0xd14: 0x4000, 0xd15: 0x4000, 0xd16: 0x4000, + 0xd19: 0x4016, 0xd1a: 0x4017, 0xd1b: 0x4000, 0xd1c: 0x4000, 0xd1d: 0x4000, + 0xd1e: 0x4000, 0xd1f: 0x4000, 0xd20: 0x4000, 0xd21: 0x4018, 0xd22: 0x4019, 0xd23: 0x401a, + 0xd24: 0x401b, 0xd25: 0x401c, 0xd26: 0x401d, 0xd27: 0x401e, 0xd28: 0x401f, 0xd29: 0x4020, + 0xd2a: 0x4021, 0xd2b: 0x4022, 0xd2c: 0x4000, 0xd2d: 0x4010, 0xd2e: 0x4000, 0xd2f: 0x4023, + 0xd30: 0x4000, 0xd31: 0x4024, 0xd32: 0x4000, 0xd33: 0x4025, 0xd34: 0x4000, 0xd35: 0x4026, + 0xd36: 0x4000, 0xd37: 0x401a, 0xd38: 0x4000, 0xd39: 0x4027, 0xd3a: 0x4000, 0xd3b: 0x4028, + 0xd3c: 0x4000, 0xd3d: 0x4020, 0xd3e: 0x4000, 0xd3f: 0x4029, + // Block 0x35, offset 0xd40 + 0xd40: 0x4000, 0xd41: 0x402a, 0xd42: 0x4000, 0xd43: 0x402b, 0xd44: 0x402c, 0xd45: 0x4000, + 0xd46: 0x4017, 0xd47: 0x4000, 0xd48: 0x402d, 0xd49: 0x4000, 0xd4a: 0x402e, 0xd4b: 0x402f, + 0xd4c: 0x4030, 0xd4d: 0x4017, 0xd4e: 0x4016, 0xd4f: 0x4017, 0xd50: 0x4000, 0xd51: 0x4000, + 0xd52: 0x4031, 0xd53: 0x4000, 0xd54: 0x4000, 0xd55: 0x4031, 0xd56: 0x4000, 0xd57: 0x4000, + 0xd58: 0x4032, 0xd59: 0x4000, 0xd5a: 0x4000, 0xd5b: 0x4032, 0xd5c: 0x4000, 0xd5d: 0x4000, + 0xd5e: 0x4033, 0xd5f: 0x402e, 0xd60: 0x4034, 0xd61: 0x4035, 0xd62: 0x4034, 0xd63: 0x4036, + 0xd64: 0x4037, 0xd65: 0x4024, 0xd66: 0x4035, 0xd67: 0x4025, 0xd68: 0x4038, 0xd69: 0x4038, + 0xd6a: 0x4039, 0xd6b: 0x4039, 0xd6c: 0x403a, 0xd6d: 0x403a, 0xd6e: 0x4000, 0xd6f: 0x4035, + 0xd70: 0x4000, 0xd71: 0x4000, 0xd72: 0x403b, 0xd73: 0x403c, 0xd74: 0x4000, 0xd75: 0x4000, + 0xd76: 0x4000, 0xd77: 0x4000, 0xd78: 0x4000, 0xd79: 0x4000, 0xd7a: 0x4000, 0xd7b: 0x403d, + 0xd7c: 0x401c, 0xd7d: 0x4000, 0xd7e: 0x4000, 0xd7f: 0x4000, + // Block 0x36, offset 0xd80 + 0xd85: 0x4000, + 0xd86: 0x4000, 0xd87: 0x4000, 0xd88: 0x4000, 0xd89: 0x4000, 0xd8a: 0x4000, 0xd8b: 0x4000, + 0xd8c: 0x4000, 0xd8d: 0x4000, 0xd8e: 0x4000, 0xd8f: 0x4000, 0xd90: 0x4000, 0xd91: 0x4000, + 0xd92: 0x4000, 0xd93: 0x4000, 0xd94: 0x4000, 0xd95: 0x4000, 0xd96: 0x4000, 0xd97: 0x4000, + 0xd98: 0x4000, 0xd99: 0x4000, 0xd9a: 0x4000, 0xd9b: 0x4000, 0xd9c: 0x4000, 0xd9d: 0x4000, + 0xd9e: 0x4000, 0xd9f: 0x4000, 0xda0: 0x4000, 0xda1: 0x4000, 0xda2: 0x4000, 0xda3: 0x4000, + 0xda4: 0x4000, 0xda5: 0x4000, 0xda6: 0x4000, 0xda7: 0x4000, 0xda8: 0x4000, 0xda9: 0x4000, + 0xdaa: 0x4000, 0xdab: 0x4000, 0xdac: 0x4000, 0xdad: 0x4000, 0xdae: 0x4000, 0xdaf: 0x4000, + 0xdb1: 0x403e, 0xdb2: 0x403e, 0xdb3: 0x403e, 0xdb4: 0x403e, 0xdb5: 0x403e, + 0xdb6: 0x403e, 0xdb7: 0x403e, 0xdb8: 0x403e, 0xdb9: 0x403e, 0xdba: 0x403e, 0xdbb: 0x403e, + 0xdbc: 0x403e, 0xdbd: 0x403e, 0xdbe: 0x403e, 0xdbf: 0x403e, + // Block 0x37, offset 0xdc0 + 0xdc0: 0x4037, 0xdc1: 0x4037, 0xdc2: 0x4037, 0xdc3: 0x4037, 0xdc4: 0x4037, 0xdc5: 0x4037, + 0xdc6: 0x4037, 0xdc7: 0x4037, 0xdc8: 0x4037, 0xdc9: 0x4037, 0xdca: 0x4037, 0xdcb: 0x4037, + 0xdcc: 0x4037, 0xdcd: 0x4037, 0xdce: 0x4037, 0xdcf: 0x400e, 0xdd0: 0x403f, 0xdd1: 0x4040, + 0xdd2: 0x4041, 0xdd3: 0x4040, 0xdd4: 0x403f, 0xdd5: 0x4042, 0xdd6: 0x4043, 0xdd7: 0x4044, + 0xdd8: 0x4040, 0xdd9: 0x4041, 0xdda: 0x4040, 0xddb: 0x4045, 0xddc: 0x4009, 0xddd: 0x4045, + 0xdde: 0x4046, 0xddf: 0x4045, 0xde0: 0x4047, 0xde1: 0x400b, 0xde2: 0x400a, 0xde3: 0x400c, + 0xde4: 0x4048, 0xde5: 0x4000, 0xde6: 0x4000, 0xde7: 0x4000, 0xde8: 0x4000, 0xde9: 0x4000, + 0xdea: 0x4000, 0xdeb: 0x4000, 0xdec: 0x4000, 0xded: 0x4000, 0xdee: 0x4000, 0xdef: 0x4000, + 0xdf0: 0x4000, 0xdf1: 0x4000, 0xdf2: 0x4000, 0xdf3: 0x4000, 0xdf4: 0x4000, 0xdf5: 0x4000, + 0xdf6: 0x4000, 0xdf7: 0x4000, 0xdf8: 0x4000, 0xdf9: 0x4000, 0xdfa: 0x4000, 0xdfb: 0x4000, + 0xdfc: 0x4000, 0xdfd: 0x4000, 0xdfe: 0x4000, 0xdff: 0x4000, + // Block 0x38, offset 0xe00 + 0xe00: 0x4000, 0xe01: 0x4000, 0xe02: 0x4000, 0xe03: 0x4000, 0xe04: 0x4000, 0xe05: 0x4000, + 0xe06: 0x4000, 0xe07: 0x4000, 0xe08: 0x4000, 0xe09: 0x4000, 0xe0a: 0x4000, 0xe0b: 0x4000, + 0xe0c: 0x4000, 0xe0d: 0x4000, 0xe0e: 0x4000, 0xe10: 0x4000, 0xe11: 0x4000, + 0xe12: 0x4000, 0xe13: 0x4000, 0xe14: 0x4000, 0xe15: 0x4000, 0xe16: 0x4000, 0xe17: 0x4000, + 0xe18: 0x4000, 0xe19: 0x4000, 0xe1a: 0x4000, 0xe1b: 0x4000, 0xe1c: 0x4000, 0xe1d: 0x4000, + 0xe1e: 0x4000, 0xe1f: 0x4000, 0xe20: 0x4000, 0xe21: 0x4000, 0xe22: 0x4000, 0xe23: 0x4000, + 0xe24: 0x4000, 0xe25: 0x4000, 0xe26: 0x4000, 0xe27: 0x4000, 0xe28: 0x4000, 0xe29: 0x4000, + 0xe2a: 0x4000, 0xe2b: 0x4000, 0xe2c: 0x4000, 0xe2d: 0x4000, 0xe2e: 0x4000, 0xe2f: 0x4000, + 0xe30: 0x4000, 0xe31: 0x4000, 0xe32: 0x4000, 0xe33: 0x4000, 0xe34: 0x4000, 0xe35: 0x4000, + 0xe36: 0x4000, 0xe37: 0x4000, 0xe38: 0x4000, 0xe39: 0x4000, 0xe3a: 0x4000, + // Block 0x39, offset 0xe40 + 0xe40: 0x4000, 0xe41: 0x4000, 0xe42: 0x4000, 0xe43: 0x4000, 0xe44: 0x4000, 0xe45: 0x4000, + 0xe46: 0x4000, 0xe47: 0x4000, 0xe48: 0x4000, 0xe49: 0x4000, 0xe4a: 0x4000, 0xe4b: 0x4000, + 0xe4c: 0x4000, 0xe4d: 0x4000, 0xe4e: 0x4000, 0xe4f: 0x4000, 0xe50: 0x4000, 0xe51: 0x4000, + 0xe52: 0x4000, 0xe53: 0x4000, 0xe54: 0x4000, 0xe55: 0x4000, 0xe56: 0x4000, 0xe57: 0x4000, + 0xe58: 0x4000, 0xe59: 0x4000, 0xe5a: 0x4000, 0xe5b: 0x4000, 0xe5c: 0x4000, 0xe5d: 0x4000, + 0xe5e: 0x4000, 0xe5f: 0x4000, 0xe60: 0x4000, 0xe61: 0x4000, 0xe62: 0x4000, 0xe63: 0x4000, + 0xe70: 0x4000, 0xe71: 0x4000, 0xe72: 0x4000, 0xe73: 0x4000, 0xe74: 0x4000, 0xe75: 0x4000, + 0xe76: 0x4000, 0xe77: 0x4000, 0xe78: 0x4000, 0xe79: 0x4000, 0xe7a: 0x4000, 0xe7b: 0x4000, + 0xe7c: 0x4000, 0xe7d: 0x4000, 0xe7e: 0x4000, 0xe7f: 0x4000, + // Block 0x3a, offset 0xe80 + 0xe80: 0x4000, 0xe81: 0x4000, 0xe82: 0x4000, 0xe83: 0x4000, 0xe84: 0x4000, 0xe85: 0x4000, + 0xe86: 0x4000, 0xe87: 0x4000, 0xe88: 0x4000, 0xe89: 0x4000, 0xe8a: 0x4000, 0xe8b: 0x4000, + 0xe8c: 0x4000, 0xe8d: 0x4000, 0xe8e: 0x4000, 0xe8f: 0x4000, 0xe90: 0x4000, 0xe91: 0x4000, + 0xe92: 0x4000, 0xe93: 0x4000, 0xe94: 0x4000, 0xe95: 0x4000, 0xe96: 0x4000, 0xe97: 0x4000, + 0xe98: 0x4000, 0xe99: 0x4000, 0xe9a: 0x4000, 0xe9b: 0x4000, 0xe9c: 0x4000, 0xe9d: 0x4000, + 0xe9e: 0x4000, 0xea0: 0x4000, 0xea1: 0x4000, 0xea2: 0x4000, 0xea3: 0x4000, + 0xea4: 0x4000, 0xea5: 0x4000, 0xea6: 0x4000, 0xea7: 0x4000, 0xea8: 0x4000, 0xea9: 0x4000, + 0xeaa: 0x4000, 0xeab: 0x4000, 0xeac: 0x4000, 0xead: 0x4000, 0xeae: 0x4000, 0xeaf: 0x4000, + 0xeb0: 0x4000, 0xeb1: 0x4000, 0xeb2: 0x4000, 0xeb3: 0x4000, 0xeb4: 0x4000, 0xeb5: 0x4000, + 0xeb6: 0x4000, 0xeb7: 0x4000, 0xeb8: 0x4000, 0xeb9: 0x4000, 0xeba: 0x4000, 0xebb: 0x4000, + 0xebc: 0x4000, 0xebd: 0x4000, 0xebe: 0x4000, 0xebf: 0x4000, + // Block 0x3b, offset 0xec0 + 0xec0: 0x4000, 0xec1: 0x4000, 0xec2: 0x4000, 0xec3: 0x4000, 0xec4: 0x4000, 0xec5: 0x4000, + 0xec6: 0x4000, 0xec7: 0x4000, 0xec8: 0x2000, 0xec9: 0x2000, 0xeca: 0x2000, 0xecb: 0x2000, + 0xecc: 0x2000, 0xecd: 0x2000, 0xece: 0x2000, 0xecf: 0x2000, 0xed0: 0x4000, 0xed1: 0x4000, + 0xed2: 0x4000, 0xed3: 0x4000, 0xed4: 0x4000, 0xed5: 0x4000, 0xed6: 0x4000, 0xed7: 0x4000, + 0xed8: 0x4000, 0xed9: 0x4000, 0xeda: 0x4000, 0xedb: 0x4000, 0xedc: 0x4000, 0xedd: 0x4000, + 0xede: 0x4000, 0xedf: 0x4000, 0xee0: 0x4000, 0xee1: 0x4000, 0xee2: 0x4000, 0xee3: 0x4000, + 0xee4: 0x4000, 0xee5: 0x4000, 0xee6: 0x4000, 0xee7: 0x4000, 0xee8: 0x4000, 0xee9: 0x4000, + 0xeea: 0x4000, 0xeeb: 0x4000, 0xeec: 0x4000, 0xeed: 0x4000, 0xeee: 0x4000, 0xeef: 0x4000, + 0xef0: 0x4000, 0xef1: 0x4000, 0xef2: 0x4000, 0xef3: 0x4000, 0xef4: 0x4000, 0xef5: 0x4000, + 0xef6: 0x4000, 0xef7: 0x4000, 0xef8: 0x4000, 0xef9: 0x4000, 0xefa: 0x4000, 0xefb: 0x4000, + 0xefc: 0x4000, 0xefd: 0x4000, 0xefe: 0x4000, 0xeff: 0x4000, + // Block 0x3c, offset 0xf00 + 0xf00: 0x4000, 0xf01: 0x4000, 0xf02: 0x4000, 0xf03: 0x4000, 0xf04: 0x4000, 0xf05: 0x4000, + 0xf06: 0x4000, 0xf07: 0x4000, 0xf08: 0x4000, 0xf09: 0x4000, 0xf0a: 0x4000, 0xf0b: 0x4000, + 0xf0c: 0x4000, 0xf0d: 0x4000, 0xf0e: 0x4000, 0xf0f: 0x4000, 0xf10: 0x4000, 0xf11: 0x4000, + 0xf12: 0x4000, 0xf13: 0x4000, 0xf14: 0x4000, 0xf15: 0x4000, 0xf16: 0x4000, 0xf17: 0x4000, + 0xf18: 0x4000, 0xf19: 0x4000, 0xf1a: 0x4000, 0xf1b: 0x4000, 0xf1c: 0x4000, 0xf1d: 0x4000, + 0xf1e: 0x4000, 0xf1f: 0x4000, 0xf20: 0x4000, 0xf21: 0x4000, 0xf22: 0x4000, 0xf23: 0x4000, + 0xf24: 0x4000, 0xf25: 0x4000, 0xf26: 0x4000, 0xf27: 0x4000, 0xf28: 0x4000, 0xf29: 0x4000, + 0xf2a: 0x4000, 0xf2b: 0x4000, 0xf2c: 0x4000, 0xf2d: 0x4000, 0xf2e: 0x4000, 0xf2f: 0x4000, + 0xf30: 0x4000, 0xf31: 0x4000, 0xf32: 0x4000, 0xf33: 0x4000, 0xf34: 0x4000, 0xf35: 0x4000, + 0xf36: 0x4000, 0xf37: 0x4000, 0xf38: 0x4000, 0xf39: 0x4000, 0xf3a: 0x4000, 0xf3b: 0x4000, + 0xf3c: 0x4000, 0xf3d: 0x4000, 0xf3e: 0x4000, + // Block 0x3d, offset 0xf40 + 0xf40: 0x4000, 0xf41: 0x4000, 0xf42: 0x4000, 0xf43: 0x4000, 0xf44: 0x4000, 0xf45: 0x4000, + 0xf46: 0x4000, 0xf47: 0x4000, 0xf48: 0x4000, 0xf49: 0x4000, 0xf4a: 0x4000, 0xf4b: 0x4000, + 0xf4c: 0x4000, 0xf50: 0x4000, 0xf51: 0x4000, + 0xf52: 0x4000, 0xf53: 0x4000, 0xf54: 0x4000, 0xf55: 0x4000, 0xf56: 0x4000, 0xf57: 0x4000, + 0xf58: 0x4000, 0xf59: 0x4000, 0xf5a: 0x4000, 0xf5b: 0x4000, 0xf5c: 0x4000, 0xf5d: 0x4000, + 0xf5e: 0x4000, 0xf5f: 0x4000, 0xf60: 0x4000, 0xf61: 0x4000, 0xf62: 0x4000, 0xf63: 0x4000, + 0xf64: 0x4000, 0xf65: 0x4000, 0xf66: 0x4000, 0xf67: 0x4000, 0xf68: 0x4000, 0xf69: 0x4000, + 0xf6a: 0x4000, 0xf6b: 0x4000, 0xf6c: 0x4000, 0xf6d: 0x4000, 0xf6e: 0x4000, 0xf6f: 0x4000, + 0xf70: 0x4000, 0xf71: 0x4000, 0xf72: 0x4000, 0xf73: 0x4000, 0xf74: 0x4000, 0xf75: 0x4000, + 0xf76: 0x4000, 0xf77: 0x4000, 0xf78: 0x4000, 0xf79: 0x4000, 0xf7a: 0x4000, 0xf7b: 0x4000, + 0xf7c: 0x4000, 0xf7d: 0x4000, 0xf7e: 0x4000, 0xf7f: 0x4000, + // Block 0x3e, offset 0xf80 + 0xf80: 0x4000, 0xf81: 0x4000, 0xf82: 0x4000, 0xf83: 0x4000, 0xf84: 0x4000, 0xf85: 0x4000, + 0xf86: 0x4000, + // Block 0x3f, offset 0xfc0 + 0xfe0: 0x4000, 0xfe1: 0x4000, 0xfe2: 0x4000, 0xfe3: 0x4000, + 0xfe4: 0x4000, 0xfe5: 0x4000, 0xfe6: 0x4000, 0xfe7: 0x4000, 0xfe8: 0x4000, 0xfe9: 0x4000, + 0xfea: 0x4000, 0xfeb: 0x4000, 0xfec: 0x4000, 0xfed: 0x4000, 0xfee: 0x4000, 0xfef: 0x4000, + 0xff0: 0x4000, 0xff1: 0x4000, 0xff2: 0x4000, 0xff3: 0x4000, 0xff4: 0x4000, 0xff5: 0x4000, + 0xff6: 0x4000, 0xff7: 0x4000, 0xff8: 0x4000, 0xff9: 0x4000, 0xffa: 0x4000, 0xffb: 0x4000, + 0xffc: 0x4000, + // Block 0x40, offset 0x1000 + 0x1000: 0x4000, 0x1001: 0x4000, 0x1002: 0x4000, 0x1003: 0x4000, 0x1004: 0x4000, 0x1005: 0x4000, + 0x1006: 0x4000, 0x1007: 0x4000, 0x1008: 0x4000, 0x1009: 0x4000, 0x100a: 0x4000, 0x100b: 0x4000, + 0x100c: 0x4000, 0x100d: 0x4000, 0x100e: 0x4000, 0x100f: 0x4000, 0x1010: 0x4000, 0x1011: 0x4000, + 0x1012: 0x4000, 0x1013: 0x4000, 0x1014: 0x4000, 0x1015: 0x4000, 0x1016: 0x4000, 0x1017: 0x4000, + 0x1018: 0x4000, 0x1019: 0x4000, 0x101a: 0x4000, 0x101b: 0x4000, 0x101c: 0x4000, 0x101d: 0x4000, + 0x101e: 0x4000, 0x101f: 0x4000, 0x1020: 0x4000, 0x1021: 0x4000, 0x1022: 0x4000, 0x1023: 0x4000, + // Block 0x41, offset 0x1040 + 0x1040: 0x2000, 0x1041: 0x2000, 0x1042: 0x2000, 0x1043: 0x2000, 0x1044: 0x2000, 0x1045: 0x2000, + 0x1046: 0x2000, 0x1047: 0x2000, 0x1048: 0x2000, 0x1049: 0x2000, 0x104a: 0x2000, 0x104b: 0x2000, + 0x104c: 0x2000, 0x104d: 0x2000, 0x104e: 0x2000, 0x104f: 0x2000, 0x1050: 0x4000, 0x1051: 0x4000, + 0x1052: 0x4000, 0x1053: 0x4000, 0x1054: 0x4000, 0x1055: 0x4000, 0x1056: 0x4000, 0x1057: 0x4000, + 0x1058: 0x4000, 0x1059: 0x4000, + 0x1070: 0x4000, 0x1071: 0x4000, 0x1072: 0x4000, 0x1073: 0x4000, 0x1074: 0x4000, 0x1075: 0x4000, + 0x1076: 0x4000, 0x1077: 0x4000, 0x1078: 0x4000, 0x1079: 0x4000, 0x107a: 0x4000, 0x107b: 0x4000, + 0x107c: 0x4000, 0x107d: 0x4000, 0x107e: 0x4000, 0x107f: 0x4000, + // Block 0x42, offset 0x1080 + 0x1080: 0x4000, 0x1081: 0x4000, 0x1082: 0x4000, 0x1083: 0x4000, 0x1084: 0x4000, 0x1085: 0x4000, + 0x1086: 0x4000, 0x1087: 0x4000, 0x1088: 0x4000, 0x1089: 0x4000, 0x108a: 0x4000, 0x108b: 0x4000, + 0x108c: 0x4000, 0x108d: 0x4000, 0x108e: 0x4000, 0x108f: 0x4000, 0x1090: 0x4000, 0x1091: 0x4000, + 0x1092: 0x4000, 0x1094: 0x4000, 0x1095: 0x4000, 0x1096: 0x4000, 0x1097: 0x4000, + 0x1098: 0x4000, 0x1099: 0x4000, 0x109a: 0x4000, 0x109b: 0x4000, 0x109c: 0x4000, 0x109d: 0x4000, + 0x109e: 0x4000, 0x109f: 0x4000, 0x10a0: 0x4000, 0x10a1: 0x4000, 0x10a2: 0x4000, 0x10a3: 0x4000, + 0x10a4: 0x4000, 0x10a5: 0x4000, 0x10a6: 0x4000, 0x10a8: 0x4000, 0x10a9: 0x4000, + 0x10aa: 0x4000, 0x10ab: 0x4000, + // Block 0x43, offset 0x10c0 + 0x10c1: 0x9012, 0x10c2: 0x9012, 0x10c3: 0x9012, 0x10c4: 0x9012, 0x10c5: 0x9012, + 0x10c6: 0x9012, 0x10c7: 0x9012, 0x10c8: 0x9012, 0x10c9: 0x9012, 0x10ca: 0x9012, 0x10cb: 0x9012, + 0x10cc: 0x9012, 0x10cd: 0x9012, 0x10ce: 0x9012, 0x10cf: 0x9012, 0x10d0: 0x9012, 0x10d1: 0x9012, + 0x10d2: 0x9012, 0x10d3: 0x9012, 0x10d4: 0x9012, 0x10d5: 0x9012, 0x10d6: 0x9012, 0x10d7: 0x9012, + 0x10d8: 0x9012, 0x10d9: 0x9012, 0x10da: 0x9012, 0x10db: 0x9012, 0x10dc: 0x9012, 0x10dd: 0x9012, + 0x10de: 0x9012, 0x10df: 0x9012, 0x10e0: 0x9049, 0x10e1: 0x9049, 0x10e2: 0x9049, 0x10e3: 0x9049, + 0x10e4: 0x9049, 0x10e5: 0x9049, 0x10e6: 0x9049, 0x10e7: 0x9049, 0x10e8: 0x9049, 0x10e9: 0x9049, + 0x10ea: 0x9049, 0x10eb: 0x9049, 0x10ec: 0x9049, 0x10ed: 0x9049, 0x10ee: 0x9049, 0x10ef: 0x9049, + 0x10f0: 0x9049, 0x10f1: 0x9049, 0x10f2: 0x9049, 0x10f3: 0x9049, 0x10f4: 0x9049, 0x10f5: 0x9049, + 0x10f6: 0x9049, 0x10f7: 0x9049, 0x10f8: 0x9049, 0x10f9: 0x9049, 0x10fa: 0x9049, 0x10fb: 0x9049, + 0x10fc: 0x9049, 0x10fd: 0x9049, 0x10fe: 0x9049, 0x10ff: 0x9049, + // Block 0x44, offset 0x1100 + 0x1100: 0x9049, 0x1101: 0x9049, 0x1102: 0x9049, 0x1103: 0x9049, 0x1104: 0x9049, 0x1105: 0x9049, + 0x1106: 0x9049, 0x1107: 0x9049, 0x1108: 0x9049, 0x1109: 0x9049, 0x110a: 0x9049, 0x110b: 0x9049, + 0x110c: 0x9049, 0x110d: 0x9049, 0x110e: 0x9049, 0x110f: 0x9049, 0x1110: 0x9049, 0x1111: 0x9049, + 0x1112: 0x9049, 0x1113: 0x9049, 0x1114: 0x9049, 0x1115: 0x9049, 0x1116: 0x9049, 0x1117: 0x9049, + 0x1118: 0x9049, 0x1119: 0x9049, 0x111a: 0x9049, 0x111b: 0x9049, 0x111c: 0x9049, 0x111d: 0x9049, + 0x111e: 0x9049, 0x111f: 0x904a, 0x1120: 0x904b, 0x1121: 0xb04c, 0x1122: 0xb04d, 0x1123: 0xb04d, + 0x1124: 0xb04e, 0x1125: 0xb04f, 0x1126: 0xb050, 0x1127: 0xb051, 0x1128: 0xb052, 0x1129: 0xb053, + 0x112a: 0xb054, 0x112b: 0xb055, 0x112c: 0xb056, 0x112d: 0xb057, 0x112e: 0xb058, 0x112f: 0xb059, + 0x1130: 0xb05a, 0x1131: 0xb05b, 0x1132: 0xb05c, 0x1133: 0xb05d, 0x1134: 0xb05e, 0x1135: 0xb05f, + 0x1136: 0xb060, 0x1137: 0xb061, 0x1138: 0xb062, 0x1139: 0xb063, 0x113a: 0xb064, 0x113b: 0xb065, + 0x113c: 0xb052, 0x113d: 0xb066, 0x113e: 0xb067, 0x113f: 0xb055, + // Block 0x45, offset 0x1140 + 0x1140: 0xb068, 0x1141: 0xb069, 0x1142: 0xb06a, 0x1143: 0xb06b, 0x1144: 0xb05a, 0x1145: 0xb056, + 0x1146: 0xb06c, 0x1147: 0xb06d, 0x1148: 0xb06b, 0x1149: 0xb06e, 0x114a: 0xb06b, 0x114b: 0xb06f, + 0x114c: 0xb06f, 0x114d: 0xb070, 0x114e: 0xb070, 0x114f: 0xb071, 0x1150: 0xb056, 0x1151: 0xb072, + 0x1152: 0xb073, 0x1153: 0xb072, 0x1154: 0xb074, 0x1155: 0xb073, 0x1156: 0xb075, 0x1157: 0xb075, + 0x1158: 0xb076, 0x1159: 0xb076, 0x115a: 0xb077, 0x115b: 0xb077, 0x115c: 0xb073, 0x115d: 0xb078, + 0x115e: 0xb079, 0x115f: 0xb067, 0x1160: 0xb07a, 0x1161: 0xb07b, 0x1162: 0xb07b, 0x1163: 0xb07b, + 0x1164: 0xb07b, 0x1165: 0xb07b, 0x1166: 0xb07b, 0x1167: 0xb07b, 0x1168: 0xb07b, 0x1169: 0xb07b, + 0x116a: 0xb07b, 0x116b: 0xb07b, 0x116c: 0xb07b, 0x116d: 0xb07b, 0x116e: 0xb07b, 0x116f: 0xb07b, + 0x1170: 0xb07c, 0x1171: 0xb07c, 0x1172: 0xb07c, 0x1173: 0xb07c, 0x1174: 0xb07c, 0x1175: 0xb07c, + 0x1176: 0xb07c, 0x1177: 0xb07c, 0x1178: 0xb07c, 0x1179: 0xb07c, 0x117a: 0xb07c, 0x117b: 0xb07c, + 0x117c: 0xb07c, 0x117d: 0xb07c, 0x117e: 0xb07c, + // Block 0x46, offset 0x1180 + 0x1182: 0xb07d, 0x1183: 0xb07e, 0x1184: 0xb07f, 0x1185: 0xb080, + 0x1186: 0xb07f, 0x1187: 0xb07e, 0x118a: 0xb081, 0x118b: 0xb082, + 0x118c: 0xb083, 0x118d: 0xb07f, 0x118e: 0xb080, 0x118f: 0xb07f, + 0x1192: 0xb084, 0x1193: 0xb085, 0x1194: 0xb084, 0x1195: 0xb086, 0x1196: 0xb084, 0x1197: 0xb087, + 0x119a: 0xb088, 0x119b: 0xb089, 0x119c: 0xb08a, + 0x11a0: 0x908b, 0x11a1: 0x908b, 0x11a2: 0x908c, 0x11a3: 0x908d, + 0x11a4: 0x908b, 0x11a5: 0x908e, 0x11a6: 0x908f, 0x11a8: 0xb090, 0x11a9: 0xb091, + 0x11aa: 0xb092, 0x11ab: 0xb091, 0x11ac: 0xb093, 0x11ad: 0xb094, 0x11ae: 0xb095, + 0x11bd: 0x2000, + // Block 0x47, offset 0x11c0 + 0x11e0: 0x4000, 0x11e1: 0x4000, 0x11e2: 0x4000, 0x11e3: 0x4000, + // Block 0x48, offset 0x1200 + 0x1200: 0x4000, 0x1201: 0x4000, 0x1202: 0x4000, 0x1203: 0x4000, 0x1204: 0x4000, 0x1205: 0x4000, + 0x1206: 0x4000, 0x1207: 0x4000, 0x1208: 0x4000, 0x1209: 0x4000, 0x120a: 0x4000, 0x120b: 0x4000, + 0x120c: 0x4000, 0x120d: 0x4000, 0x120e: 0x4000, 0x120f: 0x4000, 0x1210: 0x4000, 0x1211: 0x4000, + 0x1212: 0x4000, 0x1213: 0x4000, 0x1214: 0x4000, 0x1215: 0x4000, 0x1216: 0x4000, 0x1217: 0x4000, + 0x1218: 0x4000, 0x1219: 0x4000, 0x121a: 0x4000, 0x121b: 0x4000, 0x121c: 0x4000, 0x121d: 0x4000, + 0x121e: 0x4000, 0x121f: 0x4000, 0x1220: 0x4000, 0x1221: 0x4000, 0x1222: 0x4000, 0x1223: 0x4000, + 0x1224: 0x4000, 0x1225: 0x4000, 0x1226: 0x4000, 0x1227: 0x4000, 0x1228: 0x4000, 0x1229: 0x4000, + 0x122a: 0x4000, 0x122b: 0x4000, 0x122c: 0x4000, 0x122d: 0x4000, 0x122e: 0x4000, 0x122f: 0x4000, + 0x1230: 0x4000, 0x1231: 0x4000, 0x1232: 0x4000, 0x1233: 0x4000, 0x1234: 0x4000, 0x1235: 0x4000, + 0x1236: 0x4000, 0x1237: 0x4000, + // Block 0x49, offset 0x1240 + 0x1240: 0x4000, 0x1241: 0x4000, 0x1242: 0x4000, 0x1243: 0x4000, 0x1244: 0x4000, 0x1245: 0x4000, + 0x1246: 0x4000, 0x1247: 0x4000, 0x1248: 0x4000, 0x1249: 0x4000, 0x124a: 0x4000, 0x124b: 0x4000, + 0x124c: 0x4000, 0x124d: 0x4000, 0x124e: 0x4000, 0x124f: 0x4000, 0x1250: 0x4000, 0x1251: 0x4000, + 0x1252: 0x4000, 0x1253: 0x4000, 0x1254: 0x4000, 0x1255: 0x4000, 0x1256: 0x4000, 0x1257: 0x4000, + 0x1258: 0x4000, 0x1259: 0x4000, 0x125a: 0x4000, 0x125b: 0x4000, 0x125c: 0x4000, 0x125d: 0x4000, + 0x125e: 0x4000, 0x125f: 0x4000, 0x1260: 0x4000, 0x1261: 0x4000, 0x1262: 0x4000, 0x1263: 0x4000, + 0x1264: 0x4000, 0x1265: 0x4000, 0x1266: 0x4000, 0x1267: 0x4000, 0x1268: 0x4000, 0x1269: 0x4000, + 0x126a: 0x4000, 0x126b: 0x4000, 0x126c: 0x4000, 0x126d: 0x4000, 0x126e: 0x4000, 0x126f: 0x4000, + 0x1270: 0x4000, 0x1271: 0x4000, 0x1272: 0x4000, + // Block 0x4a, offset 0x1280 + 0x1280: 0x4000, 0x1281: 0x4000, 0x1282: 0x4000, 0x1283: 0x4000, 0x1284: 0x4000, 0x1285: 0x4000, + 0x1286: 0x4000, 0x1287: 0x4000, 0x1288: 0x4000, 0x1289: 0x4000, 0x128a: 0x4000, 0x128b: 0x4000, + 0x128c: 0x4000, 0x128d: 0x4000, 0x128e: 0x4000, 0x128f: 0x4000, 0x1290: 0x4000, 0x1291: 0x4000, + 0x1292: 0x4000, 0x1293: 0x4000, 0x1294: 0x4000, 0x1295: 0x4000, 0x1296: 0x4000, 0x1297: 0x4000, + 0x1298: 0x4000, 0x1299: 0x4000, 0x129a: 0x4000, 0x129b: 0x4000, 0x129c: 0x4000, 0x129d: 0x4000, + 0x129e: 0x4000, + // Block 0x4b, offset 0x12c0 + 0x12d0: 0x4000, 0x12d1: 0x4000, + 0x12d2: 0x4000, + 0x12e4: 0x4000, 0x12e5: 0x4000, 0x12e6: 0x4000, 0x12e7: 0x4000, + 0x12f0: 0x4000, 0x12f1: 0x4000, 0x12f2: 0x4000, 0x12f3: 0x4000, 0x12f4: 0x4000, 0x12f5: 0x4000, + 0x12f6: 0x4000, 0x12f7: 0x4000, 0x12f8: 0x4000, 0x12f9: 0x4000, 0x12fa: 0x4000, 0x12fb: 0x4000, + 0x12fc: 0x4000, 0x12fd: 0x4000, 0x12fe: 0x4000, 0x12ff: 0x4000, + // Block 0x4c, offset 0x1300 + 0x1300: 0x4000, 0x1301: 0x4000, 0x1302: 0x4000, 0x1303: 0x4000, 0x1304: 0x4000, 0x1305: 0x4000, + 0x1306: 0x4000, 0x1307: 0x4000, 0x1308: 0x4000, 0x1309: 0x4000, 0x130a: 0x4000, 0x130b: 0x4000, + 0x130c: 0x4000, 0x130d: 0x4000, 0x130e: 0x4000, 0x130f: 0x4000, 0x1310: 0x4000, 0x1311: 0x4000, + 0x1312: 0x4000, 0x1313: 0x4000, 0x1314: 0x4000, 0x1315: 0x4000, 0x1316: 0x4000, 0x1317: 0x4000, + 0x1318: 0x4000, 0x1319: 0x4000, 0x131a: 0x4000, 0x131b: 0x4000, 0x131c: 0x4000, 0x131d: 0x4000, + 0x131e: 0x4000, 0x131f: 0x4000, 0x1320: 0x4000, 0x1321: 0x4000, 0x1322: 0x4000, 0x1323: 0x4000, + 0x1324: 0x4000, 0x1325: 0x4000, 0x1326: 0x4000, 0x1327: 0x4000, 0x1328: 0x4000, 0x1329: 0x4000, + 0x132a: 0x4000, 0x132b: 0x4000, 0x132c: 0x4000, 0x132d: 0x4000, 0x132e: 0x4000, 0x132f: 0x4000, + 0x1330: 0x4000, 0x1331: 0x4000, 0x1332: 0x4000, 0x1333: 0x4000, 0x1334: 0x4000, 0x1335: 0x4000, + 0x1336: 0x4000, 0x1337: 0x4000, 0x1338: 0x4000, 0x1339: 0x4000, 0x133a: 0x4000, 0x133b: 0x4000, + // Block 0x4d, offset 0x1340 + 0x1344: 0x4000, + // Block 0x4e, offset 0x1380 + 0x138f: 0x4000, + // Block 0x4f, offset 0x13c0 + 0x13c0: 0x2000, 0x13c1: 0x2000, 0x13c2: 0x2000, 0x13c3: 0x2000, 0x13c4: 0x2000, 0x13c5: 0x2000, + 0x13c6: 0x2000, 0x13c7: 0x2000, 0x13c8: 0x2000, 0x13c9: 0x2000, 0x13ca: 0x2000, + 0x13d0: 0x2000, 0x13d1: 0x2000, + 0x13d2: 0x2000, 0x13d3: 0x2000, 0x13d4: 0x2000, 0x13d5: 0x2000, 0x13d6: 0x2000, 0x13d7: 0x2000, + 0x13d8: 0x2000, 0x13d9: 0x2000, 0x13da: 0x2000, 0x13db: 0x2000, 0x13dc: 0x2000, 0x13dd: 0x2000, + 0x13de: 0x2000, 0x13df: 0x2000, 0x13e0: 0x2000, 0x13e1: 0x2000, 0x13e2: 0x2000, 0x13e3: 0x2000, + 0x13e4: 0x2000, 0x13e5: 0x2000, 0x13e6: 0x2000, 0x13e7: 0x2000, 0x13e8: 0x2000, 0x13e9: 0x2000, + 0x13ea: 0x2000, 0x13eb: 0x2000, 0x13ec: 0x2000, 0x13ed: 0x2000, + 0x13f0: 0x2000, 0x13f1: 0x2000, 0x13f2: 0x2000, 0x13f3: 0x2000, 0x13f4: 0x2000, 0x13f5: 0x2000, + 0x13f6: 0x2000, 0x13f7: 0x2000, 0x13f8: 0x2000, 0x13f9: 0x2000, 0x13fa: 0x2000, 0x13fb: 0x2000, + 0x13fc: 0x2000, 0x13fd: 0x2000, 0x13fe: 0x2000, 0x13ff: 0x2000, + // Block 0x50, offset 0x1400 + 0x1400: 0x2000, 0x1401: 0x2000, 0x1402: 0x2000, 0x1403: 0x2000, 0x1404: 0x2000, 0x1405: 0x2000, + 0x1406: 0x2000, 0x1407: 0x2000, 0x1408: 0x2000, 0x1409: 0x2000, 0x140a: 0x2000, 0x140b: 0x2000, + 0x140c: 0x2000, 0x140d: 0x2000, 0x140e: 0x2000, 0x140f: 0x2000, 0x1410: 0x2000, 0x1411: 0x2000, + 0x1412: 0x2000, 0x1413: 0x2000, 0x1414: 0x2000, 0x1415: 0x2000, 0x1416: 0x2000, 0x1417: 0x2000, + 0x1418: 0x2000, 0x1419: 0x2000, 0x141a: 0x2000, 0x141b: 0x2000, 0x141c: 0x2000, 0x141d: 0x2000, + 0x141e: 0x2000, 0x141f: 0x2000, 0x1420: 0x2000, 0x1421: 0x2000, 0x1422: 0x2000, 0x1423: 0x2000, + 0x1424: 0x2000, 0x1425: 0x2000, 0x1426: 0x2000, 0x1427: 0x2000, 0x1428: 0x2000, 0x1429: 0x2000, + 0x1430: 0x2000, 0x1431: 0x2000, 0x1432: 0x2000, 0x1433: 0x2000, 0x1434: 0x2000, 0x1435: 0x2000, + 0x1436: 0x2000, 0x1437: 0x2000, 0x1438: 0x2000, 0x1439: 0x2000, 0x143a: 0x2000, 0x143b: 0x2000, + 0x143c: 0x2000, 0x143d: 0x2000, 0x143e: 0x2000, 0x143f: 0x2000, + // Block 0x51, offset 0x1440 + 0x1440: 0x2000, 0x1441: 0x2000, 0x1442: 0x2000, 0x1443: 0x2000, 0x1444: 0x2000, 0x1445: 0x2000, + 0x1446: 0x2000, 0x1447: 0x2000, 0x1448: 0x2000, 0x1449: 0x2000, 0x144a: 0x2000, 0x144b: 0x2000, + 0x144c: 0x2000, 0x144d: 0x2000, 0x144e: 0x4000, 0x144f: 0x2000, 0x1450: 0x2000, 0x1451: 0x4000, + 0x1452: 0x4000, 0x1453: 0x4000, 0x1454: 0x4000, 0x1455: 0x4000, 0x1456: 0x4000, 0x1457: 0x4000, + 0x1458: 0x4000, 0x1459: 0x4000, 0x145a: 0x4000, 0x145b: 0x2000, 0x145c: 0x2000, 0x145d: 0x2000, + 0x145e: 0x2000, 0x145f: 0x2000, 0x1460: 0x2000, 0x1461: 0x2000, 0x1462: 0x2000, 0x1463: 0x2000, + 0x1464: 0x2000, 0x1465: 0x2000, 0x1466: 0x2000, 0x1467: 0x2000, 0x1468: 0x2000, 0x1469: 0x2000, + 0x146a: 0x2000, 0x146b: 0x2000, 0x146c: 0x2000, + // Block 0x52, offset 0x1480 + 0x1480: 0x4000, 0x1481: 0x4000, 0x1482: 0x4000, + 0x1490: 0x4000, 0x1491: 0x4000, + 0x1492: 0x4000, 0x1493: 0x4000, 0x1494: 0x4000, 0x1495: 0x4000, 0x1496: 0x4000, 0x1497: 0x4000, + 0x1498: 0x4000, 0x1499: 0x4000, 0x149a: 0x4000, 0x149b: 0x4000, 0x149c: 0x4000, 0x149d: 0x4000, + 0x149e: 0x4000, 0x149f: 0x4000, 0x14a0: 0x4000, 0x14a1: 0x4000, 0x14a2: 0x4000, 0x14a3: 0x4000, + 0x14a4: 0x4000, 0x14a5: 0x4000, 0x14a6: 0x4000, 0x14a7: 0x4000, 0x14a8: 0x4000, 0x14a9: 0x4000, + 0x14aa: 0x4000, 0x14ab: 0x4000, 0x14ac: 0x4000, 0x14ad: 0x4000, 0x14ae: 0x4000, 0x14af: 0x4000, + 0x14b0: 0x4000, 0x14b1: 0x4000, 0x14b2: 0x4000, 0x14b3: 0x4000, 0x14b4: 0x4000, 0x14b5: 0x4000, + 0x14b6: 0x4000, 0x14b7: 0x4000, 0x14b8: 0x4000, 0x14b9: 0x4000, 0x14ba: 0x4000, 0x14bb: 0x4000, + // Block 0x53, offset 0x14c0 + 0x14c0: 0x4000, 0x14c1: 0x4000, 0x14c2: 0x4000, 0x14c3: 0x4000, 0x14c4: 0x4000, 0x14c5: 0x4000, + 0x14c6: 0x4000, 0x14c7: 0x4000, 0x14c8: 0x4000, + 0x14d0: 0x4000, 0x14d1: 0x4000, + 0x14e0: 0x4000, 0x14e1: 0x4000, 0x14e2: 0x4000, 0x14e3: 0x4000, + 0x14e4: 0x4000, 0x14e5: 0x4000, + // Block 0x54, offset 0x1500 + 0x1500: 0x4000, 0x1501: 0x4000, 0x1502: 0x4000, 0x1503: 0x4000, 0x1504: 0x4000, 0x1505: 0x4000, + 0x1506: 0x4000, 0x1507: 0x4000, 0x1508: 0x4000, 0x1509: 0x4000, 0x150a: 0x4000, 0x150b: 0x4000, + 0x150c: 0x4000, 0x150d: 0x4000, 0x150e: 0x4000, 0x150f: 0x4000, 0x1510: 0x4000, 0x1511: 0x4000, + 0x1512: 0x4000, 0x1513: 0x4000, 0x1514: 0x4000, 0x1515: 0x4000, 0x1516: 0x4000, 0x1517: 0x4000, + 0x1518: 0x4000, 0x1519: 0x4000, 0x151a: 0x4000, 0x151b: 0x4000, 0x151c: 0x4000, 0x151d: 0x4000, + 0x151e: 0x4000, 0x151f: 0x4000, 0x1520: 0x4000, + 0x152d: 0x4000, 0x152e: 0x4000, 0x152f: 0x4000, + 0x1530: 0x4000, 0x1531: 0x4000, 0x1532: 0x4000, 0x1533: 0x4000, 0x1534: 0x4000, 0x1535: 0x4000, + 0x1537: 0x4000, 0x1538: 0x4000, 0x1539: 0x4000, 0x153a: 0x4000, 0x153b: 0x4000, + 0x153c: 0x4000, 0x153d: 0x4000, 0x153e: 0x4000, 0x153f: 0x4000, + // Block 0x55, offset 0x1540 + 0x1540: 0x4000, 0x1541: 0x4000, 0x1542: 0x4000, 0x1543: 0x4000, 0x1544: 0x4000, 0x1545: 0x4000, + 0x1546: 0x4000, 0x1547: 0x4000, 0x1548: 0x4000, 0x1549: 0x4000, 0x154a: 0x4000, 0x154b: 0x4000, + 0x154c: 0x4000, 0x154d: 0x4000, 0x154e: 0x4000, 0x154f: 0x4000, 0x1550: 0x4000, 0x1551: 0x4000, + 0x1552: 0x4000, 0x1553: 0x4000, 0x1554: 0x4000, 0x1555: 0x4000, 0x1556: 0x4000, 0x1557: 0x4000, + 0x1558: 0x4000, 0x1559: 0x4000, 0x155a: 0x4000, 0x155b: 0x4000, 0x155c: 0x4000, 0x155d: 0x4000, + 0x155e: 0x4000, 0x155f: 0x4000, 0x1560: 0x4000, 0x1561: 0x4000, 0x1562: 0x4000, 0x1563: 0x4000, + 0x1564: 0x4000, 0x1565: 0x4000, 0x1566: 0x4000, 0x1567: 0x4000, 0x1568: 0x4000, 0x1569: 0x4000, + 0x156a: 0x4000, 0x156b: 0x4000, 0x156c: 0x4000, 0x156d: 0x4000, 0x156e: 0x4000, 0x156f: 0x4000, + 0x1570: 0x4000, 0x1571: 0x4000, 0x1572: 0x4000, 0x1573: 0x4000, 0x1574: 0x4000, 0x1575: 0x4000, + 0x1576: 0x4000, 0x1577: 0x4000, 0x1578: 0x4000, 0x1579: 0x4000, 0x157a: 0x4000, 0x157b: 0x4000, + 0x157c: 0x4000, 0x157e: 0x4000, 0x157f: 0x4000, + // Block 0x56, offset 0x1580 + 0x1580: 0x4000, 0x1581: 0x4000, 0x1582: 0x4000, 0x1583: 0x4000, 0x1584: 0x4000, 0x1585: 0x4000, + 0x1586: 0x4000, 0x1587: 0x4000, 0x1588: 0x4000, 0x1589: 0x4000, 0x158a: 0x4000, 0x158b: 0x4000, + 0x158c: 0x4000, 0x158d: 0x4000, 0x158e: 0x4000, 0x158f: 0x4000, 0x1590: 0x4000, 0x1591: 0x4000, + 0x1592: 0x4000, 0x1593: 0x4000, + 0x15a0: 0x4000, 0x15a1: 0x4000, 0x15a2: 0x4000, 0x15a3: 0x4000, + 0x15a4: 0x4000, 0x15a5: 0x4000, 0x15a6: 0x4000, 0x15a7: 0x4000, 0x15a8: 0x4000, 0x15a9: 0x4000, + 0x15aa: 0x4000, 0x15ab: 0x4000, 0x15ac: 0x4000, 0x15ad: 0x4000, 0x15ae: 0x4000, 0x15af: 0x4000, + 0x15b0: 0x4000, 0x15b1: 0x4000, 0x15b2: 0x4000, 0x15b3: 0x4000, 0x15b4: 0x4000, 0x15b5: 0x4000, + 0x15b6: 0x4000, 0x15b7: 0x4000, 0x15b8: 0x4000, 0x15b9: 0x4000, 0x15ba: 0x4000, 0x15bb: 0x4000, + 0x15bc: 0x4000, 0x15bd: 0x4000, 0x15be: 0x4000, 0x15bf: 0x4000, + // Block 0x57, offset 0x15c0 + 0x15c0: 0x4000, 0x15c1: 0x4000, 0x15c2: 0x4000, 0x15c3: 0x4000, 0x15c4: 0x4000, 0x15c5: 0x4000, + 0x15c6: 0x4000, 0x15c7: 0x4000, 0x15c8: 0x4000, 0x15c9: 0x4000, 0x15ca: 0x4000, + 0x15cf: 0x4000, 0x15d0: 0x4000, 0x15d1: 0x4000, + 0x15d2: 0x4000, 0x15d3: 0x4000, + 0x15e0: 0x4000, 0x15e1: 0x4000, 0x15e2: 0x4000, 0x15e3: 0x4000, + 0x15e4: 0x4000, 0x15e5: 0x4000, 0x15e6: 0x4000, 0x15e7: 0x4000, 0x15e8: 0x4000, 0x15e9: 0x4000, + 0x15ea: 0x4000, 0x15eb: 0x4000, 0x15ec: 0x4000, 0x15ed: 0x4000, 0x15ee: 0x4000, 0x15ef: 0x4000, + 0x15f0: 0x4000, 0x15f4: 0x4000, + 0x15f8: 0x4000, 0x15f9: 0x4000, 0x15fa: 0x4000, 0x15fb: 0x4000, + 0x15fc: 0x4000, 0x15fd: 0x4000, 0x15fe: 0x4000, 0x15ff: 0x4000, + // Block 0x58, offset 0x1600 + 0x1600: 0x4000, 0x1602: 0x4000, 0x1603: 0x4000, 0x1604: 0x4000, 0x1605: 0x4000, + 0x1606: 0x4000, 0x1607: 0x4000, 0x1608: 0x4000, 0x1609: 0x4000, 0x160a: 0x4000, 0x160b: 0x4000, + 0x160c: 0x4000, 0x160d: 0x4000, 0x160e: 0x4000, 0x160f: 0x4000, 0x1610: 0x4000, 0x1611: 0x4000, + 0x1612: 0x4000, 0x1613: 0x4000, 0x1614: 0x4000, 0x1615: 0x4000, 0x1616: 0x4000, 0x1617: 0x4000, + 0x1618: 0x4000, 0x1619: 0x4000, 0x161a: 0x4000, 0x161b: 0x4000, 0x161c: 0x4000, 0x161d: 0x4000, + 0x161e: 0x4000, 0x161f: 0x4000, 0x1620: 0x4000, 0x1621: 0x4000, 0x1622: 0x4000, 0x1623: 0x4000, + 0x1624: 0x4000, 0x1625: 0x4000, 0x1626: 0x4000, 0x1627: 0x4000, 0x1628: 0x4000, 0x1629: 0x4000, + 0x162a: 0x4000, 0x162b: 0x4000, 0x162c: 0x4000, 0x162d: 0x4000, 0x162e: 0x4000, 0x162f: 0x4000, + 0x1630: 0x4000, 0x1631: 0x4000, 0x1632: 0x4000, 0x1633: 0x4000, 0x1634: 0x4000, 0x1635: 0x4000, + 0x1636: 0x4000, 0x1637: 0x4000, 0x1638: 0x4000, 0x1639: 0x4000, 0x163a: 0x4000, 0x163b: 0x4000, + 0x163c: 0x4000, 0x163d: 0x4000, 0x163e: 0x4000, 0x163f: 0x4000, + // Block 0x59, offset 0x1640 + 0x1640: 0x4000, 0x1641: 0x4000, 0x1642: 0x4000, 0x1643: 0x4000, 0x1644: 0x4000, 0x1645: 0x4000, + 0x1646: 0x4000, 0x1647: 0x4000, 0x1648: 0x4000, 0x1649: 0x4000, 0x164a: 0x4000, 0x164b: 0x4000, + 0x164c: 0x4000, 0x164d: 0x4000, 0x164e: 0x4000, 0x164f: 0x4000, 0x1650: 0x4000, 0x1651: 0x4000, + 0x1652: 0x4000, 0x1653: 0x4000, 0x1654: 0x4000, 0x1655: 0x4000, 0x1656: 0x4000, 0x1657: 0x4000, + 0x1658: 0x4000, 0x1659: 0x4000, 0x165a: 0x4000, 0x165b: 0x4000, 0x165c: 0x4000, 0x165d: 0x4000, + 0x165e: 0x4000, 0x165f: 0x4000, 0x1660: 0x4000, 0x1661: 0x4000, 0x1662: 0x4000, 0x1663: 0x4000, + 0x1664: 0x4000, 0x1665: 0x4000, 0x1666: 0x4000, 0x1667: 0x4000, 0x1668: 0x4000, 0x1669: 0x4000, + 0x166a: 0x4000, 0x166b: 0x4000, 0x166c: 0x4000, 0x166d: 0x4000, 0x166e: 0x4000, 0x166f: 0x4000, + 0x1670: 0x4000, 0x1671: 0x4000, 0x1672: 0x4000, 0x1673: 0x4000, 0x1674: 0x4000, 0x1675: 0x4000, + 0x1676: 0x4000, 0x1677: 0x4000, 0x1678: 0x4000, 0x1679: 0x4000, 0x167a: 0x4000, 0x167b: 0x4000, + 0x167c: 0x4000, 0x167f: 0x4000, + // Block 0x5a, offset 0x1680 + 0x1680: 0x4000, 0x1681: 0x4000, 0x1682: 0x4000, 0x1683: 0x4000, 0x1684: 0x4000, 0x1685: 0x4000, + 0x1686: 0x4000, 0x1687: 0x4000, 0x1688: 0x4000, 0x1689: 0x4000, 0x168a: 0x4000, 0x168b: 0x4000, + 0x168c: 0x4000, 0x168d: 0x4000, 0x168e: 0x4000, 0x168f: 0x4000, 0x1690: 0x4000, 0x1691: 0x4000, + 0x1692: 0x4000, 0x1693: 0x4000, 0x1694: 0x4000, 0x1695: 0x4000, 0x1696: 0x4000, 0x1697: 0x4000, + 0x1698: 0x4000, 0x1699: 0x4000, 0x169a: 0x4000, 0x169b: 0x4000, 0x169c: 0x4000, 0x169d: 0x4000, + 0x169e: 0x4000, 0x169f: 0x4000, 0x16a0: 0x4000, 0x16a1: 0x4000, 0x16a2: 0x4000, 0x16a3: 0x4000, + 0x16a4: 0x4000, 0x16a5: 0x4000, 0x16a6: 0x4000, 0x16a7: 0x4000, 0x16a8: 0x4000, 0x16a9: 0x4000, + 0x16aa: 0x4000, 0x16ab: 0x4000, 0x16ac: 0x4000, 0x16ad: 0x4000, 0x16ae: 0x4000, 0x16af: 0x4000, + 0x16b0: 0x4000, 0x16b1: 0x4000, 0x16b2: 0x4000, 0x16b3: 0x4000, 0x16b4: 0x4000, 0x16b5: 0x4000, + 0x16b6: 0x4000, 0x16b7: 0x4000, 0x16b8: 0x4000, 0x16b9: 0x4000, 0x16ba: 0x4000, 0x16bb: 0x4000, + 0x16bc: 0x4000, 0x16bd: 0x4000, + // Block 0x5b, offset 0x16c0 + 0x16cb: 0x4000, + 0x16cc: 0x4000, 0x16cd: 0x4000, 0x16ce: 0x4000, 0x16d0: 0x4000, 0x16d1: 0x4000, + 0x16d2: 0x4000, 0x16d3: 0x4000, 0x16d4: 0x4000, 0x16d5: 0x4000, 0x16d6: 0x4000, 0x16d7: 0x4000, + 0x16d8: 0x4000, 0x16d9: 0x4000, 0x16da: 0x4000, 0x16db: 0x4000, 0x16dc: 0x4000, 0x16dd: 0x4000, + 0x16de: 0x4000, 0x16df: 0x4000, 0x16e0: 0x4000, 0x16e1: 0x4000, 0x16e2: 0x4000, 0x16e3: 0x4000, + 0x16e4: 0x4000, 0x16e5: 0x4000, 0x16e6: 0x4000, 0x16e7: 0x4000, + 0x16fa: 0x4000, + // Block 0x5c, offset 0x1700 + 0x1715: 0x4000, 0x1716: 0x4000, + 0x1724: 0x4000, + // Block 0x5d, offset 0x1740 + 0x177b: 0x4000, + 0x177c: 0x4000, 0x177d: 0x4000, 0x177e: 0x4000, 0x177f: 0x4000, + // Block 0x5e, offset 0x1780 + 0x1780: 0x4000, 0x1781: 0x4000, 0x1782: 0x4000, 0x1783: 0x4000, 0x1784: 0x4000, 0x1785: 0x4000, + 0x1786: 0x4000, 0x1787: 0x4000, 0x1788: 0x4000, 0x1789: 0x4000, 0x178a: 0x4000, 0x178b: 0x4000, + 0x178c: 0x4000, 0x178d: 0x4000, 0x178e: 0x4000, 0x178f: 0x4000, + // Block 0x5f, offset 0x17c0 + 0x17c0: 0x4000, 0x17c1: 0x4000, 0x17c2: 0x4000, 0x17c3: 0x4000, 0x17c4: 0x4000, 0x17c5: 0x4000, + 0x17cc: 0x4000, 0x17d0: 0x4000, 0x17d1: 0x4000, + 0x17d2: 0x4000, 0x17d5: 0x4000, + 0x17eb: 0x4000, 0x17ec: 0x4000, + 0x17f4: 0x4000, 0x17f5: 0x4000, + 0x17f6: 0x4000, 0x17f7: 0x4000, 0x17f8: 0x4000, 0x17f9: 0x4000, 0x17fa: 0x4000, + // Block 0x60, offset 0x1800 + 0x1820: 0x4000, 0x1821: 0x4000, 0x1822: 0x4000, 0x1823: 0x4000, + 0x1824: 0x4000, 0x1825: 0x4000, 0x1826: 0x4000, 0x1827: 0x4000, 0x1828: 0x4000, 0x1829: 0x4000, + 0x182a: 0x4000, 0x182b: 0x4000, + // Block 0x61, offset 0x1840 + 0x184d: 0x4000, 0x184e: 0x4000, 0x184f: 0x4000, 0x1850: 0x4000, 0x1851: 0x4000, + 0x1852: 0x4000, 0x1853: 0x4000, 0x1854: 0x4000, 0x1855: 0x4000, 0x1856: 0x4000, 0x1857: 0x4000, + 0x1858: 0x4000, 0x1859: 0x4000, 0x185a: 0x4000, 0x185b: 0x4000, 0x185c: 0x4000, 0x185d: 0x4000, + 0x185e: 0x4000, 0x185f: 0x4000, 0x1860: 0x4000, 0x1861: 0x4000, 0x1862: 0x4000, 0x1863: 0x4000, + 0x1864: 0x4000, 0x1865: 0x4000, 0x1866: 0x4000, 0x1867: 0x4000, 0x1868: 0x4000, 0x1869: 0x4000, + 0x186a: 0x4000, 0x186b: 0x4000, 0x186c: 0x4000, 0x186d: 0x4000, 0x186e: 0x4000, 0x186f: 0x4000, + 0x1870: 0x4000, 0x1871: 0x4000, 0x1872: 0x4000, 0x1873: 0x4000, 0x1874: 0x4000, 0x1875: 0x4000, + 0x1876: 0x4000, 0x1877: 0x4000, 0x1878: 0x4000, 0x1879: 0x4000, 0x187a: 0x4000, 0x187b: 0x4000, + 0x187c: 0x4000, 0x187d: 0x4000, 0x187e: 0x4000, 0x187f: 0x4000, + // Block 0x62, offset 0x1880 + 0x1880: 0x4000, 0x1881: 0x4000, 0x1882: 0x4000, 0x1883: 0x4000, 0x1884: 0x4000, 0x1885: 0x4000, + 0x1886: 0x4000, 0x1887: 0x4000, 0x1888: 0x4000, 0x1889: 0x4000, 0x188a: 0x4000, 0x188b: 0x4000, + 0x188c: 0x4000, 0x188d: 0x4000, 0x188e: 0x4000, 0x188f: 0x4000, 0x1890: 0x4000, 0x1891: 0x4000, + 0x1892: 0x4000, 0x1893: 0x4000, 0x1894: 0x4000, 0x1895: 0x4000, 0x1896: 0x4000, 0x1897: 0x4000, + 0x1898: 0x4000, 0x1899: 0x4000, 0x189a: 0x4000, 0x189b: 0x4000, 0x189c: 0x4000, 0x189d: 0x4000, + 0x189e: 0x4000, 0x189f: 0x4000, 0x18a0: 0x4000, 0x18a1: 0x4000, 0x18a2: 0x4000, 0x18a3: 0x4000, + 0x18a4: 0x4000, 0x18a5: 0x4000, 0x18a6: 0x4000, 0x18a7: 0x4000, 0x18a8: 0x4000, 0x18a9: 0x4000, + 0x18aa: 0x4000, 0x18ab: 0x4000, 0x18ac: 0x4000, 0x18ad: 0x4000, 0x18ae: 0x4000, 0x18af: 0x4000, + 0x18b0: 0x4000, 0x18b1: 0x4000, 0x18b3: 0x4000, 0x18b4: 0x4000, 0x18b5: 0x4000, + 0x18b6: 0x4000, 0x18ba: 0x4000, 0x18bb: 0x4000, + 0x18bc: 0x4000, 0x18bd: 0x4000, 0x18be: 0x4000, 0x18bf: 0x4000, + // Block 0x63, offset 0x18c0 + 0x18c0: 0x4000, 0x18c1: 0x4000, 0x18c2: 0x4000, 0x18c3: 0x4000, 0x18c4: 0x4000, 0x18c5: 0x4000, + 0x18c6: 0x4000, 0x18c7: 0x4000, 0x18c8: 0x4000, 0x18c9: 0x4000, 0x18ca: 0x4000, 0x18cb: 0x4000, + 0x18cc: 0x4000, 0x18cd: 0x4000, 0x18ce: 0x4000, 0x18cf: 0x4000, 0x18d0: 0x4000, 0x18d1: 0x4000, + 0x18d2: 0x4000, 0x18d3: 0x4000, 0x18d4: 0x4000, 0x18d5: 0x4000, 0x18d6: 0x4000, 0x18d7: 0x4000, + 0x18d8: 0x4000, 0x18d9: 0x4000, 0x18da: 0x4000, 0x18db: 0x4000, 0x18dc: 0x4000, 0x18dd: 0x4000, + 0x18de: 0x4000, 0x18df: 0x4000, 0x18e0: 0x4000, 0x18e1: 0x4000, 0x18e2: 0x4000, + 0x18e5: 0x4000, 0x18e6: 0x4000, 0x18e7: 0x4000, 0x18e8: 0x4000, 0x18e9: 0x4000, + 0x18ea: 0x4000, 0x18ee: 0x4000, 0x18ef: 0x4000, + 0x18f0: 0x4000, 0x18f1: 0x4000, 0x18f2: 0x4000, 0x18f3: 0x4000, 0x18f4: 0x4000, 0x18f5: 0x4000, + 0x18f6: 0x4000, 0x18f7: 0x4000, 0x18f8: 0x4000, 0x18f9: 0x4000, 0x18fa: 0x4000, 0x18fb: 0x4000, + 0x18fc: 0x4000, 0x18fd: 0x4000, 0x18fe: 0x4000, 0x18ff: 0x4000, + // Block 0x64, offset 0x1900 + 0x1900: 0x4000, 0x1901: 0x4000, 0x1902: 0x4000, 0x1903: 0x4000, 0x1904: 0x4000, 0x1905: 0x4000, + 0x1906: 0x4000, 0x1907: 0x4000, 0x1908: 0x4000, 0x1909: 0x4000, 0x190a: 0x4000, + 0x190d: 0x4000, 0x190e: 0x4000, 0x190f: 0x4000, 0x1910: 0x4000, 0x1911: 0x4000, + 0x1912: 0x4000, 0x1913: 0x4000, 0x1914: 0x4000, 0x1915: 0x4000, 0x1916: 0x4000, 0x1917: 0x4000, + 0x1918: 0x4000, 0x1919: 0x4000, 0x191a: 0x4000, 0x191b: 0x4000, 0x191c: 0x4000, 0x191d: 0x4000, + 0x191e: 0x4000, 0x191f: 0x4000, 0x1920: 0x4000, 0x1921: 0x4000, 0x1922: 0x4000, 0x1923: 0x4000, + 0x1924: 0x4000, 0x1925: 0x4000, 0x1926: 0x4000, 0x1927: 0x4000, 0x1928: 0x4000, 0x1929: 0x4000, + 0x192a: 0x4000, 0x192b: 0x4000, 0x192c: 0x4000, 0x192d: 0x4000, 0x192e: 0x4000, 0x192f: 0x4000, + 0x1930: 0x4000, 0x1931: 0x4000, 0x1932: 0x4000, 0x1933: 0x4000, 0x1934: 0x4000, 0x1935: 0x4000, + 0x1936: 0x4000, 0x1937: 0x4000, 0x1938: 0x4000, 0x1939: 0x4000, 0x193a: 0x4000, 0x193b: 0x4000, + 0x193c: 0x4000, 0x193d: 0x4000, 0x193e: 0x4000, 0x193f: 0x4000, + // Block 0x65, offset 0x1940 + 0x1970: 0x4000, 0x1971: 0x4000, 0x1972: 0x4000, 0x1973: 0x4000, + 0x1978: 0x4000, 0x1979: 0x4000, 0x197a: 0x4000, + // Block 0x66, offset 0x1980 + 0x1980: 0x4000, 0x1981: 0x4000, 0x1982: 0x4000, + 0x1990: 0x4000, 0x1991: 0x4000, + 0x1992: 0x4000, 0x1993: 0x4000, 0x1994: 0x4000, 0x1995: 0x4000, + // Block 0x67, offset 0x19c0 + 0x19c0: 0x2000, 0x19c1: 0x2000, 0x19c2: 0x2000, 0x19c3: 0x2000, 0x19c4: 0x2000, 0x19c5: 0x2000, + 0x19c6: 0x2000, 0x19c7: 0x2000, 0x19c8: 0x2000, 0x19c9: 0x2000, 0x19ca: 0x2000, 0x19cb: 0x2000, + 0x19cc: 0x2000, 0x19cd: 0x2000, 0x19ce: 0x2000, 0x19cf: 0x2000, 0x19d0: 0x2000, 0x19d1: 0x2000, + 0x19d2: 0x2000, 0x19d3: 0x2000, 0x19d4: 0x2000, 0x19d5: 0x2000, 0x19d6: 0x2000, 0x19d7: 0x2000, + 0x19d8: 0x2000, 0x19d9: 0x2000, 0x19da: 0x2000, 0x19db: 0x2000, 0x19dc: 0x2000, 0x19dd: 0x2000, + 0x19de: 0x2000, 0x19df: 0x2000, 0x19e0: 0x2000, 0x19e1: 0x2000, 0x19e2: 0x2000, 0x19e3: 0x2000, + 0x19e4: 0x2000, 0x19e5: 0x2000, 0x19e6: 0x2000, 0x19e7: 0x2000, 0x19e8: 0x2000, 0x19e9: 0x2000, + 0x19ea: 0x2000, 0x19eb: 0x2000, 0x19ec: 0x2000, 0x19ed: 0x2000, 0x19ee: 0x2000, 0x19ef: 0x2000, + 0x19f0: 0x2000, 0x19f1: 0x2000, 0x19f2: 0x2000, 0x19f3: 0x2000, 0x19f4: 0x2000, 0x19f5: 0x2000, + 0x19f6: 0x2000, 0x19f7: 0x2000, 0x19f8: 0x2000, 0x19f9: 0x2000, 0x19fa: 0x2000, 0x19fb: 0x2000, + 0x19fc: 0x2000, 0x19fd: 0x2000, +} + +// widthIndex: 22 blocks, 1408 entries, 1408 bytes +// Block 0 is the zero block. +var widthIndex = [1408]uint8{ + // Block 0x0, offset 0x0 + // Block 0x1, offset 0x40 + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0xc2: 0x01, 0xc3: 0x02, 0xc4: 0x03, 0xc5: 0x04, 0xc7: 0x05, + 0xc9: 0x06, 0xcb: 0x07, 0xcc: 0x08, 0xcd: 0x09, 0xce: 0x0a, 0xcf: 0x0b, + 0xd0: 0x0c, 0xd1: 0x0d, + 0xe1: 0x02, 0xe2: 0x03, 0xe3: 0x04, 0xe4: 0x05, 0xe5: 0x06, 0xe6: 0x06, 0xe7: 0x06, + 0xe8: 0x06, 0xe9: 0x06, 0xea: 0x07, 0xeb: 0x06, 0xec: 0x06, 0xed: 0x08, 0xee: 0x09, 0xef: 0x0a, + 0xf0: 0x0f, 0xf3: 0x12, 0xf4: 0x13, + // Block 0x4, offset 0x100 + 0x104: 0x0e, 0x105: 0x0f, + // Block 0x5, offset 0x140 + 0x140: 0x10, 0x141: 0x11, 0x142: 0x12, 0x144: 0x13, 0x145: 0x14, 0x146: 0x15, 0x147: 0x16, + 0x148: 0x17, 0x149: 0x18, 0x14a: 0x19, 0x14c: 0x1a, 0x14f: 0x1b, + 0x151: 0x1c, 0x152: 0x08, 0x153: 0x1d, 0x154: 0x1e, 0x155: 0x1f, 0x156: 0x20, 0x157: 0x21, + 0x158: 0x22, 0x159: 0x23, 0x15a: 0x24, 0x15b: 0x25, 0x15c: 0x26, 0x15d: 0x27, 0x15e: 0x28, 0x15f: 0x29, + 0x166: 0x2a, + 0x16c: 0x2b, 0x16d: 0x2c, + 0x17a: 0x2d, 0x17b: 0x2e, 0x17c: 0x0e, 0x17d: 0x0e, 0x17e: 0x0e, 0x17f: 0x2f, + // Block 0x6, offset 0x180 + 0x180: 0x30, 0x181: 0x31, 0x182: 0x32, 0x183: 0x33, 0x184: 0x34, 0x185: 0x35, 0x186: 0x36, 0x187: 0x37, + 0x188: 0x38, 0x189: 0x39, 0x18a: 0x0e, 0x18b: 0x3a, 0x18c: 0x0e, 0x18d: 0x0e, 0x18e: 0x0e, 0x18f: 0x0e, + 0x190: 0x0e, 0x191: 0x0e, 0x192: 0x0e, 0x193: 0x0e, 0x194: 0x0e, 0x195: 0x0e, 0x196: 0x0e, 0x197: 0x0e, + 0x198: 0x0e, 0x199: 0x0e, 0x19a: 0x0e, 0x19b: 0x0e, 0x19c: 0x0e, 0x19d: 0x0e, 0x19e: 0x0e, 0x19f: 0x0e, + 0x1a0: 0x0e, 0x1a1: 0x0e, 0x1a2: 0x0e, 0x1a3: 0x0e, 0x1a4: 0x0e, 0x1a5: 0x0e, 0x1a6: 0x0e, 0x1a7: 0x0e, + 0x1a8: 0x0e, 0x1a9: 0x0e, 0x1aa: 0x0e, 0x1ab: 0x0e, 0x1ac: 0x0e, 0x1ad: 0x0e, 0x1ae: 0x0e, 0x1af: 0x0e, + 0x1b0: 0x0e, 0x1b1: 0x0e, 0x1b2: 0x0e, 0x1b3: 0x0e, 0x1b4: 0x0e, 0x1b5: 0x0e, 0x1b6: 0x0e, 0x1b7: 0x0e, + 0x1b8: 0x0e, 0x1b9: 0x0e, 0x1ba: 0x0e, 0x1bb: 0x0e, 0x1bc: 0x0e, 0x1bd: 0x0e, 0x1be: 0x0e, 0x1bf: 0x0e, + // Block 0x7, offset 0x1c0 + 0x1c0: 0x0e, 0x1c1: 0x0e, 0x1c2: 0x0e, 0x1c3: 0x0e, 0x1c4: 0x0e, 0x1c5: 0x0e, 0x1c6: 0x0e, 0x1c7: 0x0e, + 0x1c8: 0x0e, 0x1c9: 0x0e, 0x1ca: 0x0e, 0x1cb: 0x0e, 0x1cc: 0x0e, 0x1cd: 0x0e, 0x1ce: 0x0e, 0x1cf: 0x0e, + 0x1d0: 0x0e, 0x1d1: 0x0e, 0x1d2: 0x0e, 0x1d3: 0x0e, 0x1d4: 0x0e, 0x1d5: 0x0e, 0x1d6: 0x0e, 0x1d7: 0x0e, + 0x1d8: 0x0e, 0x1d9: 0x0e, 0x1da: 0x0e, 0x1db: 0x0e, 0x1dc: 0x0e, 0x1dd: 0x0e, 0x1de: 0x0e, 0x1df: 0x0e, + 0x1e0: 0x0e, 0x1e1: 0x0e, 0x1e2: 0x0e, 0x1e3: 0x0e, 0x1e4: 0x0e, 0x1e5: 0x0e, 0x1e6: 0x0e, 0x1e7: 0x0e, + 0x1e8: 0x0e, 0x1e9: 0x0e, 0x1ea: 0x0e, 0x1eb: 0x0e, 0x1ec: 0x0e, 0x1ed: 0x0e, 0x1ee: 0x0e, 0x1ef: 0x0e, + 0x1f0: 0x0e, 0x1f1: 0x0e, 0x1f2: 0x0e, 0x1f3: 0x0e, 0x1f4: 0x0e, 0x1f5: 0x0e, 0x1f6: 0x0e, + 0x1f8: 0x0e, 0x1f9: 0x0e, 0x1fa: 0x0e, 0x1fb: 0x0e, 0x1fc: 0x0e, 0x1fd: 0x0e, 0x1fe: 0x0e, 0x1ff: 0x0e, + // Block 0x8, offset 0x200 + 0x200: 0x0e, 0x201: 0x0e, 0x202: 0x0e, 0x203: 0x0e, 0x204: 0x0e, 0x205: 0x0e, 0x206: 0x0e, 0x207: 0x0e, + 0x208: 0x0e, 0x209: 0x0e, 0x20a: 0x0e, 0x20b: 0x0e, 0x20c: 0x0e, 0x20d: 0x0e, 0x20e: 0x0e, 0x20f: 0x0e, + 0x210: 0x0e, 0x211: 0x0e, 0x212: 0x0e, 0x213: 0x0e, 0x214: 0x0e, 0x215: 0x0e, 0x216: 0x0e, 0x217: 0x0e, + 0x218: 0x0e, 0x219: 0x0e, 0x21a: 0x0e, 0x21b: 0x0e, 0x21c: 0x0e, 0x21d: 0x0e, 0x21e: 0x0e, 0x21f: 0x0e, + 0x220: 0x0e, 0x221: 0x0e, 0x222: 0x0e, 0x223: 0x0e, 0x224: 0x0e, 0x225: 0x0e, 0x226: 0x0e, 0x227: 0x0e, + 0x228: 0x0e, 0x229: 0x0e, 0x22a: 0x0e, 0x22b: 0x0e, 0x22c: 0x0e, 0x22d: 0x0e, 0x22e: 0x0e, 0x22f: 0x0e, + 0x230: 0x0e, 0x231: 0x0e, 0x232: 0x0e, 0x233: 0x0e, 0x234: 0x0e, 0x235: 0x0e, 0x236: 0x0e, 0x237: 0x0e, + 0x238: 0x0e, 0x239: 0x0e, 0x23a: 0x0e, 0x23b: 0x0e, 0x23c: 0x0e, 0x23d: 0x0e, 0x23e: 0x0e, 0x23f: 0x0e, + // Block 0x9, offset 0x240 + 0x240: 0x0e, 0x241: 0x0e, 0x242: 0x0e, 0x243: 0x0e, 0x244: 0x0e, 0x245: 0x0e, 0x246: 0x0e, 0x247: 0x0e, + 0x248: 0x0e, 0x249: 0x0e, 0x24a: 0x0e, 0x24b: 0x0e, 0x24c: 0x0e, 0x24d: 0x0e, 0x24e: 0x0e, 0x24f: 0x0e, + 0x250: 0x0e, 0x251: 0x0e, 0x252: 0x3b, 0x253: 0x3c, + 0x265: 0x3d, + 0x270: 0x0e, 0x271: 0x0e, 0x272: 0x0e, 0x273: 0x0e, 0x274: 0x0e, 0x275: 0x0e, 0x276: 0x0e, 0x277: 0x0e, + 0x278: 0x0e, 0x279: 0x0e, 0x27a: 0x0e, 0x27b: 0x0e, 0x27c: 0x0e, 0x27d: 0x0e, 0x27e: 0x0e, 0x27f: 0x0e, + // Block 0xa, offset 0x280 + 0x280: 0x0e, 0x281: 0x0e, 0x282: 0x0e, 0x283: 0x0e, 0x284: 0x0e, 0x285: 0x0e, 0x286: 0x0e, 0x287: 0x0e, + 0x288: 0x0e, 0x289: 0x0e, 0x28a: 0x0e, 0x28b: 0x0e, 0x28c: 0x0e, 0x28d: 0x0e, 0x28e: 0x0e, 0x28f: 0x0e, + 0x290: 0x0e, 0x291: 0x0e, 0x292: 0x0e, 0x293: 0x0e, 0x294: 0x0e, 0x295: 0x0e, 0x296: 0x0e, 0x297: 0x0e, + 0x298: 0x0e, 0x299: 0x0e, 0x29a: 0x0e, 0x29b: 0x0e, 0x29c: 0x0e, 0x29d: 0x0e, 0x29e: 0x3e, + // Block 0xb, offset 0x2c0 + 0x2c0: 0x08, 0x2c1: 0x08, 0x2c2: 0x08, 0x2c3: 0x08, 0x2c4: 0x08, 0x2c5: 0x08, 0x2c6: 0x08, 0x2c7: 0x08, + 0x2c8: 0x08, 0x2c9: 0x08, 0x2ca: 0x08, 0x2cb: 0x08, 0x2cc: 0x08, 0x2cd: 0x08, 0x2ce: 0x08, 0x2cf: 0x08, + 0x2d0: 0x08, 0x2d1: 0x08, 0x2d2: 0x08, 0x2d3: 0x08, 0x2d4: 0x08, 0x2d5: 0x08, 0x2d6: 0x08, 0x2d7: 0x08, + 0x2d8: 0x08, 0x2d9: 0x08, 0x2da: 0x08, 0x2db: 0x08, 0x2dc: 0x08, 0x2dd: 0x08, 0x2de: 0x08, 0x2df: 0x08, + 0x2e0: 0x08, 0x2e1: 0x08, 0x2e2: 0x08, 0x2e3: 0x08, 0x2e4: 0x08, 0x2e5: 0x08, 0x2e6: 0x08, 0x2e7: 0x08, + 0x2e8: 0x08, 0x2e9: 0x08, 0x2ea: 0x08, 0x2eb: 0x08, 0x2ec: 0x08, 0x2ed: 0x08, 0x2ee: 0x08, 0x2ef: 0x08, + 0x2f0: 0x08, 0x2f1: 0x08, 0x2f2: 0x08, 0x2f3: 0x08, 0x2f4: 0x08, 0x2f5: 0x08, 0x2f6: 0x08, 0x2f7: 0x08, + 0x2f8: 0x08, 0x2f9: 0x08, 0x2fa: 0x08, 0x2fb: 0x08, 0x2fc: 0x08, 0x2fd: 0x08, 0x2fe: 0x08, 0x2ff: 0x08, + // Block 0xc, offset 0x300 + 0x300: 0x08, 0x301: 0x08, 0x302: 0x08, 0x303: 0x08, 0x304: 0x08, 0x305: 0x08, 0x306: 0x08, 0x307: 0x08, + 0x308: 0x08, 0x309: 0x08, 0x30a: 0x08, 0x30b: 0x08, 0x30c: 0x08, 0x30d: 0x08, 0x30e: 0x08, 0x30f: 0x08, + 0x310: 0x08, 0x311: 0x08, 0x312: 0x08, 0x313: 0x08, 0x314: 0x08, 0x315: 0x08, 0x316: 0x08, 0x317: 0x08, + 0x318: 0x08, 0x319: 0x08, 0x31a: 0x08, 0x31b: 0x08, 0x31c: 0x08, 0x31d: 0x08, 0x31e: 0x08, 0x31f: 0x08, + 0x320: 0x08, 0x321: 0x08, 0x322: 0x08, 0x323: 0x08, 0x324: 0x0e, 0x325: 0x0e, 0x326: 0x0e, 0x327: 0x0e, + 0x328: 0x0e, 0x329: 0x0e, 0x32a: 0x0e, 0x32b: 0x0e, + 0x338: 0x3f, 0x339: 0x40, 0x33c: 0x41, 0x33d: 0x42, 0x33e: 0x43, 0x33f: 0x44, + // Block 0xd, offset 0x340 + 0x37f: 0x45, + // Block 0xe, offset 0x380 + 0x380: 0x0e, 0x381: 0x0e, 0x382: 0x0e, 0x383: 0x0e, 0x384: 0x0e, 0x385: 0x0e, 0x386: 0x0e, 0x387: 0x0e, + 0x388: 0x0e, 0x389: 0x0e, 0x38a: 0x0e, 0x38b: 0x0e, 0x38c: 0x0e, 0x38d: 0x0e, 0x38e: 0x0e, 0x38f: 0x0e, + 0x390: 0x0e, 0x391: 0x0e, 0x392: 0x0e, 0x393: 0x0e, 0x394: 0x0e, 0x395: 0x0e, 0x396: 0x0e, 0x397: 0x0e, + 0x398: 0x0e, 0x399: 0x0e, 0x39a: 0x0e, 0x39b: 0x0e, 0x39c: 0x0e, 0x39d: 0x0e, 0x39e: 0x0e, 0x39f: 0x46, + 0x3a0: 0x0e, 0x3a1: 0x0e, 0x3a2: 0x0e, 0x3a3: 0x0e, 0x3a4: 0x0e, 0x3a5: 0x0e, 0x3a6: 0x0e, 0x3a7: 0x0e, + 0x3a8: 0x0e, 0x3a9: 0x0e, 0x3aa: 0x0e, 0x3ab: 0x47, + // Block 0xf, offset 0x3c0 + 0x3c0: 0x0e, 0x3c1: 0x0e, 0x3c2: 0x0e, 0x3c3: 0x0e, 0x3c4: 0x48, 0x3c5: 0x49, 0x3c6: 0x0e, 0x3c7: 0x0e, + 0x3c8: 0x0e, 0x3c9: 0x0e, 0x3ca: 0x0e, 0x3cb: 0x4a, + // Block 0x10, offset 0x400 + 0x400: 0x4b, 0x403: 0x4c, 0x404: 0x4d, 0x405: 0x4e, 0x406: 0x4f, + 0x408: 0x50, 0x409: 0x51, 0x40c: 0x52, 0x40d: 0x53, 0x40e: 0x54, 0x40f: 0x55, + 0x410: 0x3a, 0x411: 0x56, 0x412: 0x0e, 0x413: 0x57, 0x414: 0x58, 0x415: 0x59, 0x416: 0x5a, 0x417: 0x5b, + 0x418: 0x0e, 0x419: 0x5c, 0x41a: 0x0e, 0x41b: 0x5d, 0x41f: 0x5e, + 0x424: 0x5f, 0x425: 0x60, 0x426: 0x61, 0x427: 0x62, + 0x429: 0x63, 0x42a: 0x64, + // Block 0x11, offset 0x440 + 0x456: 0x0b, 0x457: 0x06, + 0x458: 0x0c, 0x45b: 0x0d, 0x45f: 0x0e, + 0x460: 0x06, 0x461: 0x06, 0x462: 0x06, 0x463: 0x06, 0x464: 0x06, 0x465: 0x06, 0x466: 0x06, 0x467: 0x06, + 0x468: 0x06, 0x469: 0x06, 0x46a: 0x06, 0x46b: 0x06, 0x46c: 0x06, 0x46d: 0x06, 0x46e: 0x06, 0x46f: 0x06, + 0x470: 0x06, 0x471: 0x06, 0x472: 0x06, 0x473: 0x06, 0x474: 0x06, 0x475: 0x06, 0x476: 0x06, 0x477: 0x06, + 0x478: 0x06, 0x479: 0x06, 0x47a: 0x06, 0x47b: 0x06, 0x47c: 0x06, 0x47d: 0x06, 0x47e: 0x06, 0x47f: 0x06, + // Block 0x12, offset 0x480 + 0x484: 0x08, 0x485: 0x08, 0x486: 0x08, 0x487: 0x09, + // Block 0x13, offset 0x4c0 + 0x4c0: 0x08, 0x4c1: 0x08, 0x4c2: 0x08, 0x4c3: 0x08, 0x4c4: 0x08, 0x4c5: 0x08, 0x4c6: 0x08, 0x4c7: 0x08, + 0x4c8: 0x08, 0x4c9: 0x08, 0x4ca: 0x08, 0x4cb: 0x08, 0x4cc: 0x08, 0x4cd: 0x08, 0x4ce: 0x08, 0x4cf: 0x08, + 0x4d0: 0x08, 0x4d1: 0x08, 0x4d2: 0x08, 0x4d3: 0x08, 0x4d4: 0x08, 0x4d5: 0x08, 0x4d6: 0x08, 0x4d7: 0x08, + 0x4d8: 0x08, 0x4d9: 0x08, 0x4da: 0x08, 0x4db: 0x08, 0x4dc: 0x08, 0x4dd: 0x08, 0x4de: 0x08, 0x4df: 0x08, + 0x4e0: 0x08, 0x4e1: 0x08, 0x4e2: 0x08, 0x4e3: 0x08, 0x4e4: 0x08, 0x4e5: 0x08, 0x4e6: 0x08, 0x4e7: 0x08, + 0x4e8: 0x08, 0x4e9: 0x08, 0x4ea: 0x08, 0x4eb: 0x08, 0x4ec: 0x08, 0x4ed: 0x08, 0x4ee: 0x08, 0x4ef: 0x08, + 0x4f0: 0x08, 0x4f1: 0x08, 0x4f2: 0x08, 0x4f3: 0x08, 0x4f4: 0x08, 0x4f5: 0x08, 0x4f6: 0x08, 0x4f7: 0x08, + 0x4f8: 0x08, 0x4f9: 0x08, 0x4fa: 0x08, 0x4fb: 0x08, 0x4fc: 0x08, 0x4fd: 0x08, 0x4fe: 0x08, 0x4ff: 0x65, + // Block 0x14, offset 0x500 + 0x520: 0x10, + 0x530: 0x09, 0x531: 0x09, 0x532: 0x09, 0x533: 0x09, 0x534: 0x09, 0x535: 0x09, 0x536: 0x09, 0x537: 0x09, + 0x538: 0x09, 0x539: 0x09, 0x53a: 0x09, 0x53b: 0x09, 0x53c: 0x09, 0x53d: 0x09, 0x53e: 0x09, 0x53f: 0x11, + // Block 0x15, offset 0x540 + 0x540: 0x09, 0x541: 0x09, 0x542: 0x09, 0x543: 0x09, 0x544: 0x09, 0x545: 0x09, 0x546: 0x09, 0x547: 0x09, + 0x548: 0x09, 0x549: 0x09, 0x54a: 0x09, 0x54b: 0x09, 0x54c: 0x09, 0x54d: 0x09, 0x54e: 0x09, 0x54f: 0x11, +} + +// inverseData contains 4-byte entries of the following format: +// +// <0 padding> +// +// The last byte of the UTF-8-encoded rune is xor-ed with the last byte of the +// UTF-8 encoding of the original rune. Mappings often have the following +// pattern: +// +// Ａ -> A (U+FF21 -> U+0041) +// Ｂ -> B (U+FF22 -> U+0042) +// ... +// +// By xor-ing the last byte the same entry can be shared by many mappings. This +// reduces the total number of distinct entries by about two thirds. +// The resulting entry for the aforementioned mappings is +// +// { 0x01, 0xE0, 0x00, 0x00 } +// +// Using this entry to map U+FF21 (UTF-8 [EF BC A1]), we get +// +// E0 ^ A1 = 41. +// +// Similarly, for U+FF22 (UTF-8 [EF BC A2]), we get +// +// E0 ^ A2 = 42. +// +// Note that because of the xor-ing, the byte sequence stored in the entry is +// not valid UTF-8. +var inverseData = [150][4]byte{ + {0x00, 0x00, 0x00, 0x00}, + {0x03, 0xe3, 0x80, 0xa0}, + {0x03, 0xef, 0xbc, 0xa0}, + {0x03, 0xef, 0xbc, 0xe0}, + {0x03, 0xef, 0xbd, 0xe0}, + {0x03, 0xef, 0xbf, 0x02}, + {0x03, 0xef, 0xbf, 0x00}, + {0x03, 0xef, 0xbf, 0x0e}, + {0x03, 0xef, 0xbf, 0x0c}, + {0x03, 0xef, 0xbf, 0x0f}, + {0x03, 0xef, 0xbf, 0x39}, + {0x03, 0xef, 0xbf, 0x3b}, + {0x03, 0xef, 0xbf, 0x3f}, + {0x03, 0xef, 0xbf, 0x2a}, + {0x03, 0xef, 0xbf, 0x0d}, + {0x03, 0xef, 0xbf, 0x25}, + {0x03, 0xef, 0xbd, 0x1a}, + {0x03, 0xef, 0xbd, 0x26}, + {0x01, 0xa0, 0x00, 0x00}, + {0x03, 0xef, 0xbd, 0x25}, + {0x03, 0xef, 0xbd, 0x23}, + {0x03, 0xef, 0xbd, 0x2e}, + {0x03, 0xef, 0xbe, 0x07}, + {0x03, 0xef, 0xbe, 0x05}, + {0x03, 0xef, 0xbd, 0x06}, + {0x03, 0xef, 0xbd, 0x13}, + {0x03, 0xef, 0xbd, 0x0b}, + {0x03, 0xef, 0xbd, 0x16}, + {0x03, 0xef, 0xbd, 0x0c}, + {0x03, 0xef, 0xbd, 0x15}, + {0x03, 0xef, 0xbd, 0x0d}, + {0x03, 0xef, 0xbd, 0x1c}, + {0x03, 0xef, 0xbd, 0x02}, + {0x03, 0xef, 0xbd, 0x1f}, + {0x03, 0xef, 0xbd, 0x1d}, + {0x03, 0xef, 0xbd, 0x17}, + {0x03, 0xef, 0xbd, 0x08}, + {0x03, 0xef, 0xbd, 0x09}, + {0x03, 0xef, 0xbd, 0x0e}, + {0x03, 0xef, 0xbd, 0x04}, + {0x03, 0xef, 0xbd, 0x05}, + {0x03, 0xef, 0xbe, 0x3f}, + {0x03, 0xef, 0xbe, 0x00}, + {0x03, 0xef, 0xbd, 0x2c}, + {0x03, 0xef, 0xbe, 0x06}, + {0x03, 0xef, 0xbe, 0x0c}, + {0x03, 0xef, 0xbe, 0x0f}, + {0x03, 0xef, 0xbe, 0x0d}, + {0x03, 0xef, 0xbe, 0x0b}, + {0x03, 0xef, 0xbe, 0x19}, + {0x03, 0xef, 0xbe, 0x15}, + {0x03, 0xef, 0xbe, 0x11}, + {0x03, 0xef, 0xbe, 0x31}, + {0x03, 0xef, 0xbe, 0x33}, + {0x03, 0xef, 0xbd, 0x0f}, + {0x03, 0xef, 0xbe, 0x30}, + {0x03, 0xef, 0xbe, 0x3e}, + {0x03, 0xef, 0xbe, 0x32}, + {0x03, 0xef, 0xbe, 0x36}, + {0x03, 0xef, 0xbd, 0x14}, + {0x03, 0xef, 0xbe, 0x2e}, + {0x03, 0xef, 0xbd, 0x1e}, + {0x03, 0xef, 0xbe, 0x10}, + {0x03, 0xef, 0xbf, 0x13}, + {0x03, 0xef, 0xbf, 0x15}, + {0x03, 0xef, 0xbf, 0x17}, + {0x03, 0xef, 0xbf, 0x1f}, + {0x03, 0xef, 0xbf, 0x1d}, + {0x03, 0xef, 0xbf, 0x1b}, + {0x03, 0xef, 0xbf, 0x09}, + {0x03, 0xef, 0xbf, 0x0b}, + {0x03, 0xef, 0xbf, 0x37}, + {0x03, 0xef, 0xbe, 0x04}, + {0x01, 0xe0, 0x00, 0x00}, + {0x03, 0xe2, 0xa6, 0x1a}, + {0x03, 0xe2, 0xa6, 0x26}, + {0x03, 0xe3, 0x80, 0x23}, + {0x03, 0xe3, 0x80, 0x2e}, + {0x03, 0xe3, 0x80, 0x25}, + {0x03, 0xe3, 0x83, 0x1e}, + {0x03, 0xe3, 0x83, 0x14}, + {0x03, 0xe3, 0x82, 0x06}, + {0x03, 0xe3, 0x82, 0x0b}, + {0x03, 0xe3, 0x82, 0x0c}, + {0x03, 0xe3, 0x82, 0x0d}, + {0x03, 0xe3, 0x82, 0x02}, + {0x03, 0xe3, 0x83, 0x0f}, + {0x03, 0xe3, 0x83, 0x08}, + {0x03, 0xe3, 0x83, 0x09}, + {0x03, 0xe3, 0x83, 0x2c}, + {0x03, 0xe3, 0x83, 0x0c}, + {0x03, 0xe3, 0x82, 0x13}, + {0x03, 0xe3, 0x82, 0x16}, + {0x03, 0xe3, 0x82, 0x15}, + {0x03, 0xe3, 0x82, 0x1c}, + {0x03, 0xe3, 0x82, 0x1f}, + {0x03, 0xe3, 0x82, 0x1d}, + {0x03, 0xe3, 0x82, 0x1a}, + {0x03, 0xe3, 0x82, 0x17}, + {0x03, 0xe3, 0x82, 0x08}, + {0x03, 0xe3, 0x82, 0x09}, + {0x03, 0xe3, 0x82, 0x0e}, + {0x03, 0xe3, 0x82, 0x04}, + {0x03, 0xe3, 0x82, 0x05}, + {0x03, 0xe3, 0x82, 0x3f}, + {0x03, 0xe3, 0x83, 0x00}, + {0x03, 0xe3, 0x83, 0x06}, + {0x03, 0xe3, 0x83, 0x05}, + {0x03, 0xe3, 0x83, 0x0d}, + {0x03, 0xe3, 0x83, 0x0b}, + {0x03, 0xe3, 0x83, 0x07}, + {0x03, 0xe3, 0x83, 0x19}, + {0x03, 0xe3, 0x83, 0x15}, + {0x03, 0xe3, 0x83, 0x11}, + {0x03, 0xe3, 0x83, 0x31}, + {0x03, 0xe3, 0x83, 0x33}, + {0x03, 0xe3, 0x83, 0x30}, + {0x03, 0xe3, 0x83, 0x3e}, + {0x03, 0xe3, 0x83, 0x32}, + {0x03, 0xe3, 0x83, 0x36}, + {0x03, 0xe3, 0x83, 0x2e}, + {0x03, 0xe3, 0x82, 0x07}, + {0x03, 0xe3, 0x85, 0x04}, + {0x03, 0xe3, 0x84, 0x10}, + {0x03, 0xe3, 0x85, 0x30}, + {0x03, 0xe3, 0x85, 0x0d}, + {0x03, 0xe3, 0x85, 0x13}, + {0x03, 0xe3, 0x85, 0x15}, + {0x03, 0xe3, 0x85, 0x17}, + {0x03, 0xe3, 0x85, 0x1f}, + {0x03, 0xe3, 0x85, 0x1d}, + {0x03, 0xe3, 0x85, 0x1b}, + {0x03, 0xe3, 0x85, 0x09}, + {0x03, 0xe3, 0x85, 0x0f}, + {0x03, 0xe3, 0x85, 0x0b}, + {0x03, 0xe3, 0x85, 0x37}, + {0x03, 0xe3, 0x85, 0x3b}, + {0x03, 0xe3, 0x85, 0x39}, + {0x03, 0xe3, 0x85, 0x3f}, + {0x02, 0xc2, 0x02, 0x00}, + {0x02, 0xc2, 0x0e, 0x00}, + {0x02, 0xc2, 0x0c, 0x00}, + {0x02, 0xc2, 0x00, 0x00}, + {0x03, 0xe2, 0x82, 0x0f}, + {0x03, 0xe2, 0x94, 0x2a}, + {0x03, 0xe2, 0x86, 0x39}, + {0x03, 0xe2, 0x86, 0x3b}, + {0x03, 0xe2, 0x86, 0x3f}, + {0x03, 0xe2, 0x96, 0x0d}, + {0x03, 0xe2, 0x97, 0x25}, +} + +// Total table size 15320 bytes (14KiB) diff --git a/vendor/golang.org/x/text/width/tables13.0.0.go b/vendor/golang.org/x/text/width/tables13.0.0.go new file mode 100644 index 0000000000..b1fcb522cb --- /dev/null +++ b/vendor/golang.org/x/text/width/tables13.0.0.go @@ -0,0 +1,1362 @@ +// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. + +//go:build go1.16 && !go1.21 +// +build go1.16,!go1.21 + +package width + +// UnicodeVersion is the Unicode version from which the tables in this package are derived. +const UnicodeVersion = "13.0.0" + +// lookup returns the trie value for the first UTF-8 encoding in s and +// the width in bytes of this encoding. The size will be 0 if s does not +// hold enough bytes to complete the encoding. len(s) must be greater than 0. +func (t *widthTrie) lookup(s []byte) (v uint16, sz int) { + c0 := s[0] + switch { + case c0 < 0x80: // is ASCII + return widthValues[c0], 1 + case c0 < 0xC2: + return 0, 1 // Illegal UTF-8: not a starter, not ASCII. + case c0 < 0xE0: // 2-byte UTF-8 + if len(s) < 2 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c1), 2 + case c0 < 0xF0: // 3-byte UTF-8 + if len(s) < 3 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = widthIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c2), 3 + case c0 < 0xF8: // 4-byte UTF-8 + if len(s) < 4 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = widthIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + o = uint32(i)<<6 + uint32(c2) + i = widthIndex[o] + c3 := s[3] + if c3 < 0x80 || 0xC0 <= c3 { + return 0, 3 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c3), 4 + } + // Illegal rune + return 0, 1 +} + +// lookupUnsafe returns the trie value for the first UTF-8 encoding in s. +// s must start with a full and valid UTF-8 encoded rune. +func (t *widthTrie) lookupUnsafe(s []byte) uint16 { + c0 := s[0] + if c0 < 0x80 { // is ASCII + return widthValues[c0] + } + i := widthIndex[c0] + if c0 < 0xE0 { // 2-byte UTF-8 + return t.lookupValue(uint32(i), s[1]) + } + i = widthIndex[uint32(i)<<6+uint32(s[1])] + if c0 < 0xF0 { // 3-byte UTF-8 + return t.lookupValue(uint32(i), s[2]) + } + i = widthIndex[uint32(i)<<6+uint32(s[2])] + if c0 < 0xF8 { // 4-byte UTF-8 + return t.lookupValue(uint32(i), s[3]) + } + return 0 +} + +// lookupString returns the trie value for the first UTF-8 encoding in s and +// the width in bytes of this encoding. The size will be 0 if s does not +// hold enough bytes to complete the encoding. len(s) must be greater than 0. +func (t *widthTrie) lookupString(s string) (v uint16, sz int) { + c0 := s[0] + switch { + case c0 < 0x80: // is ASCII + return widthValues[c0], 1 + case c0 < 0xC2: + return 0, 1 // Illegal UTF-8: not a starter, not ASCII. + case c0 < 0xE0: // 2-byte UTF-8 + if len(s) < 2 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c1), 2 + case c0 < 0xF0: // 3-byte UTF-8 + if len(s) < 3 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = widthIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c2), 3 + case c0 < 0xF8: // 4-byte UTF-8 + if len(s) < 4 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = widthIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + o = uint32(i)<<6 + uint32(c2) + i = widthIndex[o] + c3 := s[3] + if c3 < 0x80 || 0xC0 <= c3 { + return 0, 3 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c3), 4 + } + // Illegal rune + return 0, 1 +} + +// lookupStringUnsafe returns the trie value for the first UTF-8 encoding in s. +// s must start with a full and valid UTF-8 encoded rune. +func (t *widthTrie) lookupStringUnsafe(s string) uint16 { + c0 := s[0] + if c0 < 0x80 { // is ASCII + return widthValues[c0] + } + i := widthIndex[c0] + if c0 < 0xE0 { // 2-byte UTF-8 + return t.lookupValue(uint32(i), s[1]) + } + i = widthIndex[uint32(i)<<6+uint32(s[1])] + if c0 < 0xF0 { // 3-byte UTF-8 + return t.lookupValue(uint32(i), s[2]) + } + i = widthIndex[uint32(i)<<6+uint32(s[2])] + if c0 < 0xF8 { // 4-byte UTF-8 + return t.lookupValue(uint32(i), s[3]) + } + return 0 +} + +// widthTrie. Total size: 14848 bytes (14.50 KiB). Checksum: 17e24343536472f6. +type widthTrie struct{} + +func newWidthTrie(i int) *widthTrie { + return &widthTrie{} +} + +// lookupValue determines the type of block n and looks up the value for b. +func (t *widthTrie) lookupValue(n uint32, b byte) uint16 { + switch { + default: + return uint16(widthValues[n<<6+uint32(b)]) + } +} + +// widthValues: 105 blocks, 6720 entries, 13440 bytes +// The third block is the zero block. +var widthValues = [6720]uint16{ + // Block 0x0, offset 0x0 + 0x20: 0x6001, 0x21: 0x6002, 0x22: 0x6002, 0x23: 0x6002, + 0x24: 0x6002, 0x25: 0x6002, 0x26: 0x6002, 0x27: 0x6002, 0x28: 0x6002, 0x29: 0x6002, + 0x2a: 0x6002, 0x2b: 0x6002, 0x2c: 0x6002, 0x2d: 0x6002, 0x2e: 0x6002, 0x2f: 0x6002, + 0x30: 0x6002, 0x31: 0x6002, 0x32: 0x6002, 0x33: 0x6002, 0x34: 0x6002, 0x35: 0x6002, + 0x36: 0x6002, 0x37: 0x6002, 0x38: 0x6002, 0x39: 0x6002, 0x3a: 0x6002, 0x3b: 0x6002, + 0x3c: 0x6002, 0x3d: 0x6002, 0x3e: 0x6002, 0x3f: 0x6002, + // Block 0x1, offset 0x40 + 0x40: 0x6003, 0x41: 0x6003, 0x42: 0x6003, 0x43: 0x6003, 0x44: 0x6003, 0x45: 0x6003, + 0x46: 0x6003, 0x47: 0x6003, 0x48: 0x6003, 0x49: 0x6003, 0x4a: 0x6003, 0x4b: 0x6003, + 0x4c: 0x6003, 0x4d: 0x6003, 0x4e: 0x6003, 0x4f: 0x6003, 0x50: 0x6003, 0x51: 0x6003, + 0x52: 0x6003, 0x53: 0x6003, 0x54: 0x6003, 0x55: 0x6003, 0x56: 0x6003, 0x57: 0x6003, + 0x58: 0x6003, 0x59: 0x6003, 0x5a: 0x6003, 0x5b: 0x6003, 0x5c: 0x6003, 0x5d: 0x6003, + 0x5e: 0x6003, 0x5f: 0x6003, 0x60: 0x6004, 0x61: 0x6004, 0x62: 0x6004, 0x63: 0x6004, + 0x64: 0x6004, 0x65: 0x6004, 0x66: 0x6004, 0x67: 0x6004, 0x68: 0x6004, 0x69: 0x6004, + 0x6a: 0x6004, 0x6b: 0x6004, 0x6c: 0x6004, 0x6d: 0x6004, 0x6e: 0x6004, 0x6f: 0x6004, + 0x70: 0x6004, 0x71: 0x6004, 0x72: 0x6004, 0x73: 0x6004, 0x74: 0x6004, 0x75: 0x6004, + 0x76: 0x6004, 0x77: 0x6004, 0x78: 0x6004, 0x79: 0x6004, 0x7a: 0x6004, 0x7b: 0x6004, + 0x7c: 0x6004, 0x7d: 0x6004, 0x7e: 0x6004, + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0xe1: 0x2000, 0xe2: 0x6005, 0xe3: 0x6005, + 0xe4: 0x2000, 0xe5: 0x6006, 0xe6: 0x6005, 0xe7: 0x2000, 0xe8: 0x2000, + 0xea: 0x2000, 0xec: 0x6007, 0xed: 0x2000, 0xee: 0x2000, 0xef: 0x6008, + 0xf0: 0x2000, 0xf1: 0x2000, 0xf2: 0x2000, 0xf3: 0x2000, 0xf4: 0x2000, + 0xf6: 0x2000, 0xf7: 0x2000, 0xf8: 0x2000, 0xf9: 0x2000, 0xfa: 0x2000, + 0xfc: 0x2000, 0xfd: 0x2000, 0xfe: 0x2000, 0xff: 0x2000, + // Block 0x4, offset 0x100 + 0x106: 0x2000, + 0x110: 0x2000, + 0x117: 0x2000, + 0x118: 0x2000, + 0x11e: 0x2000, 0x11f: 0x2000, 0x120: 0x2000, 0x121: 0x2000, + 0x126: 0x2000, 0x128: 0x2000, 0x129: 0x2000, + 0x12a: 0x2000, 0x12c: 0x2000, 0x12d: 0x2000, + 0x130: 0x2000, 0x132: 0x2000, 0x133: 0x2000, + 0x137: 0x2000, 0x138: 0x2000, 0x139: 0x2000, 0x13a: 0x2000, + 0x13c: 0x2000, 0x13e: 0x2000, + // Block 0x5, offset 0x140 + 0x141: 0x2000, + 0x151: 0x2000, + 0x153: 0x2000, + 0x15b: 0x2000, + 0x166: 0x2000, 0x167: 0x2000, + 0x16b: 0x2000, + 0x171: 0x2000, 0x172: 0x2000, 0x173: 0x2000, + 0x178: 0x2000, + 0x17f: 0x2000, + // Block 0x6, offset 0x180 + 0x180: 0x2000, 0x181: 0x2000, 0x182: 0x2000, 0x184: 0x2000, + 0x188: 0x2000, 0x189: 0x2000, 0x18a: 0x2000, 0x18b: 0x2000, + 0x18d: 0x2000, + 0x192: 0x2000, 0x193: 0x2000, + 0x1a6: 0x2000, 0x1a7: 0x2000, + 0x1ab: 0x2000, + // Block 0x7, offset 0x1c0 + 0x1ce: 0x2000, 0x1d0: 0x2000, + 0x1d2: 0x2000, 0x1d4: 0x2000, 0x1d6: 0x2000, + 0x1d8: 0x2000, 0x1da: 0x2000, 0x1dc: 0x2000, + // Block 0x8, offset 0x200 + 0x211: 0x2000, + 0x221: 0x2000, + // Block 0x9, offset 0x240 + 0x244: 0x2000, + 0x247: 0x2000, 0x249: 0x2000, 0x24a: 0x2000, 0x24b: 0x2000, + 0x24d: 0x2000, 0x250: 0x2000, + 0x258: 0x2000, 0x259: 0x2000, 0x25a: 0x2000, 0x25b: 0x2000, 0x25d: 0x2000, + 0x25f: 0x2000, + // Block 0xa, offset 0x280 + 0x280: 0x2000, 0x281: 0x2000, 0x282: 0x2000, 0x283: 0x2000, 0x284: 0x2000, 0x285: 0x2000, + 0x286: 0x2000, 0x287: 0x2000, 0x288: 0x2000, 0x289: 0x2000, 0x28a: 0x2000, 0x28b: 0x2000, + 0x28c: 0x2000, 0x28d: 0x2000, 0x28e: 0x2000, 0x28f: 0x2000, 0x290: 0x2000, 0x291: 0x2000, + 0x292: 0x2000, 0x293: 0x2000, 0x294: 0x2000, 0x295: 0x2000, 0x296: 0x2000, 0x297: 0x2000, + 0x298: 0x2000, 0x299: 0x2000, 0x29a: 0x2000, 0x29b: 0x2000, 0x29c: 0x2000, 0x29d: 0x2000, + 0x29e: 0x2000, 0x29f: 0x2000, 0x2a0: 0x2000, 0x2a1: 0x2000, 0x2a2: 0x2000, 0x2a3: 0x2000, + 0x2a4: 0x2000, 0x2a5: 0x2000, 0x2a6: 0x2000, 0x2a7: 0x2000, 0x2a8: 0x2000, 0x2a9: 0x2000, + 0x2aa: 0x2000, 0x2ab: 0x2000, 0x2ac: 0x2000, 0x2ad: 0x2000, 0x2ae: 0x2000, 0x2af: 0x2000, + 0x2b0: 0x2000, 0x2b1: 0x2000, 0x2b2: 0x2000, 0x2b3: 0x2000, 0x2b4: 0x2000, 0x2b5: 0x2000, + 0x2b6: 0x2000, 0x2b7: 0x2000, 0x2b8: 0x2000, 0x2b9: 0x2000, 0x2ba: 0x2000, 0x2bb: 0x2000, + 0x2bc: 0x2000, 0x2bd: 0x2000, 0x2be: 0x2000, 0x2bf: 0x2000, + // Block 0xb, offset 0x2c0 + 0x2c0: 0x2000, 0x2c1: 0x2000, 0x2c2: 0x2000, 0x2c3: 0x2000, 0x2c4: 0x2000, 0x2c5: 0x2000, + 0x2c6: 0x2000, 0x2c7: 0x2000, 0x2c8: 0x2000, 0x2c9: 0x2000, 0x2ca: 0x2000, 0x2cb: 0x2000, + 0x2cc: 0x2000, 0x2cd: 0x2000, 0x2ce: 0x2000, 0x2cf: 0x2000, 0x2d0: 0x2000, 0x2d1: 0x2000, + 0x2d2: 0x2000, 0x2d3: 0x2000, 0x2d4: 0x2000, 0x2d5: 0x2000, 0x2d6: 0x2000, 0x2d7: 0x2000, + 0x2d8: 0x2000, 0x2d9: 0x2000, 0x2da: 0x2000, 0x2db: 0x2000, 0x2dc: 0x2000, 0x2dd: 0x2000, + 0x2de: 0x2000, 0x2df: 0x2000, 0x2e0: 0x2000, 0x2e1: 0x2000, 0x2e2: 0x2000, 0x2e3: 0x2000, + 0x2e4: 0x2000, 0x2e5: 0x2000, 0x2e6: 0x2000, 0x2e7: 0x2000, 0x2e8: 0x2000, 0x2e9: 0x2000, + 0x2ea: 0x2000, 0x2eb: 0x2000, 0x2ec: 0x2000, 0x2ed: 0x2000, 0x2ee: 0x2000, 0x2ef: 0x2000, + // Block 0xc, offset 0x300 + 0x311: 0x2000, + 0x312: 0x2000, 0x313: 0x2000, 0x314: 0x2000, 0x315: 0x2000, 0x316: 0x2000, 0x317: 0x2000, + 0x318: 0x2000, 0x319: 0x2000, 0x31a: 0x2000, 0x31b: 0x2000, 0x31c: 0x2000, 0x31d: 0x2000, + 0x31e: 0x2000, 0x31f: 0x2000, 0x320: 0x2000, 0x321: 0x2000, 0x323: 0x2000, + 0x324: 0x2000, 0x325: 0x2000, 0x326: 0x2000, 0x327: 0x2000, 0x328: 0x2000, 0x329: 0x2000, + 0x331: 0x2000, 0x332: 0x2000, 0x333: 0x2000, 0x334: 0x2000, 0x335: 0x2000, + 0x336: 0x2000, 0x337: 0x2000, 0x338: 0x2000, 0x339: 0x2000, 0x33a: 0x2000, 0x33b: 0x2000, + 0x33c: 0x2000, 0x33d: 0x2000, 0x33e: 0x2000, 0x33f: 0x2000, + // Block 0xd, offset 0x340 + 0x340: 0x2000, 0x341: 0x2000, 0x343: 0x2000, 0x344: 0x2000, 0x345: 0x2000, + 0x346: 0x2000, 0x347: 0x2000, 0x348: 0x2000, 0x349: 0x2000, + // Block 0xe, offset 0x380 + 0x381: 0x2000, + 0x390: 0x2000, 0x391: 0x2000, + 0x392: 0x2000, 0x393: 0x2000, 0x394: 0x2000, 0x395: 0x2000, 0x396: 0x2000, 0x397: 0x2000, + 0x398: 0x2000, 0x399: 0x2000, 0x39a: 0x2000, 0x39b: 0x2000, 0x39c: 0x2000, 0x39d: 0x2000, + 0x39e: 0x2000, 0x39f: 0x2000, 0x3a0: 0x2000, 0x3a1: 0x2000, 0x3a2: 0x2000, 0x3a3: 0x2000, + 0x3a4: 0x2000, 0x3a5: 0x2000, 0x3a6: 0x2000, 0x3a7: 0x2000, 0x3a8: 0x2000, 0x3a9: 0x2000, + 0x3aa: 0x2000, 0x3ab: 0x2000, 0x3ac: 0x2000, 0x3ad: 0x2000, 0x3ae: 0x2000, 0x3af: 0x2000, + 0x3b0: 0x2000, 0x3b1: 0x2000, 0x3b2: 0x2000, 0x3b3: 0x2000, 0x3b4: 0x2000, 0x3b5: 0x2000, + 0x3b6: 0x2000, 0x3b7: 0x2000, 0x3b8: 0x2000, 0x3b9: 0x2000, 0x3ba: 0x2000, 0x3bb: 0x2000, + 0x3bc: 0x2000, 0x3bd: 0x2000, 0x3be: 0x2000, 0x3bf: 0x2000, + // Block 0xf, offset 0x3c0 + 0x3c0: 0x2000, 0x3c1: 0x2000, 0x3c2: 0x2000, 0x3c3: 0x2000, 0x3c4: 0x2000, 0x3c5: 0x2000, + 0x3c6: 0x2000, 0x3c7: 0x2000, 0x3c8: 0x2000, 0x3c9: 0x2000, 0x3ca: 0x2000, 0x3cb: 0x2000, + 0x3cc: 0x2000, 0x3cd: 0x2000, 0x3ce: 0x2000, 0x3cf: 0x2000, 0x3d1: 0x2000, + // Block 0x10, offset 0x400 + 0x400: 0x4000, 0x401: 0x4000, 0x402: 0x4000, 0x403: 0x4000, 0x404: 0x4000, 0x405: 0x4000, + 0x406: 0x4000, 0x407: 0x4000, 0x408: 0x4000, 0x409: 0x4000, 0x40a: 0x4000, 0x40b: 0x4000, + 0x40c: 0x4000, 0x40d: 0x4000, 0x40e: 0x4000, 0x40f: 0x4000, 0x410: 0x4000, 0x411: 0x4000, + 0x412: 0x4000, 0x413: 0x4000, 0x414: 0x4000, 0x415: 0x4000, 0x416: 0x4000, 0x417: 0x4000, + 0x418: 0x4000, 0x419: 0x4000, 0x41a: 0x4000, 0x41b: 0x4000, 0x41c: 0x4000, 0x41d: 0x4000, + 0x41e: 0x4000, 0x41f: 0x4000, 0x420: 0x4000, 0x421: 0x4000, 0x422: 0x4000, 0x423: 0x4000, + 0x424: 0x4000, 0x425: 0x4000, 0x426: 0x4000, 0x427: 0x4000, 0x428: 0x4000, 0x429: 0x4000, + 0x42a: 0x4000, 0x42b: 0x4000, 0x42c: 0x4000, 0x42d: 0x4000, 0x42e: 0x4000, 0x42f: 0x4000, + 0x430: 0x4000, 0x431: 0x4000, 0x432: 0x4000, 0x433: 0x4000, 0x434: 0x4000, 0x435: 0x4000, + 0x436: 0x4000, 0x437: 0x4000, 0x438: 0x4000, 0x439: 0x4000, 0x43a: 0x4000, 0x43b: 0x4000, + 0x43c: 0x4000, 0x43d: 0x4000, 0x43e: 0x4000, 0x43f: 0x4000, + // Block 0x11, offset 0x440 + 0x440: 0x4000, 0x441: 0x4000, 0x442: 0x4000, 0x443: 0x4000, 0x444: 0x4000, 0x445: 0x4000, + 0x446: 0x4000, 0x447: 0x4000, 0x448: 0x4000, 0x449: 0x4000, 0x44a: 0x4000, 0x44b: 0x4000, + 0x44c: 0x4000, 0x44d: 0x4000, 0x44e: 0x4000, 0x44f: 0x4000, 0x450: 0x4000, 0x451: 0x4000, + 0x452: 0x4000, 0x453: 0x4000, 0x454: 0x4000, 0x455: 0x4000, 0x456: 0x4000, 0x457: 0x4000, + 0x458: 0x4000, 0x459: 0x4000, 0x45a: 0x4000, 0x45b: 0x4000, 0x45c: 0x4000, 0x45d: 0x4000, + 0x45e: 0x4000, 0x45f: 0x4000, + // Block 0x12, offset 0x480 + 0x490: 0x2000, + 0x493: 0x2000, 0x494: 0x2000, 0x495: 0x2000, 0x496: 0x2000, + 0x498: 0x2000, 0x499: 0x2000, 0x49c: 0x2000, 0x49d: 0x2000, + 0x4a0: 0x2000, 0x4a1: 0x2000, 0x4a2: 0x2000, + 0x4a4: 0x2000, 0x4a5: 0x2000, 0x4a6: 0x2000, 0x4a7: 0x2000, + 0x4b0: 0x2000, 0x4b2: 0x2000, 0x4b3: 0x2000, 0x4b5: 0x2000, + 0x4bb: 0x2000, + 0x4be: 0x2000, + // Block 0x13, offset 0x4c0 + 0x4f4: 0x2000, + 0x4ff: 0x2000, + // Block 0x14, offset 0x500 + 0x501: 0x2000, 0x502: 0x2000, 0x503: 0x2000, 0x504: 0x2000, + 0x529: 0xa009, + 0x52c: 0x2000, + // Block 0x15, offset 0x540 + 0x543: 0x2000, 0x545: 0x2000, + 0x549: 0x2000, + 0x553: 0x2000, 0x556: 0x2000, + 0x561: 0x2000, 0x562: 0x2000, + 0x566: 0x2000, + 0x56b: 0x2000, + // Block 0x16, offset 0x580 + 0x593: 0x2000, 0x594: 0x2000, + 0x59b: 0x2000, 0x59c: 0x2000, 0x59d: 0x2000, + 0x59e: 0x2000, 0x5a0: 0x2000, 0x5a1: 0x2000, 0x5a2: 0x2000, 0x5a3: 0x2000, + 0x5a4: 0x2000, 0x5a5: 0x2000, 0x5a6: 0x2000, 0x5a7: 0x2000, 0x5a8: 0x2000, 0x5a9: 0x2000, + 0x5aa: 0x2000, 0x5ab: 0x2000, + 0x5b0: 0x2000, 0x5b1: 0x2000, 0x5b2: 0x2000, 0x5b3: 0x2000, 0x5b4: 0x2000, 0x5b5: 0x2000, + 0x5b6: 0x2000, 0x5b7: 0x2000, 0x5b8: 0x2000, 0x5b9: 0x2000, + // Block 0x17, offset 0x5c0 + 0x5c9: 0x2000, + 0x5d0: 0x200a, 0x5d1: 0x200b, + 0x5d2: 0x200a, 0x5d3: 0x200c, 0x5d4: 0x2000, 0x5d5: 0x2000, 0x5d6: 0x2000, 0x5d7: 0x2000, + 0x5d8: 0x2000, 0x5d9: 0x2000, + 0x5f8: 0x2000, 0x5f9: 0x2000, + // Block 0x18, offset 0x600 + 0x612: 0x2000, 0x614: 0x2000, + 0x627: 0x2000, + // Block 0x19, offset 0x640 + 0x640: 0x2000, 0x642: 0x2000, 0x643: 0x2000, + 0x647: 0x2000, 0x648: 0x2000, 0x64b: 0x2000, + 0x64f: 0x2000, 0x651: 0x2000, + 0x655: 0x2000, + 0x65a: 0x2000, 0x65d: 0x2000, + 0x65e: 0x2000, 0x65f: 0x2000, 0x660: 0x2000, 0x663: 0x2000, + 0x665: 0x2000, 0x667: 0x2000, 0x668: 0x2000, 0x669: 0x2000, + 0x66a: 0x2000, 0x66b: 0x2000, 0x66c: 0x2000, 0x66e: 0x2000, + 0x674: 0x2000, 0x675: 0x2000, + 0x676: 0x2000, 0x677: 0x2000, + 0x67c: 0x2000, 0x67d: 0x2000, + // Block 0x1a, offset 0x680 + 0x688: 0x2000, + 0x68c: 0x2000, + 0x692: 0x2000, + 0x6a0: 0x2000, 0x6a1: 0x2000, + 0x6a4: 0x2000, 0x6a5: 0x2000, 0x6a6: 0x2000, 0x6a7: 0x2000, + 0x6aa: 0x2000, 0x6ab: 0x2000, 0x6ae: 0x2000, 0x6af: 0x2000, + // Block 0x1b, offset 0x6c0 + 0x6c2: 0x2000, 0x6c3: 0x2000, + 0x6c6: 0x2000, 0x6c7: 0x2000, + 0x6d5: 0x2000, + 0x6d9: 0x2000, + 0x6e5: 0x2000, + 0x6ff: 0x2000, + // Block 0x1c, offset 0x700 + 0x712: 0x2000, + 0x71a: 0x4000, 0x71b: 0x4000, + 0x729: 0x4000, + 0x72a: 0x4000, + // Block 0x1d, offset 0x740 + 0x769: 0x4000, + 0x76a: 0x4000, 0x76b: 0x4000, 0x76c: 0x4000, + 0x770: 0x4000, 0x773: 0x4000, + // Block 0x1e, offset 0x780 + 0x7a0: 0x2000, 0x7a1: 0x2000, 0x7a2: 0x2000, 0x7a3: 0x2000, + 0x7a4: 0x2000, 0x7a5: 0x2000, 0x7a6: 0x2000, 0x7a7: 0x2000, 0x7a8: 0x2000, 0x7a9: 0x2000, + 0x7aa: 0x2000, 0x7ab: 0x2000, 0x7ac: 0x2000, 0x7ad: 0x2000, 0x7ae: 0x2000, 0x7af: 0x2000, + 0x7b0: 0x2000, 0x7b1: 0x2000, 0x7b2: 0x2000, 0x7b3: 0x2000, 0x7b4: 0x2000, 0x7b5: 0x2000, + 0x7b6: 0x2000, 0x7b7: 0x2000, 0x7b8: 0x2000, 0x7b9: 0x2000, 0x7ba: 0x2000, 0x7bb: 0x2000, + 0x7bc: 0x2000, 0x7bd: 0x2000, 0x7be: 0x2000, 0x7bf: 0x2000, + // Block 0x1f, offset 0x7c0 + 0x7c0: 0x2000, 0x7c1: 0x2000, 0x7c2: 0x2000, 0x7c3: 0x2000, 0x7c4: 0x2000, 0x7c5: 0x2000, + 0x7c6: 0x2000, 0x7c7: 0x2000, 0x7c8: 0x2000, 0x7c9: 0x2000, 0x7ca: 0x2000, 0x7cb: 0x2000, + 0x7cc: 0x2000, 0x7cd: 0x2000, 0x7ce: 0x2000, 0x7cf: 0x2000, 0x7d0: 0x2000, 0x7d1: 0x2000, + 0x7d2: 0x2000, 0x7d3: 0x2000, 0x7d4: 0x2000, 0x7d5: 0x2000, 0x7d6: 0x2000, 0x7d7: 0x2000, + 0x7d8: 0x2000, 0x7d9: 0x2000, 0x7da: 0x2000, 0x7db: 0x2000, 0x7dc: 0x2000, 0x7dd: 0x2000, + 0x7de: 0x2000, 0x7df: 0x2000, 0x7e0: 0x2000, 0x7e1: 0x2000, 0x7e2: 0x2000, 0x7e3: 0x2000, + 0x7e4: 0x2000, 0x7e5: 0x2000, 0x7e6: 0x2000, 0x7e7: 0x2000, 0x7e8: 0x2000, 0x7e9: 0x2000, + 0x7eb: 0x2000, 0x7ec: 0x2000, 0x7ed: 0x2000, 0x7ee: 0x2000, 0x7ef: 0x2000, + 0x7f0: 0x2000, 0x7f1: 0x2000, 0x7f2: 0x2000, 0x7f3: 0x2000, 0x7f4: 0x2000, 0x7f5: 0x2000, + 0x7f6: 0x2000, 0x7f7: 0x2000, 0x7f8: 0x2000, 0x7f9: 0x2000, 0x7fa: 0x2000, 0x7fb: 0x2000, + 0x7fc: 0x2000, 0x7fd: 0x2000, 0x7fe: 0x2000, 0x7ff: 0x2000, + // Block 0x20, offset 0x800 + 0x800: 0x2000, 0x801: 0x2000, 0x802: 0x200d, 0x803: 0x2000, 0x804: 0x2000, 0x805: 0x2000, + 0x806: 0x2000, 0x807: 0x2000, 0x808: 0x2000, 0x809: 0x2000, 0x80a: 0x2000, 0x80b: 0x2000, + 0x80c: 0x2000, 0x80d: 0x2000, 0x80e: 0x2000, 0x80f: 0x2000, 0x810: 0x2000, 0x811: 0x2000, + 0x812: 0x2000, 0x813: 0x2000, 0x814: 0x2000, 0x815: 0x2000, 0x816: 0x2000, 0x817: 0x2000, + 0x818: 0x2000, 0x819: 0x2000, 0x81a: 0x2000, 0x81b: 0x2000, 0x81c: 0x2000, 0x81d: 0x2000, + 0x81e: 0x2000, 0x81f: 0x2000, 0x820: 0x2000, 0x821: 0x2000, 0x822: 0x2000, 0x823: 0x2000, + 0x824: 0x2000, 0x825: 0x2000, 0x826: 0x2000, 0x827: 0x2000, 0x828: 0x2000, 0x829: 0x2000, + 0x82a: 0x2000, 0x82b: 0x2000, 0x82c: 0x2000, 0x82d: 0x2000, 0x82e: 0x2000, 0x82f: 0x2000, + 0x830: 0x2000, 0x831: 0x2000, 0x832: 0x2000, 0x833: 0x2000, 0x834: 0x2000, 0x835: 0x2000, + 0x836: 0x2000, 0x837: 0x2000, 0x838: 0x2000, 0x839: 0x2000, 0x83a: 0x2000, 0x83b: 0x2000, + 0x83c: 0x2000, 0x83d: 0x2000, 0x83e: 0x2000, 0x83f: 0x2000, + // Block 0x21, offset 0x840 + 0x840: 0x2000, 0x841: 0x2000, 0x842: 0x2000, 0x843: 0x2000, 0x844: 0x2000, 0x845: 0x2000, + 0x846: 0x2000, 0x847: 0x2000, 0x848: 0x2000, 0x849: 0x2000, 0x84a: 0x2000, 0x84b: 0x2000, + 0x850: 0x2000, 0x851: 0x2000, + 0x852: 0x2000, 0x853: 0x2000, 0x854: 0x2000, 0x855: 0x2000, 0x856: 0x2000, 0x857: 0x2000, + 0x858: 0x2000, 0x859: 0x2000, 0x85a: 0x2000, 0x85b: 0x2000, 0x85c: 0x2000, 0x85d: 0x2000, + 0x85e: 0x2000, 0x85f: 0x2000, 0x860: 0x2000, 0x861: 0x2000, 0x862: 0x2000, 0x863: 0x2000, + 0x864: 0x2000, 0x865: 0x2000, 0x866: 0x2000, 0x867: 0x2000, 0x868: 0x2000, 0x869: 0x2000, + 0x86a: 0x2000, 0x86b: 0x2000, 0x86c: 0x2000, 0x86d: 0x2000, 0x86e: 0x2000, 0x86f: 0x2000, + 0x870: 0x2000, 0x871: 0x2000, 0x872: 0x2000, 0x873: 0x2000, + // Block 0x22, offset 0x880 + 0x880: 0x2000, 0x881: 0x2000, 0x882: 0x2000, 0x883: 0x2000, 0x884: 0x2000, 0x885: 0x2000, + 0x886: 0x2000, 0x887: 0x2000, 0x888: 0x2000, 0x889: 0x2000, 0x88a: 0x2000, 0x88b: 0x2000, + 0x88c: 0x2000, 0x88d: 0x2000, 0x88e: 0x2000, 0x88f: 0x2000, + 0x892: 0x2000, 0x893: 0x2000, 0x894: 0x2000, 0x895: 0x2000, + 0x8a0: 0x200e, 0x8a1: 0x2000, 0x8a3: 0x2000, + 0x8a4: 0x2000, 0x8a5: 0x2000, 0x8a6: 0x2000, 0x8a7: 0x2000, 0x8a8: 0x2000, 0x8a9: 0x2000, + 0x8b2: 0x2000, 0x8b3: 0x2000, + 0x8b6: 0x2000, 0x8b7: 0x2000, + 0x8bc: 0x2000, 0x8bd: 0x2000, + // Block 0x23, offset 0x8c0 + 0x8c0: 0x2000, 0x8c1: 0x2000, + 0x8c6: 0x2000, 0x8c7: 0x2000, 0x8c8: 0x2000, 0x8cb: 0x200f, + 0x8ce: 0x2000, 0x8cf: 0x2000, 0x8d0: 0x2000, 0x8d1: 0x2000, + 0x8e2: 0x2000, 0x8e3: 0x2000, + 0x8e4: 0x2000, 0x8e5: 0x2000, + 0x8ef: 0x2000, + 0x8fd: 0x4000, 0x8fe: 0x4000, + // Block 0x24, offset 0x900 + 0x905: 0x2000, + 0x906: 0x2000, 0x909: 0x2000, + 0x90e: 0x2000, 0x90f: 0x2000, + 0x914: 0x4000, 0x915: 0x4000, + 0x91c: 0x2000, + 0x91e: 0x2000, + // Block 0x25, offset 0x940 + 0x940: 0x2000, 0x942: 0x2000, + 0x948: 0x4000, 0x949: 0x4000, 0x94a: 0x4000, 0x94b: 0x4000, + 0x94c: 0x4000, 0x94d: 0x4000, 0x94e: 0x4000, 0x94f: 0x4000, 0x950: 0x4000, 0x951: 0x4000, + 0x952: 0x4000, 0x953: 0x4000, + 0x960: 0x2000, 0x961: 0x2000, 0x963: 0x2000, + 0x964: 0x2000, 0x965: 0x2000, 0x967: 0x2000, 0x968: 0x2000, 0x969: 0x2000, + 0x96a: 0x2000, 0x96c: 0x2000, 0x96d: 0x2000, 0x96f: 0x2000, + 0x97f: 0x4000, + // Block 0x26, offset 0x980 + 0x993: 0x4000, + 0x99e: 0x2000, 0x99f: 0x2000, 0x9a1: 0x4000, + 0x9aa: 0x4000, 0x9ab: 0x4000, + 0x9bd: 0x4000, 0x9be: 0x4000, 0x9bf: 0x2000, + // Block 0x27, offset 0x9c0 + 0x9c4: 0x4000, 0x9c5: 0x4000, + 0x9c6: 0x2000, 0x9c7: 0x2000, 0x9c8: 0x2000, 0x9c9: 0x2000, 0x9ca: 0x2000, 0x9cb: 0x2000, + 0x9cc: 0x2000, 0x9cd: 0x2000, 0x9ce: 0x4000, 0x9cf: 0x2000, 0x9d0: 0x2000, 0x9d1: 0x2000, + 0x9d2: 0x2000, 0x9d3: 0x2000, 0x9d4: 0x4000, 0x9d5: 0x2000, 0x9d6: 0x2000, 0x9d7: 0x2000, + 0x9d8: 0x2000, 0x9d9: 0x2000, 0x9da: 0x2000, 0x9db: 0x2000, 0x9dc: 0x2000, 0x9dd: 0x2000, + 0x9de: 0x2000, 0x9df: 0x2000, 0x9e0: 0x2000, 0x9e1: 0x2000, 0x9e3: 0x2000, + 0x9e8: 0x2000, 0x9e9: 0x2000, + 0x9ea: 0x4000, 0x9eb: 0x2000, 0x9ec: 0x2000, 0x9ed: 0x2000, 0x9ee: 0x2000, 0x9ef: 0x2000, + 0x9f0: 0x2000, 0x9f1: 0x2000, 0x9f2: 0x4000, 0x9f3: 0x4000, 0x9f4: 0x2000, 0x9f5: 0x4000, + 0x9f6: 0x2000, 0x9f7: 0x2000, 0x9f8: 0x2000, 0x9f9: 0x2000, 0x9fa: 0x4000, 0x9fb: 0x2000, + 0x9fc: 0x2000, 0x9fd: 0x4000, 0x9fe: 0x2000, 0x9ff: 0x2000, + // Block 0x28, offset 0xa00 + 0xa05: 0x4000, + 0xa0a: 0x4000, 0xa0b: 0x4000, + 0xa28: 0x4000, + 0xa3d: 0x2000, + // Block 0x29, offset 0xa40 + 0xa4c: 0x4000, 0xa4e: 0x4000, + 0xa53: 0x4000, 0xa54: 0x4000, 0xa55: 0x4000, 0xa57: 0x4000, + 0xa76: 0x2000, 0xa77: 0x2000, 0xa78: 0x2000, 0xa79: 0x2000, 0xa7a: 0x2000, 0xa7b: 0x2000, + 0xa7c: 0x2000, 0xa7d: 0x2000, 0xa7e: 0x2000, 0xa7f: 0x2000, + // Block 0x2a, offset 0xa80 + 0xa95: 0x4000, 0xa96: 0x4000, 0xa97: 0x4000, + 0xab0: 0x4000, + 0xabf: 0x4000, + // Block 0x2b, offset 0xac0 + 0xae6: 0x6000, 0xae7: 0x6000, 0xae8: 0x6000, 0xae9: 0x6000, + 0xaea: 0x6000, 0xaeb: 0x6000, 0xaec: 0x6000, 0xaed: 0x6000, + // Block 0x2c, offset 0xb00 + 0xb05: 0x6010, + 0xb06: 0x6011, + // Block 0x2d, offset 0xb40 + 0xb5b: 0x4000, 0xb5c: 0x4000, + // Block 0x2e, offset 0xb80 + 0xb90: 0x4000, + 0xb95: 0x4000, 0xb96: 0x2000, 0xb97: 0x2000, + 0xb98: 0x2000, 0xb99: 0x2000, + // Block 0x2f, offset 0xbc0 + 0xbc0: 0x4000, 0xbc1: 0x4000, 0xbc2: 0x4000, 0xbc3: 0x4000, 0xbc4: 0x4000, 0xbc5: 0x4000, + 0xbc6: 0x4000, 0xbc7: 0x4000, 0xbc8: 0x4000, 0xbc9: 0x4000, 0xbca: 0x4000, 0xbcb: 0x4000, + 0xbcc: 0x4000, 0xbcd: 0x4000, 0xbce: 0x4000, 0xbcf: 0x4000, 0xbd0: 0x4000, 0xbd1: 0x4000, + 0xbd2: 0x4000, 0xbd3: 0x4000, 0xbd4: 0x4000, 0xbd5: 0x4000, 0xbd6: 0x4000, 0xbd7: 0x4000, + 0xbd8: 0x4000, 0xbd9: 0x4000, 0xbdb: 0x4000, 0xbdc: 0x4000, 0xbdd: 0x4000, + 0xbde: 0x4000, 0xbdf: 0x4000, 0xbe0: 0x4000, 0xbe1: 0x4000, 0xbe2: 0x4000, 0xbe3: 0x4000, + 0xbe4: 0x4000, 0xbe5: 0x4000, 0xbe6: 0x4000, 0xbe7: 0x4000, 0xbe8: 0x4000, 0xbe9: 0x4000, + 0xbea: 0x4000, 0xbeb: 0x4000, 0xbec: 0x4000, 0xbed: 0x4000, 0xbee: 0x4000, 0xbef: 0x4000, + 0xbf0: 0x4000, 0xbf1: 0x4000, 0xbf2: 0x4000, 0xbf3: 0x4000, 0xbf4: 0x4000, 0xbf5: 0x4000, + 0xbf6: 0x4000, 0xbf7: 0x4000, 0xbf8: 0x4000, 0xbf9: 0x4000, 0xbfa: 0x4000, 0xbfb: 0x4000, + 0xbfc: 0x4000, 0xbfd: 0x4000, 0xbfe: 0x4000, 0xbff: 0x4000, + // Block 0x30, offset 0xc00 + 0xc00: 0x4000, 0xc01: 0x4000, 0xc02: 0x4000, 0xc03: 0x4000, 0xc04: 0x4000, 0xc05: 0x4000, + 0xc06: 0x4000, 0xc07: 0x4000, 0xc08: 0x4000, 0xc09: 0x4000, 0xc0a: 0x4000, 0xc0b: 0x4000, + 0xc0c: 0x4000, 0xc0d: 0x4000, 0xc0e: 0x4000, 0xc0f: 0x4000, 0xc10: 0x4000, 0xc11: 0x4000, + 0xc12: 0x4000, 0xc13: 0x4000, 0xc14: 0x4000, 0xc15: 0x4000, 0xc16: 0x4000, 0xc17: 0x4000, + 0xc18: 0x4000, 0xc19: 0x4000, 0xc1a: 0x4000, 0xc1b: 0x4000, 0xc1c: 0x4000, 0xc1d: 0x4000, + 0xc1e: 0x4000, 0xc1f: 0x4000, 0xc20: 0x4000, 0xc21: 0x4000, 0xc22: 0x4000, 0xc23: 0x4000, + 0xc24: 0x4000, 0xc25: 0x4000, 0xc26: 0x4000, 0xc27: 0x4000, 0xc28: 0x4000, 0xc29: 0x4000, + 0xc2a: 0x4000, 0xc2b: 0x4000, 0xc2c: 0x4000, 0xc2d: 0x4000, 0xc2e: 0x4000, 0xc2f: 0x4000, + 0xc30: 0x4000, 0xc31: 0x4000, 0xc32: 0x4000, 0xc33: 0x4000, + // Block 0x31, offset 0xc40 + 0xc40: 0x4000, 0xc41: 0x4000, 0xc42: 0x4000, 0xc43: 0x4000, 0xc44: 0x4000, 0xc45: 0x4000, + 0xc46: 0x4000, 0xc47: 0x4000, 0xc48: 0x4000, 0xc49: 0x4000, 0xc4a: 0x4000, 0xc4b: 0x4000, + 0xc4c: 0x4000, 0xc4d: 0x4000, 0xc4e: 0x4000, 0xc4f: 0x4000, 0xc50: 0x4000, 0xc51: 0x4000, + 0xc52: 0x4000, 0xc53: 0x4000, 0xc54: 0x4000, 0xc55: 0x4000, + 0xc70: 0x4000, 0xc71: 0x4000, 0xc72: 0x4000, 0xc73: 0x4000, 0xc74: 0x4000, 0xc75: 0x4000, + 0xc76: 0x4000, 0xc77: 0x4000, 0xc78: 0x4000, 0xc79: 0x4000, 0xc7a: 0x4000, 0xc7b: 0x4000, + // Block 0x32, offset 0xc80 + 0xc80: 0x9012, 0xc81: 0x4013, 0xc82: 0x4014, 0xc83: 0x4000, 0xc84: 0x4000, 0xc85: 0x4000, + 0xc86: 0x4000, 0xc87: 0x4000, 0xc88: 0x4000, 0xc89: 0x4000, 0xc8a: 0x4000, 0xc8b: 0x4000, + 0xc8c: 0x4015, 0xc8d: 0x4015, 0xc8e: 0x4000, 0xc8f: 0x4000, 0xc90: 0x4000, 0xc91: 0x4000, + 0xc92: 0x4000, 0xc93: 0x4000, 0xc94: 0x4000, 0xc95: 0x4000, 0xc96: 0x4000, 0xc97: 0x4000, + 0xc98: 0x4000, 0xc99: 0x4000, 0xc9a: 0x4000, 0xc9b: 0x4000, 0xc9c: 0x4000, 0xc9d: 0x4000, + 0xc9e: 0x4000, 0xc9f: 0x4000, 0xca0: 0x4000, 0xca1: 0x4000, 0xca2: 0x4000, 0xca3: 0x4000, + 0xca4: 0x4000, 0xca5: 0x4000, 0xca6: 0x4000, 0xca7: 0x4000, 0xca8: 0x4000, 0xca9: 0x4000, + 0xcaa: 0x4000, 0xcab: 0x4000, 0xcac: 0x4000, 0xcad: 0x4000, 0xcae: 0x4000, 0xcaf: 0x4000, + 0xcb0: 0x4000, 0xcb1: 0x4000, 0xcb2: 0x4000, 0xcb3: 0x4000, 0xcb4: 0x4000, 0xcb5: 0x4000, + 0xcb6: 0x4000, 0xcb7: 0x4000, 0xcb8: 0x4000, 0xcb9: 0x4000, 0xcba: 0x4000, 0xcbb: 0x4000, + 0xcbc: 0x4000, 0xcbd: 0x4000, 0xcbe: 0x4000, + // Block 0x33, offset 0xcc0 + 0xcc1: 0x4000, 0xcc2: 0x4000, 0xcc3: 0x4000, 0xcc4: 0x4000, 0xcc5: 0x4000, + 0xcc6: 0x4000, 0xcc7: 0x4000, 0xcc8: 0x4000, 0xcc9: 0x4000, 0xcca: 0x4000, 0xccb: 0x4000, + 0xccc: 0x4000, 0xccd: 0x4000, 0xcce: 0x4000, 0xccf: 0x4000, 0xcd0: 0x4000, 0xcd1: 0x4000, + 0xcd2: 0x4000, 0xcd3: 0x4000, 0xcd4: 0x4000, 0xcd5: 0x4000, 0xcd6: 0x4000, 0xcd7: 0x4000, + 0xcd8: 0x4000, 0xcd9: 0x4000, 0xcda: 0x4000, 0xcdb: 0x4000, 0xcdc: 0x4000, 0xcdd: 0x4000, + 0xcde: 0x4000, 0xcdf: 0x4000, 0xce0: 0x4000, 0xce1: 0x4000, 0xce2: 0x4000, 0xce3: 0x4000, + 0xce4: 0x4000, 0xce5: 0x4000, 0xce6: 0x4000, 0xce7: 0x4000, 0xce8: 0x4000, 0xce9: 0x4000, + 0xcea: 0x4000, 0xceb: 0x4000, 0xcec: 0x4000, 0xced: 0x4000, 0xcee: 0x4000, 0xcef: 0x4000, + 0xcf0: 0x4000, 0xcf1: 0x4000, 0xcf2: 0x4000, 0xcf3: 0x4000, 0xcf4: 0x4000, 0xcf5: 0x4000, + 0xcf6: 0x4000, 0xcf7: 0x4000, 0xcf8: 0x4000, 0xcf9: 0x4000, 0xcfa: 0x4000, 0xcfb: 0x4000, + 0xcfc: 0x4000, 0xcfd: 0x4000, 0xcfe: 0x4000, 0xcff: 0x4000, + // Block 0x34, offset 0xd00 + 0xd00: 0x4000, 0xd01: 0x4000, 0xd02: 0x4000, 0xd03: 0x4000, 0xd04: 0x4000, 0xd05: 0x4000, + 0xd06: 0x4000, 0xd07: 0x4000, 0xd08: 0x4000, 0xd09: 0x4000, 0xd0a: 0x4000, 0xd0b: 0x4000, + 0xd0c: 0x4000, 0xd0d: 0x4000, 0xd0e: 0x4000, 0xd0f: 0x4000, 0xd10: 0x4000, 0xd11: 0x4000, + 0xd12: 0x4000, 0xd13: 0x4000, 0xd14: 0x4000, 0xd15: 0x4000, 0xd16: 0x4000, + 0xd19: 0x4016, 0xd1a: 0x4017, 0xd1b: 0x4000, 0xd1c: 0x4000, 0xd1d: 0x4000, + 0xd1e: 0x4000, 0xd1f: 0x4000, 0xd20: 0x4000, 0xd21: 0x4018, 0xd22: 0x4019, 0xd23: 0x401a, + 0xd24: 0x401b, 0xd25: 0x401c, 0xd26: 0x401d, 0xd27: 0x401e, 0xd28: 0x401f, 0xd29: 0x4020, + 0xd2a: 0x4021, 0xd2b: 0x4022, 0xd2c: 0x4000, 0xd2d: 0x4010, 0xd2e: 0x4000, 0xd2f: 0x4023, + 0xd30: 0x4000, 0xd31: 0x4024, 0xd32: 0x4000, 0xd33: 0x4025, 0xd34: 0x4000, 0xd35: 0x4026, + 0xd36: 0x4000, 0xd37: 0x401a, 0xd38: 0x4000, 0xd39: 0x4027, 0xd3a: 0x4000, 0xd3b: 0x4028, + 0xd3c: 0x4000, 0xd3d: 0x4020, 0xd3e: 0x4000, 0xd3f: 0x4029, + // Block 0x35, offset 0xd40 + 0xd40: 0x4000, 0xd41: 0x402a, 0xd42: 0x4000, 0xd43: 0x402b, 0xd44: 0x402c, 0xd45: 0x4000, + 0xd46: 0x4017, 0xd47: 0x4000, 0xd48: 0x402d, 0xd49: 0x4000, 0xd4a: 0x402e, 0xd4b: 0x402f, + 0xd4c: 0x4030, 0xd4d: 0x4017, 0xd4e: 0x4016, 0xd4f: 0x4017, 0xd50: 0x4000, 0xd51: 0x4000, + 0xd52: 0x4031, 0xd53: 0x4000, 0xd54: 0x4000, 0xd55: 0x4031, 0xd56: 0x4000, 0xd57: 0x4000, + 0xd58: 0x4032, 0xd59: 0x4000, 0xd5a: 0x4000, 0xd5b: 0x4032, 0xd5c: 0x4000, 0xd5d: 0x4000, + 0xd5e: 0x4033, 0xd5f: 0x402e, 0xd60: 0x4034, 0xd61: 0x4035, 0xd62: 0x4034, 0xd63: 0x4036, + 0xd64: 0x4037, 0xd65: 0x4024, 0xd66: 0x4035, 0xd67: 0x4025, 0xd68: 0x4038, 0xd69: 0x4038, + 0xd6a: 0x4039, 0xd6b: 0x4039, 0xd6c: 0x403a, 0xd6d: 0x403a, 0xd6e: 0x4000, 0xd6f: 0x4035, + 0xd70: 0x4000, 0xd71: 0x4000, 0xd72: 0x403b, 0xd73: 0x403c, 0xd74: 0x4000, 0xd75: 0x4000, + 0xd76: 0x4000, 0xd77: 0x4000, 0xd78: 0x4000, 0xd79: 0x4000, 0xd7a: 0x4000, 0xd7b: 0x403d, + 0xd7c: 0x401c, 0xd7d: 0x4000, 0xd7e: 0x4000, 0xd7f: 0x4000, + // Block 0x36, offset 0xd80 + 0xd85: 0x4000, + 0xd86: 0x4000, 0xd87: 0x4000, 0xd88: 0x4000, 0xd89: 0x4000, 0xd8a: 0x4000, 0xd8b: 0x4000, + 0xd8c: 0x4000, 0xd8d: 0x4000, 0xd8e: 0x4000, 0xd8f: 0x4000, 0xd90: 0x4000, 0xd91: 0x4000, + 0xd92: 0x4000, 0xd93: 0x4000, 0xd94: 0x4000, 0xd95: 0x4000, 0xd96: 0x4000, 0xd97: 0x4000, + 0xd98: 0x4000, 0xd99: 0x4000, 0xd9a: 0x4000, 0xd9b: 0x4000, 0xd9c: 0x4000, 0xd9d: 0x4000, + 0xd9e: 0x4000, 0xd9f: 0x4000, 0xda0: 0x4000, 0xda1: 0x4000, 0xda2: 0x4000, 0xda3: 0x4000, + 0xda4: 0x4000, 0xda5: 0x4000, 0xda6: 0x4000, 0xda7: 0x4000, 0xda8: 0x4000, 0xda9: 0x4000, + 0xdaa: 0x4000, 0xdab: 0x4000, 0xdac: 0x4000, 0xdad: 0x4000, 0xdae: 0x4000, 0xdaf: 0x4000, + 0xdb1: 0x403e, 0xdb2: 0x403e, 0xdb3: 0x403e, 0xdb4: 0x403e, 0xdb5: 0x403e, + 0xdb6: 0x403e, 0xdb7: 0x403e, 0xdb8: 0x403e, 0xdb9: 0x403e, 0xdba: 0x403e, 0xdbb: 0x403e, + 0xdbc: 0x403e, 0xdbd: 0x403e, 0xdbe: 0x403e, 0xdbf: 0x403e, + // Block 0x37, offset 0xdc0 + 0xdc0: 0x4037, 0xdc1: 0x4037, 0xdc2: 0x4037, 0xdc3: 0x4037, 0xdc4: 0x4037, 0xdc5: 0x4037, + 0xdc6: 0x4037, 0xdc7: 0x4037, 0xdc8: 0x4037, 0xdc9: 0x4037, 0xdca: 0x4037, 0xdcb: 0x4037, + 0xdcc: 0x4037, 0xdcd: 0x4037, 0xdce: 0x4037, 0xdcf: 0x400e, 0xdd0: 0x403f, 0xdd1: 0x4040, + 0xdd2: 0x4041, 0xdd3: 0x4040, 0xdd4: 0x403f, 0xdd5: 0x4042, 0xdd6: 0x4043, 0xdd7: 0x4044, + 0xdd8: 0x4040, 0xdd9: 0x4041, 0xdda: 0x4040, 0xddb: 0x4045, 0xddc: 0x4009, 0xddd: 0x4045, + 0xdde: 0x4046, 0xddf: 0x4045, 0xde0: 0x4047, 0xde1: 0x400b, 0xde2: 0x400a, 0xde3: 0x400c, + 0xde4: 0x4048, 0xde5: 0x4000, 0xde6: 0x4000, 0xde7: 0x4000, 0xde8: 0x4000, 0xde9: 0x4000, + 0xdea: 0x4000, 0xdeb: 0x4000, 0xdec: 0x4000, 0xded: 0x4000, 0xdee: 0x4000, 0xdef: 0x4000, + 0xdf0: 0x4000, 0xdf1: 0x4000, 0xdf2: 0x4000, 0xdf3: 0x4000, 0xdf4: 0x4000, 0xdf5: 0x4000, + 0xdf6: 0x4000, 0xdf7: 0x4000, 0xdf8: 0x4000, 0xdf9: 0x4000, 0xdfa: 0x4000, 0xdfb: 0x4000, + 0xdfc: 0x4000, 0xdfd: 0x4000, 0xdfe: 0x4000, 0xdff: 0x4000, + // Block 0x38, offset 0xe00 + 0xe00: 0x4000, 0xe01: 0x4000, 0xe02: 0x4000, 0xe03: 0x4000, 0xe04: 0x4000, 0xe05: 0x4000, + 0xe06: 0x4000, 0xe07: 0x4000, 0xe08: 0x4000, 0xe09: 0x4000, 0xe0a: 0x4000, 0xe0b: 0x4000, + 0xe0c: 0x4000, 0xe0d: 0x4000, 0xe0e: 0x4000, 0xe10: 0x4000, 0xe11: 0x4000, + 0xe12: 0x4000, 0xe13: 0x4000, 0xe14: 0x4000, 0xe15: 0x4000, 0xe16: 0x4000, 0xe17: 0x4000, + 0xe18: 0x4000, 0xe19: 0x4000, 0xe1a: 0x4000, 0xe1b: 0x4000, 0xe1c: 0x4000, 0xe1d: 0x4000, + 0xe1e: 0x4000, 0xe1f: 0x4000, 0xe20: 0x4000, 0xe21: 0x4000, 0xe22: 0x4000, 0xe23: 0x4000, + 0xe24: 0x4000, 0xe25: 0x4000, 0xe26: 0x4000, 0xe27: 0x4000, 0xe28: 0x4000, 0xe29: 0x4000, + 0xe2a: 0x4000, 0xe2b: 0x4000, 0xe2c: 0x4000, 0xe2d: 0x4000, 0xe2e: 0x4000, 0xe2f: 0x4000, + 0xe30: 0x4000, 0xe31: 0x4000, 0xe32: 0x4000, 0xe33: 0x4000, 0xe34: 0x4000, 0xe35: 0x4000, + 0xe36: 0x4000, 0xe37: 0x4000, 0xe38: 0x4000, 0xe39: 0x4000, 0xe3a: 0x4000, 0xe3b: 0x4000, + 0xe3c: 0x4000, 0xe3d: 0x4000, 0xe3e: 0x4000, 0xe3f: 0x4000, + // Block 0x39, offset 0xe40 + 0xe40: 0x4000, 0xe41: 0x4000, 0xe42: 0x4000, 0xe43: 0x4000, 0xe44: 0x4000, 0xe45: 0x4000, + 0xe46: 0x4000, 0xe47: 0x4000, 0xe48: 0x4000, 0xe49: 0x4000, 0xe4a: 0x4000, 0xe4b: 0x4000, + 0xe4c: 0x4000, 0xe4d: 0x4000, 0xe4e: 0x4000, 0xe4f: 0x4000, 0xe50: 0x4000, 0xe51: 0x4000, + 0xe52: 0x4000, 0xe53: 0x4000, 0xe54: 0x4000, 0xe55: 0x4000, 0xe56: 0x4000, 0xe57: 0x4000, + 0xe58: 0x4000, 0xe59: 0x4000, 0xe5a: 0x4000, 0xe5b: 0x4000, 0xe5c: 0x4000, 0xe5d: 0x4000, + 0xe5e: 0x4000, 0xe5f: 0x4000, 0xe60: 0x4000, 0xe61: 0x4000, 0xe62: 0x4000, 0xe63: 0x4000, + 0xe70: 0x4000, 0xe71: 0x4000, 0xe72: 0x4000, 0xe73: 0x4000, 0xe74: 0x4000, 0xe75: 0x4000, + 0xe76: 0x4000, 0xe77: 0x4000, 0xe78: 0x4000, 0xe79: 0x4000, 0xe7a: 0x4000, 0xe7b: 0x4000, + 0xe7c: 0x4000, 0xe7d: 0x4000, 0xe7e: 0x4000, 0xe7f: 0x4000, + // Block 0x3a, offset 0xe80 + 0xe80: 0x4000, 0xe81: 0x4000, 0xe82: 0x4000, 0xe83: 0x4000, 0xe84: 0x4000, 0xe85: 0x4000, + 0xe86: 0x4000, 0xe87: 0x4000, 0xe88: 0x4000, 0xe89: 0x4000, 0xe8a: 0x4000, 0xe8b: 0x4000, + 0xe8c: 0x4000, 0xe8d: 0x4000, 0xe8e: 0x4000, 0xe8f: 0x4000, 0xe90: 0x4000, 0xe91: 0x4000, + 0xe92: 0x4000, 0xe93: 0x4000, 0xe94: 0x4000, 0xe95: 0x4000, 0xe96: 0x4000, 0xe97: 0x4000, + 0xe98: 0x4000, 0xe99: 0x4000, 0xe9a: 0x4000, 0xe9b: 0x4000, 0xe9c: 0x4000, 0xe9d: 0x4000, + 0xe9e: 0x4000, 0xea0: 0x4000, 0xea1: 0x4000, 0xea2: 0x4000, 0xea3: 0x4000, + 0xea4: 0x4000, 0xea5: 0x4000, 0xea6: 0x4000, 0xea7: 0x4000, 0xea8: 0x4000, 0xea9: 0x4000, + 0xeaa: 0x4000, 0xeab: 0x4000, 0xeac: 0x4000, 0xead: 0x4000, 0xeae: 0x4000, 0xeaf: 0x4000, + 0xeb0: 0x4000, 0xeb1: 0x4000, 0xeb2: 0x4000, 0xeb3: 0x4000, 0xeb4: 0x4000, 0xeb5: 0x4000, + 0xeb6: 0x4000, 0xeb7: 0x4000, 0xeb8: 0x4000, 0xeb9: 0x4000, 0xeba: 0x4000, 0xebb: 0x4000, + 0xebc: 0x4000, 0xebd: 0x4000, 0xebe: 0x4000, 0xebf: 0x4000, + // Block 0x3b, offset 0xec0 + 0xec0: 0x4000, 0xec1: 0x4000, 0xec2: 0x4000, 0xec3: 0x4000, 0xec4: 0x4000, 0xec5: 0x4000, + 0xec6: 0x4000, 0xec7: 0x4000, 0xec8: 0x2000, 0xec9: 0x2000, 0xeca: 0x2000, 0xecb: 0x2000, + 0xecc: 0x2000, 0xecd: 0x2000, 0xece: 0x2000, 0xecf: 0x2000, 0xed0: 0x4000, 0xed1: 0x4000, + 0xed2: 0x4000, 0xed3: 0x4000, 0xed4: 0x4000, 0xed5: 0x4000, 0xed6: 0x4000, 0xed7: 0x4000, + 0xed8: 0x4000, 0xed9: 0x4000, 0xeda: 0x4000, 0xedb: 0x4000, 0xedc: 0x4000, 0xedd: 0x4000, + 0xede: 0x4000, 0xedf: 0x4000, 0xee0: 0x4000, 0xee1: 0x4000, 0xee2: 0x4000, 0xee3: 0x4000, + 0xee4: 0x4000, 0xee5: 0x4000, 0xee6: 0x4000, 0xee7: 0x4000, 0xee8: 0x4000, 0xee9: 0x4000, + 0xeea: 0x4000, 0xeeb: 0x4000, 0xeec: 0x4000, 0xeed: 0x4000, 0xeee: 0x4000, 0xeef: 0x4000, + 0xef0: 0x4000, 0xef1: 0x4000, 0xef2: 0x4000, 0xef3: 0x4000, 0xef4: 0x4000, 0xef5: 0x4000, + 0xef6: 0x4000, 0xef7: 0x4000, 0xef8: 0x4000, 0xef9: 0x4000, 0xefa: 0x4000, 0xefb: 0x4000, + 0xefc: 0x4000, 0xefd: 0x4000, 0xefe: 0x4000, 0xeff: 0x4000, + // Block 0x3c, offset 0xf00 + 0xf00: 0x4000, 0xf01: 0x4000, 0xf02: 0x4000, 0xf03: 0x4000, 0xf04: 0x4000, 0xf05: 0x4000, + 0xf06: 0x4000, 0xf07: 0x4000, 0xf08: 0x4000, 0xf09: 0x4000, 0xf0a: 0x4000, 0xf0b: 0x4000, + 0xf0c: 0x4000, 0xf10: 0x4000, 0xf11: 0x4000, + 0xf12: 0x4000, 0xf13: 0x4000, 0xf14: 0x4000, 0xf15: 0x4000, 0xf16: 0x4000, 0xf17: 0x4000, + 0xf18: 0x4000, 0xf19: 0x4000, 0xf1a: 0x4000, 0xf1b: 0x4000, 0xf1c: 0x4000, 0xf1d: 0x4000, + 0xf1e: 0x4000, 0xf1f: 0x4000, 0xf20: 0x4000, 0xf21: 0x4000, 0xf22: 0x4000, 0xf23: 0x4000, + 0xf24: 0x4000, 0xf25: 0x4000, 0xf26: 0x4000, 0xf27: 0x4000, 0xf28: 0x4000, 0xf29: 0x4000, + 0xf2a: 0x4000, 0xf2b: 0x4000, 0xf2c: 0x4000, 0xf2d: 0x4000, 0xf2e: 0x4000, 0xf2f: 0x4000, + 0xf30: 0x4000, 0xf31: 0x4000, 0xf32: 0x4000, 0xf33: 0x4000, 0xf34: 0x4000, 0xf35: 0x4000, + 0xf36: 0x4000, 0xf37: 0x4000, 0xf38: 0x4000, 0xf39: 0x4000, 0xf3a: 0x4000, 0xf3b: 0x4000, + 0xf3c: 0x4000, 0xf3d: 0x4000, 0xf3e: 0x4000, 0xf3f: 0x4000, + // Block 0x3d, offset 0xf40 + 0xf40: 0x4000, 0xf41: 0x4000, 0xf42: 0x4000, 0xf43: 0x4000, 0xf44: 0x4000, 0xf45: 0x4000, + 0xf46: 0x4000, + // Block 0x3e, offset 0xf80 + 0xfa0: 0x4000, 0xfa1: 0x4000, 0xfa2: 0x4000, 0xfa3: 0x4000, + 0xfa4: 0x4000, 0xfa5: 0x4000, 0xfa6: 0x4000, 0xfa7: 0x4000, 0xfa8: 0x4000, 0xfa9: 0x4000, + 0xfaa: 0x4000, 0xfab: 0x4000, 0xfac: 0x4000, 0xfad: 0x4000, 0xfae: 0x4000, 0xfaf: 0x4000, + 0xfb0: 0x4000, 0xfb1: 0x4000, 0xfb2: 0x4000, 0xfb3: 0x4000, 0xfb4: 0x4000, 0xfb5: 0x4000, + 0xfb6: 0x4000, 0xfb7: 0x4000, 0xfb8: 0x4000, 0xfb9: 0x4000, 0xfba: 0x4000, 0xfbb: 0x4000, + 0xfbc: 0x4000, + // Block 0x3f, offset 0xfc0 + 0xfc0: 0x4000, 0xfc1: 0x4000, 0xfc2: 0x4000, 0xfc3: 0x4000, 0xfc4: 0x4000, 0xfc5: 0x4000, + 0xfc6: 0x4000, 0xfc7: 0x4000, 0xfc8: 0x4000, 0xfc9: 0x4000, 0xfca: 0x4000, 0xfcb: 0x4000, + 0xfcc: 0x4000, 0xfcd: 0x4000, 0xfce: 0x4000, 0xfcf: 0x4000, 0xfd0: 0x4000, 0xfd1: 0x4000, + 0xfd2: 0x4000, 0xfd3: 0x4000, 0xfd4: 0x4000, 0xfd5: 0x4000, 0xfd6: 0x4000, 0xfd7: 0x4000, + 0xfd8: 0x4000, 0xfd9: 0x4000, 0xfda: 0x4000, 0xfdb: 0x4000, 0xfdc: 0x4000, 0xfdd: 0x4000, + 0xfde: 0x4000, 0xfdf: 0x4000, 0xfe0: 0x4000, 0xfe1: 0x4000, 0xfe2: 0x4000, 0xfe3: 0x4000, + // Block 0x40, offset 0x1000 + 0x1000: 0x2000, 0x1001: 0x2000, 0x1002: 0x2000, 0x1003: 0x2000, 0x1004: 0x2000, 0x1005: 0x2000, + 0x1006: 0x2000, 0x1007: 0x2000, 0x1008: 0x2000, 0x1009: 0x2000, 0x100a: 0x2000, 0x100b: 0x2000, + 0x100c: 0x2000, 0x100d: 0x2000, 0x100e: 0x2000, 0x100f: 0x2000, 0x1010: 0x4000, 0x1011: 0x4000, + 0x1012: 0x4000, 0x1013: 0x4000, 0x1014: 0x4000, 0x1015: 0x4000, 0x1016: 0x4000, 0x1017: 0x4000, + 0x1018: 0x4000, 0x1019: 0x4000, + 0x1030: 0x4000, 0x1031: 0x4000, 0x1032: 0x4000, 0x1033: 0x4000, 0x1034: 0x4000, 0x1035: 0x4000, + 0x1036: 0x4000, 0x1037: 0x4000, 0x1038: 0x4000, 0x1039: 0x4000, 0x103a: 0x4000, 0x103b: 0x4000, + 0x103c: 0x4000, 0x103d: 0x4000, 0x103e: 0x4000, 0x103f: 0x4000, + // Block 0x41, offset 0x1040 + 0x1040: 0x4000, 0x1041: 0x4000, 0x1042: 0x4000, 0x1043: 0x4000, 0x1044: 0x4000, 0x1045: 0x4000, + 0x1046: 0x4000, 0x1047: 0x4000, 0x1048: 0x4000, 0x1049: 0x4000, 0x104a: 0x4000, 0x104b: 0x4000, + 0x104c: 0x4000, 0x104d: 0x4000, 0x104e: 0x4000, 0x104f: 0x4000, 0x1050: 0x4000, 0x1051: 0x4000, + 0x1052: 0x4000, 0x1054: 0x4000, 0x1055: 0x4000, 0x1056: 0x4000, 0x1057: 0x4000, + 0x1058: 0x4000, 0x1059: 0x4000, 0x105a: 0x4000, 0x105b: 0x4000, 0x105c: 0x4000, 0x105d: 0x4000, + 0x105e: 0x4000, 0x105f: 0x4000, 0x1060: 0x4000, 0x1061: 0x4000, 0x1062: 0x4000, 0x1063: 0x4000, + 0x1064: 0x4000, 0x1065: 0x4000, 0x1066: 0x4000, 0x1068: 0x4000, 0x1069: 0x4000, + 0x106a: 0x4000, 0x106b: 0x4000, + // Block 0x42, offset 0x1080 + 0x1081: 0x9012, 0x1082: 0x9012, 0x1083: 0x9012, 0x1084: 0x9012, 0x1085: 0x9012, + 0x1086: 0x9012, 0x1087: 0x9012, 0x1088: 0x9012, 0x1089: 0x9012, 0x108a: 0x9012, 0x108b: 0x9012, + 0x108c: 0x9012, 0x108d: 0x9012, 0x108e: 0x9012, 0x108f: 0x9012, 0x1090: 0x9012, 0x1091: 0x9012, + 0x1092: 0x9012, 0x1093: 0x9012, 0x1094: 0x9012, 0x1095: 0x9012, 0x1096: 0x9012, 0x1097: 0x9012, + 0x1098: 0x9012, 0x1099: 0x9012, 0x109a: 0x9012, 0x109b: 0x9012, 0x109c: 0x9012, 0x109d: 0x9012, + 0x109e: 0x9012, 0x109f: 0x9012, 0x10a0: 0x9049, 0x10a1: 0x9049, 0x10a2: 0x9049, 0x10a3: 0x9049, + 0x10a4: 0x9049, 0x10a5: 0x9049, 0x10a6: 0x9049, 0x10a7: 0x9049, 0x10a8: 0x9049, 0x10a9: 0x9049, + 0x10aa: 0x9049, 0x10ab: 0x9049, 0x10ac: 0x9049, 0x10ad: 0x9049, 0x10ae: 0x9049, 0x10af: 0x9049, + 0x10b0: 0x9049, 0x10b1: 0x9049, 0x10b2: 0x9049, 0x10b3: 0x9049, 0x10b4: 0x9049, 0x10b5: 0x9049, + 0x10b6: 0x9049, 0x10b7: 0x9049, 0x10b8: 0x9049, 0x10b9: 0x9049, 0x10ba: 0x9049, 0x10bb: 0x9049, + 0x10bc: 0x9049, 0x10bd: 0x9049, 0x10be: 0x9049, 0x10bf: 0x9049, + // Block 0x43, offset 0x10c0 + 0x10c0: 0x9049, 0x10c1: 0x9049, 0x10c2: 0x9049, 0x10c3: 0x9049, 0x10c4: 0x9049, 0x10c5: 0x9049, + 0x10c6: 0x9049, 0x10c7: 0x9049, 0x10c8: 0x9049, 0x10c9: 0x9049, 0x10ca: 0x9049, 0x10cb: 0x9049, + 0x10cc: 0x9049, 0x10cd: 0x9049, 0x10ce: 0x9049, 0x10cf: 0x9049, 0x10d0: 0x9049, 0x10d1: 0x9049, + 0x10d2: 0x9049, 0x10d3: 0x9049, 0x10d4: 0x9049, 0x10d5: 0x9049, 0x10d6: 0x9049, 0x10d7: 0x9049, + 0x10d8: 0x9049, 0x10d9: 0x9049, 0x10da: 0x9049, 0x10db: 0x9049, 0x10dc: 0x9049, 0x10dd: 0x9049, + 0x10de: 0x9049, 0x10df: 0x904a, 0x10e0: 0x904b, 0x10e1: 0xb04c, 0x10e2: 0xb04d, 0x10e3: 0xb04d, + 0x10e4: 0xb04e, 0x10e5: 0xb04f, 0x10e6: 0xb050, 0x10e7: 0xb051, 0x10e8: 0xb052, 0x10e9: 0xb053, + 0x10ea: 0xb054, 0x10eb: 0xb055, 0x10ec: 0xb056, 0x10ed: 0xb057, 0x10ee: 0xb058, 0x10ef: 0xb059, + 0x10f0: 0xb05a, 0x10f1: 0xb05b, 0x10f2: 0xb05c, 0x10f3: 0xb05d, 0x10f4: 0xb05e, 0x10f5: 0xb05f, + 0x10f6: 0xb060, 0x10f7: 0xb061, 0x10f8: 0xb062, 0x10f9: 0xb063, 0x10fa: 0xb064, 0x10fb: 0xb065, + 0x10fc: 0xb052, 0x10fd: 0xb066, 0x10fe: 0xb067, 0x10ff: 0xb055, + // Block 0x44, offset 0x1100 + 0x1100: 0xb068, 0x1101: 0xb069, 0x1102: 0xb06a, 0x1103: 0xb06b, 0x1104: 0xb05a, 0x1105: 0xb056, + 0x1106: 0xb06c, 0x1107: 0xb06d, 0x1108: 0xb06b, 0x1109: 0xb06e, 0x110a: 0xb06b, 0x110b: 0xb06f, + 0x110c: 0xb06f, 0x110d: 0xb070, 0x110e: 0xb070, 0x110f: 0xb071, 0x1110: 0xb056, 0x1111: 0xb072, + 0x1112: 0xb073, 0x1113: 0xb072, 0x1114: 0xb074, 0x1115: 0xb073, 0x1116: 0xb075, 0x1117: 0xb075, + 0x1118: 0xb076, 0x1119: 0xb076, 0x111a: 0xb077, 0x111b: 0xb077, 0x111c: 0xb073, 0x111d: 0xb078, + 0x111e: 0xb079, 0x111f: 0xb067, 0x1120: 0xb07a, 0x1121: 0xb07b, 0x1122: 0xb07b, 0x1123: 0xb07b, + 0x1124: 0xb07b, 0x1125: 0xb07b, 0x1126: 0xb07b, 0x1127: 0xb07b, 0x1128: 0xb07b, 0x1129: 0xb07b, + 0x112a: 0xb07b, 0x112b: 0xb07b, 0x112c: 0xb07b, 0x112d: 0xb07b, 0x112e: 0xb07b, 0x112f: 0xb07b, + 0x1130: 0xb07c, 0x1131: 0xb07c, 0x1132: 0xb07c, 0x1133: 0xb07c, 0x1134: 0xb07c, 0x1135: 0xb07c, + 0x1136: 0xb07c, 0x1137: 0xb07c, 0x1138: 0xb07c, 0x1139: 0xb07c, 0x113a: 0xb07c, 0x113b: 0xb07c, + 0x113c: 0xb07c, 0x113d: 0xb07c, 0x113e: 0xb07c, + // Block 0x45, offset 0x1140 + 0x1142: 0xb07d, 0x1143: 0xb07e, 0x1144: 0xb07f, 0x1145: 0xb080, + 0x1146: 0xb07f, 0x1147: 0xb07e, 0x114a: 0xb081, 0x114b: 0xb082, + 0x114c: 0xb083, 0x114d: 0xb07f, 0x114e: 0xb080, 0x114f: 0xb07f, + 0x1152: 0xb084, 0x1153: 0xb085, 0x1154: 0xb084, 0x1155: 0xb086, 0x1156: 0xb084, 0x1157: 0xb087, + 0x115a: 0xb088, 0x115b: 0xb089, 0x115c: 0xb08a, + 0x1160: 0x908b, 0x1161: 0x908b, 0x1162: 0x908c, 0x1163: 0x908d, + 0x1164: 0x908b, 0x1165: 0x908e, 0x1166: 0x908f, 0x1168: 0xb090, 0x1169: 0xb091, + 0x116a: 0xb092, 0x116b: 0xb091, 0x116c: 0xb093, 0x116d: 0xb094, 0x116e: 0xb095, + 0x117d: 0x2000, + // Block 0x46, offset 0x1180 + 0x11a0: 0x4000, 0x11a1: 0x4000, 0x11a2: 0x4000, 0x11a3: 0x4000, + 0x11a4: 0x4000, + 0x11b0: 0x4000, 0x11b1: 0x4000, + // Block 0x47, offset 0x11c0 + 0x11c0: 0x4000, 0x11c1: 0x4000, 0x11c2: 0x4000, 0x11c3: 0x4000, 0x11c4: 0x4000, 0x11c5: 0x4000, + 0x11c6: 0x4000, 0x11c7: 0x4000, 0x11c8: 0x4000, 0x11c9: 0x4000, 0x11ca: 0x4000, 0x11cb: 0x4000, + 0x11cc: 0x4000, 0x11cd: 0x4000, 0x11ce: 0x4000, 0x11cf: 0x4000, 0x11d0: 0x4000, 0x11d1: 0x4000, + 0x11d2: 0x4000, 0x11d3: 0x4000, 0x11d4: 0x4000, 0x11d5: 0x4000, 0x11d6: 0x4000, 0x11d7: 0x4000, + 0x11d8: 0x4000, 0x11d9: 0x4000, 0x11da: 0x4000, 0x11db: 0x4000, 0x11dc: 0x4000, 0x11dd: 0x4000, + 0x11de: 0x4000, 0x11df: 0x4000, 0x11e0: 0x4000, 0x11e1: 0x4000, 0x11e2: 0x4000, 0x11e3: 0x4000, + 0x11e4: 0x4000, 0x11e5: 0x4000, 0x11e6: 0x4000, 0x11e7: 0x4000, 0x11e8: 0x4000, 0x11e9: 0x4000, + 0x11ea: 0x4000, 0x11eb: 0x4000, 0x11ec: 0x4000, 0x11ed: 0x4000, 0x11ee: 0x4000, 0x11ef: 0x4000, + 0x11f0: 0x4000, 0x11f1: 0x4000, 0x11f2: 0x4000, 0x11f3: 0x4000, 0x11f4: 0x4000, 0x11f5: 0x4000, + 0x11f6: 0x4000, 0x11f7: 0x4000, + // Block 0x48, offset 0x1200 + 0x1200: 0x4000, 0x1201: 0x4000, 0x1202: 0x4000, 0x1203: 0x4000, 0x1204: 0x4000, 0x1205: 0x4000, + 0x1206: 0x4000, 0x1207: 0x4000, 0x1208: 0x4000, 0x1209: 0x4000, 0x120a: 0x4000, 0x120b: 0x4000, + 0x120c: 0x4000, 0x120d: 0x4000, 0x120e: 0x4000, 0x120f: 0x4000, 0x1210: 0x4000, 0x1211: 0x4000, + 0x1212: 0x4000, 0x1213: 0x4000, 0x1214: 0x4000, 0x1215: 0x4000, + // Block 0x49, offset 0x1240 + 0x1240: 0x4000, 0x1241: 0x4000, 0x1242: 0x4000, 0x1243: 0x4000, 0x1244: 0x4000, 0x1245: 0x4000, + 0x1246: 0x4000, 0x1247: 0x4000, 0x1248: 0x4000, + // Block 0x4a, offset 0x1280 + 0x1280: 0x4000, 0x1281: 0x4000, 0x1282: 0x4000, 0x1283: 0x4000, 0x1284: 0x4000, 0x1285: 0x4000, + 0x1286: 0x4000, 0x1287: 0x4000, 0x1288: 0x4000, 0x1289: 0x4000, 0x128a: 0x4000, 0x128b: 0x4000, + 0x128c: 0x4000, 0x128d: 0x4000, 0x128e: 0x4000, 0x128f: 0x4000, 0x1290: 0x4000, 0x1291: 0x4000, + 0x1292: 0x4000, 0x1293: 0x4000, 0x1294: 0x4000, 0x1295: 0x4000, 0x1296: 0x4000, 0x1297: 0x4000, + 0x1298: 0x4000, 0x1299: 0x4000, 0x129a: 0x4000, 0x129b: 0x4000, 0x129c: 0x4000, 0x129d: 0x4000, + 0x129e: 0x4000, + // Block 0x4b, offset 0x12c0 + 0x12d0: 0x4000, 0x12d1: 0x4000, + 0x12d2: 0x4000, + 0x12e4: 0x4000, 0x12e5: 0x4000, 0x12e6: 0x4000, 0x12e7: 0x4000, + 0x12f0: 0x4000, 0x12f1: 0x4000, 0x12f2: 0x4000, 0x12f3: 0x4000, 0x12f4: 0x4000, 0x12f5: 0x4000, + 0x12f6: 0x4000, 0x12f7: 0x4000, 0x12f8: 0x4000, 0x12f9: 0x4000, 0x12fa: 0x4000, 0x12fb: 0x4000, + 0x12fc: 0x4000, 0x12fd: 0x4000, 0x12fe: 0x4000, 0x12ff: 0x4000, + // Block 0x4c, offset 0x1300 + 0x1300: 0x4000, 0x1301: 0x4000, 0x1302: 0x4000, 0x1303: 0x4000, 0x1304: 0x4000, 0x1305: 0x4000, + 0x1306: 0x4000, 0x1307: 0x4000, 0x1308: 0x4000, 0x1309: 0x4000, 0x130a: 0x4000, 0x130b: 0x4000, + 0x130c: 0x4000, 0x130d: 0x4000, 0x130e: 0x4000, 0x130f: 0x4000, 0x1310: 0x4000, 0x1311: 0x4000, + 0x1312: 0x4000, 0x1313: 0x4000, 0x1314: 0x4000, 0x1315: 0x4000, 0x1316: 0x4000, 0x1317: 0x4000, + 0x1318: 0x4000, 0x1319: 0x4000, 0x131a: 0x4000, 0x131b: 0x4000, 0x131c: 0x4000, 0x131d: 0x4000, + 0x131e: 0x4000, 0x131f: 0x4000, 0x1320: 0x4000, 0x1321: 0x4000, 0x1322: 0x4000, 0x1323: 0x4000, + 0x1324: 0x4000, 0x1325: 0x4000, 0x1326: 0x4000, 0x1327: 0x4000, 0x1328: 0x4000, 0x1329: 0x4000, + 0x132a: 0x4000, 0x132b: 0x4000, 0x132c: 0x4000, 0x132d: 0x4000, 0x132e: 0x4000, 0x132f: 0x4000, + 0x1330: 0x4000, 0x1331: 0x4000, 0x1332: 0x4000, 0x1333: 0x4000, 0x1334: 0x4000, 0x1335: 0x4000, + 0x1336: 0x4000, 0x1337: 0x4000, 0x1338: 0x4000, 0x1339: 0x4000, 0x133a: 0x4000, 0x133b: 0x4000, + // Block 0x4d, offset 0x1340 + 0x1344: 0x4000, + // Block 0x4e, offset 0x1380 + 0x138f: 0x4000, + // Block 0x4f, offset 0x13c0 + 0x13c0: 0x2000, 0x13c1: 0x2000, 0x13c2: 0x2000, 0x13c3: 0x2000, 0x13c4: 0x2000, 0x13c5: 0x2000, + 0x13c6: 0x2000, 0x13c7: 0x2000, 0x13c8: 0x2000, 0x13c9: 0x2000, 0x13ca: 0x2000, + 0x13d0: 0x2000, 0x13d1: 0x2000, + 0x13d2: 0x2000, 0x13d3: 0x2000, 0x13d4: 0x2000, 0x13d5: 0x2000, 0x13d6: 0x2000, 0x13d7: 0x2000, + 0x13d8: 0x2000, 0x13d9: 0x2000, 0x13da: 0x2000, 0x13db: 0x2000, 0x13dc: 0x2000, 0x13dd: 0x2000, + 0x13de: 0x2000, 0x13df: 0x2000, 0x13e0: 0x2000, 0x13e1: 0x2000, 0x13e2: 0x2000, 0x13e3: 0x2000, + 0x13e4: 0x2000, 0x13e5: 0x2000, 0x13e6: 0x2000, 0x13e7: 0x2000, 0x13e8: 0x2000, 0x13e9: 0x2000, + 0x13ea: 0x2000, 0x13eb: 0x2000, 0x13ec: 0x2000, 0x13ed: 0x2000, + 0x13f0: 0x2000, 0x13f1: 0x2000, 0x13f2: 0x2000, 0x13f3: 0x2000, 0x13f4: 0x2000, 0x13f5: 0x2000, + 0x13f6: 0x2000, 0x13f7: 0x2000, 0x13f8: 0x2000, 0x13f9: 0x2000, 0x13fa: 0x2000, 0x13fb: 0x2000, + 0x13fc: 0x2000, 0x13fd: 0x2000, 0x13fe: 0x2000, 0x13ff: 0x2000, + // Block 0x50, offset 0x1400 + 0x1400: 0x2000, 0x1401: 0x2000, 0x1402: 0x2000, 0x1403: 0x2000, 0x1404: 0x2000, 0x1405: 0x2000, + 0x1406: 0x2000, 0x1407: 0x2000, 0x1408: 0x2000, 0x1409: 0x2000, 0x140a: 0x2000, 0x140b: 0x2000, + 0x140c: 0x2000, 0x140d: 0x2000, 0x140e: 0x2000, 0x140f: 0x2000, 0x1410: 0x2000, 0x1411: 0x2000, + 0x1412: 0x2000, 0x1413: 0x2000, 0x1414: 0x2000, 0x1415: 0x2000, 0x1416: 0x2000, 0x1417: 0x2000, + 0x1418: 0x2000, 0x1419: 0x2000, 0x141a: 0x2000, 0x141b: 0x2000, 0x141c: 0x2000, 0x141d: 0x2000, + 0x141e: 0x2000, 0x141f: 0x2000, 0x1420: 0x2000, 0x1421: 0x2000, 0x1422: 0x2000, 0x1423: 0x2000, + 0x1424: 0x2000, 0x1425: 0x2000, 0x1426: 0x2000, 0x1427: 0x2000, 0x1428: 0x2000, 0x1429: 0x2000, + 0x1430: 0x2000, 0x1431: 0x2000, 0x1432: 0x2000, 0x1433: 0x2000, 0x1434: 0x2000, 0x1435: 0x2000, + 0x1436: 0x2000, 0x1437: 0x2000, 0x1438: 0x2000, 0x1439: 0x2000, 0x143a: 0x2000, 0x143b: 0x2000, + 0x143c: 0x2000, 0x143d: 0x2000, 0x143e: 0x2000, 0x143f: 0x2000, + // Block 0x51, offset 0x1440 + 0x1440: 0x2000, 0x1441: 0x2000, 0x1442: 0x2000, 0x1443: 0x2000, 0x1444: 0x2000, 0x1445: 0x2000, + 0x1446: 0x2000, 0x1447: 0x2000, 0x1448: 0x2000, 0x1449: 0x2000, 0x144a: 0x2000, 0x144b: 0x2000, + 0x144c: 0x2000, 0x144d: 0x2000, 0x144e: 0x4000, 0x144f: 0x2000, 0x1450: 0x2000, 0x1451: 0x4000, + 0x1452: 0x4000, 0x1453: 0x4000, 0x1454: 0x4000, 0x1455: 0x4000, 0x1456: 0x4000, 0x1457: 0x4000, + 0x1458: 0x4000, 0x1459: 0x4000, 0x145a: 0x4000, 0x145b: 0x2000, 0x145c: 0x2000, 0x145d: 0x2000, + 0x145e: 0x2000, 0x145f: 0x2000, 0x1460: 0x2000, 0x1461: 0x2000, 0x1462: 0x2000, 0x1463: 0x2000, + 0x1464: 0x2000, 0x1465: 0x2000, 0x1466: 0x2000, 0x1467: 0x2000, 0x1468: 0x2000, 0x1469: 0x2000, + 0x146a: 0x2000, 0x146b: 0x2000, 0x146c: 0x2000, + // Block 0x52, offset 0x1480 + 0x1480: 0x4000, 0x1481: 0x4000, 0x1482: 0x4000, + 0x1490: 0x4000, 0x1491: 0x4000, + 0x1492: 0x4000, 0x1493: 0x4000, 0x1494: 0x4000, 0x1495: 0x4000, 0x1496: 0x4000, 0x1497: 0x4000, + 0x1498: 0x4000, 0x1499: 0x4000, 0x149a: 0x4000, 0x149b: 0x4000, 0x149c: 0x4000, 0x149d: 0x4000, + 0x149e: 0x4000, 0x149f: 0x4000, 0x14a0: 0x4000, 0x14a1: 0x4000, 0x14a2: 0x4000, 0x14a3: 0x4000, + 0x14a4: 0x4000, 0x14a5: 0x4000, 0x14a6: 0x4000, 0x14a7: 0x4000, 0x14a8: 0x4000, 0x14a9: 0x4000, + 0x14aa: 0x4000, 0x14ab: 0x4000, 0x14ac: 0x4000, 0x14ad: 0x4000, 0x14ae: 0x4000, 0x14af: 0x4000, + 0x14b0: 0x4000, 0x14b1: 0x4000, 0x14b2: 0x4000, 0x14b3: 0x4000, 0x14b4: 0x4000, 0x14b5: 0x4000, + 0x14b6: 0x4000, 0x14b7: 0x4000, 0x14b8: 0x4000, 0x14b9: 0x4000, 0x14ba: 0x4000, 0x14bb: 0x4000, + // Block 0x53, offset 0x14c0 + 0x14c0: 0x4000, 0x14c1: 0x4000, 0x14c2: 0x4000, 0x14c3: 0x4000, 0x14c4: 0x4000, 0x14c5: 0x4000, + 0x14c6: 0x4000, 0x14c7: 0x4000, 0x14c8: 0x4000, + 0x14d0: 0x4000, 0x14d1: 0x4000, + 0x14e0: 0x4000, 0x14e1: 0x4000, 0x14e2: 0x4000, 0x14e3: 0x4000, + 0x14e4: 0x4000, 0x14e5: 0x4000, + // Block 0x54, offset 0x1500 + 0x1500: 0x4000, 0x1501: 0x4000, 0x1502: 0x4000, 0x1503: 0x4000, 0x1504: 0x4000, 0x1505: 0x4000, + 0x1506: 0x4000, 0x1507: 0x4000, 0x1508: 0x4000, 0x1509: 0x4000, 0x150a: 0x4000, 0x150b: 0x4000, + 0x150c: 0x4000, 0x150d: 0x4000, 0x150e: 0x4000, 0x150f: 0x4000, 0x1510: 0x4000, 0x1511: 0x4000, + 0x1512: 0x4000, 0x1513: 0x4000, 0x1514: 0x4000, 0x1515: 0x4000, 0x1516: 0x4000, 0x1517: 0x4000, + 0x1518: 0x4000, 0x1519: 0x4000, 0x151a: 0x4000, 0x151b: 0x4000, 0x151c: 0x4000, 0x151d: 0x4000, + 0x151e: 0x4000, 0x151f: 0x4000, 0x1520: 0x4000, + 0x152d: 0x4000, 0x152e: 0x4000, 0x152f: 0x4000, + 0x1530: 0x4000, 0x1531: 0x4000, 0x1532: 0x4000, 0x1533: 0x4000, 0x1534: 0x4000, 0x1535: 0x4000, + 0x1537: 0x4000, 0x1538: 0x4000, 0x1539: 0x4000, 0x153a: 0x4000, 0x153b: 0x4000, + 0x153c: 0x4000, 0x153d: 0x4000, 0x153e: 0x4000, 0x153f: 0x4000, + // Block 0x55, offset 0x1540 + 0x1540: 0x4000, 0x1541: 0x4000, 0x1542: 0x4000, 0x1543: 0x4000, 0x1544: 0x4000, 0x1545: 0x4000, + 0x1546: 0x4000, 0x1547: 0x4000, 0x1548: 0x4000, 0x1549: 0x4000, 0x154a: 0x4000, 0x154b: 0x4000, + 0x154c: 0x4000, 0x154d: 0x4000, 0x154e: 0x4000, 0x154f: 0x4000, 0x1550: 0x4000, 0x1551: 0x4000, + 0x1552: 0x4000, 0x1553: 0x4000, 0x1554: 0x4000, 0x1555: 0x4000, 0x1556: 0x4000, 0x1557: 0x4000, + 0x1558: 0x4000, 0x1559: 0x4000, 0x155a: 0x4000, 0x155b: 0x4000, 0x155c: 0x4000, 0x155d: 0x4000, + 0x155e: 0x4000, 0x155f: 0x4000, 0x1560: 0x4000, 0x1561: 0x4000, 0x1562: 0x4000, 0x1563: 0x4000, + 0x1564: 0x4000, 0x1565: 0x4000, 0x1566: 0x4000, 0x1567: 0x4000, 0x1568: 0x4000, 0x1569: 0x4000, + 0x156a: 0x4000, 0x156b: 0x4000, 0x156c: 0x4000, 0x156d: 0x4000, 0x156e: 0x4000, 0x156f: 0x4000, + 0x1570: 0x4000, 0x1571: 0x4000, 0x1572: 0x4000, 0x1573: 0x4000, 0x1574: 0x4000, 0x1575: 0x4000, + 0x1576: 0x4000, 0x1577: 0x4000, 0x1578: 0x4000, 0x1579: 0x4000, 0x157a: 0x4000, 0x157b: 0x4000, + 0x157c: 0x4000, 0x157e: 0x4000, 0x157f: 0x4000, + // Block 0x56, offset 0x1580 + 0x1580: 0x4000, 0x1581: 0x4000, 0x1582: 0x4000, 0x1583: 0x4000, 0x1584: 0x4000, 0x1585: 0x4000, + 0x1586: 0x4000, 0x1587: 0x4000, 0x1588: 0x4000, 0x1589: 0x4000, 0x158a: 0x4000, 0x158b: 0x4000, + 0x158c: 0x4000, 0x158d: 0x4000, 0x158e: 0x4000, 0x158f: 0x4000, 0x1590: 0x4000, 0x1591: 0x4000, + 0x1592: 0x4000, 0x1593: 0x4000, + 0x15a0: 0x4000, 0x15a1: 0x4000, 0x15a2: 0x4000, 0x15a3: 0x4000, + 0x15a4: 0x4000, 0x15a5: 0x4000, 0x15a6: 0x4000, 0x15a7: 0x4000, 0x15a8: 0x4000, 0x15a9: 0x4000, + 0x15aa: 0x4000, 0x15ab: 0x4000, 0x15ac: 0x4000, 0x15ad: 0x4000, 0x15ae: 0x4000, 0x15af: 0x4000, + 0x15b0: 0x4000, 0x15b1: 0x4000, 0x15b2: 0x4000, 0x15b3: 0x4000, 0x15b4: 0x4000, 0x15b5: 0x4000, + 0x15b6: 0x4000, 0x15b7: 0x4000, 0x15b8: 0x4000, 0x15b9: 0x4000, 0x15ba: 0x4000, 0x15bb: 0x4000, + 0x15bc: 0x4000, 0x15bd: 0x4000, 0x15be: 0x4000, 0x15bf: 0x4000, + // Block 0x57, offset 0x15c0 + 0x15c0: 0x4000, 0x15c1: 0x4000, 0x15c2: 0x4000, 0x15c3: 0x4000, 0x15c4: 0x4000, 0x15c5: 0x4000, + 0x15c6: 0x4000, 0x15c7: 0x4000, 0x15c8: 0x4000, 0x15c9: 0x4000, 0x15ca: 0x4000, + 0x15cf: 0x4000, 0x15d0: 0x4000, 0x15d1: 0x4000, + 0x15d2: 0x4000, 0x15d3: 0x4000, + 0x15e0: 0x4000, 0x15e1: 0x4000, 0x15e2: 0x4000, 0x15e3: 0x4000, + 0x15e4: 0x4000, 0x15e5: 0x4000, 0x15e6: 0x4000, 0x15e7: 0x4000, 0x15e8: 0x4000, 0x15e9: 0x4000, + 0x15ea: 0x4000, 0x15eb: 0x4000, 0x15ec: 0x4000, 0x15ed: 0x4000, 0x15ee: 0x4000, 0x15ef: 0x4000, + 0x15f0: 0x4000, 0x15f4: 0x4000, + 0x15f8: 0x4000, 0x15f9: 0x4000, 0x15fa: 0x4000, 0x15fb: 0x4000, + 0x15fc: 0x4000, 0x15fd: 0x4000, 0x15fe: 0x4000, 0x15ff: 0x4000, + // Block 0x58, offset 0x1600 + 0x1600: 0x4000, 0x1601: 0x4000, 0x1602: 0x4000, 0x1603: 0x4000, 0x1604: 0x4000, 0x1605: 0x4000, + 0x1606: 0x4000, 0x1607: 0x4000, 0x1608: 0x4000, 0x1609: 0x4000, 0x160a: 0x4000, 0x160b: 0x4000, + 0x160c: 0x4000, 0x160d: 0x4000, 0x160e: 0x4000, 0x160f: 0x4000, 0x1610: 0x4000, 0x1611: 0x4000, + 0x1612: 0x4000, 0x1613: 0x4000, 0x1614: 0x4000, 0x1615: 0x4000, 0x1616: 0x4000, 0x1617: 0x4000, + 0x1618: 0x4000, 0x1619: 0x4000, 0x161a: 0x4000, 0x161b: 0x4000, 0x161c: 0x4000, 0x161d: 0x4000, + 0x161e: 0x4000, 0x161f: 0x4000, 0x1620: 0x4000, 0x1621: 0x4000, 0x1622: 0x4000, 0x1623: 0x4000, + 0x1624: 0x4000, 0x1625: 0x4000, 0x1626: 0x4000, 0x1627: 0x4000, 0x1628: 0x4000, 0x1629: 0x4000, + 0x162a: 0x4000, 0x162b: 0x4000, 0x162c: 0x4000, 0x162d: 0x4000, 0x162e: 0x4000, 0x162f: 0x4000, + 0x1630: 0x4000, 0x1631: 0x4000, 0x1632: 0x4000, 0x1633: 0x4000, 0x1634: 0x4000, 0x1635: 0x4000, + 0x1636: 0x4000, 0x1637: 0x4000, 0x1638: 0x4000, 0x1639: 0x4000, 0x163a: 0x4000, 0x163b: 0x4000, + 0x163c: 0x4000, 0x163d: 0x4000, 0x163e: 0x4000, + // Block 0x59, offset 0x1640 + 0x1640: 0x4000, 0x1642: 0x4000, 0x1643: 0x4000, 0x1644: 0x4000, 0x1645: 0x4000, + 0x1646: 0x4000, 0x1647: 0x4000, 0x1648: 0x4000, 0x1649: 0x4000, 0x164a: 0x4000, 0x164b: 0x4000, + 0x164c: 0x4000, 0x164d: 0x4000, 0x164e: 0x4000, 0x164f: 0x4000, 0x1650: 0x4000, 0x1651: 0x4000, + 0x1652: 0x4000, 0x1653: 0x4000, 0x1654: 0x4000, 0x1655: 0x4000, 0x1656: 0x4000, 0x1657: 0x4000, + 0x1658: 0x4000, 0x1659: 0x4000, 0x165a: 0x4000, 0x165b: 0x4000, 0x165c: 0x4000, 0x165d: 0x4000, + 0x165e: 0x4000, 0x165f: 0x4000, 0x1660: 0x4000, 0x1661: 0x4000, 0x1662: 0x4000, 0x1663: 0x4000, + 0x1664: 0x4000, 0x1665: 0x4000, 0x1666: 0x4000, 0x1667: 0x4000, 0x1668: 0x4000, 0x1669: 0x4000, + 0x166a: 0x4000, 0x166b: 0x4000, 0x166c: 0x4000, 0x166d: 0x4000, 0x166e: 0x4000, 0x166f: 0x4000, + 0x1670: 0x4000, 0x1671: 0x4000, 0x1672: 0x4000, 0x1673: 0x4000, 0x1674: 0x4000, 0x1675: 0x4000, + 0x1676: 0x4000, 0x1677: 0x4000, 0x1678: 0x4000, 0x1679: 0x4000, 0x167a: 0x4000, 0x167b: 0x4000, + 0x167c: 0x4000, 0x167d: 0x4000, 0x167e: 0x4000, 0x167f: 0x4000, + // Block 0x5a, offset 0x1680 + 0x1680: 0x4000, 0x1681: 0x4000, 0x1682: 0x4000, 0x1683: 0x4000, 0x1684: 0x4000, 0x1685: 0x4000, + 0x1686: 0x4000, 0x1687: 0x4000, 0x1688: 0x4000, 0x1689: 0x4000, 0x168a: 0x4000, 0x168b: 0x4000, + 0x168c: 0x4000, 0x168d: 0x4000, 0x168e: 0x4000, 0x168f: 0x4000, 0x1690: 0x4000, 0x1691: 0x4000, + 0x1692: 0x4000, 0x1693: 0x4000, 0x1694: 0x4000, 0x1695: 0x4000, 0x1696: 0x4000, 0x1697: 0x4000, + 0x1698: 0x4000, 0x1699: 0x4000, 0x169a: 0x4000, 0x169b: 0x4000, 0x169c: 0x4000, 0x169d: 0x4000, + 0x169e: 0x4000, 0x169f: 0x4000, 0x16a0: 0x4000, 0x16a1: 0x4000, 0x16a2: 0x4000, 0x16a3: 0x4000, + 0x16a4: 0x4000, 0x16a5: 0x4000, 0x16a6: 0x4000, 0x16a7: 0x4000, 0x16a8: 0x4000, 0x16a9: 0x4000, + 0x16aa: 0x4000, 0x16ab: 0x4000, 0x16ac: 0x4000, 0x16ad: 0x4000, 0x16ae: 0x4000, 0x16af: 0x4000, + 0x16b0: 0x4000, 0x16b1: 0x4000, 0x16b2: 0x4000, 0x16b3: 0x4000, 0x16b4: 0x4000, 0x16b5: 0x4000, + 0x16b6: 0x4000, 0x16b7: 0x4000, 0x16b8: 0x4000, 0x16b9: 0x4000, 0x16ba: 0x4000, 0x16bb: 0x4000, + 0x16bc: 0x4000, 0x16bf: 0x4000, + // Block 0x5b, offset 0x16c0 + 0x16c0: 0x4000, 0x16c1: 0x4000, 0x16c2: 0x4000, 0x16c3: 0x4000, 0x16c4: 0x4000, 0x16c5: 0x4000, + 0x16c6: 0x4000, 0x16c7: 0x4000, 0x16c8: 0x4000, 0x16c9: 0x4000, 0x16ca: 0x4000, 0x16cb: 0x4000, + 0x16cc: 0x4000, 0x16cd: 0x4000, 0x16ce: 0x4000, 0x16cf: 0x4000, 0x16d0: 0x4000, 0x16d1: 0x4000, + 0x16d2: 0x4000, 0x16d3: 0x4000, 0x16d4: 0x4000, 0x16d5: 0x4000, 0x16d6: 0x4000, 0x16d7: 0x4000, + 0x16d8: 0x4000, 0x16d9: 0x4000, 0x16da: 0x4000, 0x16db: 0x4000, 0x16dc: 0x4000, 0x16dd: 0x4000, + 0x16de: 0x4000, 0x16df: 0x4000, 0x16e0: 0x4000, 0x16e1: 0x4000, 0x16e2: 0x4000, 0x16e3: 0x4000, + 0x16e4: 0x4000, 0x16e5: 0x4000, 0x16e6: 0x4000, 0x16e7: 0x4000, 0x16e8: 0x4000, 0x16e9: 0x4000, + 0x16ea: 0x4000, 0x16eb: 0x4000, 0x16ec: 0x4000, 0x16ed: 0x4000, 0x16ee: 0x4000, 0x16ef: 0x4000, + 0x16f0: 0x4000, 0x16f1: 0x4000, 0x16f2: 0x4000, 0x16f3: 0x4000, 0x16f4: 0x4000, 0x16f5: 0x4000, + 0x16f6: 0x4000, 0x16f7: 0x4000, 0x16f8: 0x4000, 0x16f9: 0x4000, 0x16fa: 0x4000, 0x16fb: 0x4000, + 0x16fc: 0x4000, 0x16fd: 0x4000, + // Block 0x5c, offset 0x1700 + 0x170b: 0x4000, + 0x170c: 0x4000, 0x170d: 0x4000, 0x170e: 0x4000, 0x1710: 0x4000, 0x1711: 0x4000, + 0x1712: 0x4000, 0x1713: 0x4000, 0x1714: 0x4000, 0x1715: 0x4000, 0x1716: 0x4000, 0x1717: 0x4000, + 0x1718: 0x4000, 0x1719: 0x4000, 0x171a: 0x4000, 0x171b: 0x4000, 0x171c: 0x4000, 0x171d: 0x4000, + 0x171e: 0x4000, 0x171f: 0x4000, 0x1720: 0x4000, 0x1721: 0x4000, 0x1722: 0x4000, 0x1723: 0x4000, + 0x1724: 0x4000, 0x1725: 0x4000, 0x1726: 0x4000, 0x1727: 0x4000, + 0x173a: 0x4000, + // Block 0x5d, offset 0x1740 + 0x1755: 0x4000, 0x1756: 0x4000, + 0x1764: 0x4000, + // Block 0x5e, offset 0x1780 + 0x17bb: 0x4000, + 0x17bc: 0x4000, 0x17bd: 0x4000, 0x17be: 0x4000, 0x17bf: 0x4000, + // Block 0x5f, offset 0x17c0 + 0x17c0: 0x4000, 0x17c1: 0x4000, 0x17c2: 0x4000, 0x17c3: 0x4000, 0x17c4: 0x4000, 0x17c5: 0x4000, + 0x17c6: 0x4000, 0x17c7: 0x4000, 0x17c8: 0x4000, 0x17c9: 0x4000, 0x17ca: 0x4000, 0x17cb: 0x4000, + 0x17cc: 0x4000, 0x17cd: 0x4000, 0x17ce: 0x4000, 0x17cf: 0x4000, + // Block 0x60, offset 0x1800 + 0x1800: 0x4000, 0x1801: 0x4000, 0x1802: 0x4000, 0x1803: 0x4000, 0x1804: 0x4000, 0x1805: 0x4000, + 0x180c: 0x4000, 0x1810: 0x4000, 0x1811: 0x4000, + 0x1812: 0x4000, 0x1815: 0x4000, 0x1816: 0x4000, 0x1817: 0x4000, + 0x182b: 0x4000, 0x182c: 0x4000, + 0x1834: 0x4000, 0x1835: 0x4000, + 0x1836: 0x4000, 0x1837: 0x4000, 0x1838: 0x4000, 0x1839: 0x4000, 0x183a: 0x4000, 0x183b: 0x4000, + 0x183c: 0x4000, + // Block 0x61, offset 0x1840 + 0x1860: 0x4000, 0x1861: 0x4000, 0x1862: 0x4000, 0x1863: 0x4000, + 0x1864: 0x4000, 0x1865: 0x4000, 0x1866: 0x4000, 0x1867: 0x4000, 0x1868: 0x4000, 0x1869: 0x4000, + 0x186a: 0x4000, 0x186b: 0x4000, + // Block 0x62, offset 0x1880 + 0x188c: 0x4000, 0x188d: 0x4000, 0x188e: 0x4000, 0x188f: 0x4000, 0x1890: 0x4000, 0x1891: 0x4000, + 0x1892: 0x4000, 0x1893: 0x4000, 0x1894: 0x4000, 0x1895: 0x4000, 0x1896: 0x4000, 0x1897: 0x4000, + 0x1898: 0x4000, 0x1899: 0x4000, 0x189a: 0x4000, 0x189b: 0x4000, 0x189c: 0x4000, 0x189d: 0x4000, + 0x189e: 0x4000, 0x189f: 0x4000, 0x18a0: 0x4000, 0x18a1: 0x4000, 0x18a2: 0x4000, 0x18a3: 0x4000, + 0x18a4: 0x4000, 0x18a5: 0x4000, 0x18a6: 0x4000, 0x18a7: 0x4000, 0x18a8: 0x4000, 0x18a9: 0x4000, + 0x18aa: 0x4000, 0x18ab: 0x4000, 0x18ac: 0x4000, 0x18ad: 0x4000, 0x18ae: 0x4000, 0x18af: 0x4000, + 0x18b0: 0x4000, 0x18b1: 0x4000, 0x18b2: 0x4000, 0x18b3: 0x4000, 0x18b4: 0x4000, 0x18b5: 0x4000, + 0x18b6: 0x4000, 0x18b7: 0x4000, 0x18b8: 0x4000, 0x18b9: 0x4000, 0x18ba: 0x4000, + 0x18bc: 0x4000, 0x18bd: 0x4000, 0x18be: 0x4000, 0x18bf: 0x4000, + // Block 0x63, offset 0x18c0 + 0x18c0: 0x4000, 0x18c1: 0x4000, 0x18c2: 0x4000, 0x18c3: 0x4000, 0x18c4: 0x4000, 0x18c5: 0x4000, + 0x18c7: 0x4000, 0x18c8: 0x4000, 0x18c9: 0x4000, 0x18ca: 0x4000, 0x18cb: 0x4000, + 0x18cc: 0x4000, 0x18cd: 0x4000, 0x18ce: 0x4000, 0x18cf: 0x4000, 0x18d0: 0x4000, 0x18d1: 0x4000, + 0x18d2: 0x4000, 0x18d3: 0x4000, 0x18d4: 0x4000, 0x18d5: 0x4000, 0x18d6: 0x4000, 0x18d7: 0x4000, + 0x18d8: 0x4000, 0x18d9: 0x4000, 0x18da: 0x4000, 0x18db: 0x4000, 0x18dc: 0x4000, 0x18dd: 0x4000, + 0x18de: 0x4000, 0x18df: 0x4000, 0x18e0: 0x4000, 0x18e1: 0x4000, 0x18e2: 0x4000, 0x18e3: 0x4000, + 0x18e4: 0x4000, 0x18e5: 0x4000, 0x18e6: 0x4000, 0x18e7: 0x4000, 0x18e8: 0x4000, 0x18e9: 0x4000, + 0x18ea: 0x4000, 0x18eb: 0x4000, 0x18ec: 0x4000, 0x18ed: 0x4000, 0x18ee: 0x4000, 0x18ef: 0x4000, + 0x18f0: 0x4000, 0x18f1: 0x4000, 0x18f2: 0x4000, 0x18f3: 0x4000, 0x18f4: 0x4000, 0x18f5: 0x4000, + 0x18f6: 0x4000, 0x18f7: 0x4000, 0x18f8: 0x4000, 0x18fa: 0x4000, 0x18fb: 0x4000, + 0x18fc: 0x4000, 0x18fd: 0x4000, 0x18fe: 0x4000, 0x18ff: 0x4000, + // Block 0x64, offset 0x1900 + 0x1900: 0x4000, 0x1901: 0x4000, 0x1902: 0x4000, 0x1903: 0x4000, 0x1904: 0x4000, 0x1905: 0x4000, + 0x1906: 0x4000, 0x1907: 0x4000, 0x1908: 0x4000, 0x1909: 0x4000, 0x190a: 0x4000, 0x190b: 0x4000, + 0x190d: 0x4000, 0x190e: 0x4000, 0x190f: 0x4000, 0x1910: 0x4000, 0x1911: 0x4000, + 0x1912: 0x4000, 0x1913: 0x4000, 0x1914: 0x4000, 0x1915: 0x4000, 0x1916: 0x4000, 0x1917: 0x4000, + 0x1918: 0x4000, 0x1919: 0x4000, 0x191a: 0x4000, 0x191b: 0x4000, 0x191c: 0x4000, 0x191d: 0x4000, + 0x191e: 0x4000, 0x191f: 0x4000, 0x1920: 0x4000, 0x1921: 0x4000, 0x1922: 0x4000, 0x1923: 0x4000, + 0x1924: 0x4000, 0x1925: 0x4000, 0x1926: 0x4000, 0x1927: 0x4000, 0x1928: 0x4000, 0x1929: 0x4000, + 0x192a: 0x4000, 0x192b: 0x4000, 0x192c: 0x4000, 0x192d: 0x4000, 0x192e: 0x4000, 0x192f: 0x4000, + 0x1930: 0x4000, 0x1931: 0x4000, 0x1932: 0x4000, 0x1933: 0x4000, 0x1934: 0x4000, 0x1935: 0x4000, + 0x1936: 0x4000, 0x1937: 0x4000, 0x1938: 0x4000, 0x1939: 0x4000, 0x193a: 0x4000, 0x193b: 0x4000, + 0x193c: 0x4000, 0x193d: 0x4000, 0x193e: 0x4000, 0x193f: 0x4000, + // Block 0x65, offset 0x1940 + 0x1970: 0x4000, 0x1971: 0x4000, 0x1972: 0x4000, 0x1973: 0x4000, 0x1974: 0x4000, + 0x1978: 0x4000, 0x1979: 0x4000, 0x197a: 0x4000, + // Block 0x66, offset 0x1980 + 0x1980: 0x4000, 0x1981: 0x4000, 0x1982: 0x4000, 0x1983: 0x4000, 0x1984: 0x4000, 0x1985: 0x4000, + 0x1986: 0x4000, + 0x1990: 0x4000, 0x1991: 0x4000, + 0x1992: 0x4000, 0x1993: 0x4000, 0x1994: 0x4000, 0x1995: 0x4000, 0x1996: 0x4000, 0x1997: 0x4000, + 0x1998: 0x4000, 0x1999: 0x4000, 0x199a: 0x4000, 0x199b: 0x4000, 0x199c: 0x4000, 0x199d: 0x4000, + 0x199e: 0x4000, 0x199f: 0x4000, 0x19a0: 0x4000, 0x19a1: 0x4000, 0x19a2: 0x4000, 0x19a3: 0x4000, + 0x19a4: 0x4000, 0x19a5: 0x4000, 0x19a6: 0x4000, 0x19a7: 0x4000, 0x19a8: 0x4000, + 0x19b0: 0x4000, 0x19b1: 0x4000, 0x19b2: 0x4000, 0x19b3: 0x4000, 0x19b4: 0x4000, 0x19b5: 0x4000, + 0x19b6: 0x4000, + // Block 0x67, offset 0x19c0 + 0x19c0: 0x4000, 0x19c1: 0x4000, 0x19c2: 0x4000, + 0x19d0: 0x4000, 0x19d1: 0x4000, + 0x19d2: 0x4000, 0x19d3: 0x4000, 0x19d4: 0x4000, 0x19d5: 0x4000, 0x19d6: 0x4000, + // Block 0x68, offset 0x1a00 + 0x1a00: 0x2000, 0x1a01: 0x2000, 0x1a02: 0x2000, 0x1a03: 0x2000, 0x1a04: 0x2000, 0x1a05: 0x2000, + 0x1a06: 0x2000, 0x1a07: 0x2000, 0x1a08: 0x2000, 0x1a09: 0x2000, 0x1a0a: 0x2000, 0x1a0b: 0x2000, + 0x1a0c: 0x2000, 0x1a0d: 0x2000, 0x1a0e: 0x2000, 0x1a0f: 0x2000, 0x1a10: 0x2000, 0x1a11: 0x2000, + 0x1a12: 0x2000, 0x1a13: 0x2000, 0x1a14: 0x2000, 0x1a15: 0x2000, 0x1a16: 0x2000, 0x1a17: 0x2000, + 0x1a18: 0x2000, 0x1a19: 0x2000, 0x1a1a: 0x2000, 0x1a1b: 0x2000, 0x1a1c: 0x2000, 0x1a1d: 0x2000, + 0x1a1e: 0x2000, 0x1a1f: 0x2000, 0x1a20: 0x2000, 0x1a21: 0x2000, 0x1a22: 0x2000, 0x1a23: 0x2000, + 0x1a24: 0x2000, 0x1a25: 0x2000, 0x1a26: 0x2000, 0x1a27: 0x2000, 0x1a28: 0x2000, 0x1a29: 0x2000, + 0x1a2a: 0x2000, 0x1a2b: 0x2000, 0x1a2c: 0x2000, 0x1a2d: 0x2000, 0x1a2e: 0x2000, 0x1a2f: 0x2000, + 0x1a30: 0x2000, 0x1a31: 0x2000, 0x1a32: 0x2000, 0x1a33: 0x2000, 0x1a34: 0x2000, 0x1a35: 0x2000, + 0x1a36: 0x2000, 0x1a37: 0x2000, 0x1a38: 0x2000, 0x1a39: 0x2000, 0x1a3a: 0x2000, 0x1a3b: 0x2000, + 0x1a3c: 0x2000, 0x1a3d: 0x2000, +} + +// widthIndex: 22 blocks, 1408 entries, 1408 bytes +// Block 0 is the zero block. +var widthIndex = [1408]uint8{ + // Block 0x0, offset 0x0 + // Block 0x1, offset 0x40 + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0xc2: 0x01, 0xc3: 0x02, 0xc4: 0x03, 0xc5: 0x04, 0xc7: 0x05, + 0xc9: 0x06, 0xcb: 0x07, 0xcc: 0x08, 0xcd: 0x09, 0xce: 0x0a, 0xcf: 0x0b, + 0xd0: 0x0c, 0xd1: 0x0d, + 0xe1: 0x02, 0xe2: 0x03, 0xe3: 0x04, 0xe4: 0x05, 0xe5: 0x06, 0xe6: 0x06, 0xe7: 0x06, + 0xe8: 0x06, 0xe9: 0x06, 0xea: 0x07, 0xeb: 0x06, 0xec: 0x06, 0xed: 0x08, 0xee: 0x09, 0xef: 0x0a, + 0xf0: 0x0f, 0xf3: 0x12, 0xf4: 0x13, + // Block 0x4, offset 0x100 + 0x104: 0x0e, 0x105: 0x0f, + // Block 0x5, offset 0x140 + 0x140: 0x10, 0x141: 0x11, 0x142: 0x12, 0x144: 0x13, 0x145: 0x14, 0x146: 0x15, 0x147: 0x16, + 0x148: 0x17, 0x149: 0x18, 0x14a: 0x19, 0x14c: 0x1a, 0x14f: 0x1b, + 0x151: 0x1c, 0x152: 0x08, 0x153: 0x1d, 0x154: 0x1e, 0x155: 0x1f, 0x156: 0x20, 0x157: 0x21, + 0x158: 0x22, 0x159: 0x23, 0x15a: 0x24, 0x15b: 0x25, 0x15c: 0x26, 0x15d: 0x27, 0x15e: 0x28, 0x15f: 0x29, + 0x166: 0x2a, + 0x16c: 0x2b, 0x16d: 0x2c, + 0x17a: 0x2d, 0x17b: 0x2e, 0x17c: 0x0e, 0x17d: 0x0e, 0x17e: 0x0e, 0x17f: 0x2f, + // Block 0x6, offset 0x180 + 0x180: 0x30, 0x181: 0x31, 0x182: 0x32, 0x183: 0x33, 0x184: 0x34, 0x185: 0x35, 0x186: 0x36, 0x187: 0x37, + 0x188: 0x38, 0x189: 0x39, 0x18a: 0x0e, 0x18b: 0x0e, 0x18c: 0x0e, 0x18d: 0x0e, 0x18e: 0x0e, 0x18f: 0x0e, + 0x190: 0x0e, 0x191: 0x0e, 0x192: 0x0e, 0x193: 0x0e, 0x194: 0x0e, 0x195: 0x0e, 0x196: 0x0e, 0x197: 0x0e, + 0x198: 0x0e, 0x199: 0x0e, 0x19a: 0x0e, 0x19b: 0x0e, 0x19c: 0x0e, 0x19d: 0x0e, 0x19e: 0x0e, 0x19f: 0x0e, + 0x1a0: 0x0e, 0x1a1: 0x0e, 0x1a2: 0x0e, 0x1a3: 0x0e, 0x1a4: 0x0e, 0x1a5: 0x0e, 0x1a6: 0x0e, 0x1a7: 0x0e, + 0x1a8: 0x0e, 0x1a9: 0x0e, 0x1aa: 0x0e, 0x1ab: 0x0e, 0x1ac: 0x0e, 0x1ad: 0x0e, 0x1ae: 0x0e, 0x1af: 0x0e, + 0x1b0: 0x0e, 0x1b1: 0x0e, 0x1b2: 0x0e, 0x1b3: 0x0e, 0x1b4: 0x0e, 0x1b5: 0x0e, 0x1b6: 0x0e, 0x1b7: 0x0e, + 0x1b8: 0x0e, 0x1b9: 0x0e, 0x1ba: 0x0e, 0x1bb: 0x0e, 0x1bc: 0x0e, 0x1bd: 0x0e, 0x1be: 0x0e, 0x1bf: 0x0e, + // Block 0x7, offset 0x1c0 + 0x1c0: 0x0e, 0x1c1: 0x0e, 0x1c2: 0x0e, 0x1c3: 0x0e, 0x1c4: 0x0e, 0x1c5: 0x0e, 0x1c6: 0x0e, 0x1c7: 0x0e, + 0x1c8: 0x0e, 0x1c9: 0x0e, 0x1ca: 0x0e, 0x1cb: 0x0e, 0x1cc: 0x0e, 0x1cd: 0x0e, 0x1ce: 0x0e, 0x1cf: 0x0e, + 0x1d0: 0x0e, 0x1d1: 0x0e, 0x1d2: 0x0e, 0x1d3: 0x0e, 0x1d4: 0x0e, 0x1d5: 0x0e, 0x1d6: 0x0e, 0x1d7: 0x0e, + 0x1d8: 0x0e, 0x1d9: 0x0e, 0x1da: 0x0e, 0x1db: 0x0e, 0x1dc: 0x0e, 0x1dd: 0x0e, 0x1de: 0x0e, 0x1df: 0x0e, + 0x1e0: 0x0e, 0x1e1: 0x0e, 0x1e2: 0x0e, 0x1e3: 0x0e, 0x1e4: 0x0e, 0x1e5: 0x0e, 0x1e6: 0x0e, 0x1e7: 0x0e, + 0x1e8: 0x0e, 0x1e9: 0x0e, 0x1ea: 0x0e, 0x1eb: 0x0e, 0x1ec: 0x0e, 0x1ed: 0x0e, 0x1ee: 0x0e, 0x1ef: 0x0e, + 0x1f0: 0x0e, 0x1f1: 0x0e, 0x1f2: 0x0e, 0x1f3: 0x0e, 0x1f4: 0x0e, 0x1f5: 0x0e, 0x1f6: 0x0e, + 0x1f8: 0x0e, 0x1f9: 0x0e, 0x1fa: 0x0e, 0x1fb: 0x0e, 0x1fc: 0x0e, 0x1fd: 0x0e, 0x1fe: 0x0e, 0x1ff: 0x0e, + // Block 0x8, offset 0x200 + 0x200: 0x0e, 0x201: 0x0e, 0x202: 0x0e, 0x203: 0x0e, 0x204: 0x0e, 0x205: 0x0e, 0x206: 0x0e, 0x207: 0x0e, + 0x208: 0x0e, 0x209: 0x0e, 0x20a: 0x0e, 0x20b: 0x0e, 0x20c: 0x0e, 0x20d: 0x0e, 0x20e: 0x0e, 0x20f: 0x0e, + 0x210: 0x0e, 0x211: 0x0e, 0x212: 0x0e, 0x213: 0x0e, 0x214: 0x0e, 0x215: 0x0e, 0x216: 0x0e, 0x217: 0x0e, + 0x218: 0x0e, 0x219: 0x0e, 0x21a: 0x0e, 0x21b: 0x0e, 0x21c: 0x0e, 0x21d: 0x0e, 0x21e: 0x0e, 0x21f: 0x0e, + 0x220: 0x0e, 0x221: 0x0e, 0x222: 0x0e, 0x223: 0x0e, 0x224: 0x0e, 0x225: 0x0e, 0x226: 0x0e, 0x227: 0x0e, + 0x228: 0x0e, 0x229: 0x0e, 0x22a: 0x0e, 0x22b: 0x0e, 0x22c: 0x0e, 0x22d: 0x0e, 0x22e: 0x0e, 0x22f: 0x0e, + 0x230: 0x0e, 0x231: 0x0e, 0x232: 0x0e, 0x233: 0x0e, 0x234: 0x0e, 0x235: 0x0e, 0x236: 0x0e, 0x237: 0x0e, + 0x238: 0x0e, 0x239: 0x0e, 0x23a: 0x0e, 0x23b: 0x0e, 0x23c: 0x0e, 0x23d: 0x0e, 0x23e: 0x0e, 0x23f: 0x0e, + // Block 0x9, offset 0x240 + 0x240: 0x0e, 0x241: 0x0e, 0x242: 0x0e, 0x243: 0x0e, 0x244: 0x0e, 0x245: 0x0e, 0x246: 0x0e, 0x247: 0x0e, + 0x248: 0x0e, 0x249: 0x0e, 0x24a: 0x0e, 0x24b: 0x0e, 0x24c: 0x0e, 0x24d: 0x0e, 0x24e: 0x0e, 0x24f: 0x0e, + 0x250: 0x0e, 0x251: 0x0e, 0x252: 0x3a, 0x253: 0x3b, + 0x265: 0x3c, + 0x270: 0x0e, 0x271: 0x0e, 0x272: 0x0e, 0x273: 0x0e, 0x274: 0x0e, 0x275: 0x0e, 0x276: 0x0e, 0x277: 0x0e, + 0x278: 0x0e, 0x279: 0x0e, 0x27a: 0x0e, 0x27b: 0x0e, 0x27c: 0x0e, 0x27d: 0x0e, 0x27e: 0x0e, 0x27f: 0x0e, + // Block 0xa, offset 0x280 + 0x280: 0x0e, 0x281: 0x0e, 0x282: 0x0e, 0x283: 0x0e, 0x284: 0x0e, 0x285: 0x0e, 0x286: 0x0e, 0x287: 0x0e, + 0x288: 0x0e, 0x289: 0x0e, 0x28a: 0x0e, 0x28b: 0x0e, 0x28c: 0x0e, 0x28d: 0x0e, 0x28e: 0x0e, 0x28f: 0x0e, + 0x290: 0x0e, 0x291: 0x0e, 0x292: 0x0e, 0x293: 0x0e, 0x294: 0x0e, 0x295: 0x0e, 0x296: 0x0e, 0x297: 0x0e, + 0x298: 0x0e, 0x299: 0x0e, 0x29a: 0x0e, 0x29b: 0x0e, 0x29c: 0x0e, 0x29d: 0x0e, 0x29e: 0x3d, + // Block 0xb, offset 0x2c0 + 0x2c0: 0x08, 0x2c1: 0x08, 0x2c2: 0x08, 0x2c3: 0x08, 0x2c4: 0x08, 0x2c5: 0x08, 0x2c6: 0x08, 0x2c7: 0x08, + 0x2c8: 0x08, 0x2c9: 0x08, 0x2ca: 0x08, 0x2cb: 0x08, 0x2cc: 0x08, 0x2cd: 0x08, 0x2ce: 0x08, 0x2cf: 0x08, + 0x2d0: 0x08, 0x2d1: 0x08, 0x2d2: 0x08, 0x2d3: 0x08, 0x2d4: 0x08, 0x2d5: 0x08, 0x2d6: 0x08, 0x2d7: 0x08, + 0x2d8: 0x08, 0x2d9: 0x08, 0x2da: 0x08, 0x2db: 0x08, 0x2dc: 0x08, 0x2dd: 0x08, 0x2de: 0x08, 0x2df: 0x08, + 0x2e0: 0x08, 0x2e1: 0x08, 0x2e2: 0x08, 0x2e3: 0x08, 0x2e4: 0x08, 0x2e5: 0x08, 0x2e6: 0x08, 0x2e7: 0x08, + 0x2e8: 0x08, 0x2e9: 0x08, 0x2ea: 0x08, 0x2eb: 0x08, 0x2ec: 0x08, 0x2ed: 0x08, 0x2ee: 0x08, 0x2ef: 0x08, + 0x2f0: 0x08, 0x2f1: 0x08, 0x2f2: 0x08, 0x2f3: 0x08, 0x2f4: 0x08, 0x2f5: 0x08, 0x2f6: 0x08, 0x2f7: 0x08, + 0x2f8: 0x08, 0x2f9: 0x08, 0x2fa: 0x08, 0x2fb: 0x08, 0x2fc: 0x08, 0x2fd: 0x08, 0x2fe: 0x08, 0x2ff: 0x08, + // Block 0xc, offset 0x300 + 0x300: 0x08, 0x301: 0x08, 0x302: 0x08, 0x303: 0x08, 0x304: 0x08, 0x305: 0x08, 0x306: 0x08, 0x307: 0x08, + 0x308: 0x08, 0x309: 0x08, 0x30a: 0x08, 0x30b: 0x08, 0x30c: 0x08, 0x30d: 0x08, 0x30e: 0x08, 0x30f: 0x08, + 0x310: 0x08, 0x311: 0x08, 0x312: 0x08, 0x313: 0x08, 0x314: 0x08, 0x315: 0x08, 0x316: 0x08, 0x317: 0x08, + 0x318: 0x08, 0x319: 0x08, 0x31a: 0x08, 0x31b: 0x08, 0x31c: 0x08, 0x31d: 0x08, 0x31e: 0x08, 0x31f: 0x08, + 0x320: 0x08, 0x321: 0x08, 0x322: 0x08, 0x323: 0x08, 0x324: 0x0e, 0x325: 0x0e, 0x326: 0x0e, 0x327: 0x0e, + 0x328: 0x0e, 0x329: 0x0e, 0x32a: 0x0e, 0x32b: 0x0e, + 0x338: 0x3e, 0x339: 0x3f, 0x33c: 0x40, 0x33d: 0x41, 0x33e: 0x42, 0x33f: 0x43, + // Block 0xd, offset 0x340 + 0x37f: 0x44, + // Block 0xe, offset 0x380 + 0x380: 0x0e, 0x381: 0x0e, 0x382: 0x0e, 0x383: 0x0e, 0x384: 0x0e, 0x385: 0x0e, 0x386: 0x0e, 0x387: 0x0e, + 0x388: 0x0e, 0x389: 0x0e, 0x38a: 0x0e, 0x38b: 0x0e, 0x38c: 0x0e, 0x38d: 0x0e, 0x38e: 0x0e, 0x38f: 0x0e, + 0x390: 0x0e, 0x391: 0x0e, 0x392: 0x0e, 0x393: 0x0e, 0x394: 0x0e, 0x395: 0x0e, 0x396: 0x0e, 0x397: 0x0e, + 0x398: 0x0e, 0x399: 0x0e, 0x39a: 0x0e, 0x39b: 0x0e, 0x39c: 0x0e, 0x39d: 0x0e, 0x39e: 0x0e, 0x39f: 0x45, + 0x3a0: 0x0e, 0x3a1: 0x0e, 0x3a2: 0x0e, 0x3a3: 0x0e, 0x3a4: 0x0e, 0x3a5: 0x0e, 0x3a6: 0x0e, 0x3a7: 0x0e, + 0x3a8: 0x0e, 0x3a9: 0x0e, 0x3aa: 0x0e, 0x3ab: 0x0e, 0x3ac: 0x0e, 0x3ad: 0x0e, 0x3ae: 0x0e, 0x3af: 0x0e, + 0x3b0: 0x0e, 0x3b1: 0x0e, 0x3b2: 0x0e, 0x3b3: 0x46, 0x3b4: 0x47, + // Block 0xf, offset 0x3c0 + 0x3c0: 0x0e, 0x3c1: 0x0e, 0x3c2: 0x0e, 0x3c3: 0x0e, 0x3c4: 0x48, 0x3c5: 0x49, 0x3c6: 0x0e, 0x3c7: 0x0e, + 0x3c8: 0x0e, 0x3c9: 0x0e, 0x3ca: 0x0e, 0x3cb: 0x4a, + // Block 0x10, offset 0x400 + 0x400: 0x4b, 0x403: 0x4c, 0x404: 0x4d, 0x405: 0x4e, 0x406: 0x4f, + 0x408: 0x50, 0x409: 0x51, 0x40c: 0x52, 0x40d: 0x53, 0x40e: 0x54, 0x40f: 0x55, + 0x410: 0x56, 0x411: 0x57, 0x412: 0x0e, 0x413: 0x58, 0x414: 0x59, 0x415: 0x5a, 0x416: 0x5b, 0x417: 0x5c, + 0x418: 0x0e, 0x419: 0x5d, 0x41a: 0x0e, 0x41b: 0x5e, 0x41f: 0x5f, + 0x424: 0x60, 0x425: 0x61, 0x426: 0x0e, 0x427: 0x62, + 0x429: 0x63, 0x42a: 0x64, 0x42b: 0x65, + // Block 0x11, offset 0x440 + 0x456: 0x0b, 0x457: 0x06, + 0x458: 0x0c, 0x45b: 0x0d, 0x45f: 0x0e, + 0x460: 0x06, 0x461: 0x06, 0x462: 0x06, 0x463: 0x06, 0x464: 0x06, 0x465: 0x06, 0x466: 0x06, 0x467: 0x06, + 0x468: 0x06, 0x469: 0x06, 0x46a: 0x06, 0x46b: 0x06, 0x46c: 0x06, 0x46d: 0x06, 0x46e: 0x06, 0x46f: 0x06, + 0x470: 0x06, 0x471: 0x06, 0x472: 0x06, 0x473: 0x06, 0x474: 0x06, 0x475: 0x06, 0x476: 0x06, 0x477: 0x06, + 0x478: 0x06, 0x479: 0x06, 0x47a: 0x06, 0x47b: 0x06, 0x47c: 0x06, 0x47d: 0x06, 0x47e: 0x06, 0x47f: 0x06, + // Block 0x12, offset 0x480 + 0x484: 0x08, 0x485: 0x08, 0x486: 0x08, 0x487: 0x09, + // Block 0x13, offset 0x4c0 + 0x4c0: 0x08, 0x4c1: 0x08, 0x4c2: 0x08, 0x4c3: 0x08, 0x4c4: 0x08, 0x4c5: 0x08, 0x4c6: 0x08, 0x4c7: 0x08, + 0x4c8: 0x08, 0x4c9: 0x08, 0x4ca: 0x08, 0x4cb: 0x08, 0x4cc: 0x08, 0x4cd: 0x08, 0x4ce: 0x08, 0x4cf: 0x08, + 0x4d0: 0x08, 0x4d1: 0x08, 0x4d2: 0x08, 0x4d3: 0x08, 0x4d4: 0x08, 0x4d5: 0x08, 0x4d6: 0x08, 0x4d7: 0x08, + 0x4d8: 0x08, 0x4d9: 0x08, 0x4da: 0x08, 0x4db: 0x08, 0x4dc: 0x08, 0x4dd: 0x08, 0x4de: 0x08, 0x4df: 0x08, + 0x4e0: 0x08, 0x4e1: 0x08, 0x4e2: 0x08, 0x4e3: 0x08, 0x4e4: 0x08, 0x4e5: 0x08, 0x4e6: 0x08, 0x4e7: 0x08, + 0x4e8: 0x08, 0x4e9: 0x08, 0x4ea: 0x08, 0x4eb: 0x08, 0x4ec: 0x08, 0x4ed: 0x08, 0x4ee: 0x08, 0x4ef: 0x08, + 0x4f0: 0x08, 0x4f1: 0x08, 0x4f2: 0x08, 0x4f3: 0x08, 0x4f4: 0x08, 0x4f5: 0x08, 0x4f6: 0x08, 0x4f7: 0x08, + 0x4f8: 0x08, 0x4f9: 0x08, 0x4fa: 0x08, 0x4fb: 0x08, 0x4fc: 0x08, 0x4fd: 0x08, 0x4fe: 0x08, 0x4ff: 0x66, + // Block 0x14, offset 0x500 + 0x520: 0x10, + 0x530: 0x09, 0x531: 0x09, 0x532: 0x09, 0x533: 0x09, 0x534: 0x09, 0x535: 0x09, 0x536: 0x09, 0x537: 0x09, + 0x538: 0x09, 0x539: 0x09, 0x53a: 0x09, 0x53b: 0x09, 0x53c: 0x09, 0x53d: 0x09, 0x53e: 0x09, 0x53f: 0x11, + // Block 0x15, offset 0x540 + 0x540: 0x09, 0x541: 0x09, 0x542: 0x09, 0x543: 0x09, 0x544: 0x09, 0x545: 0x09, 0x546: 0x09, 0x547: 0x09, + 0x548: 0x09, 0x549: 0x09, 0x54a: 0x09, 0x54b: 0x09, 0x54c: 0x09, 0x54d: 0x09, 0x54e: 0x09, 0x54f: 0x11, +} + +// inverseData contains 4-byte entries of the following format: +// +// <0 padding> +// +// The last byte of the UTF-8-encoded rune is xor-ed with the last byte of the +// UTF-8 encoding of the original rune. Mappings often have the following +// pattern: +// +// Ａ -> A (U+FF21 -> U+0041) +// Ｂ -> B (U+FF22 -> U+0042) +// ... +// +// By xor-ing the last byte the same entry can be shared by many mappings. This +// reduces the total number of distinct entries by about two thirds. +// The resulting entry for the aforementioned mappings is +// +// { 0x01, 0xE0, 0x00, 0x00 } +// +// Using this entry to map U+FF21 (UTF-8 [EF BC A1]), we get +// +// E0 ^ A1 = 41. +// +// Similarly, for U+FF22 (UTF-8 [EF BC A2]), we get +// +// E0 ^ A2 = 42. +// +// Note that because of the xor-ing, the byte sequence stored in the entry is +// not valid UTF-8. +var inverseData = [150][4]byte{ + {0x00, 0x00, 0x00, 0x00}, + {0x03, 0xe3, 0x80, 0xa0}, + {0x03, 0xef, 0xbc, 0xa0}, + {0x03, 0xef, 0xbc, 0xe0}, + {0x03, 0xef, 0xbd, 0xe0}, + {0x03, 0xef, 0xbf, 0x02}, + {0x03, 0xef, 0xbf, 0x00}, + {0x03, 0xef, 0xbf, 0x0e}, + {0x03, 0xef, 0xbf, 0x0c}, + {0x03, 0xef, 0xbf, 0x0f}, + {0x03, 0xef, 0xbf, 0x39}, + {0x03, 0xef, 0xbf, 0x3b}, + {0x03, 0xef, 0xbf, 0x3f}, + {0x03, 0xef, 0xbf, 0x2a}, + {0x03, 0xef, 0xbf, 0x0d}, + {0x03, 0xef, 0xbf, 0x25}, + {0x03, 0xef, 0xbd, 0x1a}, + {0x03, 0xef, 0xbd, 0x26}, + {0x01, 0xa0, 0x00, 0x00}, + {0x03, 0xef, 0xbd, 0x25}, + {0x03, 0xef, 0xbd, 0x23}, + {0x03, 0xef, 0xbd, 0x2e}, + {0x03, 0xef, 0xbe, 0x07}, + {0x03, 0xef, 0xbe, 0x05}, + {0x03, 0xef, 0xbd, 0x06}, + {0x03, 0xef, 0xbd, 0x13}, + {0x03, 0xef, 0xbd, 0x0b}, + {0x03, 0xef, 0xbd, 0x16}, + {0x03, 0xef, 0xbd, 0x0c}, + {0x03, 0xef, 0xbd, 0x15}, + {0x03, 0xef, 0xbd, 0x0d}, + {0x03, 0xef, 0xbd, 0x1c}, + {0x03, 0xef, 0xbd, 0x02}, + {0x03, 0xef, 0xbd, 0x1f}, + {0x03, 0xef, 0xbd, 0x1d}, + {0x03, 0xef, 0xbd, 0x17}, + {0x03, 0xef, 0xbd, 0x08}, + {0x03, 0xef, 0xbd, 0x09}, + {0x03, 0xef, 0xbd, 0x0e}, + {0x03, 0xef, 0xbd, 0x04}, + {0x03, 0xef, 0xbd, 0x05}, + {0x03, 0xef, 0xbe, 0x3f}, + {0x03, 0xef, 0xbe, 0x00}, + {0x03, 0xef, 0xbd, 0x2c}, + {0x03, 0xef, 0xbe, 0x06}, + {0x03, 0xef, 0xbe, 0x0c}, + {0x03, 0xef, 0xbe, 0x0f}, + {0x03, 0xef, 0xbe, 0x0d}, + {0x03, 0xef, 0xbe, 0x0b}, + {0x03, 0xef, 0xbe, 0x19}, + {0x03, 0xef, 0xbe, 0x15}, + {0x03, 0xef, 0xbe, 0x11}, + {0x03, 0xef, 0xbe, 0x31}, + {0x03, 0xef, 0xbe, 0x33}, + {0x03, 0xef, 0xbd, 0x0f}, + {0x03, 0xef, 0xbe, 0x30}, + {0x03, 0xef, 0xbe, 0x3e}, + {0x03, 0xef, 0xbe, 0x32}, + {0x03, 0xef, 0xbe, 0x36}, + {0x03, 0xef, 0xbd, 0x14}, + {0x03, 0xef, 0xbe, 0x2e}, + {0x03, 0xef, 0xbd, 0x1e}, + {0x03, 0xef, 0xbe, 0x10}, + {0x03, 0xef, 0xbf, 0x13}, + {0x03, 0xef, 0xbf, 0x15}, + {0x03, 0xef, 0xbf, 0x17}, + {0x03, 0xef, 0xbf, 0x1f}, + {0x03, 0xef, 0xbf, 0x1d}, + {0x03, 0xef, 0xbf, 0x1b}, + {0x03, 0xef, 0xbf, 0x09}, + {0x03, 0xef, 0xbf, 0x0b}, + {0x03, 0xef, 0xbf, 0x37}, + {0x03, 0xef, 0xbe, 0x04}, + {0x01, 0xe0, 0x00, 0x00}, + {0x03, 0xe2, 0xa6, 0x1a}, + {0x03, 0xe2, 0xa6, 0x26}, + {0x03, 0xe3, 0x80, 0x23}, + {0x03, 0xe3, 0x80, 0x2e}, + {0x03, 0xe3, 0x80, 0x25}, + {0x03, 0xe3, 0x83, 0x1e}, + {0x03, 0xe3, 0x83, 0x14}, + {0x03, 0xe3, 0x82, 0x06}, + {0x03, 0xe3, 0x82, 0x0b}, + {0x03, 0xe3, 0x82, 0x0c}, + {0x03, 0xe3, 0x82, 0x0d}, + {0x03, 0xe3, 0x82, 0x02}, + {0x03, 0xe3, 0x83, 0x0f}, + {0x03, 0xe3, 0x83, 0x08}, + {0x03, 0xe3, 0x83, 0x09}, + {0x03, 0xe3, 0x83, 0x2c}, + {0x03, 0xe3, 0x83, 0x0c}, + {0x03, 0xe3, 0x82, 0x13}, + {0x03, 0xe3, 0x82, 0x16}, + {0x03, 0xe3, 0x82, 0x15}, + {0x03, 0xe3, 0x82, 0x1c}, + {0x03, 0xe3, 0x82, 0x1f}, + {0x03, 0xe3, 0x82, 0x1d}, + {0x03, 0xe3, 0x82, 0x1a}, + {0x03, 0xe3, 0x82, 0x17}, + {0x03, 0xe3, 0x82, 0x08}, + {0x03, 0xe3, 0x82, 0x09}, + {0x03, 0xe3, 0x82, 0x0e}, + {0x03, 0xe3, 0x82, 0x04}, + {0x03, 0xe3, 0x82, 0x05}, + {0x03, 0xe3, 0x82, 0x3f}, + {0x03, 0xe3, 0x83, 0x00}, + {0x03, 0xe3, 0x83, 0x06}, + {0x03, 0xe3, 0x83, 0x05}, + {0x03, 0xe3, 0x83, 0x0d}, + {0x03, 0xe3, 0x83, 0x0b}, + {0x03, 0xe3, 0x83, 0x07}, + {0x03, 0xe3, 0x83, 0x19}, + {0x03, 0xe3, 0x83, 0x15}, + {0x03, 0xe3, 0x83, 0x11}, + {0x03, 0xe3, 0x83, 0x31}, + {0x03, 0xe3, 0x83, 0x33}, + {0x03, 0xe3, 0x83, 0x30}, + {0x03, 0xe3, 0x83, 0x3e}, + {0x03, 0xe3, 0x83, 0x32}, + {0x03, 0xe3, 0x83, 0x36}, + {0x03, 0xe3, 0x83, 0x2e}, + {0x03, 0xe3, 0x82, 0x07}, + {0x03, 0xe3, 0x85, 0x04}, + {0x03, 0xe3, 0x84, 0x10}, + {0x03, 0xe3, 0x85, 0x30}, + {0x03, 0xe3, 0x85, 0x0d}, + {0x03, 0xe3, 0x85, 0x13}, + {0x03, 0xe3, 0x85, 0x15}, + {0x03, 0xe3, 0x85, 0x17}, + {0x03, 0xe3, 0x85, 0x1f}, + {0x03, 0xe3, 0x85, 0x1d}, + {0x03, 0xe3, 0x85, 0x1b}, + {0x03, 0xe3, 0x85, 0x09}, + {0x03, 0xe3, 0x85, 0x0f}, + {0x03, 0xe3, 0x85, 0x0b}, + {0x03, 0xe3, 0x85, 0x37}, + {0x03, 0xe3, 0x85, 0x3b}, + {0x03, 0xe3, 0x85, 0x39}, + {0x03, 0xe3, 0x85, 0x3f}, + {0x02, 0xc2, 0x02, 0x00}, + {0x02, 0xc2, 0x0e, 0x00}, + {0x02, 0xc2, 0x0c, 0x00}, + {0x02, 0xc2, 0x00, 0x00}, + {0x03, 0xe2, 0x82, 0x0f}, + {0x03, 0xe2, 0x94, 0x2a}, + {0x03, 0xe2, 0x86, 0x39}, + {0x03, 0xe2, 0x86, 0x3b}, + {0x03, 0xe2, 0x86, 0x3f}, + {0x03, 0xe2, 0x96, 0x0d}, + {0x03, 0xe2, 0x97, 0x25}, +} + +// Total table size 15448 bytes (15KiB) diff --git a/vendor/golang.org/x/text/width/tables15.0.0.go b/vendor/golang.org/x/text/width/tables15.0.0.go new file mode 100644 index 0000000000..4b91e3384d --- /dev/null +++ b/vendor/golang.org/x/text/width/tables15.0.0.go @@ -0,0 +1,1368 @@ +// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. + +//go:build go1.21 +// +build go1.21 + +package width + +// UnicodeVersion is the Unicode version from which the tables in this package are derived. +const UnicodeVersion = "15.0.0" + +// lookup returns the trie value for the first UTF-8 encoding in s and +// the width in bytes of this encoding. The size will be 0 if s does not +// hold enough bytes to complete the encoding. len(s) must be greater than 0. +func (t *widthTrie) lookup(s []byte) (v uint16, sz int) { + c0 := s[0] + switch { + case c0 < 0x80: // is ASCII + return widthValues[c0], 1 + case c0 < 0xC2: + return 0, 1 // Illegal UTF-8: not a starter, not ASCII. + case c0 < 0xE0: // 2-byte UTF-8 + if len(s) < 2 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c1), 2 + case c0 < 0xF0: // 3-byte UTF-8 + if len(s) < 3 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = widthIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c2), 3 + case c0 < 0xF8: // 4-byte UTF-8 + if len(s) < 4 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = widthIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + o = uint32(i)<<6 + uint32(c2) + i = widthIndex[o] + c3 := s[3] + if c3 < 0x80 || 0xC0 <= c3 { + return 0, 3 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c3), 4 + } + // Illegal rune + return 0, 1 +} + +// lookupUnsafe returns the trie value for the first UTF-8 encoding in s. +// s must start with a full and valid UTF-8 encoded rune. +func (t *widthTrie) lookupUnsafe(s []byte) uint16 { + c0 := s[0] + if c0 < 0x80 { // is ASCII + return widthValues[c0] + } + i := widthIndex[c0] + if c0 < 0xE0 { // 2-byte UTF-8 + return t.lookupValue(uint32(i), s[1]) + } + i = widthIndex[uint32(i)<<6+uint32(s[1])] + if c0 < 0xF0 { // 3-byte UTF-8 + return t.lookupValue(uint32(i), s[2]) + } + i = widthIndex[uint32(i)<<6+uint32(s[2])] + if c0 < 0xF8 { // 4-byte UTF-8 + return t.lookupValue(uint32(i), s[3]) + } + return 0 +} + +// lookupString returns the trie value for the first UTF-8 encoding in s and +// the width in bytes of this encoding. The size will be 0 if s does not +// hold enough bytes to complete the encoding. len(s) must be greater than 0. +func (t *widthTrie) lookupString(s string) (v uint16, sz int) { + c0 := s[0] + switch { + case c0 < 0x80: // is ASCII + return widthValues[c0], 1 + case c0 < 0xC2: + return 0, 1 // Illegal UTF-8: not a starter, not ASCII. + case c0 < 0xE0: // 2-byte UTF-8 + if len(s) < 2 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c1), 2 + case c0 < 0xF0: // 3-byte UTF-8 + if len(s) < 3 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = widthIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c2), 3 + case c0 < 0xF8: // 4-byte UTF-8 + if len(s) < 4 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = widthIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + o = uint32(i)<<6 + uint32(c2) + i = widthIndex[o] + c3 := s[3] + if c3 < 0x80 || 0xC0 <= c3 { + return 0, 3 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c3), 4 + } + // Illegal rune + return 0, 1 +} + +// lookupStringUnsafe returns the trie value for the first UTF-8 encoding in s. +// s must start with a full and valid UTF-8 encoded rune. +func (t *widthTrie) lookupStringUnsafe(s string) uint16 { + c0 := s[0] + if c0 < 0x80 { // is ASCII + return widthValues[c0] + } + i := widthIndex[c0] + if c0 < 0xE0 { // 2-byte UTF-8 + return t.lookupValue(uint32(i), s[1]) + } + i = widthIndex[uint32(i)<<6+uint32(s[1])] + if c0 < 0xF0 { // 3-byte UTF-8 + return t.lookupValue(uint32(i), s[2]) + } + i = widthIndex[uint32(i)<<6+uint32(s[2])] + if c0 < 0xF8 { // 4-byte UTF-8 + return t.lookupValue(uint32(i), s[3]) + } + return 0 +} + +// widthTrie. Total size: 14912 bytes (14.56 KiB). Checksum: 4468b6cd178303d2. +type widthTrie struct{} + +func newWidthTrie(i int) *widthTrie { + return &widthTrie{} +} + +// lookupValue determines the type of block n and looks up the value for b. +func (t *widthTrie) lookupValue(n uint32, b byte) uint16 { + switch { + default: + return uint16(widthValues[n<<6+uint32(b)]) + } +} + +// widthValues: 105 blocks, 6720 entries, 13440 bytes +// The third block is the zero block. +var widthValues = [6720]uint16{ + // Block 0x0, offset 0x0 + 0x20: 0x6001, 0x21: 0x6002, 0x22: 0x6002, 0x23: 0x6002, + 0x24: 0x6002, 0x25: 0x6002, 0x26: 0x6002, 0x27: 0x6002, 0x28: 0x6002, 0x29: 0x6002, + 0x2a: 0x6002, 0x2b: 0x6002, 0x2c: 0x6002, 0x2d: 0x6002, 0x2e: 0x6002, 0x2f: 0x6002, + 0x30: 0x6002, 0x31: 0x6002, 0x32: 0x6002, 0x33: 0x6002, 0x34: 0x6002, 0x35: 0x6002, + 0x36: 0x6002, 0x37: 0x6002, 0x38: 0x6002, 0x39: 0x6002, 0x3a: 0x6002, 0x3b: 0x6002, + 0x3c: 0x6002, 0x3d: 0x6002, 0x3e: 0x6002, 0x3f: 0x6002, + // Block 0x1, offset 0x40 + 0x40: 0x6003, 0x41: 0x6003, 0x42: 0x6003, 0x43: 0x6003, 0x44: 0x6003, 0x45: 0x6003, + 0x46: 0x6003, 0x47: 0x6003, 0x48: 0x6003, 0x49: 0x6003, 0x4a: 0x6003, 0x4b: 0x6003, + 0x4c: 0x6003, 0x4d: 0x6003, 0x4e: 0x6003, 0x4f: 0x6003, 0x50: 0x6003, 0x51: 0x6003, + 0x52: 0x6003, 0x53: 0x6003, 0x54: 0x6003, 0x55: 0x6003, 0x56: 0x6003, 0x57: 0x6003, + 0x58: 0x6003, 0x59: 0x6003, 0x5a: 0x6003, 0x5b: 0x6003, 0x5c: 0x6003, 0x5d: 0x6003, + 0x5e: 0x6003, 0x5f: 0x6003, 0x60: 0x6004, 0x61: 0x6004, 0x62: 0x6004, 0x63: 0x6004, + 0x64: 0x6004, 0x65: 0x6004, 0x66: 0x6004, 0x67: 0x6004, 0x68: 0x6004, 0x69: 0x6004, + 0x6a: 0x6004, 0x6b: 0x6004, 0x6c: 0x6004, 0x6d: 0x6004, 0x6e: 0x6004, 0x6f: 0x6004, + 0x70: 0x6004, 0x71: 0x6004, 0x72: 0x6004, 0x73: 0x6004, 0x74: 0x6004, 0x75: 0x6004, + 0x76: 0x6004, 0x77: 0x6004, 0x78: 0x6004, 0x79: 0x6004, 0x7a: 0x6004, 0x7b: 0x6004, + 0x7c: 0x6004, 0x7d: 0x6004, 0x7e: 0x6004, + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0xe1: 0x2000, 0xe2: 0x6005, 0xe3: 0x6005, + 0xe4: 0x2000, 0xe5: 0x6006, 0xe6: 0x6005, 0xe7: 0x2000, 0xe8: 0x2000, + 0xea: 0x2000, 0xec: 0x6007, 0xed: 0x2000, 0xee: 0x2000, 0xef: 0x6008, + 0xf0: 0x2000, 0xf1: 0x2000, 0xf2: 0x2000, 0xf3: 0x2000, 0xf4: 0x2000, + 0xf6: 0x2000, 0xf7: 0x2000, 0xf8: 0x2000, 0xf9: 0x2000, 0xfa: 0x2000, + 0xfc: 0x2000, 0xfd: 0x2000, 0xfe: 0x2000, 0xff: 0x2000, + // Block 0x4, offset 0x100 + 0x106: 0x2000, + 0x110: 0x2000, + 0x117: 0x2000, + 0x118: 0x2000, + 0x11e: 0x2000, 0x11f: 0x2000, 0x120: 0x2000, 0x121: 0x2000, + 0x126: 0x2000, 0x128: 0x2000, 0x129: 0x2000, + 0x12a: 0x2000, 0x12c: 0x2000, 0x12d: 0x2000, + 0x130: 0x2000, 0x132: 0x2000, 0x133: 0x2000, + 0x137: 0x2000, 0x138: 0x2000, 0x139: 0x2000, 0x13a: 0x2000, + 0x13c: 0x2000, 0x13e: 0x2000, + // Block 0x5, offset 0x140 + 0x141: 0x2000, + 0x151: 0x2000, + 0x153: 0x2000, + 0x15b: 0x2000, + 0x166: 0x2000, 0x167: 0x2000, + 0x16b: 0x2000, + 0x171: 0x2000, 0x172: 0x2000, 0x173: 0x2000, + 0x178: 0x2000, + 0x17f: 0x2000, + // Block 0x6, offset 0x180 + 0x180: 0x2000, 0x181: 0x2000, 0x182: 0x2000, 0x184: 0x2000, + 0x188: 0x2000, 0x189: 0x2000, 0x18a: 0x2000, 0x18b: 0x2000, + 0x18d: 0x2000, + 0x192: 0x2000, 0x193: 0x2000, + 0x1a6: 0x2000, 0x1a7: 0x2000, + 0x1ab: 0x2000, + // Block 0x7, offset 0x1c0 + 0x1ce: 0x2000, 0x1d0: 0x2000, + 0x1d2: 0x2000, 0x1d4: 0x2000, 0x1d6: 0x2000, + 0x1d8: 0x2000, 0x1da: 0x2000, 0x1dc: 0x2000, + // Block 0x8, offset 0x200 + 0x211: 0x2000, + 0x221: 0x2000, + // Block 0x9, offset 0x240 + 0x244: 0x2000, + 0x247: 0x2000, 0x249: 0x2000, 0x24a: 0x2000, 0x24b: 0x2000, + 0x24d: 0x2000, 0x250: 0x2000, + 0x258: 0x2000, 0x259: 0x2000, 0x25a: 0x2000, 0x25b: 0x2000, 0x25d: 0x2000, + 0x25f: 0x2000, + // Block 0xa, offset 0x280 + 0x280: 0x2000, 0x281: 0x2000, 0x282: 0x2000, 0x283: 0x2000, 0x284: 0x2000, 0x285: 0x2000, + 0x286: 0x2000, 0x287: 0x2000, 0x288: 0x2000, 0x289: 0x2000, 0x28a: 0x2000, 0x28b: 0x2000, + 0x28c: 0x2000, 0x28d: 0x2000, 0x28e: 0x2000, 0x28f: 0x2000, 0x290: 0x2000, 0x291: 0x2000, + 0x292: 0x2000, 0x293: 0x2000, 0x294: 0x2000, 0x295: 0x2000, 0x296: 0x2000, 0x297: 0x2000, + 0x298: 0x2000, 0x299: 0x2000, 0x29a: 0x2000, 0x29b: 0x2000, 0x29c: 0x2000, 0x29d: 0x2000, + 0x29e: 0x2000, 0x29f: 0x2000, 0x2a0: 0x2000, 0x2a1: 0x2000, 0x2a2: 0x2000, 0x2a3: 0x2000, + 0x2a4: 0x2000, 0x2a5: 0x2000, 0x2a6: 0x2000, 0x2a7: 0x2000, 0x2a8: 0x2000, 0x2a9: 0x2000, + 0x2aa: 0x2000, 0x2ab: 0x2000, 0x2ac: 0x2000, 0x2ad: 0x2000, 0x2ae: 0x2000, 0x2af: 0x2000, + 0x2b0: 0x2000, 0x2b1: 0x2000, 0x2b2: 0x2000, 0x2b3: 0x2000, 0x2b4: 0x2000, 0x2b5: 0x2000, + 0x2b6: 0x2000, 0x2b7: 0x2000, 0x2b8: 0x2000, 0x2b9: 0x2000, 0x2ba: 0x2000, 0x2bb: 0x2000, + 0x2bc: 0x2000, 0x2bd: 0x2000, 0x2be: 0x2000, 0x2bf: 0x2000, + // Block 0xb, offset 0x2c0 + 0x2c0: 0x2000, 0x2c1: 0x2000, 0x2c2: 0x2000, 0x2c3: 0x2000, 0x2c4: 0x2000, 0x2c5: 0x2000, + 0x2c6: 0x2000, 0x2c7: 0x2000, 0x2c8: 0x2000, 0x2c9: 0x2000, 0x2ca: 0x2000, 0x2cb: 0x2000, + 0x2cc: 0x2000, 0x2cd: 0x2000, 0x2ce: 0x2000, 0x2cf: 0x2000, 0x2d0: 0x2000, 0x2d1: 0x2000, + 0x2d2: 0x2000, 0x2d3: 0x2000, 0x2d4: 0x2000, 0x2d5: 0x2000, 0x2d6: 0x2000, 0x2d7: 0x2000, + 0x2d8: 0x2000, 0x2d9: 0x2000, 0x2da: 0x2000, 0x2db: 0x2000, 0x2dc: 0x2000, 0x2dd: 0x2000, + 0x2de: 0x2000, 0x2df: 0x2000, 0x2e0: 0x2000, 0x2e1: 0x2000, 0x2e2: 0x2000, 0x2e3: 0x2000, + 0x2e4: 0x2000, 0x2e5: 0x2000, 0x2e6: 0x2000, 0x2e7: 0x2000, 0x2e8: 0x2000, 0x2e9: 0x2000, + 0x2ea: 0x2000, 0x2eb: 0x2000, 0x2ec: 0x2000, 0x2ed: 0x2000, 0x2ee: 0x2000, 0x2ef: 0x2000, + // Block 0xc, offset 0x300 + 0x311: 0x2000, + 0x312: 0x2000, 0x313: 0x2000, 0x314: 0x2000, 0x315: 0x2000, 0x316: 0x2000, 0x317: 0x2000, + 0x318: 0x2000, 0x319: 0x2000, 0x31a: 0x2000, 0x31b: 0x2000, 0x31c: 0x2000, 0x31d: 0x2000, + 0x31e: 0x2000, 0x31f: 0x2000, 0x320: 0x2000, 0x321: 0x2000, 0x323: 0x2000, + 0x324: 0x2000, 0x325: 0x2000, 0x326: 0x2000, 0x327: 0x2000, 0x328: 0x2000, 0x329: 0x2000, + 0x331: 0x2000, 0x332: 0x2000, 0x333: 0x2000, 0x334: 0x2000, 0x335: 0x2000, + 0x336: 0x2000, 0x337: 0x2000, 0x338: 0x2000, 0x339: 0x2000, 0x33a: 0x2000, 0x33b: 0x2000, + 0x33c: 0x2000, 0x33d: 0x2000, 0x33e: 0x2000, 0x33f: 0x2000, + // Block 0xd, offset 0x340 + 0x340: 0x2000, 0x341: 0x2000, 0x343: 0x2000, 0x344: 0x2000, 0x345: 0x2000, + 0x346: 0x2000, 0x347: 0x2000, 0x348: 0x2000, 0x349: 0x2000, + // Block 0xe, offset 0x380 + 0x381: 0x2000, + 0x390: 0x2000, 0x391: 0x2000, + 0x392: 0x2000, 0x393: 0x2000, 0x394: 0x2000, 0x395: 0x2000, 0x396: 0x2000, 0x397: 0x2000, + 0x398: 0x2000, 0x399: 0x2000, 0x39a: 0x2000, 0x39b: 0x2000, 0x39c: 0x2000, 0x39d: 0x2000, + 0x39e: 0x2000, 0x39f: 0x2000, 0x3a0: 0x2000, 0x3a1: 0x2000, 0x3a2: 0x2000, 0x3a3: 0x2000, + 0x3a4: 0x2000, 0x3a5: 0x2000, 0x3a6: 0x2000, 0x3a7: 0x2000, 0x3a8: 0x2000, 0x3a9: 0x2000, + 0x3aa: 0x2000, 0x3ab: 0x2000, 0x3ac: 0x2000, 0x3ad: 0x2000, 0x3ae: 0x2000, 0x3af: 0x2000, + 0x3b0: 0x2000, 0x3b1: 0x2000, 0x3b2: 0x2000, 0x3b3: 0x2000, 0x3b4: 0x2000, 0x3b5: 0x2000, + 0x3b6: 0x2000, 0x3b7: 0x2000, 0x3b8: 0x2000, 0x3b9: 0x2000, 0x3ba: 0x2000, 0x3bb: 0x2000, + 0x3bc: 0x2000, 0x3bd: 0x2000, 0x3be: 0x2000, 0x3bf: 0x2000, + // Block 0xf, offset 0x3c0 + 0x3c0: 0x2000, 0x3c1: 0x2000, 0x3c2: 0x2000, 0x3c3: 0x2000, 0x3c4: 0x2000, 0x3c5: 0x2000, + 0x3c6: 0x2000, 0x3c7: 0x2000, 0x3c8: 0x2000, 0x3c9: 0x2000, 0x3ca: 0x2000, 0x3cb: 0x2000, + 0x3cc: 0x2000, 0x3cd: 0x2000, 0x3ce: 0x2000, 0x3cf: 0x2000, 0x3d1: 0x2000, + // Block 0x10, offset 0x400 + 0x400: 0x4000, 0x401: 0x4000, 0x402: 0x4000, 0x403: 0x4000, 0x404: 0x4000, 0x405: 0x4000, + 0x406: 0x4000, 0x407: 0x4000, 0x408: 0x4000, 0x409: 0x4000, 0x40a: 0x4000, 0x40b: 0x4000, + 0x40c: 0x4000, 0x40d: 0x4000, 0x40e: 0x4000, 0x40f: 0x4000, 0x410: 0x4000, 0x411: 0x4000, + 0x412: 0x4000, 0x413: 0x4000, 0x414: 0x4000, 0x415: 0x4000, 0x416: 0x4000, 0x417: 0x4000, + 0x418: 0x4000, 0x419: 0x4000, 0x41a: 0x4000, 0x41b: 0x4000, 0x41c: 0x4000, 0x41d: 0x4000, + 0x41e: 0x4000, 0x41f: 0x4000, 0x420: 0x4000, 0x421: 0x4000, 0x422: 0x4000, 0x423: 0x4000, + 0x424: 0x4000, 0x425: 0x4000, 0x426: 0x4000, 0x427: 0x4000, 0x428: 0x4000, 0x429: 0x4000, + 0x42a: 0x4000, 0x42b: 0x4000, 0x42c: 0x4000, 0x42d: 0x4000, 0x42e: 0x4000, 0x42f: 0x4000, + 0x430: 0x4000, 0x431: 0x4000, 0x432: 0x4000, 0x433: 0x4000, 0x434: 0x4000, 0x435: 0x4000, + 0x436: 0x4000, 0x437: 0x4000, 0x438: 0x4000, 0x439: 0x4000, 0x43a: 0x4000, 0x43b: 0x4000, + 0x43c: 0x4000, 0x43d: 0x4000, 0x43e: 0x4000, 0x43f: 0x4000, + // Block 0x11, offset 0x440 + 0x440: 0x4000, 0x441: 0x4000, 0x442: 0x4000, 0x443: 0x4000, 0x444: 0x4000, 0x445: 0x4000, + 0x446: 0x4000, 0x447: 0x4000, 0x448: 0x4000, 0x449: 0x4000, 0x44a: 0x4000, 0x44b: 0x4000, + 0x44c: 0x4000, 0x44d: 0x4000, 0x44e: 0x4000, 0x44f: 0x4000, 0x450: 0x4000, 0x451: 0x4000, + 0x452: 0x4000, 0x453: 0x4000, 0x454: 0x4000, 0x455: 0x4000, 0x456: 0x4000, 0x457: 0x4000, + 0x458: 0x4000, 0x459: 0x4000, 0x45a: 0x4000, 0x45b: 0x4000, 0x45c: 0x4000, 0x45d: 0x4000, + 0x45e: 0x4000, 0x45f: 0x4000, + // Block 0x12, offset 0x480 + 0x490: 0x2000, + 0x493: 0x2000, 0x494: 0x2000, 0x495: 0x2000, 0x496: 0x2000, + 0x498: 0x2000, 0x499: 0x2000, 0x49c: 0x2000, 0x49d: 0x2000, + 0x4a0: 0x2000, 0x4a1: 0x2000, 0x4a2: 0x2000, + 0x4a4: 0x2000, 0x4a5: 0x2000, 0x4a6: 0x2000, 0x4a7: 0x2000, + 0x4b0: 0x2000, 0x4b2: 0x2000, 0x4b3: 0x2000, 0x4b5: 0x2000, + 0x4bb: 0x2000, + 0x4be: 0x2000, + // Block 0x13, offset 0x4c0 + 0x4f4: 0x2000, + 0x4ff: 0x2000, + // Block 0x14, offset 0x500 + 0x501: 0x2000, 0x502: 0x2000, 0x503: 0x2000, 0x504: 0x2000, + 0x529: 0xa009, + 0x52c: 0x2000, + // Block 0x15, offset 0x540 + 0x543: 0x2000, 0x545: 0x2000, + 0x549: 0x2000, + 0x553: 0x2000, 0x556: 0x2000, + 0x561: 0x2000, 0x562: 0x2000, + 0x566: 0x2000, + 0x56b: 0x2000, + // Block 0x16, offset 0x580 + 0x593: 0x2000, 0x594: 0x2000, + 0x59b: 0x2000, 0x59c: 0x2000, 0x59d: 0x2000, + 0x59e: 0x2000, 0x5a0: 0x2000, 0x5a1: 0x2000, 0x5a2: 0x2000, 0x5a3: 0x2000, + 0x5a4: 0x2000, 0x5a5: 0x2000, 0x5a6: 0x2000, 0x5a7: 0x2000, 0x5a8: 0x2000, 0x5a9: 0x2000, + 0x5aa: 0x2000, 0x5ab: 0x2000, + 0x5b0: 0x2000, 0x5b1: 0x2000, 0x5b2: 0x2000, 0x5b3: 0x2000, 0x5b4: 0x2000, 0x5b5: 0x2000, + 0x5b6: 0x2000, 0x5b7: 0x2000, 0x5b8: 0x2000, 0x5b9: 0x2000, + // Block 0x17, offset 0x5c0 + 0x5c9: 0x2000, + 0x5d0: 0x200a, 0x5d1: 0x200b, + 0x5d2: 0x200a, 0x5d3: 0x200c, 0x5d4: 0x2000, 0x5d5: 0x2000, 0x5d6: 0x2000, 0x5d7: 0x2000, + 0x5d8: 0x2000, 0x5d9: 0x2000, + 0x5f8: 0x2000, 0x5f9: 0x2000, + // Block 0x18, offset 0x600 + 0x612: 0x2000, 0x614: 0x2000, + 0x627: 0x2000, + // Block 0x19, offset 0x640 + 0x640: 0x2000, 0x642: 0x2000, 0x643: 0x2000, + 0x647: 0x2000, 0x648: 0x2000, 0x64b: 0x2000, + 0x64f: 0x2000, 0x651: 0x2000, + 0x655: 0x2000, + 0x65a: 0x2000, 0x65d: 0x2000, + 0x65e: 0x2000, 0x65f: 0x2000, 0x660: 0x2000, 0x663: 0x2000, + 0x665: 0x2000, 0x667: 0x2000, 0x668: 0x2000, 0x669: 0x2000, + 0x66a: 0x2000, 0x66b: 0x2000, 0x66c: 0x2000, 0x66e: 0x2000, + 0x674: 0x2000, 0x675: 0x2000, + 0x676: 0x2000, 0x677: 0x2000, + 0x67c: 0x2000, 0x67d: 0x2000, + // Block 0x1a, offset 0x680 + 0x688: 0x2000, + 0x68c: 0x2000, + 0x692: 0x2000, + 0x6a0: 0x2000, 0x6a1: 0x2000, + 0x6a4: 0x2000, 0x6a5: 0x2000, 0x6a6: 0x2000, 0x6a7: 0x2000, + 0x6aa: 0x2000, 0x6ab: 0x2000, 0x6ae: 0x2000, 0x6af: 0x2000, + // Block 0x1b, offset 0x6c0 + 0x6c2: 0x2000, 0x6c3: 0x2000, + 0x6c6: 0x2000, 0x6c7: 0x2000, + 0x6d5: 0x2000, + 0x6d9: 0x2000, + 0x6e5: 0x2000, + 0x6ff: 0x2000, + // Block 0x1c, offset 0x700 + 0x712: 0x2000, + 0x71a: 0x4000, 0x71b: 0x4000, + 0x729: 0x4000, + 0x72a: 0x4000, + // Block 0x1d, offset 0x740 + 0x769: 0x4000, + 0x76a: 0x4000, 0x76b: 0x4000, 0x76c: 0x4000, + 0x770: 0x4000, 0x773: 0x4000, + // Block 0x1e, offset 0x780 + 0x7a0: 0x2000, 0x7a1: 0x2000, 0x7a2: 0x2000, 0x7a3: 0x2000, + 0x7a4: 0x2000, 0x7a5: 0x2000, 0x7a6: 0x2000, 0x7a7: 0x2000, 0x7a8: 0x2000, 0x7a9: 0x2000, + 0x7aa: 0x2000, 0x7ab: 0x2000, 0x7ac: 0x2000, 0x7ad: 0x2000, 0x7ae: 0x2000, 0x7af: 0x2000, + 0x7b0: 0x2000, 0x7b1: 0x2000, 0x7b2: 0x2000, 0x7b3: 0x2000, 0x7b4: 0x2000, 0x7b5: 0x2000, + 0x7b6: 0x2000, 0x7b7: 0x2000, 0x7b8: 0x2000, 0x7b9: 0x2000, 0x7ba: 0x2000, 0x7bb: 0x2000, + 0x7bc: 0x2000, 0x7bd: 0x2000, 0x7be: 0x2000, 0x7bf: 0x2000, + // Block 0x1f, offset 0x7c0 + 0x7c0: 0x2000, 0x7c1: 0x2000, 0x7c2: 0x2000, 0x7c3: 0x2000, 0x7c4: 0x2000, 0x7c5: 0x2000, + 0x7c6: 0x2000, 0x7c7: 0x2000, 0x7c8: 0x2000, 0x7c9: 0x2000, 0x7ca: 0x2000, 0x7cb: 0x2000, + 0x7cc: 0x2000, 0x7cd: 0x2000, 0x7ce: 0x2000, 0x7cf: 0x2000, 0x7d0: 0x2000, 0x7d1: 0x2000, + 0x7d2: 0x2000, 0x7d3: 0x2000, 0x7d4: 0x2000, 0x7d5: 0x2000, 0x7d6: 0x2000, 0x7d7: 0x2000, + 0x7d8: 0x2000, 0x7d9: 0x2000, 0x7da: 0x2000, 0x7db: 0x2000, 0x7dc: 0x2000, 0x7dd: 0x2000, + 0x7de: 0x2000, 0x7df: 0x2000, 0x7e0: 0x2000, 0x7e1: 0x2000, 0x7e2: 0x2000, 0x7e3: 0x2000, + 0x7e4: 0x2000, 0x7e5: 0x2000, 0x7e6: 0x2000, 0x7e7: 0x2000, 0x7e8: 0x2000, 0x7e9: 0x2000, + 0x7eb: 0x2000, 0x7ec: 0x2000, 0x7ed: 0x2000, 0x7ee: 0x2000, 0x7ef: 0x2000, + 0x7f0: 0x2000, 0x7f1: 0x2000, 0x7f2: 0x2000, 0x7f3: 0x2000, 0x7f4: 0x2000, 0x7f5: 0x2000, + 0x7f6: 0x2000, 0x7f7: 0x2000, 0x7f8: 0x2000, 0x7f9: 0x2000, 0x7fa: 0x2000, 0x7fb: 0x2000, + 0x7fc: 0x2000, 0x7fd: 0x2000, 0x7fe: 0x2000, 0x7ff: 0x2000, + // Block 0x20, offset 0x800 + 0x800: 0x2000, 0x801: 0x2000, 0x802: 0x200d, 0x803: 0x2000, 0x804: 0x2000, 0x805: 0x2000, + 0x806: 0x2000, 0x807: 0x2000, 0x808: 0x2000, 0x809: 0x2000, 0x80a: 0x2000, 0x80b: 0x2000, + 0x80c: 0x2000, 0x80d: 0x2000, 0x80e: 0x2000, 0x80f: 0x2000, 0x810: 0x2000, 0x811: 0x2000, + 0x812: 0x2000, 0x813: 0x2000, 0x814: 0x2000, 0x815: 0x2000, 0x816: 0x2000, 0x817: 0x2000, + 0x818: 0x2000, 0x819: 0x2000, 0x81a: 0x2000, 0x81b: 0x2000, 0x81c: 0x2000, 0x81d: 0x2000, + 0x81e: 0x2000, 0x81f: 0x2000, 0x820: 0x2000, 0x821: 0x2000, 0x822: 0x2000, 0x823: 0x2000, + 0x824: 0x2000, 0x825: 0x2000, 0x826: 0x2000, 0x827: 0x2000, 0x828: 0x2000, 0x829: 0x2000, + 0x82a: 0x2000, 0x82b: 0x2000, 0x82c: 0x2000, 0x82d: 0x2000, 0x82e: 0x2000, 0x82f: 0x2000, + 0x830: 0x2000, 0x831: 0x2000, 0x832: 0x2000, 0x833: 0x2000, 0x834: 0x2000, 0x835: 0x2000, + 0x836: 0x2000, 0x837: 0x2000, 0x838: 0x2000, 0x839: 0x2000, 0x83a: 0x2000, 0x83b: 0x2000, + 0x83c: 0x2000, 0x83d: 0x2000, 0x83e: 0x2000, 0x83f: 0x2000, + // Block 0x21, offset 0x840 + 0x840: 0x2000, 0x841: 0x2000, 0x842: 0x2000, 0x843: 0x2000, 0x844: 0x2000, 0x845: 0x2000, + 0x846: 0x2000, 0x847: 0x2000, 0x848: 0x2000, 0x849: 0x2000, 0x84a: 0x2000, 0x84b: 0x2000, + 0x850: 0x2000, 0x851: 0x2000, + 0x852: 0x2000, 0x853: 0x2000, 0x854: 0x2000, 0x855: 0x2000, 0x856: 0x2000, 0x857: 0x2000, + 0x858: 0x2000, 0x859: 0x2000, 0x85a: 0x2000, 0x85b: 0x2000, 0x85c: 0x2000, 0x85d: 0x2000, + 0x85e: 0x2000, 0x85f: 0x2000, 0x860: 0x2000, 0x861: 0x2000, 0x862: 0x2000, 0x863: 0x2000, + 0x864: 0x2000, 0x865: 0x2000, 0x866: 0x2000, 0x867: 0x2000, 0x868: 0x2000, 0x869: 0x2000, + 0x86a: 0x2000, 0x86b: 0x2000, 0x86c: 0x2000, 0x86d: 0x2000, 0x86e: 0x2000, 0x86f: 0x2000, + 0x870: 0x2000, 0x871: 0x2000, 0x872: 0x2000, 0x873: 0x2000, + // Block 0x22, offset 0x880 + 0x880: 0x2000, 0x881: 0x2000, 0x882: 0x2000, 0x883: 0x2000, 0x884: 0x2000, 0x885: 0x2000, + 0x886: 0x2000, 0x887: 0x2000, 0x888: 0x2000, 0x889: 0x2000, 0x88a: 0x2000, 0x88b: 0x2000, + 0x88c: 0x2000, 0x88d: 0x2000, 0x88e: 0x2000, 0x88f: 0x2000, + 0x892: 0x2000, 0x893: 0x2000, 0x894: 0x2000, 0x895: 0x2000, + 0x8a0: 0x200e, 0x8a1: 0x2000, 0x8a3: 0x2000, + 0x8a4: 0x2000, 0x8a5: 0x2000, 0x8a6: 0x2000, 0x8a7: 0x2000, 0x8a8: 0x2000, 0x8a9: 0x2000, + 0x8b2: 0x2000, 0x8b3: 0x2000, + 0x8b6: 0x2000, 0x8b7: 0x2000, + 0x8bc: 0x2000, 0x8bd: 0x2000, + // Block 0x23, offset 0x8c0 + 0x8c0: 0x2000, 0x8c1: 0x2000, + 0x8c6: 0x2000, 0x8c7: 0x2000, 0x8c8: 0x2000, 0x8cb: 0x200f, + 0x8ce: 0x2000, 0x8cf: 0x2000, 0x8d0: 0x2000, 0x8d1: 0x2000, + 0x8e2: 0x2000, 0x8e3: 0x2000, + 0x8e4: 0x2000, 0x8e5: 0x2000, + 0x8ef: 0x2000, + 0x8fd: 0x4000, 0x8fe: 0x4000, + // Block 0x24, offset 0x900 + 0x905: 0x2000, + 0x906: 0x2000, 0x909: 0x2000, + 0x90e: 0x2000, 0x90f: 0x2000, + 0x914: 0x4000, 0x915: 0x4000, + 0x91c: 0x2000, + 0x91e: 0x2000, + // Block 0x25, offset 0x940 + 0x940: 0x2000, 0x942: 0x2000, + 0x948: 0x4000, 0x949: 0x4000, 0x94a: 0x4000, 0x94b: 0x4000, + 0x94c: 0x4000, 0x94d: 0x4000, 0x94e: 0x4000, 0x94f: 0x4000, 0x950: 0x4000, 0x951: 0x4000, + 0x952: 0x4000, 0x953: 0x4000, + 0x960: 0x2000, 0x961: 0x2000, 0x963: 0x2000, + 0x964: 0x2000, 0x965: 0x2000, 0x967: 0x2000, 0x968: 0x2000, 0x969: 0x2000, + 0x96a: 0x2000, 0x96c: 0x2000, 0x96d: 0x2000, 0x96f: 0x2000, + 0x97f: 0x4000, + // Block 0x26, offset 0x980 + 0x993: 0x4000, + 0x99e: 0x2000, 0x99f: 0x2000, 0x9a1: 0x4000, + 0x9aa: 0x4000, 0x9ab: 0x4000, + 0x9bd: 0x4000, 0x9be: 0x4000, 0x9bf: 0x2000, + // Block 0x27, offset 0x9c0 + 0x9c4: 0x4000, 0x9c5: 0x4000, + 0x9c6: 0x2000, 0x9c7: 0x2000, 0x9c8: 0x2000, 0x9c9: 0x2000, 0x9ca: 0x2000, 0x9cb: 0x2000, + 0x9cc: 0x2000, 0x9cd: 0x2000, 0x9ce: 0x4000, 0x9cf: 0x2000, 0x9d0: 0x2000, 0x9d1: 0x2000, + 0x9d2: 0x2000, 0x9d3: 0x2000, 0x9d4: 0x4000, 0x9d5: 0x2000, 0x9d6: 0x2000, 0x9d7: 0x2000, + 0x9d8: 0x2000, 0x9d9: 0x2000, 0x9da: 0x2000, 0x9db: 0x2000, 0x9dc: 0x2000, 0x9dd: 0x2000, + 0x9de: 0x2000, 0x9df: 0x2000, 0x9e0: 0x2000, 0x9e1: 0x2000, 0x9e3: 0x2000, + 0x9e8: 0x2000, 0x9e9: 0x2000, + 0x9ea: 0x4000, 0x9eb: 0x2000, 0x9ec: 0x2000, 0x9ed: 0x2000, 0x9ee: 0x2000, 0x9ef: 0x2000, + 0x9f0: 0x2000, 0x9f1: 0x2000, 0x9f2: 0x4000, 0x9f3: 0x4000, 0x9f4: 0x2000, 0x9f5: 0x4000, + 0x9f6: 0x2000, 0x9f7: 0x2000, 0x9f8: 0x2000, 0x9f9: 0x2000, 0x9fa: 0x4000, 0x9fb: 0x2000, + 0x9fc: 0x2000, 0x9fd: 0x4000, 0x9fe: 0x2000, 0x9ff: 0x2000, + // Block 0x28, offset 0xa00 + 0xa05: 0x4000, + 0xa0a: 0x4000, 0xa0b: 0x4000, + 0xa28: 0x4000, + 0xa3d: 0x2000, + // Block 0x29, offset 0xa40 + 0xa4c: 0x4000, 0xa4e: 0x4000, + 0xa53: 0x4000, 0xa54: 0x4000, 0xa55: 0x4000, 0xa57: 0x4000, + 0xa76: 0x2000, 0xa77: 0x2000, 0xa78: 0x2000, 0xa79: 0x2000, 0xa7a: 0x2000, 0xa7b: 0x2000, + 0xa7c: 0x2000, 0xa7d: 0x2000, 0xa7e: 0x2000, 0xa7f: 0x2000, + // Block 0x2a, offset 0xa80 + 0xa95: 0x4000, 0xa96: 0x4000, 0xa97: 0x4000, + 0xab0: 0x4000, + 0xabf: 0x4000, + // Block 0x2b, offset 0xac0 + 0xae6: 0x6000, 0xae7: 0x6000, 0xae8: 0x6000, 0xae9: 0x6000, + 0xaea: 0x6000, 0xaeb: 0x6000, 0xaec: 0x6000, 0xaed: 0x6000, + // Block 0x2c, offset 0xb00 + 0xb05: 0x6010, + 0xb06: 0x6011, + // Block 0x2d, offset 0xb40 + 0xb5b: 0x4000, 0xb5c: 0x4000, + // Block 0x2e, offset 0xb80 + 0xb90: 0x4000, + 0xb95: 0x4000, 0xb96: 0x2000, 0xb97: 0x2000, + 0xb98: 0x2000, 0xb99: 0x2000, + // Block 0x2f, offset 0xbc0 + 0xbc0: 0x4000, 0xbc1: 0x4000, 0xbc2: 0x4000, 0xbc3: 0x4000, 0xbc4: 0x4000, 0xbc5: 0x4000, + 0xbc6: 0x4000, 0xbc7: 0x4000, 0xbc8: 0x4000, 0xbc9: 0x4000, 0xbca: 0x4000, 0xbcb: 0x4000, + 0xbcc: 0x4000, 0xbcd: 0x4000, 0xbce: 0x4000, 0xbcf: 0x4000, 0xbd0: 0x4000, 0xbd1: 0x4000, + 0xbd2: 0x4000, 0xbd3: 0x4000, 0xbd4: 0x4000, 0xbd5: 0x4000, 0xbd6: 0x4000, 0xbd7: 0x4000, + 0xbd8: 0x4000, 0xbd9: 0x4000, 0xbdb: 0x4000, 0xbdc: 0x4000, 0xbdd: 0x4000, + 0xbde: 0x4000, 0xbdf: 0x4000, 0xbe0: 0x4000, 0xbe1: 0x4000, 0xbe2: 0x4000, 0xbe3: 0x4000, + 0xbe4: 0x4000, 0xbe5: 0x4000, 0xbe6: 0x4000, 0xbe7: 0x4000, 0xbe8: 0x4000, 0xbe9: 0x4000, + 0xbea: 0x4000, 0xbeb: 0x4000, 0xbec: 0x4000, 0xbed: 0x4000, 0xbee: 0x4000, 0xbef: 0x4000, + 0xbf0: 0x4000, 0xbf1: 0x4000, 0xbf2: 0x4000, 0xbf3: 0x4000, 0xbf4: 0x4000, 0xbf5: 0x4000, + 0xbf6: 0x4000, 0xbf7: 0x4000, 0xbf8: 0x4000, 0xbf9: 0x4000, 0xbfa: 0x4000, 0xbfb: 0x4000, + 0xbfc: 0x4000, 0xbfd: 0x4000, 0xbfe: 0x4000, 0xbff: 0x4000, + // Block 0x30, offset 0xc00 + 0xc00: 0x4000, 0xc01: 0x4000, 0xc02: 0x4000, 0xc03: 0x4000, 0xc04: 0x4000, 0xc05: 0x4000, + 0xc06: 0x4000, 0xc07: 0x4000, 0xc08: 0x4000, 0xc09: 0x4000, 0xc0a: 0x4000, 0xc0b: 0x4000, + 0xc0c: 0x4000, 0xc0d: 0x4000, 0xc0e: 0x4000, 0xc0f: 0x4000, 0xc10: 0x4000, 0xc11: 0x4000, + 0xc12: 0x4000, 0xc13: 0x4000, 0xc14: 0x4000, 0xc15: 0x4000, 0xc16: 0x4000, 0xc17: 0x4000, + 0xc18: 0x4000, 0xc19: 0x4000, 0xc1a: 0x4000, 0xc1b: 0x4000, 0xc1c: 0x4000, 0xc1d: 0x4000, + 0xc1e: 0x4000, 0xc1f: 0x4000, 0xc20: 0x4000, 0xc21: 0x4000, 0xc22: 0x4000, 0xc23: 0x4000, + 0xc24: 0x4000, 0xc25: 0x4000, 0xc26: 0x4000, 0xc27: 0x4000, 0xc28: 0x4000, 0xc29: 0x4000, + 0xc2a: 0x4000, 0xc2b: 0x4000, 0xc2c: 0x4000, 0xc2d: 0x4000, 0xc2e: 0x4000, 0xc2f: 0x4000, + 0xc30: 0x4000, 0xc31: 0x4000, 0xc32: 0x4000, 0xc33: 0x4000, + // Block 0x31, offset 0xc40 + 0xc40: 0x4000, 0xc41: 0x4000, 0xc42: 0x4000, 0xc43: 0x4000, 0xc44: 0x4000, 0xc45: 0x4000, + 0xc46: 0x4000, 0xc47: 0x4000, 0xc48: 0x4000, 0xc49: 0x4000, 0xc4a: 0x4000, 0xc4b: 0x4000, + 0xc4c: 0x4000, 0xc4d: 0x4000, 0xc4e: 0x4000, 0xc4f: 0x4000, 0xc50: 0x4000, 0xc51: 0x4000, + 0xc52: 0x4000, 0xc53: 0x4000, 0xc54: 0x4000, 0xc55: 0x4000, + 0xc70: 0x4000, 0xc71: 0x4000, 0xc72: 0x4000, 0xc73: 0x4000, 0xc74: 0x4000, 0xc75: 0x4000, + 0xc76: 0x4000, 0xc77: 0x4000, 0xc78: 0x4000, 0xc79: 0x4000, 0xc7a: 0x4000, 0xc7b: 0x4000, + // Block 0x32, offset 0xc80 + 0xc80: 0x9012, 0xc81: 0x4013, 0xc82: 0x4014, 0xc83: 0x4000, 0xc84: 0x4000, 0xc85: 0x4000, + 0xc86: 0x4000, 0xc87: 0x4000, 0xc88: 0x4000, 0xc89: 0x4000, 0xc8a: 0x4000, 0xc8b: 0x4000, + 0xc8c: 0x4015, 0xc8d: 0x4015, 0xc8e: 0x4000, 0xc8f: 0x4000, 0xc90: 0x4000, 0xc91: 0x4000, + 0xc92: 0x4000, 0xc93: 0x4000, 0xc94: 0x4000, 0xc95: 0x4000, 0xc96: 0x4000, 0xc97: 0x4000, + 0xc98: 0x4000, 0xc99: 0x4000, 0xc9a: 0x4000, 0xc9b: 0x4000, 0xc9c: 0x4000, 0xc9d: 0x4000, + 0xc9e: 0x4000, 0xc9f: 0x4000, 0xca0: 0x4000, 0xca1: 0x4000, 0xca2: 0x4000, 0xca3: 0x4000, + 0xca4: 0x4000, 0xca5: 0x4000, 0xca6: 0x4000, 0xca7: 0x4000, 0xca8: 0x4000, 0xca9: 0x4000, + 0xcaa: 0x4000, 0xcab: 0x4000, 0xcac: 0x4000, 0xcad: 0x4000, 0xcae: 0x4000, 0xcaf: 0x4000, + 0xcb0: 0x4000, 0xcb1: 0x4000, 0xcb2: 0x4000, 0xcb3: 0x4000, 0xcb4: 0x4000, 0xcb5: 0x4000, + 0xcb6: 0x4000, 0xcb7: 0x4000, 0xcb8: 0x4000, 0xcb9: 0x4000, 0xcba: 0x4000, 0xcbb: 0x4000, + 0xcbc: 0x4000, 0xcbd: 0x4000, 0xcbe: 0x4000, + // Block 0x33, offset 0xcc0 + 0xcc1: 0x4000, 0xcc2: 0x4000, 0xcc3: 0x4000, 0xcc4: 0x4000, 0xcc5: 0x4000, + 0xcc6: 0x4000, 0xcc7: 0x4000, 0xcc8: 0x4000, 0xcc9: 0x4000, 0xcca: 0x4000, 0xccb: 0x4000, + 0xccc: 0x4000, 0xccd: 0x4000, 0xcce: 0x4000, 0xccf: 0x4000, 0xcd0: 0x4000, 0xcd1: 0x4000, + 0xcd2: 0x4000, 0xcd3: 0x4000, 0xcd4: 0x4000, 0xcd5: 0x4000, 0xcd6: 0x4000, 0xcd7: 0x4000, + 0xcd8: 0x4000, 0xcd9: 0x4000, 0xcda: 0x4000, 0xcdb: 0x4000, 0xcdc: 0x4000, 0xcdd: 0x4000, + 0xcde: 0x4000, 0xcdf: 0x4000, 0xce0: 0x4000, 0xce1: 0x4000, 0xce2: 0x4000, 0xce3: 0x4000, + 0xce4: 0x4000, 0xce5: 0x4000, 0xce6: 0x4000, 0xce7: 0x4000, 0xce8: 0x4000, 0xce9: 0x4000, + 0xcea: 0x4000, 0xceb: 0x4000, 0xcec: 0x4000, 0xced: 0x4000, 0xcee: 0x4000, 0xcef: 0x4000, + 0xcf0: 0x4000, 0xcf1: 0x4000, 0xcf2: 0x4000, 0xcf3: 0x4000, 0xcf4: 0x4000, 0xcf5: 0x4000, + 0xcf6: 0x4000, 0xcf7: 0x4000, 0xcf8: 0x4000, 0xcf9: 0x4000, 0xcfa: 0x4000, 0xcfb: 0x4000, + 0xcfc: 0x4000, 0xcfd: 0x4000, 0xcfe: 0x4000, 0xcff: 0x4000, + // Block 0x34, offset 0xd00 + 0xd00: 0x4000, 0xd01: 0x4000, 0xd02: 0x4000, 0xd03: 0x4000, 0xd04: 0x4000, 0xd05: 0x4000, + 0xd06: 0x4000, 0xd07: 0x4000, 0xd08: 0x4000, 0xd09: 0x4000, 0xd0a: 0x4000, 0xd0b: 0x4000, + 0xd0c: 0x4000, 0xd0d: 0x4000, 0xd0e: 0x4000, 0xd0f: 0x4000, 0xd10: 0x4000, 0xd11: 0x4000, + 0xd12: 0x4000, 0xd13: 0x4000, 0xd14: 0x4000, 0xd15: 0x4000, 0xd16: 0x4000, + 0xd19: 0x4016, 0xd1a: 0x4017, 0xd1b: 0x4000, 0xd1c: 0x4000, 0xd1d: 0x4000, + 0xd1e: 0x4000, 0xd1f: 0x4000, 0xd20: 0x4000, 0xd21: 0x4018, 0xd22: 0x4019, 0xd23: 0x401a, + 0xd24: 0x401b, 0xd25: 0x401c, 0xd26: 0x401d, 0xd27: 0x401e, 0xd28: 0x401f, 0xd29: 0x4020, + 0xd2a: 0x4021, 0xd2b: 0x4022, 0xd2c: 0x4000, 0xd2d: 0x4010, 0xd2e: 0x4000, 0xd2f: 0x4023, + 0xd30: 0x4000, 0xd31: 0x4024, 0xd32: 0x4000, 0xd33: 0x4025, 0xd34: 0x4000, 0xd35: 0x4026, + 0xd36: 0x4000, 0xd37: 0x401a, 0xd38: 0x4000, 0xd39: 0x4027, 0xd3a: 0x4000, 0xd3b: 0x4028, + 0xd3c: 0x4000, 0xd3d: 0x4020, 0xd3e: 0x4000, 0xd3f: 0x4029, + // Block 0x35, offset 0xd40 + 0xd40: 0x4000, 0xd41: 0x402a, 0xd42: 0x4000, 0xd43: 0x402b, 0xd44: 0x402c, 0xd45: 0x4000, + 0xd46: 0x4017, 0xd47: 0x4000, 0xd48: 0x402d, 0xd49: 0x4000, 0xd4a: 0x402e, 0xd4b: 0x402f, + 0xd4c: 0x4030, 0xd4d: 0x4017, 0xd4e: 0x4016, 0xd4f: 0x4017, 0xd50: 0x4000, 0xd51: 0x4000, + 0xd52: 0x4031, 0xd53: 0x4000, 0xd54: 0x4000, 0xd55: 0x4031, 0xd56: 0x4000, 0xd57: 0x4000, + 0xd58: 0x4032, 0xd59: 0x4000, 0xd5a: 0x4000, 0xd5b: 0x4032, 0xd5c: 0x4000, 0xd5d: 0x4000, + 0xd5e: 0x4033, 0xd5f: 0x402e, 0xd60: 0x4034, 0xd61: 0x4035, 0xd62: 0x4034, 0xd63: 0x4036, + 0xd64: 0x4037, 0xd65: 0x4024, 0xd66: 0x4035, 0xd67: 0x4025, 0xd68: 0x4038, 0xd69: 0x4038, + 0xd6a: 0x4039, 0xd6b: 0x4039, 0xd6c: 0x403a, 0xd6d: 0x403a, 0xd6e: 0x4000, 0xd6f: 0x4035, + 0xd70: 0x4000, 0xd71: 0x4000, 0xd72: 0x403b, 0xd73: 0x403c, 0xd74: 0x4000, 0xd75: 0x4000, + 0xd76: 0x4000, 0xd77: 0x4000, 0xd78: 0x4000, 0xd79: 0x4000, 0xd7a: 0x4000, 0xd7b: 0x403d, + 0xd7c: 0x401c, 0xd7d: 0x4000, 0xd7e: 0x4000, 0xd7f: 0x4000, + // Block 0x36, offset 0xd80 + 0xd85: 0x4000, + 0xd86: 0x4000, 0xd87: 0x4000, 0xd88: 0x4000, 0xd89: 0x4000, 0xd8a: 0x4000, 0xd8b: 0x4000, + 0xd8c: 0x4000, 0xd8d: 0x4000, 0xd8e: 0x4000, 0xd8f: 0x4000, 0xd90: 0x4000, 0xd91: 0x4000, + 0xd92: 0x4000, 0xd93: 0x4000, 0xd94: 0x4000, 0xd95: 0x4000, 0xd96: 0x4000, 0xd97: 0x4000, + 0xd98: 0x4000, 0xd99: 0x4000, 0xd9a: 0x4000, 0xd9b: 0x4000, 0xd9c: 0x4000, 0xd9d: 0x4000, + 0xd9e: 0x4000, 0xd9f: 0x4000, 0xda0: 0x4000, 0xda1: 0x4000, 0xda2: 0x4000, 0xda3: 0x4000, + 0xda4: 0x4000, 0xda5: 0x4000, 0xda6: 0x4000, 0xda7: 0x4000, 0xda8: 0x4000, 0xda9: 0x4000, + 0xdaa: 0x4000, 0xdab: 0x4000, 0xdac: 0x4000, 0xdad: 0x4000, 0xdae: 0x4000, 0xdaf: 0x4000, + 0xdb1: 0x403e, 0xdb2: 0x403e, 0xdb3: 0x403e, 0xdb4: 0x403e, 0xdb5: 0x403e, + 0xdb6: 0x403e, 0xdb7: 0x403e, 0xdb8: 0x403e, 0xdb9: 0x403e, 0xdba: 0x403e, 0xdbb: 0x403e, + 0xdbc: 0x403e, 0xdbd: 0x403e, 0xdbe: 0x403e, 0xdbf: 0x403e, + // Block 0x37, offset 0xdc0 + 0xdc0: 0x4037, 0xdc1: 0x4037, 0xdc2: 0x4037, 0xdc3: 0x4037, 0xdc4: 0x4037, 0xdc5: 0x4037, + 0xdc6: 0x4037, 0xdc7: 0x4037, 0xdc8: 0x4037, 0xdc9: 0x4037, 0xdca: 0x4037, 0xdcb: 0x4037, + 0xdcc: 0x4037, 0xdcd: 0x4037, 0xdce: 0x4037, 0xdcf: 0x400e, 0xdd0: 0x403f, 0xdd1: 0x4040, + 0xdd2: 0x4041, 0xdd3: 0x4040, 0xdd4: 0x403f, 0xdd5: 0x4042, 0xdd6: 0x4043, 0xdd7: 0x4044, + 0xdd8: 0x4040, 0xdd9: 0x4041, 0xdda: 0x4040, 0xddb: 0x4045, 0xddc: 0x4009, 0xddd: 0x4045, + 0xdde: 0x4046, 0xddf: 0x4045, 0xde0: 0x4047, 0xde1: 0x400b, 0xde2: 0x400a, 0xde3: 0x400c, + 0xde4: 0x4048, 0xde5: 0x4000, 0xde6: 0x4000, 0xde7: 0x4000, 0xde8: 0x4000, 0xde9: 0x4000, + 0xdea: 0x4000, 0xdeb: 0x4000, 0xdec: 0x4000, 0xded: 0x4000, 0xdee: 0x4000, 0xdef: 0x4000, + 0xdf0: 0x4000, 0xdf1: 0x4000, 0xdf2: 0x4000, 0xdf3: 0x4000, 0xdf4: 0x4000, 0xdf5: 0x4000, + 0xdf6: 0x4000, 0xdf7: 0x4000, 0xdf8: 0x4000, 0xdf9: 0x4000, 0xdfa: 0x4000, 0xdfb: 0x4000, + 0xdfc: 0x4000, 0xdfd: 0x4000, 0xdfe: 0x4000, 0xdff: 0x4000, + // Block 0x38, offset 0xe00 + 0xe00: 0x4000, 0xe01: 0x4000, 0xe02: 0x4000, 0xe03: 0x4000, 0xe04: 0x4000, 0xe05: 0x4000, + 0xe06: 0x4000, 0xe07: 0x4000, 0xe08: 0x4000, 0xe09: 0x4000, 0xe0a: 0x4000, 0xe0b: 0x4000, + 0xe0c: 0x4000, 0xe0d: 0x4000, 0xe0e: 0x4000, 0xe10: 0x4000, 0xe11: 0x4000, + 0xe12: 0x4000, 0xe13: 0x4000, 0xe14: 0x4000, 0xe15: 0x4000, 0xe16: 0x4000, 0xe17: 0x4000, + 0xe18: 0x4000, 0xe19: 0x4000, 0xe1a: 0x4000, 0xe1b: 0x4000, 0xe1c: 0x4000, 0xe1d: 0x4000, + 0xe1e: 0x4000, 0xe1f: 0x4000, 0xe20: 0x4000, 0xe21: 0x4000, 0xe22: 0x4000, 0xe23: 0x4000, + 0xe24: 0x4000, 0xe25: 0x4000, 0xe26: 0x4000, 0xe27: 0x4000, 0xe28: 0x4000, 0xe29: 0x4000, + 0xe2a: 0x4000, 0xe2b: 0x4000, 0xe2c: 0x4000, 0xe2d: 0x4000, 0xe2e: 0x4000, 0xe2f: 0x4000, + 0xe30: 0x4000, 0xe31: 0x4000, 0xe32: 0x4000, 0xe33: 0x4000, 0xe34: 0x4000, 0xe35: 0x4000, + 0xe36: 0x4000, 0xe37: 0x4000, 0xe38: 0x4000, 0xe39: 0x4000, 0xe3a: 0x4000, 0xe3b: 0x4000, + 0xe3c: 0x4000, 0xe3d: 0x4000, 0xe3e: 0x4000, 0xe3f: 0x4000, + // Block 0x39, offset 0xe40 + 0xe40: 0x4000, 0xe41: 0x4000, 0xe42: 0x4000, 0xe43: 0x4000, 0xe44: 0x4000, 0xe45: 0x4000, + 0xe46: 0x4000, 0xe47: 0x4000, 0xe48: 0x4000, 0xe49: 0x4000, 0xe4a: 0x4000, 0xe4b: 0x4000, + 0xe4c: 0x4000, 0xe4d: 0x4000, 0xe4e: 0x4000, 0xe4f: 0x4000, 0xe50: 0x4000, 0xe51: 0x4000, + 0xe52: 0x4000, 0xe53: 0x4000, 0xe54: 0x4000, 0xe55: 0x4000, 0xe56: 0x4000, 0xe57: 0x4000, + 0xe58: 0x4000, 0xe59: 0x4000, 0xe5a: 0x4000, 0xe5b: 0x4000, 0xe5c: 0x4000, 0xe5d: 0x4000, + 0xe5e: 0x4000, 0xe5f: 0x4000, 0xe60: 0x4000, 0xe61: 0x4000, 0xe62: 0x4000, 0xe63: 0x4000, + 0xe70: 0x4000, 0xe71: 0x4000, 0xe72: 0x4000, 0xe73: 0x4000, 0xe74: 0x4000, 0xe75: 0x4000, + 0xe76: 0x4000, 0xe77: 0x4000, 0xe78: 0x4000, 0xe79: 0x4000, 0xe7a: 0x4000, 0xe7b: 0x4000, + 0xe7c: 0x4000, 0xe7d: 0x4000, 0xe7e: 0x4000, 0xe7f: 0x4000, + // Block 0x3a, offset 0xe80 + 0xe80: 0x4000, 0xe81: 0x4000, 0xe82: 0x4000, 0xe83: 0x4000, 0xe84: 0x4000, 0xe85: 0x4000, + 0xe86: 0x4000, 0xe87: 0x4000, 0xe88: 0x4000, 0xe89: 0x4000, 0xe8a: 0x4000, 0xe8b: 0x4000, + 0xe8c: 0x4000, 0xe8d: 0x4000, 0xe8e: 0x4000, 0xe8f: 0x4000, 0xe90: 0x4000, 0xe91: 0x4000, + 0xe92: 0x4000, 0xe93: 0x4000, 0xe94: 0x4000, 0xe95: 0x4000, 0xe96: 0x4000, 0xe97: 0x4000, + 0xe98: 0x4000, 0xe99: 0x4000, 0xe9a: 0x4000, 0xe9b: 0x4000, 0xe9c: 0x4000, 0xe9d: 0x4000, + 0xe9e: 0x4000, 0xea0: 0x4000, 0xea1: 0x4000, 0xea2: 0x4000, 0xea3: 0x4000, + 0xea4: 0x4000, 0xea5: 0x4000, 0xea6: 0x4000, 0xea7: 0x4000, 0xea8: 0x4000, 0xea9: 0x4000, + 0xeaa: 0x4000, 0xeab: 0x4000, 0xeac: 0x4000, 0xead: 0x4000, 0xeae: 0x4000, 0xeaf: 0x4000, + 0xeb0: 0x4000, 0xeb1: 0x4000, 0xeb2: 0x4000, 0xeb3: 0x4000, 0xeb4: 0x4000, 0xeb5: 0x4000, + 0xeb6: 0x4000, 0xeb7: 0x4000, 0xeb8: 0x4000, 0xeb9: 0x4000, 0xeba: 0x4000, 0xebb: 0x4000, + 0xebc: 0x4000, 0xebd: 0x4000, 0xebe: 0x4000, 0xebf: 0x4000, + // Block 0x3b, offset 0xec0 + 0xec0: 0x4000, 0xec1: 0x4000, 0xec2: 0x4000, 0xec3: 0x4000, 0xec4: 0x4000, 0xec5: 0x4000, + 0xec6: 0x4000, 0xec7: 0x4000, 0xec8: 0x2000, 0xec9: 0x2000, 0xeca: 0x2000, 0xecb: 0x2000, + 0xecc: 0x2000, 0xecd: 0x2000, 0xece: 0x2000, 0xecf: 0x2000, 0xed0: 0x4000, 0xed1: 0x4000, + 0xed2: 0x4000, 0xed3: 0x4000, 0xed4: 0x4000, 0xed5: 0x4000, 0xed6: 0x4000, 0xed7: 0x4000, + 0xed8: 0x4000, 0xed9: 0x4000, 0xeda: 0x4000, 0xedb: 0x4000, 0xedc: 0x4000, 0xedd: 0x4000, + 0xede: 0x4000, 0xedf: 0x4000, 0xee0: 0x4000, 0xee1: 0x4000, 0xee2: 0x4000, 0xee3: 0x4000, + 0xee4: 0x4000, 0xee5: 0x4000, 0xee6: 0x4000, 0xee7: 0x4000, 0xee8: 0x4000, 0xee9: 0x4000, + 0xeea: 0x4000, 0xeeb: 0x4000, 0xeec: 0x4000, 0xeed: 0x4000, 0xeee: 0x4000, 0xeef: 0x4000, + 0xef0: 0x4000, 0xef1: 0x4000, 0xef2: 0x4000, 0xef3: 0x4000, 0xef4: 0x4000, 0xef5: 0x4000, + 0xef6: 0x4000, 0xef7: 0x4000, 0xef8: 0x4000, 0xef9: 0x4000, 0xefa: 0x4000, 0xefb: 0x4000, + 0xefc: 0x4000, 0xefd: 0x4000, 0xefe: 0x4000, 0xeff: 0x4000, + // Block 0x3c, offset 0xf00 + 0xf00: 0x4000, 0xf01: 0x4000, 0xf02: 0x4000, 0xf03: 0x4000, 0xf04: 0x4000, 0xf05: 0x4000, + 0xf06: 0x4000, 0xf07: 0x4000, 0xf08: 0x4000, 0xf09: 0x4000, 0xf0a: 0x4000, 0xf0b: 0x4000, + 0xf0c: 0x4000, 0xf10: 0x4000, 0xf11: 0x4000, + 0xf12: 0x4000, 0xf13: 0x4000, 0xf14: 0x4000, 0xf15: 0x4000, 0xf16: 0x4000, 0xf17: 0x4000, + 0xf18: 0x4000, 0xf19: 0x4000, 0xf1a: 0x4000, 0xf1b: 0x4000, 0xf1c: 0x4000, 0xf1d: 0x4000, + 0xf1e: 0x4000, 0xf1f: 0x4000, 0xf20: 0x4000, 0xf21: 0x4000, 0xf22: 0x4000, 0xf23: 0x4000, + 0xf24: 0x4000, 0xf25: 0x4000, 0xf26: 0x4000, 0xf27: 0x4000, 0xf28: 0x4000, 0xf29: 0x4000, + 0xf2a: 0x4000, 0xf2b: 0x4000, 0xf2c: 0x4000, 0xf2d: 0x4000, 0xf2e: 0x4000, 0xf2f: 0x4000, + 0xf30: 0x4000, 0xf31: 0x4000, 0xf32: 0x4000, 0xf33: 0x4000, 0xf34: 0x4000, 0xf35: 0x4000, + 0xf36: 0x4000, 0xf37: 0x4000, 0xf38: 0x4000, 0xf39: 0x4000, 0xf3a: 0x4000, 0xf3b: 0x4000, + 0xf3c: 0x4000, 0xf3d: 0x4000, 0xf3e: 0x4000, 0xf3f: 0x4000, + // Block 0x3d, offset 0xf40 + 0xf40: 0x4000, 0xf41: 0x4000, 0xf42: 0x4000, 0xf43: 0x4000, 0xf44: 0x4000, 0xf45: 0x4000, + 0xf46: 0x4000, + // Block 0x3e, offset 0xf80 + 0xfa0: 0x4000, 0xfa1: 0x4000, 0xfa2: 0x4000, 0xfa3: 0x4000, + 0xfa4: 0x4000, 0xfa5: 0x4000, 0xfa6: 0x4000, 0xfa7: 0x4000, 0xfa8: 0x4000, 0xfa9: 0x4000, + 0xfaa: 0x4000, 0xfab: 0x4000, 0xfac: 0x4000, 0xfad: 0x4000, 0xfae: 0x4000, 0xfaf: 0x4000, + 0xfb0: 0x4000, 0xfb1: 0x4000, 0xfb2: 0x4000, 0xfb3: 0x4000, 0xfb4: 0x4000, 0xfb5: 0x4000, + 0xfb6: 0x4000, 0xfb7: 0x4000, 0xfb8: 0x4000, 0xfb9: 0x4000, 0xfba: 0x4000, 0xfbb: 0x4000, + 0xfbc: 0x4000, + // Block 0x3f, offset 0xfc0 + 0xfc0: 0x4000, 0xfc1: 0x4000, 0xfc2: 0x4000, 0xfc3: 0x4000, 0xfc4: 0x4000, 0xfc5: 0x4000, + 0xfc6: 0x4000, 0xfc7: 0x4000, 0xfc8: 0x4000, 0xfc9: 0x4000, 0xfca: 0x4000, 0xfcb: 0x4000, + 0xfcc: 0x4000, 0xfcd: 0x4000, 0xfce: 0x4000, 0xfcf: 0x4000, 0xfd0: 0x4000, 0xfd1: 0x4000, + 0xfd2: 0x4000, 0xfd3: 0x4000, 0xfd4: 0x4000, 0xfd5: 0x4000, 0xfd6: 0x4000, 0xfd7: 0x4000, + 0xfd8: 0x4000, 0xfd9: 0x4000, 0xfda: 0x4000, 0xfdb: 0x4000, 0xfdc: 0x4000, 0xfdd: 0x4000, + 0xfde: 0x4000, 0xfdf: 0x4000, 0xfe0: 0x4000, 0xfe1: 0x4000, 0xfe2: 0x4000, 0xfe3: 0x4000, + // Block 0x40, offset 0x1000 + 0x1000: 0x2000, 0x1001: 0x2000, 0x1002: 0x2000, 0x1003: 0x2000, 0x1004: 0x2000, 0x1005: 0x2000, + 0x1006: 0x2000, 0x1007: 0x2000, 0x1008: 0x2000, 0x1009: 0x2000, 0x100a: 0x2000, 0x100b: 0x2000, + 0x100c: 0x2000, 0x100d: 0x2000, 0x100e: 0x2000, 0x100f: 0x2000, 0x1010: 0x4000, 0x1011: 0x4000, + 0x1012: 0x4000, 0x1013: 0x4000, 0x1014: 0x4000, 0x1015: 0x4000, 0x1016: 0x4000, 0x1017: 0x4000, + 0x1018: 0x4000, 0x1019: 0x4000, + 0x1030: 0x4000, 0x1031: 0x4000, 0x1032: 0x4000, 0x1033: 0x4000, 0x1034: 0x4000, 0x1035: 0x4000, + 0x1036: 0x4000, 0x1037: 0x4000, 0x1038: 0x4000, 0x1039: 0x4000, 0x103a: 0x4000, 0x103b: 0x4000, + 0x103c: 0x4000, 0x103d: 0x4000, 0x103e: 0x4000, 0x103f: 0x4000, + // Block 0x41, offset 0x1040 + 0x1040: 0x4000, 0x1041: 0x4000, 0x1042: 0x4000, 0x1043: 0x4000, 0x1044: 0x4000, 0x1045: 0x4000, + 0x1046: 0x4000, 0x1047: 0x4000, 0x1048: 0x4000, 0x1049: 0x4000, 0x104a: 0x4000, 0x104b: 0x4000, + 0x104c: 0x4000, 0x104d: 0x4000, 0x104e: 0x4000, 0x104f: 0x4000, 0x1050: 0x4000, 0x1051: 0x4000, + 0x1052: 0x4000, 0x1054: 0x4000, 0x1055: 0x4000, 0x1056: 0x4000, 0x1057: 0x4000, + 0x1058: 0x4000, 0x1059: 0x4000, 0x105a: 0x4000, 0x105b: 0x4000, 0x105c: 0x4000, 0x105d: 0x4000, + 0x105e: 0x4000, 0x105f: 0x4000, 0x1060: 0x4000, 0x1061: 0x4000, 0x1062: 0x4000, 0x1063: 0x4000, + 0x1064: 0x4000, 0x1065: 0x4000, 0x1066: 0x4000, 0x1068: 0x4000, 0x1069: 0x4000, + 0x106a: 0x4000, 0x106b: 0x4000, + // Block 0x42, offset 0x1080 + 0x1081: 0x9012, 0x1082: 0x9012, 0x1083: 0x9012, 0x1084: 0x9012, 0x1085: 0x9012, + 0x1086: 0x9012, 0x1087: 0x9012, 0x1088: 0x9012, 0x1089: 0x9012, 0x108a: 0x9012, 0x108b: 0x9012, + 0x108c: 0x9012, 0x108d: 0x9012, 0x108e: 0x9012, 0x108f: 0x9012, 0x1090: 0x9012, 0x1091: 0x9012, + 0x1092: 0x9012, 0x1093: 0x9012, 0x1094: 0x9012, 0x1095: 0x9012, 0x1096: 0x9012, 0x1097: 0x9012, + 0x1098: 0x9012, 0x1099: 0x9012, 0x109a: 0x9012, 0x109b: 0x9012, 0x109c: 0x9012, 0x109d: 0x9012, + 0x109e: 0x9012, 0x109f: 0x9012, 0x10a0: 0x9049, 0x10a1: 0x9049, 0x10a2: 0x9049, 0x10a3: 0x9049, + 0x10a4: 0x9049, 0x10a5: 0x9049, 0x10a6: 0x9049, 0x10a7: 0x9049, 0x10a8: 0x9049, 0x10a9: 0x9049, + 0x10aa: 0x9049, 0x10ab: 0x9049, 0x10ac: 0x9049, 0x10ad: 0x9049, 0x10ae: 0x9049, 0x10af: 0x9049, + 0x10b0: 0x9049, 0x10b1: 0x9049, 0x10b2: 0x9049, 0x10b3: 0x9049, 0x10b4: 0x9049, 0x10b5: 0x9049, + 0x10b6: 0x9049, 0x10b7: 0x9049, 0x10b8: 0x9049, 0x10b9: 0x9049, 0x10ba: 0x9049, 0x10bb: 0x9049, + 0x10bc: 0x9049, 0x10bd: 0x9049, 0x10be: 0x9049, 0x10bf: 0x9049, + // Block 0x43, offset 0x10c0 + 0x10c0: 0x9049, 0x10c1: 0x9049, 0x10c2: 0x9049, 0x10c3: 0x9049, 0x10c4: 0x9049, 0x10c5: 0x9049, + 0x10c6: 0x9049, 0x10c7: 0x9049, 0x10c8: 0x9049, 0x10c9: 0x9049, 0x10ca: 0x9049, 0x10cb: 0x9049, + 0x10cc: 0x9049, 0x10cd: 0x9049, 0x10ce: 0x9049, 0x10cf: 0x9049, 0x10d0: 0x9049, 0x10d1: 0x9049, + 0x10d2: 0x9049, 0x10d3: 0x9049, 0x10d4: 0x9049, 0x10d5: 0x9049, 0x10d6: 0x9049, 0x10d7: 0x9049, + 0x10d8: 0x9049, 0x10d9: 0x9049, 0x10da: 0x9049, 0x10db: 0x9049, 0x10dc: 0x9049, 0x10dd: 0x9049, + 0x10de: 0x9049, 0x10df: 0x904a, 0x10e0: 0x904b, 0x10e1: 0xb04c, 0x10e2: 0xb04d, 0x10e3: 0xb04d, + 0x10e4: 0xb04e, 0x10e5: 0xb04f, 0x10e6: 0xb050, 0x10e7: 0xb051, 0x10e8: 0xb052, 0x10e9: 0xb053, + 0x10ea: 0xb054, 0x10eb: 0xb055, 0x10ec: 0xb056, 0x10ed: 0xb057, 0x10ee: 0xb058, 0x10ef: 0xb059, + 0x10f0: 0xb05a, 0x10f1: 0xb05b, 0x10f2: 0xb05c, 0x10f3: 0xb05d, 0x10f4: 0xb05e, 0x10f5: 0xb05f, + 0x10f6: 0xb060, 0x10f7: 0xb061, 0x10f8: 0xb062, 0x10f9: 0xb063, 0x10fa: 0xb064, 0x10fb: 0xb065, + 0x10fc: 0xb052, 0x10fd: 0xb066, 0x10fe: 0xb067, 0x10ff: 0xb055, + // Block 0x44, offset 0x1100 + 0x1100: 0xb068, 0x1101: 0xb069, 0x1102: 0xb06a, 0x1103: 0xb06b, 0x1104: 0xb05a, 0x1105: 0xb056, + 0x1106: 0xb06c, 0x1107: 0xb06d, 0x1108: 0xb06b, 0x1109: 0xb06e, 0x110a: 0xb06b, 0x110b: 0xb06f, + 0x110c: 0xb06f, 0x110d: 0xb070, 0x110e: 0xb070, 0x110f: 0xb071, 0x1110: 0xb056, 0x1111: 0xb072, + 0x1112: 0xb073, 0x1113: 0xb072, 0x1114: 0xb074, 0x1115: 0xb073, 0x1116: 0xb075, 0x1117: 0xb075, + 0x1118: 0xb076, 0x1119: 0xb076, 0x111a: 0xb077, 0x111b: 0xb077, 0x111c: 0xb073, 0x111d: 0xb078, + 0x111e: 0xb079, 0x111f: 0xb067, 0x1120: 0xb07a, 0x1121: 0xb07b, 0x1122: 0xb07b, 0x1123: 0xb07b, + 0x1124: 0xb07b, 0x1125: 0xb07b, 0x1126: 0xb07b, 0x1127: 0xb07b, 0x1128: 0xb07b, 0x1129: 0xb07b, + 0x112a: 0xb07b, 0x112b: 0xb07b, 0x112c: 0xb07b, 0x112d: 0xb07b, 0x112e: 0xb07b, 0x112f: 0xb07b, + 0x1130: 0xb07c, 0x1131: 0xb07c, 0x1132: 0xb07c, 0x1133: 0xb07c, 0x1134: 0xb07c, 0x1135: 0xb07c, + 0x1136: 0xb07c, 0x1137: 0xb07c, 0x1138: 0xb07c, 0x1139: 0xb07c, 0x113a: 0xb07c, 0x113b: 0xb07c, + 0x113c: 0xb07c, 0x113d: 0xb07c, 0x113e: 0xb07c, + // Block 0x45, offset 0x1140 + 0x1142: 0xb07d, 0x1143: 0xb07e, 0x1144: 0xb07f, 0x1145: 0xb080, + 0x1146: 0xb07f, 0x1147: 0xb07e, 0x114a: 0xb081, 0x114b: 0xb082, + 0x114c: 0xb083, 0x114d: 0xb07f, 0x114e: 0xb080, 0x114f: 0xb07f, + 0x1152: 0xb084, 0x1153: 0xb085, 0x1154: 0xb084, 0x1155: 0xb086, 0x1156: 0xb084, 0x1157: 0xb087, + 0x115a: 0xb088, 0x115b: 0xb089, 0x115c: 0xb08a, + 0x1160: 0x908b, 0x1161: 0x908b, 0x1162: 0x908c, 0x1163: 0x908d, + 0x1164: 0x908b, 0x1165: 0x908e, 0x1166: 0x908f, 0x1168: 0xb090, 0x1169: 0xb091, + 0x116a: 0xb092, 0x116b: 0xb091, 0x116c: 0xb093, 0x116d: 0xb094, 0x116e: 0xb095, + 0x117d: 0x2000, + // Block 0x46, offset 0x1180 + 0x11a0: 0x4000, 0x11a1: 0x4000, 0x11a2: 0x4000, 0x11a3: 0x4000, + 0x11a4: 0x4000, + 0x11b0: 0x4000, 0x11b1: 0x4000, + // Block 0x47, offset 0x11c0 + 0x11c0: 0x4000, 0x11c1: 0x4000, 0x11c2: 0x4000, 0x11c3: 0x4000, 0x11c4: 0x4000, 0x11c5: 0x4000, + 0x11c6: 0x4000, 0x11c7: 0x4000, 0x11c8: 0x4000, 0x11c9: 0x4000, 0x11ca: 0x4000, 0x11cb: 0x4000, + 0x11cc: 0x4000, 0x11cd: 0x4000, 0x11ce: 0x4000, 0x11cf: 0x4000, 0x11d0: 0x4000, 0x11d1: 0x4000, + 0x11d2: 0x4000, 0x11d3: 0x4000, 0x11d4: 0x4000, 0x11d5: 0x4000, 0x11d6: 0x4000, 0x11d7: 0x4000, + 0x11d8: 0x4000, 0x11d9: 0x4000, 0x11da: 0x4000, 0x11db: 0x4000, 0x11dc: 0x4000, 0x11dd: 0x4000, + 0x11de: 0x4000, 0x11df: 0x4000, 0x11e0: 0x4000, 0x11e1: 0x4000, 0x11e2: 0x4000, 0x11e3: 0x4000, + 0x11e4: 0x4000, 0x11e5: 0x4000, 0x11e6: 0x4000, 0x11e7: 0x4000, 0x11e8: 0x4000, 0x11e9: 0x4000, + 0x11ea: 0x4000, 0x11eb: 0x4000, 0x11ec: 0x4000, 0x11ed: 0x4000, 0x11ee: 0x4000, 0x11ef: 0x4000, + 0x11f0: 0x4000, 0x11f1: 0x4000, 0x11f2: 0x4000, 0x11f3: 0x4000, 0x11f4: 0x4000, 0x11f5: 0x4000, + 0x11f6: 0x4000, 0x11f7: 0x4000, + // Block 0x48, offset 0x1200 + 0x1200: 0x4000, 0x1201: 0x4000, 0x1202: 0x4000, 0x1203: 0x4000, 0x1204: 0x4000, 0x1205: 0x4000, + 0x1206: 0x4000, 0x1207: 0x4000, 0x1208: 0x4000, 0x1209: 0x4000, 0x120a: 0x4000, 0x120b: 0x4000, + 0x120c: 0x4000, 0x120d: 0x4000, 0x120e: 0x4000, 0x120f: 0x4000, 0x1210: 0x4000, 0x1211: 0x4000, + 0x1212: 0x4000, 0x1213: 0x4000, 0x1214: 0x4000, 0x1215: 0x4000, + // Block 0x49, offset 0x1240 + 0x1240: 0x4000, 0x1241: 0x4000, 0x1242: 0x4000, 0x1243: 0x4000, 0x1244: 0x4000, 0x1245: 0x4000, + 0x1246: 0x4000, 0x1247: 0x4000, 0x1248: 0x4000, + // Block 0x4a, offset 0x1280 + 0x12b0: 0x4000, 0x12b1: 0x4000, 0x12b2: 0x4000, 0x12b3: 0x4000, 0x12b5: 0x4000, + 0x12b6: 0x4000, 0x12b7: 0x4000, 0x12b8: 0x4000, 0x12b9: 0x4000, 0x12ba: 0x4000, 0x12bb: 0x4000, + 0x12bd: 0x4000, 0x12be: 0x4000, + // Block 0x4b, offset 0x12c0 + 0x12c0: 0x4000, 0x12c1: 0x4000, 0x12c2: 0x4000, 0x12c3: 0x4000, 0x12c4: 0x4000, 0x12c5: 0x4000, + 0x12c6: 0x4000, 0x12c7: 0x4000, 0x12c8: 0x4000, 0x12c9: 0x4000, 0x12ca: 0x4000, 0x12cb: 0x4000, + 0x12cc: 0x4000, 0x12cd: 0x4000, 0x12ce: 0x4000, 0x12cf: 0x4000, 0x12d0: 0x4000, 0x12d1: 0x4000, + 0x12d2: 0x4000, 0x12d3: 0x4000, 0x12d4: 0x4000, 0x12d5: 0x4000, 0x12d6: 0x4000, 0x12d7: 0x4000, + 0x12d8: 0x4000, 0x12d9: 0x4000, 0x12da: 0x4000, 0x12db: 0x4000, 0x12dc: 0x4000, 0x12dd: 0x4000, + 0x12de: 0x4000, 0x12df: 0x4000, 0x12e0: 0x4000, 0x12e1: 0x4000, 0x12e2: 0x4000, + 0x12f2: 0x4000, + // Block 0x4c, offset 0x1300 + 0x1310: 0x4000, 0x1311: 0x4000, + 0x1312: 0x4000, 0x1315: 0x4000, + 0x1324: 0x4000, 0x1325: 0x4000, 0x1326: 0x4000, 0x1327: 0x4000, + 0x1330: 0x4000, 0x1331: 0x4000, 0x1332: 0x4000, 0x1333: 0x4000, 0x1334: 0x4000, 0x1335: 0x4000, + 0x1336: 0x4000, 0x1337: 0x4000, 0x1338: 0x4000, 0x1339: 0x4000, 0x133a: 0x4000, 0x133b: 0x4000, + 0x133c: 0x4000, 0x133d: 0x4000, 0x133e: 0x4000, 0x133f: 0x4000, + // Block 0x4d, offset 0x1340 + 0x1340: 0x4000, 0x1341: 0x4000, 0x1342: 0x4000, 0x1343: 0x4000, 0x1344: 0x4000, 0x1345: 0x4000, + 0x1346: 0x4000, 0x1347: 0x4000, 0x1348: 0x4000, 0x1349: 0x4000, 0x134a: 0x4000, 0x134b: 0x4000, + 0x134c: 0x4000, 0x134d: 0x4000, 0x134e: 0x4000, 0x134f: 0x4000, 0x1350: 0x4000, 0x1351: 0x4000, + 0x1352: 0x4000, 0x1353: 0x4000, 0x1354: 0x4000, 0x1355: 0x4000, 0x1356: 0x4000, 0x1357: 0x4000, + 0x1358: 0x4000, 0x1359: 0x4000, 0x135a: 0x4000, 0x135b: 0x4000, 0x135c: 0x4000, 0x135d: 0x4000, + 0x135e: 0x4000, 0x135f: 0x4000, 0x1360: 0x4000, 0x1361: 0x4000, 0x1362: 0x4000, 0x1363: 0x4000, + 0x1364: 0x4000, 0x1365: 0x4000, 0x1366: 0x4000, 0x1367: 0x4000, 0x1368: 0x4000, 0x1369: 0x4000, + 0x136a: 0x4000, 0x136b: 0x4000, 0x136c: 0x4000, 0x136d: 0x4000, 0x136e: 0x4000, 0x136f: 0x4000, + 0x1370: 0x4000, 0x1371: 0x4000, 0x1372: 0x4000, 0x1373: 0x4000, 0x1374: 0x4000, 0x1375: 0x4000, + 0x1376: 0x4000, 0x1377: 0x4000, 0x1378: 0x4000, 0x1379: 0x4000, 0x137a: 0x4000, 0x137b: 0x4000, + // Block 0x4e, offset 0x1380 + 0x1384: 0x4000, + // Block 0x4f, offset 0x13c0 + 0x13cf: 0x4000, + // Block 0x50, offset 0x1400 + 0x1400: 0x2000, 0x1401: 0x2000, 0x1402: 0x2000, 0x1403: 0x2000, 0x1404: 0x2000, 0x1405: 0x2000, + 0x1406: 0x2000, 0x1407: 0x2000, 0x1408: 0x2000, 0x1409: 0x2000, 0x140a: 0x2000, + 0x1410: 0x2000, 0x1411: 0x2000, + 0x1412: 0x2000, 0x1413: 0x2000, 0x1414: 0x2000, 0x1415: 0x2000, 0x1416: 0x2000, 0x1417: 0x2000, + 0x1418: 0x2000, 0x1419: 0x2000, 0x141a: 0x2000, 0x141b: 0x2000, 0x141c: 0x2000, 0x141d: 0x2000, + 0x141e: 0x2000, 0x141f: 0x2000, 0x1420: 0x2000, 0x1421: 0x2000, 0x1422: 0x2000, 0x1423: 0x2000, + 0x1424: 0x2000, 0x1425: 0x2000, 0x1426: 0x2000, 0x1427: 0x2000, 0x1428: 0x2000, 0x1429: 0x2000, + 0x142a: 0x2000, 0x142b: 0x2000, 0x142c: 0x2000, 0x142d: 0x2000, + 0x1430: 0x2000, 0x1431: 0x2000, 0x1432: 0x2000, 0x1433: 0x2000, 0x1434: 0x2000, 0x1435: 0x2000, + 0x1436: 0x2000, 0x1437: 0x2000, 0x1438: 0x2000, 0x1439: 0x2000, 0x143a: 0x2000, 0x143b: 0x2000, + 0x143c: 0x2000, 0x143d: 0x2000, 0x143e: 0x2000, 0x143f: 0x2000, + // Block 0x51, offset 0x1440 + 0x1440: 0x2000, 0x1441: 0x2000, 0x1442: 0x2000, 0x1443: 0x2000, 0x1444: 0x2000, 0x1445: 0x2000, + 0x1446: 0x2000, 0x1447: 0x2000, 0x1448: 0x2000, 0x1449: 0x2000, 0x144a: 0x2000, 0x144b: 0x2000, + 0x144c: 0x2000, 0x144d: 0x2000, 0x144e: 0x2000, 0x144f: 0x2000, 0x1450: 0x2000, 0x1451: 0x2000, + 0x1452: 0x2000, 0x1453: 0x2000, 0x1454: 0x2000, 0x1455: 0x2000, 0x1456: 0x2000, 0x1457: 0x2000, + 0x1458: 0x2000, 0x1459: 0x2000, 0x145a: 0x2000, 0x145b: 0x2000, 0x145c: 0x2000, 0x145d: 0x2000, + 0x145e: 0x2000, 0x145f: 0x2000, 0x1460: 0x2000, 0x1461: 0x2000, 0x1462: 0x2000, 0x1463: 0x2000, + 0x1464: 0x2000, 0x1465: 0x2000, 0x1466: 0x2000, 0x1467: 0x2000, 0x1468: 0x2000, 0x1469: 0x2000, + 0x1470: 0x2000, 0x1471: 0x2000, 0x1472: 0x2000, 0x1473: 0x2000, 0x1474: 0x2000, 0x1475: 0x2000, + 0x1476: 0x2000, 0x1477: 0x2000, 0x1478: 0x2000, 0x1479: 0x2000, 0x147a: 0x2000, 0x147b: 0x2000, + 0x147c: 0x2000, 0x147d: 0x2000, 0x147e: 0x2000, 0x147f: 0x2000, + // Block 0x52, offset 0x1480 + 0x1480: 0x2000, 0x1481: 0x2000, 0x1482: 0x2000, 0x1483: 0x2000, 0x1484: 0x2000, 0x1485: 0x2000, + 0x1486: 0x2000, 0x1487: 0x2000, 0x1488: 0x2000, 0x1489: 0x2000, 0x148a: 0x2000, 0x148b: 0x2000, + 0x148c: 0x2000, 0x148d: 0x2000, 0x148e: 0x4000, 0x148f: 0x2000, 0x1490: 0x2000, 0x1491: 0x4000, + 0x1492: 0x4000, 0x1493: 0x4000, 0x1494: 0x4000, 0x1495: 0x4000, 0x1496: 0x4000, 0x1497: 0x4000, + 0x1498: 0x4000, 0x1499: 0x4000, 0x149a: 0x4000, 0x149b: 0x2000, 0x149c: 0x2000, 0x149d: 0x2000, + 0x149e: 0x2000, 0x149f: 0x2000, 0x14a0: 0x2000, 0x14a1: 0x2000, 0x14a2: 0x2000, 0x14a3: 0x2000, + 0x14a4: 0x2000, 0x14a5: 0x2000, 0x14a6: 0x2000, 0x14a7: 0x2000, 0x14a8: 0x2000, 0x14a9: 0x2000, + 0x14aa: 0x2000, 0x14ab: 0x2000, 0x14ac: 0x2000, + // Block 0x53, offset 0x14c0 + 0x14c0: 0x4000, 0x14c1: 0x4000, 0x14c2: 0x4000, + 0x14d0: 0x4000, 0x14d1: 0x4000, + 0x14d2: 0x4000, 0x14d3: 0x4000, 0x14d4: 0x4000, 0x14d5: 0x4000, 0x14d6: 0x4000, 0x14d7: 0x4000, + 0x14d8: 0x4000, 0x14d9: 0x4000, 0x14da: 0x4000, 0x14db: 0x4000, 0x14dc: 0x4000, 0x14dd: 0x4000, + 0x14de: 0x4000, 0x14df: 0x4000, 0x14e0: 0x4000, 0x14e1: 0x4000, 0x14e2: 0x4000, 0x14e3: 0x4000, + 0x14e4: 0x4000, 0x14e5: 0x4000, 0x14e6: 0x4000, 0x14e7: 0x4000, 0x14e8: 0x4000, 0x14e9: 0x4000, + 0x14ea: 0x4000, 0x14eb: 0x4000, 0x14ec: 0x4000, 0x14ed: 0x4000, 0x14ee: 0x4000, 0x14ef: 0x4000, + 0x14f0: 0x4000, 0x14f1: 0x4000, 0x14f2: 0x4000, 0x14f3: 0x4000, 0x14f4: 0x4000, 0x14f5: 0x4000, + 0x14f6: 0x4000, 0x14f7: 0x4000, 0x14f8: 0x4000, 0x14f9: 0x4000, 0x14fa: 0x4000, 0x14fb: 0x4000, + // Block 0x54, offset 0x1500 + 0x1500: 0x4000, 0x1501: 0x4000, 0x1502: 0x4000, 0x1503: 0x4000, 0x1504: 0x4000, 0x1505: 0x4000, + 0x1506: 0x4000, 0x1507: 0x4000, 0x1508: 0x4000, + 0x1510: 0x4000, 0x1511: 0x4000, + 0x1520: 0x4000, 0x1521: 0x4000, 0x1522: 0x4000, 0x1523: 0x4000, + 0x1524: 0x4000, 0x1525: 0x4000, + // Block 0x55, offset 0x1540 + 0x1540: 0x4000, 0x1541: 0x4000, 0x1542: 0x4000, 0x1543: 0x4000, 0x1544: 0x4000, 0x1545: 0x4000, + 0x1546: 0x4000, 0x1547: 0x4000, 0x1548: 0x4000, 0x1549: 0x4000, 0x154a: 0x4000, 0x154b: 0x4000, + 0x154c: 0x4000, 0x154d: 0x4000, 0x154e: 0x4000, 0x154f: 0x4000, 0x1550: 0x4000, 0x1551: 0x4000, + 0x1552: 0x4000, 0x1553: 0x4000, 0x1554: 0x4000, 0x1555: 0x4000, 0x1556: 0x4000, 0x1557: 0x4000, + 0x1558: 0x4000, 0x1559: 0x4000, 0x155a: 0x4000, 0x155b: 0x4000, 0x155c: 0x4000, 0x155d: 0x4000, + 0x155e: 0x4000, 0x155f: 0x4000, 0x1560: 0x4000, + 0x156d: 0x4000, 0x156e: 0x4000, 0x156f: 0x4000, + 0x1570: 0x4000, 0x1571: 0x4000, 0x1572: 0x4000, 0x1573: 0x4000, 0x1574: 0x4000, 0x1575: 0x4000, + 0x1577: 0x4000, 0x1578: 0x4000, 0x1579: 0x4000, 0x157a: 0x4000, 0x157b: 0x4000, + 0x157c: 0x4000, 0x157d: 0x4000, 0x157e: 0x4000, 0x157f: 0x4000, + // Block 0x56, offset 0x1580 + 0x1580: 0x4000, 0x1581: 0x4000, 0x1582: 0x4000, 0x1583: 0x4000, 0x1584: 0x4000, 0x1585: 0x4000, + 0x1586: 0x4000, 0x1587: 0x4000, 0x1588: 0x4000, 0x1589: 0x4000, 0x158a: 0x4000, 0x158b: 0x4000, + 0x158c: 0x4000, 0x158d: 0x4000, 0x158e: 0x4000, 0x158f: 0x4000, 0x1590: 0x4000, 0x1591: 0x4000, + 0x1592: 0x4000, 0x1593: 0x4000, 0x1594: 0x4000, 0x1595: 0x4000, 0x1596: 0x4000, 0x1597: 0x4000, + 0x1598: 0x4000, 0x1599: 0x4000, 0x159a: 0x4000, 0x159b: 0x4000, 0x159c: 0x4000, 0x159d: 0x4000, + 0x159e: 0x4000, 0x159f: 0x4000, 0x15a0: 0x4000, 0x15a1: 0x4000, 0x15a2: 0x4000, 0x15a3: 0x4000, + 0x15a4: 0x4000, 0x15a5: 0x4000, 0x15a6: 0x4000, 0x15a7: 0x4000, 0x15a8: 0x4000, 0x15a9: 0x4000, + 0x15aa: 0x4000, 0x15ab: 0x4000, 0x15ac: 0x4000, 0x15ad: 0x4000, 0x15ae: 0x4000, 0x15af: 0x4000, + 0x15b0: 0x4000, 0x15b1: 0x4000, 0x15b2: 0x4000, 0x15b3: 0x4000, 0x15b4: 0x4000, 0x15b5: 0x4000, + 0x15b6: 0x4000, 0x15b7: 0x4000, 0x15b8: 0x4000, 0x15b9: 0x4000, 0x15ba: 0x4000, 0x15bb: 0x4000, + 0x15bc: 0x4000, 0x15be: 0x4000, 0x15bf: 0x4000, + // Block 0x57, offset 0x15c0 + 0x15c0: 0x4000, 0x15c1: 0x4000, 0x15c2: 0x4000, 0x15c3: 0x4000, 0x15c4: 0x4000, 0x15c5: 0x4000, + 0x15c6: 0x4000, 0x15c7: 0x4000, 0x15c8: 0x4000, 0x15c9: 0x4000, 0x15ca: 0x4000, 0x15cb: 0x4000, + 0x15cc: 0x4000, 0x15cd: 0x4000, 0x15ce: 0x4000, 0x15cf: 0x4000, 0x15d0: 0x4000, 0x15d1: 0x4000, + 0x15d2: 0x4000, 0x15d3: 0x4000, + 0x15e0: 0x4000, 0x15e1: 0x4000, 0x15e2: 0x4000, 0x15e3: 0x4000, + 0x15e4: 0x4000, 0x15e5: 0x4000, 0x15e6: 0x4000, 0x15e7: 0x4000, 0x15e8: 0x4000, 0x15e9: 0x4000, + 0x15ea: 0x4000, 0x15eb: 0x4000, 0x15ec: 0x4000, 0x15ed: 0x4000, 0x15ee: 0x4000, 0x15ef: 0x4000, + 0x15f0: 0x4000, 0x15f1: 0x4000, 0x15f2: 0x4000, 0x15f3: 0x4000, 0x15f4: 0x4000, 0x15f5: 0x4000, + 0x15f6: 0x4000, 0x15f7: 0x4000, 0x15f8: 0x4000, 0x15f9: 0x4000, 0x15fa: 0x4000, 0x15fb: 0x4000, + 0x15fc: 0x4000, 0x15fd: 0x4000, 0x15fe: 0x4000, 0x15ff: 0x4000, + // Block 0x58, offset 0x1600 + 0x1600: 0x4000, 0x1601: 0x4000, 0x1602: 0x4000, 0x1603: 0x4000, 0x1604: 0x4000, 0x1605: 0x4000, + 0x1606: 0x4000, 0x1607: 0x4000, 0x1608: 0x4000, 0x1609: 0x4000, 0x160a: 0x4000, + 0x160f: 0x4000, 0x1610: 0x4000, 0x1611: 0x4000, + 0x1612: 0x4000, 0x1613: 0x4000, + 0x1620: 0x4000, 0x1621: 0x4000, 0x1622: 0x4000, 0x1623: 0x4000, + 0x1624: 0x4000, 0x1625: 0x4000, 0x1626: 0x4000, 0x1627: 0x4000, 0x1628: 0x4000, 0x1629: 0x4000, + 0x162a: 0x4000, 0x162b: 0x4000, 0x162c: 0x4000, 0x162d: 0x4000, 0x162e: 0x4000, 0x162f: 0x4000, + 0x1630: 0x4000, 0x1634: 0x4000, + 0x1638: 0x4000, 0x1639: 0x4000, 0x163a: 0x4000, 0x163b: 0x4000, + 0x163c: 0x4000, 0x163d: 0x4000, 0x163e: 0x4000, 0x163f: 0x4000, + // Block 0x59, offset 0x1640 + 0x1640: 0x4000, 0x1641: 0x4000, 0x1642: 0x4000, 0x1643: 0x4000, 0x1644: 0x4000, 0x1645: 0x4000, + 0x1646: 0x4000, 0x1647: 0x4000, 0x1648: 0x4000, 0x1649: 0x4000, 0x164a: 0x4000, 0x164b: 0x4000, + 0x164c: 0x4000, 0x164d: 0x4000, 0x164e: 0x4000, 0x164f: 0x4000, 0x1650: 0x4000, 0x1651: 0x4000, + 0x1652: 0x4000, 0x1653: 0x4000, 0x1654: 0x4000, 0x1655: 0x4000, 0x1656: 0x4000, 0x1657: 0x4000, + 0x1658: 0x4000, 0x1659: 0x4000, 0x165a: 0x4000, 0x165b: 0x4000, 0x165c: 0x4000, 0x165d: 0x4000, + 0x165e: 0x4000, 0x165f: 0x4000, 0x1660: 0x4000, 0x1661: 0x4000, 0x1662: 0x4000, 0x1663: 0x4000, + 0x1664: 0x4000, 0x1665: 0x4000, 0x1666: 0x4000, 0x1667: 0x4000, 0x1668: 0x4000, 0x1669: 0x4000, + 0x166a: 0x4000, 0x166b: 0x4000, 0x166c: 0x4000, 0x166d: 0x4000, 0x166e: 0x4000, 0x166f: 0x4000, + 0x1670: 0x4000, 0x1671: 0x4000, 0x1672: 0x4000, 0x1673: 0x4000, 0x1674: 0x4000, 0x1675: 0x4000, + 0x1676: 0x4000, 0x1677: 0x4000, 0x1678: 0x4000, 0x1679: 0x4000, 0x167a: 0x4000, 0x167b: 0x4000, + 0x167c: 0x4000, 0x167d: 0x4000, 0x167e: 0x4000, + // Block 0x5a, offset 0x1680 + 0x1680: 0x4000, 0x1682: 0x4000, 0x1683: 0x4000, 0x1684: 0x4000, 0x1685: 0x4000, + 0x1686: 0x4000, 0x1687: 0x4000, 0x1688: 0x4000, 0x1689: 0x4000, 0x168a: 0x4000, 0x168b: 0x4000, + 0x168c: 0x4000, 0x168d: 0x4000, 0x168e: 0x4000, 0x168f: 0x4000, 0x1690: 0x4000, 0x1691: 0x4000, + 0x1692: 0x4000, 0x1693: 0x4000, 0x1694: 0x4000, 0x1695: 0x4000, 0x1696: 0x4000, 0x1697: 0x4000, + 0x1698: 0x4000, 0x1699: 0x4000, 0x169a: 0x4000, 0x169b: 0x4000, 0x169c: 0x4000, 0x169d: 0x4000, + 0x169e: 0x4000, 0x169f: 0x4000, 0x16a0: 0x4000, 0x16a1: 0x4000, 0x16a2: 0x4000, 0x16a3: 0x4000, + 0x16a4: 0x4000, 0x16a5: 0x4000, 0x16a6: 0x4000, 0x16a7: 0x4000, 0x16a8: 0x4000, 0x16a9: 0x4000, + 0x16aa: 0x4000, 0x16ab: 0x4000, 0x16ac: 0x4000, 0x16ad: 0x4000, 0x16ae: 0x4000, 0x16af: 0x4000, + 0x16b0: 0x4000, 0x16b1: 0x4000, 0x16b2: 0x4000, 0x16b3: 0x4000, 0x16b4: 0x4000, 0x16b5: 0x4000, + 0x16b6: 0x4000, 0x16b7: 0x4000, 0x16b8: 0x4000, 0x16b9: 0x4000, 0x16ba: 0x4000, 0x16bb: 0x4000, + 0x16bc: 0x4000, 0x16bd: 0x4000, 0x16be: 0x4000, 0x16bf: 0x4000, + // Block 0x5b, offset 0x16c0 + 0x16c0: 0x4000, 0x16c1: 0x4000, 0x16c2: 0x4000, 0x16c3: 0x4000, 0x16c4: 0x4000, 0x16c5: 0x4000, + 0x16c6: 0x4000, 0x16c7: 0x4000, 0x16c8: 0x4000, 0x16c9: 0x4000, 0x16ca: 0x4000, 0x16cb: 0x4000, + 0x16cc: 0x4000, 0x16cd: 0x4000, 0x16ce: 0x4000, 0x16cf: 0x4000, 0x16d0: 0x4000, 0x16d1: 0x4000, + 0x16d2: 0x4000, 0x16d3: 0x4000, 0x16d4: 0x4000, 0x16d5: 0x4000, 0x16d6: 0x4000, 0x16d7: 0x4000, + 0x16d8: 0x4000, 0x16d9: 0x4000, 0x16da: 0x4000, 0x16db: 0x4000, 0x16dc: 0x4000, 0x16dd: 0x4000, + 0x16de: 0x4000, 0x16df: 0x4000, 0x16e0: 0x4000, 0x16e1: 0x4000, 0x16e2: 0x4000, 0x16e3: 0x4000, + 0x16e4: 0x4000, 0x16e5: 0x4000, 0x16e6: 0x4000, 0x16e7: 0x4000, 0x16e8: 0x4000, 0x16e9: 0x4000, + 0x16ea: 0x4000, 0x16eb: 0x4000, 0x16ec: 0x4000, 0x16ed: 0x4000, 0x16ee: 0x4000, 0x16ef: 0x4000, + 0x16f0: 0x4000, 0x16f1: 0x4000, 0x16f2: 0x4000, 0x16f3: 0x4000, 0x16f4: 0x4000, 0x16f5: 0x4000, + 0x16f6: 0x4000, 0x16f7: 0x4000, 0x16f8: 0x4000, 0x16f9: 0x4000, 0x16fa: 0x4000, 0x16fb: 0x4000, + 0x16fc: 0x4000, 0x16ff: 0x4000, + // Block 0x5c, offset 0x1700 + 0x1700: 0x4000, 0x1701: 0x4000, 0x1702: 0x4000, 0x1703: 0x4000, 0x1704: 0x4000, 0x1705: 0x4000, + 0x1706: 0x4000, 0x1707: 0x4000, 0x1708: 0x4000, 0x1709: 0x4000, 0x170a: 0x4000, 0x170b: 0x4000, + 0x170c: 0x4000, 0x170d: 0x4000, 0x170e: 0x4000, 0x170f: 0x4000, 0x1710: 0x4000, 0x1711: 0x4000, + 0x1712: 0x4000, 0x1713: 0x4000, 0x1714: 0x4000, 0x1715: 0x4000, 0x1716: 0x4000, 0x1717: 0x4000, + 0x1718: 0x4000, 0x1719: 0x4000, 0x171a: 0x4000, 0x171b: 0x4000, 0x171c: 0x4000, 0x171d: 0x4000, + 0x171e: 0x4000, 0x171f: 0x4000, 0x1720: 0x4000, 0x1721: 0x4000, 0x1722: 0x4000, 0x1723: 0x4000, + 0x1724: 0x4000, 0x1725: 0x4000, 0x1726: 0x4000, 0x1727: 0x4000, 0x1728: 0x4000, 0x1729: 0x4000, + 0x172a: 0x4000, 0x172b: 0x4000, 0x172c: 0x4000, 0x172d: 0x4000, 0x172e: 0x4000, 0x172f: 0x4000, + 0x1730: 0x4000, 0x1731: 0x4000, 0x1732: 0x4000, 0x1733: 0x4000, 0x1734: 0x4000, 0x1735: 0x4000, + 0x1736: 0x4000, 0x1737: 0x4000, 0x1738: 0x4000, 0x1739: 0x4000, 0x173a: 0x4000, 0x173b: 0x4000, + 0x173c: 0x4000, 0x173d: 0x4000, + // Block 0x5d, offset 0x1740 + 0x174b: 0x4000, + 0x174c: 0x4000, 0x174d: 0x4000, 0x174e: 0x4000, 0x1750: 0x4000, 0x1751: 0x4000, + 0x1752: 0x4000, 0x1753: 0x4000, 0x1754: 0x4000, 0x1755: 0x4000, 0x1756: 0x4000, 0x1757: 0x4000, + 0x1758: 0x4000, 0x1759: 0x4000, 0x175a: 0x4000, 0x175b: 0x4000, 0x175c: 0x4000, 0x175d: 0x4000, + 0x175e: 0x4000, 0x175f: 0x4000, 0x1760: 0x4000, 0x1761: 0x4000, 0x1762: 0x4000, 0x1763: 0x4000, + 0x1764: 0x4000, 0x1765: 0x4000, 0x1766: 0x4000, 0x1767: 0x4000, + 0x177a: 0x4000, + // Block 0x5e, offset 0x1780 + 0x1795: 0x4000, 0x1796: 0x4000, + 0x17a4: 0x4000, + // Block 0x5f, offset 0x17c0 + 0x17fb: 0x4000, + 0x17fc: 0x4000, 0x17fd: 0x4000, 0x17fe: 0x4000, 0x17ff: 0x4000, + // Block 0x60, offset 0x1800 + 0x1800: 0x4000, 0x1801: 0x4000, 0x1802: 0x4000, 0x1803: 0x4000, 0x1804: 0x4000, 0x1805: 0x4000, + 0x1806: 0x4000, 0x1807: 0x4000, 0x1808: 0x4000, 0x1809: 0x4000, 0x180a: 0x4000, 0x180b: 0x4000, + 0x180c: 0x4000, 0x180d: 0x4000, 0x180e: 0x4000, 0x180f: 0x4000, + // Block 0x61, offset 0x1840 + 0x1840: 0x4000, 0x1841: 0x4000, 0x1842: 0x4000, 0x1843: 0x4000, 0x1844: 0x4000, 0x1845: 0x4000, + 0x184c: 0x4000, 0x1850: 0x4000, 0x1851: 0x4000, + 0x1852: 0x4000, 0x1855: 0x4000, 0x1856: 0x4000, 0x1857: 0x4000, + 0x185c: 0x4000, 0x185d: 0x4000, + 0x185e: 0x4000, 0x185f: 0x4000, + 0x186b: 0x4000, 0x186c: 0x4000, + 0x1874: 0x4000, 0x1875: 0x4000, + 0x1876: 0x4000, 0x1877: 0x4000, 0x1878: 0x4000, 0x1879: 0x4000, 0x187a: 0x4000, 0x187b: 0x4000, + 0x187c: 0x4000, + // Block 0x62, offset 0x1880 + 0x18a0: 0x4000, 0x18a1: 0x4000, 0x18a2: 0x4000, 0x18a3: 0x4000, + 0x18a4: 0x4000, 0x18a5: 0x4000, 0x18a6: 0x4000, 0x18a7: 0x4000, 0x18a8: 0x4000, 0x18a9: 0x4000, + 0x18aa: 0x4000, 0x18ab: 0x4000, + 0x18b0: 0x4000, + // Block 0x63, offset 0x18c0 + 0x18cc: 0x4000, 0x18cd: 0x4000, 0x18ce: 0x4000, 0x18cf: 0x4000, 0x18d0: 0x4000, 0x18d1: 0x4000, + 0x18d2: 0x4000, 0x18d3: 0x4000, 0x18d4: 0x4000, 0x18d5: 0x4000, 0x18d6: 0x4000, 0x18d7: 0x4000, + 0x18d8: 0x4000, 0x18d9: 0x4000, 0x18da: 0x4000, 0x18db: 0x4000, 0x18dc: 0x4000, 0x18dd: 0x4000, + 0x18de: 0x4000, 0x18df: 0x4000, 0x18e0: 0x4000, 0x18e1: 0x4000, 0x18e2: 0x4000, 0x18e3: 0x4000, + 0x18e4: 0x4000, 0x18e5: 0x4000, 0x18e6: 0x4000, 0x18e7: 0x4000, 0x18e8: 0x4000, 0x18e9: 0x4000, + 0x18ea: 0x4000, 0x18eb: 0x4000, 0x18ec: 0x4000, 0x18ed: 0x4000, 0x18ee: 0x4000, 0x18ef: 0x4000, + 0x18f0: 0x4000, 0x18f1: 0x4000, 0x18f2: 0x4000, 0x18f3: 0x4000, 0x18f4: 0x4000, 0x18f5: 0x4000, + 0x18f6: 0x4000, 0x18f7: 0x4000, 0x18f8: 0x4000, 0x18f9: 0x4000, 0x18fa: 0x4000, + 0x18fc: 0x4000, 0x18fd: 0x4000, 0x18fe: 0x4000, 0x18ff: 0x4000, + // Block 0x64, offset 0x1900 + 0x1900: 0x4000, 0x1901: 0x4000, 0x1902: 0x4000, 0x1903: 0x4000, 0x1904: 0x4000, 0x1905: 0x4000, + 0x1907: 0x4000, 0x1908: 0x4000, 0x1909: 0x4000, 0x190a: 0x4000, 0x190b: 0x4000, + 0x190c: 0x4000, 0x190d: 0x4000, 0x190e: 0x4000, 0x190f: 0x4000, 0x1910: 0x4000, 0x1911: 0x4000, + 0x1912: 0x4000, 0x1913: 0x4000, 0x1914: 0x4000, 0x1915: 0x4000, 0x1916: 0x4000, 0x1917: 0x4000, + 0x1918: 0x4000, 0x1919: 0x4000, 0x191a: 0x4000, 0x191b: 0x4000, 0x191c: 0x4000, 0x191d: 0x4000, + 0x191e: 0x4000, 0x191f: 0x4000, 0x1920: 0x4000, 0x1921: 0x4000, 0x1922: 0x4000, 0x1923: 0x4000, + 0x1924: 0x4000, 0x1925: 0x4000, 0x1926: 0x4000, 0x1927: 0x4000, 0x1928: 0x4000, 0x1929: 0x4000, + 0x192a: 0x4000, 0x192b: 0x4000, 0x192c: 0x4000, 0x192d: 0x4000, 0x192e: 0x4000, 0x192f: 0x4000, + 0x1930: 0x4000, 0x1931: 0x4000, 0x1932: 0x4000, 0x1933: 0x4000, 0x1934: 0x4000, 0x1935: 0x4000, + 0x1936: 0x4000, 0x1937: 0x4000, 0x1938: 0x4000, 0x1939: 0x4000, 0x193a: 0x4000, 0x193b: 0x4000, + 0x193c: 0x4000, 0x193d: 0x4000, 0x193e: 0x4000, 0x193f: 0x4000, + // Block 0x65, offset 0x1940 + 0x1970: 0x4000, 0x1971: 0x4000, 0x1972: 0x4000, 0x1973: 0x4000, 0x1974: 0x4000, 0x1975: 0x4000, + 0x1976: 0x4000, 0x1977: 0x4000, 0x1978: 0x4000, 0x1979: 0x4000, 0x197a: 0x4000, 0x197b: 0x4000, + 0x197c: 0x4000, + // Block 0x66, offset 0x1980 + 0x1980: 0x4000, 0x1981: 0x4000, 0x1982: 0x4000, 0x1983: 0x4000, 0x1984: 0x4000, 0x1985: 0x4000, + 0x1986: 0x4000, 0x1987: 0x4000, 0x1988: 0x4000, + 0x1990: 0x4000, 0x1991: 0x4000, + 0x1992: 0x4000, 0x1993: 0x4000, 0x1994: 0x4000, 0x1995: 0x4000, 0x1996: 0x4000, 0x1997: 0x4000, + 0x1998: 0x4000, 0x1999: 0x4000, 0x199a: 0x4000, 0x199b: 0x4000, 0x199c: 0x4000, 0x199d: 0x4000, + 0x199e: 0x4000, 0x199f: 0x4000, 0x19a0: 0x4000, 0x19a1: 0x4000, 0x19a2: 0x4000, 0x19a3: 0x4000, + 0x19a4: 0x4000, 0x19a5: 0x4000, 0x19a6: 0x4000, 0x19a7: 0x4000, 0x19a8: 0x4000, 0x19a9: 0x4000, + 0x19aa: 0x4000, 0x19ab: 0x4000, 0x19ac: 0x4000, 0x19ad: 0x4000, 0x19ae: 0x4000, 0x19af: 0x4000, + 0x19b0: 0x4000, 0x19b1: 0x4000, 0x19b2: 0x4000, 0x19b3: 0x4000, 0x19b4: 0x4000, 0x19b5: 0x4000, + 0x19b6: 0x4000, 0x19b7: 0x4000, 0x19b8: 0x4000, 0x19b9: 0x4000, 0x19ba: 0x4000, 0x19bb: 0x4000, + 0x19bc: 0x4000, 0x19bd: 0x4000, 0x19bf: 0x4000, + // Block 0x67, offset 0x19c0 + 0x19c0: 0x4000, 0x19c1: 0x4000, 0x19c2: 0x4000, 0x19c3: 0x4000, 0x19c4: 0x4000, 0x19c5: 0x4000, + 0x19ce: 0x4000, 0x19cf: 0x4000, 0x19d0: 0x4000, 0x19d1: 0x4000, + 0x19d2: 0x4000, 0x19d3: 0x4000, 0x19d4: 0x4000, 0x19d5: 0x4000, 0x19d6: 0x4000, 0x19d7: 0x4000, + 0x19d8: 0x4000, 0x19d9: 0x4000, 0x19da: 0x4000, 0x19db: 0x4000, + 0x19e0: 0x4000, 0x19e1: 0x4000, 0x19e2: 0x4000, 0x19e3: 0x4000, + 0x19e4: 0x4000, 0x19e5: 0x4000, 0x19e6: 0x4000, 0x19e7: 0x4000, 0x19e8: 0x4000, + 0x19f0: 0x4000, 0x19f1: 0x4000, 0x19f2: 0x4000, 0x19f3: 0x4000, 0x19f4: 0x4000, 0x19f5: 0x4000, + 0x19f6: 0x4000, 0x19f7: 0x4000, 0x19f8: 0x4000, + // Block 0x68, offset 0x1a00 + 0x1a00: 0x2000, 0x1a01: 0x2000, 0x1a02: 0x2000, 0x1a03: 0x2000, 0x1a04: 0x2000, 0x1a05: 0x2000, + 0x1a06: 0x2000, 0x1a07: 0x2000, 0x1a08: 0x2000, 0x1a09: 0x2000, 0x1a0a: 0x2000, 0x1a0b: 0x2000, + 0x1a0c: 0x2000, 0x1a0d: 0x2000, 0x1a0e: 0x2000, 0x1a0f: 0x2000, 0x1a10: 0x2000, 0x1a11: 0x2000, + 0x1a12: 0x2000, 0x1a13: 0x2000, 0x1a14: 0x2000, 0x1a15: 0x2000, 0x1a16: 0x2000, 0x1a17: 0x2000, + 0x1a18: 0x2000, 0x1a19: 0x2000, 0x1a1a: 0x2000, 0x1a1b: 0x2000, 0x1a1c: 0x2000, 0x1a1d: 0x2000, + 0x1a1e: 0x2000, 0x1a1f: 0x2000, 0x1a20: 0x2000, 0x1a21: 0x2000, 0x1a22: 0x2000, 0x1a23: 0x2000, + 0x1a24: 0x2000, 0x1a25: 0x2000, 0x1a26: 0x2000, 0x1a27: 0x2000, 0x1a28: 0x2000, 0x1a29: 0x2000, + 0x1a2a: 0x2000, 0x1a2b: 0x2000, 0x1a2c: 0x2000, 0x1a2d: 0x2000, 0x1a2e: 0x2000, 0x1a2f: 0x2000, + 0x1a30: 0x2000, 0x1a31: 0x2000, 0x1a32: 0x2000, 0x1a33: 0x2000, 0x1a34: 0x2000, 0x1a35: 0x2000, + 0x1a36: 0x2000, 0x1a37: 0x2000, 0x1a38: 0x2000, 0x1a39: 0x2000, 0x1a3a: 0x2000, 0x1a3b: 0x2000, + 0x1a3c: 0x2000, 0x1a3d: 0x2000, +} + +// widthIndex: 23 blocks, 1472 entries, 1472 bytes +// Block 0 is the zero block. +var widthIndex = [1472]uint8{ + // Block 0x0, offset 0x0 + // Block 0x1, offset 0x40 + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0xc2: 0x01, 0xc3: 0x02, 0xc4: 0x03, 0xc5: 0x04, 0xc7: 0x05, + 0xc9: 0x06, 0xcb: 0x07, 0xcc: 0x08, 0xcd: 0x09, 0xce: 0x0a, 0xcf: 0x0b, + 0xd0: 0x0c, 0xd1: 0x0d, + 0xe1: 0x02, 0xe2: 0x03, 0xe3: 0x04, 0xe4: 0x05, 0xe5: 0x06, 0xe6: 0x06, 0xe7: 0x06, + 0xe8: 0x06, 0xe9: 0x06, 0xea: 0x07, 0xeb: 0x06, 0xec: 0x06, 0xed: 0x08, 0xee: 0x09, 0xef: 0x0a, + 0xf0: 0x10, 0xf3: 0x13, 0xf4: 0x14, + // Block 0x4, offset 0x100 + 0x104: 0x0e, 0x105: 0x0f, + // Block 0x5, offset 0x140 + 0x140: 0x10, 0x141: 0x11, 0x142: 0x12, 0x144: 0x13, 0x145: 0x14, 0x146: 0x15, 0x147: 0x16, + 0x148: 0x17, 0x149: 0x18, 0x14a: 0x19, 0x14c: 0x1a, 0x14f: 0x1b, + 0x151: 0x1c, 0x152: 0x08, 0x153: 0x1d, 0x154: 0x1e, 0x155: 0x1f, 0x156: 0x20, 0x157: 0x21, + 0x158: 0x22, 0x159: 0x23, 0x15a: 0x24, 0x15b: 0x25, 0x15c: 0x26, 0x15d: 0x27, 0x15e: 0x28, 0x15f: 0x29, + 0x166: 0x2a, + 0x16c: 0x2b, 0x16d: 0x2c, + 0x17a: 0x2d, 0x17b: 0x2e, 0x17c: 0x0e, 0x17d: 0x0e, 0x17e: 0x0e, 0x17f: 0x2f, + // Block 0x6, offset 0x180 + 0x180: 0x30, 0x181: 0x31, 0x182: 0x32, 0x183: 0x33, 0x184: 0x34, 0x185: 0x35, 0x186: 0x36, 0x187: 0x37, + 0x188: 0x38, 0x189: 0x39, 0x18a: 0x0e, 0x18b: 0x0e, 0x18c: 0x0e, 0x18d: 0x0e, 0x18e: 0x0e, 0x18f: 0x0e, + 0x190: 0x0e, 0x191: 0x0e, 0x192: 0x0e, 0x193: 0x0e, 0x194: 0x0e, 0x195: 0x0e, 0x196: 0x0e, 0x197: 0x0e, + 0x198: 0x0e, 0x199: 0x0e, 0x19a: 0x0e, 0x19b: 0x0e, 0x19c: 0x0e, 0x19d: 0x0e, 0x19e: 0x0e, 0x19f: 0x0e, + 0x1a0: 0x0e, 0x1a1: 0x0e, 0x1a2: 0x0e, 0x1a3: 0x0e, 0x1a4: 0x0e, 0x1a5: 0x0e, 0x1a6: 0x0e, 0x1a7: 0x0e, + 0x1a8: 0x0e, 0x1a9: 0x0e, 0x1aa: 0x0e, 0x1ab: 0x0e, 0x1ac: 0x0e, 0x1ad: 0x0e, 0x1ae: 0x0e, 0x1af: 0x0e, + 0x1b0: 0x0e, 0x1b1: 0x0e, 0x1b2: 0x0e, 0x1b3: 0x0e, 0x1b4: 0x0e, 0x1b5: 0x0e, 0x1b6: 0x0e, 0x1b7: 0x0e, + 0x1b8: 0x0e, 0x1b9: 0x0e, 0x1ba: 0x0e, 0x1bb: 0x0e, 0x1bc: 0x0e, 0x1bd: 0x0e, 0x1be: 0x0e, 0x1bf: 0x0e, + // Block 0x7, offset 0x1c0 + 0x1c0: 0x0e, 0x1c1: 0x0e, 0x1c2: 0x0e, 0x1c3: 0x0e, 0x1c4: 0x0e, 0x1c5: 0x0e, 0x1c6: 0x0e, 0x1c7: 0x0e, + 0x1c8: 0x0e, 0x1c9: 0x0e, 0x1ca: 0x0e, 0x1cb: 0x0e, 0x1cc: 0x0e, 0x1cd: 0x0e, 0x1ce: 0x0e, 0x1cf: 0x0e, + 0x1d0: 0x0e, 0x1d1: 0x0e, 0x1d2: 0x0e, 0x1d3: 0x0e, 0x1d4: 0x0e, 0x1d5: 0x0e, 0x1d6: 0x0e, 0x1d7: 0x0e, + 0x1d8: 0x0e, 0x1d9: 0x0e, 0x1da: 0x0e, 0x1db: 0x0e, 0x1dc: 0x0e, 0x1dd: 0x0e, 0x1de: 0x0e, 0x1df: 0x0e, + 0x1e0: 0x0e, 0x1e1: 0x0e, 0x1e2: 0x0e, 0x1e3: 0x0e, 0x1e4: 0x0e, 0x1e5: 0x0e, 0x1e6: 0x0e, 0x1e7: 0x0e, + 0x1e8: 0x0e, 0x1e9: 0x0e, 0x1ea: 0x0e, 0x1eb: 0x0e, 0x1ec: 0x0e, 0x1ed: 0x0e, 0x1ee: 0x0e, 0x1ef: 0x0e, + 0x1f0: 0x0e, 0x1f1: 0x0e, 0x1f2: 0x0e, 0x1f3: 0x0e, 0x1f4: 0x0e, 0x1f5: 0x0e, 0x1f6: 0x0e, + 0x1f8: 0x0e, 0x1f9: 0x0e, 0x1fa: 0x0e, 0x1fb: 0x0e, 0x1fc: 0x0e, 0x1fd: 0x0e, 0x1fe: 0x0e, 0x1ff: 0x0e, + // Block 0x8, offset 0x200 + 0x200: 0x0e, 0x201: 0x0e, 0x202: 0x0e, 0x203: 0x0e, 0x204: 0x0e, 0x205: 0x0e, 0x206: 0x0e, 0x207: 0x0e, + 0x208: 0x0e, 0x209: 0x0e, 0x20a: 0x0e, 0x20b: 0x0e, 0x20c: 0x0e, 0x20d: 0x0e, 0x20e: 0x0e, 0x20f: 0x0e, + 0x210: 0x0e, 0x211: 0x0e, 0x212: 0x0e, 0x213: 0x0e, 0x214: 0x0e, 0x215: 0x0e, 0x216: 0x0e, 0x217: 0x0e, + 0x218: 0x0e, 0x219: 0x0e, 0x21a: 0x0e, 0x21b: 0x0e, 0x21c: 0x0e, 0x21d: 0x0e, 0x21e: 0x0e, 0x21f: 0x0e, + 0x220: 0x0e, 0x221: 0x0e, 0x222: 0x0e, 0x223: 0x0e, 0x224: 0x0e, 0x225: 0x0e, 0x226: 0x0e, 0x227: 0x0e, + 0x228: 0x0e, 0x229: 0x0e, 0x22a: 0x0e, 0x22b: 0x0e, 0x22c: 0x0e, 0x22d: 0x0e, 0x22e: 0x0e, 0x22f: 0x0e, + 0x230: 0x0e, 0x231: 0x0e, 0x232: 0x0e, 0x233: 0x0e, 0x234: 0x0e, 0x235: 0x0e, 0x236: 0x0e, 0x237: 0x0e, + 0x238: 0x0e, 0x239: 0x0e, 0x23a: 0x0e, 0x23b: 0x0e, 0x23c: 0x0e, 0x23d: 0x0e, 0x23e: 0x0e, 0x23f: 0x0e, + // Block 0x9, offset 0x240 + 0x240: 0x0e, 0x241: 0x0e, 0x242: 0x0e, 0x243: 0x0e, 0x244: 0x0e, 0x245: 0x0e, 0x246: 0x0e, 0x247: 0x0e, + 0x248: 0x0e, 0x249: 0x0e, 0x24a: 0x0e, 0x24b: 0x0e, 0x24c: 0x0e, 0x24d: 0x0e, 0x24e: 0x0e, 0x24f: 0x0e, + 0x250: 0x0e, 0x251: 0x0e, 0x252: 0x3a, 0x253: 0x3b, + 0x265: 0x3c, + 0x270: 0x0e, 0x271: 0x0e, 0x272: 0x0e, 0x273: 0x0e, 0x274: 0x0e, 0x275: 0x0e, 0x276: 0x0e, 0x277: 0x0e, + 0x278: 0x0e, 0x279: 0x0e, 0x27a: 0x0e, 0x27b: 0x0e, 0x27c: 0x0e, 0x27d: 0x0e, 0x27e: 0x0e, 0x27f: 0x0e, + // Block 0xa, offset 0x280 + 0x280: 0x0e, 0x281: 0x0e, 0x282: 0x0e, 0x283: 0x0e, 0x284: 0x0e, 0x285: 0x0e, 0x286: 0x0e, 0x287: 0x0e, + 0x288: 0x0e, 0x289: 0x0e, 0x28a: 0x0e, 0x28b: 0x0e, 0x28c: 0x0e, 0x28d: 0x0e, 0x28e: 0x0e, 0x28f: 0x0e, + 0x290: 0x0e, 0x291: 0x0e, 0x292: 0x0e, 0x293: 0x0e, 0x294: 0x0e, 0x295: 0x0e, 0x296: 0x0e, 0x297: 0x0e, + 0x298: 0x0e, 0x299: 0x0e, 0x29a: 0x0e, 0x29b: 0x0e, 0x29c: 0x0e, 0x29d: 0x0e, 0x29e: 0x3d, + // Block 0xb, offset 0x2c0 + 0x2c0: 0x08, 0x2c1: 0x08, 0x2c2: 0x08, 0x2c3: 0x08, 0x2c4: 0x08, 0x2c5: 0x08, 0x2c6: 0x08, 0x2c7: 0x08, + 0x2c8: 0x08, 0x2c9: 0x08, 0x2ca: 0x08, 0x2cb: 0x08, 0x2cc: 0x08, 0x2cd: 0x08, 0x2ce: 0x08, 0x2cf: 0x08, + 0x2d0: 0x08, 0x2d1: 0x08, 0x2d2: 0x08, 0x2d3: 0x08, 0x2d4: 0x08, 0x2d5: 0x08, 0x2d6: 0x08, 0x2d7: 0x08, + 0x2d8: 0x08, 0x2d9: 0x08, 0x2da: 0x08, 0x2db: 0x08, 0x2dc: 0x08, 0x2dd: 0x08, 0x2de: 0x08, 0x2df: 0x08, + 0x2e0: 0x08, 0x2e1: 0x08, 0x2e2: 0x08, 0x2e3: 0x08, 0x2e4: 0x08, 0x2e5: 0x08, 0x2e6: 0x08, 0x2e7: 0x08, + 0x2e8: 0x08, 0x2e9: 0x08, 0x2ea: 0x08, 0x2eb: 0x08, 0x2ec: 0x08, 0x2ed: 0x08, 0x2ee: 0x08, 0x2ef: 0x08, + 0x2f0: 0x08, 0x2f1: 0x08, 0x2f2: 0x08, 0x2f3: 0x08, 0x2f4: 0x08, 0x2f5: 0x08, 0x2f6: 0x08, 0x2f7: 0x08, + 0x2f8: 0x08, 0x2f9: 0x08, 0x2fa: 0x08, 0x2fb: 0x08, 0x2fc: 0x08, 0x2fd: 0x08, 0x2fe: 0x08, 0x2ff: 0x08, + // Block 0xc, offset 0x300 + 0x300: 0x08, 0x301: 0x08, 0x302: 0x08, 0x303: 0x08, 0x304: 0x08, 0x305: 0x08, 0x306: 0x08, 0x307: 0x08, + 0x308: 0x08, 0x309: 0x08, 0x30a: 0x08, 0x30b: 0x08, 0x30c: 0x08, 0x30d: 0x08, 0x30e: 0x08, 0x30f: 0x08, + 0x310: 0x08, 0x311: 0x08, 0x312: 0x08, 0x313: 0x08, 0x314: 0x08, 0x315: 0x08, 0x316: 0x08, 0x317: 0x08, + 0x318: 0x08, 0x319: 0x08, 0x31a: 0x08, 0x31b: 0x08, 0x31c: 0x08, 0x31d: 0x08, 0x31e: 0x08, 0x31f: 0x08, + 0x320: 0x08, 0x321: 0x08, 0x322: 0x08, 0x323: 0x08, 0x324: 0x0e, 0x325: 0x0e, 0x326: 0x0e, 0x327: 0x0e, + 0x328: 0x0e, 0x329: 0x0e, 0x32a: 0x0e, 0x32b: 0x0e, + 0x338: 0x3e, 0x339: 0x3f, 0x33c: 0x40, 0x33d: 0x41, 0x33e: 0x42, 0x33f: 0x43, + // Block 0xd, offset 0x340 + 0x37f: 0x44, + // Block 0xe, offset 0x380 + 0x380: 0x0e, 0x381: 0x0e, 0x382: 0x0e, 0x383: 0x0e, 0x384: 0x0e, 0x385: 0x0e, 0x386: 0x0e, 0x387: 0x0e, + 0x388: 0x0e, 0x389: 0x0e, 0x38a: 0x0e, 0x38b: 0x0e, 0x38c: 0x0e, 0x38d: 0x0e, 0x38e: 0x0e, 0x38f: 0x0e, + 0x390: 0x0e, 0x391: 0x0e, 0x392: 0x0e, 0x393: 0x0e, 0x394: 0x0e, 0x395: 0x0e, 0x396: 0x0e, 0x397: 0x0e, + 0x398: 0x0e, 0x399: 0x0e, 0x39a: 0x0e, 0x39b: 0x0e, 0x39c: 0x0e, 0x39d: 0x0e, 0x39e: 0x0e, 0x39f: 0x45, + 0x3a0: 0x0e, 0x3a1: 0x0e, 0x3a2: 0x0e, 0x3a3: 0x0e, 0x3a4: 0x0e, 0x3a5: 0x0e, 0x3a6: 0x0e, 0x3a7: 0x0e, + 0x3a8: 0x0e, 0x3a9: 0x0e, 0x3aa: 0x0e, 0x3ab: 0x0e, 0x3ac: 0x0e, 0x3ad: 0x0e, 0x3ae: 0x0e, 0x3af: 0x0e, + 0x3b0: 0x0e, 0x3b1: 0x0e, 0x3b2: 0x0e, 0x3b3: 0x46, 0x3b4: 0x47, + // Block 0xf, offset 0x3c0 + 0x3ff: 0x48, + // Block 0x10, offset 0x400 + 0x400: 0x0e, 0x401: 0x0e, 0x402: 0x0e, 0x403: 0x0e, 0x404: 0x49, 0x405: 0x4a, 0x406: 0x0e, 0x407: 0x0e, + 0x408: 0x0e, 0x409: 0x0e, 0x40a: 0x0e, 0x40b: 0x4b, + // Block 0x11, offset 0x440 + 0x440: 0x4c, 0x443: 0x4d, 0x444: 0x4e, 0x445: 0x4f, 0x446: 0x50, + 0x448: 0x51, 0x449: 0x52, 0x44c: 0x53, 0x44d: 0x54, 0x44e: 0x55, 0x44f: 0x56, + 0x450: 0x57, 0x451: 0x58, 0x452: 0x0e, 0x453: 0x59, 0x454: 0x5a, 0x455: 0x5b, 0x456: 0x5c, 0x457: 0x5d, + 0x458: 0x0e, 0x459: 0x5e, 0x45a: 0x0e, 0x45b: 0x5f, 0x45f: 0x60, + 0x464: 0x61, 0x465: 0x62, 0x466: 0x0e, 0x467: 0x0e, + 0x469: 0x63, 0x46a: 0x64, 0x46b: 0x65, + // Block 0x12, offset 0x480 + 0x496: 0x0b, 0x497: 0x06, + 0x498: 0x0c, 0x49a: 0x0d, 0x49b: 0x0e, 0x49f: 0x0f, + 0x4a0: 0x06, 0x4a1: 0x06, 0x4a2: 0x06, 0x4a3: 0x06, 0x4a4: 0x06, 0x4a5: 0x06, 0x4a6: 0x06, 0x4a7: 0x06, + 0x4a8: 0x06, 0x4a9: 0x06, 0x4aa: 0x06, 0x4ab: 0x06, 0x4ac: 0x06, 0x4ad: 0x06, 0x4ae: 0x06, 0x4af: 0x06, + 0x4b0: 0x06, 0x4b1: 0x06, 0x4b2: 0x06, 0x4b3: 0x06, 0x4b4: 0x06, 0x4b5: 0x06, 0x4b6: 0x06, 0x4b7: 0x06, + 0x4b8: 0x06, 0x4b9: 0x06, 0x4ba: 0x06, 0x4bb: 0x06, 0x4bc: 0x06, 0x4bd: 0x06, 0x4be: 0x06, 0x4bf: 0x06, + // Block 0x13, offset 0x4c0 + 0x4c4: 0x08, 0x4c5: 0x08, 0x4c6: 0x08, 0x4c7: 0x09, + // Block 0x14, offset 0x500 + 0x500: 0x08, 0x501: 0x08, 0x502: 0x08, 0x503: 0x08, 0x504: 0x08, 0x505: 0x08, 0x506: 0x08, 0x507: 0x08, + 0x508: 0x08, 0x509: 0x08, 0x50a: 0x08, 0x50b: 0x08, 0x50c: 0x08, 0x50d: 0x08, 0x50e: 0x08, 0x50f: 0x08, + 0x510: 0x08, 0x511: 0x08, 0x512: 0x08, 0x513: 0x08, 0x514: 0x08, 0x515: 0x08, 0x516: 0x08, 0x517: 0x08, + 0x518: 0x08, 0x519: 0x08, 0x51a: 0x08, 0x51b: 0x08, 0x51c: 0x08, 0x51d: 0x08, 0x51e: 0x08, 0x51f: 0x08, + 0x520: 0x08, 0x521: 0x08, 0x522: 0x08, 0x523: 0x08, 0x524: 0x08, 0x525: 0x08, 0x526: 0x08, 0x527: 0x08, + 0x528: 0x08, 0x529: 0x08, 0x52a: 0x08, 0x52b: 0x08, 0x52c: 0x08, 0x52d: 0x08, 0x52e: 0x08, 0x52f: 0x08, + 0x530: 0x08, 0x531: 0x08, 0x532: 0x08, 0x533: 0x08, 0x534: 0x08, 0x535: 0x08, 0x536: 0x08, 0x537: 0x08, + 0x538: 0x08, 0x539: 0x08, 0x53a: 0x08, 0x53b: 0x08, 0x53c: 0x08, 0x53d: 0x08, 0x53e: 0x08, 0x53f: 0x66, + // Block 0x15, offset 0x540 + 0x560: 0x11, + 0x570: 0x09, 0x571: 0x09, 0x572: 0x09, 0x573: 0x09, 0x574: 0x09, 0x575: 0x09, 0x576: 0x09, 0x577: 0x09, + 0x578: 0x09, 0x579: 0x09, 0x57a: 0x09, 0x57b: 0x09, 0x57c: 0x09, 0x57d: 0x09, 0x57e: 0x09, 0x57f: 0x12, + // Block 0x16, offset 0x580 + 0x580: 0x09, 0x581: 0x09, 0x582: 0x09, 0x583: 0x09, 0x584: 0x09, 0x585: 0x09, 0x586: 0x09, 0x587: 0x09, + 0x588: 0x09, 0x589: 0x09, 0x58a: 0x09, 0x58b: 0x09, 0x58c: 0x09, 0x58d: 0x09, 0x58e: 0x09, 0x58f: 0x12, +} + +// inverseData contains 4-byte entries of the following format: +// +// <0 padding> +// +// The last byte of the UTF-8-encoded rune is xor-ed with the last byte of the +// UTF-8 encoding of the original rune. Mappings often have the following +// pattern: +// +// Ａ -> A (U+FF21 -> U+0041) +// Ｂ -> B (U+FF22 -> U+0042) +// ... +// +// By xor-ing the last byte the same entry can be shared by many mappings. This +// reduces the total number of distinct entries by about two thirds. +// The resulting entry for the aforementioned mappings is +// +// { 0x01, 0xE0, 0x00, 0x00 } +// +// Using this entry to map U+FF21 (UTF-8 [EF BC A1]), we get +// +// E0 ^ A1 = 41. +// +// Similarly, for U+FF22 (UTF-8 [EF BC A2]), we get +// +// E0 ^ A2 = 42. +// +// Note that because of the xor-ing, the byte sequence stored in the entry is +// not valid UTF-8. +var inverseData = [150][4]byte{ + {0x00, 0x00, 0x00, 0x00}, + {0x03, 0xe3, 0x80, 0xa0}, + {0x03, 0xef, 0xbc, 0xa0}, + {0x03, 0xef, 0xbc, 0xe0}, + {0x03, 0xef, 0xbd, 0xe0}, + {0x03, 0xef, 0xbf, 0x02}, + {0x03, 0xef, 0xbf, 0x00}, + {0x03, 0xef, 0xbf, 0x0e}, + {0x03, 0xef, 0xbf, 0x0c}, + {0x03, 0xef, 0xbf, 0x0f}, + {0x03, 0xef, 0xbf, 0x39}, + {0x03, 0xef, 0xbf, 0x3b}, + {0x03, 0xef, 0xbf, 0x3f}, + {0x03, 0xef, 0xbf, 0x2a}, + {0x03, 0xef, 0xbf, 0x0d}, + {0x03, 0xef, 0xbf, 0x25}, + {0x03, 0xef, 0xbd, 0x1a}, + {0x03, 0xef, 0xbd, 0x26}, + {0x01, 0xa0, 0x00, 0x00}, + {0x03, 0xef, 0xbd, 0x25}, + {0x03, 0xef, 0xbd, 0x23}, + {0x03, 0xef, 0xbd, 0x2e}, + {0x03, 0xef, 0xbe, 0x07}, + {0x03, 0xef, 0xbe, 0x05}, + {0x03, 0xef, 0xbd, 0x06}, + {0x03, 0xef, 0xbd, 0x13}, + {0x03, 0xef, 0xbd, 0x0b}, + {0x03, 0xef, 0xbd, 0x16}, + {0x03, 0xef, 0xbd, 0x0c}, + {0x03, 0xef, 0xbd, 0x15}, + {0x03, 0xef, 0xbd, 0x0d}, + {0x03, 0xef, 0xbd, 0x1c}, + {0x03, 0xef, 0xbd, 0x02}, + {0x03, 0xef, 0xbd, 0x1f}, + {0x03, 0xef, 0xbd, 0x1d}, + {0x03, 0xef, 0xbd, 0x17}, + {0x03, 0xef, 0xbd, 0x08}, + {0x03, 0xef, 0xbd, 0x09}, + {0x03, 0xef, 0xbd, 0x0e}, + {0x03, 0xef, 0xbd, 0x04}, + {0x03, 0xef, 0xbd, 0x05}, + {0x03, 0xef, 0xbe, 0x3f}, + {0x03, 0xef, 0xbe, 0x00}, + {0x03, 0xef, 0xbd, 0x2c}, + {0x03, 0xef, 0xbe, 0x06}, + {0x03, 0xef, 0xbe, 0x0c}, + {0x03, 0xef, 0xbe, 0x0f}, + {0x03, 0xef, 0xbe, 0x0d}, + {0x03, 0xef, 0xbe, 0x0b}, + {0x03, 0xef, 0xbe, 0x19}, + {0x03, 0xef, 0xbe, 0x15}, + {0x03, 0xef, 0xbe, 0x11}, + {0x03, 0xef, 0xbe, 0x31}, + {0x03, 0xef, 0xbe, 0x33}, + {0x03, 0xef, 0xbd, 0x0f}, + {0x03, 0xef, 0xbe, 0x30}, + {0x03, 0xef, 0xbe, 0x3e}, + {0x03, 0xef, 0xbe, 0x32}, + {0x03, 0xef, 0xbe, 0x36}, + {0x03, 0xef, 0xbd, 0x14}, + {0x03, 0xef, 0xbe, 0x2e}, + {0x03, 0xef, 0xbd, 0x1e}, + {0x03, 0xef, 0xbe, 0x10}, + {0x03, 0xef, 0xbf, 0x13}, + {0x03, 0xef, 0xbf, 0x15}, + {0x03, 0xef, 0xbf, 0x17}, + {0x03, 0xef, 0xbf, 0x1f}, + {0x03, 0xef, 0xbf, 0x1d}, + {0x03, 0xef, 0xbf, 0x1b}, + {0x03, 0xef, 0xbf, 0x09}, + {0x03, 0xef, 0xbf, 0x0b}, + {0x03, 0xef, 0xbf, 0x37}, + {0x03, 0xef, 0xbe, 0x04}, + {0x01, 0xe0, 0x00, 0x00}, + {0x03, 0xe2, 0xa6, 0x1a}, + {0x03, 0xe2, 0xa6, 0x26}, + {0x03, 0xe3, 0x80, 0x23}, + {0x03, 0xe3, 0x80, 0x2e}, + {0x03, 0xe3, 0x80, 0x25}, + {0x03, 0xe3, 0x83, 0x1e}, + {0x03, 0xe3, 0x83, 0x14}, + {0x03, 0xe3, 0x82, 0x06}, + {0x03, 0xe3, 0x82, 0x0b}, + {0x03, 0xe3, 0x82, 0x0c}, + {0x03, 0xe3, 0x82, 0x0d}, + {0x03, 0xe3, 0x82, 0x02}, + {0x03, 0xe3, 0x83, 0x0f}, + {0x03, 0xe3, 0x83, 0x08}, + {0x03, 0xe3, 0x83, 0x09}, + {0x03, 0xe3, 0x83, 0x2c}, + {0x03, 0xe3, 0x83, 0x0c}, + {0x03, 0xe3, 0x82, 0x13}, + {0x03, 0xe3, 0x82, 0x16}, + {0x03, 0xe3, 0x82, 0x15}, + {0x03, 0xe3, 0x82, 0x1c}, + {0x03, 0xe3, 0x82, 0x1f}, + {0x03, 0xe3, 0x82, 0x1d}, + {0x03, 0xe3, 0x82, 0x1a}, + {0x03, 0xe3, 0x82, 0x17}, + {0x03, 0xe3, 0x82, 0x08}, + {0x03, 0xe3, 0x82, 0x09}, + {0x03, 0xe3, 0x82, 0x0e}, + {0x03, 0xe3, 0x82, 0x04}, + {0x03, 0xe3, 0x82, 0x05}, + {0x03, 0xe3, 0x82, 0x3f}, + {0x03, 0xe3, 0x83, 0x00}, + {0x03, 0xe3, 0x83, 0x06}, + {0x03, 0xe3, 0x83, 0x05}, + {0x03, 0xe3, 0x83, 0x0d}, + {0x03, 0xe3, 0x83, 0x0b}, + {0x03, 0xe3, 0x83, 0x07}, + {0x03, 0xe3, 0x83, 0x19}, + {0x03, 0xe3, 0x83, 0x15}, + {0x03, 0xe3, 0x83, 0x11}, + {0x03, 0xe3, 0x83, 0x31}, + {0x03, 0xe3, 0x83, 0x33}, + {0x03, 0xe3, 0x83, 0x30}, + {0x03, 0xe3, 0x83, 0x3e}, + {0x03, 0xe3, 0x83, 0x32}, + {0x03, 0xe3, 0x83, 0x36}, + {0x03, 0xe3, 0x83, 0x2e}, + {0x03, 0xe3, 0x82, 0x07}, + {0x03, 0xe3, 0x85, 0x04}, + {0x03, 0xe3, 0x84, 0x10}, + {0x03, 0xe3, 0x85, 0x30}, + {0x03, 0xe3, 0x85, 0x0d}, + {0x03, 0xe3, 0x85, 0x13}, + {0x03, 0xe3, 0x85, 0x15}, + {0x03, 0xe3, 0x85, 0x17}, + {0x03, 0xe3, 0x85, 0x1f}, + {0x03, 0xe3, 0x85, 0x1d}, + {0x03, 0xe3, 0x85, 0x1b}, + {0x03, 0xe3, 0x85, 0x09}, + {0x03, 0xe3, 0x85, 0x0f}, + {0x03, 0xe3, 0x85, 0x0b}, + {0x03, 0xe3, 0x85, 0x37}, + {0x03, 0xe3, 0x85, 0x3b}, + {0x03, 0xe3, 0x85, 0x39}, + {0x03, 0xe3, 0x85, 0x3f}, + {0x02, 0xc2, 0x02, 0x00}, + {0x02, 0xc2, 0x0e, 0x00}, + {0x02, 0xc2, 0x0c, 0x00}, + {0x02, 0xc2, 0x00, 0x00}, + {0x03, 0xe2, 0x82, 0x0f}, + {0x03, 0xe2, 0x94, 0x2a}, + {0x03, 0xe2, 0x86, 0x39}, + {0x03, 0xe2, 0x86, 0x3b}, + {0x03, 0xe2, 0x86, 0x3f}, + {0x03, 0xe2, 0x96, 0x0d}, + {0x03, 0xe2, 0x97, 0x25}, +} + +// Total table size 15512 bytes (15KiB) diff --git a/vendor/golang.org/x/text/width/tables9.0.0.go b/vendor/golang.org/x/text/width/tables9.0.0.go new file mode 100644 index 0000000000..6781f3d960 --- /dev/null +++ b/vendor/golang.org/x/text/width/tables9.0.0.go @@ -0,0 +1,1297 @@ +// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. + +//go:build !go1.10 +// +build !go1.10 + +package width + +// UnicodeVersion is the Unicode version from which the tables in this package are derived. +const UnicodeVersion = "9.0.0" + +// lookup returns the trie value for the first UTF-8 encoding in s and +// the width in bytes of this encoding. The size will be 0 if s does not +// hold enough bytes to complete the encoding. len(s) must be greater than 0. +func (t *widthTrie) lookup(s []byte) (v uint16, sz int) { + c0 := s[0] + switch { + case c0 < 0x80: // is ASCII + return widthValues[c0], 1 + case c0 < 0xC2: + return 0, 1 // Illegal UTF-8: not a starter, not ASCII. + case c0 < 0xE0: // 2-byte UTF-8 + if len(s) < 2 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c1), 2 + case c0 < 0xF0: // 3-byte UTF-8 + if len(s) < 3 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = widthIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c2), 3 + case c0 < 0xF8: // 4-byte UTF-8 + if len(s) < 4 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = widthIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + o = uint32(i)<<6 + uint32(c2) + i = widthIndex[o] + c3 := s[3] + if c3 < 0x80 || 0xC0 <= c3 { + return 0, 3 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c3), 4 + } + // Illegal rune + return 0, 1 +} + +// lookupUnsafe returns the trie value for the first UTF-8 encoding in s. +// s must start with a full and valid UTF-8 encoded rune. +func (t *widthTrie) lookupUnsafe(s []byte) uint16 { + c0 := s[0] + if c0 < 0x80 { // is ASCII + return widthValues[c0] + } + i := widthIndex[c0] + if c0 < 0xE0 { // 2-byte UTF-8 + return t.lookupValue(uint32(i), s[1]) + } + i = widthIndex[uint32(i)<<6+uint32(s[1])] + if c0 < 0xF0 { // 3-byte UTF-8 + return t.lookupValue(uint32(i), s[2]) + } + i = widthIndex[uint32(i)<<6+uint32(s[2])] + if c0 < 0xF8 { // 4-byte UTF-8 + return t.lookupValue(uint32(i), s[3]) + } + return 0 +} + +// lookupString returns the trie value for the first UTF-8 encoding in s and +// the width in bytes of this encoding. The size will be 0 if s does not +// hold enough bytes to complete the encoding. len(s) must be greater than 0. +func (t *widthTrie) lookupString(s string) (v uint16, sz int) { + c0 := s[0] + switch { + case c0 < 0x80: // is ASCII + return widthValues[c0], 1 + case c0 < 0xC2: + return 0, 1 // Illegal UTF-8: not a starter, not ASCII. + case c0 < 0xE0: // 2-byte UTF-8 + if len(s) < 2 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c1), 2 + case c0 < 0xF0: // 3-byte UTF-8 + if len(s) < 3 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = widthIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c2), 3 + case c0 < 0xF8: // 4-byte UTF-8 + if len(s) < 4 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = widthIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + o = uint32(i)<<6 + uint32(c2) + i = widthIndex[o] + c3 := s[3] + if c3 < 0x80 || 0xC0 <= c3 { + return 0, 3 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c3), 4 + } + // Illegal rune + return 0, 1 +} + +// lookupStringUnsafe returns the trie value for the first UTF-8 encoding in s. +// s must start with a full and valid UTF-8 encoded rune. +func (t *widthTrie) lookupStringUnsafe(s string) uint16 { + c0 := s[0] + if c0 < 0x80 { // is ASCII + return widthValues[c0] + } + i := widthIndex[c0] + if c0 < 0xE0 { // 2-byte UTF-8 + return t.lookupValue(uint32(i), s[1]) + } + i = widthIndex[uint32(i)<<6+uint32(s[1])] + if c0 < 0xF0 { // 3-byte UTF-8 + return t.lookupValue(uint32(i), s[2]) + } + i = widthIndex[uint32(i)<<6+uint32(s[2])] + if c0 < 0xF8 { // 4-byte UTF-8 + return t.lookupValue(uint32(i), s[3]) + } + return 0 +} + +// widthTrie. Total size: 14080 bytes (13.75 KiB). Checksum: 3b8aeb3dc03667a3. +type widthTrie struct{} + +func newWidthTrie(i int) *widthTrie { + return &widthTrie{} +} + +// lookupValue determines the type of block n and looks up the value for b. +func (t *widthTrie) lookupValue(n uint32, b byte) uint16 { + switch { + default: + return uint16(widthValues[n<<6+uint32(b)]) + } +} + +// widthValues: 99 blocks, 6336 entries, 12672 bytes +// The third block is the zero block. +var widthValues = [6336]uint16{ + // Block 0x0, offset 0x0 + 0x20: 0x6001, 0x21: 0x6002, 0x22: 0x6002, 0x23: 0x6002, + 0x24: 0x6002, 0x25: 0x6002, 0x26: 0x6002, 0x27: 0x6002, 0x28: 0x6002, 0x29: 0x6002, + 0x2a: 0x6002, 0x2b: 0x6002, 0x2c: 0x6002, 0x2d: 0x6002, 0x2e: 0x6002, 0x2f: 0x6002, + 0x30: 0x6002, 0x31: 0x6002, 0x32: 0x6002, 0x33: 0x6002, 0x34: 0x6002, 0x35: 0x6002, + 0x36: 0x6002, 0x37: 0x6002, 0x38: 0x6002, 0x39: 0x6002, 0x3a: 0x6002, 0x3b: 0x6002, + 0x3c: 0x6002, 0x3d: 0x6002, 0x3e: 0x6002, 0x3f: 0x6002, + // Block 0x1, offset 0x40 + 0x40: 0x6003, 0x41: 0x6003, 0x42: 0x6003, 0x43: 0x6003, 0x44: 0x6003, 0x45: 0x6003, + 0x46: 0x6003, 0x47: 0x6003, 0x48: 0x6003, 0x49: 0x6003, 0x4a: 0x6003, 0x4b: 0x6003, + 0x4c: 0x6003, 0x4d: 0x6003, 0x4e: 0x6003, 0x4f: 0x6003, 0x50: 0x6003, 0x51: 0x6003, + 0x52: 0x6003, 0x53: 0x6003, 0x54: 0x6003, 0x55: 0x6003, 0x56: 0x6003, 0x57: 0x6003, + 0x58: 0x6003, 0x59: 0x6003, 0x5a: 0x6003, 0x5b: 0x6003, 0x5c: 0x6003, 0x5d: 0x6003, + 0x5e: 0x6003, 0x5f: 0x6003, 0x60: 0x6004, 0x61: 0x6004, 0x62: 0x6004, 0x63: 0x6004, + 0x64: 0x6004, 0x65: 0x6004, 0x66: 0x6004, 0x67: 0x6004, 0x68: 0x6004, 0x69: 0x6004, + 0x6a: 0x6004, 0x6b: 0x6004, 0x6c: 0x6004, 0x6d: 0x6004, 0x6e: 0x6004, 0x6f: 0x6004, + 0x70: 0x6004, 0x71: 0x6004, 0x72: 0x6004, 0x73: 0x6004, 0x74: 0x6004, 0x75: 0x6004, + 0x76: 0x6004, 0x77: 0x6004, 0x78: 0x6004, 0x79: 0x6004, 0x7a: 0x6004, 0x7b: 0x6004, + 0x7c: 0x6004, 0x7d: 0x6004, 0x7e: 0x6004, + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0xe1: 0x2000, 0xe2: 0x6005, 0xe3: 0x6005, + 0xe4: 0x2000, 0xe5: 0x6006, 0xe6: 0x6005, 0xe7: 0x2000, 0xe8: 0x2000, + 0xea: 0x2000, 0xec: 0x6007, 0xed: 0x2000, 0xee: 0x2000, 0xef: 0x6008, + 0xf0: 0x2000, 0xf1: 0x2000, 0xf2: 0x2000, 0xf3: 0x2000, 0xf4: 0x2000, + 0xf6: 0x2000, 0xf7: 0x2000, 0xf8: 0x2000, 0xf9: 0x2000, 0xfa: 0x2000, + 0xfc: 0x2000, 0xfd: 0x2000, 0xfe: 0x2000, 0xff: 0x2000, + // Block 0x4, offset 0x100 + 0x106: 0x2000, + 0x110: 0x2000, + 0x117: 0x2000, + 0x118: 0x2000, + 0x11e: 0x2000, 0x11f: 0x2000, 0x120: 0x2000, 0x121: 0x2000, + 0x126: 0x2000, 0x128: 0x2000, 0x129: 0x2000, + 0x12a: 0x2000, 0x12c: 0x2000, 0x12d: 0x2000, + 0x130: 0x2000, 0x132: 0x2000, 0x133: 0x2000, + 0x137: 0x2000, 0x138: 0x2000, 0x139: 0x2000, 0x13a: 0x2000, + 0x13c: 0x2000, 0x13e: 0x2000, + // Block 0x5, offset 0x140 + 0x141: 0x2000, + 0x151: 0x2000, + 0x153: 0x2000, + 0x15b: 0x2000, + 0x166: 0x2000, 0x167: 0x2000, + 0x16b: 0x2000, + 0x171: 0x2000, 0x172: 0x2000, 0x173: 0x2000, + 0x178: 0x2000, + 0x17f: 0x2000, + // Block 0x6, offset 0x180 + 0x180: 0x2000, 0x181: 0x2000, 0x182: 0x2000, 0x184: 0x2000, + 0x188: 0x2000, 0x189: 0x2000, 0x18a: 0x2000, 0x18b: 0x2000, + 0x18d: 0x2000, + 0x192: 0x2000, 0x193: 0x2000, + 0x1a6: 0x2000, 0x1a7: 0x2000, + 0x1ab: 0x2000, + // Block 0x7, offset 0x1c0 + 0x1ce: 0x2000, 0x1d0: 0x2000, + 0x1d2: 0x2000, 0x1d4: 0x2000, 0x1d6: 0x2000, + 0x1d8: 0x2000, 0x1da: 0x2000, 0x1dc: 0x2000, + // Block 0x8, offset 0x200 + 0x211: 0x2000, + 0x221: 0x2000, + // Block 0x9, offset 0x240 + 0x244: 0x2000, + 0x247: 0x2000, 0x249: 0x2000, 0x24a: 0x2000, 0x24b: 0x2000, + 0x24d: 0x2000, 0x250: 0x2000, + 0x258: 0x2000, 0x259: 0x2000, 0x25a: 0x2000, 0x25b: 0x2000, 0x25d: 0x2000, + 0x25f: 0x2000, + // Block 0xa, offset 0x280 + 0x280: 0x2000, 0x281: 0x2000, 0x282: 0x2000, 0x283: 0x2000, 0x284: 0x2000, 0x285: 0x2000, + 0x286: 0x2000, 0x287: 0x2000, 0x288: 0x2000, 0x289: 0x2000, 0x28a: 0x2000, 0x28b: 0x2000, + 0x28c: 0x2000, 0x28d: 0x2000, 0x28e: 0x2000, 0x28f: 0x2000, 0x290: 0x2000, 0x291: 0x2000, + 0x292: 0x2000, 0x293: 0x2000, 0x294: 0x2000, 0x295: 0x2000, 0x296: 0x2000, 0x297: 0x2000, + 0x298: 0x2000, 0x299: 0x2000, 0x29a: 0x2000, 0x29b: 0x2000, 0x29c: 0x2000, 0x29d: 0x2000, + 0x29e: 0x2000, 0x29f: 0x2000, 0x2a0: 0x2000, 0x2a1: 0x2000, 0x2a2: 0x2000, 0x2a3: 0x2000, + 0x2a4: 0x2000, 0x2a5: 0x2000, 0x2a6: 0x2000, 0x2a7: 0x2000, 0x2a8: 0x2000, 0x2a9: 0x2000, + 0x2aa: 0x2000, 0x2ab: 0x2000, 0x2ac: 0x2000, 0x2ad: 0x2000, 0x2ae: 0x2000, 0x2af: 0x2000, + 0x2b0: 0x2000, 0x2b1: 0x2000, 0x2b2: 0x2000, 0x2b3: 0x2000, 0x2b4: 0x2000, 0x2b5: 0x2000, + 0x2b6: 0x2000, 0x2b7: 0x2000, 0x2b8: 0x2000, 0x2b9: 0x2000, 0x2ba: 0x2000, 0x2bb: 0x2000, + 0x2bc: 0x2000, 0x2bd: 0x2000, 0x2be: 0x2000, 0x2bf: 0x2000, + // Block 0xb, offset 0x2c0 + 0x2c0: 0x2000, 0x2c1: 0x2000, 0x2c2: 0x2000, 0x2c3: 0x2000, 0x2c4: 0x2000, 0x2c5: 0x2000, + 0x2c6: 0x2000, 0x2c7: 0x2000, 0x2c8: 0x2000, 0x2c9: 0x2000, 0x2ca: 0x2000, 0x2cb: 0x2000, + 0x2cc: 0x2000, 0x2cd: 0x2000, 0x2ce: 0x2000, 0x2cf: 0x2000, 0x2d0: 0x2000, 0x2d1: 0x2000, + 0x2d2: 0x2000, 0x2d3: 0x2000, 0x2d4: 0x2000, 0x2d5: 0x2000, 0x2d6: 0x2000, 0x2d7: 0x2000, + 0x2d8: 0x2000, 0x2d9: 0x2000, 0x2da: 0x2000, 0x2db: 0x2000, 0x2dc: 0x2000, 0x2dd: 0x2000, + 0x2de: 0x2000, 0x2df: 0x2000, 0x2e0: 0x2000, 0x2e1: 0x2000, 0x2e2: 0x2000, 0x2e3: 0x2000, + 0x2e4: 0x2000, 0x2e5: 0x2000, 0x2e6: 0x2000, 0x2e7: 0x2000, 0x2e8: 0x2000, 0x2e9: 0x2000, + 0x2ea: 0x2000, 0x2eb: 0x2000, 0x2ec: 0x2000, 0x2ed: 0x2000, 0x2ee: 0x2000, 0x2ef: 0x2000, + // Block 0xc, offset 0x300 + 0x311: 0x2000, + 0x312: 0x2000, 0x313: 0x2000, 0x314: 0x2000, 0x315: 0x2000, 0x316: 0x2000, 0x317: 0x2000, + 0x318: 0x2000, 0x319: 0x2000, 0x31a: 0x2000, 0x31b: 0x2000, 0x31c: 0x2000, 0x31d: 0x2000, + 0x31e: 0x2000, 0x31f: 0x2000, 0x320: 0x2000, 0x321: 0x2000, 0x323: 0x2000, + 0x324: 0x2000, 0x325: 0x2000, 0x326: 0x2000, 0x327: 0x2000, 0x328: 0x2000, 0x329: 0x2000, + 0x331: 0x2000, 0x332: 0x2000, 0x333: 0x2000, 0x334: 0x2000, 0x335: 0x2000, + 0x336: 0x2000, 0x337: 0x2000, 0x338: 0x2000, 0x339: 0x2000, 0x33a: 0x2000, 0x33b: 0x2000, + 0x33c: 0x2000, 0x33d: 0x2000, 0x33e: 0x2000, 0x33f: 0x2000, + // Block 0xd, offset 0x340 + 0x340: 0x2000, 0x341: 0x2000, 0x343: 0x2000, 0x344: 0x2000, 0x345: 0x2000, + 0x346: 0x2000, 0x347: 0x2000, 0x348: 0x2000, 0x349: 0x2000, + // Block 0xe, offset 0x380 + 0x381: 0x2000, + 0x390: 0x2000, 0x391: 0x2000, + 0x392: 0x2000, 0x393: 0x2000, 0x394: 0x2000, 0x395: 0x2000, 0x396: 0x2000, 0x397: 0x2000, + 0x398: 0x2000, 0x399: 0x2000, 0x39a: 0x2000, 0x39b: 0x2000, 0x39c: 0x2000, 0x39d: 0x2000, + 0x39e: 0x2000, 0x39f: 0x2000, 0x3a0: 0x2000, 0x3a1: 0x2000, 0x3a2: 0x2000, 0x3a3: 0x2000, + 0x3a4: 0x2000, 0x3a5: 0x2000, 0x3a6: 0x2000, 0x3a7: 0x2000, 0x3a8: 0x2000, 0x3a9: 0x2000, + 0x3aa: 0x2000, 0x3ab: 0x2000, 0x3ac: 0x2000, 0x3ad: 0x2000, 0x3ae: 0x2000, 0x3af: 0x2000, + 0x3b0: 0x2000, 0x3b1: 0x2000, 0x3b2: 0x2000, 0x3b3: 0x2000, 0x3b4: 0x2000, 0x3b5: 0x2000, + 0x3b6: 0x2000, 0x3b7: 0x2000, 0x3b8: 0x2000, 0x3b9: 0x2000, 0x3ba: 0x2000, 0x3bb: 0x2000, + 0x3bc: 0x2000, 0x3bd: 0x2000, 0x3be: 0x2000, 0x3bf: 0x2000, + // Block 0xf, offset 0x3c0 + 0x3c0: 0x2000, 0x3c1: 0x2000, 0x3c2: 0x2000, 0x3c3: 0x2000, 0x3c4: 0x2000, 0x3c5: 0x2000, + 0x3c6: 0x2000, 0x3c7: 0x2000, 0x3c8: 0x2000, 0x3c9: 0x2000, 0x3ca: 0x2000, 0x3cb: 0x2000, + 0x3cc: 0x2000, 0x3cd: 0x2000, 0x3ce: 0x2000, 0x3cf: 0x2000, 0x3d1: 0x2000, + // Block 0x10, offset 0x400 + 0x400: 0x4000, 0x401: 0x4000, 0x402: 0x4000, 0x403: 0x4000, 0x404: 0x4000, 0x405: 0x4000, + 0x406: 0x4000, 0x407: 0x4000, 0x408: 0x4000, 0x409: 0x4000, 0x40a: 0x4000, 0x40b: 0x4000, + 0x40c: 0x4000, 0x40d: 0x4000, 0x40e: 0x4000, 0x40f: 0x4000, 0x410: 0x4000, 0x411: 0x4000, + 0x412: 0x4000, 0x413: 0x4000, 0x414: 0x4000, 0x415: 0x4000, 0x416: 0x4000, 0x417: 0x4000, + 0x418: 0x4000, 0x419: 0x4000, 0x41a: 0x4000, 0x41b: 0x4000, 0x41c: 0x4000, 0x41d: 0x4000, + 0x41e: 0x4000, 0x41f: 0x4000, 0x420: 0x4000, 0x421: 0x4000, 0x422: 0x4000, 0x423: 0x4000, + 0x424: 0x4000, 0x425: 0x4000, 0x426: 0x4000, 0x427: 0x4000, 0x428: 0x4000, 0x429: 0x4000, + 0x42a: 0x4000, 0x42b: 0x4000, 0x42c: 0x4000, 0x42d: 0x4000, 0x42e: 0x4000, 0x42f: 0x4000, + 0x430: 0x4000, 0x431: 0x4000, 0x432: 0x4000, 0x433: 0x4000, 0x434: 0x4000, 0x435: 0x4000, + 0x436: 0x4000, 0x437: 0x4000, 0x438: 0x4000, 0x439: 0x4000, 0x43a: 0x4000, 0x43b: 0x4000, + 0x43c: 0x4000, 0x43d: 0x4000, 0x43e: 0x4000, 0x43f: 0x4000, + // Block 0x11, offset 0x440 + 0x440: 0x4000, 0x441: 0x4000, 0x442: 0x4000, 0x443: 0x4000, 0x444: 0x4000, 0x445: 0x4000, + 0x446: 0x4000, 0x447: 0x4000, 0x448: 0x4000, 0x449: 0x4000, 0x44a: 0x4000, 0x44b: 0x4000, + 0x44c: 0x4000, 0x44d: 0x4000, 0x44e: 0x4000, 0x44f: 0x4000, 0x450: 0x4000, 0x451: 0x4000, + 0x452: 0x4000, 0x453: 0x4000, 0x454: 0x4000, 0x455: 0x4000, 0x456: 0x4000, 0x457: 0x4000, + 0x458: 0x4000, 0x459: 0x4000, 0x45a: 0x4000, 0x45b: 0x4000, 0x45c: 0x4000, 0x45d: 0x4000, + 0x45e: 0x4000, 0x45f: 0x4000, + // Block 0x12, offset 0x480 + 0x490: 0x2000, + 0x493: 0x2000, 0x494: 0x2000, 0x495: 0x2000, 0x496: 0x2000, + 0x498: 0x2000, 0x499: 0x2000, 0x49c: 0x2000, 0x49d: 0x2000, + 0x4a0: 0x2000, 0x4a1: 0x2000, 0x4a2: 0x2000, + 0x4a4: 0x2000, 0x4a5: 0x2000, 0x4a6: 0x2000, 0x4a7: 0x2000, + 0x4b0: 0x2000, 0x4b2: 0x2000, 0x4b3: 0x2000, 0x4b5: 0x2000, + 0x4bb: 0x2000, + 0x4be: 0x2000, + // Block 0x13, offset 0x4c0 + 0x4f4: 0x2000, + 0x4ff: 0x2000, + // Block 0x14, offset 0x500 + 0x501: 0x2000, 0x502: 0x2000, 0x503: 0x2000, 0x504: 0x2000, + 0x529: 0xa009, + 0x52c: 0x2000, + // Block 0x15, offset 0x540 + 0x543: 0x2000, 0x545: 0x2000, + 0x549: 0x2000, + 0x553: 0x2000, 0x556: 0x2000, + 0x561: 0x2000, 0x562: 0x2000, + 0x566: 0x2000, + 0x56b: 0x2000, + // Block 0x16, offset 0x580 + 0x593: 0x2000, 0x594: 0x2000, + 0x59b: 0x2000, 0x59c: 0x2000, 0x59d: 0x2000, + 0x59e: 0x2000, 0x5a0: 0x2000, 0x5a1: 0x2000, 0x5a2: 0x2000, 0x5a3: 0x2000, + 0x5a4: 0x2000, 0x5a5: 0x2000, 0x5a6: 0x2000, 0x5a7: 0x2000, 0x5a8: 0x2000, 0x5a9: 0x2000, + 0x5aa: 0x2000, 0x5ab: 0x2000, + 0x5b0: 0x2000, 0x5b1: 0x2000, 0x5b2: 0x2000, 0x5b3: 0x2000, 0x5b4: 0x2000, 0x5b5: 0x2000, + 0x5b6: 0x2000, 0x5b7: 0x2000, 0x5b8: 0x2000, 0x5b9: 0x2000, + // Block 0x17, offset 0x5c0 + 0x5c9: 0x2000, + 0x5d0: 0x200a, 0x5d1: 0x200b, + 0x5d2: 0x200a, 0x5d3: 0x200c, 0x5d4: 0x2000, 0x5d5: 0x2000, 0x5d6: 0x2000, 0x5d7: 0x2000, + 0x5d8: 0x2000, 0x5d9: 0x2000, + 0x5f8: 0x2000, 0x5f9: 0x2000, + // Block 0x18, offset 0x600 + 0x612: 0x2000, 0x614: 0x2000, + 0x627: 0x2000, + // Block 0x19, offset 0x640 + 0x640: 0x2000, 0x642: 0x2000, 0x643: 0x2000, + 0x647: 0x2000, 0x648: 0x2000, 0x64b: 0x2000, + 0x64f: 0x2000, 0x651: 0x2000, + 0x655: 0x2000, + 0x65a: 0x2000, 0x65d: 0x2000, + 0x65e: 0x2000, 0x65f: 0x2000, 0x660: 0x2000, 0x663: 0x2000, + 0x665: 0x2000, 0x667: 0x2000, 0x668: 0x2000, 0x669: 0x2000, + 0x66a: 0x2000, 0x66b: 0x2000, 0x66c: 0x2000, 0x66e: 0x2000, + 0x674: 0x2000, 0x675: 0x2000, + 0x676: 0x2000, 0x677: 0x2000, + 0x67c: 0x2000, 0x67d: 0x2000, + // Block 0x1a, offset 0x680 + 0x688: 0x2000, + 0x68c: 0x2000, + 0x692: 0x2000, + 0x6a0: 0x2000, 0x6a1: 0x2000, + 0x6a4: 0x2000, 0x6a5: 0x2000, 0x6a6: 0x2000, 0x6a7: 0x2000, + 0x6aa: 0x2000, 0x6ab: 0x2000, 0x6ae: 0x2000, 0x6af: 0x2000, + // Block 0x1b, offset 0x6c0 + 0x6c2: 0x2000, 0x6c3: 0x2000, + 0x6c6: 0x2000, 0x6c7: 0x2000, + 0x6d5: 0x2000, + 0x6d9: 0x2000, + 0x6e5: 0x2000, + 0x6ff: 0x2000, + // Block 0x1c, offset 0x700 + 0x712: 0x2000, + 0x71a: 0x4000, 0x71b: 0x4000, + 0x729: 0x4000, + 0x72a: 0x4000, + // Block 0x1d, offset 0x740 + 0x769: 0x4000, + 0x76a: 0x4000, 0x76b: 0x4000, 0x76c: 0x4000, + 0x770: 0x4000, 0x773: 0x4000, + // Block 0x1e, offset 0x780 + 0x7a0: 0x2000, 0x7a1: 0x2000, 0x7a2: 0x2000, 0x7a3: 0x2000, + 0x7a4: 0x2000, 0x7a5: 0x2000, 0x7a6: 0x2000, 0x7a7: 0x2000, 0x7a8: 0x2000, 0x7a9: 0x2000, + 0x7aa: 0x2000, 0x7ab: 0x2000, 0x7ac: 0x2000, 0x7ad: 0x2000, 0x7ae: 0x2000, 0x7af: 0x2000, + 0x7b0: 0x2000, 0x7b1: 0x2000, 0x7b2: 0x2000, 0x7b3: 0x2000, 0x7b4: 0x2000, 0x7b5: 0x2000, + 0x7b6: 0x2000, 0x7b7: 0x2000, 0x7b8: 0x2000, 0x7b9: 0x2000, 0x7ba: 0x2000, 0x7bb: 0x2000, + 0x7bc: 0x2000, 0x7bd: 0x2000, 0x7be: 0x2000, 0x7bf: 0x2000, + // Block 0x1f, offset 0x7c0 + 0x7c0: 0x2000, 0x7c1: 0x2000, 0x7c2: 0x2000, 0x7c3: 0x2000, 0x7c4: 0x2000, 0x7c5: 0x2000, + 0x7c6: 0x2000, 0x7c7: 0x2000, 0x7c8: 0x2000, 0x7c9: 0x2000, 0x7ca: 0x2000, 0x7cb: 0x2000, + 0x7cc: 0x2000, 0x7cd: 0x2000, 0x7ce: 0x2000, 0x7cf: 0x2000, 0x7d0: 0x2000, 0x7d1: 0x2000, + 0x7d2: 0x2000, 0x7d3: 0x2000, 0x7d4: 0x2000, 0x7d5: 0x2000, 0x7d6: 0x2000, 0x7d7: 0x2000, + 0x7d8: 0x2000, 0x7d9: 0x2000, 0x7da: 0x2000, 0x7db: 0x2000, 0x7dc: 0x2000, 0x7dd: 0x2000, + 0x7de: 0x2000, 0x7df: 0x2000, 0x7e0: 0x2000, 0x7e1: 0x2000, 0x7e2: 0x2000, 0x7e3: 0x2000, + 0x7e4: 0x2000, 0x7e5: 0x2000, 0x7e6: 0x2000, 0x7e7: 0x2000, 0x7e8: 0x2000, 0x7e9: 0x2000, + 0x7eb: 0x2000, 0x7ec: 0x2000, 0x7ed: 0x2000, 0x7ee: 0x2000, 0x7ef: 0x2000, + 0x7f0: 0x2000, 0x7f1: 0x2000, 0x7f2: 0x2000, 0x7f3: 0x2000, 0x7f4: 0x2000, 0x7f5: 0x2000, + 0x7f6: 0x2000, 0x7f7: 0x2000, 0x7f8: 0x2000, 0x7f9: 0x2000, 0x7fa: 0x2000, 0x7fb: 0x2000, + 0x7fc: 0x2000, 0x7fd: 0x2000, 0x7fe: 0x2000, 0x7ff: 0x2000, + // Block 0x20, offset 0x800 + 0x800: 0x2000, 0x801: 0x2000, 0x802: 0x200d, 0x803: 0x2000, 0x804: 0x2000, 0x805: 0x2000, + 0x806: 0x2000, 0x807: 0x2000, 0x808: 0x2000, 0x809: 0x2000, 0x80a: 0x2000, 0x80b: 0x2000, + 0x80c: 0x2000, 0x80d: 0x2000, 0x80e: 0x2000, 0x80f: 0x2000, 0x810: 0x2000, 0x811: 0x2000, + 0x812: 0x2000, 0x813: 0x2000, 0x814: 0x2000, 0x815: 0x2000, 0x816: 0x2000, 0x817: 0x2000, + 0x818: 0x2000, 0x819: 0x2000, 0x81a: 0x2000, 0x81b: 0x2000, 0x81c: 0x2000, 0x81d: 0x2000, + 0x81e: 0x2000, 0x81f: 0x2000, 0x820: 0x2000, 0x821: 0x2000, 0x822: 0x2000, 0x823: 0x2000, + 0x824: 0x2000, 0x825: 0x2000, 0x826: 0x2000, 0x827: 0x2000, 0x828: 0x2000, 0x829: 0x2000, + 0x82a: 0x2000, 0x82b: 0x2000, 0x82c: 0x2000, 0x82d: 0x2000, 0x82e: 0x2000, 0x82f: 0x2000, + 0x830: 0x2000, 0x831: 0x2000, 0x832: 0x2000, 0x833: 0x2000, 0x834: 0x2000, 0x835: 0x2000, + 0x836: 0x2000, 0x837: 0x2000, 0x838: 0x2000, 0x839: 0x2000, 0x83a: 0x2000, 0x83b: 0x2000, + 0x83c: 0x2000, 0x83d: 0x2000, 0x83e: 0x2000, 0x83f: 0x2000, + // Block 0x21, offset 0x840 + 0x840: 0x2000, 0x841: 0x2000, 0x842: 0x2000, 0x843: 0x2000, 0x844: 0x2000, 0x845: 0x2000, + 0x846: 0x2000, 0x847: 0x2000, 0x848: 0x2000, 0x849: 0x2000, 0x84a: 0x2000, 0x84b: 0x2000, + 0x850: 0x2000, 0x851: 0x2000, + 0x852: 0x2000, 0x853: 0x2000, 0x854: 0x2000, 0x855: 0x2000, 0x856: 0x2000, 0x857: 0x2000, + 0x858: 0x2000, 0x859: 0x2000, 0x85a: 0x2000, 0x85b: 0x2000, 0x85c: 0x2000, 0x85d: 0x2000, + 0x85e: 0x2000, 0x85f: 0x2000, 0x860: 0x2000, 0x861: 0x2000, 0x862: 0x2000, 0x863: 0x2000, + 0x864: 0x2000, 0x865: 0x2000, 0x866: 0x2000, 0x867: 0x2000, 0x868: 0x2000, 0x869: 0x2000, + 0x86a: 0x2000, 0x86b: 0x2000, 0x86c: 0x2000, 0x86d: 0x2000, 0x86e: 0x2000, 0x86f: 0x2000, + 0x870: 0x2000, 0x871: 0x2000, 0x872: 0x2000, 0x873: 0x2000, + // Block 0x22, offset 0x880 + 0x880: 0x2000, 0x881: 0x2000, 0x882: 0x2000, 0x883: 0x2000, 0x884: 0x2000, 0x885: 0x2000, + 0x886: 0x2000, 0x887: 0x2000, 0x888: 0x2000, 0x889: 0x2000, 0x88a: 0x2000, 0x88b: 0x2000, + 0x88c: 0x2000, 0x88d: 0x2000, 0x88e: 0x2000, 0x88f: 0x2000, + 0x892: 0x2000, 0x893: 0x2000, 0x894: 0x2000, 0x895: 0x2000, + 0x8a0: 0x200e, 0x8a1: 0x2000, 0x8a3: 0x2000, + 0x8a4: 0x2000, 0x8a5: 0x2000, 0x8a6: 0x2000, 0x8a7: 0x2000, 0x8a8: 0x2000, 0x8a9: 0x2000, + 0x8b2: 0x2000, 0x8b3: 0x2000, + 0x8b6: 0x2000, 0x8b7: 0x2000, + 0x8bc: 0x2000, 0x8bd: 0x2000, + // Block 0x23, offset 0x8c0 + 0x8c0: 0x2000, 0x8c1: 0x2000, + 0x8c6: 0x2000, 0x8c7: 0x2000, 0x8c8: 0x2000, 0x8cb: 0x200f, + 0x8ce: 0x2000, 0x8cf: 0x2000, 0x8d0: 0x2000, 0x8d1: 0x2000, + 0x8e2: 0x2000, 0x8e3: 0x2000, + 0x8e4: 0x2000, 0x8e5: 0x2000, + 0x8ef: 0x2000, + 0x8fd: 0x4000, 0x8fe: 0x4000, + // Block 0x24, offset 0x900 + 0x905: 0x2000, + 0x906: 0x2000, 0x909: 0x2000, + 0x90e: 0x2000, 0x90f: 0x2000, + 0x914: 0x4000, 0x915: 0x4000, + 0x91c: 0x2000, + 0x91e: 0x2000, + // Block 0x25, offset 0x940 + 0x940: 0x2000, 0x942: 0x2000, + 0x948: 0x4000, 0x949: 0x4000, 0x94a: 0x4000, 0x94b: 0x4000, + 0x94c: 0x4000, 0x94d: 0x4000, 0x94e: 0x4000, 0x94f: 0x4000, 0x950: 0x4000, 0x951: 0x4000, + 0x952: 0x4000, 0x953: 0x4000, + 0x960: 0x2000, 0x961: 0x2000, 0x963: 0x2000, + 0x964: 0x2000, 0x965: 0x2000, 0x967: 0x2000, 0x968: 0x2000, 0x969: 0x2000, + 0x96a: 0x2000, 0x96c: 0x2000, 0x96d: 0x2000, 0x96f: 0x2000, + 0x97f: 0x4000, + // Block 0x26, offset 0x980 + 0x993: 0x4000, + 0x99e: 0x2000, 0x99f: 0x2000, 0x9a1: 0x4000, + 0x9aa: 0x4000, 0x9ab: 0x4000, + 0x9bd: 0x4000, 0x9be: 0x4000, 0x9bf: 0x2000, + // Block 0x27, offset 0x9c0 + 0x9c4: 0x4000, 0x9c5: 0x4000, + 0x9c6: 0x2000, 0x9c7: 0x2000, 0x9c8: 0x2000, 0x9c9: 0x2000, 0x9ca: 0x2000, 0x9cb: 0x2000, + 0x9cc: 0x2000, 0x9cd: 0x2000, 0x9ce: 0x4000, 0x9cf: 0x2000, 0x9d0: 0x2000, 0x9d1: 0x2000, + 0x9d2: 0x2000, 0x9d3: 0x2000, 0x9d4: 0x4000, 0x9d5: 0x2000, 0x9d6: 0x2000, 0x9d7: 0x2000, + 0x9d8: 0x2000, 0x9d9: 0x2000, 0x9da: 0x2000, 0x9db: 0x2000, 0x9dc: 0x2000, 0x9dd: 0x2000, + 0x9de: 0x2000, 0x9df: 0x2000, 0x9e0: 0x2000, 0x9e1: 0x2000, 0x9e3: 0x2000, + 0x9e8: 0x2000, 0x9e9: 0x2000, + 0x9ea: 0x4000, 0x9eb: 0x2000, 0x9ec: 0x2000, 0x9ed: 0x2000, 0x9ee: 0x2000, 0x9ef: 0x2000, + 0x9f0: 0x2000, 0x9f1: 0x2000, 0x9f2: 0x4000, 0x9f3: 0x4000, 0x9f4: 0x2000, 0x9f5: 0x4000, + 0x9f6: 0x2000, 0x9f7: 0x2000, 0x9f8: 0x2000, 0x9f9: 0x2000, 0x9fa: 0x4000, 0x9fb: 0x2000, + 0x9fc: 0x2000, 0x9fd: 0x4000, 0x9fe: 0x2000, 0x9ff: 0x2000, + // Block 0x28, offset 0xa00 + 0xa05: 0x4000, + 0xa0a: 0x4000, 0xa0b: 0x4000, + 0xa28: 0x4000, + 0xa3d: 0x2000, + // Block 0x29, offset 0xa40 + 0xa4c: 0x4000, 0xa4e: 0x4000, + 0xa53: 0x4000, 0xa54: 0x4000, 0xa55: 0x4000, 0xa57: 0x4000, + 0xa76: 0x2000, 0xa77: 0x2000, 0xa78: 0x2000, 0xa79: 0x2000, 0xa7a: 0x2000, 0xa7b: 0x2000, + 0xa7c: 0x2000, 0xa7d: 0x2000, 0xa7e: 0x2000, 0xa7f: 0x2000, + // Block 0x2a, offset 0xa80 + 0xa95: 0x4000, 0xa96: 0x4000, 0xa97: 0x4000, + 0xab0: 0x4000, + 0xabf: 0x4000, + // Block 0x2b, offset 0xac0 + 0xae6: 0x6000, 0xae7: 0x6000, 0xae8: 0x6000, 0xae9: 0x6000, + 0xaea: 0x6000, 0xaeb: 0x6000, 0xaec: 0x6000, 0xaed: 0x6000, + // Block 0x2c, offset 0xb00 + 0xb05: 0x6010, + 0xb06: 0x6011, + // Block 0x2d, offset 0xb40 + 0xb5b: 0x4000, 0xb5c: 0x4000, + // Block 0x2e, offset 0xb80 + 0xb90: 0x4000, + 0xb95: 0x4000, 0xb96: 0x2000, 0xb97: 0x2000, + 0xb98: 0x2000, 0xb99: 0x2000, + // Block 0x2f, offset 0xbc0 + 0xbc0: 0x4000, 0xbc1: 0x4000, 0xbc2: 0x4000, 0xbc3: 0x4000, 0xbc4: 0x4000, 0xbc5: 0x4000, + 0xbc6: 0x4000, 0xbc7: 0x4000, 0xbc8: 0x4000, 0xbc9: 0x4000, 0xbca: 0x4000, 0xbcb: 0x4000, + 0xbcc: 0x4000, 0xbcd: 0x4000, 0xbce: 0x4000, 0xbcf: 0x4000, 0xbd0: 0x4000, 0xbd1: 0x4000, + 0xbd2: 0x4000, 0xbd3: 0x4000, 0xbd4: 0x4000, 0xbd5: 0x4000, 0xbd6: 0x4000, 0xbd7: 0x4000, + 0xbd8: 0x4000, 0xbd9: 0x4000, 0xbdb: 0x4000, 0xbdc: 0x4000, 0xbdd: 0x4000, + 0xbde: 0x4000, 0xbdf: 0x4000, 0xbe0: 0x4000, 0xbe1: 0x4000, 0xbe2: 0x4000, 0xbe3: 0x4000, + 0xbe4: 0x4000, 0xbe5: 0x4000, 0xbe6: 0x4000, 0xbe7: 0x4000, 0xbe8: 0x4000, 0xbe9: 0x4000, + 0xbea: 0x4000, 0xbeb: 0x4000, 0xbec: 0x4000, 0xbed: 0x4000, 0xbee: 0x4000, 0xbef: 0x4000, + 0xbf0: 0x4000, 0xbf1: 0x4000, 0xbf2: 0x4000, 0xbf3: 0x4000, 0xbf4: 0x4000, 0xbf5: 0x4000, + 0xbf6: 0x4000, 0xbf7: 0x4000, 0xbf8: 0x4000, 0xbf9: 0x4000, 0xbfa: 0x4000, 0xbfb: 0x4000, + 0xbfc: 0x4000, 0xbfd: 0x4000, 0xbfe: 0x4000, 0xbff: 0x4000, + // Block 0x30, offset 0xc00 + 0xc00: 0x4000, 0xc01: 0x4000, 0xc02: 0x4000, 0xc03: 0x4000, 0xc04: 0x4000, 0xc05: 0x4000, + 0xc06: 0x4000, 0xc07: 0x4000, 0xc08: 0x4000, 0xc09: 0x4000, 0xc0a: 0x4000, 0xc0b: 0x4000, + 0xc0c: 0x4000, 0xc0d: 0x4000, 0xc0e: 0x4000, 0xc0f: 0x4000, 0xc10: 0x4000, 0xc11: 0x4000, + 0xc12: 0x4000, 0xc13: 0x4000, 0xc14: 0x4000, 0xc15: 0x4000, 0xc16: 0x4000, 0xc17: 0x4000, + 0xc18: 0x4000, 0xc19: 0x4000, 0xc1a: 0x4000, 0xc1b: 0x4000, 0xc1c: 0x4000, 0xc1d: 0x4000, + 0xc1e: 0x4000, 0xc1f: 0x4000, 0xc20: 0x4000, 0xc21: 0x4000, 0xc22: 0x4000, 0xc23: 0x4000, + 0xc24: 0x4000, 0xc25: 0x4000, 0xc26: 0x4000, 0xc27: 0x4000, 0xc28: 0x4000, 0xc29: 0x4000, + 0xc2a: 0x4000, 0xc2b: 0x4000, 0xc2c: 0x4000, 0xc2d: 0x4000, 0xc2e: 0x4000, 0xc2f: 0x4000, + 0xc30: 0x4000, 0xc31: 0x4000, 0xc32: 0x4000, 0xc33: 0x4000, + // Block 0x31, offset 0xc40 + 0xc40: 0x4000, 0xc41: 0x4000, 0xc42: 0x4000, 0xc43: 0x4000, 0xc44: 0x4000, 0xc45: 0x4000, + 0xc46: 0x4000, 0xc47: 0x4000, 0xc48: 0x4000, 0xc49: 0x4000, 0xc4a: 0x4000, 0xc4b: 0x4000, + 0xc4c: 0x4000, 0xc4d: 0x4000, 0xc4e: 0x4000, 0xc4f: 0x4000, 0xc50: 0x4000, 0xc51: 0x4000, + 0xc52: 0x4000, 0xc53: 0x4000, 0xc54: 0x4000, 0xc55: 0x4000, + 0xc70: 0x4000, 0xc71: 0x4000, 0xc72: 0x4000, 0xc73: 0x4000, 0xc74: 0x4000, 0xc75: 0x4000, + 0xc76: 0x4000, 0xc77: 0x4000, 0xc78: 0x4000, 0xc79: 0x4000, 0xc7a: 0x4000, 0xc7b: 0x4000, + // Block 0x32, offset 0xc80 + 0xc80: 0x9012, 0xc81: 0x4013, 0xc82: 0x4014, 0xc83: 0x4000, 0xc84: 0x4000, 0xc85: 0x4000, + 0xc86: 0x4000, 0xc87: 0x4000, 0xc88: 0x4000, 0xc89: 0x4000, 0xc8a: 0x4000, 0xc8b: 0x4000, + 0xc8c: 0x4015, 0xc8d: 0x4015, 0xc8e: 0x4000, 0xc8f: 0x4000, 0xc90: 0x4000, 0xc91: 0x4000, + 0xc92: 0x4000, 0xc93: 0x4000, 0xc94: 0x4000, 0xc95: 0x4000, 0xc96: 0x4000, 0xc97: 0x4000, + 0xc98: 0x4000, 0xc99: 0x4000, 0xc9a: 0x4000, 0xc9b: 0x4000, 0xc9c: 0x4000, 0xc9d: 0x4000, + 0xc9e: 0x4000, 0xc9f: 0x4000, 0xca0: 0x4000, 0xca1: 0x4000, 0xca2: 0x4000, 0xca3: 0x4000, + 0xca4: 0x4000, 0xca5: 0x4000, 0xca6: 0x4000, 0xca7: 0x4000, 0xca8: 0x4000, 0xca9: 0x4000, + 0xcaa: 0x4000, 0xcab: 0x4000, 0xcac: 0x4000, 0xcad: 0x4000, 0xcae: 0x4000, 0xcaf: 0x4000, + 0xcb0: 0x4000, 0xcb1: 0x4000, 0xcb2: 0x4000, 0xcb3: 0x4000, 0xcb4: 0x4000, 0xcb5: 0x4000, + 0xcb6: 0x4000, 0xcb7: 0x4000, 0xcb8: 0x4000, 0xcb9: 0x4000, 0xcba: 0x4000, 0xcbb: 0x4000, + 0xcbc: 0x4000, 0xcbd: 0x4000, 0xcbe: 0x4000, + // Block 0x33, offset 0xcc0 + 0xcc1: 0x4000, 0xcc2: 0x4000, 0xcc3: 0x4000, 0xcc4: 0x4000, 0xcc5: 0x4000, + 0xcc6: 0x4000, 0xcc7: 0x4000, 0xcc8: 0x4000, 0xcc9: 0x4000, 0xcca: 0x4000, 0xccb: 0x4000, + 0xccc: 0x4000, 0xccd: 0x4000, 0xcce: 0x4000, 0xccf: 0x4000, 0xcd0: 0x4000, 0xcd1: 0x4000, + 0xcd2: 0x4000, 0xcd3: 0x4000, 0xcd4: 0x4000, 0xcd5: 0x4000, 0xcd6: 0x4000, 0xcd7: 0x4000, + 0xcd8: 0x4000, 0xcd9: 0x4000, 0xcda: 0x4000, 0xcdb: 0x4000, 0xcdc: 0x4000, 0xcdd: 0x4000, + 0xcde: 0x4000, 0xcdf: 0x4000, 0xce0: 0x4000, 0xce1: 0x4000, 0xce2: 0x4000, 0xce3: 0x4000, + 0xce4: 0x4000, 0xce5: 0x4000, 0xce6: 0x4000, 0xce7: 0x4000, 0xce8: 0x4000, 0xce9: 0x4000, + 0xcea: 0x4000, 0xceb: 0x4000, 0xcec: 0x4000, 0xced: 0x4000, 0xcee: 0x4000, 0xcef: 0x4000, + 0xcf0: 0x4000, 0xcf1: 0x4000, 0xcf2: 0x4000, 0xcf3: 0x4000, 0xcf4: 0x4000, 0xcf5: 0x4000, + 0xcf6: 0x4000, 0xcf7: 0x4000, 0xcf8: 0x4000, 0xcf9: 0x4000, 0xcfa: 0x4000, 0xcfb: 0x4000, + 0xcfc: 0x4000, 0xcfd: 0x4000, 0xcfe: 0x4000, 0xcff: 0x4000, + // Block 0x34, offset 0xd00 + 0xd00: 0x4000, 0xd01: 0x4000, 0xd02: 0x4000, 0xd03: 0x4000, 0xd04: 0x4000, 0xd05: 0x4000, + 0xd06: 0x4000, 0xd07: 0x4000, 0xd08: 0x4000, 0xd09: 0x4000, 0xd0a: 0x4000, 0xd0b: 0x4000, + 0xd0c: 0x4000, 0xd0d: 0x4000, 0xd0e: 0x4000, 0xd0f: 0x4000, 0xd10: 0x4000, 0xd11: 0x4000, + 0xd12: 0x4000, 0xd13: 0x4000, 0xd14: 0x4000, 0xd15: 0x4000, 0xd16: 0x4000, + 0xd19: 0x4016, 0xd1a: 0x4017, 0xd1b: 0x4000, 0xd1c: 0x4000, 0xd1d: 0x4000, + 0xd1e: 0x4000, 0xd1f: 0x4000, 0xd20: 0x4000, 0xd21: 0x4018, 0xd22: 0x4019, 0xd23: 0x401a, + 0xd24: 0x401b, 0xd25: 0x401c, 0xd26: 0x401d, 0xd27: 0x401e, 0xd28: 0x401f, 0xd29: 0x4020, + 0xd2a: 0x4021, 0xd2b: 0x4022, 0xd2c: 0x4000, 0xd2d: 0x4010, 0xd2e: 0x4000, 0xd2f: 0x4023, + 0xd30: 0x4000, 0xd31: 0x4024, 0xd32: 0x4000, 0xd33: 0x4025, 0xd34: 0x4000, 0xd35: 0x4026, + 0xd36: 0x4000, 0xd37: 0x401a, 0xd38: 0x4000, 0xd39: 0x4027, 0xd3a: 0x4000, 0xd3b: 0x4028, + 0xd3c: 0x4000, 0xd3d: 0x4020, 0xd3e: 0x4000, 0xd3f: 0x4029, + // Block 0x35, offset 0xd40 + 0xd40: 0x4000, 0xd41: 0x402a, 0xd42: 0x4000, 0xd43: 0x402b, 0xd44: 0x402c, 0xd45: 0x4000, + 0xd46: 0x4017, 0xd47: 0x4000, 0xd48: 0x402d, 0xd49: 0x4000, 0xd4a: 0x402e, 0xd4b: 0x402f, + 0xd4c: 0x4030, 0xd4d: 0x4017, 0xd4e: 0x4016, 0xd4f: 0x4017, 0xd50: 0x4000, 0xd51: 0x4000, + 0xd52: 0x4031, 0xd53: 0x4000, 0xd54: 0x4000, 0xd55: 0x4031, 0xd56: 0x4000, 0xd57: 0x4000, + 0xd58: 0x4032, 0xd59: 0x4000, 0xd5a: 0x4000, 0xd5b: 0x4032, 0xd5c: 0x4000, 0xd5d: 0x4000, + 0xd5e: 0x4033, 0xd5f: 0x402e, 0xd60: 0x4034, 0xd61: 0x4035, 0xd62: 0x4034, 0xd63: 0x4036, + 0xd64: 0x4037, 0xd65: 0x4024, 0xd66: 0x4035, 0xd67: 0x4025, 0xd68: 0x4038, 0xd69: 0x4038, + 0xd6a: 0x4039, 0xd6b: 0x4039, 0xd6c: 0x403a, 0xd6d: 0x403a, 0xd6e: 0x4000, 0xd6f: 0x4035, + 0xd70: 0x4000, 0xd71: 0x4000, 0xd72: 0x403b, 0xd73: 0x403c, 0xd74: 0x4000, 0xd75: 0x4000, + 0xd76: 0x4000, 0xd77: 0x4000, 0xd78: 0x4000, 0xd79: 0x4000, 0xd7a: 0x4000, 0xd7b: 0x403d, + 0xd7c: 0x401c, 0xd7d: 0x4000, 0xd7e: 0x4000, 0xd7f: 0x4000, + // Block 0x36, offset 0xd80 + 0xd85: 0x4000, + 0xd86: 0x4000, 0xd87: 0x4000, 0xd88: 0x4000, 0xd89: 0x4000, 0xd8a: 0x4000, 0xd8b: 0x4000, + 0xd8c: 0x4000, 0xd8d: 0x4000, 0xd8e: 0x4000, 0xd8f: 0x4000, 0xd90: 0x4000, 0xd91: 0x4000, + 0xd92: 0x4000, 0xd93: 0x4000, 0xd94: 0x4000, 0xd95: 0x4000, 0xd96: 0x4000, 0xd97: 0x4000, + 0xd98: 0x4000, 0xd99: 0x4000, 0xd9a: 0x4000, 0xd9b: 0x4000, 0xd9c: 0x4000, 0xd9d: 0x4000, + 0xd9e: 0x4000, 0xd9f: 0x4000, 0xda0: 0x4000, 0xda1: 0x4000, 0xda2: 0x4000, 0xda3: 0x4000, + 0xda4: 0x4000, 0xda5: 0x4000, 0xda6: 0x4000, 0xda7: 0x4000, 0xda8: 0x4000, 0xda9: 0x4000, + 0xdaa: 0x4000, 0xdab: 0x4000, 0xdac: 0x4000, 0xdad: 0x4000, + 0xdb1: 0x403e, 0xdb2: 0x403e, 0xdb3: 0x403e, 0xdb4: 0x403e, 0xdb5: 0x403e, + 0xdb6: 0x403e, 0xdb7: 0x403e, 0xdb8: 0x403e, 0xdb9: 0x403e, 0xdba: 0x403e, 0xdbb: 0x403e, + 0xdbc: 0x403e, 0xdbd: 0x403e, 0xdbe: 0x403e, 0xdbf: 0x403e, + // Block 0x37, offset 0xdc0 + 0xdc0: 0x4037, 0xdc1: 0x4037, 0xdc2: 0x4037, 0xdc3: 0x4037, 0xdc4: 0x4037, 0xdc5: 0x4037, + 0xdc6: 0x4037, 0xdc7: 0x4037, 0xdc8: 0x4037, 0xdc9: 0x4037, 0xdca: 0x4037, 0xdcb: 0x4037, + 0xdcc: 0x4037, 0xdcd: 0x4037, 0xdce: 0x4037, 0xdcf: 0x400e, 0xdd0: 0x403f, 0xdd1: 0x4040, + 0xdd2: 0x4041, 0xdd3: 0x4040, 0xdd4: 0x403f, 0xdd5: 0x4042, 0xdd6: 0x4043, 0xdd7: 0x4044, + 0xdd8: 0x4040, 0xdd9: 0x4041, 0xdda: 0x4040, 0xddb: 0x4045, 0xddc: 0x4009, 0xddd: 0x4045, + 0xdde: 0x4046, 0xddf: 0x4045, 0xde0: 0x4047, 0xde1: 0x400b, 0xde2: 0x400a, 0xde3: 0x400c, + 0xde4: 0x4048, 0xde5: 0x4000, 0xde6: 0x4000, 0xde7: 0x4000, 0xde8: 0x4000, 0xde9: 0x4000, + 0xdea: 0x4000, 0xdeb: 0x4000, 0xdec: 0x4000, 0xded: 0x4000, 0xdee: 0x4000, 0xdef: 0x4000, + 0xdf0: 0x4000, 0xdf1: 0x4000, 0xdf2: 0x4000, 0xdf3: 0x4000, 0xdf4: 0x4000, 0xdf5: 0x4000, + 0xdf6: 0x4000, 0xdf7: 0x4000, 0xdf8: 0x4000, 0xdf9: 0x4000, 0xdfa: 0x4000, 0xdfb: 0x4000, + 0xdfc: 0x4000, 0xdfd: 0x4000, 0xdfe: 0x4000, 0xdff: 0x4000, + // Block 0x38, offset 0xe00 + 0xe00: 0x4000, 0xe01: 0x4000, 0xe02: 0x4000, 0xe03: 0x4000, 0xe04: 0x4000, 0xe05: 0x4000, + 0xe06: 0x4000, 0xe07: 0x4000, 0xe08: 0x4000, 0xe09: 0x4000, 0xe0a: 0x4000, 0xe0b: 0x4000, + 0xe0c: 0x4000, 0xe0d: 0x4000, 0xe0e: 0x4000, 0xe10: 0x4000, 0xe11: 0x4000, + 0xe12: 0x4000, 0xe13: 0x4000, 0xe14: 0x4000, 0xe15: 0x4000, 0xe16: 0x4000, 0xe17: 0x4000, + 0xe18: 0x4000, 0xe19: 0x4000, 0xe1a: 0x4000, 0xe1b: 0x4000, 0xe1c: 0x4000, 0xe1d: 0x4000, + 0xe1e: 0x4000, 0xe1f: 0x4000, 0xe20: 0x4000, 0xe21: 0x4000, 0xe22: 0x4000, 0xe23: 0x4000, + 0xe24: 0x4000, 0xe25: 0x4000, 0xe26: 0x4000, 0xe27: 0x4000, 0xe28: 0x4000, 0xe29: 0x4000, + 0xe2a: 0x4000, 0xe2b: 0x4000, 0xe2c: 0x4000, 0xe2d: 0x4000, 0xe2e: 0x4000, 0xe2f: 0x4000, + 0xe30: 0x4000, 0xe31: 0x4000, 0xe32: 0x4000, 0xe33: 0x4000, 0xe34: 0x4000, 0xe35: 0x4000, + 0xe36: 0x4000, 0xe37: 0x4000, 0xe38: 0x4000, 0xe39: 0x4000, 0xe3a: 0x4000, + // Block 0x39, offset 0xe40 + 0xe40: 0x4000, 0xe41: 0x4000, 0xe42: 0x4000, 0xe43: 0x4000, 0xe44: 0x4000, 0xe45: 0x4000, + 0xe46: 0x4000, 0xe47: 0x4000, 0xe48: 0x4000, 0xe49: 0x4000, 0xe4a: 0x4000, 0xe4b: 0x4000, + 0xe4c: 0x4000, 0xe4d: 0x4000, 0xe4e: 0x4000, 0xe4f: 0x4000, 0xe50: 0x4000, 0xe51: 0x4000, + 0xe52: 0x4000, 0xe53: 0x4000, 0xe54: 0x4000, 0xe55: 0x4000, 0xe56: 0x4000, 0xe57: 0x4000, + 0xe58: 0x4000, 0xe59: 0x4000, 0xe5a: 0x4000, 0xe5b: 0x4000, 0xe5c: 0x4000, 0xe5d: 0x4000, + 0xe5e: 0x4000, 0xe5f: 0x4000, 0xe60: 0x4000, 0xe61: 0x4000, 0xe62: 0x4000, 0xe63: 0x4000, + 0xe70: 0x4000, 0xe71: 0x4000, 0xe72: 0x4000, 0xe73: 0x4000, 0xe74: 0x4000, 0xe75: 0x4000, + 0xe76: 0x4000, 0xe77: 0x4000, 0xe78: 0x4000, 0xe79: 0x4000, 0xe7a: 0x4000, 0xe7b: 0x4000, + 0xe7c: 0x4000, 0xe7d: 0x4000, 0xe7e: 0x4000, 0xe7f: 0x4000, + // Block 0x3a, offset 0xe80 + 0xe80: 0x4000, 0xe81: 0x4000, 0xe82: 0x4000, 0xe83: 0x4000, 0xe84: 0x4000, 0xe85: 0x4000, + 0xe86: 0x4000, 0xe87: 0x4000, 0xe88: 0x4000, 0xe89: 0x4000, 0xe8a: 0x4000, 0xe8b: 0x4000, + 0xe8c: 0x4000, 0xe8d: 0x4000, 0xe8e: 0x4000, 0xe8f: 0x4000, 0xe90: 0x4000, 0xe91: 0x4000, + 0xe92: 0x4000, 0xe93: 0x4000, 0xe94: 0x4000, 0xe95: 0x4000, 0xe96: 0x4000, 0xe97: 0x4000, + 0xe98: 0x4000, 0xe99: 0x4000, 0xe9a: 0x4000, 0xe9b: 0x4000, 0xe9c: 0x4000, 0xe9d: 0x4000, + 0xe9e: 0x4000, 0xea0: 0x4000, 0xea1: 0x4000, 0xea2: 0x4000, 0xea3: 0x4000, + 0xea4: 0x4000, 0xea5: 0x4000, 0xea6: 0x4000, 0xea7: 0x4000, 0xea8: 0x4000, 0xea9: 0x4000, + 0xeaa: 0x4000, 0xeab: 0x4000, 0xeac: 0x4000, 0xead: 0x4000, 0xeae: 0x4000, 0xeaf: 0x4000, + 0xeb0: 0x4000, 0xeb1: 0x4000, 0xeb2: 0x4000, 0xeb3: 0x4000, 0xeb4: 0x4000, 0xeb5: 0x4000, + 0xeb6: 0x4000, 0xeb7: 0x4000, 0xeb8: 0x4000, 0xeb9: 0x4000, 0xeba: 0x4000, 0xebb: 0x4000, + 0xebc: 0x4000, 0xebd: 0x4000, 0xebe: 0x4000, 0xebf: 0x4000, + // Block 0x3b, offset 0xec0 + 0xec0: 0x4000, 0xec1: 0x4000, 0xec2: 0x4000, 0xec3: 0x4000, 0xec4: 0x4000, 0xec5: 0x4000, + 0xec6: 0x4000, 0xec7: 0x4000, 0xec8: 0x2000, 0xec9: 0x2000, 0xeca: 0x2000, 0xecb: 0x2000, + 0xecc: 0x2000, 0xecd: 0x2000, 0xece: 0x2000, 0xecf: 0x2000, 0xed0: 0x4000, 0xed1: 0x4000, + 0xed2: 0x4000, 0xed3: 0x4000, 0xed4: 0x4000, 0xed5: 0x4000, 0xed6: 0x4000, 0xed7: 0x4000, + 0xed8: 0x4000, 0xed9: 0x4000, 0xeda: 0x4000, 0xedb: 0x4000, 0xedc: 0x4000, 0xedd: 0x4000, + 0xede: 0x4000, 0xedf: 0x4000, 0xee0: 0x4000, 0xee1: 0x4000, 0xee2: 0x4000, 0xee3: 0x4000, + 0xee4: 0x4000, 0xee5: 0x4000, 0xee6: 0x4000, 0xee7: 0x4000, 0xee8: 0x4000, 0xee9: 0x4000, + 0xeea: 0x4000, 0xeeb: 0x4000, 0xeec: 0x4000, 0xeed: 0x4000, 0xeee: 0x4000, 0xeef: 0x4000, + 0xef0: 0x4000, 0xef1: 0x4000, 0xef2: 0x4000, 0xef3: 0x4000, 0xef4: 0x4000, 0xef5: 0x4000, + 0xef6: 0x4000, 0xef7: 0x4000, 0xef8: 0x4000, 0xef9: 0x4000, 0xefa: 0x4000, 0xefb: 0x4000, + 0xefc: 0x4000, 0xefd: 0x4000, 0xefe: 0x4000, 0xeff: 0x4000, + // Block 0x3c, offset 0xf00 + 0xf00: 0x4000, 0xf01: 0x4000, 0xf02: 0x4000, 0xf03: 0x4000, 0xf04: 0x4000, 0xf05: 0x4000, + 0xf06: 0x4000, 0xf07: 0x4000, 0xf08: 0x4000, 0xf09: 0x4000, 0xf0a: 0x4000, 0xf0b: 0x4000, + 0xf0c: 0x4000, 0xf0d: 0x4000, 0xf0e: 0x4000, 0xf0f: 0x4000, 0xf10: 0x4000, 0xf11: 0x4000, + 0xf12: 0x4000, 0xf13: 0x4000, 0xf14: 0x4000, 0xf15: 0x4000, 0xf16: 0x4000, 0xf17: 0x4000, + 0xf18: 0x4000, 0xf19: 0x4000, 0xf1a: 0x4000, 0xf1b: 0x4000, 0xf1c: 0x4000, 0xf1d: 0x4000, + 0xf1e: 0x4000, 0xf1f: 0x4000, 0xf20: 0x4000, 0xf21: 0x4000, 0xf22: 0x4000, 0xf23: 0x4000, + 0xf24: 0x4000, 0xf25: 0x4000, 0xf26: 0x4000, 0xf27: 0x4000, 0xf28: 0x4000, 0xf29: 0x4000, + 0xf2a: 0x4000, 0xf2b: 0x4000, 0xf2c: 0x4000, 0xf2d: 0x4000, 0xf2e: 0x4000, 0xf2f: 0x4000, + 0xf30: 0x4000, 0xf31: 0x4000, 0xf32: 0x4000, 0xf33: 0x4000, 0xf34: 0x4000, 0xf35: 0x4000, + 0xf36: 0x4000, 0xf37: 0x4000, 0xf38: 0x4000, 0xf39: 0x4000, 0xf3a: 0x4000, 0xf3b: 0x4000, + 0xf3c: 0x4000, 0xf3d: 0x4000, 0xf3e: 0x4000, + // Block 0x3d, offset 0xf40 + 0xf40: 0x4000, 0xf41: 0x4000, 0xf42: 0x4000, 0xf43: 0x4000, 0xf44: 0x4000, 0xf45: 0x4000, + 0xf46: 0x4000, 0xf47: 0x4000, 0xf48: 0x4000, 0xf49: 0x4000, 0xf4a: 0x4000, 0xf4b: 0x4000, + 0xf4c: 0x4000, 0xf50: 0x4000, 0xf51: 0x4000, + 0xf52: 0x4000, 0xf53: 0x4000, 0xf54: 0x4000, 0xf55: 0x4000, 0xf56: 0x4000, 0xf57: 0x4000, + 0xf58: 0x4000, 0xf59: 0x4000, 0xf5a: 0x4000, 0xf5b: 0x4000, 0xf5c: 0x4000, 0xf5d: 0x4000, + 0xf5e: 0x4000, 0xf5f: 0x4000, 0xf60: 0x4000, 0xf61: 0x4000, 0xf62: 0x4000, 0xf63: 0x4000, + 0xf64: 0x4000, 0xf65: 0x4000, 0xf66: 0x4000, 0xf67: 0x4000, 0xf68: 0x4000, 0xf69: 0x4000, + 0xf6a: 0x4000, 0xf6b: 0x4000, 0xf6c: 0x4000, 0xf6d: 0x4000, 0xf6e: 0x4000, 0xf6f: 0x4000, + 0xf70: 0x4000, 0xf71: 0x4000, 0xf72: 0x4000, 0xf73: 0x4000, 0xf74: 0x4000, 0xf75: 0x4000, + 0xf76: 0x4000, 0xf77: 0x4000, 0xf78: 0x4000, 0xf79: 0x4000, 0xf7a: 0x4000, 0xf7b: 0x4000, + 0xf7c: 0x4000, 0xf7d: 0x4000, 0xf7e: 0x4000, 0xf7f: 0x4000, + // Block 0x3e, offset 0xf80 + 0xf80: 0x4000, 0xf81: 0x4000, 0xf82: 0x4000, 0xf83: 0x4000, 0xf84: 0x4000, 0xf85: 0x4000, + 0xf86: 0x4000, + // Block 0x3f, offset 0xfc0 + 0xfe0: 0x4000, 0xfe1: 0x4000, 0xfe2: 0x4000, 0xfe3: 0x4000, + 0xfe4: 0x4000, 0xfe5: 0x4000, 0xfe6: 0x4000, 0xfe7: 0x4000, 0xfe8: 0x4000, 0xfe9: 0x4000, + 0xfea: 0x4000, 0xfeb: 0x4000, 0xfec: 0x4000, 0xfed: 0x4000, 0xfee: 0x4000, 0xfef: 0x4000, + 0xff0: 0x4000, 0xff1: 0x4000, 0xff2: 0x4000, 0xff3: 0x4000, 0xff4: 0x4000, 0xff5: 0x4000, + 0xff6: 0x4000, 0xff7: 0x4000, 0xff8: 0x4000, 0xff9: 0x4000, 0xffa: 0x4000, 0xffb: 0x4000, + 0xffc: 0x4000, + // Block 0x40, offset 0x1000 + 0x1000: 0x4000, 0x1001: 0x4000, 0x1002: 0x4000, 0x1003: 0x4000, 0x1004: 0x4000, 0x1005: 0x4000, + 0x1006: 0x4000, 0x1007: 0x4000, 0x1008: 0x4000, 0x1009: 0x4000, 0x100a: 0x4000, 0x100b: 0x4000, + 0x100c: 0x4000, 0x100d: 0x4000, 0x100e: 0x4000, 0x100f: 0x4000, 0x1010: 0x4000, 0x1011: 0x4000, + 0x1012: 0x4000, 0x1013: 0x4000, 0x1014: 0x4000, 0x1015: 0x4000, 0x1016: 0x4000, 0x1017: 0x4000, + 0x1018: 0x4000, 0x1019: 0x4000, 0x101a: 0x4000, 0x101b: 0x4000, 0x101c: 0x4000, 0x101d: 0x4000, + 0x101e: 0x4000, 0x101f: 0x4000, 0x1020: 0x4000, 0x1021: 0x4000, 0x1022: 0x4000, 0x1023: 0x4000, + // Block 0x41, offset 0x1040 + 0x1040: 0x2000, 0x1041: 0x2000, 0x1042: 0x2000, 0x1043: 0x2000, 0x1044: 0x2000, 0x1045: 0x2000, + 0x1046: 0x2000, 0x1047: 0x2000, 0x1048: 0x2000, 0x1049: 0x2000, 0x104a: 0x2000, 0x104b: 0x2000, + 0x104c: 0x2000, 0x104d: 0x2000, 0x104e: 0x2000, 0x104f: 0x2000, 0x1050: 0x4000, 0x1051: 0x4000, + 0x1052: 0x4000, 0x1053: 0x4000, 0x1054: 0x4000, 0x1055: 0x4000, 0x1056: 0x4000, 0x1057: 0x4000, + 0x1058: 0x4000, 0x1059: 0x4000, + 0x1070: 0x4000, 0x1071: 0x4000, 0x1072: 0x4000, 0x1073: 0x4000, 0x1074: 0x4000, 0x1075: 0x4000, + 0x1076: 0x4000, 0x1077: 0x4000, 0x1078: 0x4000, 0x1079: 0x4000, 0x107a: 0x4000, 0x107b: 0x4000, + 0x107c: 0x4000, 0x107d: 0x4000, 0x107e: 0x4000, 0x107f: 0x4000, + // Block 0x42, offset 0x1080 + 0x1080: 0x4000, 0x1081: 0x4000, 0x1082: 0x4000, 0x1083: 0x4000, 0x1084: 0x4000, 0x1085: 0x4000, + 0x1086: 0x4000, 0x1087: 0x4000, 0x1088: 0x4000, 0x1089: 0x4000, 0x108a: 0x4000, 0x108b: 0x4000, + 0x108c: 0x4000, 0x108d: 0x4000, 0x108e: 0x4000, 0x108f: 0x4000, 0x1090: 0x4000, 0x1091: 0x4000, + 0x1092: 0x4000, 0x1094: 0x4000, 0x1095: 0x4000, 0x1096: 0x4000, 0x1097: 0x4000, + 0x1098: 0x4000, 0x1099: 0x4000, 0x109a: 0x4000, 0x109b: 0x4000, 0x109c: 0x4000, 0x109d: 0x4000, + 0x109e: 0x4000, 0x109f: 0x4000, 0x10a0: 0x4000, 0x10a1: 0x4000, 0x10a2: 0x4000, 0x10a3: 0x4000, + 0x10a4: 0x4000, 0x10a5: 0x4000, 0x10a6: 0x4000, 0x10a8: 0x4000, 0x10a9: 0x4000, + 0x10aa: 0x4000, 0x10ab: 0x4000, + // Block 0x43, offset 0x10c0 + 0x10c1: 0x9012, 0x10c2: 0x9012, 0x10c3: 0x9012, 0x10c4: 0x9012, 0x10c5: 0x9012, + 0x10c6: 0x9012, 0x10c7: 0x9012, 0x10c8: 0x9012, 0x10c9: 0x9012, 0x10ca: 0x9012, 0x10cb: 0x9012, + 0x10cc: 0x9012, 0x10cd: 0x9012, 0x10ce: 0x9012, 0x10cf: 0x9012, 0x10d0: 0x9012, 0x10d1: 0x9012, + 0x10d2: 0x9012, 0x10d3: 0x9012, 0x10d4: 0x9012, 0x10d5: 0x9012, 0x10d6: 0x9012, 0x10d7: 0x9012, + 0x10d8: 0x9012, 0x10d9: 0x9012, 0x10da: 0x9012, 0x10db: 0x9012, 0x10dc: 0x9012, 0x10dd: 0x9012, + 0x10de: 0x9012, 0x10df: 0x9012, 0x10e0: 0x9049, 0x10e1: 0x9049, 0x10e2: 0x9049, 0x10e3: 0x9049, + 0x10e4: 0x9049, 0x10e5: 0x9049, 0x10e6: 0x9049, 0x10e7: 0x9049, 0x10e8: 0x9049, 0x10e9: 0x9049, + 0x10ea: 0x9049, 0x10eb: 0x9049, 0x10ec: 0x9049, 0x10ed: 0x9049, 0x10ee: 0x9049, 0x10ef: 0x9049, + 0x10f0: 0x9049, 0x10f1: 0x9049, 0x10f2: 0x9049, 0x10f3: 0x9049, 0x10f4: 0x9049, 0x10f5: 0x9049, + 0x10f6: 0x9049, 0x10f7: 0x9049, 0x10f8: 0x9049, 0x10f9: 0x9049, 0x10fa: 0x9049, 0x10fb: 0x9049, + 0x10fc: 0x9049, 0x10fd: 0x9049, 0x10fe: 0x9049, 0x10ff: 0x9049, + // Block 0x44, offset 0x1100 + 0x1100: 0x9049, 0x1101: 0x9049, 0x1102: 0x9049, 0x1103: 0x9049, 0x1104: 0x9049, 0x1105: 0x9049, + 0x1106: 0x9049, 0x1107: 0x9049, 0x1108: 0x9049, 0x1109: 0x9049, 0x110a: 0x9049, 0x110b: 0x9049, + 0x110c: 0x9049, 0x110d: 0x9049, 0x110e: 0x9049, 0x110f: 0x9049, 0x1110: 0x9049, 0x1111: 0x9049, + 0x1112: 0x9049, 0x1113: 0x9049, 0x1114: 0x9049, 0x1115: 0x9049, 0x1116: 0x9049, 0x1117: 0x9049, + 0x1118: 0x9049, 0x1119: 0x9049, 0x111a: 0x9049, 0x111b: 0x9049, 0x111c: 0x9049, 0x111d: 0x9049, + 0x111e: 0x9049, 0x111f: 0x904a, 0x1120: 0x904b, 0x1121: 0xb04c, 0x1122: 0xb04d, 0x1123: 0xb04d, + 0x1124: 0xb04e, 0x1125: 0xb04f, 0x1126: 0xb050, 0x1127: 0xb051, 0x1128: 0xb052, 0x1129: 0xb053, + 0x112a: 0xb054, 0x112b: 0xb055, 0x112c: 0xb056, 0x112d: 0xb057, 0x112e: 0xb058, 0x112f: 0xb059, + 0x1130: 0xb05a, 0x1131: 0xb05b, 0x1132: 0xb05c, 0x1133: 0xb05d, 0x1134: 0xb05e, 0x1135: 0xb05f, + 0x1136: 0xb060, 0x1137: 0xb061, 0x1138: 0xb062, 0x1139: 0xb063, 0x113a: 0xb064, 0x113b: 0xb065, + 0x113c: 0xb052, 0x113d: 0xb066, 0x113e: 0xb067, 0x113f: 0xb055, + // Block 0x45, offset 0x1140 + 0x1140: 0xb068, 0x1141: 0xb069, 0x1142: 0xb06a, 0x1143: 0xb06b, 0x1144: 0xb05a, 0x1145: 0xb056, + 0x1146: 0xb06c, 0x1147: 0xb06d, 0x1148: 0xb06b, 0x1149: 0xb06e, 0x114a: 0xb06b, 0x114b: 0xb06f, + 0x114c: 0xb06f, 0x114d: 0xb070, 0x114e: 0xb070, 0x114f: 0xb071, 0x1150: 0xb056, 0x1151: 0xb072, + 0x1152: 0xb073, 0x1153: 0xb072, 0x1154: 0xb074, 0x1155: 0xb073, 0x1156: 0xb075, 0x1157: 0xb075, + 0x1158: 0xb076, 0x1159: 0xb076, 0x115a: 0xb077, 0x115b: 0xb077, 0x115c: 0xb073, 0x115d: 0xb078, + 0x115e: 0xb079, 0x115f: 0xb067, 0x1160: 0xb07a, 0x1161: 0xb07b, 0x1162: 0xb07b, 0x1163: 0xb07b, + 0x1164: 0xb07b, 0x1165: 0xb07b, 0x1166: 0xb07b, 0x1167: 0xb07b, 0x1168: 0xb07b, 0x1169: 0xb07b, + 0x116a: 0xb07b, 0x116b: 0xb07b, 0x116c: 0xb07b, 0x116d: 0xb07b, 0x116e: 0xb07b, 0x116f: 0xb07b, + 0x1170: 0xb07c, 0x1171: 0xb07c, 0x1172: 0xb07c, 0x1173: 0xb07c, 0x1174: 0xb07c, 0x1175: 0xb07c, + 0x1176: 0xb07c, 0x1177: 0xb07c, 0x1178: 0xb07c, 0x1179: 0xb07c, 0x117a: 0xb07c, 0x117b: 0xb07c, + 0x117c: 0xb07c, 0x117d: 0xb07c, 0x117e: 0xb07c, + // Block 0x46, offset 0x1180 + 0x1182: 0xb07d, 0x1183: 0xb07e, 0x1184: 0xb07f, 0x1185: 0xb080, + 0x1186: 0xb07f, 0x1187: 0xb07e, 0x118a: 0xb081, 0x118b: 0xb082, + 0x118c: 0xb083, 0x118d: 0xb07f, 0x118e: 0xb080, 0x118f: 0xb07f, + 0x1192: 0xb084, 0x1193: 0xb085, 0x1194: 0xb084, 0x1195: 0xb086, 0x1196: 0xb084, 0x1197: 0xb087, + 0x119a: 0xb088, 0x119b: 0xb089, 0x119c: 0xb08a, + 0x11a0: 0x908b, 0x11a1: 0x908b, 0x11a2: 0x908c, 0x11a3: 0x908d, + 0x11a4: 0x908b, 0x11a5: 0x908e, 0x11a6: 0x908f, 0x11a8: 0xb090, 0x11a9: 0xb091, + 0x11aa: 0xb092, 0x11ab: 0xb091, 0x11ac: 0xb093, 0x11ad: 0xb094, 0x11ae: 0xb095, + 0x11bd: 0x2000, + // Block 0x47, offset 0x11c0 + 0x11e0: 0x4000, + // Block 0x48, offset 0x1200 + 0x1200: 0x4000, 0x1201: 0x4000, 0x1202: 0x4000, 0x1203: 0x4000, 0x1204: 0x4000, 0x1205: 0x4000, + 0x1206: 0x4000, 0x1207: 0x4000, 0x1208: 0x4000, 0x1209: 0x4000, 0x120a: 0x4000, 0x120b: 0x4000, + 0x120c: 0x4000, 0x120d: 0x4000, 0x120e: 0x4000, 0x120f: 0x4000, 0x1210: 0x4000, 0x1211: 0x4000, + 0x1212: 0x4000, 0x1213: 0x4000, 0x1214: 0x4000, 0x1215: 0x4000, 0x1216: 0x4000, 0x1217: 0x4000, + 0x1218: 0x4000, 0x1219: 0x4000, 0x121a: 0x4000, 0x121b: 0x4000, 0x121c: 0x4000, 0x121d: 0x4000, + 0x121e: 0x4000, 0x121f: 0x4000, 0x1220: 0x4000, 0x1221: 0x4000, 0x1222: 0x4000, 0x1223: 0x4000, + 0x1224: 0x4000, 0x1225: 0x4000, 0x1226: 0x4000, 0x1227: 0x4000, 0x1228: 0x4000, 0x1229: 0x4000, + 0x122a: 0x4000, 0x122b: 0x4000, 0x122c: 0x4000, + // Block 0x49, offset 0x1240 + 0x1240: 0x4000, 0x1241: 0x4000, 0x1242: 0x4000, 0x1243: 0x4000, 0x1244: 0x4000, 0x1245: 0x4000, + 0x1246: 0x4000, 0x1247: 0x4000, 0x1248: 0x4000, 0x1249: 0x4000, 0x124a: 0x4000, 0x124b: 0x4000, + 0x124c: 0x4000, 0x124d: 0x4000, 0x124e: 0x4000, 0x124f: 0x4000, 0x1250: 0x4000, 0x1251: 0x4000, + 0x1252: 0x4000, 0x1253: 0x4000, 0x1254: 0x4000, 0x1255: 0x4000, 0x1256: 0x4000, 0x1257: 0x4000, + 0x1258: 0x4000, 0x1259: 0x4000, 0x125a: 0x4000, 0x125b: 0x4000, 0x125c: 0x4000, 0x125d: 0x4000, + 0x125e: 0x4000, 0x125f: 0x4000, 0x1260: 0x4000, 0x1261: 0x4000, 0x1262: 0x4000, 0x1263: 0x4000, + 0x1264: 0x4000, 0x1265: 0x4000, 0x1266: 0x4000, 0x1267: 0x4000, 0x1268: 0x4000, 0x1269: 0x4000, + 0x126a: 0x4000, 0x126b: 0x4000, 0x126c: 0x4000, 0x126d: 0x4000, 0x126e: 0x4000, 0x126f: 0x4000, + 0x1270: 0x4000, 0x1271: 0x4000, 0x1272: 0x4000, + // Block 0x4a, offset 0x1280 + 0x1280: 0x4000, 0x1281: 0x4000, + // Block 0x4b, offset 0x12c0 + 0x12c4: 0x4000, + // Block 0x4c, offset 0x1300 + 0x130f: 0x4000, + // Block 0x4d, offset 0x1340 + 0x1340: 0x2000, 0x1341: 0x2000, 0x1342: 0x2000, 0x1343: 0x2000, 0x1344: 0x2000, 0x1345: 0x2000, + 0x1346: 0x2000, 0x1347: 0x2000, 0x1348: 0x2000, 0x1349: 0x2000, 0x134a: 0x2000, + 0x1350: 0x2000, 0x1351: 0x2000, + 0x1352: 0x2000, 0x1353: 0x2000, 0x1354: 0x2000, 0x1355: 0x2000, 0x1356: 0x2000, 0x1357: 0x2000, + 0x1358: 0x2000, 0x1359: 0x2000, 0x135a: 0x2000, 0x135b: 0x2000, 0x135c: 0x2000, 0x135d: 0x2000, + 0x135e: 0x2000, 0x135f: 0x2000, 0x1360: 0x2000, 0x1361: 0x2000, 0x1362: 0x2000, 0x1363: 0x2000, + 0x1364: 0x2000, 0x1365: 0x2000, 0x1366: 0x2000, 0x1367: 0x2000, 0x1368: 0x2000, 0x1369: 0x2000, + 0x136a: 0x2000, 0x136b: 0x2000, 0x136c: 0x2000, 0x136d: 0x2000, + 0x1370: 0x2000, 0x1371: 0x2000, 0x1372: 0x2000, 0x1373: 0x2000, 0x1374: 0x2000, 0x1375: 0x2000, + 0x1376: 0x2000, 0x1377: 0x2000, 0x1378: 0x2000, 0x1379: 0x2000, 0x137a: 0x2000, 0x137b: 0x2000, + 0x137c: 0x2000, 0x137d: 0x2000, 0x137e: 0x2000, 0x137f: 0x2000, + // Block 0x4e, offset 0x1380 + 0x1380: 0x2000, 0x1381: 0x2000, 0x1382: 0x2000, 0x1383: 0x2000, 0x1384: 0x2000, 0x1385: 0x2000, + 0x1386: 0x2000, 0x1387: 0x2000, 0x1388: 0x2000, 0x1389: 0x2000, 0x138a: 0x2000, 0x138b: 0x2000, + 0x138c: 0x2000, 0x138d: 0x2000, 0x138e: 0x2000, 0x138f: 0x2000, 0x1390: 0x2000, 0x1391: 0x2000, + 0x1392: 0x2000, 0x1393: 0x2000, 0x1394: 0x2000, 0x1395: 0x2000, 0x1396: 0x2000, 0x1397: 0x2000, + 0x1398: 0x2000, 0x1399: 0x2000, 0x139a: 0x2000, 0x139b: 0x2000, 0x139c: 0x2000, 0x139d: 0x2000, + 0x139e: 0x2000, 0x139f: 0x2000, 0x13a0: 0x2000, 0x13a1: 0x2000, 0x13a2: 0x2000, 0x13a3: 0x2000, + 0x13a4: 0x2000, 0x13a5: 0x2000, 0x13a6: 0x2000, 0x13a7: 0x2000, 0x13a8: 0x2000, 0x13a9: 0x2000, + 0x13b0: 0x2000, 0x13b1: 0x2000, 0x13b2: 0x2000, 0x13b3: 0x2000, 0x13b4: 0x2000, 0x13b5: 0x2000, + 0x13b6: 0x2000, 0x13b7: 0x2000, 0x13b8: 0x2000, 0x13b9: 0x2000, 0x13ba: 0x2000, 0x13bb: 0x2000, + 0x13bc: 0x2000, 0x13bd: 0x2000, 0x13be: 0x2000, 0x13bf: 0x2000, + // Block 0x4f, offset 0x13c0 + 0x13c0: 0x2000, 0x13c1: 0x2000, 0x13c2: 0x2000, 0x13c3: 0x2000, 0x13c4: 0x2000, 0x13c5: 0x2000, + 0x13c6: 0x2000, 0x13c7: 0x2000, 0x13c8: 0x2000, 0x13c9: 0x2000, 0x13ca: 0x2000, 0x13cb: 0x2000, + 0x13cc: 0x2000, 0x13cd: 0x2000, 0x13ce: 0x4000, 0x13cf: 0x2000, 0x13d0: 0x2000, 0x13d1: 0x4000, + 0x13d2: 0x4000, 0x13d3: 0x4000, 0x13d4: 0x4000, 0x13d5: 0x4000, 0x13d6: 0x4000, 0x13d7: 0x4000, + 0x13d8: 0x4000, 0x13d9: 0x4000, 0x13da: 0x4000, 0x13db: 0x2000, 0x13dc: 0x2000, 0x13dd: 0x2000, + 0x13de: 0x2000, 0x13df: 0x2000, 0x13e0: 0x2000, 0x13e1: 0x2000, 0x13e2: 0x2000, 0x13e3: 0x2000, + 0x13e4: 0x2000, 0x13e5: 0x2000, 0x13e6: 0x2000, 0x13e7: 0x2000, 0x13e8: 0x2000, 0x13e9: 0x2000, + 0x13ea: 0x2000, 0x13eb: 0x2000, 0x13ec: 0x2000, + // Block 0x50, offset 0x1400 + 0x1400: 0x4000, 0x1401: 0x4000, 0x1402: 0x4000, + 0x1410: 0x4000, 0x1411: 0x4000, + 0x1412: 0x4000, 0x1413: 0x4000, 0x1414: 0x4000, 0x1415: 0x4000, 0x1416: 0x4000, 0x1417: 0x4000, + 0x1418: 0x4000, 0x1419: 0x4000, 0x141a: 0x4000, 0x141b: 0x4000, 0x141c: 0x4000, 0x141d: 0x4000, + 0x141e: 0x4000, 0x141f: 0x4000, 0x1420: 0x4000, 0x1421: 0x4000, 0x1422: 0x4000, 0x1423: 0x4000, + 0x1424: 0x4000, 0x1425: 0x4000, 0x1426: 0x4000, 0x1427: 0x4000, 0x1428: 0x4000, 0x1429: 0x4000, + 0x142a: 0x4000, 0x142b: 0x4000, 0x142c: 0x4000, 0x142d: 0x4000, 0x142e: 0x4000, 0x142f: 0x4000, + 0x1430: 0x4000, 0x1431: 0x4000, 0x1432: 0x4000, 0x1433: 0x4000, 0x1434: 0x4000, 0x1435: 0x4000, + 0x1436: 0x4000, 0x1437: 0x4000, 0x1438: 0x4000, 0x1439: 0x4000, 0x143a: 0x4000, 0x143b: 0x4000, + // Block 0x51, offset 0x1440 + 0x1440: 0x4000, 0x1441: 0x4000, 0x1442: 0x4000, 0x1443: 0x4000, 0x1444: 0x4000, 0x1445: 0x4000, + 0x1446: 0x4000, 0x1447: 0x4000, 0x1448: 0x4000, + 0x1450: 0x4000, 0x1451: 0x4000, + // Block 0x52, offset 0x1480 + 0x1480: 0x4000, 0x1481: 0x4000, 0x1482: 0x4000, 0x1483: 0x4000, 0x1484: 0x4000, 0x1485: 0x4000, + 0x1486: 0x4000, 0x1487: 0x4000, 0x1488: 0x4000, 0x1489: 0x4000, 0x148a: 0x4000, 0x148b: 0x4000, + 0x148c: 0x4000, 0x148d: 0x4000, 0x148e: 0x4000, 0x148f: 0x4000, 0x1490: 0x4000, 0x1491: 0x4000, + 0x1492: 0x4000, 0x1493: 0x4000, 0x1494: 0x4000, 0x1495: 0x4000, 0x1496: 0x4000, 0x1497: 0x4000, + 0x1498: 0x4000, 0x1499: 0x4000, 0x149a: 0x4000, 0x149b: 0x4000, 0x149c: 0x4000, 0x149d: 0x4000, + 0x149e: 0x4000, 0x149f: 0x4000, 0x14a0: 0x4000, + 0x14ad: 0x4000, 0x14ae: 0x4000, 0x14af: 0x4000, + 0x14b0: 0x4000, 0x14b1: 0x4000, 0x14b2: 0x4000, 0x14b3: 0x4000, 0x14b4: 0x4000, 0x14b5: 0x4000, + 0x14b7: 0x4000, 0x14b8: 0x4000, 0x14b9: 0x4000, 0x14ba: 0x4000, 0x14bb: 0x4000, + 0x14bc: 0x4000, 0x14bd: 0x4000, 0x14be: 0x4000, 0x14bf: 0x4000, + // Block 0x53, offset 0x14c0 + 0x14c0: 0x4000, 0x14c1: 0x4000, 0x14c2: 0x4000, 0x14c3: 0x4000, 0x14c4: 0x4000, 0x14c5: 0x4000, + 0x14c6: 0x4000, 0x14c7: 0x4000, 0x14c8: 0x4000, 0x14c9: 0x4000, 0x14ca: 0x4000, 0x14cb: 0x4000, + 0x14cc: 0x4000, 0x14cd: 0x4000, 0x14ce: 0x4000, 0x14cf: 0x4000, 0x14d0: 0x4000, 0x14d1: 0x4000, + 0x14d2: 0x4000, 0x14d3: 0x4000, 0x14d4: 0x4000, 0x14d5: 0x4000, 0x14d6: 0x4000, 0x14d7: 0x4000, + 0x14d8: 0x4000, 0x14d9: 0x4000, 0x14da: 0x4000, 0x14db: 0x4000, 0x14dc: 0x4000, 0x14dd: 0x4000, + 0x14de: 0x4000, 0x14df: 0x4000, 0x14e0: 0x4000, 0x14e1: 0x4000, 0x14e2: 0x4000, 0x14e3: 0x4000, + 0x14e4: 0x4000, 0x14e5: 0x4000, 0x14e6: 0x4000, 0x14e7: 0x4000, 0x14e8: 0x4000, 0x14e9: 0x4000, + 0x14ea: 0x4000, 0x14eb: 0x4000, 0x14ec: 0x4000, 0x14ed: 0x4000, 0x14ee: 0x4000, 0x14ef: 0x4000, + 0x14f0: 0x4000, 0x14f1: 0x4000, 0x14f2: 0x4000, 0x14f3: 0x4000, 0x14f4: 0x4000, 0x14f5: 0x4000, + 0x14f6: 0x4000, 0x14f7: 0x4000, 0x14f8: 0x4000, 0x14f9: 0x4000, 0x14fa: 0x4000, 0x14fb: 0x4000, + 0x14fc: 0x4000, 0x14fe: 0x4000, 0x14ff: 0x4000, + // Block 0x54, offset 0x1500 + 0x1500: 0x4000, 0x1501: 0x4000, 0x1502: 0x4000, 0x1503: 0x4000, 0x1504: 0x4000, 0x1505: 0x4000, + 0x1506: 0x4000, 0x1507: 0x4000, 0x1508: 0x4000, 0x1509: 0x4000, 0x150a: 0x4000, 0x150b: 0x4000, + 0x150c: 0x4000, 0x150d: 0x4000, 0x150e: 0x4000, 0x150f: 0x4000, 0x1510: 0x4000, 0x1511: 0x4000, + 0x1512: 0x4000, 0x1513: 0x4000, + 0x1520: 0x4000, 0x1521: 0x4000, 0x1522: 0x4000, 0x1523: 0x4000, + 0x1524: 0x4000, 0x1525: 0x4000, 0x1526: 0x4000, 0x1527: 0x4000, 0x1528: 0x4000, 0x1529: 0x4000, + 0x152a: 0x4000, 0x152b: 0x4000, 0x152c: 0x4000, 0x152d: 0x4000, 0x152e: 0x4000, 0x152f: 0x4000, + 0x1530: 0x4000, 0x1531: 0x4000, 0x1532: 0x4000, 0x1533: 0x4000, 0x1534: 0x4000, 0x1535: 0x4000, + 0x1536: 0x4000, 0x1537: 0x4000, 0x1538: 0x4000, 0x1539: 0x4000, 0x153a: 0x4000, 0x153b: 0x4000, + 0x153c: 0x4000, 0x153d: 0x4000, 0x153e: 0x4000, 0x153f: 0x4000, + // Block 0x55, offset 0x1540 + 0x1540: 0x4000, 0x1541: 0x4000, 0x1542: 0x4000, 0x1543: 0x4000, 0x1544: 0x4000, 0x1545: 0x4000, + 0x1546: 0x4000, 0x1547: 0x4000, 0x1548: 0x4000, 0x1549: 0x4000, 0x154a: 0x4000, + 0x154f: 0x4000, 0x1550: 0x4000, 0x1551: 0x4000, + 0x1552: 0x4000, 0x1553: 0x4000, + 0x1560: 0x4000, 0x1561: 0x4000, 0x1562: 0x4000, 0x1563: 0x4000, + 0x1564: 0x4000, 0x1565: 0x4000, 0x1566: 0x4000, 0x1567: 0x4000, 0x1568: 0x4000, 0x1569: 0x4000, + 0x156a: 0x4000, 0x156b: 0x4000, 0x156c: 0x4000, 0x156d: 0x4000, 0x156e: 0x4000, 0x156f: 0x4000, + 0x1570: 0x4000, 0x1574: 0x4000, + 0x1578: 0x4000, 0x1579: 0x4000, 0x157a: 0x4000, 0x157b: 0x4000, + 0x157c: 0x4000, 0x157d: 0x4000, 0x157e: 0x4000, 0x157f: 0x4000, + // Block 0x56, offset 0x1580 + 0x1580: 0x4000, 0x1582: 0x4000, 0x1583: 0x4000, 0x1584: 0x4000, 0x1585: 0x4000, + 0x1586: 0x4000, 0x1587: 0x4000, 0x1588: 0x4000, 0x1589: 0x4000, 0x158a: 0x4000, 0x158b: 0x4000, + 0x158c: 0x4000, 0x158d: 0x4000, 0x158e: 0x4000, 0x158f: 0x4000, 0x1590: 0x4000, 0x1591: 0x4000, + 0x1592: 0x4000, 0x1593: 0x4000, 0x1594: 0x4000, 0x1595: 0x4000, 0x1596: 0x4000, 0x1597: 0x4000, + 0x1598: 0x4000, 0x1599: 0x4000, 0x159a: 0x4000, 0x159b: 0x4000, 0x159c: 0x4000, 0x159d: 0x4000, + 0x159e: 0x4000, 0x159f: 0x4000, 0x15a0: 0x4000, 0x15a1: 0x4000, 0x15a2: 0x4000, 0x15a3: 0x4000, + 0x15a4: 0x4000, 0x15a5: 0x4000, 0x15a6: 0x4000, 0x15a7: 0x4000, 0x15a8: 0x4000, 0x15a9: 0x4000, + 0x15aa: 0x4000, 0x15ab: 0x4000, 0x15ac: 0x4000, 0x15ad: 0x4000, 0x15ae: 0x4000, 0x15af: 0x4000, + 0x15b0: 0x4000, 0x15b1: 0x4000, 0x15b2: 0x4000, 0x15b3: 0x4000, 0x15b4: 0x4000, 0x15b5: 0x4000, + 0x15b6: 0x4000, 0x15b7: 0x4000, 0x15b8: 0x4000, 0x15b9: 0x4000, 0x15ba: 0x4000, 0x15bb: 0x4000, + 0x15bc: 0x4000, 0x15bd: 0x4000, 0x15be: 0x4000, 0x15bf: 0x4000, + // Block 0x57, offset 0x15c0 + 0x15c0: 0x4000, 0x15c1: 0x4000, 0x15c2: 0x4000, 0x15c3: 0x4000, 0x15c4: 0x4000, 0x15c5: 0x4000, + 0x15c6: 0x4000, 0x15c7: 0x4000, 0x15c8: 0x4000, 0x15c9: 0x4000, 0x15ca: 0x4000, 0x15cb: 0x4000, + 0x15cc: 0x4000, 0x15cd: 0x4000, 0x15ce: 0x4000, 0x15cf: 0x4000, 0x15d0: 0x4000, 0x15d1: 0x4000, + 0x15d2: 0x4000, 0x15d3: 0x4000, 0x15d4: 0x4000, 0x15d5: 0x4000, 0x15d6: 0x4000, 0x15d7: 0x4000, + 0x15d8: 0x4000, 0x15d9: 0x4000, 0x15da: 0x4000, 0x15db: 0x4000, 0x15dc: 0x4000, 0x15dd: 0x4000, + 0x15de: 0x4000, 0x15df: 0x4000, 0x15e0: 0x4000, 0x15e1: 0x4000, 0x15e2: 0x4000, 0x15e3: 0x4000, + 0x15e4: 0x4000, 0x15e5: 0x4000, 0x15e6: 0x4000, 0x15e7: 0x4000, 0x15e8: 0x4000, 0x15e9: 0x4000, + 0x15ea: 0x4000, 0x15eb: 0x4000, 0x15ec: 0x4000, 0x15ed: 0x4000, 0x15ee: 0x4000, 0x15ef: 0x4000, + 0x15f0: 0x4000, 0x15f1: 0x4000, 0x15f2: 0x4000, 0x15f3: 0x4000, 0x15f4: 0x4000, 0x15f5: 0x4000, + 0x15f6: 0x4000, 0x15f7: 0x4000, 0x15f8: 0x4000, 0x15f9: 0x4000, 0x15fa: 0x4000, 0x15fb: 0x4000, + 0x15fc: 0x4000, 0x15ff: 0x4000, + // Block 0x58, offset 0x1600 + 0x1600: 0x4000, 0x1601: 0x4000, 0x1602: 0x4000, 0x1603: 0x4000, 0x1604: 0x4000, 0x1605: 0x4000, + 0x1606: 0x4000, 0x1607: 0x4000, 0x1608: 0x4000, 0x1609: 0x4000, 0x160a: 0x4000, 0x160b: 0x4000, + 0x160c: 0x4000, 0x160d: 0x4000, 0x160e: 0x4000, 0x160f: 0x4000, 0x1610: 0x4000, 0x1611: 0x4000, + 0x1612: 0x4000, 0x1613: 0x4000, 0x1614: 0x4000, 0x1615: 0x4000, 0x1616: 0x4000, 0x1617: 0x4000, + 0x1618: 0x4000, 0x1619: 0x4000, 0x161a: 0x4000, 0x161b: 0x4000, 0x161c: 0x4000, 0x161d: 0x4000, + 0x161e: 0x4000, 0x161f: 0x4000, 0x1620: 0x4000, 0x1621: 0x4000, 0x1622: 0x4000, 0x1623: 0x4000, + 0x1624: 0x4000, 0x1625: 0x4000, 0x1626: 0x4000, 0x1627: 0x4000, 0x1628: 0x4000, 0x1629: 0x4000, + 0x162a: 0x4000, 0x162b: 0x4000, 0x162c: 0x4000, 0x162d: 0x4000, 0x162e: 0x4000, 0x162f: 0x4000, + 0x1630: 0x4000, 0x1631: 0x4000, 0x1632: 0x4000, 0x1633: 0x4000, 0x1634: 0x4000, 0x1635: 0x4000, + 0x1636: 0x4000, 0x1637: 0x4000, 0x1638: 0x4000, 0x1639: 0x4000, 0x163a: 0x4000, 0x163b: 0x4000, + 0x163c: 0x4000, 0x163d: 0x4000, + // Block 0x59, offset 0x1640 + 0x164b: 0x4000, + 0x164c: 0x4000, 0x164d: 0x4000, 0x164e: 0x4000, 0x1650: 0x4000, 0x1651: 0x4000, + 0x1652: 0x4000, 0x1653: 0x4000, 0x1654: 0x4000, 0x1655: 0x4000, 0x1656: 0x4000, 0x1657: 0x4000, + 0x1658: 0x4000, 0x1659: 0x4000, 0x165a: 0x4000, 0x165b: 0x4000, 0x165c: 0x4000, 0x165d: 0x4000, + 0x165e: 0x4000, 0x165f: 0x4000, 0x1660: 0x4000, 0x1661: 0x4000, 0x1662: 0x4000, 0x1663: 0x4000, + 0x1664: 0x4000, 0x1665: 0x4000, 0x1666: 0x4000, 0x1667: 0x4000, + 0x167a: 0x4000, + // Block 0x5a, offset 0x1680 + 0x1695: 0x4000, 0x1696: 0x4000, + 0x16a4: 0x4000, + // Block 0x5b, offset 0x16c0 + 0x16fb: 0x4000, + 0x16fc: 0x4000, 0x16fd: 0x4000, 0x16fe: 0x4000, 0x16ff: 0x4000, + // Block 0x5c, offset 0x1700 + 0x1700: 0x4000, 0x1701: 0x4000, 0x1702: 0x4000, 0x1703: 0x4000, 0x1704: 0x4000, 0x1705: 0x4000, + 0x1706: 0x4000, 0x1707: 0x4000, 0x1708: 0x4000, 0x1709: 0x4000, 0x170a: 0x4000, 0x170b: 0x4000, + 0x170c: 0x4000, 0x170d: 0x4000, 0x170e: 0x4000, 0x170f: 0x4000, + // Block 0x5d, offset 0x1740 + 0x1740: 0x4000, 0x1741: 0x4000, 0x1742: 0x4000, 0x1743: 0x4000, 0x1744: 0x4000, 0x1745: 0x4000, + 0x174c: 0x4000, 0x1750: 0x4000, 0x1751: 0x4000, + 0x1752: 0x4000, + 0x176b: 0x4000, 0x176c: 0x4000, + 0x1774: 0x4000, 0x1775: 0x4000, + 0x1776: 0x4000, + // Block 0x5e, offset 0x1780 + 0x1790: 0x4000, 0x1791: 0x4000, + 0x1792: 0x4000, 0x1793: 0x4000, 0x1794: 0x4000, 0x1795: 0x4000, 0x1796: 0x4000, 0x1797: 0x4000, + 0x1798: 0x4000, 0x1799: 0x4000, 0x179a: 0x4000, 0x179b: 0x4000, 0x179c: 0x4000, 0x179d: 0x4000, + 0x179e: 0x4000, 0x17a0: 0x4000, 0x17a1: 0x4000, 0x17a2: 0x4000, 0x17a3: 0x4000, + 0x17a4: 0x4000, 0x17a5: 0x4000, 0x17a6: 0x4000, 0x17a7: 0x4000, + 0x17b0: 0x4000, 0x17b3: 0x4000, 0x17b4: 0x4000, 0x17b5: 0x4000, + 0x17b6: 0x4000, 0x17b7: 0x4000, 0x17b8: 0x4000, 0x17b9: 0x4000, 0x17ba: 0x4000, 0x17bb: 0x4000, + 0x17bc: 0x4000, 0x17bd: 0x4000, 0x17be: 0x4000, + // Block 0x5f, offset 0x17c0 + 0x17c0: 0x4000, 0x17c1: 0x4000, 0x17c2: 0x4000, 0x17c3: 0x4000, 0x17c4: 0x4000, 0x17c5: 0x4000, + 0x17c6: 0x4000, 0x17c7: 0x4000, 0x17c8: 0x4000, 0x17c9: 0x4000, 0x17ca: 0x4000, 0x17cb: 0x4000, + 0x17d0: 0x4000, 0x17d1: 0x4000, + 0x17d2: 0x4000, 0x17d3: 0x4000, 0x17d4: 0x4000, 0x17d5: 0x4000, 0x17d6: 0x4000, 0x17d7: 0x4000, + 0x17d8: 0x4000, 0x17d9: 0x4000, 0x17da: 0x4000, 0x17db: 0x4000, 0x17dc: 0x4000, 0x17dd: 0x4000, + 0x17de: 0x4000, + // Block 0x60, offset 0x1800 + 0x1800: 0x4000, 0x1801: 0x4000, 0x1802: 0x4000, 0x1803: 0x4000, 0x1804: 0x4000, 0x1805: 0x4000, + 0x1806: 0x4000, 0x1807: 0x4000, 0x1808: 0x4000, 0x1809: 0x4000, 0x180a: 0x4000, 0x180b: 0x4000, + 0x180c: 0x4000, 0x180d: 0x4000, 0x180e: 0x4000, 0x180f: 0x4000, 0x1810: 0x4000, 0x1811: 0x4000, + // Block 0x61, offset 0x1840 + 0x1840: 0x4000, + // Block 0x62, offset 0x1880 + 0x1880: 0x2000, 0x1881: 0x2000, 0x1882: 0x2000, 0x1883: 0x2000, 0x1884: 0x2000, 0x1885: 0x2000, + 0x1886: 0x2000, 0x1887: 0x2000, 0x1888: 0x2000, 0x1889: 0x2000, 0x188a: 0x2000, 0x188b: 0x2000, + 0x188c: 0x2000, 0x188d: 0x2000, 0x188e: 0x2000, 0x188f: 0x2000, 0x1890: 0x2000, 0x1891: 0x2000, + 0x1892: 0x2000, 0x1893: 0x2000, 0x1894: 0x2000, 0x1895: 0x2000, 0x1896: 0x2000, 0x1897: 0x2000, + 0x1898: 0x2000, 0x1899: 0x2000, 0x189a: 0x2000, 0x189b: 0x2000, 0x189c: 0x2000, 0x189d: 0x2000, + 0x189e: 0x2000, 0x189f: 0x2000, 0x18a0: 0x2000, 0x18a1: 0x2000, 0x18a2: 0x2000, 0x18a3: 0x2000, + 0x18a4: 0x2000, 0x18a5: 0x2000, 0x18a6: 0x2000, 0x18a7: 0x2000, 0x18a8: 0x2000, 0x18a9: 0x2000, + 0x18aa: 0x2000, 0x18ab: 0x2000, 0x18ac: 0x2000, 0x18ad: 0x2000, 0x18ae: 0x2000, 0x18af: 0x2000, + 0x18b0: 0x2000, 0x18b1: 0x2000, 0x18b2: 0x2000, 0x18b3: 0x2000, 0x18b4: 0x2000, 0x18b5: 0x2000, + 0x18b6: 0x2000, 0x18b7: 0x2000, 0x18b8: 0x2000, 0x18b9: 0x2000, 0x18ba: 0x2000, 0x18bb: 0x2000, + 0x18bc: 0x2000, 0x18bd: 0x2000, +} + +// widthIndex: 22 blocks, 1408 entries, 1408 bytes +// Block 0 is the zero block. +var widthIndex = [1408]uint8{ + // Block 0x0, offset 0x0 + // Block 0x1, offset 0x40 + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0xc2: 0x01, 0xc3: 0x02, 0xc4: 0x03, 0xc5: 0x04, 0xc7: 0x05, + 0xc9: 0x06, 0xcb: 0x07, 0xcc: 0x08, 0xcd: 0x09, 0xce: 0x0a, 0xcf: 0x0b, + 0xd0: 0x0c, 0xd1: 0x0d, + 0xe1: 0x02, 0xe2: 0x03, 0xe3: 0x04, 0xe4: 0x05, 0xe5: 0x06, 0xe6: 0x06, 0xe7: 0x06, + 0xe8: 0x06, 0xe9: 0x06, 0xea: 0x07, 0xeb: 0x06, 0xec: 0x06, 0xed: 0x08, 0xee: 0x09, 0xef: 0x0a, + 0xf0: 0x0f, 0xf3: 0x12, 0xf4: 0x13, + // Block 0x4, offset 0x100 + 0x104: 0x0e, 0x105: 0x0f, + // Block 0x5, offset 0x140 + 0x140: 0x10, 0x141: 0x11, 0x142: 0x12, 0x144: 0x13, 0x145: 0x14, 0x146: 0x15, 0x147: 0x16, + 0x148: 0x17, 0x149: 0x18, 0x14a: 0x19, 0x14c: 0x1a, 0x14f: 0x1b, + 0x151: 0x1c, 0x152: 0x08, 0x153: 0x1d, 0x154: 0x1e, 0x155: 0x1f, 0x156: 0x20, 0x157: 0x21, + 0x158: 0x22, 0x159: 0x23, 0x15a: 0x24, 0x15b: 0x25, 0x15c: 0x26, 0x15d: 0x27, 0x15e: 0x28, 0x15f: 0x29, + 0x166: 0x2a, + 0x16c: 0x2b, 0x16d: 0x2c, + 0x17a: 0x2d, 0x17b: 0x2e, 0x17c: 0x0e, 0x17d: 0x0e, 0x17e: 0x0e, 0x17f: 0x2f, + // Block 0x6, offset 0x180 + 0x180: 0x30, 0x181: 0x31, 0x182: 0x32, 0x183: 0x33, 0x184: 0x34, 0x185: 0x35, 0x186: 0x36, 0x187: 0x37, + 0x188: 0x38, 0x189: 0x39, 0x18a: 0x0e, 0x18b: 0x3a, 0x18c: 0x0e, 0x18d: 0x0e, 0x18e: 0x0e, 0x18f: 0x0e, + 0x190: 0x0e, 0x191: 0x0e, 0x192: 0x0e, 0x193: 0x0e, 0x194: 0x0e, 0x195: 0x0e, 0x196: 0x0e, 0x197: 0x0e, + 0x198: 0x0e, 0x199: 0x0e, 0x19a: 0x0e, 0x19b: 0x0e, 0x19c: 0x0e, 0x19d: 0x0e, 0x19e: 0x0e, 0x19f: 0x0e, + 0x1a0: 0x0e, 0x1a1: 0x0e, 0x1a2: 0x0e, 0x1a3: 0x0e, 0x1a4: 0x0e, 0x1a5: 0x0e, 0x1a6: 0x0e, 0x1a7: 0x0e, + 0x1a8: 0x0e, 0x1a9: 0x0e, 0x1aa: 0x0e, 0x1ab: 0x0e, 0x1ac: 0x0e, 0x1ad: 0x0e, 0x1ae: 0x0e, 0x1af: 0x0e, + 0x1b0: 0x0e, 0x1b1: 0x0e, 0x1b2: 0x0e, 0x1b3: 0x0e, 0x1b4: 0x0e, 0x1b5: 0x0e, 0x1b6: 0x0e, 0x1b7: 0x0e, + 0x1b8: 0x0e, 0x1b9: 0x0e, 0x1ba: 0x0e, 0x1bb: 0x0e, 0x1bc: 0x0e, 0x1bd: 0x0e, 0x1be: 0x0e, 0x1bf: 0x0e, + // Block 0x7, offset 0x1c0 + 0x1c0: 0x0e, 0x1c1: 0x0e, 0x1c2: 0x0e, 0x1c3: 0x0e, 0x1c4: 0x0e, 0x1c5: 0x0e, 0x1c6: 0x0e, 0x1c7: 0x0e, + 0x1c8: 0x0e, 0x1c9: 0x0e, 0x1ca: 0x0e, 0x1cb: 0x0e, 0x1cc: 0x0e, 0x1cd: 0x0e, 0x1ce: 0x0e, 0x1cf: 0x0e, + 0x1d0: 0x0e, 0x1d1: 0x0e, 0x1d2: 0x0e, 0x1d3: 0x0e, 0x1d4: 0x0e, 0x1d5: 0x0e, 0x1d6: 0x0e, 0x1d7: 0x0e, + 0x1d8: 0x0e, 0x1d9: 0x0e, 0x1da: 0x0e, 0x1db: 0x0e, 0x1dc: 0x0e, 0x1dd: 0x0e, 0x1de: 0x0e, 0x1df: 0x0e, + 0x1e0: 0x0e, 0x1e1: 0x0e, 0x1e2: 0x0e, 0x1e3: 0x0e, 0x1e4: 0x0e, 0x1e5: 0x0e, 0x1e6: 0x0e, 0x1e7: 0x0e, + 0x1e8: 0x0e, 0x1e9: 0x0e, 0x1ea: 0x0e, 0x1eb: 0x0e, 0x1ec: 0x0e, 0x1ed: 0x0e, 0x1ee: 0x0e, 0x1ef: 0x0e, + 0x1f0: 0x0e, 0x1f1: 0x0e, 0x1f2: 0x0e, 0x1f3: 0x0e, 0x1f4: 0x0e, 0x1f5: 0x0e, 0x1f6: 0x0e, + 0x1f8: 0x0e, 0x1f9: 0x0e, 0x1fa: 0x0e, 0x1fb: 0x0e, 0x1fc: 0x0e, 0x1fd: 0x0e, 0x1fe: 0x0e, 0x1ff: 0x0e, + // Block 0x8, offset 0x200 + 0x200: 0x0e, 0x201: 0x0e, 0x202: 0x0e, 0x203: 0x0e, 0x204: 0x0e, 0x205: 0x0e, 0x206: 0x0e, 0x207: 0x0e, + 0x208: 0x0e, 0x209: 0x0e, 0x20a: 0x0e, 0x20b: 0x0e, 0x20c: 0x0e, 0x20d: 0x0e, 0x20e: 0x0e, 0x20f: 0x0e, + 0x210: 0x0e, 0x211: 0x0e, 0x212: 0x0e, 0x213: 0x0e, 0x214: 0x0e, 0x215: 0x0e, 0x216: 0x0e, 0x217: 0x0e, + 0x218: 0x0e, 0x219: 0x0e, 0x21a: 0x0e, 0x21b: 0x0e, 0x21c: 0x0e, 0x21d: 0x0e, 0x21e: 0x0e, 0x21f: 0x0e, + 0x220: 0x0e, 0x221: 0x0e, 0x222: 0x0e, 0x223: 0x0e, 0x224: 0x0e, 0x225: 0x0e, 0x226: 0x0e, 0x227: 0x0e, + 0x228: 0x0e, 0x229: 0x0e, 0x22a: 0x0e, 0x22b: 0x0e, 0x22c: 0x0e, 0x22d: 0x0e, 0x22e: 0x0e, 0x22f: 0x0e, + 0x230: 0x0e, 0x231: 0x0e, 0x232: 0x0e, 0x233: 0x0e, 0x234: 0x0e, 0x235: 0x0e, 0x236: 0x0e, 0x237: 0x0e, + 0x238: 0x0e, 0x239: 0x0e, 0x23a: 0x0e, 0x23b: 0x0e, 0x23c: 0x0e, 0x23d: 0x0e, 0x23e: 0x0e, 0x23f: 0x0e, + // Block 0x9, offset 0x240 + 0x240: 0x0e, 0x241: 0x0e, 0x242: 0x0e, 0x243: 0x0e, 0x244: 0x0e, 0x245: 0x0e, 0x246: 0x0e, 0x247: 0x0e, + 0x248: 0x0e, 0x249: 0x0e, 0x24a: 0x0e, 0x24b: 0x0e, 0x24c: 0x0e, 0x24d: 0x0e, 0x24e: 0x0e, 0x24f: 0x0e, + 0x250: 0x0e, 0x251: 0x0e, 0x252: 0x3b, 0x253: 0x3c, + 0x265: 0x3d, + 0x270: 0x0e, 0x271: 0x0e, 0x272: 0x0e, 0x273: 0x0e, 0x274: 0x0e, 0x275: 0x0e, 0x276: 0x0e, 0x277: 0x0e, + 0x278: 0x0e, 0x279: 0x0e, 0x27a: 0x0e, 0x27b: 0x0e, 0x27c: 0x0e, 0x27d: 0x0e, 0x27e: 0x0e, 0x27f: 0x0e, + // Block 0xa, offset 0x280 + 0x280: 0x0e, 0x281: 0x0e, 0x282: 0x0e, 0x283: 0x0e, 0x284: 0x0e, 0x285: 0x0e, 0x286: 0x0e, 0x287: 0x0e, + 0x288: 0x0e, 0x289: 0x0e, 0x28a: 0x0e, 0x28b: 0x0e, 0x28c: 0x0e, 0x28d: 0x0e, 0x28e: 0x0e, 0x28f: 0x0e, + 0x290: 0x0e, 0x291: 0x0e, 0x292: 0x0e, 0x293: 0x0e, 0x294: 0x0e, 0x295: 0x0e, 0x296: 0x0e, 0x297: 0x0e, + 0x298: 0x0e, 0x299: 0x0e, 0x29a: 0x0e, 0x29b: 0x0e, 0x29c: 0x0e, 0x29d: 0x0e, 0x29e: 0x3e, + // Block 0xb, offset 0x2c0 + 0x2c0: 0x08, 0x2c1: 0x08, 0x2c2: 0x08, 0x2c3: 0x08, 0x2c4: 0x08, 0x2c5: 0x08, 0x2c6: 0x08, 0x2c7: 0x08, + 0x2c8: 0x08, 0x2c9: 0x08, 0x2ca: 0x08, 0x2cb: 0x08, 0x2cc: 0x08, 0x2cd: 0x08, 0x2ce: 0x08, 0x2cf: 0x08, + 0x2d0: 0x08, 0x2d1: 0x08, 0x2d2: 0x08, 0x2d3: 0x08, 0x2d4: 0x08, 0x2d5: 0x08, 0x2d6: 0x08, 0x2d7: 0x08, + 0x2d8: 0x08, 0x2d9: 0x08, 0x2da: 0x08, 0x2db: 0x08, 0x2dc: 0x08, 0x2dd: 0x08, 0x2de: 0x08, 0x2df: 0x08, + 0x2e0: 0x08, 0x2e1: 0x08, 0x2e2: 0x08, 0x2e3: 0x08, 0x2e4: 0x08, 0x2e5: 0x08, 0x2e6: 0x08, 0x2e7: 0x08, + 0x2e8: 0x08, 0x2e9: 0x08, 0x2ea: 0x08, 0x2eb: 0x08, 0x2ec: 0x08, 0x2ed: 0x08, 0x2ee: 0x08, 0x2ef: 0x08, + 0x2f0: 0x08, 0x2f1: 0x08, 0x2f2: 0x08, 0x2f3: 0x08, 0x2f4: 0x08, 0x2f5: 0x08, 0x2f6: 0x08, 0x2f7: 0x08, + 0x2f8: 0x08, 0x2f9: 0x08, 0x2fa: 0x08, 0x2fb: 0x08, 0x2fc: 0x08, 0x2fd: 0x08, 0x2fe: 0x08, 0x2ff: 0x08, + // Block 0xc, offset 0x300 + 0x300: 0x08, 0x301: 0x08, 0x302: 0x08, 0x303: 0x08, 0x304: 0x08, 0x305: 0x08, 0x306: 0x08, 0x307: 0x08, + 0x308: 0x08, 0x309: 0x08, 0x30a: 0x08, 0x30b: 0x08, 0x30c: 0x08, 0x30d: 0x08, 0x30e: 0x08, 0x30f: 0x08, + 0x310: 0x08, 0x311: 0x08, 0x312: 0x08, 0x313: 0x08, 0x314: 0x08, 0x315: 0x08, 0x316: 0x08, 0x317: 0x08, + 0x318: 0x08, 0x319: 0x08, 0x31a: 0x08, 0x31b: 0x08, 0x31c: 0x08, 0x31d: 0x08, 0x31e: 0x08, 0x31f: 0x08, + 0x320: 0x08, 0x321: 0x08, 0x322: 0x08, 0x323: 0x08, 0x324: 0x0e, 0x325: 0x0e, 0x326: 0x0e, 0x327: 0x0e, + 0x328: 0x0e, 0x329: 0x0e, 0x32a: 0x0e, 0x32b: 0x0e, + 0x338: 0x3f, 0x339: 0x40, 0x33c: 0x41, 0x33d: 0x42, 0x33e: 0x43, 0x33f: 0x44, + // Block 0xd, offset 0x340 + 0x37f: 0x45, + // Block 0xe, offset 0x380 + 0x380: 0x0e, 0x381: 0x0e, 0x382: 0x0e, 0x383: 0x0e, 0x384: 0x0e, 0x385: 0x0e, 0x386: 0x0e, 0x387: 0x0e, + 0x388: 0x0e, 0x389: 0x0e, 0x38a: 0x0e, 0x38b: 0x0e, 0x38c: 0x0e, 0x38d: 0x0e, 0x38e: 0x0e, 0x38f: 0x0e, + 0x390: 0x0e, 0x391: 0x0e, 0x392: 0x0e, 0x393: 0x0e, 0x394: 0x0e, 0x395: 0x0e, 0x396: 0x0e, 0x397: 0x0e, + 0x398: 0x0e, 0x399: 0x0e, 0x39a: 0x0e, 0x39b: 0x0e, 0x39c: 0x0e, 0x39d: 0x0e, 0x39e: 0x0e, 0x39f: 0x46, + 0x3a0: 0x0e, 0x3a1: 0x0e, 0x3a2: 0x0e, 0x3a3: 0x0e, 0x3a4: 0x0e, 0x3a5: 0x0e, 0x3a6: 0x0e, 0x3a7: 0x0e, + 0x3a8: 0x0e, 0x3a9: 0x0e, 0x3aa: 0x0e, 0x3ab: 0x47, + // Block 0xf, offset 0x3c0 + 0x3c0: 0x48, + // Block 0x10, offset 0x400 + 0x400: 0x49, 0x403: 0x4a, 0x404: 0x4b, 0x405: 0x4c, 0x406: 0x4d, + 0x408: 0x4e, 0x409: 0x4f, 0x40c: 0x50, 0x40d: 0x51, 0x40e: 0x52, 0x40f: 0x53, + 0x410: 0x3a, 0x411: 0x54, 0x412: 0x0e, 0x413: 0x55, 0x414: 0x56, 0x415: 0x57, 0x416: 0x58, 0x417: 0x59, + 0x418: 0x0e, 0x419: 0x5a, 0x41a: 0x0e, 0x41b: 0x5b, + 0x424: 0x5c, 0x425: 0x5d, 0x426: 0x5e, 0x427: 0x5f, + // Block 0x11, offset 0x440 + 0x456: 0x0b, 0x457: 0x06, + 0x458: 0x0c, 0x45b: 0x0d, 0x45f: 0x0e, + 0x460: 0x06, 0x461: 0x06, 0x462: 0x06, 0x463: 0x06, 0x464: 0x06, 0x465: 0x06, 0x466: 0x06, 0x467: 0x06, + 0x468: 0x06, 0x469: 0x06, 0x46a: 0x06, 0x46b: 0x06, 0x46c: 0x06, 0x46d: 0x06, 0x46e: 0x06, 0x46f: 0x06, + 0x470: 0x06, 0x471: 0x06, 0x472: 0x06, 0x473: 0x06, 0x474: 0x06, 0x475: 0x06, 0x476: 0x06, 0x477: 0x06, + 0x478: 0x06, 0x479: 0x06, 0x47a: 0x06, 0x47b: 0x06, 0x47c: 0x06, 0x47d: 0x06, 0x47e: 0x06, 0x47f: 0x06, + // Block 0x12, offset 0x480 + 0x484: 0x08, 0x485: 0x08, 0x486: 0x08, 0x487: 0x09, + // Block 0x13, offset 0x4c0 + 0x4c0: 0x08, 0x4c1: 0x08, 0x4c2: 0x08, 0x4c3: 0x08, 0x4c4: 0x08, 0x4c5: 0x08, 0x4c6: 0x08, 0x4c7: 0x08, + 0x4c8: 0x08, 0x4c9: 0x08, 0x4ca: 0x08, 0x4cb: 0x08, 0x4cc: 0x08, 0x4cd: 0x08, 0x4ce: 0x08, 0x4cf: 0x08, + 0x4d0: 0x08, 0x4d1: 0x08, 0x4d2: 0x08, 0x4d3: 0x08, 0x4d4: 0x08, 0x4d5: 0x08, 0x4d6: 0x08, 0x4d7: 0x08, + 0x4d8: 0x08, 0x4d9: 0x08, 0x4da: 0x08, 0x4db: 0x08, 0x4dc: 0x08, 0x4dd: 0x08, 0x4de: 0x08, 0x4df: 0x08, + 0x4e0: 0x08, 0x4e1: 0x08, 0x4e2: 0x08, 0x4e3: 0x08, 0x4e4: 0x08, 0x4e5: 0x08, 0x4e6: 0x08, 0x4e7: 0x08, + 0x4e8: 0x08, 0x4e9: 0x08, 0x4ea: 0x08, 0x4eb: 0x08, 0x4ec: 0x08, 0x4ed: 0x08, 0x4ee: 0x08, 0x4ef: 0x08, + 0x4f0: 0x08, 0x4f1: 0x08, 0x4f2: 0x08, 0x4f3: 0x08, 0x4f4: 0x08, 0x4f5: 0x08, 0x4f6: 0x08, 0x4f7: 0x08, + 0x4f8: 0x08, 0x4f9: 0x08, 0x4fa: 0x08, 0x4fb: 0x08, 0x4fc: 0x08, 0x4fd: 0x08, 0x4fe: 0x08, 0x4ff: 0x60, + // Block 0x14, offset 0x500 + 0x520: 0x10, + 0x530: 0x09, 0x531: 0x09, 0x532: 0x09, 0x533: 0x09, 0x534: 0x09, 0x535: 0x09, 0x536: 0x09, 0x537: 0x09, + 0x538: 0x09, 0x539: 0x09, 0x53a: 0x09, 0x53b: 0x09, 0x53c: 0x09, 0x53d: 0x09, 0x53e: 0x09, 0x53f: 0x11, + // Block 0x15, offset 0x540 + 0x540: 0x09, 0x541: 0x09, 0x542: 0x09, 0x543: 0x09, 0x544: 0x09, 0x545: 0x09, 0x546: 0x09, 0x547: 0x09, + 0x548: 0x09, 0x549: 0x09, 0x54a: 0x09, 0x54b: 0x09, 0x54c: 0x09, 0x54d: 0x09, 0x54e: 0x09, 0x54f: 0x11, +} + +// inverseData contains 4-byte entries of the following format: +// +// <0 padding> +// +// The last byte of the UTF-8-encoded rune is xor-ed with the last byte of the +// UTF-8 encoding of the original rune. Mappings often have the following +// pattern: +// +// Ａ -> A (U+FF21 -> U+0041) +// Ｂ -> B (U+FF22 -> U+0042) +// ... +// +// By xor-ing the last byte the same entry can be shared by many mappings. This +// reduces the total number of distinct entries by about two thirds. +// The resulting entry for the aforementioned mappings is +// +// { 0x01, 0xE0, 0x00, 0x00 } +// +// Using this entry to map U+FF21 (UTF-8 [EF BC A1]), we get +// +// E0 ^ A1 = 41. +// +// Similarly, for U+FF22 (UTF-8 [EF BC A2]), we get +// +// E0 ^ A2 = 42. +// +// Note that because of the xor-ing, the byte sequence stored in the entry is +// not valid UTF-8. +var inverseData = [150][4]byte{ + {0x00, 0x00, 0x00, 0x00}, + {0x03, 0xe3, 0x80, 0xa0}, + {0x03, 0xef, 0xbc, 0xa0}, + {0x03, 0xef, 0xbc, 0xe0}, + {0x03, 0xef, 0xbd, 0xe0}, + {0x03, 0xef, 0xbf, 0x02}, + {0x03, 0xef, 0xbf, 0x00}, + {0x03, 0xef, 0xbf, 0x0e}, + {0x03, 0xef, 0xbf, 0x0c}, + {0x03, 0xef, 0xbf, 0x0f}, + {0x03, 0xef, 0xbf, 0x39}, + {0x03, 0xef, 0xbf, 0x3b}, + {0x03, 0xef, 0xbf, 0x3f}, + {0x03, 0xef, 0xbf, 0x2a}, + {0x03, 0xef, 0xbf, 0x0d}, + {0x03, 0xef, 0xbf, 0x25}, + {0x03, 0xef, 0xbd, 0x1a}, + {0x03, 0xef, 0xbd, 0x26}, + {0x01, 0xa0, 0x00, 0x00}, + {0x03, 0xef, 0xbd, 0x25}, + {0x03, 0xef, 0xbd, 0x23}, + {0x03, 0xef, 0xbd, 0x2e}, + {0x03, 0xef, 0xbe, 0x07}, + {0x03, 0xef, 0xbe, 0x05}, + {0x03, 0xef, 0xbd, 0x06}, + {0x03, 0xef, 0xbd, 0x13}, + {0x03, 0xef, 0xbd, 0x0b}, + {0x03, 0xef, 0xbd, 0x16}, + {0x03, 0xef, 0xbd, 0x0c}, + {0x03, 0xef, 0xbd, 0x15}, + {0x03, 0xef, 0xbd, 0x0d}, + {0x03, 0xef, 0xbd, 0x1c}, + {0x03, 0xef, 0xbd, 0x02}, + {0x03, 0xef, 0xbd, 0x1f}, + {0x03, 0xef, 0xbd, 0x1d}, + {0x03, 0xef, 0xbd, 0x17}, + {0x03, 0xef, 0xbd, 0x08}, + {0x03, 0xef, 0xbd, 0x09}, + {0x03, 0xef, 0xbd, 0x0e}, + {0x03, 0xef, 0xbd, 0x04}, + {0x03, 0xef, 0xbd, 0x05}, + {0x03, 0xef, 0xbe, 0x3f}, + {0x03, 0xef, 0xbe, 0x00}, + {0x03, 0xef, 0xbd, 0x2c}, + {0x03, 0xef, 0xbe, 0x06}, + {0x03, 0xef, 0xbe, 0x0c}, + {0x03, 0xef, 0xbe, 0x0f}, + {0x03, 0xef, 0xbe, 0x0d}, + {0x03, 0xef, 0xbe, 0x0b}, + {0x03, 0xef, 0xbe, 0x19}, + {0x03, 0xef, 0xbe, 0x15}, + {0x03, 0xef, 0xbe, 0x11}, + {0x03, 0xef, 0xbe, 0x31}, + {0x03, 0xef, 0xbe, 0x33}, + {0x03, 0xef, 0xbd, 0x0f}, + {0x03, 0xef, 0xbe, 0x30}, + {0x03, 0xef, 0xbe, 0x3e}, + {0x03, 0xef, 0xbe, 0x32}, + {0x03, 0xef, 0xbe, 0x36}, + {0x03, 0xef, 0xbd, 0x14}, + {0x03, 0xef, 0xbe, 0x2e}, + {0x03, 0xef, 0xbd, 0x1e}, + {0x03, 0xef, 0xbe, 0x10}, + {0x03, 0xef, 0xbf, 0x13}, + {0x03, 0xef, 0xbf, 0x15}, + {0x03, 0xef, 0xbf, 0x17}, + {0x03, 0xef, 0xbf, 0x1f}, + {0x03, 0xef, 0xbf, 0x1d}, + {0x03, 0xef, 0xbf, 0x1b}, + {0x03, 0xef, 0xbf, 0x09}, + {0x03, 0xef, 0xbf, 0x0b}, + {0x03, 0xef, 0xbf, 0x37}, + {0x03, 0xef, 0xbe, 0x04}, + {0x01, 0xe0, 0x00, 0x00}, + {0x03, 0xe2, 0xa6, 0x1a}, + {0x03, 0xe2, 0xa6, 0x26}, + {0x03, 0xe3, 0x80, 0x23}, + {0x03, 0xe3, 0x80, 0x2e}, + {0x03, 0xe3, 0x80, 0x25}, + {0x03, 0xe3, 0x83, 0x1e}, + {0x03, 0xe3, 0x83, 0x14}, + {0x03, 0xe3, 0x82, 0x06}, + {0x03, 0xe3, 0x82, 0x0b}, + {0x03, 0xe3, 0x82, 0x0c}, + {0x03, 0xe3, 0x82, 0x0d}, + {0x03, 0xe3, 0x82, 0x02}, + {0x03, 0xe3, 0x83, 0x0f}, + {0x03, 0xe3, 0x83, 0x08}, + {0x03, 0xe3, 0x83, 0x09}, + {0x03, 0xe3, 0x83, 0x2c}, + {0x03, 0xe3, 0x83, 0x0c}, + {0x03, 0xe3, 0x82, 0x13}, + {0x03, 0xe3, 0x82, 0x16}, + {0x03, 0xe3, 0x82, 0x15}, + {0x03, 0xe3, 0x82, 0x1c}, + {0x03, 0xe3, 0x82, 0x1f}, + {0x03, 0xe3, 0x82, 0x1d}, + {0x03, 0xe3, 0x82, 0x1a}, + {0x03, 0xe3, 0x82, 0x17}, + {0x03, 0xe3, 0x82, 0x08}, + {0x03, 0xe3, 0x82, 0x09}, + {0x03, 0xe3, 0x82, 0x0e}, + {0x03, 0xe3, 0x82, 0x04}, + {0x03, 0xe3, 0x82, 0x05}, + {0x03, 0xe3, 0x82, 0x3f}, + {0x03, 0xe3, 0x83, 0x00}, + {0x03, 0xe3, 0x83, 0x06}, + {0x03, 0xe3, 0x83, 0x05}, + {0x03, 0xe3, 0x83, 0x0d}, + {0x03, 0xe3, 0x83, 0x0b}, + {0x03, 0xe3, 0x83, 0x07}, + {0x03, 0xe3, 0x83, 0x19}, + {0x03, 0xe3, 0x83, 0x15}, + {0x03, 0xe3, 0x83, 0x11}, + {0x03, 0xe3, 0x83, 0x31}, + {0x03, 0xe3, 0x83, 0x33}, + {0x03, 0xe3, 0x83, 0x30}, + {0x03, 0xe3, 0x83, 0x3e}, + {0x03, 0xe3, 0x83, 0x32}, + {0x03, 0xe3, 0x83, 0x36}, + {0x03, 0xe3, 0x83, 0x2e}, + {0x03, 0xe3, 0x82, 0x07}, + {0x03, 0xe3, 0x85, 0x04}, + {0x03, 0xe3, 0x84, 0x10}, + {0x03, 0xe3, 0x85, 0x30}, + {0x03, 0xe3, 0x85, 0x0d}, + {0x03, 0xe3, 0x85, 0x13}, + {0x03, 0xe3, 0x85, 0x15}, + {0x03, 0xe3, 0x85, 0x17}, + {0x03, 0xe3, 0x85, 0x1f}, + {0x03, 0xe3, 0x85, 0x1d}, + {0x03, 0xe3, 0x85, 0x1b}, + {0x03, 0xe3, 0x85, 0x09}, + {0x03, 0xe3, 0x85, 0x0f}, + {0x03, 0xe3, 0x85, 0x0b}, + {0x03, 0xe3, 0x85, 0x37}, + {0x03, 0xe3, 0x85, 0x3b}, + {0x03, 0xe3, 0x85, 0x39}, + {0x03, 0xe3, 0x85, 0x3f}, + {0x02, 0xc2, 0x02, 0x00}, + {0x02, 0xc2, 0x0e, 0x00}, + {0x02, 0xc2, 0x0c, 0x00}, + {0x02, 0xc2, 0x00, 0x00}, + {0x03, 0xe2, 0x82, 0x0f}, + {0x03, 0xe2, 0x94, 0x2a}, + {0x03, 0xe2, 0x86, 0x39}, + {0x03, 0xe2, 0x86, 0x3b}, + {0x03, 0xe2, 0x86, 0x3f}, + {0x03, 0xe2, 0x96, 0x0d}, + {0x03, 0xe2, 0x97, 0x25}, +} + +// Total table size 14680 bytes (14KiB) diff --git a/vendor/golang.org/x/text/width/transform.go b/vendor/golang.org/x/text/width/transform.go new file mode 100644 index 0000000000..0049f700a2 --- /dev/null +++ b/vendor/golang.org/x/text/width/transform.go @@ -0,0 +1,239 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package width + +import ( + "unicode/utf8" + + "golang.org/x/text/transform" +) + +type foldTransform struct { + transform.NopResetter +} + +func (foldTransform) Span(src []byte, atEOF bool) (n int, err error) { + for n < len(src) { + if src[n] < utf8.RuneSelf { + // ASCII fast path. + for n++; n < len(src) && src[n] < utf8.RuneSelf; n++ { + } + continue + } + v, size := trie.lookup(src[n:]) + if size == 0 { // incomplete UTF-8 encoding + if !atEOF { + err = transform.ErrShortSrc + } else { + n = len(src) + } + break + } + if elem(v)&tagNeedsFold != 0 { + err = transform.ErrEndOfSpan + break + } + n += size + } + return n, err +} + +func (foldTransform) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { + for nSrc < len(src) { + if src[nSrc] < utf8.RuneSelf { + // ASCII fast path. + start, end := nSrc, len(src) + if d := len(dst) - nDst; d < end-start { + end = nSrc + d + } + for nSrc++; nSrc < end && src[nSrc] < utf8.RuneSelf; nSrc++ { + } + n := copy(dst[nDst:], src[start:nSrc]) + if nDst += n; nDst == len(dst) { + nSrc = start + n + if nSrc == len(src) { + return nDst, nSrc, nil + } + if src[nSrc] < utf8.RuneSelf { + return nDst, nSrc, transform.ErrShortDst + } + } + continue + } + v, size := trie.lookup(src[nSrc:]) + if size == 0 { // incomplete UTF-8 encoding + if !atEOF { + return nDst, nSrc, transform.ErrShortSrc + } + size = 1 // gobble 1 byte + } + if elem(v)&tagNeedsFold == 0 { + if size != copy(dst[nDst:], src[nSrc:nSrc+size]) { + return nDst, nSrc, transform.ErrShortDst + } + nDst += size + } else { + data := inverseData[byte(v)] + if len(dst)-nDst < int(data[0]) { + return nDst, nSrc, transform.ErrShortDst + } + i := 1 + for end := int(data[0]); i < end; i++ { + dst[nDst] = data[i] + nDst++ + } + dst[nDst] = data[i] ^ src[nSrc+size-1] + nDst++ + } + nSrc += size + } + return nDst, nSrc, nil +} + +type narrowTransform struct { + transform.NopResetter +} + +func (narrowTransform) Span(src []byte, atEOF bool) (n int, err error) { + for n < len(src) { + if src[n] < utf8.RuneSelf { + // ASCII fast path. + for n++; n < len(src) && src[n] < utf8.RuneSelf; n++ { + } + continue + } + v, size := trie.lookup(src[n:]) + if size == 0 { // incomplete UTF-8 encoding + if !atEOF { + err = transform.ErrShortSrc + } else { + n = len(src) + } + break + } + if k := elem(v).kind(); byte(v) == 0 || k != EastAsianFullwidth && k != EastAsianWide && k != EastAsianAmbiguous { + } else { + err = transform.ErrEndOfSpan + break + } + n += size + } + return n, err +} + +func (narrowTransform) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { + for nSrc < len(src) { + if src[nSrc] < utf8.RuneSelf { + // ASCII fast path. + start, end := nSrc, len(src) + if d := len(dst) - nDst; d < end-start { + end = nSrc + d + } + for nSrc++; nSrc < end && src[nSrc] < utf8.RuneSelf; nSrc++ { + } + n := copy(dst[nDst:], src[start:nSrc]) + if nDst += n; nDst == len(dst) { + nSrc = start + n + if nSrc == len(src) { + return nDst, nSrc, nil + } + if src[nSrc] < utf8.RuneSelf { + return nDst, nSrc, transform.ErrShortDst + } + } + continue + } + v, size := trie.lookup(src[nSrc:]) + if size == 0 { // incomplete UTF-8 encoding + if !atEOF { + return nDst, nSrc, transform.ErrShortSrc + } + size = 1 // gobble 1 byte + } + if k := elem(v).kind(); byte(v) == 0 || k != EastAsianFullwidth && k != EastAsianWide && k != EastAsianAmbiguous { + if size != copy(dst[nDst:], src[nSrc:nSrc+size]) { + return nDst, nSrc, transform.ErrShortDst + } + nDst += size + } else { + data := inverseData[byte(v)] + if len(dst)-nDst < int(data[0]) { + return nDst, nSrc, transform.ErrShortDst + } + i := 1 + for end := int(data[0]); i < end; i++ { + dst[nDst] = data[i] + nDst++ + } + dst[nDst] = data[i] ^ src[nSrc+size-1] + nDst++ + } + nSrc += size + } + return nDst, nSrc, nil +} + +type wideTransform struct { + transform.NopResetter +} + +func (wideTransform) Span(src []byte, atEOF bool) (n int, err error) { + for n < len(src) { + // TODO: Consider ASCII fast path. Special-casing ASCII handling can + // reduce the ns/op of BenchmarkWideASCII by about 30%. This is probably + // not enough to warrant the extra code and complexity. + v, size := trie.lookup(src[n:]) + if size == 0 { // incomplete UTF-8 encoding + if !atEOF { + err = transform.ErrShortSrc + } else { + n = len(src) + } + break + } + if k := elem(v).kind(); byte(v) == 0 || k != EastAsianHalfwidth && k != EastAsianNarrow { + } else { + err = transform.ErrEndOfSpan + break + } + n += size + } + return n, err +} + +func (wideTransform) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { + for nSrc < len(src) { + // TODO: Consider ASCII fast path. Special-casing ASCII handling can + // reduce the ns/op of BenchmarkWideASCII by about 30%. This is probably + // not enough to warrant the extra code and complexity. + v, size := trie.lookup(src[nSrc:]) + if size == 0 { // incomplete UTF-8 encoding + if !atEOF { + return nDst, nSrc, transform.ErrShortSrc + } + size = 1 // gobble 1 byte + } + if k := elem(v).kind(); byte(v) == 0 || k != EastAsianHalfwidth && k != EastAsianNarrow { + if size != copy(dst[nDst:], src[nSrc:nSrc+size]) { + return nDst, nSrc, transform.ErrShortDst + } + nDst += size + } else { + data := inverseData[byte(v)] + if len(dst)-nDst < int(data[0]) { + return nDst, nSrc, transform.ErrShortDst + } + i := 1 + for end := int(data[0]); i < end; i++ { + dst[nDst] = data[i] + nDst++ + } + dst[nDst] = data[i] ^ src[nSrc+size-1] + nDst++ + } + nSrc += size + } + return nDst, nSrc, nil +} diff --git a/vendor/golang.org/x/text/width/trieval.go b/vendor/golang.org/x/text/width/trieval.go new file mode 100644 index 0000000000..ca8e45fd19 --- /dev/null +++ b/vendor/golang.org/x/text/width/trieval.go @@ -0,0 +1,30 @@ +// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. + +package width + +// elem is an entry of the width trie. The high byte is used to encode the type +// of the rune. The low byte is used to store the index to a mapping entry in +// the inverseData array. +type elem uint16 + +const ( + tagNeutral elem = iota << typeShift + tagAmbiguous + tagWide + tagNarrow + tagFullwidth + tagHalfwidth +) + +const ( + numTypeBits = 3 + typeShift = 16 - numTypeBits + + // tagNeedsFold is true for all fullwidth and halfwidth runes except for + // the Won sign U+20A9. + tagNeedsFold = 0x1000 + + // The Korean Won sign is halfwidth, but SHOULD NOT be mapped to a wide + // variant. + wonSign rune = 0x20A9 +) diff --git a/vendor/golang.org/x/text/width/width.go b/vendor/golang.org/x/text/width/width.go new file mode 100644 index 0000000000..29c7509be7 --- /dev/null +++ b/vendor/golang.org/x/text/width/width.go @@ -0,0 +1,206 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:generate stringer -type=Kind +//go:generate go run gen.go gen_common.go gen_trieval.go + +// Package width provides functionality for handling different widths in text. +// +// Wide characters behave like ideographs; they tend to allow line breaks after +// each character and remain upright in vertical text layout. Narrow characters +// are kept together in words or runs that are rotated sideways in vertical text +// layout. +// +// For more information, see https://unicode.org/reports/tr11/. +package width // import "golang.org/x/text/width" + +import ( + "unicode/utf8" + + "golang.org/x/text/transform" +) + +// TODO +// 1) Reduce table size by compressing blocks. +// 2) API proposition for computing display length +// (approximation, fixed pitch only). +// 3) Implement display length. + +// Kind indicates the type of width property as defined in https://unicode.org/reports/tr11/. +type Kind int + +const ( + // Neutral characters do not occur in legacy East Asian character sets. + Neutral Kind = iota + + // EastAsianAmbiguous characters that can be sometimes wide and sometimes + // narrow and require additional information not contained in the character + // code to further resolve their width. + EastAsianAmbiguous + + // EastAsianWide characters are wide in its usual form. They occur only in + // the context of East Asian typography. These runes may have explicit + // halfwidth counterparts. + EastAsianWide + + // EastAsianNarrow characters are narrow in its usual form. They often have + // fullwidth counterparts. + EastAsianNarrow + + // Note: there exist Narrow runes that do not have fullwidth or wide + // counterparts, despite what the definition says (e.g. U+27E6). + + // EastAsianFullwidth characters have a compatibility decompositions of type + // wide that map to a narrow counterpart. + EastAsianFullwidth + + // EastAsianHalfwidth characters have a compatibility decomposition of type + // narrow that map to a wide or ambiguous counterpart, plus U+20A9 ₩ WON + // SIGN. + EastAsianHalfwidth + + // Note: there exist runes that have a halfwidth counterparts but that are + // classified as Ambiguous, rather than wide (e.g. U+2190). +) + +// TODO: the generated tries need to return size 1 for invalid runes for the +// width to be computed correctly (each byte should render width 1) + +var trie = newWidthTrie(0) + +// Lookup reports the Properties of the first rune in b and the number of bytes +// of its UTF-8 encoding. +func Lookup(b []byte) (p Properties, size int) { + v, sz := trie.lookup(b) + return Properties{elem(v), b[sz-1]}, sz +} + +// LookupString reports the Properties of the first rune in s and the number of +// bytes of its UTF-8 encoding. +func LookupString(s string) (p Properties, size int) { + v, sz := trie.lookupString(s) + return Properties{elem(v), s[sz-1]}, sz +} + +// LookupRune reports the Properties of rune r. +func LookupRune(r rune) Properties { + var buf [4]byte + n := utf8.EncodeRune(buf[:], r) + v, _ := trie.lookup(buf[:n]) + last := byte(r) + if r >= utf8.RuneSelf { + last = 0x80 + byte(r&0x3f) + } + return Properties{elem(v), last} +} + +// Properties provides access to width properties of a rune. +type Properties struct { + elem elem + last byte +} + +func (e elem) kind() Kind { + return Kind(e >> typeShift) +} + +// Kind returns the Kind of a rune as defined in Unicode TR #11. +// See https://unicode.org/reports/tr11/ for more details. +func (p Properties) Kind() Kind { + return p.elem.kind() +} + +// Folded returns the folded variant of a rune or 0 if the rune is canonical. +func (p Properties) Folded() rune { + if p.elem&tagNeedsFold != 0 { + buf := inverseData[byte(p.elem)] + buf[buf[0]] ^= p.last + r, _ := utf8.DecodeRune(buf[1 : 1+buf[0]]) + return r + } + return 0 +} + +// Narrow returns the narrow variant of a rune or 0 if the rune is already +// narrow or doesn't have a narrow variant. +func (p Properties) Narrow() rune { + if k := p.elem.kind(); byte(p.elem) != 0 && (k == EastAsianFullwidth || k == EastAsianWide || k == EastAsianAmbiguous) { + buf := inverseData[byte(p.elem)] + buf[buf[0]] ^= p.last + r, _ := utf8.DecodeRune(buf[1 : 1+buf[0]]) + return r + } + return 0 +} + +// Wide returns the wide variant of a rune or 0 if the rune is already +// wide or doesn't have a wide variant. +func (p Properties) Wide() rune { + if k := p.elem.kind(); byte(p.elem) != 0 && (k == EastAsianHalfwidth || k == EastAsianNarrow) { + buf := inverseData[byte(p.elem)] + buf[buf[0]] ^= p.last + r, _ := utf8.DecodeRune(buf[1 : 1+buf[0]]) + return r + } + return 0 +} + +// TODO for Properties: +// - Add Fullwidth/Halfwidth or Inverted methods for computing variants +// mapping. +// - Add width information (including information on non-spacing runes). + +// Transformer implements the transform.Transformer interface. +type Transformer struct { + t transform.SpanningTransformer +} + +// Reset implements the transform.Transformer interface. +func (t Transformer) Reset() { t.t.Reset() } + +// Transform implements the transform.Transformer interface. +func (t Transformer) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { + return t.t.Transform(dst, src, atEOF) +} + +// Span implements the transform.SpanningTransformer interface. +func (t Transformer) Span(src []byte, atEOF bool) (n int, err error) { + return t.t.Span(src, atEOF) +} + +// Bytes returns a new byte slice with the result of applying t to b. +func (t Transformer) Bytes(b []byte) []byte { + b, _, _ = transform.Bytes(t, b) + return b +} + +// String returns a string with the result of applying t to s. +func (t Transformer) String(s string) string { + s, _, _ = transform.String(t, s) + return s +} + +var ( + // Fold is a transform that maps all runes to their canonical width. + // + // Note that the NFKC and NFKD transforms in golang.org/x/text/unicode/norm + // provide a more generic folding mechanism. + Fold Transformer = Transformer{foldTransform{}} + + // Widen is a transform that maps runes to their wide variant, if + // available. + Widen Transformer = Transformer{wideTransform{}} + + // Narrow is a transform that maps runes to their narrow variant, if + // available. + Narrow Transformer = Transformer{narrowTransform{}} +) + +// TODO: Consider the following options: +// - Treat Ambiguous runes that have a halfwidth counterpart as wide, or some +// generalized variant of this. +// - Consider a wide Won character to be the default width (or some generalized +// variant of this). +// - Filter the set of characters that gets converted (the preferred approach is +// to allow applying filters to transforms). From 5aa3f1fa818d8c4e70b495c57cad1b48f6ff6400 Mon Sep 17 00:00:00 2001 From: Hemant Kumar Date: Tue, 9 Jan 2024 10:22:41 -0500 Subject: [PATCH 2/2] UPSTREAM: Update vendor --- go.mod | 2 +- go.sum | 8 +- .../github.com/PuerkitoBio/purell/.gitignore | 5 - .../github.com/PuerkitoBio/purell/.travis.yml | 12 - vendor/github.com/PuerkitoBio/purell/LICENSE | 12 - .../github.com/PuerkitoBio/purell/README.md | 188 --- .../github.com/PuerkitoBio/purell/purell.go | 379 ----- .../github.com/PuerkitoBio/urlesc/.travis.yml | 15 - vendor/github.com/PuerkitoBio/urlesc/LICENSE | 27 - .../github.com/PuerkitoBio/urlesc/README.md | 16 - .../github.com/PuerkitoBio/urlesc/urlesc.go | 180 --- .../jsonreference/internal/normalize_url.go | 63 + .../go-openapi/jsonreference/reference.go | 6 +- .../github.com/go-openapi/swag/.gitattributes | 2 + .../github.com/go-openapi/swag/.golangci.yml | 15 + vendor/github.com/go-openapi/swag/.travis.yml | 37 - vendor/github.com/go-openapi/swag/doc.go | 15 +- vendor/github.com/go-openapi/swag/file.go | 33 + vendor/github.com/go-openapi/swag/loading.go | 11 +- .../github.com/go-openapi/swag/post_go18.go | 1 + .../github.com/go-openapi/swag/post_go19.go | 1 + vendor/github.com/go-openapi/swag/pre_go18.go | 1 + vendor/github.com/go-openapi/swag/pre_go19.go | 1 + vendor/github.com/go-openapi/swag/util.go | 17 +- vendor/github.com/go-openapi/swag/yaml.go | 252 ++- .../google/gnostic/jsonschema/display.go | 17 +- .../google/gnostic/jsonschema/models.go | 8 +- .../google/gnostic/jsonschema/reader.go | 1 - .../google/gnostic/jsonschema/writer.go | 30 +- .../google/gnostic/openapiv2/OpenAPIv2.go | 7 +- .../google/gnostic/openapiv3/OpenAPIv3.go | 7 +- .../google/gnostic/openapiv3/OpenAPIv3.pb.go | 13 +- .../google/gnostic/openapiv3/OpenAPIv3.proto | 2 +- .../google/gnostic/openapiv3/README.md | 4 + .../gnostic/openapiv3/annotations.pb.go | 183 +++ .../gnostic/openapiv3/annotations.proto | 60 + .../mailru/easyjson/jlexer/lexer.go | 14 + .../github.com/vmware/govmomi/find/finder.go | 14 +- .../github.com/vmware/govmomi/list/lister.go | 4 +- .../vmware/govmomi/lookup/client.go | 4 +- .../vmware/govmomi/simulator/container.go | 14 + vendor/golang.org/x/text/width/kind_string.go | 28 - .../golang.org/x/text/width/tables10.0.0.go | 1329 ---------------- .../golang.org/x/text/width/tables11.0.0.go | 1341 ---------------- .../golang.org/x/text/width/tables12.0.0.go | 1361 ---------------- .../golang.org/x/text/width/tables13.0.0.go | 1362 ---------------- .../golang.org/x/text/width/tables15.0.0.go | 1368 ----------------- vendor/golang.org/x/text/width/tables9.0.0.go | 1297 ---------------- vendor/golang.org/x/text/width/transform.go | 239 --- vendor/golang.org/x/text/width/trieval.go | 30 - vendor/golang.org/x/text/width/width.go | 206 --- vendor/modules.txt | 19 +- 52 files changed, 745 insertions(+), 9516 deletions(-) delete mode 100644 vendor/github.com/PuerkitoBio/purell/.gitignore delete mode 100644 vendor/github.com/PuerkitoBio/purell/.travis.yml delete mode 100644 vendor/github.com/PuerkitoBio/purell/LICENSE delete mode 100644 vendor/github.com/PuerkitoBio/purell/README.md delete mode 100644 vendor/github.com/PuerkitoBio/purell/purell.go delete mode 100644 vendor/github.com/PuerkitoBio/urlesc/.travis.yml delete mode 100644 vendor/github.com/PuerkitoBio/urlesc/LICENSE delete mode 100644 vendor/github.com/PuerkitoBio/urlesc/README.md delete mode 100644 vendor/github.com/PuerkitoBio/urlesc/urlesc.go create mode 100644 vendor/github.com/go-openapi/jsonreference/internal/normalize_url.go create mode 100644 vendor/github.com/go-openapi/swag/.gitattributes delete mode 100644 vendor/github.com/go-openapi/swag/.travis.yml create mode 100644 vendor/github.com/go-openapi/swag/file.go create mode 100644 vendor/github.com/google/gnostic/openapiv3/annotations.pb.go create mode 100644 vendor/github.com/google/gnostic/openapiv3/annotations.proto delete mode 100644 vendor/golang.org/x/text/width/kind_string.go delete mode 100644 vendor/golang.org/x/text/width/tables10.0.0.go delete mode 100644 vendor/golang.org/x/text/width/tables11.0.0.go delete mode 100644 vendor/golang.org/x/text/width/tables12.0.0.go delete mode 100644 vendor/golang.org/x/text/width/tables13.0.0.go delete mode 100644 vendor/golang.org/x/text/width/tables15.0.0.go delete mode 100644 vendor/golang.org/x/text/width/tables9.0.0.go delete mode 100644 vendor/golang.org/x/text/width/transform.go delete mode 100644 vendor/golang.org/x/text/width/trieval.go delete mode 100644 vendor/golang.org/x/text/width/width.go diff --git a/go.mod b/go.mod index f106981290..7c844f12a2 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/hashicorp/go-version v1.6.0 github.com/kubernetes-csi/csi-lib-utils v0.11.0 github.com/kubernetes-csi/csi-proxy/client v1.0.1 - github.com/kubernetes-csi/external-snapshotter/client/v6 v6.1.0 + github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0 github.com/onsi/ginkgo/v2 v2.8.3 github.com/onsi/gomega v1.27.0 github.com/prometheus/client_golang v1.14.0 diff --git a/go.sum b/go.sum index e8c489d3e0..097c19ea06 100644 --- a/go.sum +++ b/go.sum @@ -412,8 +412,8 @@ github.com/kubernetes-csi/csi-lib-utils v0.11.0 h1:FHWOBtAZBA/hVk7v/qaXgG9Sxv0/n github.com/kubernetes-csi/csi-lib-utils v0.11.0/go.mod h1:BmGZZB16L18+9+Lgg9YWwBKfNEHIDdgGfAyuW6p2NV0= github.com/kubernetes-csi/csi-proxy/client v1.0.1 h1:BPK9e5Fy0GcDRjDc9hqu7TnouSRujG6IvbH+PXSDOsY= github.com/kubernetes-csi/csi-proxy/client v1.0.1/go.mod h1:URLOkEbRhOwKVvGvug6HSKRTpLSFuQ/Gt3xahDag8qc= -github.com/kubernetes-csi/external-snapshotter/client/v6 v6.1.0 h1:yeuon3bOuOADwiWl2CyYrU4vbmYbAzGLCTscE1yLNHk= -github.com/kubernetes-csi/external-snapshotter/client/v6 v6.1.0/go.mod h1:eVY6gNtSrhsblGAqKFDG3CrkCLFAjsDvOpPpt+EaS6k= +github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0 h1:nHHjmvjitIiyPlUHk/ofpgvBcNcawJLtf4PYHORLjAA= +github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0/go.mod h1:YBCo4DoEeDndqvAn6eeu0vWM7QdXmHEeI9cFWplmBys= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY= @@ -724,6 +724,7 @@ golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -902,6 +903,7 @@ golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1128,6 +1130,7 @@ k8s.io/client-go v0.26.10 h1:4mDzl+1IrfRxh4Ro0s65JRGJp14w77gSMUTjACYWVRo= k8s.io/client-go v0.26.10/go.mod h1:sh74ig838gCckU4ElYclWb24lTesPdEDPnlyg5vcbkA= k8s.io/cloud-provider v0.26.10 h1:KEKR5IN508u6qKTIp8hiQshdwjp2vAmUf1dq00YeqwE= k8s.io/cloud-provider v0.26.10/go.mod h1:s8jaxZgFcipPVnGMxLzWbCG46BYK8ExpBaqMjtUswVg= +k8s.io/code-generator v0.26.10/go.mod h1:+IHzChHYqL6v5M5KVRglocWMzdSzH3I2jRXZK05yZ9I= k8s.io/component-base v0.26.10 h1:vl3Gfe5aC09mNxfnQtTng7u3rnBVrShOK3MAkqEleb0= k8s.io/component-base v0.26.10/go.mod h1:/IDdENUHG5uGxqcofZajovYXE9KSPzJ4yQbkYQt7oN0= k8s.io/component-helpers v0.26.10 h1:KEwLNxzTE65R2kNz4UZ26h1G9O8xd6+iXVz7jkLgEYc= @@ -1139,6 +1142,7 @@ k8s.io/csi-translation-lib v0.26.10/go.mod h1:qx+y88RGweEWgYpmU5HwnQA69NzguWAGWH k8s.io/dynamic-resource-allocation v0.26.10 h1:w+m8eEq7Ududi1r7LTlXfzl1C/aXS/7sMLlFdpCd6v4= k8s.io/dynamic-resource-allocation v0.26.10/go.mod h1:piJ6x7/p0cb+xJIuX/MnGO+bKKX1QOErxY+Z1PgerrU= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/gengo v0.0.0-20220902162205-c0856e24416d/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= diff --git a/vendor/github.com/PuerkitoBio/purell/.gitignore b/vendor/github.com/PuerkitoBio/purell/.gitignore deleted file mode 100644 index 748e4c8073..0000000000 --- a/vendor/github.com/PuerkitoBio/purell/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -*.sublime-* -.DS_Store -*.swp -*.swo -tags diff --git a/vendor/github.com/PuerkitoBio/purell/.travis.yml b/vendor/github.com/PuerkitoBio/purell/.travis.yml deleted file mode 100644 index cf31e6af6d..0000000000 --- a/vendor/github.com/PuerkitoBio/purell/.travis.yml +++ /dev/null @@ -1,12 +0,0 @@ -language: go - -go: - - 1.4.x - - 1.5.x - - 1.6.x - - 1.7.x - - 1.8.x - - 1.9.x - - "1.10.x" - - "1.11.x" - - tip diff --git a/vendor/github.com/PuerkitoBio/purell/LICENSE b/vendor/github.com/PuerkitoBio/purell/LICENSE deleted file mode 100644 index 4b9986dea7..0000000000 --- a/vendor/github.com/PuerkitoBio/purell/LICENSE +++ /dev/null @@ -1,12 +0,0 @@ -Copyright (c) 2012, Martin Angers -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - -* Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/PuerkitoBio/purell/README.md b/vendor/github.com/PuerkitoBio/purell/README.md deleted file mode 100644 index 07de0c4986..0000000000 --- a/vendor/github.com/PuerkitoBio/purell/README.md +++ /dev/null @@ -1,188 +0,0 @@ -# Purell - -Purell is a tiny Go library to normalize URLs. It returns a pure URL. Pure-ell. Sanitizer and all. Yeah, I know... - -Based on the [wikipedia paper][wiki] and the [RFC 3986 document][rfc]. - -[![build status](https://travis-ci.org/PuerkitoBio/purell.svg?branch=master)](http://travis-ci.org/PuerkitoBio/purell) - -## Install - -`go get github.com/PuerkitoBio/purell` - -## Changelog - -* **v1.1.1** : Fix failing test due to Go1.12 changes (thanks to @ianlancetaylor). -* **2016-11-14 (v1.1.0)** : IDN: Conform to RFC 5895: Fold character width (thanks to @beeker1121). -* **2016-07-27 (v1.0.0)** : Normalize IDN to ASCII (thanks to @zenovich). -* **2015-02-08** : Add fix for relative paths issue ([PR #5][pr5]) and add fix for unnecessary encoding of reserved characters ([see issue #7][iss7]). -* **v0.2.0** : Add benchmarks, Attempt IDN support. -* **v0.1.0** : Initial release. - -## Examples - -From `example_test.go` (note that in your code, you would import "github.com/PuerkitoBio/purell", and would prefix references to its methods and constants with "purell."): - -```go -package purell - -import ( - "fmt" - "net/url" -) - -func ExampleNormalizeURLString() { - if normalized, err := NormalizeURLString("hTTp://someWEBsite.com:80/Amazing%3f/url/", - FlagLowercaseScheme|FlagLowercaseHost|FlagUppercaseEscapes); err != nil { - panic(err) - } else { - fmt.Print(normalized) - } - // Output: http://somewebsite.com:80/Amazing%3F/url/ -} - -func ExampleMustNormalizeURLString() { - normalized := MustNormalizeURLString("hTTpS://someWEBsite.com:443/Amazing%fa/url/", - FlagsUnsafeGreedy) - fmt.Print(normalized) - - // Output: http://somewebsite.com/Amazing%FA/url -} - -func ExampleNormalizeURL() { - if u, err := url.Parse("Http://SomeUrl.com:8080/a/b/.././c///g?c=3&a=1&b=9&c=0#target"); err != nil { - panic(err) - } else { - normalized := NormalizeURL(u, FlagsUsuallySafeGreedy|FlagRemoveDuplicateSlashes|FlagRemoveFragment) - fmt.Print(normalized) - } - - // Output: http://someurl.com:8080/a/c/g?c=3&a=1&b=9&c=0 -} -``` - -## API - -As seen in the examples above, purell offers three methods, `NormalizeURLString(string, NormalizationFlags) (string, error)`, `MustNormalizeURLString(string, NormalizationFlags) (string)` and `NormalizeURL(*url.URL, NormalizationFlags) (string)`. They all normalize the provided URL based on the specified flags. Here are the available flags: - -```go -const ( - // Safe normalizations - FlagLowercaseScheme NormalizationFlags = 1 << iota // HTTP://host -> http://host, applied by default in Go1.1 - FlagLowercaseHost // http://HOST -> http://host - FlagUppercaseEscapes // http://host/t%ef -> http://host/t%EF - FlagDecodeUnnecessaryEscapes // http://host/t%41 -> http://host/tA - FlagEncodeNecessaryEscapes // http://host/!"#$ -> http://host/%21%22#$ - FlagRemoveDefaultPort // http://host:80 -> http://host - FlagRemoveEmptyQuerySeparator // http://host/path? -> http://host/path - - // Usually safe normalizations - FlagRemoveTrailingSlash // http://host/path/ -> http://host/path - FlagAddTrailingSlash // http://host/path -> http://host/path/ (should choose only one of these add/remove trailing slash flags) - FlagRemoveDotSegments // http://host/path/./a/b/../c -> http://host/path/a/c - - // Unsafe normalizations - FlagRemoveDirectoryIndex // http://host/path/index.html -> http://host/path/ - FlagRemoveFragment // http://host/path#fragment -> http://host/path - FlagForceHTTP // https://host -> http://host - FlagRemoveDuplicateSlashes // http://host/path//a///b -> http://host/path/a/b - FlagRemoveWWW // http://www.host/ -> http://host/ - FlagAddWWW // http://host/ -> http://www.host/ (should choose only one of these add/remove WWW flags) - FlagSortQuery // http://host/path?c=3&b=2&a=1&b=1 -> http://host/path?a=1&b=1&b=2&c=3 - - // Normalizations not in the wikipedia article, required to cover tests cases - // submitted by jehiah - FlagDecodeDWORDHost // http://1113982867 -> http://66.102.7.147 - FlagDecodeOctalHost // http://0102.0146.07.0223 -> http://66.102.7.147 - FlagDecodeHexHost // http://0x42660793 -> http://66.102.7.147 - FlagRemoveUnnecessaryHostDots // http://.host../path -> http://host/path - FlagRemoveEmptyPortSeparator // http://host:/path -> http://host/path - - // Convenience set of safe normalizations - FlagsSafe NormalizationFlags = FlagLowercaseHost | FlagLowercaseScheme | FlagUppercaseEscapes | FlagDecodeUnnecessaryEscapes | FlagEncodeNecessaryEscapes | FlagRemoveDefaultPort | FlagRemoveEmptyQuerySeparator - - // For convenience sets, "greedy" uses the "remove trailing slash" and "remove www. prefix" flags, - // while "non-greedy" uses the "add (or keep) the trailing slash" and "add www. prefix". - - // Convenience set of usually safe normalizations (includes FlagsSafe) - FlagsUsuallySafeGreedy NormalizationFlags = FlagsSafe | FlagRemoveTrailingSlash | FlagRemoveDotSegments - FlagsUsuallySafeNonGreedy NormalizationFlags = FlagsSafe | FlagAddTrailingSlash | FlagRemoveDotSegments - - // Convenience set of unsafe normalizations (includes FlagsUsuallySafe) - FlagsUnsafeGreedy NormalizationFlags = FlagsUsuallySafeGreedy | FlagRemoveDirectoryIndex | FlagRemoveFragment | FlagForceHTTP | FlagRemoveDuplicateSlashes | FlagRemoveWWW | FlagSortQuery - FlagsUnsafeNonGreedy NormalizationFlags = FlagsUsuallySafeNonGreedy | FlagRemoveDirectoryIndex | FlagRemoveFragment | FlagForceHTTP | FlagRemoveDuplicateSlashes | FlagAddWWW | FlagSortQuery - - // Convenience set of all available flags - FlagsAllGreedy = FlagsUnsafeGreedy | FlagDecodeDWORDHost | FlagDecodeOctalHost | FlagDecodeHexHost | FlagRemoveUnnecessaryHostDots | FlagRemoveEmptyPortSeparator - FlagsAllNonGreedy = FlagsUnsafeNonGreedy | FlagDecodeDWORDHost | FlagDecodeOctalHost | FlagDecodeHexHost | FlagRemoveUnnecessaryHostDots | FlagRemoveEmptyPortSeparator -) -``` - -For convenience, the set of flags `FlagsSafe`, `FlagsUsuallySafe[Greedy|NonGreedy]`, `FlagsUnsafe[Greedy|NonGreedy]` and `FlagsAll[Greedy|NonGreedy]` are provided for the similarly grouped normalizations on [wikipedia's URL normalization page][wiki]. You can add (using the bitwise OR `|` operator) or remove (using the bitwise AND NOT `&^` operator) individual flags from the sets if required, to build your own custom set. - -The [full godoc reference is available on gopkgdoc][godoc]. - -Some things to note: - -* `FlagDecodeUnnecessaryEscapes`, `FlagEncodeNecessaryEscapes`, `FlagUppercaseEscapes` and `FlagRemoveEmptyQuerySeparator` are always implicitly set, because internally, the URL string is parsed as an URL object, which automatically decodes unnecessary escapes, uppercases and encodes necessary ones, and removes empty query separators (an unnecessary `?` at the end of the url). So this operation cannot **not** be done. For this reason, `FlagRemoveEmptyQuerySeparator` (as well as the other three) has been included in the `FlagsSafe` convenience set, instead of `FlagsUnsafe`, where Wikipedia puts it. - -* The `FlagDecodeUnnecessaryEscapes` decodes the following escapes (*from -> to*): - - %24 -> $ - - %26 -> & - - %2B-%3B -> +,-./0123456789:; - - %3D -> = - - %40-%5A -> @ABCDEFGHIJKLMNOPQRSTUVWXYZ - - %5F -> _ - - %61-%7A -> abcdefghijklmnopqrstuvwxyz - - %7E -> ~ - - -* When the `NormalizeURL` function is used (passing an URL object), this source URL object is modified (that is, after the call, the URL object will be modified to reflect the normalization). - -* The *replace IP with domain name* normalization (`http://208.77.188.166/ → http://www.example.com/`) is obviously not possible for a library without making some network requests. This is not implemented in purell. - -* The *remove unused query string parameters* and *remove default query parameters* are also not implemented, since this is a very case-specific normalization, and it is quite trivial to do with an URL object. - -### Safe vs Usually Safe vs Unsafe - -Purell allows you to control the level of risk you take while normalizing an URL. You can aggressively normalize, play it totally safe, or anything in between. - -Consider the following URL: - -`HTTPS://www.RooT.com/toto/t%45%1f///a/./b/../c/?z=3&w=2&a=4&w=1#invalid` - -Normalizing with the `FlagsSafe` gives: - -`https://www.root.com/toto/tE%1F///a/./b/../c/?z=3&w=2&a=4&w=1#invalid` - -With the `FlagsUsuallySafeGreedy`: - -`https://www.root.com/toto/tE%1F///a/c?z=3&w=2&a=4&w=1#invalid` - -And with `FlagsUnsafeGreedy`: - -`http://root.com/toto/tE%1F/a/c?a=4&w=1&w=2&z=3` - -## TODOs - -* Add a class/default instance to allow specifying custom directory index names? At the moment, removing directory index removes `(^|/)((?:default|index)\.\w{1,4})$`. - -## Thanks / Contributions - -@rogpeppe -@jehiah -@opennota -@pchristopher1275 -@zenovich -@beeker1121 - -## License - -The [BSD 3-Clause license][bsd]. - -[bsd]: http://opensource.org/licenses/BSD-3-Clause -[wiki]: http://en.wikipedia.org/wiki/URL_normalization -[rfc]: http://tools.ietf.org/html/rfc3986#section-6 -[godoc]: http://go.pkgdoc.org/github.com/PuerkitoBio/purell -[pr5]: https://github.com/PuerkitoBio/purell/pull/5 -[iss7]: https://github.com/PuerkitoBio/purell/issues/7 diff --git a/vendor/github.com/PuerkitoBio/purell/purell.go b/vendor/github.com/PuerkitoBio/purell/purell.go deleted file mode 100644 index 6d0fc190a1..0000000000 --- a/vendor/github.com/PuerkitoBio/purell/purell.go +++ /dev/null @@ -1,379 +0,0 @@ -/* -Package purell offers URL normalization as described on the wikipedia page: -http://en.wikipedia.org/wiki/URL_normalization -*/ -package purell - -import ( - "bytes" - "fmt" - "net/url" - "regexp" - "sort" - "strconv" - "strings" - - "github.com/PuerkitoBio/urlesc" - "golang.org/x/net/idna" - "golang.org/x/text/unicode/norm" - "golang.org/x/text/width" -) - -// A set of normalization flags determines how a URL will -// be normalized. -type NormalizationFlags uint - -const ( - // Safe normalizations - FlagLowercaseScheme NormalizationFlags = 1 << iota // HTTP://host -> http://host, applied by default in Go1.1 - FlagLowercaseHost // http://HOST -> http://host - FlagUppercaseEscapes // http://host/t%ef -> http://host/t%EF - FlagDecodeUnnecessaryEscapes // http://host/t%41 -> http://host/tA - FlagEncodeNecessaryEscapes // http://host/!"#$ -> http://host/%21%22#$ - FlagRemoveDefaultPort // http://host:80 -> http://host - FlagRemoveEmptyQuerySeparator // http://host/path? -> http://host/path - - // Usually safe normalizations - FlagRemoveTrailingSlash // http://host/path/ -> http://host/path - FlagAddTrailingSlash // http://host/path -> http://host/path/ (should choose only one of these add/remove trailing slash flags) - FlagRemoveDotSegments // http://host/path/./a/b/../c -> http://host/path/a/c - - // Unsafe normalizations - FlagRemoveDirectoryIndex // http://host/path/index.html -> http://host/path/ - FlagRemoveFragment // http://host/path#fragment -> http://host/path - FlagForceHTTP // https://host -> http://host - FlagRemoveDuplicateSlashes // http://host/path//a///b -> http://host/path/a/b - FlagRemoveWWW // http://www.host/ -> http://host/ - FlagAddWWW // http://host/ -> http://www.host/ (should choose only one of these add/remove WWW flags) - FlagSortQuery // http://host/path?c=3&b=2&a=1&b=1 -> http://host/path?a=1&b=1&b=2&c=3 - - // Normalizations not in the wikipedia article, required to cover tests cases - // submitted by jehiah - FlagDecodeDWORDHost // http://1113982867 -> http://66.102.7.147 - FlagDecodeOctalHost // http://0102.0146.07.0223 -> http://66.102.7.147 - FlagDecodeHexHost // http://0x42660793 -> http://66.102.7.147 - FlagRemoveUnnecessaryHostDots // http://.host../path -> http://host/path - FlagRemoveEmptyPortSeparator // http://host:/path -> http://host/path - - // Convenience set of safe normalizations - FlagsSafe NormalizationFlags = FlagLowercaseHost | FlagLowercaseScheme | FlagUppercaseEscapes | FlagDecodeUnnecessaryEscapes | FlagEncodeNecessaryEscapes | FlagRemoveDefaultPort | FlagRemoveEmptyQuerySeparator - - // For convenience sets, "greedy" uses the "remove trailing slash" and "remove www. prefix" flags, - // while "non-greedy" uses the "add (or keep) the trailing slash" and "add www. prefix". - - // Convenience set of usually safe normalizations (includes FlagsSafe) - FlagsUsuallySafeGreedy NormalizationFlags = FlagsSafe | FlagRemoveTrailingSlash | FlagRemoveDotSegments - FlagsUsuallySafeNonGreedy NormalizationFlags = FlagsSafe | FlagAddTrailingSlash | FlagRemoveDotSegments - - // Convenience set of unsafe normalizations (includes FlagsUsuallySafe) - FlagsUnsafeGreedy NormalizationFlags = FlagsUsuallySafeGreedy | FlagRemoveDirectoryIndex | FlagRemoveFragment | FlagForceHTTP | FlagRemoveDuplicateSlashes | FlagRemoveWWW | FlagSortQuery - FlagsUnsafeNonGreedy NormalizationFlags = FlagsUsuallySafeNonGreedy | FlagRemoveDirectoryIndex | FlagRemoveFragment | FlagForceHTTP | FlagRemoveDuplicateSlashes | FlagAddWWW | FlagSortQuery - - // Convenience set of all available flags - FlagsAllGreedy = FlagsUnsafeGreedy | FlagDecodeDWORDHost | FlagDecodeOctalHost | FlagDecodeHexHost | FlagRemoveUnnecessaryHostDots | FlagRemoveEmptyPortSeparator - FlagsAllNonGreedy = FlagsUnsafeNonGreedy | FlagDecodeDWORDHost | FlagDecodeOctalHost | FlagDecodeHexHost | FlagRemoveUnnecessaryHostDots | FlagRemoveEmptyPortSeparator -) - -const ( - defaultHttpPort = ":80" - defaultHttpsPort = ":443" -) - -// Regular expressions used by the normalizations -var rxPort = regexp.MustCompile(`(:\d+)/?$`) -var rxDirIndex = regexp.MustCompile(`(^|/)((?:default|index)\.\w{1,4})$`) -var rxDupSlashes = regexp.MustCompile(`/{2,}`) -var rxDWORDHost = regexp.MustCompile(`^(\d+)((?:\.+)?(?:\:\d*)?)$`) -var rxOctalHost = regexp.MustCompile(`^(0\d*)\.(0\d*)\.(0\d*)\.(0\d*)((?:\.+)?(?:\:\d*)?)$`) -var rxHexHost = regexp.MustCompile(`^0x([0-9A-Fa-f]+)((?:\.+)?(?:\:\d*)?)$`) -var rxHostDots = regexp.MustCompile(`^(.+?)(:\d+)?$`) -var rxEmptyPort = regexp.MustCompile(`:+$`) - -// Map of flags to implementation function. -// FlagDecodeUnnecessaryEscapes has no action, since it is done automatically -// by parsing the string as an URL. Same for FlagUppercaseEscapes and FlagRemoveEmptyQuerySeparator. - -// Since maps have undefined traversing order, make a slice of ordered keys -var flagsOrder = []NormalizationFlags{ - FlagLowercaseScheme, - FlagLowercaseHost, - FlagRemoveDefaultPort, - FlagRemoveDirectoryIndex, - FlagRemoveDotSegments, - FlagRemoveFragment, - FlagForceHTTP, // Must be after remove default port (because https=443/http=80) - FlagRemoveDuplicateSlashes, - FlagRemoveWWW, - FlagAddWWW, - FlagSortQuery, - FlagDecodeDWORDHost, - FlagDecodeOctalHost, - FlagDecodeHexHost, - FlagRemoveUnnecessaryHostDots, - FlagRemoveEmptyPortSeparator, - FlagRemoveTrailingSlash, // These two (add/remove trailing slash) must be last - FlagAddTrailingSlash, -} - -// ... and then the map, where order is unimportant -var flags = map[NormalizationFlags]func(*url.URL){ - FlagLowercaseScheme: lowercaseScheme, - FlagLowercaseHost: lowercaseHost, - FlagRemoveDefaultPort: removeDefaultPort, - FlagRemoveDirectoryIndex: removeDirectoryIndex, - FlagRemoveDotSegments: removeDotSegments, - FlagRemoveFragment: removeFragment, - FlagForceHTTP: forceHTTP, - FlagRemoveDuplicateSlashes: removeDuplicateSlashes, - FlagRemoveWWW: removeWWW, - FlagAddWWW: addWWW, - FlagSortQuery: sortQuery, - FlagDecodeDWORDHost: decodeDWORDHost, - FlagDecodeOctalHost: decodeOctalHost, - FlagDecodeHexHost: decodeHexHost, - FlagRemoveUnnecessaryHostDots: removeUnncessaryHostDots, - FlagRemoveEmptyPortSeparator: removeEmptyPortSeparator, - FlagRemoveTrailingSlash: removeTrailingSlash, - FlagAddTrailingSlash: addTrailingSlash, -} - -// MustNormalizeURLString returns the normalized string, and panics if an error occurs. -// It takes an URL string as input, as well as the normalization flags. -func MustNormalizeURLString(u string, f NormalizationFlags) string { - result, e := NormalizeURLString(u, f) - if e != nil { - panic(e) - } - return result -} - -// NormalizeURLString returns the normalized string, or an error if it can't be parsed into an URL object. -// It takes an URL string as input, as well as the normalization flags. -func NormalizeURLString(u string, f NormalizationFlags) (string, error) { - parsed, err := url.Parse(u) - if err != nil { - return "", err - } - - if f&FlagLowercaseHost == FlagLowercaseHost { - parsed.Host = strings.ToLower(parsed.Host) - } - - // The idna package doesn't fully conform to RFC 5895 - // (https://tools.ietf.org/html/rfc5895), so we do it here. - // Taken from Go 1.8 cycle source, courtesy of bradfitz. - // TODO: Remove when (if?) idna package conforms to RFC 5895. - parsed.Host = width.Fold.String(parsed.Host) - parsed.Host = norm.NFC.String(parsed.Host) - if parsed.Host, err = idna.ToASCII(parsed.Host); err != nil { - return "", err - } - - return NormalizeURL(parsed, f), nil -} - -// NormalizeURL returns the normalized string. -// It takes a parsed URL object as input, as well as the normalization flags. -func NormalizeURL(u *url.URL, f NormalizationFlags) string { - for _, k := range flagsOrder { - if f&k == k { - flags[k](u) - } - } - return urlesc.Escape(u) -} - -func lowercaseScheme(u *url.URL) { - if len(u.Scheme) > 0 { - u.Scheme = strings.ToLower(u.Scheme) - } -} - -func lowercaseHost(u *url.URL) { - if len(u.Host) > 0 { - u.Host = strings.ToLower(u.Host) - } -} - -func removeDefaultPort(u *url.URL) { - if len(u.Host) > 0 { - scheme := strings.ToLower(u.Scheme) - u.Host = rxPort.ReplaceAllStringFunc(u.Host, func(val string) string { - if (scheme == "http" && val == defaultHttpPort) || (scheme == "https" && val == defaultHttpsPort) { - return "" - } - return val - }) - } -} - -func removeTrailingSlash(u *url.URL) { - if l := len(u.Path); l > 0 { - if strings.HasSuffix(u.Path, "/") { - u.Path = u.Path[:l-1] - } - } else if l = len(u.Host); l > 0 { - if strings.HasSuffix(u.Host, "/") { - u.Host = u.Host[:l-1] - } - } -} - -func addTrailingSlash(u *url.URL) { - if l := len(u.Path); l > 0 { - if !strings.HasSuffix(u.Path, "/") { - u.Path += "/" - } - } else if l = len(u.Host); l > 0 { - if !strings.HasSuffix(u.Host, "/") { - u.Host += "/" - } - } -} - -func removeDotSegments(u *url.URL) { - if len(u.Path) > 0 { - var dotFree []string - var lastIsDot bool - - sections := strings.Split(u.Path, "/") - for _, s := range sections { - if s == ".." { - if len(dotFree) > 0 { - dotFree = dotFree[:len(dotFree)-1] - } - } else if s != "." { - dotFree = append(dotFree, s) - } - lastIsDot = (s == "." || s == "..") - } - // Special case if host does not end with / and new path does not begin with / - u.Path = strings.Join(dotFree, "/") - if u.Host != "" && !strings.HasSuffix(u.Host, "/") && !strings.HasPrefix(u.Path, "/") { - u.Path = "/" + u.Path - } - // Special case if the last segment was a dot, make sure the path ends with a slash - if lastIsDot && !strings.HasSuffix(u.Path, "/") { - u.Path += "/" - } - } -} - -func removeDirectoryIndex(u *url.URL) { - if len(u.Path) > 0 { - u.Path = rxDirIndex.ReplaceAllString(u.Path, "$1") - } -} - -func removeFragment(u *url.URL) { - u.Fragment = "" -} - -func forceHTTP(u *url.URL) { - if strings.ToLower(u.Scheme) == "https" { - u.Scheme = "http" - } -} - -func removeDuplicateSlashes(u *url.URL) { - if len(u.Path) > 0 { - u.Path = rxDupSlashes.ReplaceAllString(u.Path, "/") - } -} - -func removeWWW(u *url.URL) { - if len(u.Host) > 0 && strings.HasPrefix(strings.ToLower(u.Host), "www.") { - u.Host = u.Host[4:] - } -} - -func addWWW(u *url.URL) { - if len(u.Host) > 0 && !strings.HasPrefix(strings.ToLower(u.Host), "www.") { - u.Host = "www." + u.Host - } -} - -func sortQuery(u *url.URL) { - q := u.Query() - - if len(q) > 0 { - arKeys := make([]string, len(q)) - i := 0 - for k := range q { - arKeys[i] = k - i++ - } - sort.Strings(arKeys) - buf := new(bytes.Buffer) - for _, k := range arKeys { - sort.Strings(q[k]) - for _, v := range q[k] { - if buf.Len() > 0 { - buf.WriteRune('&') - } - buf.WriteString(fmt.Sprintf("%s=%s", k, urlesc.QueryEscape(v))) - } - } - - // Rebuild the raw query string - u.RawQuery = buf.String() - } -} - -func decodeDWORDHost(u *url.URL) { - if len(u.Host) > 0 { - if matches := rxDWORDHost.FindStringSubmatch(u.Host); len(matches) > 2 { - var parts [4]int64 - - dword, _ := strconv.ParseInt(matches[1], 10, 0) - for i, shift := range []uint{24, 16, 8, 0} { - parts[i] = dword >> shift & 0xFF - } - u.Host = fmt.Sprintf("%d.%d.%d.%d%s", parts[0], parts[1], parts[2], parts[3], matches[2]) - } - } -} - -func decodeOctalHost(u *url.URL) { - if len(u.Host) > 0 { - if matches := rxOctalHost.FindStringSubmatch(u.Host); len(matches) > 5 { - var parts [4]int64 - - for i := 1; i <= 4; i++ { - parts[i-1], _ = strconv.ParseInt(matches[i], 8, 0) - } - u.Host = fmt.Sprintf("%d.%d.%d.%d%s", parts[0], parts[1], parts[2], parts[3], matches[5]) - } - } -} - -func decodeHexHost(u *url.URL) { - if len(u.Host) > 0 { - if matches := rxHexHost.FindStringSubmatch(u.Host); len(matches) > 2 { - // Conversion is safe because of regex validation - parsed, _ := strconv.ParseInt(matches[1], 16, 0) - // Set host as DWORD (base 10) encoded host - u.Host = fmt.Sprintf("%d%s", parsed, matches[2]) - // The rest is the same as decoding a DWORD host - decodeDWORDHost(u) - } - } -} - -func removeUnncessaryHostDots(u *url.URL) { - if len(u.Host) > 0 { - if matches := rxHostDots.FindStringSubmatch(u.Host); len(matches) > 1 { - // Trim the leading and trailing dots - u.Host = strings.Trim(matches[1], ".") - if len(matches) > 2 { - u.Host += matches[2] - } - } - } -} - -func removeEmptyPortSeparator(u *url.URL) { - if len(u.Host) > 0 { - u.Host = rxEmptyPort.ReplaceAllString(u.Host, "") - } -} diff --git a/vendor/github.com/PuerkitoBio/urlesc/.travis.yml b/vendor/github.com/PuerkitoBio/urlesc/.travis.yml deleted file mode 100644 index ba6b225f91..0000000000 --- a/vendor/github.com/PuerkitoBio/urlesc/.travis.yml +++ /dev/null @@ -1,15 +0,0 @@ -language: go - -go: - - 1.4.x - - 1.5.x - - 1.6.x - - 1.7.x - - 1.8.x - - tip - -install: - - go build . - -script: - - go test -v diff --git a/vendor/github.com/PuerkitoBio/urlesc/LICENSE b/vendor/github.com/PuerkitoBio/urlesc/LICENSE deleted file mode 100644 index 7448756763..0000000000 --- a/vendor/github.com/PuerkitoBio/urlesc/LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -Copyright (c) 2012 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/PuerkitoBio/urlesc/README.md b/vendor/github.com/PuerkitoBio/urlesc/README.md deleted file mode 100644 index 57aff0a539..0000000000 --- a/vendor/github.com/PuerkitoBio/urlesc/README.md +++ /dev/null @@ -1,16 +0,0 @@ -urlesc [![Build Status](https://travis-ci.org/PuerkitoBio/urlesc.svg?branch=master)](https://travis-ci.org/PuerkitoBio/urlesc) [![GoDoc](http://godoc.org/github.com/PuerkitoBio/urlesc?status.svg)](http://godoc.org/github.com/PuerkitoBio/urlesc) -====== - -Package urlesc implements query escaping as per RFC 3986. - -It contains some parts of the net/url package, modified so as to allow -some reserved characters incorrectly escaped by net/url (see [issue 5684](https://github.com/golang/go/issues/5684)). - -## Install - - go get github.com/PuerkitoBio/urlesc - -## License - -Go license (BSD-3-Clause) - diff --git a/vendor/github.com/PuerkitoBio/urlesc/urlesc.go b/vendor/github.com/PuerkitoBio/urlesc/urlesc.go deleted file mode 100644 index 1b84624594..0000000000 --- a/vendor/github.com/PuerkitoBio/urlesc/urlesc.go +++ /dev/null @@ -1,180 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package urlesc implements query escaping as per RFC 3986. -// It contains some parts of the net/url package, modified so as to allow -// some reserved characters incorrectly escaped by net/url. -// See https://github.com/golang/go/issues/5684 -package urlesc - -import ( - "bytes" - "net/url" - "strings" -) - -type encoding int - -const ( - encodePath encoding = 1 + iota - encodeUserPassword - encodeQueryComponent - encodeFragment -) - -// Return true if the specified character should be escaped when -// appearing in a URL string, according to RFC 3986. -func shouldEscape(c byte, mode encoding) bool { - // §2.3 Unreserved characters (alphanum) - if 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' { - return false - } - - switch c { - case '-', '.', '_', '~': // §2.3 Unreserved characters (mark) - return false - - // §2.2 Reserved characters (reserved) - case ':', '/', '?', '#', '[', ']', '@', // gen-delims - '!', '$', '&', '\'', '(', ')', '*', '+', ',', ';', '=': // sub-delims - // Different sections of the URL allow a few of - // the reserved characters to appear unescaped. - switch mode { - case encodePath: // §3.3 - // The RFC allows sub-delims and : @. - // '/', '[' and ']' can be used to assign meaning to individual path - // segments. This package only manipulates the path as a whole, - // so we allow those as well. That leaves only ? and # to escape. - return c == '?' || c == '#' - - case encodeUserPassword: // §3.2.1 - // The RFC allows : and sub-delims in - // userinfo. The parsing of userinfo treats ':' as special so we must escape - // all the gen-delims. - return c == ':' || c == '/' || c == '?' || c == '#' || c == '[' || c == ']' || c == '@' - - case encodeQueryComponent: // §3.4 - // The RFC allows / and ?. - return c != '/' && c != '?' - - case encodeFragment: // §4.1 - // The RFC text is silent but the grammar allows - // everything, so escape nothing but # - return c == '#' - } - } - - // Everything else must be escaped. - return true -} - -// QueryEscape escapes the string so it can be safely placed -// inside a URL query. -func QueryEscape(s string) string { - return escape(s, encodeQueryComponent) -} - -func escape(s string, mode encoding) string { - spaceCount, hexCount := 0, 0 - for i := 0; i < len(s); i++ { - c := s[i] - if shouldEscape(c, mode) { - if c == ' ' && mode == encodeQueryComponent { - spaceCount++ - } else { - hexCount++ - } - } - } - - if spaceCount == 0 && hexCount == 0 { - return s - } - - t := make([]byte, len(s)+2*hexCount) - j := 0 - for i := 0; i < len(s); i++ { - switch c := s[i]; { - case c == ' ' && mode == encodeQueryComponent: - t[j] = '+' - j++ - case shouldEscape(c, mode): - t[j] = '%' - t[j+1] = "0123456789ABCDEF"[c>>4] - t[j+2] = "0123456789ABCDEF"[c&15] - j += 3 - default: - t[j] = s[i] - j++ - } - } - return string(t) -} - -var uiReplacer = strings.NewReplacer( - "%21", "!", - "%27", "'", - "%28", "(", - "%29", ")", - "%2A", "*", -) - -// unescapeUserinfo unescapes some characters that need not to be escaped as per RFC3986. -func unescapeUserinfo(s string) string { - return uiReplacer.Replace(s) -} - -// Escape reassembles the URL into a valid URL string. -// The general form of the result is one of: -// -// scheme:opaque -// scheme://userinfo@host/path?query#fragment -// -// If u.Opaque is non-empty, String uses the first form; -// otherwise it uses the second form. -// -// In the second form, the following rules apply: -// - if u.Scheme is empty, scheme: is omitted. -// - if u.User is nil, userinfo@ is omitted. -// - if u.Host is empty, host/ is omitted. -// - if u.Scheme and u.Host are empty and u.User is nil, -// the entire scheme://userinfo@host/ is omitted. -// - if u.Host is non-empty and u.Path begins with a /, -// the form host/path does not add its own /. -// - if u.RawQuery is empty, ?query is omitted. -// - if u.Fragment is empty, #fragment is omitted. -func Escape(u *url.URL) string { - var buf bytes.Buffer - if u.Scheme != "" { - buf.WriteString(u.Scheme) - buf.WriteByte(':') - } - if u.Opaque != "" { - buf.WriteString(u.Opaque) - } else { - if u.Scheme != "" || u.Host != "" || u.User != nil { - buf.WriteString("//") - if ui := u.User; ui != nil { - buf.WriteString(unescapeUserinfo(ui.String())) - buf.WriteByte('@') - } - if h := u.Host; h != "" { - buf.WriteString(h) - } - } - if u.Path != "" && u.Path[0] != '/' && u.Host != "" { - buf.WriteByte('/') - } - buf.WriteString(escape(u.Path, encodePath)) - } - if u.RawQuery != "" { - buf.WriteByte('?') - buf.WriteString(u.RawQuery) - } - if u.Fragment != "" { - buf.WriteByte('#') - buf.WriteString(escape(u.Fragment, encodeFragment)) - } - return buf.String() -} diff --git a/vendor/github.com/go-openapi/jsonreference/internal/normalize_url.go b/vendor/github.com/go-openapi/jsonreference/internal/normalize_url.go new file mode 100644 index 0000000000..8956c30884 --- /dev/null +++ b/vendor/github.com/go-openapi/jsonreference/internal/normalize_url.go @@ -0,0 +1,63 @@ +package internal + +import ( + "net/url" + "regexp" + "strings" +) + +const ( + defaultHttpPort = ":80" + defaultHttpsPort = ":443" +) + +// Regular expressions used by the normalizations +var rxPort = regexp.MustCompile(`(:\d+)/?$`) +var rxDupSlashes = regexp.MustCompile(`/{2,}`) + +// NormalizeURL will normalize the specified URL +// This was added to replace a previous call to the no longer maintained purell library: +// The call that was used looked like the following: +// url.Parse(purell.NormalizeURL(parsed, purell.FlagsSafe|purell.FlagRemoveDuplicateSlashes)) +// +// To explain all that was included in the call above, purell.FlagsSafe was really just the following: +// - FlagLowercaseScheme +// - FlagLowercaseHost +// - FlagRemoveDefaultPort +// - FlagRemoveDuplicateSlashes (and this was mixed in with the |) +func NormalizeURL(u *url.URL) { + lowercaseScheme(u) + lowercaseHost(u) + removeDefaultPort(u) + removeDuplicateSlashes(u) +} + +func lowercaseScheme(u *url.URL) { + if len(u.Scheme) > 0 { + u.Scheme = strings.ToLower(u.Scheme) + } +} + +func lowercaseHost(u *url.URL) { + if len(u.Host) > 0 { + u.Host = strings.ToLower(u.Host) + } +} + +func removeDefaultPort(u *url.URL) { + if len(u.Host) > 0 { + scheme := strings.ToLower(u.Scheme) + u.Host = rxPort.ReplaceAllStringFunc(u.Host, func(val string) string { + if (scheme == "http" && val == defaultHttpPort) || (scheme == "https" && val == defaultHttpsPort) { + return "" + } + return val + }) + } +} + +func removeDuplicateSlashes(u *url.URL) { + if len(u.Path) > 0 { + u.Path = rxDupSlashes.ReplaceAllString(u.Path, "/") + } +} diff --git a/vendor/github.com/go-openapi/jsonreference/reference.go b/vendor/github.com/go-openapi/jsonreference/reference.go index 3bc0a6e26f..cfdef03e5d 100644 --- a/vendor/github.com/go-openapi/jsonreference/reference.go +++ b/vendor/github.com/go-openapi/jsonreference/reference.go @@ -30,8 +30,8 @@ import ( "net/url" "strings" - "github.com/PuerkitoBio/purell" "github.com/go-openapi/jsonpointer" + "github.com/go-openapi/jsonreference/internal" ) const ( @@ -114,7 +114,9 @@ func (r *Ref) parse(jsonReferenceString string) error { return err } - r.referenceURL, _ = url.Parse(purell.NormalizeURL(parsed, purell.FlagsSafe|purell.FlagRemoveDuplicateSlashes)) + internal.NormalizeURL(parsed) + + r.referenceURL = parsed refURL := r.referenceURL if refURL.Scheme != "" && refURL.Host != "" { diff --git a/vendor/github.com/go-openapi/swag/.gitattributes b/vendor/github.com/go-openapi/swag/.gitattributes new file mode 100644 index 0000000000..49ad52766a --- /dev/null +++ b/vendor/github.com/go-openapi/swag/.gitattributes @@ -0,0 +1,2 @@ +# gofmt always uses LF, whereas Git uses CRLF on Windows. +*.go text eol=lf diff --git a/vendor/github.com/go-openapi/swag/.golangci.yml b/vendor/github.com/go-openapi/swag/.golangci.yml index 813c47aa64..bf503e4000 100644 --- a/vendor/github.com/go-openapi/swag/.golangci.yml +++ b/vendor/github.com/go-openapi/swag/.golangci.yml @@ -37,3 +37,18 @@ linters: - gci - gocognit - paralleltest + - thelper + - ifshort + - gomoddirectives + - cyclop + - forcetypeassert + - ireturn + - tagliatelle + - varnamelen + - goimports + - tenv + - golint + - exhaustruct + - nilnil + - nonamedreturns + - nosnakecase diff --git a/vendor/github.com/go-openapi/swag/.travis.yml b/vendor/github.com/go-openapi/swag/.travis.yml deleted file mode 100644 index fc25a88728..0000000000 --- a/vendor/github.com/go-openapi/swag/.travis.yml +++ /dev/null @@ -1,37 +0,0 @@ -after_success: -- bash <(curl -s https://codecov.io/bash) -go: -- 1.14.x -- 1.x -arch: -- amd64 -jobs: - include: - # include arch ppc, but only for latest go version - skip testing for race - - go: 1.x - arch: ppc64le - install: ~ - script: - - go test -v - - #- go: 1.x - # arch: arm - # install: ~ - # script: - # - go test -v - - # include linting job, but only for latest go version and amd64 arch - - go: 1.x - arch: amd64 - install: - go get github.com/golangci/golangci-lint/cmd/golangci-lint - script: - - golangci-lint run --new-from-rev master -install: -- GO111MODULE=off go get -u gotest.tools/gotestsum -language: go -notifications: - slack: - secure: QUWvCkBBK09GF7YtEvHHVt70JOkdlNBG0nIKu/5qc4/nW5HP8I2w0SEf/XR2je0eED1Qe3L/AfMCWwrEj+IUZc3l4v+ju8X8R3Lomhme0Eb0jd1MTMCuPcBT47YCj0M7RON7vXtbFfm1hFJ/jLe5+9FXz0hpXsR24PJc5ZIi/ogNwkaPqG4BmndzecpSh0vc2FJPZUD9LT0I09REY/vXR0oQAalLkW0asGD5taHZTUZq/kBpsNxaAFrLM23i4mUcf33M5fjLpvx5LRICrX/57XpBrDh2TooBU6Qj3CgoY0uPRYUmSNxbVx1czNzl2JtEpb5yjoxfVPQeg0BvQM00G8LJINISR+ohrjhkZmAqchDupAX+yFrxTtORa78CtnIL6z/aTNlgwwVD8kvL/1pFA/JWYmKDmz93mV/+6wubGzNSQCstzjkFA4/iZEKewKUoRIAi/fxyscP6L/rCpmY/4llZZvrnyTqVbt6URWpopUpH4rwYqreXAtJxJsfBJIeSmUIiDIOMGkCTvyTEW3fWGmGoqWtSHLoaWDyAIGb7azb+KvfpWtEcoPFWfSWU+LGee0A/YsUhBl7ADB9A0CJEuR8q4BPpKpfLwPKSiKSAXL7zDkyjExyhtgqbSl2jS+rKIHOZNL8JkCcTP2MKMVd563C5rC5FMKqu3S9m2b6380E= -script: -- gotestsum -f short-verbose -- -race -coverprofile=coverage.txt -covermode=atomic ./... diff --git a/vendor/github.com/go-openapi/swag/doc.go b/vendor/github.com/go-openapi/swag/doc.go index 8d2c8c5014..55094cb74c 100644 --- a/vendor/github.com/go-openapi/swag/doc.go +++ b/vendor/github.com/go-openapi/swag/doc.go @@ -17,16 +17,15 @@ Package swag contains a bunch of helper functions for go-openapi and go-swagger You may also use it standalone for your projects. - * convert between value and pointers for builtin types - * convert from string to builtin types (wraps strconv) - * fast json concatenation - * search in path - * load from file or http - * name mangling - + - convert between value and pointers for builtin types + - convert from string to builtin types (wraps strconv) + - fast json concatenation + - search in path + - load from file or http + - name mangling This repo has only few dependencies outside of the standard library: - * YAML utilities depend on gopkg.in/yaml.v2 + - YAML utilities depend on gopkg.in/yaml.v2 */ package swag diff --git a/vendor/github.com/go-openapi/swag/file.go b/vendor/github.com/go-openapi/swag/file.go new file mode 100644 index 0000000000..16accc55f8 --- /dev/null +++ b/vendor/github.com/go-openapi/swag/file.go @@ -0,0 +1,33 @@ +// Copyright 2015 go-swagger maintainers +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package swag + +import "mime/multipart" + +// File represents an uploaded file. +type File struct { + Data multipart.File + Header *multipart.FileHeader +} + +// Read bytes from the file +func (f *File) Read(p []byte) (n int, err error) { + return f.Data.Read(p) +} + +// Close the file +func (f *File) Close() error { + return f.Data.Close() +} diff --git a/vendor/github.com/go-openapi/swag/loading.go b/vendor/github.com/go-openapi/swag/loading.go index 9a60409725..00038c3773 100644 --- a/vendor/github.com/go-openapi/swag/loading.go +++ b/vendor/github.com/go-openapi/swag/loading.go @@ -16,10 +16,11 @@ package swag import ( "fmt" - "io/ioutil" + "io" "log" "net/http" "net/url" + "os" "path/filepath" "runtime" "strings" @@ -40,13 +41,13 @@ var LoadHTTPCustomHeaders = map[string]string{} // LoadFromFileOrHTTP loads the bytes from a file or a remote http server based on the path passed in func LoadFromFileOrHTTP(path string) ([]byte, error) { - return LoadStrategy(path, ioutil.ReadFile, loadHTTPBytes(LoadHTTPTimeout))(path) + return LoadStrategy(path, os.ReadFile, loadHTTPBytes(LoadHTTPTimeout))(path) } // LoadFromFileOrHTTPWithTimeout loads the bytes from a file or a remote http server based on the path passed in // timeout arg allows for per request overriding of the request timeout func LoadFromFileOrHTTPWithTimeout(path string, timeout time.Duration) ([]byte, error) { - return LoadStrategy(path, ioutil.ReadFile, loadHTTPBytes(timeout))(path) + return LoadStrategy(path, os.ReadFile, loadHTTPBytes(timeout))(path) } // LoadStrategy returns a loader function for a given path or uri @@ -86,7 +87,7 @@ func LoadStrategy(path string, local, remote func(string) ([]byte, error)) func( func loadHTTPBytes(timeout time.Duration) func(path string) ([]byte, error) { return func(path string) ([]byte, error) { client := &http.Client{Timeout: timeout} - req, err := http.NewRequest("GET", path, nil) // nolint: noctx + req, err := http.NewRequest(http.MethodGet, path, nil) //nolint:noctx if err != nil { return nil, err } @@ -115,6 +116,6 @@ func loadHTTPBytes(timeout time.Duration) func(path string) ([]byte, error) { return nil, fmt.Errorf("could not access document at %q [%s] ", path, resp.Status) } - return ioutil.ReadAll(resp.Body) + return io.ReadAll(resp.Body) } } diff --git a/vendor/github.com/go-openapi/swag/post_go18.go b/vendor/github.com/go-openapi/swag/post_go18.go index c2e686d313..f5228b82c0 100644 --- a/vendor/github.com/go-openapi/swag/post_go18.go +++ b/vendor/github.com/go-openapi/swag/post_go18.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build go1.8 // +build go1.8 package swag diff --git a/vendor/github.com/go-openapi/swag/post_go19.go b/vendor/github.com/go-openapi/swag/post_go19.go index eb2f2d8bc7..7c7da9c088 100644 --- a/vendor/github.com/go-openapi/swag/post_go19.go +++ b/vendor/github.com/go-openapi/swag/post_go19.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build go1.9 // +build go1.9 package swag diff --git a/vendor/github.com/go-openapi/swag/pre_go18.go b/vendor/github.com/go-openapi/swag/pre_go18.go index 6607f33935..2757d9b95f 100644 --- a/vendor/github.com/go-openapi/swag/pre_go18.go +++ b/vendor/github.com/go-openapi/swag/pre_go18.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !go1.8 // +build !go1.8 package swag diff --git a/vendor/github.com/go-openapi/swag/pre_go19.go b/vendor/github.com/go-openapi/swag/pre_go19.go index 4bae187d1e..0565db377b 100644 --- a/vendor/github.com/go-openapi/swag/pre_go19.go +++ b/vendor/github.com/go-openapi/swag/pre_go19.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !go1.9 // +build !go1.9 package swag diff --git a/vendor/github.com/go-openapi/swag/util.go b/vendor/github.com/go-openapi/swag/util.go index 193702f2ce..f78ab684a0 100644 --- a/vendor/github.com/go-openapi/swag/util.go +++ b/vendor/github.com/go-openapi/swag/util.go @@ -99,10 +99,11 @@ const ( ) // JoinByFormat joins a string array by a known format (e.g. swagger's collectionFormat attribute): -// ssv: space separated value -// tsv: tab separated value -// pipes: pipe (|) separated value -// csv: comma separated value (default) +// +// ssv: space separated value +// tsv: tab separated value +// pipes: pipe (|) separated value +// csv: comma separated value (default) func JoinByFormat(data []string, format string) []string { if len(data) == 0 { return data @@ -124,11 +125,11 @@ func JoinByFormat(data []string, format string) []string { } // SplitByFormat splits a string by a known format: -// ssv: space separated value -// tsv: tab separated value -// pipes: pipe (|) separated value -// csv: comma separated value (default) // +// ssv: space separated value +// tsv: tab separated value +// pipes: pipe (|) separated value +// csv: comma separated value (default) func SplitByFormat(data, format string) []string { if data == "" { return nil diff --git a/vendor/github.com/go-openapi/swag/yaml.go b/vendor/github.com/go-openapi/swag/yaml.go index ec96914405..f09ee609f3 100644 --- a/vendor/github.com/go-openapi/swag/yaml.go +++ b/vendor/github.com/go-openapi/swag/yaml.go @@ -22,7 +22,7 @@ import ( "github.com/mailru/easyjson/jlexer" "github.com/mailru/easyjson/jwriter" - yaml "gopkg.in/yaml.v2" + yaml "gopkg.in/yaml.v3" ) // YAMLMatcher matches yaml @@ -43,16 +43,126 @@ func YAMLToJSON(data interface{}) (json.RawMessage, error) { // BytesToYAMLDoc converts a byte slice into a YAML document func BytesToYAMLDoc(data []byte) (interface{}, error) { - var canary map[interface{}]interface{} // validate this is an object and not a different type - if err := yaml.Unmarshal(data, &canary); err != nil { + var document yaml.Node // preserve order that is present in the document + if err := yaml.Unmarshal(data, &document); err != nil { return nil, err } + if document.Kind != yaml.DocumentNode || len(document.Content) != 1 || document.Content[0].Kind != yaml.MappingNode { + return nil, fmt.Errorf("only YAML documents that are objects are supported") + } + return &document, nil +} - var document yaml.MapSlice // preserve order that is present in the document - if err := yaml.Unmarshal(data, &document); err != nil { - return nil, err +func yamlNode(root *yaml.Node) (interface{}, error) { + switch root.Kind { + case yaml.DocumentNode: + return yamlDocument(root) + case yaml.SequenceNode: + return yamlSequence(root) + case yaml.MappingNode: + return yamlMapping(root) + case yaml.ScalarNode: + return yamlScalar(root) + case yaml.AliasNode: + return yamlNode(root.Alias) + default: + return nil, fmt.Errorf("unsupported YAML node type: %v", root.Kind) + } +} + +func yamlDocument(node *yaml.Node) (interface{}, error) { + if len(node.Content) != 1 { + return nil, fmt.Errorf("unexpected YAML Document node content length: %d", len(node.Content)) + } + return yamlNode(node.Content[0]) +} + +func yamlMapping(node *yaml.Node) (interface{}, error) { + m := make(JSONMapSlice, len(node.Content)/2) + + var j int + for i := 0; i < len(node.Content); i += 2 { + var nmi JSONMapItem + k, err := yamlStringScalarC(node.Content[i]) + if err != nil { + return nil, fmt.Errorf("unable to decode YAML map key: %w", err) + } + nmi.Key = k + v, err := yamlNode(node.Content[i+1]) + if err != nil { + return nil, fmt.Errorf("unable to process YAML map value for key %q: %w", k, err) + } + nmi.Value = v + m[j] = nmi + j++ + } + return m, nil +} + +func yamlSequence(node *yaml.Node) (interface{}, error) { + s := make([]interface{}, 0) + + for i := 0; i < len(node.Content); i++ { + + v, err := yamlNode(node.Content[i]) + if err != nil { + return nil, fmt.Errorf("unable to decode YAML sequence value: %w", err) + } + s = append(s, v) + } + return s, nil +} + +const ( // See https://yaml.org/type/ + yamlStringScalar = "tag:yaml.org,2002:str" + yamlIntScalar = "tag:yaml.org,2002:int" + yamlBoolScalar = "tag:yaml.org,2002:bool" + yamlFloatScalar = "tag:yaml.org,2002:float" + yamlTimestamp = "tag:yaml.org,2002:timestamp" + yamlNull = "tag:yaml.org,2002:null" +) + +func yamlScalar(node *yaml.Node) (interface{}, error) { + switch node.LongTag() { + case yamlStringScalar: + return node.Value, nil + case yamlBoolScalar: + b, err := strconv.ParseBool(node.Value) + if err != nil { + return nil, fmt.Errorf("unable to process scalar node. Got %q. Expecting bool content: %w", node.Value, err) + } + return b, nil + case yamlIntScalar: + i, err := strconv.ParseInt(node.Value, 10, 64) + if err != nil { + return nil, fmt.Errorf("unable to process scalar node. Got %q. Expecting integer content: %w", node.Value, err) + } + return i, nil + case yamlFloatScalar: + f, err := strconv.ParseFloat(node.Value, 64) + if err != nil { + return nil, fmt.Errorf("unable to process scalar node. Got %q. Expecting float content: %w", node.Value, err) + } + return f, nil + case yamlTimestamp: + return node.Value, nil + case yamlNull: + return nil, nil + default: + return nil, fmt.Errorf("YAML tag %q is not supported", node.LongTag()) + } +} + +func yamlStringScalarC(node *yaml.Node) (string, error) { + if node.Kind != yaml.ScalarNode { + return "", fmt.Errorf("expecting a string scalar but got %q", node.Kind) + } + switch node.LongTag() { + case yamlStringScalar, yamlIntScalar, yamlFloatScalar: + return node.Value, nil + default: + return "", fmt.Errorf("YAML tag %q is not supported as map key", node.LongTag()) } - return document, nil } // JSONMapSlice represent a JSON object, with the order of keys maintained @@ -105,6 +215,113 @@ func (s *JSONMapSlice) UnmarshalEasyJSON(in *jlexer.Lexer) { *s = result } +func (s JSONMapSlice) MarshalYAML() (interface{}, error) { + var n yaml.Node + n.Kind = yaml.DocumentNode + var nodes []*yaml.Node + for _, item := range s { + nn, err := json2yaml(item.Value) + if err != nil { + return nil, err + } + ns := []*yaml.Node{ + { + Kind: yaml.ScalarNode, + Tag: yamlStringScalar, + Value: item.Key, + }, + nn, + } + nodes = append(nodes, ns...) + } + + n.Content = []*yaml.Node{ + { + Kind: yaml.MappingNode, + Content: nodes, + }, + } + + return yaml.Marshal(&n) +} + +func json2yaml(item interface{}) (*yaml.Node, error) { + switch val := item.(type) { + case JSONMapSlice: + var n yaml.Node + n.Kind = yaml.MappingNode + for i := range val { + childNode, err := json2yaml(&val[i].Value) + if err != nil { + return nil, err + } + n.Content = append(n.Content, &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: yamlStringScalar, + Value: val[i].Key, + }, childNode) + } + return &n, nil + case map[string]interface{}: + var n yaml.Node + n.Kind = yaml.MappingNode + for k, v := range val { + childNode, err := json2yaml(v) + if err != nil { + return nil, err + } + n.Content = append(n.Content, &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: yamlStringScalar, + Value: k, + }, childNode) + } + return &n, nil + case []interface{}: + var n yaml.Node + n.Kind = yaml.SequenceNode + for i := range val { + childNode, err := json2yaml(val[i]) + if err != nil { + return nil, err + } + n.Content = append(n.Content, childNode) + } + return &n, nil + case string: + return &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: yamlStringScalar, + Value: val, + }, nil + case float64: + return &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: yamlFloatScalar, + Value: strconv.FormatFloat(val, 'f', -1, 64), + }, nil + case int64: + return &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: yamlIntScalar, + Value: strconv.FormatInt(val, 10), + }, nil + case uint64: + return &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: yamlIntScalar, + Value: strconv.FormatUint(val, 10), + }, nil + case bool: + return &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: yamlBoolScalar, + Value: strconv.FormatBool(val), + }, nil + } + return nil, nil +} + // JSONMapItem represents the value of a key in a JSON object held by JSONMapSlice type JSONMapItem struct { Key string @@ -173,23 +390,10 @@ func transformData(input interface{}) (out interface{}, err error) { } switch in := input.(type) { - case yaml.MapSlice: - - o := make(JSONMapSlice, len(in)) - for i, mi := range in { - var nmi JSONMapItem - if nmi.Key, err = format(mi.Key); err != nil { - return nil, err - } - - v, ert := transformData(mi.Value) - if ert != nil { - return nil, ert - } - nmi.Value = v - o[i] = nmi - } - return o, nil + case yaml.Node: + return yamlNode(&in) + case *yaml.Node: + return yamlNode(in) case map[interface{}]interface{}: o := make(JSONMapSlice, 0, len(in)) for ke, va := range in { diff --git a/vendor/github.com/google/gnostic/jsonschema/display.go b/vendor/github.com/google/gnostic/jsonschema/display.go index 028a760a91..8677ed49a0 100644 --- a/vendor/github.com/google/gnostic/jsonschema/display.go +++ b/vendor/github.com/google/gnostic/jsonschema/display.go @@ -46,8 +46,23 @@ func (schema *Schema) describeSchema(indent string) string { if schema.Schema != nil { result += indent + "$schema: " + *(schema.Schema) + "\n" } + if schema.ReadOnly != nil && *schema.ReadOnly { + result += indent + fmt.Sprintf("readOnly: %+v\n", *(schema.ReadOnly)) + } + if schema.WriteOnly != nil && *schema.WriteOnly { + result += indent + fmt.Sprintf("writeOnly: %+v\n", *(schema.WriteOnly)) + } if schema.ID != nil { - result += indent + "id: " + *(schema.ID) + "\n" + switch strings.TrimSuffix(*schema.Schema, "#") { + case "http://json-schema.org/draft-04/schema#": + fallthrough + case "#": + fallthrough + case "": + result += indent + "id: " + *(schema.ID) + "\n" + default: + result += indent + "$id: " + *(schema.ID) + "\n" + } } if schema.MultipleOf != nil { result += indent + fmt.Sprintf("multipleOf: %+v\n", *(schema.MultipleOf)) diff --git a/vendor/github.com/google/gnostic/jsonschema/models.go b/vendor/github.com/google/gnostic/jsonschema/models.go index 4781bdc5f5..0d877249ab 100644 --- a/vendor/github.com/google/gnostic/jsonschema/models.go +++ b/vendor/github.com/google/gnostic/jsonschema/models.go @@ -23,9 +23,11 @@ import "gopkg.in/yaml.v3" // All fields are pointers and are nil if the associated values // are not specified. type Schema struct { - Schema *string // $schema - ID *string // id keyword used for $ref resolution scope - Ref *string // $ref, i.e. JSON Pointers + Schema *string // $schema + ID *string // id keyword used for $ref resolution scope + Ref *string // $ref, i.e. JSON Pointers + ReadOnly *bool + WriteOnly *bool // http://json-schema.org/latest/json-schema-validation.html // 5.1. Validation keywords for numeric instances (number and integer) diff --git a/vendor/github.com/google/gnostic/jsonschema/reader.go b/vendor/github.com/google/gnostic/jsonschema/reader.go index b8583d4660..a909a34128 100644 --- a/vendor/github.com/google/gnostic/jsonschema/reader.go +++ b/vendor/github.com/google/gnostic/jsonschema/reader.go @@ -165,7 +165,6 @@ func NewSchemaFromObject(jsonData *yaml.Node) *Schema { default: fmt.Printf("schemaValue: unexpected node %+v\n", jsonData) - return nil } return nil diff --git a/vendor/github.com/google/gnostic/jsonschema/writer.go b/vendor/github.com/google/gnostic/jsonschema/writer.go index 340dc5f933..15b1f90506 100644 --- a/vendor/github.com/google/gnostic/jsonschema/writer.go +++ b/vendor/github.com/google/gnostic/jsonschema/writer.go @@ -16,6 +16,7 @@ package jsonschema import ( "fmt" + "strings" "gopkg.in/yaml.v3" ) @@ -33,7 +34,11 @@ func renderMappingNode(node *yaml.Node, indent string) (result string) { value := node.Content[i+1] switch value.Kind { case yaml.ScalarNode: - result += "\"" + value.Value + "\"" + if value.Tag == "!!bool" { + result += value.Value + } else { + result += "\"" + value.Value + "\"" + } case yaml.MappingNode: result += renderMappingNode(value, innerIndent) case yaml.SequenceNode: @@ -58,7 +63,11 @@ func renderSequenceNode(node *yaml.Node, indent string) (result string) { item := node.Content[i] switch item.Kind { case yaml.ScalarNode: - result += innerIndent + "\"" + item.Value + "\"" + if item.Tag == "!!bool" { + result += innerIndent + item.Value + } else { + result += innerIndent + "\"" + item.Value + "\"" + } case yaml.MappingNode: result += innerIndent + renderMappingNode(item, innerIndent) + "" default: @@ -260,11 +269,26 @@ func (schema *Schema) nodeValue() *yaml.Node { content = appendPair(content, "title", nodeForString(*schema.Title)) } if schema.ID != nil { - content = appendPair(content, "id", nodeForString(*schema.ID)) + switch strings.TrimSuffix(*schema.Schema, "#") { + case "http://json-schema.org/draft-04/schema": + fallthrough + case "#": + fallthrough + case "": + content = appendPair(content, "id", nodeForString(*schema.ID)) + default: + content = appendPair(content, "$id", nodeForString(*schema.ID)) + } } if schema.Schema != nil { content = appendPair(content, "$schema", nodeForString(*schema.Schema)) } + if schema.ReadOnly != nil && *schema.ReadOnly { + content = appendPair(content, "readOnly", nodeForBoolean(*schema.ReadOnly)) + } + if schema.WriteOnly != nil && *schema.WriteOnly { + content = appendPair(content, "writeOnly", nodeForBoolean(*schema.WriteOnly)) + } if schema.Type != nil { content = appendPair(content, "type", schema.Type.nodeValue()) } diff --git a/vendor/github.com/google/gnostic/openapiv2/OpenAPIv2.go b/vendor/github.com/google/gnostic/openapiv2/OpenAPIv2.go index 0f17907667..28c2777d51 100644 --- a/vendor/github.com/google/gnostic/openapiv2/OpenAPIv2.go +++ b/vendor/github.com/google/gnostic/openapiv2/OpenAPIv2.go @@ -7887,7 +7887,12 @@ func (m *Oauth2Scopes) ToRawInfo() *yaml.Node { if m == nil { return info } - // &{Name:additionalProperties Type:NamedString StringEnumValues:[] MapType:string Repeated:true Pattern: Implicit:true Description:} + if m.AdditionalProperties != nil { + for _, item := range m.AdditionalProperties { + info.Content = append(info.Content, compiler.NewScalarNodeForString(item.Name)) + info.Content = append(info.Content, compiler.NewScalarNodeForString(item.Value)) + } + } return info } diff --git a/vendor/github.com/google/gnostic/openapiv3/OpenAPIv3.go b/vendor/github.com/google/gnostic/openapiv3/OpenAPIv3.go index 5f4a7025ea..d54a84db7c 100644 --- a/vendor/github.com/google/gnostic/openapiv3/OpenAPIv3.go +++ b/vendor/github.com/google/gnostic/openapiv3/OpenAPIv3.go @@ -8560,7 +8560,12 @@ func (m *Strings) ToRawInfo() *yaml.Node { if m == nil { return info } - // &{Name:additionalProperties Type:NamedString StringEnumValues:[] MapType:string Repeated:true Pattern: Implicit:true Description:} + if m.AdditionalProperties != nil { + for _, item := range m.AdditionalProperties { + info.Content = append(info.Content, compiler.NewScalarNodeForString(item.Name)) + info.Content = append(info.Content, compiler.NewScalarNodeForString(item.Value)) + } + } return info } diff --git a/vendor/github.com/google/gnostic/openapiv3/OpenAPIv3.pb.go b/vendor/github.com/google/gnostic/openapiv3/OpenAPIv3.pb.go index 499e7f932d..90a56f5526 100644 --- a/vendor/github.com/google/gnostic/openapiv3/OpenAPIv3.pb.go +++ b/vendor/github.com/google/gnostic/openapiv3/OpenAPIv3.pb.go @@ -16,8 +16,8 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.26.0 -// protoc v3.18.1 +// protoc-gen-go v1.28.0 +// protoc v3.19.4 // source: openapiv3/OpenAPIv3.proto package openapi_v3 @@ -6760,12 +6760,13 @@ var file_openapiv3_OpenAPIv3_proto_rawDesc = []byte{ 0x5f, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x33, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x64, 0x41, 0x6e, 0x79, 0x52, 0x16, 0x73, 0x70, 0x65, 0x63, 0x69, 0x66, 0x69, 0x63, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x42, 0x3e, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x42, 0x56, 0x0a, 0x0e, 0x6f, 0x72, 0x67, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x61, 0x70, 0x69, 0x5f, 0x76, 0x33, 0x42, 0x0c, 0x4f, 0x70, 0x65, 0x6e, 0x41, 0x50, 0x49, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, - 0x5a, 0x16, 0x2e, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x61, 0x70, 0x69, 0x76, 0x33, 0x3b, 0x6f, 0x70, - 0x65, 0x6e, 0x61, 0x70, 0x69, 0x5f, 0x76, 0x33, 0xa2, 0x02, 0x03, 0x4f, 0x41, 0x53, 0x62, 0x06, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x5a, 0x2e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2f, 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x2f, 0x6f, 0x70, 0x65, 0x6e, + 0x61, 0x70, 0x69, 0x76, 0x33, 0x3b, 0x6f, 0x70, 0x65, 0x6e, 0x61, 0x70, 0x69, 0x5f, 0x76, 0x33, + 0xa2, 0x02, 0x03, 0x4f, 0x41, 0x53, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/vendor/github.com/google/gnostic/openapiv3/OpenAPIv3.proto b/vendor/github.com/google/gnostic/openapiv3/OpenAPIv3.proto index 1be335b89b..7aede5ed90 100644 --- a/vendor/github.com/google/gnostic/openapiv3/OpenAPIv3.proto +++ b/vendor/github.com/google/gnostic/openapiv3/OpenAPIv3.proto @@ -42,7 +42,7 @@ option java_package = "org.openapi_v3"; option objc_class_prefix = "OAS"; // The Go package name. -option go_package = "./openapiv3;openapi_v3"; +option go_package = "github.com/google/gnostic/openapiv3;openapi_v3"; message AdditionalPropertiesItem { oneof oneof { diff --git a/vendor/github.com/google/gnostic/openapiv3/README.md b/vendor/github.com/google/gnostic/openapiv3/README.md index 5ee12d92e2..83603b82aa 100644 --- a/vendor/github.com/google/gnostic/openapiv3/README.md +++ b/vendor/github.com/google/gnostic/openapiv3/README.md @@ -19,3 +19,7 @@ for OpenAPI. The schema-generator directory contains support code which generates openapi-3.1.json from the OpenAPI 3.1 specification document (Markdown). + +### How to rebuild + +`protoc -I=. -I=third_party --go_out=. --go_opt=paths=source_relative openapiv3/*.proto` \ No newline at end of file diff --git a/vendor/github.com/google/gnostic/openapiv3/annotations.pb.go b/vendor/github.com/google/gnostic/openapiv3/annotations.pb.go new file mode 100644 index 0000000000..ae242f3043 --- /dev/null +++ b/vendor/github.com/google/gnostic/openapiv3/annotations.pb.go @@ -0,0 +1,183 @@ +// Copyright 2022 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.28.0 +// protoc v3.19.4 +// source: openapiv3/annotations.proto + +package openapi_v3 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + descriptorpb "google.golang.org/protobuf/types/descriptorpb" + reflect "reflect" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +var file_openapiv3_annotations_proto_extTypes = []protoimpl.ExtensionInfo{ + { + ExtendedType: (*descriptorpb.FileOptions)(nil), + ExtensionType: (*Document)(nil), + Field: 1143, + Name: "openapi.v3.document", + Tag: "bytes,1143,opt,name=document", + Filename: "openapiv3/annotations.proto", + }, + { + ExtendedType: (*descriptorpb.MethodOptions)(nil), + ExtensionType: (*Operation)(nil), + Field: 1143, + Name: "openapi.v3.operation", + Tag: "bytes,1143,opt,name=operation", + Filename: "openapiv3/annotations.proto", + }, + { + ExtendedType: (*descriptorpb.MessageOptions)(nil), + ExtensionType: (*Schema)(nil), + Field: 1143, + Name: "openapi.v3.schema", + Tag: "bytes,1143,opt,name=schema", + Filename: "openapiv3/annotations.proto", + }, + { + ExtendedType: (*descriptorpb.FieldOptions)(nil), + ExtensionType: (*Schema)(nil), + Field: 1143, + Name: "openapi.v3.property", + Tag: "bytes,1143,opt,name=property", + Filename: "openapiv3/annotations.proto", + }, +} + +// Extension fields to descriptorpb.FileOptions. +var ( + // optional openapi.v3.Document document = 1143; + E_Document = &file_openapiv3_annotations_proto_extTypes[0] +) + +// Extension fields to descriptorpb.MethodOptions. +var ( + // optional openapi.v3.Operation operation = 1143; + E_Operation = &file_openapiv3_annotations_proto_extTypes[1] +) + +// Extension fields to descriptorpb.MessageOptions. +var ( + // optional openapi.v3.Schema schema = 1143; + E_Schema = &file_openapiv3_annotations_proto_extTypes[2] +) + +// Extension fields to descriptorpb.FieldOptions. +var ( + // optional openapi.v3.Schema property = 1143; + E_Property = &file_openapiv3_annotations_proto_extTypes[3] +) + +var File_openapiv3_annotations_proto protoreflect.FileDescriptor + +var file_openapiv3_annotations_proto_rawDesc = []byte{ + 0x0a, 0x1b, 0x6f, 0x70, 0x65, 0x6e, 0x61, 0x70, 0x69, 0x76, 0x33, 0x2f, 0x61, 0x6e, 0x6e, 0x6f, + 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x6f, + 0x70, 0x65, 0x6e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x33, 0x1a, 0x19, 0x6f, 0x70, 0x65, 0x6e, 0x61, + 0x70, 0x69, 0x76, 0x33, 0x2f, 0x4f, 0x70, 0x65, 0x6e, 0x41, 0x50, 0x49, 0x76, 0x33, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x20, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x3a, 0x4f, 0x0a, 0x08, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, + 0x6e, 0x74, 0x12, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x18, 0xf7, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x61, 0x70, + 0x69, 0x2e, 0x76, 0x33, 0x2e, 0x44, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x08, 0x64, + 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x3a, 0x54, 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x4f, 0x70, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x18, 0xf7, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6f, 0x70, + 0x65, 0x6e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x33, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x52, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x4c, 0x0a, + 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, 0x1f, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, + 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0xf7, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x12, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x33, 0x2e, 0x53, 0x63, 0x68, + 0x65, 0x6d, 0x61, 0x52, 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x3a, 0x4e, 0x0a, 0x08, 0x70, + 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x12, 0x1d, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x4f, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0xf7, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, + 0x6f, 0x70, 0x65, 0x6e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x33, 0x2e, 0x53, 0x63, 0x68, 0x65, 0x6d, + 0x61, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x42, 0x5a, 0x0a, 0x0e, 0x6f, + 0x72, 0x67, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x61, 0x70, 0x69, 0x5f, 0x76, 0x33, 0x42, 0x10, 0x41, + 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, + 0x01, 0x5a, 0x2e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x67, 0x6e, 0x6f, 0x73, 0x74, 0x69, 0x63, 0x2f, 0x6f, 0x70, 0x65, + 0x6e, 0x61, 0x70, 0x69, 0x76, 0x33, 0x3b, 0x6f, 0x70, 0x65, 0x6e, 0x61, 0x70, 0x69, 0x5f, 0x76, + 0x33, 0xa2, 0x02, 0x03, 0x4f, 0x41, 0x53, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var file_openapiv3_annotations_proto_goTypes = []interface{}{ + (*descriptorpb.FileOptions)(nil), // 0: google.protobuf.FileOptions + (*descriptorpb.MethodOptions)(nil), // 1: google.protobuf.MethodOptions + (*descriptorpb.MessageOptions)(nil), // 2: google.protobuf.MessageOptions + (*descriptorpb.FieldOptions)(nil), // 3: google.protobuf.FieldOptions + (*Document)(nil), // 4: openapi.v3.Document + (*Operation)(nil), // 5: openapi.v3.Operation + (*Schema)(nil), // 6: openapi.v3.Schema +} +var file_openapiv3_annotations_proto_depIdxs = []int32{ + 0, // 0: openapi.v3.document:extendee -> google.protobuf.FileOptions + 1, // 1: openapi.v3.operation:extendee -> google.protobuf.MethodOptions + 2, // 2: openapi.v3.schema:extendee -> google.protobuf.MessageOptions + 3, // 3: openapi.v3.property:extendee -> google.protobuf.FieldOptions + 4, // 4: openapi.v3.document:type_name -> openapi.v3.Document + 5, // 5: openapi.v3.operation:type_name -> openapi.v3.Operation + 6, // 6: openapi.v3.schema:type_name -> openapi.v3.Schema + 6, // 7: openapi.v3.property:type_name -> openapi.v3.Schema + 8, // [8:8] is the sub-list for method output_type + 8, // [8:8] is the sub-list for method input_type + 4, // [4:8] is the sub-list for extension type_name + 0, // [0:4] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_openapiv3_annotations_proto_init() } +func file_openapiv3_annotations_proto_init() { + if File_openapiv3_annotations_proto != nil { + return + } + file_openapiv3_OpenAPIv3_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_openapiv3_annotations_proto_rawDesc, + NumEnums: 0, + NumMessages: 0, + NumExtensions: 4, + NumServices: 0, + }, + GoTypes: file_openapiv3_annotations_proto_goTypes, + DependencyIndexes: file_openapiv3_annotations_proto_depIdxs, + ExtensionInfos: file_openapiv3_annotations_proto_extTypes, + }.Build() + File_openapiv3_annotations_proto = out.File + file_openapiv3_annotations_proto_rawDesc = nil + file_openapiv3_annotations_proto_goTypes = nil + file_openapiv3_annotations_proto_depIdxs = nil +} diff --git a/vendor/github.com/google/gnostic/openapiv3/annotations.proto b/vendor/github.com/google/gnostic/openapiv3/annotations.proto new file mode 100644 index 0000000000..0bd87810db --- /dev/null +++ b/vendor/github.com/google/gnostic/openapiv3/annotations.proto @@ -0,0 +1,60 @@ +// Copyright 2022 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package openapi.v3; + +import "openapiv3/OpenAPIv3.proto"; +import "google/protobuf/descriptor.proto"; + +// This option lets the proto compiler generate Java code inside the package +// name (see below) instead of inside an outer class. It creates a simpler +// developer experience by reducing one-level of name nesting and be +// consistent with most programming languages that don't support outer classes. +option java_multiple_files = true; + +// The Java outer classname should be the filename in UpperCamelCase. This +// class is only used to hold proto descriptor, so developers don't need to +// work with it directly. +option java_outer_classname = "AnnotationsProto"; + +// The Java package name must be proto package name with proper prefix. +option java_package = "org.openapi_v3"; + +// A reasonable prefix for the Objective-C symbols generated from the package. +// It should at a minimum be 3 characters long, all uppercase, and convention +// is to use an abbreviation of the package name. Something short, but +// hopefully unique enough to not conflict with things that may come along in +// the future. 'GPB' is reserved for the protocol buffer implementation itself. +option objc_class_prefix = "OAS"; + +// The Go package name. +option go_package = "github.com/google/gnostic/openapiv3;openapi_v3"; + +extend google.protobuf.FileOptions { + Document document = 1143; +} + +extend google.protobuf.MethodOptions { + Operation operation = 1143; +} + +extend google.protobuf.MessageOptions { + Schema schema = 1143; +} + +extend google.protobuf.FieldOptions { + Schema property = 1143; +} \ No newline at end of file diff --git a/vendor/github.com/mailru/easyjson/jlexer/lexer.go b/vendor/github.com/mailru/easyjson/jlexer/lexer.go index a42e9d65ad..b5f5e26132 100644 --- a/vendor/github.com/mailru/easyjson/jlexer/lexer.go +++ b/vendor/github.com/mailru/easyjson/jlexer/lexer.go @@ -401,6 +401,7 @@ func (r *Lexer) scanToken() { // consume resets the current token to allow scanning the next one. func (r *Lexer) consume() { r.token.kind = tokenUndef + r.token.byteValueCloned = false r.token.delimValue = 0 } @@ -528,6 +529,7 @@ func (r *Lexer) Skip() { func (r *Lexer) SkipRecursive() { r.scanToken() var start, end byte + startPos := r.start switch r.token.delimValue { case '{': @@ -553,6 +555,14 @@ func (r *Lexer) SkipRecursive() { level-- if level == 0 { r.pos += i + 1 + if !json.Valid(r.Data[startPos:r.pos]) { + r.pos = len(r.Data) + r.fatalError = &LexerError{ + Reason: "skipped array/object json value is invalid", + Offset: r.pos, + Data: string(r.Data[r.pos:]), + } + } return } case c == '\\' && inQuotes: @@ -702,6 +712,10 @@ func (r *Lexer) Bytes() []byte { r.errInvalidToken("string") return nil } + if err := r.unescapeStringToken(); err != nil { + r.errInvalidToken("string") + return nil + } ret := make([]byte, base64.StdEncoding.DecodedLen(len(r.token.byteValue))) n, err := base64.StdEncoding.Decode(ret, r.token.byteValue) if err != nil { diff --git a/vendor/github.com/vmware/govmomi/find/finder.go b/vendor/github.com/vmware/govmomi/find/finder.go index 61ac780c45..4830fc26eb 100644 --- a/vendor/github.com/vmware/govmomi/find/finder.go +++ b/vendor/github.com/vmware/govmomi/find/finder.go @@ -1,5 +1,5 @@ /* -Copyright (c) 2014-2020 VMware, Inc. All Rights Reserved. +Copyright (c) 2014-2023 VMware, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -784,6 +784,11 @@ func (f *Finder) NetworkList(ctx context.Context, path string) ([]object.Network } if len(ns) == 0 { + net, nerr := f.networkByID(ctx, path) + if nerr == nil { + return []object.NetworkReference{net}, nil + } + return nil, &NotFoundError{"network", path} } @@ -798,18 +803,13 @@ func (f *Finder) NetworkList(ctx context.Context, path string) ([]object.Network // Examples: // - Name: "dvpg-1" // - Inventory Path: "vds-1/dvpg-1" +// - Cluster Path: "/dc-1/host/cluster-1/dvpg-1" // - ManagedObject ID: "DistributedVirtualPortgroup:dvportgroup-53" // - Logical Switch UUID: "da2a59b8-2450-4cb2-b5cc-79c4c1d2144c" // - Segment ID: "/infra/segments/vnet_ce50e69b-1784-4a14-9206-ffd7f1f146f7" func (f *Finder) Network(ctx context.Context, path string) (object.NetworkReference, error) { networks, err := f.NetworkList(ctx, path) if err != nil { - if _, ok := err.(*NotFoundError); ok { - net, nerr := f.networkByID(ctx, path) - if nerr == nil { - return net, nil - } - } return nil, err } diff --git a/vendor/github.com/vmware/govmomi/list/lister.go b/vendor/github.com/vmware/govmomi/list/lister.go index 9a4caed686..92a40e8ba8 100644 --- a/vendor/github.com/vmware/govmomi/list/lister.go +++ b/vendor/github.com/vmware/govmomi/list/lister.go @@ -1,5 +1,5 @@ /* -Copyright (c) 2014-2016 VMware, Inc. All Rights Reserved. +Copyright (c) 2014-2023 VMware, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -312,6 +312,7 @@ func (l Lister) ListComputeResource(ctx context.Context) ([]Element, error) { fields := []string{ "host", + "network", "resourcePool", } @@ -327,6 +328,7 @@ func (l Lister) ListComputeResource(ctx context.Context) ([]Element, error) { childTypes := []string{ "HostSystem", + "Network", "ResourcePool", } diff --git a/vendor/github.com/vmware/govmomi/lookup/client.go b/vendor/github.com/vmware/govmomi/lookup/client.go index b3c19846a1..4cc73e0d33 100644 --- a/vendor/github.com/vmware/govmomi/lookup/client.go +++ b/vendor/github.com/vmware/govmomi/lookup/client.go @@ -125,7 +125,9 @@ func EndpointURL(ctx context.Context, c *vim25.Client, path string, filter *type path = endpoint.Url if u, err := url.Parse(path); err == nil { - if c.Thumbprint(u.Host) == "" { + // Set thumbprint only for endpoints on hosts outside this vCenter. + // Platform Services may live on multiple hosts. + if c.URL().Host != u.Host && c.Thumbprint(u.Host) == "" { c.SetThumbprint(u.Host, endpointThumbprint(endpoint)) } } diff --git a/vendor/github.com/vmware/govmomi/simulator/container.go b/vendor/github.com/vmware/govmomi/simulator/container.go index 2fc8515f68..fec1c0f486 100644 --- a/vendor/github.com/vmware/govmomi/simulator/container.go +++ b/vendor/github.com/vmware/govmomi/simulator/container.go @@ -114,6 +114,20 @@ func (c *container) inspect(vm *VirtualMachine) error { net := &vm.Guest.Net[0] net.IpAddress = []string{s.IPAddress} net.MacAddress = s.MacAddress + net.IpConfig = &types.NetIpConfigInfo{ + IpAddress: []types.NetIpConfigInfoIpAddress{{ + IpAddress: s.IPAddress, + PrefixLength: int32(s.IPPrefixLen), + State: string(types.NetIpConfigInfoIpAddressStatusPreferred), + }}, + } + } + + for _, d := range vm.Config.Hardware.Device { + if eth, ok := d.(types.BaseVirtualEthernetCard); ok { + eth.GetVirtualEthernetCard().MacAddress = s.MacAddress + break + } } } diff --git a/vendor/golang.org/x/text/width/kind_string.go b/vendor/golang.org/x/text/width/kind_string.go deleted file mode 100644 index dd3febd43b..0000000000 --- a/vendor/golang.org/x/text/width/kind_string.go +++ /dev/null @@ -1,28 +0,0 @@ -// Code generated by "stringer -type=Kind"; DO NOT EDIT. - -package width - -import "strconv" - -func _() { - // An "invalid array index" compiler error signifies that the constant values have changed. - // Re-run the stringer command to generate them again. - var x [1]struct{} - _ = x[Neutral-0] - _ = x[EastAsianAmbiguous-1] - _ = x[EastAsianWide-2] - _ = x[EastAsianNarrow-3] - _ = x[EastAsianFullwidth-4] - _ = x[EastAsianHalfwidth-5] -} - -const _Kind_name = "NeutralEastAsianAmbiguousEastAsianWideEastAsianNarrowEastAsianFullwidthEastAsianHalfwidth" - -var _Kind_index = [...]uint8{0, 7, 25, 38, 53, 71, 89} - -func (i Kind) String() string { - if i < 0 || i >= Kind(len(_Kind_index)-1) { - return "Kind(" + strconv.FormatInt(int64(i), 10) + ")" - } - return _Kind_name[_Kind_index[i]:_Kind_index[i+1]] -} diff --git a/vendor/golang.org/x/text/width/tables10.0.0.go b/vendor/golang.org/x/text/width/tables10.0.0.go deleted file mode 100644 index cd9d91cafb..0000000000 --- a/vendor/golang.org/x/text/width/tables10.0.0.go +++ /dev/null @@ -1,1329 +0,0 @@ -// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. - -//go:build go1.10 && !go1.13 -// +build go1.10,!go1.13 - -package width - -// UnicodeVersion is the Unicode version from which the tables in this package are derived. -const UnicodeVersion = "10.0.0" - -// lookup returns the trie value for the first UTF-8 encoding in s and -// the width in bytes of this encoding. The size will be 0 if s does not -// hold enough bytes to complete the encoding. len(s) must be greater than 0. -func (t *widthTrie) lookup(s []byte) (v uint16, sz int) { - c0 := s[0] - switch { - case c0 < 0x80: // is ASCII - return widthValues[c0], 1 - case c0 < 0xC2: - return 0, 1 // Illegal UTF-8: not a starter, not ASCII. - case c0 < 0xE0: // 2-byte UTF-8 - if len(s) < 2 { - return 0, 0 - } - i := widthIndex[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c1), 2 - case c0 < 0xF0: // 3-byte UTF-8 - if len(s) < 3 { - return 0, 0 - } - i := widthIndex[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - o := uint32(i)<<6 + uint32(c1) - i = widthIndex[o] - c2 := s[2] - if c2 < 0x80 || 0xC0 <= c2 { - return 0, 2 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c2), 3 - case c0 < 0xF8: // 4-byte UTF-8 - if len(s) < 4 { - return 0, 0 - } - i := widthIndex[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - o := uint32(i)<<6 + uint32(c1) - i = widthIndex[o] - c2 := s[2] - if c2 < 0x80 || 0xC0 <= c2 { - return 0, 2 // Illegal UTF-8: not a continuation byte. - } - o = uint32(i)<<6 + uint32(c2) - i = widthIndex[o] - c3 := s[3] - if c3 < 0x80 || 0xC0 <= c3 { - return 0, 3 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c3), 4 - } - // Illegal rune - return 0, 1 -} - -// lookupUnsafe returns the trie value for the first UTF-8 encoding in s. -// s must start with a full and valid UTF-8 encoded rune. -func (t *widthTrie) lookupUnsafe(s []byte) uint16 { - c0 := s[0] - if c0 < 0x80 { // is ASCII - return widthValues[c0] - } - i := widthIndex[c0] - if c0 < 0xE0 { // 2-byte UTF-8 - return t.lookupValue(uint32(i), s[1]) - } - i = widthIndex[uint32(i)<<6+uint32(s[1])] - if c0 < 0xF0 { // 3-byte UTF-8 - return t.lookupValue(uint32(i), s[2]) - } - i = widthIndex[uint32(i)<<6+uint32(s[2])] - if c0 < 0xF8 { // 4-byte UTF-8 - return t.lookupValue(uint32(i), s[3]) - } - return 0 -} - -// lookupString returns the trie value for the first UTF-8 encoding in s and -// the width in bytes of this encoding. The size will be 0 if s does not -// hold enough bytes to complete the encoding. len(s) must be greater than 0. -func (t *widthTrie) lookupString(s string) (v uint16, sz int) { - c0 := s[0] - switch { - case c0 < 0x80: // is ASCII - return widthValues[c0], 1 - case c0 < 0xC2: - return 0, 1 // Illegal UTF-8: not a starter, not ASCII. - case c0 < 0xE0: // 2-byte UTF-8 - if len(s) < 2 { - return 0, 0 - } - i := widthIndex[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c1), 2 - case c0 < 0xF0: // 3-byte UTF-8 - if len(s) < 3 { - return 0, 0 - } - i := widthIndex[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - o := uint32(i)<<6 + uint32(c1) - i = widthIndex[o] - c2 := s[2] - if c2 < 0x80 || 0xC0 <= c2 { - return 0, 2 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c2), 3 - case c0 < 0xF8: // 4-byte UTF-8 - if len(s) < 4 { - return 0, 0 - } - i := widthIndex[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - o := uint32(i)<<6 + uint32(c1) - i = widthIndex[o] - c2 := s[2] - if c2 < 0x80 || 0xC0 <= c2 { - return 0, 2 // Illegal UTF-8: not a continuation byte. - } - o = uint32(i)<<6 + uint32(c2) - i = widthIndex[o] - c3 := s[3] - if c3 < 0x80 || 0xC0 <= c3 { - return 0, 3 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c3), 4 - } - // Illegal rune - return 0, 1 -} - -// lookupStringUnsafe returns the trie value for the first UTF-8 encoding in s. -// s must start with a full and valid UTF-8 encoded rune. -func (t *widthTrie) lookupStringUnsafe(s string) uint16 { - c0 := s[0] - if c0 < 0x80 { // is ASCII - return widthValues[c0] - } - i := widthIndex[c0] - if c0 < 0xE0 { // 2-byte UTF-8 - return t.lookupValue(uint32(i), s[1]) - } - i = widthIndex[uint32(i)<<6+uint32(s[1])] - if c0 < 0xF0 { // 3-byte UTF-8 - return t.lookupValue(uint32(i), s[2]) - } - i = widthIndex[uint32(i)<<6+uint32(s[2])] - if c0 < 0xF8 { // 4-byte UTF-8 - return t.lookupValue(uint32(i), s[3]) - } - return 0 -} - -// widthTrie. Total size: 14336 bytes (14.00 KiB). Checksum: c59df54630d3dc4a. -type widthTrie struct{} - -func newWidthTrie(i int) *widthTrie { - return &widthTrie{} -} - -// lookupValue determines the type of block n and looks up the value for b. -func (t *widthTrie) lookupValue(n uint32, b byte) uint16 { - switch { - default: - return uint16(widthValues[n<<6+uint32(b)]) - } -} - -// widthValues: 101 blocks, 6464 entries, 12928 bytes -// The third block is the zero block. -var widthValues = [6464]uint16{ - // Block 0x0, offset 0x0 - 0x20: 0x6001, 0x21: 0x6002, 0x22: 0x6002, 0x23: 0x6002, - 0x24: 0x6002, 0x25: 0x6002, 0x26: 0x6002, 0x27: 0x6002, 0x28: 0x6002, 0x29: 0x6002, - 0x2a: 0x6002, 0x2b: 0x6002, 0x2c: 0x6002, 0x2d: 0x6002, 0x2e: 0x6002, 0x2f: 0x6002, - 0x30: 0x6002, 0x31: 0x6002, 0x32: 0x6002, 0x33: 0x6002, 0x34: 0x6002, 0x35: 0x6002, - 0x36: 0x6002, 0x37: 0x6002, 0x38: 0x6002, 0x39: 0x6002, 0x3a: 0x6002, 0x3b: 0x6002, - 0x3c: 0x6002, 0x3d: 0x6002, 0x3e: 0x6002, 0x3f: 0x6002, - // Block 0x1, offset 0x40 - 0x40: 0x6003, 0x41: 0x6003, 0x42: 0x6003, 0x43: 0x6003, 0x44: 0x6003, 0x45: 0x6003, - 0x46: 0x6003, 0x47: 0x6003, 0x48: 0x6003, 0x49: 0x6003, 0x4a: 0x6003, 0x4b: 0x6003, - 0x4c: 0x6003, 0x4d: 0x6003, 0x4e: 0x6003, 0x4f: 0x6003, 0x50: 0x6003, 0x51: 0x6003, - 0x52: 0x6003, 0x53: 0x6003, 0x54: 0x6003, 0x55: 0x6003, 0x56: 0x6003, 0x57: 0x6003, - 0x58: 0x6003, 0x59: 0x6003, 0x5a: 0x6003, 0x5b: 0x6003, 0x5c: 0x6003, 0x5d: 0x6003, - 0x5e: 0x6003, 0x5f: 0x6003, 0x60: 0x6004, 0x61: 0x6004, 0x62: 0x6004, 0x63: 0x6004, - 0x64: 0x6004, 0x65: 0x6004, 0x66: 0x6004, 0x67: 0x6004, 0x68: 0x6004, 0x69: 0x6004, - 0x6a: 0x6004, 0x6b: 0x6004, 0x6c: 0x6004, 0x6d: 0x6004, 0x6e: 0x6004, 0x6f: 0x6004, - 0x70: 0x6004, 0x71: 0x6004, 0x72: 0x6004, 0x73: 0x6004, 0x74: 0x6004, 0x75: 0x6004, - 0x76: 0x6004, 0x77: 0x6004, 0x78: 0x6004, 0x79: 0x6004, 0x7a: 0x6004, 0x7b: 0x6004, - 0x7c: 0x6004, 0x7d: 0x6004, 0x7e: 0x6004, - // Block 0x2, offset 0x80 - // Block 0x3, offset 0xc0 - 0xe1: 0x2000, 0xe2: 0x6005, 0xe3: 0x6005, - 0xe4: 0x2000, 0xe5: 0x6006, 0xe6: 0x6005, 0xe7: 0x2000, 0xe8: 0x2000, - 0xea: 0x2000, 0xec: 0x6007, 0xed: 0x2000, 0xee: 0x2000, 0xef: 0x6008, - 0xf0: 0x2000, 0xf1: 0x2000, 0xf2: 0x2000, 0xf3: 0x2000, 0xf4: 0x2000, - 0xf6: 0x2000, 0xf7: 0x2000, 0xf8: 0x2000, 0xf9: 0x2000, 0xfa: 0x2000, - 0xfc: 0x2000, 0xfd: 0x2000, 0xfe: 0x2000, 0xff: 0x2000, - // Block 0x4, offset 0x100 - 0x106: 0x2000, - 0x110: 0x2000, - 0x117: 0x2000, - 0x118: 0x2000, - 0x11e: 0x2000, 0x11f: 0x2000, 0x120: 0x2000, 0x121: 0x2000, - 0x126: 0x2000, 0x128: 0x2000, 0x129: 0x2000, - 0x12a: 0x2000, 0x12c: 0x2000, 0x12d: 0x2000, - 0x130: 0x2000, 0x132: 0x2000, 0x133: 0x2000, - 0x137: 0x2000, 0x138: 0x2000, 0x139: 0x2000, 0x13a: 0x2000, - 0x13c: 0x2000, 0x13e: 0x2000, - // Block 0x5, offset 0x140 - 0x141: 0x2000, - 0x151: 0x2000, - 0x153: 0x2000, - 0x15b: 0x2000, - 0x166: 0x2000, 0x167: 0x2000, - 0x16b: 0x2000, - 0x171: 0x2000, 0x172: 0x2000, 0x173: 0x2000, - 0x178: 0x2000, - 0x17f: 0x2000, - // Block 0x6, offset 0x180 - 0x180: 0x2000, 0x181: 0x2000, 0x182: 0x2000, 0x184: 0x2000, - 0x188: 0x2000, 0x189: 0x2000, 0x18a: 0x2000, 0x18b: 0x2000, - 0x18d: 0x2000, - 0x192: 0x2000, 0x193: 0x2000, - 0x1a6: 0x2000, 0x1a7: 0x2000, - 0x1ab: 0x2000, - // Block 0x7, offset 0x1c0 - 0x1ce: 0x2000, 0x1d0: 0x2000, - 0x1d2: 0x2000, 0x1d4: 0x2000, 0x1d6: 0x2000, - 0x1d8: 0x2000, 0x1da: 0x2000, 0x1dc: 0x2000, - // Block 0x8, offset 0x200 - 0x211: 0x2000, - 0x221: 0x2000, - // Block 0x9, offset 0x240 - 0x244: 0x2000, - 0x247: 0x2000, 0x249: 0x2000, 0x24a: 0x2000, 0x24b: 0x2000, - 0x24d: 0x2000, 0x250: 0x2000, - 0x258: 0x2000, 0x259: 0x2000, 0x25a: 0x2000, 0x25b: 0x2000, 0x25d: 0x2000, - 0x25f: 0x2000, - // Block 0xa, offset 0x280 - 0x280: 0x2000, 0x281: 0x2000, 0x282: 0x2000, 0x283: 0x2000, 0x284: 0x2000, 0x285: 0x2000, - 0x286: 0x2000, 0x287: 0x2000, 0x288: 0x2000, 0x289: 0x2000, 0x28a: 0x2000, 0x28b: 0x2000, - 0x28c: 0x2000, 0x28d: 0x2000, 0x28e: 0x2000, 0x28f: 0x2000, 0x290: 0x2000, 0x291: 0x2000, - 0x292: 0x2000, 0x293: 0x2000, 0x294: 0x2000, 0x295: 0x2000, 0x296: 0x2000, 0x297: 0x2000, - 0x298: 0x2000, 0x299: 0x2000, 0x29a: 0x2000, 0x29b: 0x2000, 0x29c: 0x2000, 0x29d: 0x2000, - 0x29e: 0x2000, 0x29f: 0x2000, 0x2a0: 0x2000, 0x2a1: 0x2000, 0x2a2: 0x2000, 0x2a3: 0x2000, - 0x2a4: 0x2000, 0x2a5: 0x2000, 0x2a6: 0x2000, 0x2a7: 0x2000, 0x2a8: 0x2000, 0x2a9: 0x2000, - 0x2aa: 0x2000, 0x2ab: 0x2000, 0x2ac: 0x2000, 0x2ad: 0x2000, 0x2ae: 0x2000, 0x2af: 0x2000, - 0x2b0: 0x2000, 0x2b1: 0x2000, 0x2b2: 0x2000, 0x2b3: 0x2000, 0x2b4: 0x2000, 0x2b5: 0x2000, - 0x2b6: 0x2000, 0x2b7: 0x2000, 0x2b8: 0x2000, 0x2b9: 0x2000, 0x2ba: 0x2000, 0x2bb: 0x2000, - 0x2bc: 0x2000, 0x2bd: 0x2000, 0x2be: 0x2000, 0x2bf: 0x2000, - // Block 0xb, offset 0x2c0 - 0x2c0: 0x2000, 0x2c1: 0x2000, 0x2c2: 0x2000, 0x2c3: 0x2000, 0x2c4: 0x2000, 0x2c5: 0x2000, - 0x2c6: 0x2000, 0x2c7: 0x2000, 0x2c8: 0x2000, 0x2c9: 0x2000, 0x2ca: 0x2000, 0x2cb: 0x2000, - 0x2cc: 0x2000, 0x2cd: 0x2000, 0x2ce: 0x2000, 0x2cf: 0x2000, 0x2d0: 0x2000, 0x2d1: 0x2000, - 0x2d2: 0x2000, 0x2d3: 0x2000, 0x2d4: 0x2000, 0x2d5: 0x2000, 0x2d6: 0x2000, 0x2d7: 0x2000, - 0x2d8: 0x2000, 0x2d9: 0x2000, 0x2da: 0x2000, 0x2db: 0x2000, 0x2dc: 0x2000, 0x2dd: 0x2000, - 0x2de: 0x2000, 0x2df: 0x2000, 0x2e0: 0x2000, 0x2e1: 0x2000, 0x2e2: 0x2000, 0x2e3: 0x2000, - 0x2e4: 0x2000, 0x2e5: 0x2000, 0x2e6: 0x2000, 0x2e7: 0x2000, 0x2e8: 0x2000, 0x2e9: 0x2000, - 0x2ea: 0x2000, 0x2eb: 0x2000, 0x2ec: 0x2000, 0x2ed: 0x2000, 0x2ee: 0x2000, 0x2ef: 0x2000, - // Block 0xc, offset 0x300 - 0x311: 0x2000, - 0x312: 0x2000, 0x313: 0x2000, 0x314: 0x2000, 0x315: 0x2000, 0x316: 0x2000, 0x317: 0x2000, - 0x318: 0x2000, 0x319: 0x2000, 0x31a: 0x2000, 0x31b: 0x2000, 0x31c: 0x2000, 0x31d: 0x2000, - 0x31e: 0x2000, 0x31f: 0x2000, 0x320: 0x2000, 0x321: 0x2000, 0x323: 0x2000, - 0x324: 0x2000, 0x325: 0x2000, 0x326: 0x2000, 0x327: 0x2000, 0x328: 0x2000, 0x329: 0x2000, - 0x331: 0x2000, 0x332: 0x2000, 0x333: 0x2000, 0x334: 0x2000, 0x335: 0x2000, - 0x336: 0x2000, 0x337: 0x2000, 0x338: 0x2000, 0x339: 0x2000, 0x33a: 0x2000, 0x33b: 0x2000, - 0x33c: 0x2000, 0x33d: 0x2000, 0x33e: 0x2000, 0x33f: 0x2000, - // Block 0xd, offset 0x340 - 0x340: 0x2000, 0x341: 0x2000, 0x343: 0x2000, 0x344: 0x2000, 0x345: 0x2000, - 0x346: 0x2000, 0x347: 0x2000, 0x348: 0x2000, 0x349: 0x2000, - // Block 0xe, offset 0x380 - 0x381: 0x2000, - 0x390: 0x2000, 0x391: 0x2000, - 0x392: 0x2000, 0x393: 0x2000, 0x394: 0x2000, 0x395: 0x2000, 0x396: 0x2000, 0x397: 0x2000, - 0x398: 0x2000, 0x399: 0x2000, 0x39a: 0x2000, 0x39b: 0x2000, 0x39c: 0x2000, 0x39d: 0x2000, - 0x39e: 0x2000, 0x39f: 0x2000, 0x3a0: 0x2000, 0x3a1: 0x2000, 0x3a2: 0x2000, 0x3a3: 0x2000, - 0x3a4: 0x2000, 0x3a5: 0x2000, 0x3a6: 0x2000, 0x3a7: 0x2000, 0x3a8: 0x2000, 0x3a9: 0x2000, - 0x3aa: 0x2000, 0x3ab: 0x2000, 0x3ac: 0x2000, 0x3ad: 0x2000, 0x3ae: 0x2000, 0x3af: 0x2000, - 0x3b0: 0x2000, 0x3b1: 0x2000, 0x3b2: 0x2000, 0x3b3: 0x2000, 0x3b4: 0x2000, 0x3b5: 0x2000, - 0x3b6: 0x2000, 0x3b7: 0x2000, 0x3b8: 0x2000, 0x3b9: 0x2000, 0x3ba: 0x2000, 0x3bb: 0x2000, - 0x3bc: 0x2000, 0x3bd: 0x2000, 0x3be: 0x2000, 0x3bf: 0x2000, - // Block 0xf, offset 0x3c0 - 0x3c0: 0x2000, 0x3c1: 0x2000, 0x3c2: 0x2000, 0x3c3: 0x2000, 0x3c4: 0x2000, 0x3c5: 0x2000, - 0x3c6: 0x2000, 0x3c7: 0x2000, 0x3c8: 0x2000, 0x3c9: 0x2000, 0x3ca: 0x2000, 0x3cb: 0x2000, - 0x3cc: 0x2000, 0x3cd: 0x2000, 0x3ce: 0x2000, 0x3cf: 0x2000, 0x3d1: 0x2000, - // Block 0x10, offset 0x400 - 0x400: 0x4000, 0x401: 0x4000, 0x402: 0x4000, 0x403: 0x4000, 0x404: 0x4000, 0x405: 0x4000, - 0x406: 0x4000, 0x407: 0x4000, 0x408: 0x4000, 0x409: 0x4000, 0x40a: 0x4000, 0x40b: 0x4000, - 0x40c: 0x4000, 0x40d: 0x4000, 0x40e: 0x4000, 0x40f: 0x4000, 0x410: 0x4000, 0x411: 0x4000, - 0x412: 0x4000, 0x413: 0x4000, 0x414: 0x4000, 0x415: 0x4000, 0x416: 0x4000, 0x417: 0x4000, - 0x418: 0x4000, 0x419: 0x4000, 0x41a: 0x4000, 0x41b: 0x4000, 0x41c: 0x4000, 0x41d: 0x4000, - 0x41e: 0x4000, 0x41f: 0x4000, 0x420: 0x4000, 0x421: 0x4000, 0x422: 0x4000, 0x423: 0x4000, - 0x424: 0x4000, 0x425: 0x4000, 0x426: 0x4000, 0x427: 0x4000, 0x428: 0x4000, 0x429: 0x4000, - 0x42a: 0x4000, 0x42b: 0x4000, 0x42c: 0x4000, 0x42d: 0x4000, 0x42e: 0x4000, 0x42f: 0x4000, - 0x430: 0x4000, 0x431: 0x4000, 0x432: 0x4000, 0x433: 0x4000, 0x434: 0x4000, 0x435: 0x4000, - 0x436: 0x4000, 0x437: 0x4000, 0x438: 0x4000, 0x439: 0x4000, 0x43a: 0x4000, 0x43b: 0x4000, - 0x43c: 0x4000, 0x43d: 0x4000, 0x43e: 0x4000, 0x43f: 0x4000, - // Block 0x11, offset 0x440 - 0x440: 0x4000, 0x441: 0x4000, 0x442: 0x4000, 0x443: 0x4000, 0x444: 0x4000, 0x445: 0x4000, - 0x446: 0x4000, 0x447: 0x4000, 0x448: 0x4000, 0x449: 0x4000, 0x44a: 0x4000, 0x44b: 0x4000, - 0x44c: 0x4000, 0x44d: 0x4000, 0x44e: 0x4000, 0x44f: 0x4000, 0x450: 0x4000, 0x451: 0x4000, - 0x452: 0x4000, 0x453: 0x4000, 0x454: 0x4000, 0x455: 0x4000, 0x456: 0x4000, 0x457: 0x4000, - 0x458: 0x4000, 0x459: 0x4000, 0x45a: 0x4000, 0x45b: 0x4000, 0x45c: 0x4000, 0x45d: 0x4000, - 0x45e: 0x4000, 0x45f: 0x4000, - // Block 0x12, offset 0x480 - 0x490: 0x2000, - 0x493: 0x2000, 0x494: 0x2000, 0x495: 0x2000, 0x496: 0x2000, - 0x498: 0x2000, 0x499: 0x2000, 0x49c: 0x2000, 0x49d: 0x2000, - 0x4a0: 0x2000, 0x4a1: 0x2000, 0x4a2: 0x2000, - 0x4a4: 0x2000, 0x4a5: 0x2000, 0x4a6: 0x2000, 0x4a7: 0x2000, - 0x4b0: 0x2000, 0x4b2: 0x2000, 0x4b3: 0x2000, 0x4b5: 0x2000, - 0x4bb: 0x2000, - 0x4be: 0x2000, - // Block 0x13, offset 0x4c0 - 0x4f4: 0x2000, - 0x4ff: 0x2000, - // Block 0x14, offset 0x500 - 0x501: 0x2000, 0x502: 0x2000, 0x503: 0x2000, 0x504: 0x2000, - 0x529: 0xa009, - 0x52c: 0x2000, - // Block 0x15, offset 0x540 - 0x543: 0x2000, 0x545: 0x2000, - 0x549: 0x2000, - 0x553: 0x2000, 0x556: 0x2000, - 0x561: 0x2000, 0x562: 0x2000, - 0x566: 0x2000, - 0x56b: 0x2000, - // Block 0x16, offset 0x580 - 0x593: 0x2000, 0x594: 0x2000, - 0x59b: 0x2000, 0x59c: 0x2000, 0x59d: 0x2000, - 0x59e: 0x2000, 0x5a0: 0x2000, 0x5a1: 0x2000, 0x5a2: 0x2000, 0x5a3: 0x2000, - 0x5a4: 0x2000, 0x5a5: 0x2000, 0x5a6: 0x2000, 0x5a7: 0x2000, 0x5a8: 0x2000, 0x5a9: 0x2000, - 0x5aa: 0x2000, 0x5ab: 0x2000, - 0x5b0: 0x2000, 0x5b1: 0x2000, 0x5b2: 0x2000, 0x5b3: 0x2000, 0x5b4: 0x2000, 0x5b5: 0x2000, - 0x5b6: 0x2000, 0x5b7: 0x2000, 0x5b8: 0x2000, 0x5b9: 0x2000, - // Block 0x17, offset 0x5c0 - 0x5c9: 0x2000, - 0x5d0: 0x200a, 0x5d1: 0x200b, - 0x5d2: 0x200a, 0x5d3: 0x200c, 0x5d4: 0x2000, 0x5d5: 0x2000, 0x5d6: 0x2000, 0x5d7: 0x2000, - 0x5d8: 0x2000, 0x5d9: 0x2000, - 0x5f8: 0x2000, 0x5f9: 0x2000, - // Block 0x18, offset 0x600 - 0x612: 0x2000, 0x614: 0x2000, - 0x627: 0x2000, - // Block 0x19, offset 0x640 - 0x640: 0x2000, 0x642: 0x2000, 0x643: 0x2000, - 0x647: 0x2000, 0x648: 0x2000, 0x64b: 0x2000, - 0x64f: 0x2000, 0x651: 0x2000, - 0x655: 0x2000, - 0x65a: 0x2000, 0x65d: 0x2000, - 0x65e: 0x2000, 0x65f: 0x2000, 0x660: 0x2000, 0x663: 0x2000, - 0x665: 0x2000, 0x667: 0x2000, 0x668: 0x2000, 0x669: 0x2000, - 0x66a: 0x2000, 0x66b: 0x2000, 0x66c: 0x2000, 0x66e: 0x2000, - 0x674: 0x2000, 0x675: 0x2000, - 0x676: 0x2000, 0x677: 0x2000, - 0x67c: 0x2000, 0x67d: 0x2000, - // Block 0x1a, offset 0x680 - 0x688: 0x2000, - 0x68c: 0x2000, - 0x692: 0x2000, - 0x6a0: 0x2000, 0x6a1: 0x2000, - 0x6a4: 0x2000, 0x6a5: 0x2000, 0x6a6: 0x2000, 0x6a7: 0x2000, - 0x6aa: 0x2000, 0x6ab: 0x2000, 0x6ae: 0x2000, 0x6af: 0x2000, - // Block 0x1b, offset 0x6c0 - 0x6c2: 0x2000, 0x6c3: 0x2000, - 0x6c6: 0x2000, 0x6c7: 0x2000, - 0x6d5: 0x2000, - 0x6d9: 0x2000, - 0x6e5: 0x2000, - 0x6ff: 0x2000, - // Block 0x1c, offset 0x700 - 0x712: 0x2000, - 0x71a: 0x4000, 0x71b: 0x4000, - 0x729: 0x4000, - 0x72a: 0x4000, - // Block 0x1d, offset 0x740 - 0x769: 0x4000, - 0x76a: 0x4000, 0x76b: 0x4000, 0x76c: 0x4000, - 0x770: 0x4000, 0x773: 0x4000, - // Block 0x1e, offset 0x780 - 0x7a0: 0x2000, 0x7a1: 0x2000, 0x7a2: 0x2000, 0x7a3: 0x2000, - 0x7a4: 0x2000, 0x7a5: 0x2000, 0x7a6: 0x2000, 0x7a7: 0x2000, 0x7a8: 0x2000, 0x7a9: 0x2000, - 0x7aa: 0x2000, 0x7ab: 0x2000, 0x7ac: 0x2000, 0x7ad: 0x2000, 0x7ae: 0x2000, 0x7af: 0x2000, - 0x7b0: 0x2000, 0x7b1: 0x2000, 0x7b2: 0x2000, 0x7b3: 0x2000, 0x7b4: 0x2000, 0x7b5: 0x2000, - 0x7b6: 0x2000, 0x7b7: 0x2000, 0x7b8: 0x2000, 0x7b9: 0x2000, 0x7ba: 0x2000, 0x7bb: 0x2000, - 0x7bc: 0x2000, 0x7bd: 0x2000, 0x7be: 0x2000, 0x7bf: 0x2000, - // Block 0x1f, offset 0x7c0 - 0x7c0: 0x2000, 0x7c1: 0x2000, 0x7c2: 0x2000, 0x7c3: 0x2000, 0x7c4: 0x2000, 0x7c5: 0x2000, - 0x7c6: 0x2000, 0x7c7: 0x2000, 0x7c8: 0x2000, 0x7c9: 0x2000, 0x7ca: 0x2000, 0x7cb: 0x2000, - 0x7cc: 0x2000, 0x7cd: 0x2000, 0x7ce: 0x2000, 0x7cf: 0x2000, 0x7d0: 0x2000, 0x7d1: 0x2000, - 0x7d2: 0x2000, 0x7d3: 0x2000, 0x7d4: 0x2000, 0x7d5: 0x2000, 0x7d6: 0x2000, 0x7d7: 0x2000, - 0x7d8: 0x2000, 0x7d9: 0x2000, 0x7da: 0x2000, 0x7db: 0x2000, 0x7dc: 0x2000, 0x7dd: 0x2000, - 0x7de: 0x2000, 0x7df: 0x2000, 0x7e0: 0x2000, 0x7e1: 0x2000, 0x7e2: 0x2000, 0x7e3: 0x2000, - 0x7e4: 0x2000, 0x7e5: 0x2000, 0x7e6: 0x2000, 0x7e7: 0x2000, 0x7e8: 0x2000, 0x7e9: 0x2000, - 0x7eb: 0x2000, 0x7ec: 0x2000, 0x7ed: 0x2000, 0x7ee: 0x2000, 0x7ef: 0x2000, - 0x7f0: 0x2000, 0x7f1: 0x2000, 0x7f2: 0x2000, 0x7f3: 0x2000, 0x7f4: 0x2000, 0x7f5: 0x2000, - 0x7f6: 0x2000, 0x7f7: 0x2000, 0x7f8: 0x2000, 0x7f9: 0x2000, 0x7fa: 0x2000, 0x7fb: 0x2000, - 0x7fc: 0x2000, 0x7fd: 0x2000, 0x7fe: 0x2000, 0x7ff: 0x2000, - // Block 0x20, offset 0x800 - 0x800: 0x2000, 0x801: 0x2000, 0x802: 0x200d, 0x803: 0x2000, 0x804: 0x2000, 0x805: 0x2000, - 0x806: 0x2000, 0x807: 0x2000, 0x808: 0x2000, 0x809: 0x2000, 0x80a: 0x2000, 0x80b: 0x2000, - 0x80c: 0x2000, 0x80d: 0x2000, 0x80e: 0x2000, 0x80f: 0x2000, 0x810: 0x2000, 0x811: 0x2000, - 0x812: 0x2000, 0x813: 0x2000, 0x814: 0x2000, 0x815: 0x2000, 0x816: 0x2000, 0x817: 0x2000, - 0x818: 0x2000, 0x819: 0x2000, 0x81a: 0x2000, 0x81b: 0x2000, 0x81c: 0x2000, 0x81d: 0x2000, - 0x81e: 0x2000, 0x81f: 0x2000, 0x820: 0x2000, 0x821: 0x2000, 0x822: 0x2000, 0x823: 0x2000, - 0x824: 0x2000, 0x825: 0x2000, 0x826: 0x2000, 0x827: 0x2000, 0x828: 0x2000, 0x829: 0x2000, - 0x82a: 0x2000, 0x82b: 0x2000, 0x82c: 0x2000, 0x82d: 0x2000, 0x82e: 0x2000, 0x82f: 0x2000, - 0x830: 0x2000, 0x831: 0x2000, 0x832: 0x2000, 0x833: 0x2000, 0x834: 0x2000, 0x835: 0x2000, - 0x836: 0x2000, 0x837: 0x2000, 0x838: 0x2000, 0x839: 0x2000, 0x83a: 0x2000, 0x83b: 0x2000, - 0x83c: 0x2000, 0x83d: 0x2000, 0x83e: 0x2000, 0x83f: 0x2000, - // Block 0x21, offset 0x840 - 0x840: 0x2000, 0x841: 0x2000, 0x842: 0x2000, 0x843: 0x2000, 0x844: 0x2000, 0x845: 0x2000, - 0x846: 0x2000, 0x847: 0x2000, 0x848: 0x2000, 0x849: 0x2000, 0x84a: 0x2000, 0x84b: 0x2000, - 0x850: 0x2000, 0x851: 0x2000, - 0x852: 0x2000, 0x853: 0x2000, 0x854: 0x2000, 0x855: 0x2000, 0x856: 0x2000, 0x857: 0x2000, - 0x858: 0x2000, 0x859: 0x2000, 0x85a: 0x2000, 0x85b: 0x2000, 0x85c: 0x2000, 0x85d: 0x2000, - 0x85e: 0x2000, 0x85f: 0x2000, 0x860: 0x2000, 0x861: 0x2000, 0x862: 0x2000, 0x863: 0x2000, - 0x864: 0x2000, 0x865: 0x2000, 0x866: 0x2000, 0x867: 0x2000, 0x868: 0x2000, 0x869: 0x2000, - 0x86a: 0x2000, 0x86b: 0x2000, 0x86c: 0x2000, 0x86d: 0x2000, 0x86e: 0x2000, 0x86f: 0x2000, - 0x870: 0x2000, 0x871: 0x2000, 0x872: 0x2000, 0x873: 0x2000, - // Block 0x22, offset 0x880 - 0x880: 0x2000, 0x881: 0x2000, 0x882: 0x2000, 0x883: 0x2000, 0x884: 0x2000, 0x885: 0x2000, - 0x886: 0x2000, 0x887: 0x2000, 0x888: 0x2000, 0x889: 0x2000, 0x88a: 0x2000, 0x88b: 0x2000, - 0x88c: 0x2000, 0x88d: 0x2000, 0x88e: 0x2000, 0x88f: 0x2000, - 0x892: 0x2000, 0x893: 0x2000, 0x894: 0x2000, 0x895: 0x2000, - 0x8a0: 0x200e, 0x8a1: 0x2000, 0x8a3: 0x2000, - 0x8a4: 0x2000, 0x8a5: 0x2000, 0x8a6: 0x2000, 0x8a7: 0x2000, 0x8a8: 0x2000, 0x8a9: 0x2000, - 0x8b2: 0x2000, 0x8b3: 0x2000, - 0x8b6: 0x2000, 0x8b7: 0x2000, - 0x8bc: 0x2000, 0x8bd: 0x2000, - // Block 0x23, offset 0x8c0 - 0x8c0: 0x2000, 0x8c1: 0x2000, - 0x8c6: 0x2000, 0x8c7: 0x2000, 0x8c8: 0x2000, 0x8cb: 0x200f, - 0x8ce: 0x2000, 0x8cf: 0x2000, 0x8d0: 0x2000, 0x8d1: 0x2000, - 0x8e2: 0x2000, 0x8e3: 0x2000, - 0x8e4: 0x2000, 0x8e5: 0x2000, - 0x8ef: 0x2000, - 0x8fd: 0x4000, 0x8fe: 0x4000, - // Block 0x24, offset 0x900 - 0x905: 0x2000, - 0x906: 0x2000, 0x909: 0x2000, - 0x90e: 0x2000, 0x90f: 0x2000, - 0x914: 0x4000, 0x915: 0x4000, - 0x91c: 0x2000, - 0x91e: 0x2000, - // Block 0x25, offset 0x940 - 0x940: 0x2000, 0x942: 0x2000, - 0x948: 0x4000, 0x949: 0x4000, 0x94a: 0x4000, 0x94b: 0x4000, - 0x94c: 0x4000, 0x94d: 0x4000, 0x94e: 0x4000, 0x94f: 0x4000, 0x950: 0x4000, 0x951: 0x4000, - 0x952: 0x4000, 0x953: 0x4000, - 0x960: 0x2000, 0x961: 0x2000, 0x963: 0x2000, - 0x964: 0x2000, 0x965: 0x2000, 0x967: 0x2000, 0x968: 0x2000, 0x969: 0x2000, - 0x96a: 0x2000, 0x96c: 0x2000, 0x96d: 0x2000, 0x96f: 0x2000, - 0x97f: 0x4000, - // Block 0x26, offset 0x980 - 0x993: 0x4000, - 0x99e: 0x2000, 0x99f: 0x2000, 0x9a1: 0x4000, - 0x9aa: 0x4000, 0x9ab: 0x4000, - 0x9bd: 0x4000, 0x9be: 0x4000, 0x9bf: 0x2000, - // Block 0x27, offset 0x9c0 - 0x9c4: 0x4000, 0x9c5: 0x4000, - 0x9c6: 0x2000, 0x9c7: 0x2000, 0x9c8: 0x2000, 0x9c9: 0x2000, 0x9ca: 0x2000, 0x9cb: 0x2000, - 0x9cc: 0x2000, 0x9cd: 0x2000, 0x9ce: 0x4000, 0x9cf: 0x2000, 0x9d0: 0x2000, 0x9d1: 0x2000, - 0x9d2: 0x2000, 0x9d3: 0x2000, 0x9d4: 0x4000, 0x9d5: 0x2000, 0x9d6: 0x2000, 0x9d7: 0x2000, - 0x9d8: 0x2000, 0x9d9: 0x2000, 0x9da: 0x2000, 0x9db: 0x2000, 0x9dc: 0x2000, 0x9dd: 0x2000, - 0x9de: 0x2000, 0x9df: 0x2000, 0x9e0: 0x2000, 0x9e1: 0x2000, 0x9e3: 0x2000, - 0x9e8: 0x2000, 0x9e9: 0x2000, - 0x9ea: 0x4000, 0x9eb: 0x2000, 0x9ec: 0x2000, 0x9ed: 0x2000, 0x9ee: 0x2000, 0x9ef: 0x2000, - 0x9f0: 0x2000, 0x9f1: 0x2000, 0x9f2: 0x4000, 0x9f3: 0x4000, 0x9f4: 0x2000, 0x9f5: 0x4000, - 0x9f6: 0x2000, 0x9f7: 0x2000, 0x9f8: 0x2000, 0x9f9: 0x2000, 0x9fa: 0x4000, 0x9fb: 0x2000, - 0x9fc: 0x2000, 0x9fd: 0x4000, 0x9fe: 0x2000, 0x9ff: 0x2000, - // Block 0x28, offset 0xa00 - 0xa05: 0x4000, - 0xa0a: 0x4000, 0xa0b: 0x4000, - 0xa28: 0x4000, - 0xa3d: 0x2000, - // Block 0x29, offset 0xa40 - 0xa4c: 0x4000, 0xa4e: 0x4000, - 0xa53: 0x4000, 0xa54: 0x4000, 0xa55: 0x4000, 0xa57: 0x4000, - 0xa76: 0x2000, 0xa77: 0x2000, 0xa78: 0x2000, 0xa79: 0x2000, 0xa7a: 0x2000, 0xa7b: 0x2000, - 0xa7c: 0x2000, 0xa7d: 0x2000, 0xa7e: 0x2000, 0xa7f: 0x2000, - // Block 0x2a, offset 0xa80 - 0xa95: 0x4000, 0xa96: 0x4000, 0xa97: 0x4000, - 0xab0: 0x4000, - 0xabf: 0x4000, - // Block 0x2b, offset 0xac0 - 0xae6: 0x6000, 0xae7: 0x6000, 0xae8: 0x6000, 0xae9: 0x6000, - 0xaea: 0x6000, 0xaeb: 0x6000, 0xaec: 0x6000, 0xaed: 0x6000, - // Block 0x2c, offset 0xb00 - 0xb05: 0x6010, - 0xb06: 0x6011, - // Block 0x2d, offset 0xb40 - 0xb5b: 0x4000, 0xb5c: 0x4000, - // Block 0x2e, offset 0xb80 - 0xb90: 0x4000, - 0xb95: 0x4000, 0xb96: 0x2000, 0xb97: 0x2000, - 0xb98: 0x2000, 0xb99: 0x2000, - // Block 0x2f, offset 0xbc0 - 0xbc0: 0x4000, 0xbc1: 0x4000, 0xbc2: 0x4000, 0xbc3: 0x4000, 0xbc4: 0x4000, 0xbc5: 0x4000, - 0xbc6: 0x4000, 0xbc7: 0x4000, 0xbc8: 0x4000, 0xbc9: 0x4000, 0xbca: 0x4000, 0xbcb: 0x4000, - 0xbcc: 0x4000, 0xbcd: 0x4000, 0xbce: 0x4000, 0xbcf: 0x4000, 0xbd0: 0x4000, 0xbd1: 0x4000, - 0xbd2: 0x4000, 0xbd3: 0x4000, 0xbd4: 0x4000, 0xbd5: 0x4000, 0xbd6: 0x4000, 0xbd7: 0x4000, - 0xbd8: 0x4000, 0xbd9: 0x4000, 0xbdb: 0x4000, 0xbdc: 0x4000, 0xbdd: 0x4000, - 0xbde: 0x4000, 0xbdf: 0x4000, 0xbe0: 0x4000, 0xbe1: 0x4000, 0xbe2: 0x4000, 0xbe3: 0x4000, - 0xbe4: 0x4000, 0xbe5: 0x4000, 0xbe6: 0x4000, 0xbe7: 0x4000, 0xbe8: 0x4000, 0xbe9: 0x4000, - 0xbea: 0x4000, 0xbeb: 0x4000, 0xbec: 0x4000, 0xbed: 0x4000, 0xbee: 0x4000, 0xbef: 0x4000, - 0xbf0: 0x4000, 0xbf1: 0x4000, 0xbf2: 0x4000, 0xbf3: 0x4000, 0xbf4: 0x4000, 0xbf5: 0x4000, - 0xbf6: 0x4000, 0xbf7: 0x4000, 0xbf8: 0x4000, 0xbf9: 0x4000, 0xbfa: 0x4000, 0xbfb: 0x4000, - 0xbfc: 0x4000, 0xbfd: 0x4000, 0xbfe: 0x4000, 0xbff: 0x4000, - // Block 0x30, offset 0xc00 - 0xc00: 0x4000, 0xc01: 0x4000, 0xc02: 0x4000, 0xc03: 0x4000, 0xc04: 0x4000, 0xc05: 0x4000, - 0xc06: 0x4000, 0xc07: 0x4000, 0xc08: 0x4000, 0xc09: 0x4000, 0xc0a: 0x4000, 0xc0b: 0x4000, - 0xc0c: 0x4000, 0xc0d: 0x4000, 0xc0e: 0x4000, 0xc0f: 0x4000, 0xc10: 0x4000, 0xc11: 0x4000, - 0xc12: 0x4000, 0xc13: 0x4000, 0xc14: 0x4000, 0xc15: 0x4000, 0xc16: 0x4000, 0xc17: 0x4000, - 0xc18: 0x4000, 0xc19: 0x4000, 0xc1a: 0x4000, 0xc1b: 0x4000, 0xc1c: 0x4000, 0xc1d: 0x4000, - 0xc1e: 0x4000, 0xc1f: 0x4000, 0xc20: 0x4000, 0xc21: 0x4000, 0xc22: 0x4000, 0xc23: 0x4000, - 0xc24: 0x4000, 0xc25: 0x4000, 0xc26: 0x4000, 0xc27: 0x4000, 0xc28: 0x4000, 0xc29: 0x4000, - 0xc2a: 0x4000, 0xc2b: 0x4000, 0xc2c: 0x4000, 0xc2d: 0x4000, 0xc2e: 0x4000, 0xc2f: 0x4000, - 0xc30: 0x4000, 0xc31: 0x4000, 0xc32: 0x4000, 0xc33: 0x4000, - // Block 0x31, offset 0xc40 - 0xc40: 0x4000, 0xc41: 0x4000, 0xc42: 0x4000, 0xc43: 0x4000, 0xc44: 0x4000, 0xc45: 0x4000, - 0xc46: 0x4000, 0xc47: 0x4000, 0xc48: 0x4000, 0xc49: 0x4000, 0xc4a: 0x4000, 0xc4b: 0x4000, - 0xc4c: 0x4000, 0xc4d: 0x4000, 0xc4e: 0x4000, 0xc4f: 0x4000, 0xc50: 0x4000, 0xc51: 0x4000, - 0xc52: 0x4000, 0xc53: 0x4000, 0xc54: 0x4000, 0xc55: 0x4000, - 0xc70: 0x4000, 0xc71: 0x4000, 0xc72: 0x4000, 0xc73: 0x4000, 0xc74: 0x4000, 0xc75: 0x4000, - 0xc76: 0x4000, 0xc77: 0x4000, 0xc78: 0x4000, 0xc79: 0x4000, 0xc7a: 0x4000, 0xc7b: 0x4000, - // Block 0x32, offset 0xc80 - 0xc80: 0x9012, 0xc81: 0x4013, 0xc82: 0x4014, 0xc83: 0x4000, 0xc84: 0x4000, 0xc85: 0x4000, - 0xc86: 0x4000, 0xc87: 0x4000, 0xc88: 0x4000, 0xc89: 0x4000, 0xc8a: 0x4000, 0xc8b: 0x4000, - 0xc8c: 0x4015, 0xc8d: 0x4015, 0xc8e: 0x4000, 0xc8f: 0x4000, 0xc90: 0x4000, 0xc91: 0x4000, - 0xc92: 0x4000, 0xc93: 0x4000, 0xc94: 0x4000, 0xc95: 0x4000, 0xc96: 0x4000, 0xc97: 0x4000, - 0xc98: 0x4000, 0xc99: 0x4000, 0xc9a: 0x4000, 0xc9b: 0x4000, 0xc9c: 0x4000, 0xc9d: 0x4000, - 0xc9e: 0x4000, 0xc9f: 0x4000, 0xca0: 0x4000, 0xca1: 0x4000, 0xca2: 0x4000, 0xca3: 0x4000, - 0xca4: 0x4000, 0xca5: 0x4000, 0xca6: 0x4000, 0xca7: 0x4000, 0xca8: 0x4000, 0xca9: 0x4000, - 0xcaa: 0x4000, 0xcab: 0x4000, 0xcac: 0x4000, 0xcad: 0x4000, 0xcae: 0x4000, 0xcaf: 0x4000, - 0xcb0: 0x4000, 0xcb1: 0x4000, 0xcb2: 0x4000, 0xcb3: 0x4000, 0xcb4: 0x4000, 0xcb5: 0x4000, - 0xcb6: 0x4000, 0xcb7: 0x4000, 0xcb8: 0x4000, 0xcb9: 0x4000, 0xcba: 0x4000, 0xcbb: 0x4000, - 0xcbc: 0x4000, 0xcbd: 0x4000, 0xcbe: 0x4000, - // Block 0x33, offset 0xcc0 - 0xcc1: 0x4000, 0xcc2: 0x4000, 0xcc3: 0x4000, 0xcc4: 0x4000, 0xcc5: 0x4000, - 0xcc6: 0x4000, 0xcc7: 0x4000, 0xcc8: 0x4000, 0xcc9: 0x4000, 0xcca: 0x4000, 0xccb: 0x4000, - 0xccc: 0x4000, 0xccd: 0x4000, 0xcce: 0x4000, 0xccf: 0x4000, 0xcd0: 0x4000, 0xcd1: 0x4000, - 0xcd2: 0x4000, 0xcd3: 0x4000, 0xcd4: 0x4000, 0xcd5: 0x4000, 0xcd6: 0x4000, 0xcd7: 0x4000, - 0xcd8: 0x4000, 0xcd9: 0x4000, 0xcda: 0x4000, 0xcdb: 0x4000, 0xcdc: 0x4000, 0xcdd: 0x4000, - 0xcde: 0x4000, 0xcdf: 0x4000, 0xce0: 0x4000, 0xce1: 0x4000, 0xce2: 0x4000, 0xce3: 0x4000, - 0xce4: 0x4000, 0xce5: 0x4000, 0xce6: 0x4000, 0xce7: 0x4000, 0xce8: 0x4000, 0xce9: 0x4000, - 0xcea: 0x4000, 0xceb: 0x4000, 0xcec: 0x4000, 0xced: 0x4000, 0xcee: 0x4000, 0xcef: 0x4000, - 0xcf0: 0x4000, 0xcf1: 0x4000, 0xcf2: 0x4000, 0xcf3: 0x4000, 0xcf4: 0x4000, 0xcf5: 0x4000, - 0xcf6: 0x4000, 0xcf7: 0x4000, 0xcf8: 0x4000, 0xcf9: 0x4000, 0xcfa: 0x4000, 0xcfb: 0x4000, - 0xcfc: 0x4000, 0xcfd: 0x4000, 0xcfe: 0x4000, 0xcff: 0x4000, - // Block 0x34, offset 0xd00 - 0xd00: 0x4000, 0xd01: 0x4000, 0xd02: 0x4000, 0xd03: 0x4000, 0xd04: 0x4000, 0xd05: 0x4000, - 0xd06: 0x4000, 0xd07: 0x4000, 0xd08: 0x4000, 0xd09: 0x4000, 0xd0a: 0x4000, 0xd0b: 0x4000, - 0xd0c: 0x4000, 0xd0d: 0x4000, 0xd0e: 0x4000, 0xd0f: 0x4000, 0xd10: 0x4000, 0xd11: 0x4000, - 0xd12: 0x4000, 0xd13: 0x4000, 0xd14: 0x4000, 0xd15: 0x4000, 0xd16: 0x4000, - 0xd19: 0x4016, 0xd1a: 0x4017, 0xd1b: 0x4000, 0xd1c: 0x4000, 0xd1d: 0x4000, - 0xd1e: 0x4000, 0xd1f: 0x4000, 0xd20: 0x4000, 0xd21: 0x4018, 0xd22: 0x4019, 0xd23: 0x401a, - 0xd24: 0x401b, 0xd25: 0x401c, 0xd26: 0x401d, 0xd27: 0x401e, 0xd28: 0x401f, 0xd29: 0x4020, - 0xd2a: 0x4021, 0xd2b: 0x4022, 0xd2c: 0x4000, 0xd2d: 0x4010, 0xd2e: 0x4000, 0xd2f: 0x4023, - 0xd30: 0x4000, 0xd31: 0x4024, 0xd32: 0x4000, 0xd33: 0x4025, 0xd34: 0x4000, 0xd35: 0x4026, - 0xd36: 0x4000, 0xd37: 0x401a, 0xd38: 0x4000, 0xd39: 0x4027, 0xd3a: 0x4000, 0xd3b: 0x4028, - 0xd3c: 0x4000, 0xd3d: 0x4020, 0xd3e: 0x4000, 0xd3f: 0x4029, - // Block 0x35, offset 0xd40 - 0xd40: 0x4000, 0xd41: 0x402a, 0xd42: 0x4000, 0xd43: 0x402b, 0xd44: 0x402c, 0xd45: 0x4000, - 0xd46: 0x4017, 0xd47: 0x4000, 0xd48: 0x402d, 0xd49: 0x4000, 0xd4a: 0x402e, 0xd4b: 0x402f, - 0xd4c: 0x4030, 0xd4d: 0x4017, 0xd4e: 0x4016, 0xd4f: 0x4017, 0xd50: 0x4000, 0xd51: 0x4000, - 0xd52: 0x4031, 0xd53: 0x4000, 0xd54: 0x4000, 0xd55: 0x4031, 0xd56: 0x4000, 0xd57: 0x4000, - 0xd58: 0x4032, 0xd59: 0x4000, 0xd5a: 0x4000, 0xd5b: 0x4032, 0xd5c: 0x4000, 0xd5d: 0x4000, - 0xd5e: 0x4033, 0xd5f: 0x402e, 0xd60: 0x4034, 0xd61: 0x4035, 0xd62: 0x4034, 0xd63: 0x4036, - 0xd64: 0x4037, 0xd65: 0x4024, 0xd66: 0x4035, 0xd67: 0x4025, 0xd68: 0x4038, 0xd69: 0x4038, - 0xd6a: 0x4039, 0xd6b: 0x4039, 0xd6c: 0x403a, 0xd6d: 0x403a, 0xd6e: 0x4000, 0xd6f: 0x4035, - 0xd70: 0x4000, 0xd71: 0x4000, 0xd72: 0x403b, 0xd73: 0x403c, 0xd74: 0x4000, 0xd75: 0x4000, - 0xd76: 0x4000, 0xd77: 0x4000, 0xd78: 0x4000, 0xd79: 0x4000, 0xd7a: 0x4000, 0xd7b: 0x403d, - 0xd7c: 0x401c, 0xd7d: 0x4000, 0xd7e: 0x4000, 0xd7f: 0x4000, - // Block 0x36, offset 0xd80 - 0xd85: 0x4000, - 0xd86: 0x4000, 0xd87: 0x4000, 0xd88: 0x4000, 0xd89: 0x4000, 0xd8a: 0x4000, 0xd8b: 0x4000, - 0xd8c: 0x4000, 0xd8d: 0x4000, 0xd8e: 0x4000, 0xd8f: 0x4000, 0xd90: 0x4000, 0xd91: 0x4000, - 0xd92: 0x4000, 0xd93: 0x4000, 0xd94: 0x4000, 0xd95: 0x4000, 0xd96: 0x4000, 0xd97: 0x4000, - 0xd98: 0x4000, 0xd99: 0x4000, 0xd9a: 0x4000, 0xd9b: 0x4000, 0xd9c: 0x4000, 0xd9d: 0x4000, - 0xd9e: 0x4000, 0xd9f: 0x4000, 0xda0: 0x4000, 0xda1: 0x4000, 0xda2: 0x4000, 0xda3: 0x4000, - 0xda4: 0x4000, 0xda5: 0x4000, 0xda6: 0x4000, 0xda7: 0x4000, 0xda8: 0x4000, 0xda9: 0x4000, - 0xdaa: 0x4000, 0xdab: 0x4000, 0xdac: 0x4000, 0xdad: 0x4000, 0xdae: 0x4000, - 0xdb1: 0x403e, 0xdb2: 0x403e, 0xdb3: 0x403e, 0xdb4: 0x403e, 0xdb5: 0x403e, - 0xdb6: 0x403e, 0xdb7: 0x403e, 0xdb8: 0x403e, 0xdb9: 0x403e, 0xdba: 0x403e, 0xdbb: 0x403e, - 0xdbc: 0x403e, 0xdbd: 0x403e, 0xdbe: 0x403e, 0xdbf: 0x403e, - // Block 0x37, offset 0xdc0 - 0xdc0: 0x4037, 0xdc1: 0x4037, 0xdc2: 0x4037, 0xdc3: 0x4037, 0xdc4: 0x4037, 0xdc5: 0x4037, - 0xdc6: 0x4037, 0xdc7: 0x4037, 0xdc8: 0x4037, 0xdc9: 0x4037, 0xdca: 0x4037, 0xdcb: 0x4037, - 0xdcc: 0x4037, 0xdcd: 0x4037, 0xdce: 0x4037, 0xdcf: 0x400e, 0xdd0: 0x403f, 0xdd1: 0x4040, - 0xdd2: 0x4041, 0xdd3: 0x4040, 0xdd4: 0x403f, 0xdd5: 0x4042, 0xdd6: 0x4043, 0xdd7: 0x4044, - 0xdd8: 0x4040, 0xdd9: 0x4041, 0xdda: 0x4040, 0xddb: 0x4045, 0xddc: 0x4009, 0xddd: 0x4045, - 0xdde: 0x4046, 0xddf: 0x4045, 0xde0: 0x4047, 0xde1: 0x400b, 0xde2: 0x400a, 0xde3: 0x400c, - 0xde4: 0x4048, 0xde5: 0x4000, 0xde6: 0x4000, 0xde7: 0x4000, 0xde8: 0x4000, 0xde9: 0x4000, - 0xdea: 0x4000, 0xdeb: 0x4000, 0xdec: 0x4000, 0xded: 0x4000, 0xdee: 0x4000, 0xdef: 0x4000, - 0xdf0: 0x4000, 0xdf1: 0x4000, 0xdf2: 0x4000, 0xdf3: 0x4000, 0xdf4: 0x4000, 0xdf5: 0x4000, - 0xdf6: 0x4000, 0xdf7: 0x4000, 0xdf8: 0x4000, 0xdf9: 0x4000, 0xdfa: 0x4000, 0xdfb: 0x4000, - 0xdfc: 0x4000, 0xdfd: 0x4000, 0xdfe: 0x4000, 0xdff: 0x4000, - // Block 0x38, offset 0xe00 - 0xe00: 0x4000, 0xe01: 0x4000, 0xe02: 0x4000, 0xe03: 0x4000, 0xe04: 0x4000, 0xe05: 0x4000, - 0xe06: 0x4000, 0xe07: 0x4000, 0xe08: 0x4000, 0xe09: 0x4000, 0xe0a: 0x4000, 0xe0b: 0x4000, - 0xe0c: 0x4000, 0xe0d: 0x4000, 0xe0e: 0x4000, 0xe10: 0x4000, 0xe11: 0x4000, - 0xe12: 0x4000, 0xe13: 0x4000, 0xe14: 0x4000, 0xe15: 0x4000, 0xe16: 0x4000, 0xe17: 0x4000, - 0xe18: 0x4000, 0xe19: 0x4000, 0xe1a: 0x4000, 0xe1b: 0x4000, 0xe1c: 0x4000, 0xe1d: 0x4000, - 0xe1e: 0x4000, 0xe1f: 0x4000, 0xe20: 0x4000, 0xe21: 0x4000, 0xe22: 0x4000, 0xe23: 0x4000, - 0xe24: 0x4000, 0xe25: 0x4000, 0xe26: 0x4000, 0xe27: 0x4000, 0xe28: 0x4000, 0xe29: 0x4000, - 0xe2a: 0x4000, 0xe2b: 0x4000, 0xe2c: 0x4000, 0xe2d: 0x4000, 0xe2e: 0x4000, 0xe2f: 0x4000, - 0xe30: 0x4000, 0xe31: 0x4000, 0xe32: 0x4000, 0xe33: 0x4000, 0xe34: 0x4000, 0xe35: 0x4000, - 0xe36: 0x4000, 0xe37: 0x4000, 0xe38: 0x4000, 0xe39: 0x4000, 0xe3a: 0x4000, - // Block 0x39, offset 0xe40 - 0xe40: 0x4000, 0xe41: 0x4000, 0xe42: 0x4000, 0xe43: 0x4000, 0xe44: 0x4000, 0xe45: 0x4000, - 0xe46: 0x4000, 0xe47: 0x4000, 0xe48: 0x4000, 0xe49: 0x4000, 0xe4a: 0x4000, 0xe4b: 0x4000, - 0xe4c: 0x4000, 0xe4d: 0x4000, 0xe4e: 0x4000, 0xe4f: 0x4000, 0xe50: 0x4000, 0xe51: 0x4000, - 0xe52: 0x4000, 0xe53: 0x4000, 0xe54: 0x4000, 0xe55: 0x4000, 0xe56: 0x4000, 0xe57: 0x4000, - 0xe58: 0x4000, 0xe59: 0x4000, 0xe5a: 0x4000, 0xe5b: 0x4000, 0xe5c: 0x4000, 0xe5d: 0x4000, - 0xe5e: 0x4000, 0xe5f: 0x4000, 0xe60: 0x4000, 0xe61: 0x4000, 0xe62: 0x4000, 0xe63: 0x4000, - 0xe70: 0x4000, 0xe71: 0x4000, 0xe72: 0x4000, 0xe73: 0x4000, 0xe74: 0x4000, 0xe75: 0x4000, - 0xe76: 0x4000, 0xe77: 0x4000, 0xe78: 0x4000, 0xe79: 0x4000, 0xe7a: 0x4000, 0xe7b: 0x4000, - 0xe7c: 0x4000, 0xe7d: 0x4000, 0xe7e: 0x4000, 0xe7f: 0x4000, - // Block 0x3a, offset 0xe80 - 0xe80: 0x4000, 0xe81: 0x4000, 0xe82: 0x4000, 0xe83: 0x4000, 0xe84: 0x4000, 0xe85: 0x4000, - 0xe86: 0x4000, 0xe87: 0x4000, 0xe88: 0x4000, 0xe89: 0x4000, 0xe8a: 0x4000, 0xe8b: 0x4000, - 0xe8c: 0x4000, 0xe8d: 0x4000, 0xe8e: 0x4000, 0xe8f: 0x4000, 0xe90: 0x4000, 0xe91: 0x4000, - 0xe92: 0x4000, 0xe93: 0x4000, 0xe94: 0x4000, 0xe95: 0x4000, 0xe96: 0x4000, 0xe97: 0x4000, - 0xe98: 0x4000, 0xe99: 0x4000, 0xe9a: 0x4000, 0xe9b: 0x4000, 0xe9c: 0x4000, 0xe9d: 0x4000, - 0xe9e: 0x4000, 0xea0: 0x4000, 0xea1: 0x4000, 0xea2: 0x4000, 0xea3: 0x4000, - 0xea4: 0x4000, 0xea5: 0x4000, 0xea6: 0x4000, 0xea7: 0x4000, 0xea8: 0x4000, 0xea9: 0x4000, - 0xeaa: 0x4000, 0xeab: 0x4000, 0xeac: 0x4000, 0xead: 0x4000, 0xeae: 0x4000, 0xeaf: 0x4000, - 0xeb0: 0x4000, 0xeb1: 0x4000, 0xeb2: 0x4000, 0xeb3: 0x4000, 0xeb4: 0x4000, 0xeb5: 0x4000, - 0xeb6: 0x4000, 0xeb7: 0x4000, 0xeb8: 0x4000, 0xeb9: 0x4000, 0xeba: 0x4000, 0xebb: 0x4000, - 0xebc: 0x4000, 0xebd: 0x4000, 0xebe: 0x4000, 0xebf: 0x4000, - // Block 0x3b, offset 0xec0 - 0xec0: 0x4000, 0xec1: 0x4000, 0xec2: 0x4000, 0xec3: 0x4000, 0xec4: 0x4000, 0xec5: 0x4000, - 0xec6: 0x4000, 0xec7: 0x4000, 0xec8: 0x2000, 0xec9: 0x2000, 0xeca: 0x2000, 0xecb: 0x2000, - 0xecc: 0x2000, 0xecd: 0x2000, 0xece: 0x2000, 0xecf: 0x2000, 0xed0: 0x4000, 0xed1: 0x4000, - 0xed2: 0x4000, 0xed3: 0x4000, 0xed4: 0x4000, 0xed5: 0x4000, 0xed6: 0x4000, 0xed7: 0x4000, - 0xed8: 0x4000, 0xed9: 0x4000, 0xeda: 0x4000, 0xedb: 0x4000, 0xedc: 0x4000, 0xedd: 0x4000, - 0xede: 0x4000, 0xedf: 0x4000, 0xee0: 0x4000, 0xee1: 0x4000, 0xee2: 0x4000, 0xee3: 0x4000, - 0xee4: 0x4000, 0xee5: 0x4000, 0xee6: 0x4000, 0xee7: 0x4000, 0xee8: 0x4000, 0xee9: 0x4000, - 0xeea: 0x4000, 0xeeb: 0x4000, 0xeec: 0x4000, 0xeed: 0x4000, 0xeee: 0x4000, 0xeef: 0x4000, - 0xef0: 0x4000, 0xef1: 0x4000, 0xef2: 0x4000, 0xef3: 0x4000, 0xef4: 0x4000, 0xef5: 0x4000, - 0xef6: 0x4000, 0xef7: 0x4000, 0xef8: 0x4000, 0xef9: 0x4000, 0xefa: 0x4000, 0xefb: 0x4000, - 0xefc: 0x4000, 0xefd: 0x4000, 0xefe: 0x4000, 0xeff: 0x4000, - // Block 0x3c, offset 0xf00 - 0xf00: 0x4000, 0xf01: 0x4000, 0xf02: 0x4000, 0xf03: 0x4000, 0xf04: 0x4000, 0xf05: 0x4000, - 0xf06: 0x4000, 0xf07: 0x4000, 0xf08: 0x4000, 0xf09: 0x4000, 0xf0a: 0x4000, 0xf0b: 0x4000, - 0xf0c: 0x4000, 0xf0d: 0x4000, 0xf0e: 0x4000, 0xf0f: 0x4000, 0xf10: 0x4000, 0xf11: 0x4000, - 0xf12: 0x4000, 0xf13: 0x4000, 0xf14: 0x4000, 0xf15: 0x4000, 0xf16: 0x4000, 0xf17: 0x4000, - 0xf18: 0x4000, 0xf19: 0x4000, 0xf1a: 0x4000, 0xf1b: 0x4000, 0xf1c: 0x4000, 0xf1d: 0x4000, - 0xf1e: 0x4000, 0xf1f: 0x4000, 0xf20: 0x4000, 0xf21: 0x4000, 0xf22: 0x4000, 0xf23: 0x4000, - 0xf24: 0x4000, 0xf25: 0x4000, 0xf26: 0x4000, 0xf27: 0x4000, 0xf28: 0x4000, 0xf29: 0x4000, - 0xf2a: 0x4000, 0xf2b: 0x4000, 0xf2c: 0x4000, 0xf2d: 0x4000, 0xf2e: 0x4000, 0xf2f: 0x4000, - 0xf30: 0x4000, 0xf31: 0x4000, 0xf32: 0x4000, 0xf33: 0x4000, 0xf34: 0x4000, 0xf35: 0x4000, - 0xf36: 0x4000, 0xf37: 0x4000, 0xf38: 0x4000, 0xf39: 0x4000, 0xf3a: 0x4000, 0xf3b: 0x4000, - 0xf3c: 0x4000, 0xf3d: 0x4000, 0xf3e: 0x4000, - // Block 0x3d, offset 0xf40 - 0xf40: 0x4000, 0xf41: 0x4000, 0xf42: 0x4000, 0xf43: 0x4000, 0xf44: 0x4000, 0xf45: 0x4000, - 0xf46: 0x4000, 0xf47: 0x4000, 0xf48: 0x4000, 0xf49: 0x4000, 0xf4a: 0x4000, 0xf4b: 0x4000, - 0xf4c: 0x4000, 0xf50: 0x4000, 0xf51: 0x4000, - 0xf52: 0x4000, 0xf53: 0x4000, 0xf54: 0x4000, 0xf55: 0x4000, 0xf56: 0x4000, 0xf57: 0x4000, - 0xf58: 0x4000, 0xf59: 0x4000, 0xf5a: 0x4000, 0xf5b: 0x4000, 0xf5c: 0x4000, 0xf5d: 0x4000, - 0xf5e: 0x4000, 0xf5f: 0x4000, 0xf60: 0x4000, 0xf61: 0x4000, 0xf62: 0x4000, 0xf63: 0x4000, - 0xf64: 0x4000, 0xf65: 0x4000, 0xf66: 0x4000, 0xf67: 0x4000, 0xf68: 0x4000, 0xf69: 0x4000, - 0xf6a: 0x4000, 0xf6b: 0x4000, 0xf6c: 0x4000, 0xf6d: 0x4000, 0xf6e: 0x4000, 0xf6f: 0x4000, - 0xf70: 0x4000, 0xf71: 0x4000, 0xf72: 0x4000, 0xf73: 0x4000, 0xf74: 0x4000, 0xf75: 0x4000, - 0xf76: 0x4000, 0xf77: 0x4000, 0xf78: 0x4000, 0xf79: 0x4000, 0xf7a: 0x4000, 0xf7b: 0x4000, - 0xf7c: 0x4000, 0xf7d: 0x4000, 0xf7e: 0x4000, 0xf7f: 0x4000, - // Block 0x3e, offset 0xf80 - 0xf80: 0x4000, 0xf81: 0x4000, 0xf82: 0x4000, 0xf83: 0x4000, 0xf84: 0x4000, 0xf85: 0x4000, - 0xf86: 0x4000, - // Block 0x3f, offset 0xfc0 - 0xfe0: 0x4000, 0xfe1: 0x4000, 0xfe2: 0x4000, 0xfe3: 0x4000, - 0xfe4: 0x4000, 0xfe5: 0x4000, 0xfe6: 0x4000, 0xfe7: 0x4000, 0xfe8: 0x4000, 0xfe9: 0x4000, - 0xfea: 0x4000, 0xfeb: 0x4000, 0xfec: 0x4000, 0xfed: 0x4000, 0xfee: 0x4000, 0xfef: 0x4000, - 0xff0: 0x4000, 0xff1: 0x4000, 0xff2: 0x4000, 0xff3: 0x4000, 0xff4: 0x4000, 0xff5: 0x4000, - 0xff6: 0x4000, 0xff7: 0x4000, 0xff8: 0x4000, 0xff9: 0x4000, 0xffa: 0x4000, 0xffb: 0x4000, - 0xffc: 0x4000, - // Block 0x40, offset 0x1000 - 0x1000: 0x4000, 0x1001: 0x4000, 0x1002: 0x4000, 0x1003: 0x4000, 0x1004: 0x4000, 0x1005: 0x4000, - 0x1006: 0x4000, 0x1007: 0x4000, 0x1008: 0x4000, 0x1009: 0x4000, 0x100a: 0x4000, 0x100b: 0x4000, - 0x100c: 0x4000, 0x100d: 0x4000, 0x100e: 0x4000, 0x100f: 0x4000, 0x1010: 0x4000, 0x1011: 0x4000, - 0x1012: 0x4000, 0x1013: 0x4000, 0x1014: 0x4000, 0x1015: 0x4000, 0x1016: 0x4000, 0x1017: 0x4000, - 0x1018: 0x4000, 0x1019: 0x4000, 0x101a: 0x4000, 0x101b: 0x4000, 0x101c: 0x4000, 0x101d: 0x4000, - 0x101e: 0x4000, 0x101f: 0x4000, 0x1020: 0x4000, 0x1021: 0x4000, 0x1022: 0x4000, 0x1023: 0x4000, - // Block 0x41, offset 0x1040 - 0x1040: 0x2000, 0x1041: 0x2000, 0x1042: 0x2000, 0x1043: 0x2000, 0x1044: 0x2000, 0x1045: 0x2000, - 0x1046: 0x2000, 0x1047: 0x2000, 0x1048: 0x2000, 0x1049: 0x2000, 0x104a: 0x2000, 0x104b: 0x2000, - 0x104c: 0x2000, 0x104d: 0x2000, 0x104e: 0x2000, 0x104f: 0x2000, 0x1050: 0x4000, 0x1051: 0x4000, - 0x1052: 0x4000, 0x1053: 0x4000, 0x1054: 0x4000, 0x1055: 0x4000, 0x1056: 0x4000, 0x1057: 0x4000, - 0x1058: 0x4000, 0x1059: 0x4000, - 0x1070: 0x4000, 0x1071: 0x4000, 0x1072: 0x4000, 0x1073: 0x4000, 0x1074: 0x4000, 0x1075: 0x4000, - 0x1076: 0x4000, 0x1077: 0x4000, 0x1078: 0x4000, 0x1079: 0x4000, 0x107a: 0x4000, 0x107b: 0x4000, - 0x107c: 0x4000, 0x107d: 0x4000, 0x107e: 0x4000, 0x107f: 0x4000, - // Block 0x42, offset 0x1080 - 0x1080: 0x4000, 0x1081: 0x4000, 0x1082: 0x4000, 0x1083: 0x4000, 0x1084: 0x4000, 0x1085: 0x4000, - 0x1086: 0x4000, 0x1087: 0x4000, 0x1088: 0x4000, 0x1089: 0x4000, 0x108a: 0x4000, 0x108b: 0x4000, - 0x108c: 0x4000, 0x108d: 0x4000, 0x108e: 0x4000, 0x108f: 0x4000, 0x1090: 0x4000, 0x1091: 0x4000, - 0x1092: 0x4000, 0x1094: 0x4000, 0x1095: 0x4000, 0x1096: 0x4000, 0x1097: 0x4000, - 0x1098: 0x4000, 0x1099: 0x4000, 0x109a: 0x4000, 0x109b: 0x4000, 0x109c: 0x4000, 0x109d: 0x4000, - 0x109e: 0x4000, 0x109f: 0x4000, 0x10a0: 0x4000, 0x10a1: 0x4000, 0x10a2: 0x4000, 0x10a3: 0x4000, - 0x10a4: 0x4000, 0x10a5: 0x4000, 0x10a6: 0x4000, 0x10a8: 0x4000, 0x10a9: 0x4000, - 0x10aa: 0x4000, 0x10ab: 0x4000, - // Block 0x43, offset 0x10c0 - 0x10c1: 0x9012, 0x10c2: 0x9012, 0x10c3: 0x9012, 0x10c4: 0x9012, 0x10c5: 0x9012, - 0x10c6: 0x9012, 0x10c7: 0x9012, 0x10c8: 0x9012, 0x10c9: 0x9012, 0x10ca: 0x9012, 0x10cb: 0x9012, - 0x10cc: 0x9012, 0x10cd: 0x9012, 0x10ce: 0x9012, 0x10cf: 0x9012, 0x10d0: 0x9012, 0x10d1: 0x9012, - 0x10d2: 0x9012, 0x10d3: 0x9012, 0x10d4: 0x9012, 0x10d5: 0x9012, 0x10d6: 0x9012, 0x10d7: 0x9012, - 0x10d8: 0x9012, 0x10d9: 0x9012, 0x10da: 0x9012, 0x10db: 0x9012, 0x10dc: 0x9012, 0x10dd: 0x9012, - 0x10de: 0x9012, 0x10df: 0x9012, 0x10e0: 0x9049, 0x10e1: 0x9049, 0x10e2: 0x9049, 0x10e3: 0x9049, - 0x10e4: 0x9049, 0x10e5: 0x9049, 0x10e6: 0x9049, 0x10e7: 0x9049, 0x10e8: 0x9049, 0x10e9: 0x9049, - 0x10ea: 0x9049, 0x10eb: 0x9049, 0x10ec: 0x9049, 0x10ed: 0x9049, 0x10ee: 0x9049, 0x10ef: 0x9049, - 0x10f0: 0x9049, 0x10f1: 0x9049, 0x10f2: 0x9049, 0x10f3: 0x9049, 0x10f4: 0x9049, 0x10f5: 0x9049, - 0x10f6: 0x9049, 0x10f7: 0x9049, 0x10f8: 0x9049, 0x10f9: 0x9049, 0x10fa: 0x9049, 0x10fb: 0x9049, - 0x10fc: 0x9049, 0x10fd: 0x9049, 0x10fe: 0x9049, 0x10ff: 0x9049, - // Block 0x44, offset 0x1100 - 0x1100: 0x9049, 0x1101: 0x9049, 0x1102: 0x9049, 0x1103: 0x9049, 0x1104: 0x9049, 0x1105: 0x9049, - 0x1106: 0x9049, 0x1107: 0x9049, 0x1108: 0x9049, 0x1109: 0x9049, 0x110a: 0x9049, 0x110b: 0x9049, - 0x110c: 0x9049, 0x110d: 0x9049, 0x110e: 0x9049, 0x110f: 0x9049, 0x1110: 0x9049, 0x1111: 0x9049, - 0x1112: 0x9049, 0x1113: 0x9049, 0x1114: 0x9049, 0x1115: 0x9049, 0x1116: 0x9049, 0x1117: 0x9049, - 0x1118: 0x9049, 0x1119: 0x9049, 0x111a: 0x9049, 0x111b: 0x9049, 0x111c: 0x9049, 0x111d: 0x9049, - 0x111e: 0x9049, 0x111f: 0x904a, 0x1120: 0x904b, 0x1121: 0xb04c, 0x1122: 0xb04d, 0x1123: 0xb04d, - 0x1124: 0xb04e, 0x1125: 0xb04f, 0x1126: 0xb050, 0x1127: 0xb051, 0x1128: 0xb052, 0x1129: 0xb053, - 0x112a: 0xb054, 0x112b: 0xb055, 0x112c: 0xb056, 0x112d: 0xb057, 0x112e: 0xb058, 0x112f: 0xb059, - 0x1130: 0xb05a, 0x1131: 0xb05b, 0x1132: 0xb05c, 0x1133: 0xb05d, 0x1134: 0xb05e, 0x1135: 0xb05f, - 0x1136: 0xb060, 0x1137: 0xb061, 0x1138: 0xb062, 0x1139: 0xb063, 0x113a: 0xb064, 0x113b: 0xb065, - 0x113c: 0xb052, 0x113d: 0xb066, 0x113e: 0xb067, 0x113f: 0xb055, - // Block 0x45, offset 0x1140 - 0x1140: 0xb068, 0x1141: 0xb069, 0x1142: 0xb06a, 0x1143: 0xb06b, 0x1144: 0xb05a, 0x1145: 0xb056, - 0x1146: 0xb06c, 0x1147: 0xb06d, 0x1148: 0xb06b, 0x1149: 0xb06e, 0x114a: 0xb06b, 0x114b: 0xb06f, - 0x114c: 0xb06f, 0x114d: 0xb070, 0x114e: 0xb070, 0x114f: 0xb071, 0x1150: 0xb056, 0x1151: 0xb072, - 0x1152: 0xb073, 0x1153: 0xb072, 0x1154: 0xb074, 0x1155: 0xb073, 0x1156: 0xb075, 0x1157: 0xb075, - 0x1158: 0xb076, 0x1159: 0xb076, 0x115a: 0xb077, 0x115b: 0xb077, 0x115c: 0xb073, 0x115d: 0xb078, - 0x115e: 0xb079, 0x115f: 0xb067, 0x1160: 0xb07a, 0x1161: 0xb07b, 0x1162: 0xb07b, 0x1163: 0xb07b, - 0x1164: 0xb07b, 0x1165: 0xb07b, 0x1166: 0xb07b, 0x1167: 0xb07b, 0x1168: 0xb07b, 0x1169: 0xb07b, - 0x116a: 0xb07b, 0x116b: 0xb07b, 0x116c: 0xb07b, 0x116d: 0xb07b, 0x116e: 0xb07b, 0x116f: 0xb07b, - 0x1170: 0xb07c, 0x1171: 0xb07c, 0x1172: 0xb07c, 0x1173: 0xb07c, 0x1174: 0xb07c, 0x1175: 0xb07c, - 0x1176: 0xb07c, 0x1177: 0xb07c, 0x1178: 0xb07c, 0x1179: 0xb07c, 0x117a: 0xb07c, 0x117b: 0xb07c, - 0x117c: 0xb07c, 0x117d: 0xb07c, 0x117e: 0xb07c, - // Block 0x46, offset 0x1180 - 0x1182: 0xb07d, 0x1183: 0xb07e, 0x1184: 0xb07f, 0x1185: 0xb080, - 0x1186: 0xb07f, 0x1187: 0xb07e, 0x118a: 0xb081, 0x118b: 0xb082, - 0x118c: 0xb083, 0x118d: 0xb07f, 0x118e: 0xb080, 0x118f: 0xb07f, - 0x1192: 0xb084, 0x1193: 0xb085, 0x1194: 0xb084, 0x1195: 0xb086, 0x1196: 0xb084, 0x1197: 0xb087, - 0x119a: 0xb088, 0x119b: 0xb089, 0x119c: 0xb08a, - 0x11a0: 0x908b, 0x11a1: 0x908b, 0x11a2: 0x908c, 0x11a3: 0x908d, - 0x11a4: 0x908b, 0x11a5: 0x908e, 0x11a6: 0x908f, 0x11a8: 0xb090, 0x11a9: 0xb091, - 0x11aa: 0xb092, 0x11ab: 0xb091, 0x11ac: 0xb093, 0x11ad: 0xb094, 0x11ae: 0xb095, - 0x11bd: 0x2000, - // Block 0x47, offset 0x11c0 - 0x11e0: 0x4000, 0x11e1: 0x4000, - // Block 0x48, offset 0x1200 - 0x1200: 0x4000, 0x1201: 0x4000, 0x1202: 0x4000, 0x1203: 0x4000, 0x1204: 0x4000, 0x1205: 0x4000, - 0x1206: 0x4000, 0x1207: 0x4000, 0x1208: 0x4000, 0x1209: 0x4000, 0x120a: 0x4000, 0x120b: 0x4000, - 0x120c: 0x4000, 0x120d: 0x4000, 0x120e: 0x4000, 0x120f: 0x4000, 0x1210: 0x4000, 0x1211: 0x4000, - 0x1212: 0x4000, 0x1213: 0x4000, 0x1214: 0x4000, 0x1215: 0x4000, 0x1216: 0x4000, 0x1217: 0x4000, - 0x1218: 0x4000, 0x1219: 0x4000, 0x121a: 0x4000, 0x121b: 0x4000, 0x121c: 0x4000, 0x121d: 0x4000, - 0x121e: 0x4000, 0x121f: 0x4000, 0x1220: 0x4000, 0x1221: 0x4000, 0x1222: 0x4000, 0x1223: 0x4000, - 0x1224: 0x4000, 0x1225: 0x4000, 0x1226: 0x4000, 0x1227: 0x4000, 0x1228: 0x4000, 0x1229: 0x4000, - 0x122a: 0x4000, 0x122b: 0x4000, 0x122c: 0x4000, - // Block 0x49, offset 0x1240 - 0x1240: 0x4000, 0x1241: 0x4000, 0x1242: 0x4000, 0x1243: 0x4000, 0x1244: 0x4000, 0x1245: 0x4000, - 0x1246: 0x4000, 0x1247: 0x4000, 0x1248: 0x4000, 0x1249: 0x4000, 0x124a: 0x4000, 0x124b: 0x4000, - 0x124c: 0x4000, 0x124d: 0x4000, 0x124e: 0x4000, 0x124f: 0x4000, 0x1250: 0x4000, 0x1251: 0x4000, - 0x1252: 0x4000, 0x1253: 0x4000, 0x1254: 0x4000, 0x1255: 0x4000, 0x1256: 0x4000, 0x1257: 0x4000, - 0x1258: 0x4000, 0x1259: 0x4000, 0x125a: 0x4000, 0x125b: 0x4000, 0x125c: 0x4000, 0x125d: 0x4000, - 0x125e: 0x4000, 0x125f: 0x4000, 0x1260: 0x4000, 0x1261: 0x4000, 0x1262: 0x4000, 0x1263: 0x4000, - 0x1264: 0x4000, 0x1265: 0x4000, 0x1266: 0x4000, 0x1267: 0x4000, 0x1268: 0x4000, 0x1269: 0x4000, - 0x126a: 0x4000, 0x126b: 0x4000, 0x126c: 0x4000, 0x126d: 0x4000, 0x126e: 0x4000, 0x126f: 0x4000, - 0x1270: 0x4000, 0x1271: 0x4000, 0x1272: 0x4000, - // Block 0x4a, offset 0x1280 - 0x1280: 0x4000, 0x1281: 0x4000, 0x1282: 0x4000, 0x1283: 0x4000, 0x1284: 0x4000, 0x1285: 0x4000, - 0x1286: 0x4000, 0x1287: 0x4000, 0x1288: 0x4000, 0x1289: 0x4000, 0x128a: 0x4000, 0x128b: 0x4000, - 0x128c: 0x4000, 0x128d: 0x4000, 0x128e: 0x4000, 0x128f: 0x4000, 0x1290: 0x4000, 0x1291: 0x4000, - 0x1292: 0x4000, 0x1293: 0x4000, 0x1294: 0x4000, 0x1295: 0x4000, 0x1296: 0x4000, 0x1297: 0x4000, - 0x1298: 0x4000, 0x1299: 0x4000, 0x129a: 0x4000, 0x129b: 0x4000, 0x129c: 0x4000, 0x129d: 0x4000, - 0x129e: 0x4000, - // Block 0x4b, offset 0x12c0 - 0x12f0: 0x4000, 0x12f1: 0x4000, 0x12f2: 0x4000, 0x12f3: 0x4000, 0x12f4: 0x4000, 0x12f5: 0x4000, - 0x12f6: 0x4000, 0x12f7: 0x4000, 0x12f8: 0x4000, 0x12f9: 0x4000, 0x12fa: 0x4000, 0x12fb: 0x4000, - 0x12fc: 0x4000, 0x12fd: 0x4000, 0x12fe: 0x4000, 0x12ff: 0x4000, - // Block 0x4c, offset 0x1300 - 0x1300: 0x4000, 0x1301: 0x4000, 0x1302: 0x4000, 0x1303: 0x4000, 0x1304: 0x4000, 0x1305: 0x4000, - 0x1306: 0x4000, 0x1307: 0x4000, 0x1308: 0x4000, 0x1309: 0x4000, 0x130a: 0x4000, 0x130b: 0x4000, - 0x130c: 0x4000, 0x130d: 0x4000, 0x130e: 0x4000, 0x130f: 0x4000, 0x1310: 0x4000, 0x1311: 0x4000, - 0x1312: 0x4000, 0x1313: 0x4000, 0x1314: 0x4000, 0x1315: 0x4000, 0x1316: 0x4000, 0x1317: 0x4000, - 0x1318: 0x4000, 0x1319: 0x4000, 0x131a: 0x4000, 0x131b: 0x4000, 0x131c: 0x4000, 0x131d: 0x4000, - 0x131e: 0x4000, 0x131f: 0x4000, 0x1320: 0x4000, 0x1321: 0x4000, 0x1322: 0x4000, 0x1323: 0x4000, - 0x1324: 0x4000, 0x1325: 0x4000, 0x1326: 0x4000, 0x1327: 0x4000, 0x1328: 0x4000, 0x1329: 0x4000, - 0x132a: 0x4000, 0x132b: 0x4000, 0x132c: 0x4000, 0x132d: 0x4000, 0x132e: 0x4000, 0x132f: 0x4000, - 0x1330: 0x4000, 0x1331: 0x4000, 0x1332: 0x4000, 0x1333: 0x4000, 0x1334: 0x4000, 0x1335: 0x4000, - 0x1336: 0x4000, 0x1337: 0x4000, 0x1338: 0x4000, 0x1339: 0x4000, 0x133a: 0x4000, 0x133b: 0x4000, - // Block 0x4d, offset 0x1340 - 0x1344: 0x4000, - // Block 0x4e, offset 0x1380 - 0x138f: 0x4000, - // Block 0x4f, offset 0x13c0 - 0x13c0: 0x2000, 0x13c1: 0x2000, 0x13c2: 0x2000, 0x13c3: 0x2000, 0x13c4: 0x2000, 0x13c5: 0x2000, - 0x13c6: 0x2000, 0x13c7: 0x2000, 0x13c8: 0x2000, 0x13c9: 0x2000, 0x13ca: 0x2000, - 0x13d0: 0x2000, 0x13d1: 0x2000, - 0x13d2: 0x2000, 0x13d3: 0x2000, 0x13d4: 0x2000, 0x13d5: 0x2000, 0x13d6: 0x2000, 0x13d7: 0x2000, - 0x13d8: 0x2000, 0x13d9: 0x2000, 0x13da: 0x2000, 0x13db: 0x2000, 0x13dc: 0x2000, 0x13dd: 0x2000, - 0x13de: 0x2000, 0x13df: 0x2000, 0x13e0: 0x2000, 0x13e1: 0x2000, 0x13e2: 0x2000, 0x13e3: 0x2000, - 0x13e4: 0x2000, 0x13e5: 0x2000, 0x13e6: 0x2000, 0x13e7: 0x2000, 0x13e8: 0x2000, 0x13e9: 0x2000, - 0x13ea: 0x2000, 0x13eb: 0x2000, 0x13ec: 0x2000, 0x13ed: 0x2000, - 0x13f0: 0x2000, 0x13f1: 0x2000, 0x13f2: 0x2000, 0x13f3: 0x2000, 0x13f4: 0x2000, 0x13f5: 0x2000, - 0x13f6: 0x2000, 0x13f7: 0x2000, 0x13f8: 0x2000, 0x13f9: 0x2000, 0x13fa: 0x2000, 0x13fb: 0x2000, - 0x13fc: 0x2000, 0x13fd: 0x2000, 0x13fe: 0x2000, 0x13ff: 0x2000, - // Block 0x50, offset 0x1400 - 0x1400: 0x2000, 0x1401: 0x2000, 0x1402: 0x2000, 0x1403: 0x2000, 0x1404: 0x2000, 0x1405: 0x2000, - 0x1406: 0x2000, 0x1407: 0x2000, 0x1408: 0x2000, 0x1409: 0x2000, 0x140a: 0x2000, 0x140b: 0x2000, - 0x140c: 0x2000, 0x140d: 0x2000, 0x140e: 0x2000, 0x140f: 0x2000, 0x1410: 0x2000, 0x1411: 0x2000, - 0x1412: 0x2000, 0x1413: 0x2000, 0x1414: 0x2000, 0x1415: 0x2000, 0x1416: 0x2000, 0x1417: 0x2000, - 0x1418: 0x2000, 0x1419: 0x2000, 0x141a: 0x2000, 0x141b: 0x2000, 0x141c: 0x2000, 0x141d: 0x2000, - 0x141e: 0x2000, 0x141f: 0x2000, 0x1420: 0x2000, 0x1421: 0x2000, 0x1422: 0x2000, 0x1423: 0x2000, - 0x1424: 0x2000, 0x1425: 0x2000, 0x1426: 0x2000, 0x1427: 0x2000, 0x1428: 0x2000, 0x1429: 0x2000, - 0x1430: 0x2000, 0x1431: 0x2000, 0x1432: 0x2000, 0x1433: 0x2000, 0x1434: 0x2000, 0x1435: 0x2000, - 0x1436: 0x2000, 0x1437: 0x2000, 0x1438: 0x2000, 0x1439: 0x2000, 0x143a: 0x2000, 0x143b: 0x2000, - 0x143c: 0x2000, 0x143d: 0x2000, 0x143e: 0x2000, 0x143f: 0x2000, - // Block 0x51, offset 0x1440 - 0x1440: 0x2000, 0x1441: 0x2000, 0x1442: 0x2000, 0x1443: 0x2000, 0x1444: 0x2000, 0x1445: 0x2000, - 0x1446: 0x2000, 0x1447: 0x2000, 0x1448: 0x2000, 0x1449: 0x2000, 0x144a: 0x2000, 0x144b: 0x2000, - 0x144c: 0x2000, 0x144d: 0x2000, 0x144e: 0x4000, 0x144f: 0x2000, 0x1450: 0x2000, 0x1451: 0x4000, - 0x1452: 0x4000, 0x1453: 0x4000, 0x1454: 0x4000, 0x1455: 0x4000, 0x1456: 0x4000, 0x1457: 0x4000, - 0x1458: 0x4000, 0x1459: 0x4000, 0x145a: 0x4000, 0x145b: 0x2000, 0x145c: 0x2000, 0x145d: 0x2000, - 0x145e: 0x2000, 0x145f: 0x2000, 0x1460: 0x2000, 0x1461: 0x2000, 0x1462: 0x2000, 0x1463: 0x2000, - 0x1464: 0x2000, 0x1465: 0x2000, 0x1466: 0x2000, 0x1467: 0x2000, 0x1468: 0x2000, 0x1469: 0x2000, - 0x146a: 0x2000, 0x146b: 0x2000, 0x146c: 0x2000, - // Block 0x52, offset 0x1480 - 0x1480: 0x4000, 0x1481: 0x4000, 0x1482: 0x4000, - 0x1490: 0x4000, 0x1491: 0x4000, - 0x1492: 0x4000, 0x1493: 0x4000, 0x1494: 0x4000, 0x1495: 0x4000, 0x1496: 0x4000, 0x1497: 0x4000, - 0x1498: 0x4000, 0x1499: 0x4000, 0x149a: 0x4000, 0x149b: 0x4000, 0x149c: 0x4000, 0x149d: 0x4000, - 0x149e: 0x4000, 0x149f: 0x4000, 0x14a0: 0x4000, 0x14a1: 0x4000, 0x14a2: 0x4000, 0x14a3: 0x4000, - 0x14a4: 0x4000, 0x14a5: 0x4000, 0x14a6: 0x4000, 0x14a7: 0x4000, 0x14a8: 0x4000, 0x14a9: 0x4000, - 0x14aa: 0x4000, 0x14ab: 0x4000, 0x14ac: 0x4000, 0x14ad: 0x4000, 0x14ae: 0x4000, 0x14af: 0x4000, - 0x14b0: 0x4000, 0x14b1: 0x4000, 0x14b2: 0x4000, 0x14b3: 0x4000, 0x14b4: 0x4000, 0x14b5: 0x4000, - 0x14b6: 0x4000, 0x14b7: 0x4000, 0x14b8: 0x4000, 0x14b9: 0x4000, 0x14ba: 0x4000, 0x14bb: 0x4000, - // Block 0x53, offset 0x14c0 - 0x14c0: 0x4000, 0x14c1: 0x4000, 0x14c2: 0x4000, 0x14c3: 0x4000, 0x14c4: 0x4000, 0x14c5: 0x4000, - 0x14c6: 0x4000, 0x14c7: 0x4000, 0x14c8: 0x4000, - 0x14d0: 0x4000, 0x14d1: 0x4000, - 0x14e0: 0x4000, 0x14e1: 0x4000, 0x14e2: 0x4000, 0x14e3: 0x4000, - 0x14e4: 0x4000, 0x14e5: 0x4000, - // Block 0x54, offset 0x1500 - 0x1500: 0x4000, 0x1501: 0x4000, 0x1502: 0x4000, 0x1503: 0x4000, 0x1504: 0x4000, 0x1505: 0x4000, - 0x1506: 0x4000, 0x1507: 0x4000, 0x1508: 0x4000, 0x1509: 0x4000, 0x150a: 0x4000, 0x150b: 0x4000, - 0x150c: 0x4000, 0x150d: 0x4000, 0x150e: 0x4000, 0x150f: 0x4000, 0x1510: 0x4000, 0x1511: 0x4000, - 0x1512: 0x4000, 0x1513: 0x4000, 0x1514: 0x4000, 0x1515: 0x4000, 0x1516: 0x4000, 0x1517: 0x4000, - 0x1518: 0x4000, 0x1519: 0x4000, 0x151a: 0x4000, 0x151b: 0x4000, 0x151c: 0x4000, 0x151d: 0x4000, - 0x151e: 0x4000, 0x151f: 0x4000, 0x1520: 0x4000, - 0x152d: 0x4000, 0x152e: 0x4000, 0x152f: 0x4000, - 0x1530: 0x4000, 0x1531: 0x4000, 0x1532: 0x4000, 0x1533: 0x4000, 0x1534: 0x4000, 0x1535: 0x4000, - 0x1537: 0x4000, 0x1538: 0x4000, 0x1539: 0x4000, 0x153a: 0x4000, 0x153b: 0x4000, - 0x153c: 0x4000, 0x153d: 0x4000, 0x153e: 0x4000, 0x153f: 0x4000, - // Block 0x55, offset 0x1540 - 0x1540: 0x4000, 0x1541: 0x4000, 0x1542: 0x4000, 0x1543: 0x4000, 0x1544: 0x4000, 0x1545: 0x4000, - 0x1546: 0x4000, 0x1547: 0x4000, 0x1548: 0x4000, 0x1549: 0x4000, 0x154a: 0x4000, 0x154b: 0x4000, - 0x154c: 0x4000, 0x154d: 0x4000, 0x154e: 0x4000, 0x154f: 0x4000, 0x1550: 0x4000, 0x1551: 0x4000, - 0x1552: 0x4000, 0x1553: 0x4000, 0x1554: 0x4000, 0x1555: 0x4000, 0x1556: 0x4000, 0x1557: 0x4000, - 0x1558: 0x4000, 0x1559: 0x4000, 0x155a: 0x4000, 0x155b: 0x4000, 0x155c: 0x4000, 0x155d: 0x4000, - 0x155e: 0x4000, 0x155f: 0x4000, 0x1560: 0x4000, 0x1561: 0x4000, 0x1562: 0x4000, 0x1563: 0x4000, - 0x1564: 0x4000, 0x1565: 0x4000, 0x1566: 0x4000, 0x1567: 0x4000, 0x1568: 0x4000, 0x1569: 0x4000, - 0x156a: 0x4000, 0x156b: 0x4000, 0x156c: 0x4000, 0x156d: 0x4000, 0x156e: 0x4000, 0x156f: 0x4000, - 0x1570: 0x4000, 0x1571: 0x4000, 0x1572: 0x4000, 0x1573: 0x4000, 0x1574: 0x4000, 0x1575: 0x4000, - 0x1576: 0x4000, 0x1577: 0x4000, 0x1578: 0x4000, 0x1579: 0x4000, 0x157a: 0x4000, 0x157b: 0x4000, - 0x157c: 0x4000, 0x157e: 0x4000, 0x157f: 0x4000, - // Block 0x56, offset 0x1580 - 0x1580: 0x4000, 0x1581: 0x4000, 0x1582: 0x4000, 0x1583: 0x4000, 0x1584: 0x4000, 0x1585: 0x4000, - 0x1586: 0x4000, 0x1587: 0x4000, 0x1588: 0x4000, 0x1589: 0x4000, 0x158a: 0x4000, 0x158b: 0x4000, - 0x158c: 0x4000, 0x158d: 0x4000, 0x158e: 0x4000, 0x158f: 0x4000, 0x1590: 0x4000, 0x1591: 0x4000, - 0x1592: 0x4000, 0x1593: 0x4000, - 0x15a0: 0x4000, 0x15a1: 0x4000, 0x15a2: 0x4000, 0x15a3: 0x4000, - 0x15a4: 0x4000, 0x15a5: 0x4000, 0x15a6: 0x4000, 0x15a7: 0x4000, 0x15a8: 0x4000, 0x15a9: 0x4000, - 0x15aa: 0x4000, 0x15ab: 0x4000, 0x15ac: 0x4000, 0x15ad: 0x4000, 0x15ae: 0x4000, 0x15af: 0x4000, - 0x15b0: 0x4000, 0x15b1: 0x4000, 0x15b2: 0x4000, 0x15b3: 0x4000, 0x15b4: 0x4000, 0x15b5: 0x4000, - 0x15b6: 0x4000, 0x15b7: 0x4000, 0x15b8: 0x4000, 0x15b9: 0x4000, 0x15ba: 0x4000, 0x15bb: 0x4000, - 0x15bc: 0x4000, 0x15bd: 0x4000, 0x15be: 0x4000, 0x15bf: 0x4000, - // Block 0x57, offset 0x15c0 - 0x15c0: 0x4000, 0x15c1: 0x4000, 0x15c2: 0x4000, 0x15c3: 0x4000, 0x15c4: 0x4000, 0x15c5: 0x4000, - 0x15c6: 0x4000, 0x15c7: 0x4000, 0x15c8: 0x4000, 0x15c9: 0x4000, 0x15ca: 0x4000, - 0x15cf: 0x4000, 0x15d0: 0x4000, 0x15d1: 0x4000, - 0x15d2: 0x4000, 0x15d3: 0x4000, - 0x15e0: 0x4000, 0x15e1: 0x4000, 0x15e2: 0x4000, 0x15e3: 0x4000, - 0x15e4: 0x4000, 0x15e5: 0x4000, 0x15e6: 0x4000, 0x15e7: 0x4000, 0x15e8: 0x4000, 0x15e9: 0x4000, - 0x15ea: 0x4000, 0x15eb: 0x4000, 0x15ec: 0x4000, 0x15ed: 0x4000, 0x15ee: 0x4000, 0x15ef: 0x4000, - 0x15f0: 0x4000, 0x15f4: 0x4000, - 0x15f8: 0x4000, 0x15f9: 0x4000, 0x15fa: 0x4000, 0x15fb: 0x4000, - 0x15fc: 0x4000, 0x15fd: 0x4000, 0x15fe: 0x4000, 0x15ff: 0x4000, - // Block 0x58, offset 0x1600 - 0x1600: 0x4000, 0x1602: 0x4000, 0x1603: 0x4000, 0x1604: 0x4000, 0x1605: 0x4000, - 0x1606: 0x4000, 0x1607: 0x4000, 0x1608: 0x4000, 0x1609: 0x4000, 0x160a: 0x4000, 0x160b: 0x4000, - 0x160c: 0x4000, 0x160d: 0x4000, 0x160e: 0x4000, 0x160f: 0x4000, 0x1610: 0x4000, 0x1611: 0x4000, - 0x1612: 0x4000, 0x1613: 0x4000, 0x1614: 0x4000, 0x1615: 0x4000, 0x1616: 0x4000, 0x1617: 0x4000, - 0x1618: 0x4000, 0x1619: 0x4000, 0x161a: 0x4000, 0x161b: 0x4000, 0x161c: 0x4000, 0x161d: 0x4000, - 0x161e: 0x4000, 0x161f: 0x4000, 0x1620: 0x4000, 0x1621: 0x4000, 0x1622: 0x4000, 0x1623: 0x4000, - 0x1624: 0x4000, 0x1625: 0x4000, 0x1626: 0x4000, 0x1627: 0x4000, 0x1628: 0x4000, 0x1629: 0x4000, - 0x162a: 0x4000, 0x162b: 0x4000, 0x162c: 0x4000, 0x162d: 0x4000, 0x162e: 0x4000, 0x162f: 0x4000, - 0x1630: 0x4000, 0x1631: 0x4000, 0x1632: 0x4000, 0x1633: 0x4000, 0x1634: 0x4000, 0x1635: 0x4000, - 0x1636: 0x4000, 0x1637: 0x4000, 0x1638: 0x4000, 0x1639: 0x4000, 0x163a: 0x4000, 0x163b: 0x4000, - 0x163c: 0x4000, 0x163d: 0x4000, 0x163e: 0x4000, 0x163f: 0x4000, - // Block 0x59, offset 0x1640 - 0x1640: 0x4000, 0x1641: 0x4000, 0x1642: 0x4000, 0x1643: 0x4000, 0x1644: 0x4000, 0x1645: 0x4000, - 0x1646: 0x4000, 0x1647: 0x4000, 0x1648: 0x4000, 0x1649: 0x4000, 0x164a: 0x4000, 0x164b: 0x4000, - 0x164c: 0x4000, 0x164d: 0x4000, 0x164e: 0x4000, 0x164f: 0x4000, 0x1650: 0x4000, 0x1651: 0x4000, - 0x1652: 0x4000, 0x1653: 0x4000, 0x1654: 0x4000, 0x1655: 0x4000, 0x1656: 0x4000, 0x1657: 0x4000, - 0x1658: 0x4000, 0x1659: 0x4000, 0x165a: 0x4000, 0x165b: 0x4000, 0x165c: 0x4000, 0x165d: 0x4000, - 0x165e: 0x4000, 0x165f: 0x4000, 0x1660: 0x4000, 0x1661: 0x4000, 0x1662: 0x4000, 0x1663: 0x4000, - 0x1664: 0x4000, 0x1665: 0x4000, 0x1666: 0x4000, 0x1667: 0x4000, 0x1668: 0x4000, 0x1669: 0x4000, - 0x166a: 0x4000, 0x166b: 0x4000, 0x166c: 0x4000, 0x166d: 0x4000, 0x166e: 0x4000, 0x166f: 0x4000, - 0x1670: 0x4000, 0x1671: 0x4000, 0x1672: 0x4000, 0x1673: 0x4000, 0x1674: 0x4000, 0x1675: 0x4000, - 0x1676: 0x4000, 0x1677: 0x4000, 0x1678: 0x4000, 0x1679: 0x4000, 0x167a: 0x4000, 0x167b: 0x4000, - 0x167c: 0x4000, 0x167f: 0x4000, - // Block 0x5a, offset 0x1680 - 0x1680: 0x4000, 0x1681: 0x4000, 0x1682: 0x4000, 0x1683: 0x4000, 0x1684: 0x4000, 0x1685: 0x4000, - 0x1686: 0x4000, 0x1687: 0x4000, 0x1688: 0x4000, 0x1689: 0x4000, 0x168a: 0x4000, 0x168b: 0x4000, - 0x168c: 0x4000, 0x168d: 0x4000, 0x168e: 0x4000, 0x168f: 0x4000, 0x1690: 0x4000, 0x1691: 0x4000, - 0x1692: 0x4000, 0x1693: 0x4000, 0x1694: 0x4000, 0x1695: 0x4000, 0x1696: 0x4000, 0x1697: 0x4000, - 0x1698: 0x4000, 0x1699: 0x4000, 0x169a: 0x4000, 0x169b: 0x4000, 0x169c: 0x4000, 0x169d: 0x4000, - 0x169e: 0x4000, 0x169f: 0x4000, 0x16a0: 0x4000, 0x16a1: 0x4000, 0x16a2: 0x4000, 0x16a3: 0x4000, - 0x16a4: 0x4000, 0x16a5: 0x4000, 0x16a6: 0x4000, 0x16a7: 0x4000, 0x16a8: 0x4000, 0x16a9: 0x4000, - 0x16aa: 0x4000, 0x16ab: 0x4000, 0x16ac: 0x4000, 0x16ad: 0x4000, 0x16ae: 0x4000, 0x16af: 0x4000, - 0x16b0: 0x4000, 0x16b1: 0x4000, 0x16b2: 0x4000, 0x16b3: 0x4000, 0x16b4: 0x4000, 0x16b5: 0x4000, - 0x16b6: 0x4000, 0x16b7: 0x4000, 0x16b8: 0x4000, 0x16b9: 0x4000, 0x16ba: 0x4000, 0x16bb: 0x4000, - 0x16bc: 0x4000, 0x16bd: 0x4000, - // Block 0x5b, offset 0x16c0 - 0x16cb: 0x4000, - 0x16cc: 0x4000, 0x16cd: 0x4000, 0x16ce: 0x4000, 0x16d0: 0x4000, 0x16d1: 0x4000, - 0x16d2: 0x4000, 0x16d3: 0x4000, 0x16d4: 0x4000, 0x16d5: 0x4000, 0x16d6: 0x4000, 0x16d7: 0x4000, - 0x16d8: 0x4000, 0x16d9: 0x4000, 0x16da: 0x4000, 0x16db: 0x4000, 0x16dc: 0x4000, 0x16dd: 0x4000, - 0x16de: 0x4000, 0x16df: 0x4000, 0x16e0: 0x4000, 0x16e1: 0x4000, 0x16e2: 0x4000, 0x16e3: 0x4000, - 0x16e4: 0x4000, 0x16e5: 0x4000, 0x16e6: 0x4000, 0x16e7: 0x4000, - 0x16fa: 0x4000, - // Block 0x5c, offset 0x1700 - 0x1715: 0x4000, 0x1716: 0x4000, - 0x1724: 0x4000, - // Block 0x5d, offset 0x1740 - 0x177b: 0x4000, - 0x177c: 0x4000, 0x177d: 0x4000, 0x177e: 0x4000, 0x177f: 0x4000, - // Block 0x5e, offset 0x1780 - 0x1780: 0x4000, 0x1781: 0x4000, 0x1782: 0x4000, 0x1783: 0x4000, 0x1784: 0x4000, 0x1785: 0x4000, - 0x1786: 0x4000, 0x1787: 0x4000, 0x1788: 0x4000, 0x1789: 0x4000, 0x178a: 0x4000, 0x178b: 0x4000, - 0x178c: 0x4000, 0x178d: 0x4000, 0x178e: 0x4000, 0x178f: 0x4000, - // Block 0x5f, offset 0x17c0 - 0x17c0: 0x4000, 0x17c1: 0x4000, 0x17c2: 0x4000, 0x17c3: 0x4000, 0x17c4: 0x4000, 0x17c5: 0x4000, - 0x17cc: 0x4000, 0x17d0: 0x4000, 0x17d1: 0x4000, - 0x17d2: 0x4000, - 0x17eb: 0x4000, 0x17ec: 0x4000, - 0x17f4: 0x4000, 0x17f5: 0x4000, - 0x17f6: 0x4000, 0x17f7: 0x4000, 0x17f8: 0x4000, - // Block 0x60, offset 0x1800 - 0x1810: 0x4000, 0x1811: 0x4000, - 0x1812: 0x4000, 0x1813: 0x4000, 0x1814: 0x4000, 0x1815: 0x4000, 0x1816: 0x4000, 0x1817: 0x4000, - 0x1818: 0x4000, 0x1819: 0x4000, 0x181a: 0x4000, 0x181b: 0x4000, 0x181c: 0x4000, 0x181d: 0x4000, - 0x181e: 0x4000, 0x181f: 0x4000, 0x1820: 0x4000, 0x1821: 0x4000, 0x1822: 0x4000, 0x1823: 0x4000, - 0x1824: 0x4000, 0x1825: 0x4000, 0x1826: 0x4000, 0x1827: 0x4000, 0x1828: 0x4000, 0x1829: 0x4000, - 0x182a: 0x4000, 0x182b: 0x4000, 0x182c: 0x4000, 0x182d: 0x4000, 0x182e: 0x4000, 0x182f: 0x4000, - 0x1830: 0x4000, 0x1831: 0x4000, 0x1832: 0x4000, 0x1833: 0x4000, 0x1834: 0x4000, 0x1835: 0x4000, - 0x1836: 0x4000, 0x1837: 0x4000, 0x1838: 0x4000, 0x1839: 0x4000, 0x183a: 0x4000, 0x183b: 0x4000, - 0x183c: 0x4000, 0x183d: 0x4000, 0x183e: 0x4000, - // Block 0x61, offset 0x1840 - 0x1840: 0x4000, 0x1841: 0x4000, 0x1842: 0x4000, 0x1843: 0x4000, 0x1844: 0x4000, 0x1845: 0x4000, - 0x1846: 0x4000, 0x1847: 0x4000, 0x1848: 0x4000, 0x1849: 0x4000, 0x184a: 0x4000, 0x184b: 0x4000, - 0x184c: 0x4000, 0x1850: 0x4000, 0x1851: 0x4000, - 0x1852: 0x4000, 0x1853: 0x4000, 0x1854: 0x4000, 0x1855: 0x4000, 0x1856: 0x4000, 0x1857: 0x4000, - 0x1858: 0x4000, 0x1859: 0x4000, 0x185a: 0x4000, 0x185b: 0x4000, 0x185c: 0x4000, 0x185d: 0x4000, - 0x185e: 0x4000, 0x185f: 0x4000, 0x1860: 0x4000, 0x1861: 0x4000, 0x1862: 0x4000, 0x1863: 0x4000, - 0x1864: 0x4000, 0x1865: 0x4000, 0x1866: 0x4000, 0x1867: 0x4000, 0x1868: 0x4000, 0x1869: 0x4000, - 0x186a: 0x4000, 0x186b: 0x4000, - // Block 0x62, offset 0x1880 - 0x1880: 0x4000, 0x1881: 0x4000, 0x1882: 0x4000, 0x1883: 0x4000, 0x1884: 0x4000, 0x1885: 0x4000, - 0x1886: 0x4000, 0x1887: 0x4000, 0x1888: 0x4000, 0x1889: 0x4000, 0x188a: 0x4000, 0x188b: 0x4000, - 0x188c: 0x4000, 0x188d: 0x4000, 0x188e: 0x4000, 0x188f: 0x4000, 0x1890: 0x4000, 0x1891: 0x4000, - 0x1892: 0x4000, 0x1893: 0x4000, 0x1894: 0x4000, 0x1895: 0x4000, 0x1896: 0x4000, 0x1897: 0x4000, - // Block 0x63, offset 0x18c0 - 0x18c0: 0x4000, - 0x18d0: 0x4000, 0x18d1: 0x4000, - 0x18d2: 0x4000, 0x18d3: 0x4000, 0x18d4: 0x4000, 0x18d5: 0x4000, 0x18d6: 0x4000, 0x18d7: 0x4000, - 0x18d8: 0x4000, 0x18d9: 0x4000, 0x18da: 0x4000, 0x18db: 0x4000, 0x18dc: 0x4000, 0x18dd: 0x4000, - 0x18de: 0x4000, 0x18df: 0x4000, 0x18e0: 0x4000, 0x18e1: 0x4000, 0x18e2: 0x4000, 0x18e3: 0x4000, - 0x18e4: 0x4000, 0x18e5: 0x4000, 0x18e6: 0x4000, - // Block 0x64, offset 0x1900 - 0x1900: 0x2000, 0x1901: 0x2000, 0x1902: 0x2000, 0x1903: 0x2000, 0x1904: 0x2000, 0x1905: 0x2000, - 0x1906: 0x2000, 0x1907: 0x2000, 0x1908: 0x2000, 0x1909: 0x2000, 0x190a: 0x2000, 0x190b: 0x2000, - 0x190c: 0x2000, 0x190d: 0x2000, 0x190e: 0x2000, 0x190f: 0x2000, 0x1910: 0x2000, 0x1911: 0x2000, - 0x1912: 0x2000, 0x1913: 0x2000, 0x1914: 0x2000, 0x1915: 0x2000, 0x1916: 0x2000, 0x1917: 0x2000, - 0x1918: 0x2000, 0x1919: 0x2000, 0x191a: 0x2000, 0x191b: 0x2000, 0x191c: 0x2000, 0x191d: 0x2000, - 0x191e: 0x2000, 0x191f: 0x2000, 0x1920: 0x2000, 0x1921: 0x2000, 0x1922: 0x2000, 0x1923: 0x2000, - 0x1924: 0x2000, 0x1925: 0x2000, 0x1926: 0x2000, 0x1927: 0x2000, 0x1928: 0x2000, 0x1929: 0x2000, - 0x192a: 0x2000, 0x192b: 0x2000, 0x192c: 0x2000, 0x192d: 0x2000, 0x192e: 0x2000, 0x192f: 0x2000, - 0x1930: 0x2000, 0x1931: 0x2000, 0x1932: 0x2000, 0x1933: 0x2000, 0x1934: 0x2000, 0x1935: 0x2000, - 0x1936: 0x2000, 0x1937: 0x2000, 0x1938: 0x2000, 0x1939: 0x2000, 0x193a: 0x2000, 0x193b: 0x2000, - 0x193c: 0x2000, 0x193d: 0x2000, -} - -// widthIndex: 22 blocks, 1408 entries, 1408 bytes -// Block 0 is the zero block. -var widthIndex = [1408]uint8{ - // Block 0x0, offset 0x0 - // Block 0x1, offset 0x40 - // Block 0x2, offset 0x80 - // Block 0x3, offset 0xc0 - 0xc2: 0x01, 0xc3: 0x02, 0xc4: 0x03, 0xc5: 0x04, 0xc7: 0x05, - 0xc9: 0x06, 0xcb: 0x07, 0xcc: 0x08, 0xcd: 0x09, 0xce: 0x0a, 0xcf: 0x0b, - 0xd0: 0x0c, 0xd1: 0x0d, - 0xe1: 0x02, 0xe2: 0x03, 0xe3: 0x04, 0xe4: 0x05, 0xe5: 0x06, 0xe6: 0x06, 0xe7: 0x06, - 0xe8: 0x06, 0xe9: 0x06, 0xea: 0x07, 0xeb: 0x06, 0xec: 0x06, 0xed: 0x08, 0xee: 0x09, 0xef: 0x0a, - 0xf0: 0x0f, 0xf3: 0x12, 0xf4: 0x13, - // Block 0x4, offset 0x100 - 0x104: 0x0e, 0x105: 0x0f, - // Block 0x5, offset 0x140 - 0x140: 0x10, 0x141: 0x11, 0x142: 0x12, 0x144: 0x13, 0x145: 0x14, 0x146: 0x15, 0x147: 0x16, - 0x148: 0x17, 0x149: 0x18, 0x14a: 0x19, 0x14c: 0x1a, 0x14f: 0x1b, - 0x151: 0x1c, 0x152: 0x08, 0x153: 0x1d, 0x154: 0x1e, 0x155: 0x1f, 0x156: 0x20, 0x157: 0x21, - 0x158: 0x22, 0x159: 0x23, 0x15a: 0x24, 0x15b: 0x25, 0x15c: 0x26, 0x15d: 0x27, 0x15e: 0x28, 0x15f: 0x29, - 0x166: 0x2a, - 0x16c: 0x2b, 0x16d: 0x2c, - 0x17a: 0x2d, 0x17b: 0x2e, 0x17c: 0x0e, 0x17d: 0x0e, 0x17e: 0x0e, 0x17f: 0x2f, - // Block 0x6, offset 0x180 - 0x180: 0x30, 0x181: 0x31, 0x182: 0x32, 0x183: 0x33, 0x184: 0x34, 0x185: 0x35, 0x186: 0x36, 0x187: 0x37, - 0x188: 0x38, 0x189: 0x39, 0x18a: 0x0e, 0x18b: 0x3a, 0x18c: 0x0e, 0x18d: 0x0e, 0x18e: 0x0e, 0x18f: 0x0e, - 0x190: 0x0e, 0x191: 0x0e, 0x192: 0x0e, 0x193: 0x0e, 0x194: 0x0e, 0x195: 0x0e, 0x196: 0x0e, 0x197: 0x0e, - 0x198: 0x0e, 0x199: 0x0e, 0x19a: 0x0e, 0x19b: 0x0e, 0x19c: 0x0e, 0x19d: 0x0e, 0x19e: 0x0e, 0x19f: 0x0e, - 0x1a0: 0x0e, 0x1a1: 0x0e, 0x1a2: 0x0e, 0x1a3: 0x0e, 0x1a4: 0x0e, 0x1a5: 0x0e, 0x1a6: 0x0e, 0x1a7: 0x0e, - 0x1a8: 0x0e, 0x1a9: 0x0e, 0x1aa: 0x0e, 0x1ab: 0x0e, 0x1ac: 0x0e, 0x1ad: 0x0e, 0x1ae: 0x0e, 0x1af: 0x0e, - 0x1b0: 0x0e, 0x1b1: 0x0e, 0x1b2: 0x0e, 0x1b3: 0x0e, 0x1b4: 0x0e, 0x1b5: 0x0e, 0x1b6: 0x0e, 0x1b7: 0x0e, - 0x1b8: 0x0e, 0x1b9: 0x0e, 0x1ba: 0x0e, 0x1bb: 0x0e, 0x1bc: 0x0e, 0x1bd: 0x0e, 0x1be: 0x0e, 0x1bf: 0x0e, - // Block 0x7, offset 0x1c0 - 0x1c0: 0x0e, 0x1c1: 0x0e, 0x1c2: 0x0e, 0x1c3: 0x0e, 0x1c4: 0x0e, 0x1c5: 0x0e, 0x1c6: 0x0e, 0x1c7: 0x0e, - 0x1c8: 0x0e, 0x1c9: 0x0e, 0x1ca: 0x0e, 0x1cb: 0x0e, 0x1cc: 0x0e, 0x1cd: 0x0e, 0x1ce: 0x0e, 0x1cf: 0x0e, - 0x1d0: 0x0e, 0x1d1: 0x0e, 0x1d2: 0x0e, 0x1d3: 0x0e, 0x1d4: 0x0e, 0x1d5: 0x0e, 0x1d6: 0x0e, 0x1d7: 0x0e, - 0x1d8: 0x0e, 0x1d9: 0x0e, 0x1da: 0x0e, 0x1db: 0x0e, 0x1dc: 0x0e, 0x1dd: 0x0e, 0x1de: 0x0e, 0x1df: 0x0e, - 0x1e0: 0x0e, 0x1e1: 0x0e, 0x1e2: 0x0e, 0x1e3: 0x0e, 0x1e4: 0x0e, 0x1e5: 0x0e, 0x1e6: 0x0e, 0x1e7: 0x0e, - 0x1e8: 0x0e, 0x1e9: 0x0e, 0x1ea: 0x0e, 0x1eb: 0x0e, 0x1ec: 0x0e, 0x1ed: 0x0e, 0x1ee: 0x0e, 0x1ef: 0x0e, - 0x1f0: 0x0e, 0x1f1: 0x0e, 0x1f2: 0x0e, 0x1f3: 0x0e, 0x1f4: 0x0e, 0x1f5: 0x0e, 0x1f6: 0x0e, - 0x1f8: 0x0e, 0x1f9: 0x0e, 0x1fa: 0x0e, 0x1fb: 0x0e, 0x1fc: 0x0e, 0x1fd: 0x0e, 0x1fe: 0x0e, 0x1ff: 0x0e, - // Block 0x8, offset 0x200 - 0x200: 0x0e, 0x201: 0x0e, 0x202: 0x0e, 0x203: 0x0e, 0x204: 0x0e, 0x205: 0x0e, 0x206: 0x0e, 0x207: 0x0e, - 0x208: 0x0e, 0x209: 0x0e, 0x20a: 0x0e, 0x20b: 0x0e, 0x20c: 0x0e, 0x20d: 0x0e, 0x20e: 0x0e, 0x20f: 0x0e, - 0x210: 0x0e, 0x211: 0x0e, 0x212: 0x0e, 0x213: 0x0e, 0x214: 0x0e, 0x215: 0x0e, 0x216: 0x0e, 0x217: 0x0e, - 0x218: 0x0e, 0x219: 0x0e, 0x21a: 0x0e, 0x21b: 0x0e, 0x21c: 0x0e, 0x21d: 0x0e, 0x21e: 0x0e, 0x21f: 0x0e, - 0x220: 0x0e, 0x221: 0x0e, 0x222: 0x0e, 0x223: 0x0e, 0x224: 0x0e, 0x225: 0x0e, 0x226: 0x0e, 0x227: 0x0e, - 0x228: 0x0e, 0x229: 0x0e, 0x22a: 0x0e, 0x22b: 0x0e, 0x22c: 0x0e, 0x22d: 0x0e, 0x22e: 0x0e, 0x22f: 0x0e, - 0x230: 0x0e, 0x231: 0x0e, 0x232: 0x0e, 0x233: 0x0e, 0x234: 0x0e, 0x235: 0x0e, 0x236: 0x0e, 0x237: 0x0e, - 0x238: 0x0e, 0x239: 0x0e, 0x23a: 0x0e, 0x23b: 0x0e, 0x23c: 0x0e, 0x23d: 0x0e, 0x23e: 0x0e, 0x23f: 0x0e, - // Block 0x9, offset 0x240 - 0x240: 0x0e, 0x241: 0x0e, 0x242: 0x0e, 0x243: 0x0e, 0x244: 0x0e, 0x245: 0x0e, 0x246: 0x0e, 0x247: 0x0e, - 0x248: 0x0e, 0x249: 0x0e, 0x24a: 0x0e, 0x24b: 0x0e, 0x24c: 0x0e, 0x24d: 0x0e, 0x24e: 0x0e, 0x24f: 0x0e, - 0x250: 0x0e, 0x251: 0x0e, 0x252: 0x3b, 0x253: 0x3c, - 0x265: 0x3d, - 0x270: 0x0e, 0x271: 0x0e, 0x272: 0x0e, 0x273: 0x0e, 0x274: 0x0e, 0x275: 0x0e, 0x276: 0x0e, 0x277: 0x0e, - 0x278: 0x0e, 0x279: 0x0e, 0x27a: 0x0e, 0x27b: 0x0e, 0x27c: 0x0e, 0x27d: 0x0e, 0x27e: 0x0e, 0x27f: 0x0e, - // Block 0xa, offset 0x280 - 0x280: 0x0e, 0x281: 0x0e, 0x282: 0x0e, 0x283: 0x0e, 0x284: 0x0e, 0x285: 0x0e, 0x286: 0x0e, 0x287: 0x0e, - 0x288: 0x0e, 0x289: 0x0e, 0x28a: 0x0e, 0x28b: 0x0e, 0x28c: 0x0e, 0x28d: 0x0e, 0x28e: 0x0e, 0x28f: 0x0e, - 0x290: 0x0e, 0x291: 0x0e, 0x292: 0x0e, 0x293: 0x0e, 0x294: 0x0e, 0x295: 0x0e, 0x296: 0x0e, 0x297: 0x0e, - 0x298: 0x0e, 0x299: 0x0e, 0x29a: 0x0e, 0x29b: 0x0e, 0x29c: 0x0e, 0x29d: 0x0e, 0x29e: 0x3e, - // Block 0xb, offset 0x2c0 - 0x2c0: 0x08, 0x2c1: 0x08, 0x2c2: 0x08, 0x2c3: 0x08, 0x2c4: 0x08, 0x2c5: 0x08, 0x2c6: 0x08, 0x2c7: 0x08, - 0x2c8: 0x08, 0x2c9: 0x08, 0x2ca: 0x08, 0x2cb: 0x08, 0x2cc: 0x08, 0x2cd: 0x08, 0x2ce: 0x08, 0x2cf: 0x08, - 0x2d0: 0x08, 0x2d1: 0x08, 0x2d2: 0x08, 0x2d3: 0x08, 0x2d4: 0x08, 0x2d5: 0x08, 0x2d6: 0x08, 0x2d7: 0x08, - 0x2d8: 0x08, 0x2d9: 0x08, 0x2da: 0x08, 0x2db: 0x08, 0x2dc: 0x08, 0x2dd: 0x08, 0x2de: 0x08, 0x2df: 0x08, - 0x2e0: 0x08, 0x2e1: 0x08, 0x2e2: 0x08, 0x2e3: 0x08, 0x2e4: 0x08, 0x2e5: 0x08, 0x2e6: 0x08, 0x2e7: 0x08, - 0x2e8: 0x08, 0x2e9: 0x08, 0x2ea: 0x08, 0x2eb: 0x08, 0x2ec: 0x08, 0x2ed: 0x08, 0x2ee: 0x08, 0x2ef: 0x08, - 0x2f0: 0x08, 0x2f1: 0x08, 0x2f2: 0x08, 0x2f3: 0x08, 0x2f4: 0x08, 0x2f5: 0x08, 0x2f6: 0x08, 0x2f7: 0x08, - 0x2f8: 0x08, 0x2f9: 0x08, 0x2fa: 0x08, 0x2fb: 0x08, 0x2fc: 0x08, 0x2fd: 0x08, 0x2fe: 0x08, 0x2ff: 0x08, - // Block 0xc, offset 0x300 - 0x300: 0x08, 0x301: 0x08, 0x302: 0x08, 0x303: 0x08, 0x304: 0x08, 0x305: 0x08, 0x306: 0x08, 0x307: 0x08, - 0x308: 0x08, 0x309: 0x08, 0x30a: 0x08, 0x30b: 0x08, 0x30c: 0x08, 0x30d: 0x08, 0x30e: 0x08, 0x30f: 0x08, - 0x310: 0x08, 0x311: 0x08, 0x312: 0x08, 0x313: 0x08, 0x314: 0x08, 0x315: 0x08, 0x316: 0x08, 0x317: 0x08, - 0x318: 0x08, 0x319: 0x08, 0x31a: 0x08, 0x31b: 0x08, 0x31c: 0x08, 0x31d: 0x08, 0x31e: 0x08, 0x31f: 0x08, - 0x320: 0x08, 0x321: 0x08, 0x322: 0x08, 0x323: 0x08, 0x324: 0x0e, 0x325: 0x0e, 0x326: 0x0e, 0x327: 0x0e, - 0x328: 0x0e, 0x329: 0x0e, 0x32a: 0x0e, 0x32b: 0x0e, - 0x338: 0x3f, 0x339: 0x40, 0x33c: 0x41, 0x33d: 0x42, 0x33e: 0x43, 0x33f: 0x44, - // Block 0xd, offset 0x340 - 0x37f: 0x45, - // Block 0xe, offset 0x380 - 0x380: 0x0e, 0x381: 0x0e, 0x382: 0x0e, 0x383: 0x0e, 0x384: 0x0e, 0x385: 0x0e, 0x386: 0x0e, 0x387: 0x0e, - 0x388: 0x0e, 0x389: 0x0e, 0x38a: 0x0e, 0x38b: 0x0e, 0x38c: 0x0e, 0x38d: 0x0e, 0x38e: 0x0e, 0x38f: 0x0e, - 0x390: 0x0e, 0x391: 0x0e, 0x392: 0x0e, 0x393: 0x0e, 0x394: 0x0e, 0x395: 0x0e, 0x396: 0x0e, 0x397: 0x0e, - 0x398: 0x0e, 0x399: 0x0e, 0x39a: 0x0e, 0x39b: 0x0e, 0x39c: 0x0e, 0x39d: 0x0e, 0x39e: 0x0e, 0x39f: 0x46, - 0x3a0: 0x0e, 0x3a1: 0x0e, 0x3a2: 0x0e, 0x3a3: 0x0e, 0x3a4: 0x0e, 0x3a5: 0x0e, 0x3a6: 0x0e, 0x3a7: 0x0e, - 0x3a8: 0x0e, 0x3a9: 0x0e, 0x3aa: 0x0e, 0x3ab: 0x47, - // Block 0xf, offset 0x3c0 - 0x3c0: 0x0e, 0x3c1: 0x0e, 0x3c2: 0x0e, 0x3c3: 0x0e, 0x3c4: 0x48, 0x3c5: 0x49, 0x3c6: 0x0e, 0x3c7: 0x0e, - 0x3c8: 0x0e, 0x3c9: 0x0e, 0x3ca: 0x0e, 0x3cb: 0x4a, - // Block 0x10, offset 0x400 - 0x400: 0x4b, 0x403: 0x4c, 0x404: 0x4d, 0x405: 0x4e, 0x406: 0x4f, - 0x408: 0x50, 0x409: 0x51, 0x40c: 0x52, 0x40d: 0x53, 0x40e: 0x54, 0x40f: 0x55, - 0x410: 0x3a, 0x411: 0x56, 0x412: 0x0e, 0x413: 0x57, 0x414: 0x58, 0x415: 0x59, 0x416: 0x5a, 0x417: 0x5b, - 0x418: 0x0e, 0x419: 0x5c, 0x41a: 0x0e, 0x41b: 0x5d, - 0x424: 0x5e, 0x425: 0x5f, 0x426: 0x60, 0x427: 0x61, - // Block 0x11, offset 0x440 - 0x456: 0x0b, 0x457: 0x06, - 0x458: 0x0c, 0x45b: 0x0d, 0x45f: 0x0e, - 0x460: 0x06, 0x461: 0x06, 0x462: 0x06, 0x463: 0x06, 0x464: 0x06, 0x465: 0x06, 0x466: 0x06, 0x467: 0x06, - 0x468: 0x06, 0x469: 0x06, 0x46a: 0x06, 0x46b: 0x06, 0x46c: 0x06, 0x46d: 0x06, 0x46e: 0x06, 0x46f: 0x06, - 0x470: 0x06, 0x471: 0x06, 0x472: 0x06, 0x473: 0x06, 0x474: 0x06, 0x475: 0x06, 0x476: 0x06, 0x477: 0x06, - 0x478: 0x06, 0x479: 0x06, 0x47a: 0x06, 0x47b: 0x06, 0x47c: 0x06, 0x47d: 0x06, 0x47e: 0x06, 0x47f: 0x06, - // Block 0x12, offset 0x480 - 0x484: 0x08, 0x485: 0x08, 0x486: 0x08, 0x487: 0x09, - // Block 0x13, offset 0x4c0 - 0x4c0: 0x08, 0x4c1: 0x08, 0x4c2: 0x08, 0x4c3: 0x08, 0x4c4: 0x08, 0x4c5: 0x08, 0x4c6: 0x08, 0x4c7: 0x08, - 0x4c8: 0x08, 0x4c9: 0x08, 0x4ca: 0x08, 0x4cb: 0x08, 0x4cc: 0x08, 0x4cd: 0x08, 0x4ce: 0x08, 0x4cf: 0x08, - 0x4d0: 0x08, 0x4d1: 0x08, 0x4d2: 0x08, 0x4d3: 0x08, 0x4d4: 0x08, 0x4d5: 0x08, 0x4d6: 0x08, 0x4d7: 0x08, - 0x4d8: 0x08, 0x4d9: 0x08, 0x4da: 0x08, 0x4db: 0x08, 0x4dc: 0x08, 0x4dd: 0x08, 0x4de: 0x08, 0x4df: 0x08, - 0x4e0: 0x08, 0x4e1: 0x08, 0x4e2: 0x08, 0x4e3: 0x08, 0x4e4: 0x08, 0x4e5: 0x08, 0x4e6: 0x08, 0x4e7: 0x08, - 0x4e8: 0x08, 0x4e9: 0x08, 0x4ea: 0x08, 0x4eb: 0x08, 0x4ec: 0x08, 0x4ed: 0x08, 0x4ee: 0x08, 0x4ef: 0x08, - 0x4f0: 0x08, 0x4f1: 0x08, 0x4f2: 0x08, 0x4f3: 0x08, 0x4f4: 0x08, 0x4f5: 0x08, 0x4f6: 0x08, 0x4f7: 0x08, - 0x4f8: 0x08, 0x4f9: 0x08, 0x4fa: 0x08, 0x4fb: 0x08, 0x4fc: 0x08, 0x4fd: 0x08, 0x4fe: 0x08, 0x4ff: 0x62, - // Block 0x14, offset 0x500 - 0x520: 0x10, - 0x530: 0x09, 0x531: 0x09, 0x532: 0x09, 0x533: 0x09, 0x534: 0x09, 0x535: 0x09, 0x536: 0x09, 0x537: 0x09, - 0x538: 0x09, 0x539: 0x09, 0x53a: 0x09, 0x53b: 0x09, 0x53c: 0x09, 0x53d: 0x09, 0x53e: 0x09, 0x53f: 0x11, - // Block 0x15, offset 0x540 - 0x540: 0x09, 0x541: 0x09, 0x542: 0x09, 0x543: 0x09, 0x544: 0x09, 0x545: 0x09, 0x546: 0x09, 0x547: 0x09, - 0x548: 0x09, 0x549: 0x09, 0x54a: 0x09, 0x54b: 0x09, 0x54c: 0x09, 0x54d: 0x09, 0x54e: 0x09, 0x54f: 0x11, -} - -// inverseData contains 4-byte entries of the following format: -// -// <0 padding> -// -// The last byte of the UTF-8-encoded rune is xor-ed with the last byte of the -// UTF-8 encoding of the original rune. Mappings often have the following -// pattern: -// -// Ａ -> A (U+FF21 -> U+0041) -// Ｂ -> B (U+FF22 -> U+0042) -// ... -// -// By xor-ing the last byte the same entry can be shared by many mappings. This -// reduces the total number of distinct entries by about two thirds. -// The resulting entry for the aforementioned mappings is -// -// { 0x01, 0xE0, 0x00, 0x00 } -// -// Using this entry to map U+FF21 (UTF-8 [EF BC A1]), we get -// -// E0 ^ A1 = 41. -// -// Similarly, for U+FF22 (UTF-8 [EF BC A2]), we get -// -// E0 ^ A2 = 42. -// -// Note that because of the xor-ing, the byte sequence stored in the entry is -// not valid UTF-8. -var inverseData = [150][4]byte{ - {0x00, 0x00, 0x00, 0x00}, - {0x03, 0xe3, 0x80, 0xa0}, - {0x03, 0xef, 0xbc, 0xa0}, - {0x03, 0xef, 0xbc, 0xe0}, - {0x03, 0xef, 0xbd, 0xe0}, - {0x03, 0xef, 0xbf, 0x02}, - {0x03, 0xef, 0xbf, 0x00}, - {0x03, 0xef, 0xbf, 0x0e}, - {0x03, 0xef, 0xbf, 0x0c}, - {0x03, 0xef, 0xbf, 0x0f}, - {0x03, 0xef, 0xbf, 0x39}, - {0x03, 0xef, 0xbf, 0x3b}, - {0x03, 0xef, 0xbf, 0x3f}, - {0x03, 0xef, 0xbf, 0x2a}, - {0x03, 0xef, 0xbf, 0x0d}, - {0x03, 0xef, 0xbf, 0x25}, - {0x03, 0xef, 0xbd, 0x1a}, - {0x03, 0xef, 0xbd, 0x26}, - {0x01, 0xa0, 0x00, 0x00}, - {0x03, 0xef, 0xbd, 0x25}, - {0x03, 0xef, 0xbd, 0x23}, - {0x03, 0xef, 0xbd, 0x2e}, - {0x03, 0xef, 0xbe, 0x07}, - {0x03, 0xef, 0xbe, 0x05}, - {0x03, 0xef, 0xbd, 0x06}, - {0x03, 0xef, 0xbd, 0x13}, - {0x03, 0xef, 0xbd, 0x0b}, - {0x03, 0xef, 0xbd, 0x16}, - {0x03, 0xef, 0xbd, 0x0c}, - {0x03, 0xef, 0xbd, 0x15}, - {0x03, 0xef, 0xbd, 0x0d}, - {0x03, 0xef, 0xbd, 0x1c}, - {0x03, 0xef, 0xbd, 0x02}, - {0x03, 0xef, 0xbd, 0x1f}, - {0x03, 0xef, 0xbd, 0x1d}, - {0x03, 0xef, 0xbd, 0x17}, - {0x03, 0xef, 0xbd, 0x08}, - {0x03, 0xef, 0xbd, 0x09}, - {0x03, 0xef, 0xbd, 0x0e}, - {0x03, 0xef, 0xbd, 0x04}, - {0x03, 0xef, 0xbd, 0x05}, - {0x03, 0xef, 0xbe, 0x3f}, - {0x03, 0xef, 0xbe, 0x00}, - {0x03, 0xef, 0xbd, 0x2c}, - {0x03, 0xef, 0xbe, 0x06}, - {0x03, 0xef, 0xbe, 0x0c}, - {0x03, 0xef, 0xbe, 0x0f}, - {0x03, 0xef, 0xbe, 0x0d}, - {0x03, 0xef, 0xbe, 0x0b}, - {0x03, 0xef, 0xbe, 0x19}, - {0x03, 0xef, 0xbe, 0x15}, - {0x03, 0xef, 0xbe, 0x11}, - {0x03, 0xef, 0xbe, 0x31}, - {0x03, 0xef, 0xbe, 0x33}, - {0x03, 0xef, 0xbd, 0x0f}, - {0x03, 0xef, 0xbe, 0x30}, - {0x03, 0xef, 0xbe, 0x3e}, - {0x03, 0xef, 0xbe, 0x32}, - {0x03, 0xef, 0xbe, 0x36}, - {0x03, 0xef, 0xbd, 0x14}, - {0x03, 0xef, 0xbe, 0x2e}, - {0x03, 0xef, 0xbd, 0x1e}, - {0x03, 0xef, 0xbe, 0x10}, - {0x03, 0xef, 0xbf, 0x13}, - {0x03, 0xef, 0xbf, 0x15}, - {0x03, 0xef, 0xbf, 0x17}, - {0x03, 0xef, 0xbf, 0x1f}, - {0x03, 0xef, 0xbf, 0x1d}, - {0x03, 0xef, 0xbf, 0x1b}, - {0x03, 0xef, 0xbf, 0x09}, - {0x03, 0xef, 0xbf, 0x0b}, - {0x03, 0xef, 0xbf, 0x37}, - {0x03, 0xef, 0xbe, 0x04}, - {0x01, 0xe0, 0x00, 0x00}, - {0x03, 0xe2, 0xa6, 0x1a}, - {0x03, 0xe2, 0xa6, 0x26}, - {0x03, 0xe3, 0x80, 0x23}, - {0x03, 0xe3, 0x80, 0x2e}, - {0x03, 0xe3, 0x80, 0x25}, - {0x03, 0xe3, 0x83, 0x1e}, - {0x03, 0xe3, 0x83, 0x14}, - {0x03, 0xe3, 0x82, 0x06}, - {0x03, 0xe3, 0x82, 0x0b}, - {0x03, 0xe3, 0x82, 0x0c}, - {0x03, 0xe3, 0x82, 0x0d}, - {0x03, 0xe3, 0x82, 0x02}, - {0x03, 0xe3, 0x83, 0x0f}, - {0x03, 0xe3, 0x83, 0x08}, - {0x03, 0xe3, 0x83, 0x09}, - {0x03, 0xe3, 0x83, 0x2c}, - {0x03, 0xe3, 0x83, 0x0c}, - {0x03, 0xe3, 0x82, 0x13}, - {0x03, 0xe3, 0x82, 0x16}, - {0x03, 0xe3, 0x82, 0x15}, - {0x03, 0xe3, 0x82, 0x1c}, - {0x03, 0xe3, 0x82, 0x1f}, - {0x03, 0xe3, 0x82, 0x1d}, - {0x03, 0xe3, 0x82, 0x1a}, - {0x03, 0xe3, 0x82, 0x17}, - {0x03, 0xe3, 0x82, 0x08}, - {0x03, 0xe3, 0x82, 0x09}, - {0x03, 0xe3, 0x82, 0x0e}, - {0x03, 0xe3, 0x82, 0x04}, - {0x03, 0xe3, 0x82, 0x05}, - {0x03, 0xe3, 0x82, 0x3f}, - {0x03, 0xe3, 0x83, 0x00}, - {0x03, 0xe3, 0x83, 0x06}, - {0x03, 0xe3, 0x83, 0x05}, - {0x03, 0xe3, 0x83, 0x0d}, - {0x03, 0xe3, 0x83, 0x0b}, - {0x03, 0xe3, 0x83, 0x07}, - {0x03, 0xe3, 0x83, 0x19}, - {0x03, 0xe3, 0x83, 0x15}, - {0x03, 0xe3, 0x83, 0x11}, - {0x03, 0xe3, 0x83, 0x31}, - {0x03, 0xe3, 0x83, 0x33}, - {0x03, 0xe3, 0x83, 0x30}, - {0x03, 0xe3, 0x83, 0x3e}, - {0x03, 0xe3, 0x83, 0x32}, - {0x03, 0xe3, 0x83, 0x36}, - {0x03, 0xe3, 0x83, 0x2e}, - {0x03, 0xe3, 0x82, 0x07}, - {0x03, 0xe3, 0x85, 0x04}, - {0x03, 0xe3, 0x84, 0x10}, - {0x03, 0xe3, 0x85, 0x30}, - {0x03, 0xe3, 0x85, 0x0d}, - {0x03, 0xe3, 0x85, 0x13}, - {0x03, 0xe3, 0x85, 0x15}, - {0x03, 0xe3, 0x85, 0x17}, - {0x03, 0xe3, 0x85, 0x1f}, - {0x03, 0xe3, 0x85, 0x1d}, - {0x03, 0xe3, 0x85, 0x1b}, - {0x03, 0xe3, 0x85, 0x09}, - {0x03, 0xe3, 0x85, 0x0f}, - {0x03, 0xe3, 0x85, 0x0b}, - {0x03, 0xe3, 0x85, 0x37}, - {0x03, 0xe3, 0x85, 0x3b}, - {0x03, 0xe3, 0x85, 0x39}, - {0x03, 0xe3, 0x85, 0x3f}, - {0x02, 0xc2, 0x02, 0x00}, - {0x02, 0xc2, 0x0e, 0x00}, - {0x02, 0xc2, 0x0c, 0x00}, - {0x02, 0xc2, 0x00, 0x00}, - {0x03, 0xe2, 0x82, 0x0f}, - {0x03, 0xe2, 0x94, 0x2a}, - {0x03, 0xe2, 0x86, 0x39}, - {0x03, 0xe2, 0x86, 0x3b}, - {0x03, 0xe2, 0x86, 0x3f}, - {0x03, 0xe2, 0x96, 0x0d}, - {0x03, 0xe2, 0x97, 0x25}, -} - -// Total table size 14936 bytes (14KiB) diff --git a/vendor/golang.org/x/text/width/tables11.0.0.go b/vendor/golang.org/x/text/width/tables11.0.0.go deleted file mode 100644 index 327eaef9b7..0000000000 --- a/vendor/golang.org/x/text/width/tables11.0.0.go +++ /dev/null @@ -1,1341 +0,0 @@ -// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. - -//go:build go1.13 && !go1.14 -// +build go1.13,!go1.14 - -package width - -// UnicodeVersion is the Unicode version from which the tables in this package are derived. -const UnicodeVersion = "11.0.0" - -// lookup returns the trie value for the first UTF-8 encoding in s and -// the width in bytes of this encoding. The size will be 0 if s does not -// hold enough bytes to complete the encoding. len(s) must be greater than 0. -func (t *widthTrie) lookup(s []byte) (v uint16, sz int) { - c0 := s[0] - switch { - case c0 < 0x80: // is ASCII - return widthValues[c0], 1 - case c0 < 0xC2: - return 0, 1 // Illegal UTF-8: not a starter, not ASCII. - case c0 < 0xE0: // 2-byte UTF-8 - if len(s) < 2 { - return 0, 0 - } - i := widthIndex[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c1), 2 - case c0 < 0xF0: // 3-byte UTF-8 - if len(s) < 3 { - return 0, 0 - } - i := widthIndex[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - o := uint32(i)<<6 + uint32(c1) - i = widthIndex[o] - c2 := s[2] - if c2 < 0x80 || 0xC0 <= c2 { - return 0, 2 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c2), 3 - case c0 < 0xF8: // 4-byte UTF-8 - if len(s) < 4 { - return 0, 0 - } - i := widthIndex[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - o := uint32(i)<<6 + uint32(c1) - i = widthIndex[o] - c2 := s[2] - if c2 < 0x80 || 0xC0 <= c2 { - return 0, 2 // Illegal UTF-8: not a continuation byte. - } - o = uint32(i)<<6 + uint32(c2) - i = widthIndex[o] - c3 := s[3] - if c3 < 0x80 || 0xC0 <= c3 { - return 0, 3 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c3), 4 - } - // Illegal rune - return 0, 1 -} - -// lookupUnsafe returns the trie value for the first UTF-8 encoding in s. -// s must start with a full and valid UTF-8 encoded rune. -func (t *widthTrie) lookupUnsafe(s []byte) uint16 { - c0 := s[0] - if c0 < 0x80 { // is ASCII - return widthValues[c0] - } - i := widthIndex[c0] - if c0 < 0xE0 { // 2-byte UTF-8 - return t.lookupValue(uint32(i), s[1]) - } - i = widthIndex[uint32(i)<<6+uint32(s[1])] - if c0 < 0xF0 { // 3-byte UTF-8 - return t.lookupValue(uint32(i), s[2]) - } - i = widthIndex[uint32(i)<<6+uint32(s[2])] - if c0 < 0xF8 { // 4-byte UTF-8 - return t.lookupValue(uint32(i), s[3]) - } - return 0 -} - -// lookupString returns the trie value for the first UTF-8 encoding in s and -// the width in bytes of this encoding. The size will be 0 if s does not -// hold enough bytes to complete the encoding. len(s) must be greater than 0. -func (t *widthTrie) lookupString(s string) (v uint16, sz int) { - c0 := s[0] - switch { - case c0 < 0x80: // is ASCII - return widthValues[c0], 1 - case c0 < 0xC2: - return 0, 1 // Illegal UTF-8: not a starter, not ASCII. - case c0 < 0xE0: // 2-byte UTF-8 - if len(s) < 2 { - return 0, 0 - } - i := widthIndex[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c1), 2 - case c0 < 0xF0: // 3-byte UTF-8 - if len(s) < 3 { - return 0, 0 - } - i := widthIndex[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - o := uint32(i)<<6 + uint32(c1) - i = widthIndex[o] - c2 := s[2] - if c2 < 0x80 || 0xC0 <= c2 { - return 0, 2 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c2), 3 - case c0 < 0xF8: // 4-byte UTF-8 - if len(s) < 4 { - return 0, 0 - } - i := widthIndex[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - o := uint32(i)<<6 + uint32(c1) - i = widthIndex[o] - c2 := s[2] - if c2 < 0x80 || 0xC0 <= c2 { - return 0, 2 // Illegal UTF-8: not a continuation byte. - } - o = uint32(i)<<6 + uint32(c2) - i = widthIndex[o] - c3 := s[3] - if c3 < 0x80 || 0xC0 <= c3 { - return 0, 3 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c3), 4 - } - // Illegal rune - return 0, 1 -} - -// lookupStringUnsafe returns the trie value for the first UTF-8 encoding in s. -// s must start with a full and valid UTF-8 encoded rune. -func (t *widthTrie) lookupStringUnsafe(s string) uint16 { - c0 := s[0] - if c0 < 0x80 { // is ASCII - return widthValues[c0] - } - i := widthIndex[c0] - if c0 < 0xE0 { // 2-byte UTF-8 - return t.lookupValue(uint32(i), s[1]) - } - i = widthIndex[uint32(i)<<6+uint32(s[1])] - if c0 < 0xF0 { // 3-byte UTF-8 - return t.lookupValue(uint32(i), s[2]) - } - i = widthIndex[uint32(i)<<6+uint32(s[2])] - if c0 < 0xF8 { // 4-byte UTF-8 - return t.lookupValue(uint32(i), s[3]) - } - return 0 -} - -// widthTrie. Total size: 14336 bytes (14.00 KiB). Checksum: c0f7712776e71cd4. -type widthTrie struct{} - -func newWidthTrie(i int) *widthTrie { - return &widthTrie{} -} - -// lookupValue determines the type of block n and looks up the value for b. -func (t *widthTrie) lookupValue(n uint32, b byte) uint16 { - switch { - default: - return uint16(widthValues[n<<6+uint32(b)]) - } -} - -// widthValues: 101 blocks, 6464 entries, 12928 bytes -// The third block is the zero block. -var widthValues = [6464]uint16{ - // Block 0x0, offset 0x0 - 0x20: 0x6001, 0x21: 0x6002, 0x22: 0x6002, 0x23: 0x6002, - 0x24: 0x6002, 0x25: 0x6002, 0x26: 0x6002, 0x27: 0x6002, 0x28: 0x6002, 0x29: 0x6002, - 0x2a: 0x6002, 0x2b: 0x6002, 0x2c: 0x6002, 0x2d: 0x6002, 0x2e: 0x6002, 0x2f: 0x6002, - 0x30: 0x6002, 0x31: 0x6002, 0x32: 0x6002, 0x33: 0x6002, 0x34: 0x6002, 0x35: 0x6002, - 0x36: 0x6002, 0x37: 0x6002, 0x38: 0x6002, 0x39: 0x6002, 0x3a: 0x6002, 0x3b: 0x6002, - 0x3c: 0x6002, 0x3d: 0x6002, 0x3e: 0x6002, 0x3f: 0x6002, - // Block 0x1, offset 0x40 - 0x40: 0x6003, 0x41: 0x6003, 0x42: 0x6003, 0x43: 0x6003, 0x44: 0x6003, 0x45: 0x6003, - 0x46: 0x6003, 0x47: 0x6003, 0x48: 0x6003, 0x49: 0x6003, 0x4a: 0x6003, 0x4b: 0x6003, - 0x4c: 0x6003, 0x4d: 0x6003, 0x4e: 0x6003, 0x4f: 0x6003, 0x50: 0x6003, 0x51: 0x6003, - 0x52: 0x6003, 0x53: 0x6003, 0x54: 0x6003, 0x55: 0x6003, 0x56: 0x6003, 0x57: 0x6003, - 0x58: 0x6003, 0x59: 0x6003, 0x5a: 0x6003, 0x5b: 0x6003, 0x5c: 0x6003, 0x5d: 0x6003, - 0x5e: 0x6003, 0x5f: 0x6003, 0x60: 0x6004, 0x61: 0x6004, 0x62: 0x6004, 0x63: 0x6004, - 0x64: 0x6004, 0x65: 0x6004, 0x66: 0x6004, 0x67: 0x6004, 0x68: 0x6004, 0x69: 0x6004, - 0x6a: 0x6004, 0x6b: 0x6004, 0x6c: 0x6004, 0x6d: 0x6004, 0x6e: 0x6004, 0x6f: 0x6004, - 0x70: 0x6004, 0x71: 0x6004, 0x72: 0x6004, 0x73: 0x6004, 0x74: 0x6004, 0x75: 0x6004, - 0x76: 0x6004, 0x77: 0x6004, 0x78: 0x6004, 0x79: 0x6004, 0x7a: 0x6004, 0x7b: 0x6004, - 0x7c: 0x6004, 0x7d: 0x6004, 0x7e: 0x6004, - // Block 0x2, offset 0x80 - // Block 0x3, offset 0xc0 - 0xe1: 0x2000, 0xe2: 0x6005, 0xe3: 0x6005, - 0xe4: 0x2000, 0xe5: 0x6006, 0xe6: 0x6005, 0xe7: 0x2000, 0xe8: 0x2000, - 0xea: 0x2000, 0xec: 0x6007, 0xed: 0x2000, 0xee: 0x2000, 0xef: 0x6008, - 0xf0: 0x2000, 0xf1: 0x2000, 0xf2: 0x2000, 0xf3: 0x2000, 0xf4: 0x2000, - 0xf6: 0x2000, 0xf7: 0x2000, 0xf8: 0x2000, 0xf9: 0x2000, 0xfa: 0x2000, - 0xfc: 0x2000, 0xfd: 0x2000, 0xfe: 0x2000, 0xff: 0x2000, - // Block 0x4, offset 0x100 - 0x106: 0x2000, - 0x110: 0x2000, - 0x117: 0x2000, - 0x118: 0x2000, - 0x11e: 0x2000, 0x11f: 0x2000, 0x120: 0x2000, 0x121: 0x2000, - 0x126: 0x2000, 0x128: 0x2000, 0x129: 0x2000, - 0x12a: 0x2000, 0x12c: 0x2000, 0x12d: 0x2000, - 0x130: 0x2000, 0x132: 0x2000, 0x133: 0x2000, - 0x137: 0x2000, 0x138: 0x2000, 0x139: 0x2000, 0x13a: 0x2000, - 0x13c: 0x2000, 0x13e: 0x2000, - // Block 0x5, offset 0x140 - 0x141: 0x2000, - 0x151: 0x2000, - 0x153: 0x2000, - 0x15b: 0x2000, - 0x166: 0x2000, 0x167: 0x2000, - 0x16b: 0x2000, - 0x171: 0x2000, 0x172: 0x2000, 0x173: 0x2000, - 0x178: 0x2000, - 0x17f: 0x2000, - // Block 0x6, offset 0x180 - 0x180: 0x2000, 0x181: 0x2000, 0x182: 0x2000, 0x184: 0x2000, - 0x188: 0x2000, 0x189: 0x2000, 0x18a: 0x2000, 0x18b: 0x2000, - 0x18d: 0x2000, - 0x192: 0x2000, 0x193: 0x2000, - 0x1a6: 0x2000, 0x1a7: 0x2000, - 0x1ab: 0x2000, - // Block 0x7, offset 0x1c0 - 0x1ce: 0x2000, 0x1d0: 0x2000, - 0x1d2: 0x2000, 0x1d4: 0x2000, 0x1d6: 0x2000, - 0x1d8: 0x2000, 0x1da: 0x2000, 0x1dc: 0x2000, - // Block 0x8, offset 0x200 - 0x211: 0x2000, - 0x221: 0x2000, - // Block 0x9, offset 0x240 - 0x244: 0x2000, - 0x247: 0x2000, 0x249: 0x2000, 0x24a: 0x2000, 0x24b: 0x2000, - 0x24d: 0x2000, 0x250: 0x2000, - 0x258: 0x2000, 0x259: 0x2000, 0x25a: 0x2000, 0x25b: 0x2000, 0x25d: 0x2000, - 0x25f: 0x2000, - // Block 0xa, offset 0x280 - 0x280: 0x2000, 0x281: 0x2000, 0x282: 0x2000, 0x283: 0x2000, 0x284: 0x2000, 0x285: 0x2000, - 0x286: 0x2000, 0x287: 0x2000, 0x288: 0x2000, 0x289: 0x2000, 0x28a: 0x2000, 0x28b: 0x2000, - 0x28c: 0x2000, 0x28d: 0x2000, 0x28e: 0x2000, 0x28f: 0x2000, 0x290: 0x2000, 0x291: 0x2000, - 0x292: 0x2000, 0x293: 0x2000, 0x294: 0x2000, 0x295: 0x2000, 0x296: 0x2000, 0x297: 0x2000, - 0x298: 0x2000, 0x299: 0x2000, 0x29a: 0x2000, 0x29b: 0x2000, 0x29c: 0x2000, 0x29d: 0x2000, - 0x29e: 0x2000, 0x29f: 0x2000, 0x2a0: 0x2000, 0x2a1: 0x2000, 0x2a2: 0x2000, 0x2a3: 0x2000, - 0x2a4: 0x2000, 0x2a5: 0x2000, 0x2a6: 0x2000, 0x2a7: 0x2000, 0x2a8: 0x2000, 0x2a9: 0x2000, - 0x2aa: 0x2000, 0x2ab: 0x2000, 0x2ac: 0x2000, 0x2ad: 0x2000, 0x2ae: 0x2000, 0x2af: 0x2000, - 0x2b0: 0x2000, 0x2b1: 0x2000, 0x2b2: 0x2000, 0x2b3: 0x2000, 0x2b4: 0x2000, 0x2b5: 0x2000, - 0x2b6: 0x2000, 0x2b7: 0x2000, 0x2b8: 0x2000, 0x2b9: 0x2000, 0x2ba: 0x2000, 0x2bb: 0x2000, - 0x2bc: 0x2000, 0x2bd: 0x2000, 0x2be: 0x2000, 0x2bf: 0x2000, - // Block 0xb, offset 0x2c0 - 0x2c0: 0x2000, 0x2c1: 0x2000, 0x2c2: 0x2000, 0x2c3: 0x2000, 0x2c4: 0x2000, 0x2c5: 0x2000, - 0x2c6: 0x2000, 0x2c7: 0x2000, 0x2c8: 0x2000, 0x2c9: 0x2000, 0x2ca: 0x2000, 0x2cb: 0x2000, - 0x2cc: 0x2000, 0x2cd: 0x2000, 0x2ce: 0x2000, 0x2cf: 0x2000, 0x2d0: 0x2000, 0x2d1: 0x2000, - 0x2d2: 0x2000, 0x2d3: 0x2000, 0x2d4: 0x2000, 0x2d5: 0x2000, 0x2d6: 0x2000, 0x2d7: 0x2000, - 0x2d8: 0x2000, 0x2d9: 0x2000, 0x2da: 0x2000, 0x2db: 0x2000, 0x2dc: 0x2000, 0x2dd: 0x2000, - 0x2de: 0x2000, 0x2df: 0x2000, 0x2e0: 0x2000, 0x2e1: 0x2000, 0x2e2: 0x2000, 0x2e3: 0x2000, - 0x2e4: 0x2000, 0x2e5: 0x2000, 0x2e6: 0x2000, 0x2e7: 0x2000, 0x2e8: 0x2000, 0x2e9: 0x2000, - 0x2ea: 0x2000, 0x2eb: 0x2000, 0x2ec: 0x2000, 0x2ed: 0x2000, 0x2ee: 0x2000, 0x2ef: 0x2000, - // Block 0xc, offset 0x300 - 0x311: 0x2000, - 0x312: 0x2000, 0x313: 0x2000, 0x314: 0x2000, 0x315: 0x2000, 0x316: 0x2000, 0x317: 0x2000, - 0x318: 0x2000, 0x319: 0x2000, 0x31a: 0x2000, 0x31b: 0x2000, 0x31c: 0x2000, 0x31d: 0x2000, - 0x31e: 0x2000, 0x31f: 0x2000, 0x320: 0x2000, 0x321: 0x2000, 0x323: 0x2000, - 0x324: 0x2000, 0x325: 0x2000, 0x326: 0x2000, 0x327: 0x2000, 0x328: 0x2000, 0x329: 0x2000, - 0x331: 0x2000, 0x332: 0x2000, 0x333: 0x2000, 0x334: 0x2000, 0x335: 0x2000, - 0x336: 0x2000, 0x337: 0x2000, 0x338: 0x2000, 0x339: 0x2000, 0x33a: 0x2000, 0x33b: 0x2000, - 0x33c: 0x2000, 0x33d: 0x2000, 0x33e: 0x2000, 0x33f: 0x2000, - // Block 0xd, offset 0x340 - 0x340: 0x2000, 0x341: 0x2000, 0x343: 0x2000, 0x344: 0x2000, 0x345: 0x2000, - 0x346: 0x2000, 0x347: 0x2000, 0x348: 0x2000, 0x349: 0x2000, - // Block 0xe, offset 0x380 - 0x381: 0x2000, - 0x390: 0x2000, 0x391: 0x2000, - 0x392: 0x2000, 0x393: 0x2000, 0x394: 0x2000, 0x395: 0x2000, 0x396: 0x2000, 0x397: 0x2000, - 0x398: 0x2000, 0x399: 0x2000, 0x39a: 0x2000, 0x39b: 0x2000, 0x39c: 0x2000, 0x39d: 0x2000, - 0x39e: 0x2000, 0x39f: 0x2000, 0x3a0: 0x2000, 0x3a1: 0x2000, 0x3a2: 0x2000, 0x3a3: 0x2000, - 0x3a4: 0x2000, 0x3a5: 0x2000, 0x3a6: 0x2000, 0x3a7: 0x2000, 0x3a8: 0x2000, 0x3a9: 0x2000, - 0x3aa: 0x2000, 0x3ab: 0x2000, 0x3ac: 0x2000, 0x3ad: 0x2000, 0x3ae: 0x2000, 0x3af: 0x2000, - 0x3b0: 0x2000, 0x3b1: 0x2000, 0x3b2: 0x2000, 0x3b3: 0x2000, 0x3b4: 0x2000, 0x3b5: 0x2000, - 0x3b6: 0x2000, 0x3b7: 0x2000, 0x3b8: 0x2000, 0x3b9: 0x2000, 0x3ba: 0x2000, 0x3bb: 0x2000, - 0x3bc: 0x2000, 0x3bd: 0x2000, 0x3be: 0x2000, 0x3bf: 0x2000, - // Block 0xf, offset 0x3c0 - 0x3c0: 0x2000, 0x3c1: 0x2000, 0x3c2: 0x2000, 0x3c3: 0x2000, 0x3c4: 0x2000, 0x3c5: 0x2000, - 0x3c6: 0x2000, 0x3c7: 0x2000, 0x3c8: 0x2000, 0x3c9: 0x2000, 0x3ca: 0x2000, 0x3cb: 0x2000, - 0x3cc: 0x2000, 0x3cd: 0x2000, 0x3ce: 0x2000, 0x3cf: 0x2000, 0x3d1: 0x2000, - // Block 0x10, offset 0x400 - 0x400: 0x4000, 0x401: 0x4000, 0x402: 0x4000, 0x403: 0x4000, 0x404: 0x4000, 0x405: 0x4000, - 0x406: 0x4000, 0x407: 0x4000, 0x408: 0x4000, 0x409: 0x4000, 0x40a: 0x4000, 0x40b: 0x4000, - 0x40c: 0x4000, 0x40d: 0x4000, 0x40e: 0x4000, 0x40f: 0x4000, 0x410: 0x4000, 0x411: 0x4000, - 0x412: 0x4000, 0x413: 0x4000, 0x414: 0x4000, 0x415: 0x4000, 0x416: 0x4000, 0x417: 0x4000, - 0x418: 0x4000, 0x419: 0x4000, 0x41a: 0x4000, 0x41b: 0x4000, 0x41c: 0x4000, 0x41d: 0x4000, - 0x41e: 0x4000, 0x41f: 0x4000, 0x420: 0x4000, 0x421: 0x4000, 0x422: 0x4000, 0x423: 0x4000, - 0x424: 0x4000, 0x425: 0x4000, 0x426: 0x4000, 0x427: 0x4000, 0x428: 0x4000, 0x429: 0x4000, - 0x42a: 0x4000, 0x42b: 0x4000, 0x42c: 0x4000, 0x42d: 0x4000, 0x42e: 0x4000, 0x42f: 0x4000, - 0x430: 0x4000, 0x431: 0x4000, 0x432: 0x4000, 0x433: 0x4000, 0x434: 0x4000, 0x435: 0x4000, - 0x436: 0x4000, 0x437: 0x4000, 0x438: 0x4000, 0x439: 0x4000, 0x43a: 0x4000, 0x43b: 0x4000, - 0x43c: 0x4000, 0x43d: 0x4000, 0x43e: 0x4000, 0x43f: 0x4000, - // Block 0x11, offset 0x440 - 0x440: 0x4000, 0x441: 0x4000, 0x442: 0x4000, 0x443: 0x4000, 0x444: 0x4000, 0x445: 0x4000, - 0x446: 0x4000, 0x447: 0x4000, 0x448: 0x4000, 0x449: 0x4000, 0x44a: 0x4000, 0x44b: 0x4000, - 0x44c: 0x4000, 0x44d: 0x4000, 0x44e: 0x4000, 0x44f: 0x4000, 0x450: 0x4000, 0x451: 0x4000, - 0x452: 0x4000, 0x453: 0x4000, 0x454: 0x4000, 0x455: 0x4000, 0x456: 0x4000, 0x457: 0x4000, - 0x458: 0x4000, 0x459: 0x4000, 0x45a: 0x4000, 0x45b: 0x4000, 0x45c: 0x4000, 0x45d: 0x4000, - 0x45e: 0x4000, 0x45f: 0x4000, - // Block 0x12, offset 0x480 - 0x490: 0x2000, - 0x493: 0x2000, 0x494: 0x2000, 0x495: 0x2000, 0x496: 0x2000, - 0x498: 0x2000, 0x499: 0x2000, 0x49c: 0x2000, 0x49d: 0x2000, - 0x4a0: 0x2000, 0x4a1: 0x2000, 0x4a2: 0x2000, - 0x4a4: 0x2000, 0x4a5: 0x2000, 0x4a6: 0x2000, 0x4a7: 0x2000, - 0x4b0: 0x2000, 0x4b2: 0x2000, 0x4b3: 0x2000, 0x4b5: 0x2000, - 0x4bb: 0x2000, - 0x4be: 0x2000, - // Block 0x13, offset 0x4c0 - 0x4f4: 0x2000, - 0x4ff: 0x2000, - // Block 0x14, offset 0x500 - 0x501: 0x2000, 0x502: 0x2000, 0x503: 0x2000, 0x504: 0x2000, - 0x529: 0xa009, - 0x52c: 0x2000, - // Block 0x15, offset 0x540 - 0x543: 0x2000, 0x545: 0x2000, - 0x549: 0x2000, - 0x553: 0x2000, 0x556: 0x2000, - 0x561: 0x2000, 0x562: 0x2000, - 0x566: 0x2000, - 0x56b: 0x2000, - // Block 0x16, offset 0x580 - 0x593: 0x2000, 0x594: 0x2000, - 0x59b: 0x2000, 0x59c: 0x2000, 0x59d: 0x2000, - 0x59e: 0x2000, 0x5a0: 0x2000, 0x5a1: 0x2000, 0x5a2: 0x2000, 0x5a3: 0x2000, - 0x5a4: 0x2000, 0x5a5: 0x2000, 0x5a6: 0x2000, 0x5a7: 0x2000, 0x5a8: 0x2000, 0x5a9: 0x2000, - 0x5aa: 0x2000, 0x5ab: 0x2000, - 0x5b0: 0x2000, 0x5b1: 0x2000, 0x5b2: 0x2000, 0x5b3: 0x2000, 0x5b4: 0x2000, 0x5b5: 0x2000, - 0x5b6: 0x2000, 0x5b7: 0x2000, 0x5b8: 0x2000, 0x5b9: 0x2000, - // Block 0x17, offset 0x5c0 - 0x5c9: 0x2000, - 0x5d0: 0x200a, 0x5d1: 0x200b, - 0x5d2: 0x200a, 0x5d3: 0x200c, 0x5d4: 0x2000, 0x5d5: 0x2000, 0x5d6: 0x2000, 0x5d7: 0x2000, - 0x5d8: 0x2000, 0x5d9: 0x2000, - 0x5f8: 0x2000, 0x5f9: 0x2000, - // Block 0x18, offset 0x600 - 0x612: 0x2000, 0x614: 0x2000, - 0x627: 0x2000, - // Block 0x19, offset 0x640 - 0x640: 0x2000, 0x642: 0x2000, 0x643: 0x2000, - 0x647: 0x2000, 0x648: 0x2000, 0x64b: 0x2000, - 0x64f: 0x2000, 0x651: 0x2000, - 0x655: 0x2000, - 0x65a: 0x2000, 0x65d: 0x2000, - 0x65e: 0x2000, 0x65f: 0x2000, 0x660: 0x2000, 0x663: 0x2000, - 0x665: 0x2000, 0x667: 0x2000, 0x668: 0x2000, 0x669: 0x2000, - 0x66a: 0x2000, 0x66b: 0x2000, 0x66c: 0x2000, 0x66e: 0x2000, - 0x674: 0x2000, 0x675: 0x2000, - 0x676: 0x2000, 0x677: 0x2000, - 0x67c: 0x2000, 0x67d: 0x2000, - // Block 0x1a, offset 0x680 - 0x688: 0x2000, - 0x68c: 0x2000, - 0x692: 0x2000, - 0x6a0: 0x2000, 0x6a1: 0x2000, - 0x6a4: 0x2000, 0x6a5: 0x2000, 0x6a6: 0x2000, 0x6a7: 0x2000, - 0x6aa: 0x2000, 0x6ab: 0x2000, 0x6ae: 0x2000, 0x6af: 0x2000, - // Block 0x1b, offset 0x6c0 - 0x6c2: 0x2000, 0x6c3: 0x2000, - 0x6c6: 0x2000, 0x6c7: 0x2000, - 0x6d5: 0x2000, - 0x6d9: 0x2000, - 0x6e5: 0x2000, - 0x6ff: 0x2000, - // Block 0x1c, offset 0x700 - 0x712: 0x2000, - 0x71a: 0x4000, 0x71b: 0x4000, - 0x729: 0x4000, - 0x72a: 0x4000, - // Block 0x1d, offset 0x740 - 0x769: 0x4000, - 0x76a: 0x4000, 0x76b: 0x4000, 0x76c: 0x4000, - 0x770: 0x4000, 0x773: 0x4000, - // Block 0x1e, offset 0x780 - 0x7a0: 0x2000, 0x7a1: 0x2000, 0x7a2: 0x2000, 0x7a3: 0x2000, - 0x7a4: 0x2000, 0x7a5: 0x2000, 0x7a6: 0x2000, 0x7a7: 0x2000, 0x7a8: 0x2000, 0x7a9: 0x2000, - 0x7aa: 0x2000, 0x7ab: 0x2000, 0x7ac: 0x2000, 0x7ad: 0x2000, 0x7ae: 0x2000, 0x7af: 0x2000, - 0x7b0: 0x2000, 0x7b1: 0x2000, 0x7b2: 0x2000, 0x7b3: 0x2000, 0x7b4: 0x2000, 0x7b5: 0x2000, - 0x7b6: 0x2000, 0x7b7: 0x2000, 0x7b8: 0x2000, 0x7b9: 0x2000, 0x7ba: 0x2000, 0x7bb: 0x2000, - 0x7bc: 0x2000, 0x7bd: 0x2000, 0x7be: 0x2000, 0x7bf: 0x2000, - // Block 0x1f, offset 0x7c0 - 0x7c0: 0x2000, 0x7c1: 0x2000, 0x7c2: 0x2000, 0x7c3: 0x2000, 0x7c4: 0x2000, 0x7c5: 0x2000, - 0x7c6: 0x2000, 0x7c7: 0x2000, 0x7c8: 0x2000, 0x7c9: 0x2000, 0x7ca: 0x2000, 0x7cb: 0x2000, - 0x7cc: 0x2000, 0x7cd: 0x2000, 0x7ce: 0x2000, 0x7cf: 0x2000, 0x7d0: 0x2000, 0x7d1: 0x2000, - 0x7d2: 0x2000, 0x7d3: 0x2000, 0x7d4: 0x2000, 0x7d5: 0x2000, 0x7d6: 0x2000, 0x7d7: 0x2000, - 0x7d8: 0x2000, 0x7d9: 0x2000, 0x7da: 0x2000, 0x7db: 0x2000, 0x7dc: 0x2000, 0x7dd: 0x2000, - 0x7de: 0x2000, 0x7df: 0x2000, 0x7e0: 0x2000, 0x7e1: 0x2000, 0x7e2: 0x2000, 0x7e3: 0x2000, - 0x7e4: 0x2000, 0x7e5: 0x2000, 0x7e6: 0x2000, 0x7e7: 0x2000, 0x7e8: 0x2000, 0x7e9: 0x2000, - 0x7eb: 0x2000, 0x7ec: 0x2000, 0x7ed: 0x2000, 0x7ee: 0x2000, 0x7ef: 0x2000, - 0x7f0: 0x2000, 0x7f1: 0x2000, 0x7f2: 0x2000, 0x7f3: 0x2000, 0x7f4: 0x2000, 0x7f5: 0x2000, - 0x7f6: 0x2000, 0x7f7: 0x2000, 0x7f8: 0x2000, 0x7f9: 0x2000, 0x7fa: 0x2000, 0x7fb: 0x2000, - 0x7fc: 0x2000, 0x7fd: 0x2000, 0x7fe: 0x2000, 0x7ff: 0x2000, - // Block 0x20, offset 0x800 - 0x800: 0x2000, 0x801: 0x2000, 0x802: 0x200d, 0x803: 0x2000, 0x804: 0x2000, 0x805: 0x2000, - 0x806: 0x2000, 0x807: 0x2000, 0x808: 0x2000, 0x809: 0x2000, 0x80a: 0x2000, 0x80b: 0x2000, - 0x80c: 0x2000, 0x80d: 0x2000, 0x80e: 0x2000, 0x80f: 0x2000, 0x810: 0x2000, 0x811: 0x2000, - 0x812: 0x2000, 0x813: 0x2000, 0x814: 0x2000, 0x815: 0x2000, 0x816: 0x2000, 0x817: 0x2000, - 0x818: 0x2000, 0x819: 0x2000, 0x81a: 0x2000, 0x81b: 0x2000, 0x81c: 0x2000, 0x81d: 0x2000, - 0x81e: 0x2000, 0x81f: 0x2000, 0x820: 0x2000, 0x821: 0x2000, 0x822: 0x2000, 0x823: 0x2000, - 0x824: 0x2000, 0x825: 0x2000, 0x826: 0x2000, 0x827: 0x2000, 0x828: 0x2000, 0x829: 0x2000, - 0x82a: 0x2000, 0x82b: 0x2000, 0x82c: 0x2000, 0x82d: 0x2000, 0x82e: 0x2000, 0x82f: 0x2000, - 0x830: 0x2000, 0x831: 0x2000, 0x832: 0x2000, 0x833: 0x2000, 0x834: 0x2000, 0x835: 0x2000, - 0x836: 0x2000, 0x837: 0x2000, 0x838: 0x2000, 0x839: 0x2000, 0x83a: 0x2000, 0x83b: 0x2000, - 0x83c: 0x2000, 0x83d: 0x2000, 0x83e: 0x2000, 0x83f: 0x2000, - // Block 0x21, offset 0x840 - 0x840: 0x2000, 0x841: 0x2000, 0x842: 0x2000, 0x843: 0x2000, 0x844: 0x2000, 0x845: 0x2000, - 0x846: 0x2000, 0x847: 0x2000, 0x848: 0x2000, 0x849: 0x2000, 0x84a: 0x2000, 0x84b: 0x2000, - 0x850: 0x2000, 0x851: 0x2000, - 0x852: 0x2000, 0x853: 0x2000, 0x854: 0x2000, 0x855: 0x2000, 0x856: 0x2000, 0x857: 0x2000, - 0x858: 0x2000, 0x859: 0x2000, 0x85a: 0x2000, 0x85b: 0x2000, 0x85c: 0x2000, 0x85d: 0x2000, - 0x85e: 0x2000, 0x85f: 0x2000, 0x860: 0x2000, 0x861: 0x2000, 0x862: 0x2000, 0x863: 0x2000, - 0x864: 0x2000, 0x865: 0x2000, 0x866: 0x2000, 0x867: 0x2000, 0x868: 0x2000, 0x869: 0x2000, - 0x86a: 0x2000, 0x86b: 0x2000, 0x86c: 0x2000, 0x86d: 0x2000, 0x86e: 0x2000, 0x86f: 0x2000, - 0x870: 0x2000, 0x871: 0x2000, 0x872: 0x2000, 0x873: 0x2000, - // Block 0x22, offset 0x880 - 0x880: 0x2000, 0x881: 0x2000, 0x882: 0x2000, 0x883: 0x2000, 0x884: 0x2000, 0x885: 0x2000, - 0x886: 0x2000, 0x887: 0x2000, 0x888: 0x2000, 0x889: 0x2000, 0x88a: 0x2000, 0x88b: 0x2000, - 0x88c: 0x2000, 0x88d: 0x2000, 0x88e: 0x2000, 0x88f: 0x2000, - 0x892: 0x2000, 0x893: 0x2000, 0x894: 0x2000, 0x895: 0x2000, - 0x8a0: 0x200e, 0x8a1: 0x2000, 0x8a3: 0x2000, - 0x8a4: 0x2000, 0x8a5: 0x2000, 0x8a6: 0x2000, 0x8a7: 0x2000, 0x8a8: 0x2000, 0x8a9: 0x2000, - 0x8b2: 0x2000, 0x8b3: 0x2000, - 0x8b6: 0x2000, 0x8b7: 0x2000, - 0x8bc: 0x2000, 0x8bd: 0x2000, - // Block 0x23, offset 0x8c0 - 0x8c0: 0x2000, 0x8c1: 0x2000, - 0x8c6: 0x2000, 0x8c7: 0x2000, 0x8c8: 0x2000, 0x8cb: 0x200f, - 0x8ce: 0x2000, 0x8cf: 0x2000, 0x8d0: 0x2000, 0x8d1: 0x2000, - 0x8e2: 0x2000, 0x8e3: 0x2000, - 0x8e4: 0x2000, 0x8e5: 0x2000, - 0x8ef: 0x2000, - 0x8fd: 0x4000, 0x8fe: 0x4000, - // Block 0x24, offset 0x900 - 0x905: 0x2000, - 0x906: 0x2000, 0x909: 0x2000, - 0x90e: 0x2000, 0x90f: 0x2000, - 0x914: 0x4000, 0x915: 0x4000, - 0x91c: 0x2000, - 0x91e: 0x2000, - // Block 0x25, offset 0x940 - 0x940: 0x2000, 0x942: 0x2000, - 0x948: 0x4000, 0x949: 0x4000, 0x94a: 0x4000, 0x94b: 0x4000, - 0x94c: 0x4000, 0x94d: 0x4000, 0x94e: 0x4000, 0x94f: 0x4000, 0x950: 0x4000, 0x951: 0x4000, - 0x952: 0x4000, 0x953: 0x4000, - 0x960: 0x2000, 0x961: 0x2000, 0x963: 0x2000, - 0x964: 0x2000, 0x965: 0x2000, 0x967: 0x2000, 0x968: 0x2000, 0x969: 0x2000, - 0x96a: 0x2000, 0x96c: 0x2000, 0x96d: 0x2000, 0x96f: 0x2000, - 0x97f: 0x4000, - // Block 0x26, offset 0x980 - 0x993: 0x4000, - 0x99e: 0x2000, 0x99f: 0x2000, 0x9a1: 0x4000, - 0x9aa: 0x4000, 0x9ab: 0x4000, - 0x9bd: 0x4000, 0x9be: 0x4000, 0x9bf: 0x2000, - // Block 0x27, offset 0x9c0 - 0x9c4: 0x4000, 0x9c5: 0x4000, - 0x9c6: 0x2000, 0x9c7: 0x2000, 0x9c8: 0x2000, 0x9c9: 0x2000, 0x9ca: 0x2000, 0x9cb: 0x2000, - 0x9cc: 0x2000, 0x9cd: 0x2000, 0x9ce: 0x4000, 0x9cf: 0x2000, 0x9d0: 0x2000, 0x9d1: 0x2000, - 0x9d2: 0x2000, 0x9d3: 0x2000, 0x9d4: 0x4000, 0x9d5: 0x2000, 0x9d6: 0x2000, 0x9d7: 0x2000, - 0x9d8: 0x2000, 0x9d9: 0x2000, 0x9da: 0x2000, 0x9db: 0x2000, 0x9dc: 0x2000, 0x9dd: 0x2000, - 0x9de: 0x2000, 0x9df: 0x2000, 0x9e0: 0x2000, 0x9e1: 0x2000, 0x9e3: 0x2000, - 0x9e8: 0x2000, 0x9e9: 0x2000, - 0x9ea: 0x4000, 0x9eb: 0x2000, 0x9ec: 0x2000, 0x9ed: 0x2000, 0x9ee: 0x2000, 0x9ef: 0x2000, - 0x9f0: 0x2000, 0x9f1: 0x2000, 0x9f2: 0x4000, 0x9f3: 0x4000, 0x9f4: 0x2000, 0x9f5: 0x4000, - 0x9f6: 0x2000, 0x9f7: 0x2000, 0x9f8: 0x2000, 0x9f9: 0x2000, 0x9fa: 0x4000, 0x9fb: 0x2000, - 0x9fc: 0x2000, 0x9fd: 0x4000, 0x9fe: 0x2000, 0x9ff: 0x2000, - // Block 0x28, offset 0xa00 - 0xa05: 0x4000, - 0xa0a: 0x4000, 0xa0b: 0x4000, - 0xa28: 0x4000, - 0xa3d: 0x2000, - // Block 0x29, offset 0xa40 - 0xa4c: 0x4000, 0xa4e: 0x4000, - 0xa53: 0x4000, 0xa54: 0x4000, 0xa55: 0x4000, 0xa57: 0x4000, - 0xa76: 0x2000, 0xa77: 0x2000, 0xa78: 0x2000, 0xa79: 0x2000, 0xa7a: 0x2000, 0xa7b: 0x2000, - 0xa7c: 0x2000, 0xa7d: 0x2000, 0xa7e: 0x2000, 0xa7f: 0x2000, - // Block 0x2a, offset 0xa80 - 0xa95: 0x4000, 0xa96: 0x4000, 0xa97: 0x4000, - 0xab0: 0x4000, - 0xabf: 0x4000, - // Block 0x2b, offset 0xac0 - 0xae6: 0x6000, 0xae7: 0x6000, 0xae8: 0x6000, 0xae9: 0x6000, - 0xaea: 0x6000, 0xaeb: 0x6000, 0xaec: 0x6000, 0xaed: 0x6000, - // Block 0x2c, offset 0xb00 - 0xb05: 0x6010, - 0xb06: 0x6011, - // Block 0x2d, offset 0xb40 - 0xb5b: 0x4000, 0xb5c: 0x4000, - // Block 0x2e, offset 0xb80 - 0xb90: 0x4000, - 0xb95: 0x4000, 0xb96: 0x2000, 0xb97: 0x2000, - 0xb98: 0x2000, 0xb99: 0x2000, - // Block 0x2f, offset 0xbc0 - 0xbc0: 0x4000, 0xbc1: 0x4000, 0xbc2: 0x4000, 0xbc3: 0x4000, 0xbc4: 0x4000, 0xbc5: 0x4000, - 0xbc6: 0x4000, 0xbc7: 0x4000, 0xbc8: 0x4000, 0xbc9: 0x4000, 0xbca: 0x4000, 0xbcb: 0x4000, - 0xbcc: 0x4000, 0xbcd: 0x4000, 0xbce: 0x4000, 0xbcf: 0x4000, 0xbd0: 0x4000, 0xbd1: 0x4000, - 0xbd2: 0x4000, 0xbd3: 0x4000, 0xbd4: 0x4000, 0xbd5: 0x4000, 0xbd6: 0x4000, 0xbd7: 0x4000, - 0xbd8: 0x4000, 0xbd9: 0x4000, 0xbdb: 0x4000, 0xbdc: 0x4000, 0xbdd: 0x4000, - 0xbde: 0x4000, 0xbdf: 0x4000, 0xbe0: 0x4000, 0xbe1: 0x4000, 0xbe2: 0x4000, 0xbe3: 0x4000, - 0xbe4: 0x4000, 0xbe5: 0x4000, 0xbe6: 0x4000, 0xbe7: 0x4000, 0xbe8: 0x4000, 0xbe9: 0x4000, - 0xbea: 0x4000, 0xbeb: 0x4000, 0xbec: 0x4000, 0xbed: 0x4000, 0xbee: 0x4000, 0xbef: 0x4000, - 0xbf0: 0x4000, 0xbf1: 0x4000, 0xbf2: 0x4000, 0xbf3: 0x4000, 0xbf4: 0x4000, 0xbf5: 0x4000, - 0xbf6: 0x4000, 0xbf7: 0x4000, 0xbf8: 0x4000, 0xbf9: 0x4000, 0xbfa: 0x4000, 0xbfb: 0x4000, - 0xbfc: 0x4000, 0xbfd: 0x4000, 0xbfe: 0x4000, 0xbff: 0x4000, - // Block 0x30, offset 0xc00 - 0xc00: 0x4000, 0xc01: 0x4000, 0xc02: 0x4000, 0xc03: 0x4000, 0xc04: 0x4000, 0xc05: 0x4000, - 0xc06: 0x4000, 0xc07: 0x4000, 0xc08: 0x4000, 0xc09: 0x4000, 0xc0a: 0x4000, 0xc0b: 0x4000, - 0xc0c: 0x4000, 0xc0d: 0x4000, 0xc0e: 0x4000, 0xc0f: 0x4000, 0xc10: 0x4000, 0xc11: 0x4000, - 0xc12: 0x4000, 0xc13: 0x4000, 0xc14: 0x4000, 0xc15: 0x4000, 0xc16: 0x4000, 0xc17: 0x4000, - 0xc18: 0x4000, 0xc19: 0x4000, 0xc1a: 0x4000, 0xc1b: 0x4000, 0xc1c: 0x4000, 0xc1d: 0x4000, - 0xc1e: 0x4000, 0xc1f: 0x4000, 0xc20: 0x4000, 0xc21: 0x4000, 0xc22: 0x4000, 0xc23: 0x4000, - 0xc24: 0x4000, 0xc25: 0x4000, 0xc26: 0x4000, 0xc27: 0x4000, 0xc28: 0x4000, 0xc29: 0x4000, - 0xc2a: 0x4000, 0xc2b: 0x4000, 0xc2c: 0x4000, 0xc2d: 0x4000, 0xc2e: 0x4000, 0xc2f: 0x4000, - 0xc30: 0x4000, 0xc31: 0x4000, 0xc32: 0x4000, 0xc33: 0x4000, - // Block 0x31, offset 0xc40 - 0xc40: 0x4000, 0xc41: 0x4000, 0xc42: 0x4000, 0xc43: 0x4000, 0xc44: 0x4000, 0xc45: 0x4000, - 0xc46: 0x4000, 0xc47: 0x4000, 0xc48: 0x4000, 0xc49: 0x4000, 0xc4a: 0x4000, 0xc4b: 0x4000, - 0xc4c: 0x4000, 0xc4d: 0x4000, 0xc4e: 0x4000, 0xc4f: 0x4000, 0xc50: 0x4000, 0xc51: 0x4000, - 0xc52: 0x4000, 0xc53: 0x4000, 0xc54: 0x4000, 0xc55: 0x4000, - 0xc70: 0x4000, 0xc71: 0x4000, 0xc72: 0x4000, 0xc73: 0x4000, 0xc74: 0x4000, 0xc75: 0x4000, - 0xc76: 0x4000, 0xc77: 0x4000, 0xc78: 0x4000, 0xc79: 0x4000, 0xc7a: 0x4000, 0xc7b: 0x4000, - // Block 0x32, offset 0xc80 - 0xc80: 0x9012, 0xc81: 0x4013, 0xc82: 0x4014, 0xc83: 0x4000, 0xc84: 0x4000, 0xc85: 0x4000, - 0xc86: 0x4000, 0xc87: 0x4000, 0xc88: 0x4000, 0xc89: 0x4000, 0xc8a: 0x4000, 0xc8b: 0x4000, - 0xc8c: 0x4015, 0xc8d: 0x4015, 0xc8e: 0x4000, 0xc8f: 0x4000, 0xc90: 0x4000, 0xc91: 0x4000, - 0xc92: 0x4000, 0xc93: 0x4000, 0xc94: 0x4000, 0xc95: 0x4000, 0xc96: 0x4000, 0xc97: 0x4000, - 0xc98: 0x4000, 0xc99: 0x4000, 0xc9a: 0x4000, 0xc9b: 0x4000, 0xc9c: 0x4000, 0xc9d: 0x4000, - 0xc9e: 0x4000, 0xc9f: 0x4000, 0xca0: 0x4000, 0xca1: 0x4000, 0xca2: 0x4000, 0xca3: 0x4000, - 0xca4: 0x4000, 0xca5: 0x4000, 0xca6: 0x4000, 0xca7: 0x4000, 0xca8: 0x4000, 0xca9: 0x4000, - 0xcaa: 0x4000, 0xcab: 0x4000, 0xcac: 0x4000, 0xcad: 0x4000, 0xcae: 0x4000, 0xcaf: 0x4000, - 0xcb0: 0x4000, 0xcb1: 0x4000, 0xcb2: 0x4000, 0xcb3: 0x4000, 0xcb4: 0x4000, 0xcb5: 0x4000, - 0xcb6: 0x4000, 0xcb7: 0x4000, 0xcb8: 0x4000, 0xcb9: 0x4000, 0xcba: 0x4000, 0xcbb: 0x4000, - 0xcbc: 0x4000, 0xcbd: 0x4000, 0xcbe: 0x4000, - // Block 0x33, offset 0xcc0 - 0xcc1: 0x4000, 0xcc2: 0x4000, 0xcc3: 0x4000, 0xcc4: 0x4000, 0xcc5: 0x4000, - 0xcc6: 0x4000, 0xcc7: 0x4000, 0xcc8: 0x4000, 0xcc9: 0x4000, 0xcca: 0x4000, 0xccb: 0x4000, - 0xccc: 0x4000, 0xccd: 0x4000, 0xcce: 0x4000, 0xccf: 0x4000, 0xcd0: 0x4000, 0xcd1: 0x4000, - 0xcd2: 0x4000, 0xcd3: 0x4000, 0xcd4: 0x4000, 0xcd5: 0x4000, 0xcd6: 0x4000, 0xcd7: 0x4000, - 0xcd8: 0x4000, 0xcd9: 0x4000, 0xcda: 0x4000, 0xcdb: 0x4000, 0xcdc: 0x4000, 0xcdd: 0x4000, - 0xcde: 0x4000, 0xcdf: 0x4000, 0xce0: 0x4000, 0xce1: 0x4000, 0xce2: 0x4000, 0xce3: 0x4000, - 0xce4: 0x4000, 0xce5: 0x4000, 0xce6: 0x4000, 0xce7: 0x4000, 0xce8: 0x4000, 0xce9: 0x4000, - 0xcea: 0x4000, 0xceb: 0x4000, 0xcec: 0x4000, 0xced: 0x4000, 0xcee: 0x4000, 0xcef: 0x4000, - 0xcf0: 0x4000, 0xcf1: 0x4000, 0xcf2: 0x4000, 0xcf3: 0x4000, 0xcf4: 0x4000, 0xcf5: 0x4000, - 0xcf6: 0x4000, 0xcf7: 0x4000, 0xcf8: 0x4000, 0xcf9: 0x4000, 0xcfa: 0x4000, 0xcfb: 0x4000, - 0xcfc: 0x4000, 0xcfd: 0x4000, 0xcfe: 0x4000, 0xcff: 0x4000, - // Block 0x34, offset 0xd00 - 0xd00: 0x4000, 0xd01: 0x4000, 0xd02: 0x4000, 0xd03: 0x4000, 0xd04: 0x4000, 0xd05: 0x4000, - 0xd06: 0x4000, 0xd07: 0x4000, 0xd08: 0x4000, 0xd09: 0x4000, 0xd0a: 0x4000, 0xd0b: 0x4000, - 0xd0c: 0x4000, 0xd0d: 0x4000, 0xd0e: 0x4000, 0xd0f: 0x4000, 0xd10: 0x4000, 0xd11: 0x4000, - 0xd12: 0x4000, 0xd13: 0x4000, 0xd14: 0x4000, 0xd15: 0x4000, 0xd16: 0x4000, - 0xd19: 0x4016, 0xd1a: 0x4017, 0xd1b: 0x4000, 0xd1c: 0x4000, 0xd1d: 0x4000, - 0xd1e: 0x4000, 0xd1f: 0x4000, 0xd20: 0x4000, 0xd21: 0x4018, 0xd22: 0x4019, 0xd23: 0x401a, - 0xd24: 0x401b, 0xd25: 0x401c, 0xd26: 0x401d, 0xd27: 0x401e, 0xd28: 0x401f, 0xd29: 0x4020, - 0xd2a: 0x4021, 0xd2b: 0x4022, 0xd2c: 0x4000, 0xd2d: 0x4010, 0xd2e: 0x4000, 0xd2f: 0x4023, - 0xd30: 0x4000, 0xd31: 0x4024, 0xd32: 0x4000, 0xd33: 0x4025, 0xd34: 0x4000, 0xd35: 0x4026, - 0xd36: 0x4000, 0xd37: 0x401a, 0xd38: 0x4000, 0xd39: 0x4027, 0xd3a: 0x4000, 0xd3b: 0x4028, - 0xd3c: 0x4000, 0xd3d: 0x4020, 0xd3e: 0x4000, 0xd3f: 0x4029, - // Block 0x35, offset 0xd40 - 0xd40: 0x4000, 0xd41: 0x402a, 0xd42: 0x4000, 0xd43: 0x402b, 0xd44: 0x402c, 0xd45: 0x4000, - 0xd46: 0x4017, 0xd47: 0x4000, 0xd48: 0x402d, 0xd49: 0x4000, 0xd4a: 0x402e, 0xd4b: 0x402f, - 0xd4c: 0x4030, 0xd4d: 0x4017, 0xd4e: 0x4016, 0xd4f: 0x4017, 0xd50: 0x4000, 0xd51: 0x4000, - 0xd52: 0x4031, 0xd53: 0x4000, 0xd54: 0x4000, 0xd55: 0x4031, 0xd56: 0x4000, 0xd57: 0x4000, - 0xd58: 0x4032, 0xd59: 0x4000, 0xd5a: 0x4000, 0xd5b: 0x4032, 0xd5c: 0x4000, 0xd5d: 0x4000, - 0xd5e: 0x4033, 0xd5f: 0x402e, 0xd60: 0x4034, 0xd61: 0x4035, 0xd62: 0x4034, 0xd63: 0x4036, - 0xd64: 0x4037, 0xd65: 0x4024, 0xd66: 0x4035, 0xd67: 0x4025, 0xd68: 0x4038, 0xd69: 0x4038, - 0xd6a: 0x4039, 0xd6b: 0x4039, 0xd6c: 0x403a, 0xd6d: 0x403a, 0xd6e: 0x4000, 0xd6f: 0x4035, - 0xd70: 0x4000, 0xd71: 0x4000, 0xd72: 0x403b, 0xd73: 0x403c, 0xd74: 0x4000, 0xd75: 0x4000, - 0xd76: 0x4000, 0xd77: 0x4000, 0xd78: 0x4000, 0xd79: 0x4000, 0xd7a: 0x4000, 0xd7b: 0x403d, - 0xd7c: 0x401c, 0xd7d: 0x4000, 0xd7e: 0x4000, 0xd7f: 0x4000, - // Block 0x36, offset 0xd80 - 0xd85: 0x4000, - 0xd86: 0x4000, 0xd87: 0x4000, 0xd88: 0x4000, 0xd89: 0x4000, 0xd8a: 0x4000, 0xd8b: 0x4000, - 0xd8c: 0x4000, 0xd8d: 0x4000, 0xd8e: 0x4000, 0xd8f: 0x4000, 0xd90: 0x4000, 0xd91: 0x4000, - 0xd92: 0x4000, 0xd93: 0x4000, 0xd94: 0x4000, 0xd95: 0x4000, 0xd96: 0x4000, 0xd97: 0x4000, - 0xd98: 0x4000, 0xd99: 0x4000, 0xd9a: 0x4000, 0xd9b: 0x4000, 0xd9c: 0x4000, 0xd9d: 0x4000, - 0xd9e: 0x4000, 0xd9f: 0x4000, 0xda0: 0x4000, 0xda1: 0x4000, 0xda2: 0x4000, 0xda3: 0x4000, - 0xda4: 0x4000, 0xda5: 0x4000, 0xda6: 0x4000, 0xda7: 0x4000, 0xda8: 0x4000, 0xda9: 0x4000, - 0xdaa: 0x4000, 0xdab: 0x4000, 0xdac: 0x4000, 0xdad: 0x4000, 0xdae: 0x4000, 0xdaf: 0x4000, - 0xdb1: 0x403e, 0xdb2: 0x403e, 0xdb3: 0x403e, 0xdb4: 0x403e, 0xdb5: 0x403e, - 0xdb6: 0x403e, 0xdb7: 0x403e, 0xdb8: 0x403e, 0xdb9: 0x403e, 0xdba: 0x403e, 0xdbb: 0x403e, - 0xdbc: 0x403e, 0xdbd: 0x403e, 0xdbe: 0x403e, 0xdbf: 0x403e, - // Block 0x37, offset 0xdc0 - 0xdc0: 0x4037, 0xdc1: 0x4037, 0xdc2: 0x4037, 0xdc3: 0x4037, 0xdc4: 0x4037, 0xdc5: 0x4037, - 0xdc6: 0x4037, 0xdc7: 0x4037, 0xdc8: 0x4037, 0xdc9: 0x4037, 0xdca: 0x4037, 0xdcb: 0x4037, - 0xdcc: 0x4037, 0xdcd: 0x4037, 0xdce: 0x4037, 0xdcf: 0x400e, 0xdd0: 0x403f, 0xdd1: 0x4040, - 0xdd2: 0x4041, 0xdd3: 0x4040, 0xdd4: 0x403f, 0xdd5: 0x4042, 0xdd6: 0x4043, 0xdd7: 0x4044, - 0xdd8: 0x4040, 0xdd9: 0x4041, 0xdda: 0x4040, 0xddb: 0x4045, 0xddc: 0x4009, 0xddd: 0x4045, - 0xdde: 0x4046, 0xddf: 0x4045, 0xde0: 0x4047, 0xde1: 0x400b, 0xde2: 0x400a, 0xde3: 0x400c, - 0xde4: 0x4048, 0xde5: 0x4000, 0xde6: 0x4000, 0xde7: 0x4000, 0xde8: 0x4000, 0xde9: 0x4000, - 0xdea: 0x4000, 0xdeb: 0x4000, 0xdec: 0x4000, 0xded: 0x4000, 0xdee: 0x4000, 0xdef: 0x4000, - 0xdf0: 0x4000, 0xdf1: 0x4000, 0xdf2: 0x4000, 0xdf3: 0x4000, 0xdf4: 0x4000, 0xdf5: 0x4000, - 0xdf6: 0x4000, 0xdf7: 0x4000, 0xdf8: 0x4000, 0xdf9: 0x4000, 0xdfa: 0x4000, 0xdfb: 0x4000, - 0xdfc: 0x4000, 0xdfd: 0x4000, 0xdfe: 0x4000, 0xdff: 0x4000, - // Block 0x38, offset 0xe00 - 0xe00: 0x4000, 0xe01: 0x4000, 0xe02: 0x4000, 0xe03: 0x4000, 0xe04: 0x4000, 0xe05: 0x4000, - 0xe06: 0x4000, 0xe07: 0x4000, 0xe08: 0x4000, 0xe09: 0x4000, 0xe0a: 0x4000, 0xe0b: 0x4000, - 0xe0c: 0x4000, 0xe0d: 0x4000, 0xe0e: 0x4000, 0xe10: 0x4000, 0xe11: 0x4000, - 0xe12: 0x4000, 0xe13: 0x4000, 0xe14: 0x4000, 0xe15: 0x4000, 0xe16: 0x4000, 0xe17: 0x4000, - 0xe18: 0x4000, 0xe19: 0x4000, 0xe1a: 0x4000, 0xe1b: 0x4000, 0xe1c: 0x4000, 0xe1d: 0x4000, - 0xe1e: 0x4000, 0xe1f: 0x4000, 0xe20: 0x4000, 0xe21: 0x4000, 0xe22: 0x4000, 0xe23: 0x4000, - 0xe24: 0x4000, 0xe25: 0x4000, 0xe26: 0x4000, 0xe27: 0x4000, 0xe28: 0x4000, 0xe29: 0x4000, - 0xe2a: 0x4000, 0xe2b: 0x4000, 0xe2c: 0x4000, 0xe2d: 0x4000, 0xe2e: 0x4000, 0xe2f: 0x4000, - 0xe30: 0x4000, 0xe31: 0x4000, 0xe32: 0x4000, 0xe33: 0x4000, 0xe34: 0x4000, 0xe35: 0x4000, - 0xe36: 0x4000, 0xe37: 0x4000, 0xe38: 0x4000, 0xe39: 0x4000, 0xe3a: 0x4000, - // Block 0x39, offset 0xe40 - 0xe40: 0x4000, 0xe41: 0x4000, 0xe42: 0x4000, 0xe43: 0x4000, 0xe44: 0x4000, 0xe45: 0x4000, - 0xe46: 0x4000, 0xe47: 0x4000, 0xe48: 0x4000, 0xe49: 0x4000, 0xe4a: 0x4000, 0xe4b: 0x4000, - 0xe4c: 0x4000, 0xe4d: 0x4000, 0xe4e: 0x4000, 0xe4f: 0x4000, 0xe50: 0x4000, 0xe51: 0x4000, - 0xe52: 0x4000, 0xe53: 0x4000, 0xe54: 0x4000, 0xe55: 0x4000, 0xe56: 0x4000, 0xe57: 0x4000, - 0xe58: 0x4000, 0xe59: 0x4000, 0xe5a: 0x4000, 0xe5b: 0x4000, 0xe5c: 0x4000, 0xe5d: 0x4000, - 0xe5e: 0x4000, 0xe5f: 0x4000, 0xe60: 0x4000, 0xe61: 0x4000, 0xe62: 0x4000, 0xe63: 0x4000, - 0xe70: 0x4000, 0xe71: 0x4000, 0xe72: 0x4000, 0xe73: 0x4000, 0xe74: 0x4000, 0xe75: 0x4000, - 0xe76: 0x4000, 0xe77: 0x4000, 0xe78: 0x4000, 0xe79: 0x4000, 0xe7a: 0x4000, 0xe7b: 0x4000, - 0xe7c: 0x4000, 0xe7d: 0x4000, 0xe7e: 0x4000, 0xe7f: 0x4000, - // Block 0x3a, offset 0xe80 - 0xe80: 0x4000, 0xe81: 0x4000, 0xe82: 0x4000, 0xe83: 0x4000, 0xe84: 0x4000, 0xe85: 0x4000, - 0xe86: 0x4000, 0xe87: 0x4000, 0xe88: 0x4000, 0xe89: 0x4000, 0xe8a: 0x4000, 0xe8b: 0x4000, - 0xe8c: 0x4000, 0xe8d: 0x4000, 0xe8e: 0x4000, 0xe8f: 0x4000, 0xe90: 0x4000, 0xe91: 0x4000, - 0xe92: 0x4000, 0xe93: 0x4000, 0xe94: 0x4000, 0xe95: 0x4000, 0xe96: 0x4000, 0xe97: 0x4000, - 0xe98: 0x4000, 0xe99: 0x4000, 0xe9a: 0x4000, 0xe9b: 0x4000, 0xe9c: 0x4000, 0xe9d: 0x4000, - 0xe9e: 0x4000, 0xea0: 0x4000, 0xea1: 0x4000, 0xea2: 0x4000, 0xea3: 0x4000, - 0xea4: 0x4000, 0xea5: 0x4000, 0xea6: 0x4000, 0xea7: 0x4000, 0xea8: 0x4000, 0xea9: 0x4000, - 0xeaa: 0x4000, 0xeab: 0x4000, 0xeac: 0x4000, 0xead: 0x4000, 0xeae: 0x4000, 0xeaf: 0x4000, - 0xeb0: 0x4000, 0xeb1: 0x4000, 0xeb2: 0x4000, 0xeb3: 0x4000, 0xeb4: 0x4000, 0xeb5: 0x4000, - 0xeb6: 0x4000, 0xeb7: 0x4000, 0xeb8: 0x4000, 0xeb9: 0x4000, 0xeba: 0x4000, 0xebb: 0x4000, - 0xebc: 0x4000, 0xebd: 0x4000, 0xebe: 0x4000, 0xebf: 0x4000, - // Block 0x3b, offset 0xec0 - 0xec0: 0x4000, 0xec1: 0x4000, 0xec2: 0x4000, 0xec3: 0x4000, 0xec4: 0x4000, 0xec5: 0x4000, - 0xec6: 0x4000, 0xec7: 0x4000, 0xec8: 0x2000, 0xec9: 0x2000, 0xeca: 0x2000, 0xecb: 0x2000, - 0xecc: 0x2000, 0xecd: 0x2000, 0xece: 0x2000, 0xecf: 0x2000, 0xed0: 0x4000, 0xed1: 0x4000, - 0xed2: 0x4000, 0xed3: 0x4000, 0xed4: 0x4000, 0xed5: 0x4000, 0xed6: 0x4000, 0xed7: 0x4000, - 0xed8: 0x4000, 0xed9: 0x4000, 0xeda: 0x4000, 0xedb: 0x4000, 0xedc: 0x4000, 0xedd: 0x4000, - 0xede: 0x4000, 0xedf: 0x4000, 0xee0: 0x4000, 0xee1: 0x4000, 0xee2: 0x4000, 0xee3: 0x4000, - 0xee4: 0x4000, 0xee5: 0x4000, 0xee6: 0x4000, 0xee7: 0x4000, 0xee8: 0x4000, 0xee9: 0x4000, - 0xeea: 0x4000, 0xeeb: 0x4000, 0xeec: 0x4000, 0xeed: 0x4000, 0xeee: 0x4000, 0xeef: 0x4000, - 0xef0: 0x4000, 0xef1: 0x4000, 0xef2: 0x4000, 0xef3: 0x4000, 0xef4: 0x4000, 0xef5: 0x4000, - 0xef6: 0x4000, 0xef7: 0x4000, 0xef8: 0x4000, 0xef9: 0x4000, 0xefa: 0x4000, 0xefb: 0x4000, - 0xefc: 0x4000, 0xefd: 0x4000, 0xefe: 0x4000, 0xeff: 0x4000, - // Block 0x3c, offset 0xf00 - 0xf00: 0x4000, 0xf01: 0x4000, 0xf02: 0x4000, 0xf03: 0x4000, 0xf04: 0x4000, 0xf05: 0x4000, - 0xf06: 0x4000, 0xf07: 0x4000, 0xf08: 0x4000, 0xf09: 0x4000, 0xf0a: 0x4000, 0xf0b: 0x4000, - 0xf0c: 0x4000, 0xf0d: 0x4000, 0xf0e: 0x4000, 0xf0f: 0x4000, 0xf10: 0x4000, 0xf11: 0x4000, - 0xf12: 0x4000, 0xf13: 0x4000, 0xf14: 0x4000, 0xf15: 0x4000, 0xf16: 0x4000, 0xf17: 0x4000, - 0xf18: 0x4000, 0xf19: 0x4000, 0xf1a: 0x4000, 0xf1b: 0x4000, 0xf1c: 0x4000, 0xf1d: 0x4000, - 0xf1e: 0x4000, 0xf1f: 0x4000, 0xf20: 0x4000, 0xf21: 0x4000, 0xf22: 0x4000, 0xf23: 0x4000, - 0xf24: 0x4000, 0xf25: 0x4000, 0xf26: 0x4000, 0xf27: 0x4000, 0xf28: 0x4000, 0xf29: 0x4000, - 0xf2a: 0x4000, 0xf2b: 0x4000, 0xf2c: 0x4000, 0xf2d: 0x4000, 0xf2e: 0x4000, 0xf2f: 0x4000, - 0xf30: 0x4000, 0xf31: 0x4000, 0xf32: 0x4000, 0xf33: 0x4000, 0xf34: 0x4000, 0xf35: 0x4000, - 0xf36: 0x4000, 0xf37: 0x4000, 0xf38: 0x4000, 0xf39: 0x4000, 0xf3a: 0x4000, 0xf3b: 0x4000, - 0xf3c: 0x4000, 0xf3d: 0x4000, 0xf3e: 0x4000, - // Block 0x3d, offset 0xf40 - 0xf40: 0x4000, 0xf41: 0x4000, 0xf42: 0x4000, 0xf43: 0x4000, 0xf44: 0x4000, 0xf45: 0x4000, - 0xf46: 0x4000, 0xf47: 0x4000, 0xf48: 0x4000, 0xf49: 0x4000, 0xf4a: 0x4000, 0xf4b: 0x4000, - 0xf4c: 0x4000, 0xf50: 0x4000, 0xf51: 0x4000, - 0xf52: 0x4000, 0xf53: 0x4000, 0xf54: 0x4000, 0xf55: 0x4000, 0xf56: 0x4000, 0xf57: 0x4000, - 0xf58: 0x4000, 0xf59: 0x4000, 0xf5a: 0x4000, 0xf5b: 0x4000, 0xf5c: 0x4000, 0xf5d: 0x4000, - 0xf5e: 0x4000, 0xf5f: 0x4000, 0xf60: 0x4000, 0xf61: 0x4000, 0xf62: 0x4000, 0xf63: 0x4000, - 0xf64: 0x4000, 0xf65: 0x4000, 0xf66: 0x4000, 0xf67: 0x4000, 0xf68: 0x4000, 0xf69: 0x4000, - 0xf6a: 0x4000, 0xf6b: 0x4000, 0xf6c: 0x4000, 0xf6d: 0x4000, 0xf6e: 0x4000, 0xf6f: 0x4000, - 0xf70: 0x4000, 0xf71: 0x4000, 0xf72: 0x4000, 0xf73: 0x4000, 0xf74: 0x4000, 0xf75: 0x4000, - 0xf76: 0x4000, 0xf77: 0x4000, 0xf78: 0x4000, 0xf79: 0x4000, 0xf7a: 0x4000, 0xf7b: 0x4000, - 0xf7c: 0x4000, 0xf7d: 0x4000, 0xf7e: 0x4000, 0xf7f: 0x4000, - // Block 0x3e, offset 0xf80 - 0xf80: 0x4000, 0xf81: 0x4000, 0xf82: 0x4000, 0xf83: 0x4000, 0xf84: 0x4000, 0xf85: 0x4000, - 0xf86: 0x4000, - // Block 0x3f, offset 0xfc0 - 0xfe0: 0x4000, 0xfe1: 0x4000, 0xfe2: 0x4000, 0xfe3: 0x4000, - 0xfe4: 0x4000, 0xfe5: 0x4000, 0xfe6: 0x4000, 0xfe7: 0x4000, 0xfe8: 0x4000, 0xfe9: 0x4000, - 0xfea: 0x4000, 0xfeb: 0x4000, 0xfec: 0x4000, 0xfed: 0x4000, 0xfee: 0x4000, 0xfef: 0x4000, - 0xff0: 0x4000, 0xff1: 0x4000, 0xff2: 0x4000, 0xff3: 0x4000, 0xff4: 0x4000, 0xff5: 0x4000, - 0xff6: 0x4000, 0xff7: 0x4000, 0xff8: 0x4000, 0xff9: 0x4000, 0xffa: 0x4000, 0xffb: 0x4000, - 0xffc: 0x4000, - // Block 0x40, offset 0x1000 - 0x1000: 0x4000, 0x1001: 0x4000, 0x1002: 0x4000, 0x1003: 0x4000, 0x1004: 0x4000, 0x1005: 0x4000, - 0x1006: 0x4000, 0x1007: 0x4000, 0x1008: 0x4000, 0x1009: 0x4000, 0x100a: 0x4000, 0x100b: 0x4000, - 0x100c: 0x4000, 0x100d: 0x4000, 0x100e: 0x4000, 0x100f: 0x4000, 0x1010: 0x4000, 0x1011: 0x4000, - 0x1012: 0x4000, 0x1013: 0x4000, 0x1014: 0x4000, 0x1015: 0x4000, 0x1016: 0x4000, 0x1017: 0x4000, - 0x1018: 0x4000, 0x1019: 0x4000, 0x101a: 0x4000, 0x101b: 0x4000, 0x101c: 0x4000, 0x101d: 0x4000, - 0x101e: 0x4000, 0x101f: 0x4000, 0x1020: 0x4000, 0x1021: 0x4000, 0x1022: 0x4000, 0x1023: 0x4000, - // Block 0x41, offset 0x1040 - 0x1040: 0x2000, 0x1041: 0x2000, 0x1042: 0x2000, 0x1043: 0x2000, 0x1044: 0x2000, 0x1045: 0x2000, - 0x1046: 0x2000, 0x1047: 0x2000, 0x1048: 0x2000, 0x1049: 0x2000, 0x104a: 0x2000, 0x104b: 0x2000, - 0x104c: 0x2000, 0x104d: 0x2000, 0x104e: 0x2000, 0x104f: 0x2000, 0x1050: 0x4000, 0x1051: 0x4000, - 0x1052: 0x4000, 0x1053: 0x4000, 0x1054: 0x4000, 0x1055: 0x4000, 0x1056: 0x4000, 0x1057: 0x4000, - 0x1058: 0x4000, 0x1059: 0x4000, - 0x1070: 0x4000, 0x1071: 0x4000, 0x1072: 0x4000, 0x1073: 0x4000, 0x1074: 0x4000, 0x1075: 0x4000, - 0x1076: 0x4000, 0x1077: 0x4000, 0x1078: 0x4000, 0x1079: 0x4000, 0x107a: 0x4000, 0x107b: 0x4000, - 0x107c: 0x4000, 0x107d: 0x4000, 0x107e: 0x4000, 0x107f: 0x4000, - // Block 0x42, offset 0x1080 - 0x1080: 0x4000, 0x1081: 0x4000, 0x1082: 0x4000, 0x1083: 0x4000, 0x1084: 0x4000, 0x1085: 0x4000, - 0x1086: 0x4000, 0x1087: 0x4000, 0x1088: 0x4000, 0x1089: 0x4000, 0x108a: 0x4000, 0x108b: 0x4000, - 0x108c: 0x4000, 0x108d: 0x4000, 0x108e: 0x4000, 0x108f: 0x4000, 0x1090: 0x4000, 0x1091: 0x4000, - 0x1092: 0x4000, 0x1094: 0x4000, 0x1095: 0x4000, 0x1096: 0x4000, 0x1097: 0x4000, - 0x1098: 0x4000, 0x1099: 0x4000, 0x109a: 0x4000, 0x109b: 0x4000, 0x109c: 0x4000, 0x109d: 0x4000, - 0x109e: 0x4000, 0x109f: 0x4000, 0x10a0: 0x4000, 0x10a1: 0x4000, 0x10a2: 0x4000, 0x10a3: 0x4000, - 0x10a4: 0x4000, 0x10a5: 0x4000, 0x10a6: 0x4000, 0x10a8: 0x4000, 0x10a9: 0x4000, - 0x10aa: 0x4000, 0x10ab: 0x4000, - // Block 0x43, offset 0x10c0 - 0x10c1: 0x9012, 0x10c2: 0x9012, 0x10c3: 0x9012, 0x10c4: 0x9012, 0x10c5: 0x9012, - 0x10c6: 0x9012, 0x10c7: 0x9012, 0x10c8: 0x9012, 0x10c9: 0x9012, 0x10ca: 0x9012, 0x10cb: 0x9012, - 0x10cc: 0x9012, 0x10cd: 0x9012, 0x10ce: 0x9012, 0x10cf: 0x9012, 0x10d0: 0x9012, 0x10d1: 0x9012, - 0x10d2: 0x9012, 0x10d3: 0x9012, 0x10d4: 0x9012, 0x10d5: 0x9012, 0x10d6: 0x9012, 0x10d7: 0x9012, - 0x10d8: 0x9012, 0x10d9: 0x9012, 0x10da: 0x9012, 0x10db: 0x9012, 0x10dc: 0x9012, 0x10dd: 0x9012, - 0x10de: 0x9012, 0x10df: 0x9012, 0x10e0: 0x9049, 0x10e1: 0x9049, 0x10e2: 0x9049, 0x10e3: 0x9049, - 0x10e4: 0x9049, 0x10e5: 0x9049, 0x10e6: 0x9049, 0x10e7: 0x9049, 0x10e8: 0x9049, 0x10e9: 0x9049, - 0x10ea: 0x9049, 0x10eb: 0x9049, 0x10ec: 0x9049, 0x10ed: 0x9049, 0x10ee: 0x9049, 0x10ef: 0x9049, - 0x10f0: 0x9049, 0x10f1: 0x9049, 0x10f2: 0x9049, 0x10f3: 0x9049, 0x10f4: 0x9049, 0x10f5: 0x9049, - 0x10f6: 0x9049, 0x10f7: 0x9049, 0x10f8: 0x9049, 0x10f9: 0x9049, 0x10fa: 0x9049, 0x10fb: 0x9049, - 0x10fc: 0x9049, 0x10fd: 0x9049, 0x10fe: 0x9049, 0x10ff: 0x9049, - // Block 0x44, offset 0x1100 - 0x1100: 0x9049, 0x1101: 0x9049, 0x1102: 0x9049, 0x1103: 0x9049, 0x1104: 0x9049, 0x1105: 0x9049, - 0x1106: 0x9049, 0x1107: 0x9049, 0x1108: 0x9049, 0x1109: 0x9049, 0x110a: 0x9049, 0x110b: 0x9049, - 0x110c: 0x9049, 0x110d: 0x9049, 0x110e: 0x9049, 0x110f: 0x9049, 0x1110: 0x9049, 0x1111: 0x9049, - 0x1112: 0x9049, 0x1113: 0x9049, 0x1114: 0x9049, 0x1115: 0x9049, 0x1116: 0x9049, 0x1117: 0x9049, - 0x1118: 0x9049, 0x1119: 0x9049, 0x111a: 0x9049, 0x111b: 0x9049, 0x111c: 0x9049, 0x111d: 0x9049, - 0x111e: 0x9049, 0x111f: 0x904a, 0x1120: 0x904b, 0x1121: 0xb04c, 0x1122: 0xb04d, 0x1123: 0xb04d, - 0x1124: 0xb04e, 0x1125: 0xb04f, 0x1126: 0xb050, 0x1127: 0xb051, 0x1128: 0xb052, 0x1129: 0xb053, - 0x112a: 0xb054, 0x112b: 0xb055, 0x112c: 0xb056, 0x112d: 0xb057, 0x112e: 0xb058, 0x112f: 0xb059, - 0x1130: 0xb05a, 0x1131: 0xb05b, 0x1132: 0xb05c, 0x1133: 0xb05d, 0x1134: 0xb05e, 0x1135: 0xb05f, - 0x1136: 0xb060, 0x1137: 0xb061, 0x1138: 0xb062, 0x1139: 0xb063, 0x113a: 0xb064, 0x113b: 0xb065, - 0x113c: 0xb052, 0x113d: 0xb066, 0x113e: 0xb067, 0x113f: 0xb055, - // Block 0x45, offset 0x1140 - 0x1140: 0xb068, 0x1141: 0xb069, 0x1142: 0xb06a, 0x1143: 0xb06b, 0x1144: 0xb05a, 0x1145: 0xb056, - 0x1146: 0xb06c, 0x1147: 0xb06d, 0x1148: 0xb06b, 0x1149: 0xb06e, 0x114a: 0xb06b, 0x114b: 0xb06f, - 0x114c: 0xb06f, 0x114d: 0xb070, 0x114e: 0xb070, 0x114f: 0xb071, 0x1150: 0xb056, 0x1151: 0xb072, - 0x1152: 0xb073, 0x1153: 0xb072, 0x1154: 0xb074, 0x1155: 0xb073, 0x1156: 0xb075, 0x1157: 0xb075, - 0x1158: 0xb076, 0x1159: 0xb076, 0x115a: 0xb077, 0x115b: 0xb077, 0x115c: 0xb073, 0x115d: 0xb078, - 0x115e: 0xb079, 0x115f: 0xb067, 0x1160: 0xb07a, 0x1161: 0xb07b, 0x1162: 0xb07b, 0x1163: 0xb07b, - 0x1164: 0xb07b, 0x1165: 0xb07b, 0x1166: 0xb07b, 0x1167: 0xb07b, 0x1168: 0xb07b, 0x1169: 0xb07b, - 0x116a: 0xb07b, 0x116b: 0xb07b, 0x116c: 0xb07b, 0x116d: 0xb07b, 0x116e: 0xb07b, 0x116f: 0xb07b, - 0x1170: 0xb07c, 0x1171: 0xb07c, 0x1172: 0xb07c, 0x1173: 0xb07c, 0x1174: 0xb07c, 0x1175: 0xb07c, - 0x1176: 0xb07c, 0x1177: 0xb07c, 0x1178: 0xb07c, 0x1179: 0xb07c, 0x117a: 0xb07c, 0x117b: 0xb07c, - 0x117c: 0xb07c, 0x117d: 0xb07c, 0x117e: 0xb07c, - // Block 0x46, offset 0x1180 - 0x1182: 0xb07d, 0x1183: 0xb07e, 0x1184: 0xb07f, 0x1185: 0xb080, - 0x1186: 0xb07f, 0x1187: 0xb07e, 0x118a: 0xb081, 0x118b: 0xb082, - 0x118c: 0xb083, 0x118d: 0xb07f, 0x118e: 0xb080, 0x118f: 0xb07f, - 0x1192: 0xb084, 0x1193: 0xb085, 0x1194: 0xb084, 0x1195: 0xb086, 0x1196: 0xb084, 0x1197: 0xb087, - 0x119a: 0xb088, 0x119b: 0xb089, 0x119c: 0xb08a, - 0x11a0: 0x908b, 0x11a1: 0x908b, 0x11a2: 0x908c, 0x11a3: 0x908d, - 0x11a4: 0x908b, 0x11a5: 0x908e, 0x11a6: 0x908f, 0x11a8: 0xb090, 0x11a9: 0xb091, - 0x11aa: 0xb092, 0x11ab: 0xb091, 0x11ac: 0xb093, 0x11ad: 0xb094, 0x11ae: 0xb095, - 0x11bd: 0x2000, - // Block 0x47, offset 0x11c0 - 0x11e0: 0x4000, 0x11e1: 0x4000, - // Block 0x48, offset 0x1200 - 0x1200: 0x4000, 0x1201: 0x4000, 0x1202: 0x4000, 0x1203: 0x4000, 0x1204: 0x4000, 0x1205: 0x4000, - 0x1206: 0x4000, 0x1207: 0x4000, 0x1208: 0x4000, 0x1209: 0x4000, 0x120a: 0x4000, 0x120b: 0x4000, - 0x120c: 0x4000, 0x120d: 0x4000, 0x120e: 0x4000, 0x120f: 0x4000, 0x1210: 0x4000, 0x1211: 0x4000, - 0x1212: 0x4000, 0x1213: 0x4000, 0x1214: 0x4000, 0x1215: 0x4000, 0x1216: 0x4000, 0x1217: 0x4000, - 0x1218: 0x4000, 0x1219: 0x4000, 0x121a: 0x4000, 0x121b: 0x4000, 0x121c: 0x4000, 0x121d: 0x4000, - 0x121e: 0x4000, 0x121f: 0x4000, 0x1220: 0x4000, 0x1221: 0x4000, 0x1222: 0x4000, 0x1223: 0x4000, - 0x1224: 0x4000, 0x1225: 0x4000, 0x1226: 0x4000, 0x1227: 0x4000, 0x1228: 0x4000, 0x1229: 0x4000, - 0x122a: 0x4000, 0x122b: 0x4000, 0x122c: 0x4000, 0x122d: 0x4000, 0x122e: 0x4000, 0x122f: 0x4000, - 0x1230: 0x4000, 0x1231: 0x4000, - // Block 0x49, offset 0x1240 - 0x1240: 0x4000, 0x1241: 0x4000, 0x1242: 0x4000, 0x1243: 0x4000, 0x1244: 0x4000, 0x1245: 0x4000, - 0x1246: 0x4000, 0x1247: 0x4000, 0x1248: 0x4000, 0x1249: 0x4000, 0x124a: 0x4000, 0x124b: 0x4000, - 0x124c: 0x4000, 0x124d: 0x4000, 0x124e: 0x4000, 0x124f: 0x4000, 0x1250: 0x4000, 0x1251: 0x4000, - 0x1252: 0x4000, 0x1253: 0x4000, 0x1254: 0x4000, 0x1255: 0x4000, 0x1256: 0x4000, 0x1257: 0x4000, - 0x1258: 0x4000, 0x1259: 0x4000, 0x125a: 0x4000, 0x125b: 0x4000, 0x125c: 0x4000, 0x125d: 0x4000, - 0x125e: 0x4000, 0x125f: 0x4000, 0x1260: 0x4000, 0x1261: 0x4000, 0x1262: 0x4000, 0x1263: 0x4000, - 0x1264: 0x4000, 0x1265: 0x4000, 0x1266: 0x4000, 0x1267: 0x4000, 0x1268: 0x4000, 0x1269: 0x4000, - 0x126a: 0x4000, 0x126b: 0x4000, 0x126c: 0x4000, 0x126d: 0x4000, 0x126e: 0x4000, 0x126f: 0x4000, - 0x1270: 0x4000, 0x1271: 0x4000, 0x1272: 0x4000, - // Block 0x4a, offset 0x1280 - 0x1280: 0x4000, 0x1281: 0x4000, 0x1282: 0x4000, 0x1283: 0x4000, 0x1284: 0x4000, 0x1285: 0x4000, - 0x1286: 0x4000, 0x1287: 0x4000, 0x1288: 0x4000, 0x1289: 0x4000, 0x128a: 0x4000, 0x128b: 0x4000, - 0x128c: 0x4000, 0x128d: 0x4000, 0x128e: 0x4000, 0x128f: 0x4000, 0x1290: 0x4000, 0x1291: 0x4000, - 0x1292: 0x4000, 0x1293: 0x4000, 0x1294: 0x4000, 0x1295: 0x4000, 0x1296: 0x4000, 0x1297: 0x4000, - 0x1298: 0x4000, 0x1299: 0x4000, 0x129a: 0x4000, 0x129b: 0x4000, 0x129c: 0x4000, 0x129d: 0x4000, - 0x129e: 0x4000, - // Block 0x4b, offset 0x12c0 - 0x12f0: 0x4000, 0x12f1: 0x4000, 0x12f2: 0x4000, 0x12f3: 0x4000, 0x12f4: 0x4000, 0x12f5: 0x4000, - 0x12f6: 0x4000, 0x12f7: 0x4000, 0x12f8: 0x4000, 0x12f9: 0x4000, 0x12fa: 0x4000, 0x12fb: 0x4000, - 0x12fc: 0x4000, 0x12fd: 0x4000, 0x12fe: 0x4000, 0x12ff: 0x4000, - // Block 0x4c, offset 0x1300 - 0x1300: 0x4000, 0x1301: 0x4000, 0x1302: 0x4000, 0x1303: 0x4000, 0x1304: 0x4000, 0x1305: 0x4000, - 0x1306: 0x4000, 0x1307: 0x4000, 0x1308: 0x4000, 0x1309: 0x4000, 0x130a: 0x4000, 0x130b: 0x4000, - 0x130c: 0x4000, 0x130d: 0x4000, 0x130e: 0x4000, 0x130f: 0x4000, 0x1310: 0x4000, 0x1311: 0x4000, - 0x1312: 0x4000, 0x1313: 0x4000, 0x1314: 0x4000, 0x1315: 0x4000, 0x1316: 0x4000, 0x1317: 0x4000, - 0x1318: 0x4000, 0x1319: 0x4000, 0x131a: 0x4000, 0x131b: 0x4000, 0x131c: 0x4000, 0x131d: 0x4000, - 0x131e: 0x4000, 0x131f: 0x4000, 0x1320: 0x4000, 0x1321: 0x4000, 0x1322: 0x4000, 0x1323: 0x4000, - 0x1324: 0x4000, 0x1325: 0x4000, 0x1326: 0x4000, 0x1327: 0x4000, 0x1328: 0x4000, 0x1329: 0x4000, - 0x132a: 0x4000, 0x132b: 0x4000, 0x132c: 0x4000, 0x132d: 0x4000, 0x132e: 0x4000, 0x132f: 0x4000, - 0x1330: 0x4000, 0x1331: 0x4000, 0x1332: 0x4000, 0x1333: 0x4000, 0x1334: 0x4000, 0x1335: 0x4000, - 0x1336: 0x4000, 0x1337: 0x4000, 0x1338: 0x4000, 0x1339: 0x4000, 0x133a: 0x4000, 0x133b: 0x4000, - // Block 0x4d, offset 0x1340 - 0x1344: 0x4000, - // Block 0x4e, offset 0x1380 - 0x138f: 0x4000, - // Block 0x4f, offset 0x13c0 - 0x13c0: 0x2000, 0x13c1: 0x2000, 0x13c2: 0x2000, 0x13c3: 0x2000, 0x13c4: 0x2000, 0x13c5: 0x2000, - 0x13c6: 0x2000, 0x13c7: 0x2000, 0x13c8: 0x2000, 0x13c9: 0x2000, 0x13ca: 0x2000, - 0x13d0: 0x2000, 0x13d1: 0x2000, - 0x13d2: 0x2000, 0x13d3: 0x2000, 0x13d4: 0x2000, 0x13d5: 0x2000, 0x13d6: 0x2000, 0x13d7: 0x2000, - 0x13d8: 0x2000, 0x13d9: 0x2000, 0x13da: 0x2000, 0x13db: 0x2000, 0x13dc: 0x2000, 0x13dd: 0x2000, - 0x13de: 0x2000, 0x13df: 0x2000, 0x13e0: 0x2000, 0x13e1: 0x2000, 0x13e2: 0x2000, 0x13e3: 0x2000, - 0x13e4: 0x2000, 0x13e5: 0x2000, 0x13e6: 0x2000, 0x13e7: 0x2000, 0x13e8: 0x2000, 0x13e9: 0x2000, - 0x13ea: 0x2000, 0x13eb: 0x2000, 0x13ec: 0x2000, 0x13ed: 0x2000, - 0x13f0: 0x2000, 0x13f1: 0x2000, 0x13f2: 0x2000, 0x13f3: 0x2000, 0x13f4: 0x2000, 0x13f5: 0x2000, - 0x13f6: 0x2000, 0x13f7: 0x2000, 0x13f8: 0x2000, 0x13f9: 0x2000, 0x13fa: 0x2000, 0x13fb: 0x2000, - 0x13fc: 0x2000, 0x13fd: 0x2000, 0x13fe: 0x2000, 0x13ff: 0x2000, - // Block 0x50, offset 0x1400 - 0x1400: 0x2000, 0x1401: 0x2000, 0x1402: 0x2000, 0x1403: 0x2000, 0x1404: 0x2000, 0x1405: 0x2000, - 0x1406: 0x2000, 0x1407: 0x2000, 0x1408: 0x2000, 0x1409: 0x2000, 0x140a: 0x2000, 0x140b: 0x2000, - 0x140c: 0x2000, 0x140d: 0x2000, 0x140e: 0x2000, 0x140f: 0x2000, 0x1410: 0x2000, 0x1411: 0x2000, - 0x1412: 0x2000, 0x1413: 0x2000, 0x1414: 0x2000, 0x1415: 0x2000, 0x1416: 0x2000, 0x1417: 0x2000, - 0x1418: 0x2000, 0x1419: 0x2000, 0x141a: 0x2000, 0x141b: 0x2000, 0x141c: 0x2000, 0x141d: 0x2000, - 0x141e: 0x2000, 0x141f: 0x2000, 0x1420: 0x2000, 0x1421: 0x2000, 0x1422: 0x2000, 0x1423: 0x2000, - 0x1424: 0x2000, 0x1425: 0x2000, 0x1426: 0x2000, 0x1427: 0x2000, 0x1428: 0x2000, 0x1429: 0x2000, - 0x1430: 0x2000, 0x1431: 0x2000, 0x1432: 0x2000, 0x1433: 0x2000, 0x1434: 0x2000, 0x1435: 0x2000, - 0x1436: 0x2000, 0x1437: 0x2000, 0x1438: 0x2000, 0x1439: 0x2000, 0x143a: 0x2000, 0x143b: 0x2000, - 0x143c: 0x2000, 0x143d: 0x2000, 0x143e: 0x2000, 0x143f: 0x2000, - // Block 0x51, offset 0x1440 - 0x1440: 0x2000, 0x1441: 0x2000, 0x1442: 0x2000, 0x1443: 0x2000, 0x1444: 0x2000, 0x1445: 0x2000, - 0x1446: 0x2000, 0x1447: 0x2000, 0x1448: 0x2000, 0x1449: 0x2000, 0x144a: 0x2000, 0x144b: 0x2000, - 0x144c: 0x2000, 0x144d: 0x2000, 0x144e: 0x4000, 0x144f: 0x2000, 0x1450: 0x2000, 0x1451: 0x4000, - 0x1452: 0x4000, 0x1453: 0x4000, 0x1454: 0x4000, 0x1455: 0x4000, 0x1456: 0x4000, 0x1457: 0x4000, - 0x1458: 0x4000, 0x1459: 0x4000, 0x145a: 0x4000, 0x145b: 0x2000, 0x145c: 0x2000, 0x145d: 0x2000, - 0x145e: 0x2000, 0x145f: 0x2000, 0x1460: 0x2000, 0x1461: 0x2000, 0x1462: 0x2000, 0x1463: 0x2000, - 0x1464: 0x2000, 0x1465: 0x2000, 0x1466: 0x2000, 0x1467: 0x2000, 0x1468: 0x2000, 0x1469: 0x2000, - 0x146a: 0x2000, 0x146b: 0x2000, 0x146c: 0x2000, - // Block 0x52, offset 0x1480 - 0x1480: 0x4000, 0x1481: 0x4000, 0x1482: 0x4000, - 0x1490: 0x4000, 0x1491: 0x4000, - 0x1492: 0x4000, 0x1493: 0x4000, 0x1494: 0x4000, 0x1495: 0x4000, 0x1496: 0x4000, 0x1497: 0x4000, - 0x1498: 0x4000, 0x1499: 0x4000, 0x149a: 0x4000, 0x149b: 0x4000, 0x149c: 0x4000, 0x149d: 0x4000, - 0x149e: 0x4000, 0x149f: 0x4000, 0x14a0: 0x4000, 0x14a1: 0x4000, 0x14a2: 0x4000, 0x14a3: 0x4000, - 0x14a4: 0x4000, 0x14a5: 0x4000, 0x14a6: 0x4000, 0x14a7: 0x4000, 0x14a8: 0x4000, 0x14a9: 0x4000, - 0x14aa: 0x4000, 0x14ab: 0x4000, 0x14ac: 0x4000, 0x14ad: 0x4000, 0x14ae: 0x4000, 0x14af: 0x4000, - 0x14b0: 0x4000, 0x14b1: 0x4000, 0x14b2: 0x4000, 0x14b3: 0x4000, 0x14b4: 0x4000, 0x14b5: 0x4000, - 0x14b6: 0x4000, 0x14b7: 0x4000, 0x14b8: 0x4000, 0x14b9: 0x4000, 0x14ba: 0x4000, 0x14bb: 0x4000, - // Block 0x53, offset 0x14c0 - 0x14c0: 0x4000, 0x14c1: 0x4000, 0x14c2: 0x4000, 0x14c3: 0x4000, 0x14c4: 0x4000, 0x14c5: 0x4000, - 0x14c6: 0x4000, 0x14c7: 0x4000, 0x14c8: 0x4000, - 0x14d0: 0x4000, 0x14d1: 0x4000, - 0x14e0: 0x4000, 0x14e1: 0x4000, 0x14e2: 0x4000, 0x14e3: 0x4000, - 0x14e4: 0x4000, 0x14e5: 0x4000, - // Block 0x54, offset 0x1500 - 0x1500: 0x4000, 0x1501: 0x4000, 0x1502: 0x4000, 0x1503: 0x4000, 0x1504: 0x4000, 0x1505: 0x4000, - 0x1506: 0x4000, 0x1507: 0x4000, 0x1508: 0x4000, 0x1509: 0x4000, 0x150a: 0x4000, 0x150b: 0x4000, - 0x150c: 0x4000, 0x150d: 0x4000, 0x150e: 0x4000, 0x150f: 0x4000, 0x1510: 0x4000, 0x1511: 0x4000, - 0x1512: 0x4000, 0x1513: 0x4000, 0x1514: 0x4000, 0x1515: 0x4000, 0x1516: 0x4000, 0x1517: 0x4000, - 0x1518: 0x4000, 0x1519: 0x4000, 0x151a: 0x4000, 0x151b: 0x4000, 0x151c: 0x4000, 0x151d: 0x4000, - 0x151e: 0x4000, 0x151f: 0x4000, 0x1520: 0x4000, - 0x152d: 0x4000, 0x152e: 0x4000, 0x152f: 0x4000, - 0x1530: 0x4000, 0x1531: 0x4000, 0x1532: 0x4000, 0x1533: 0x4000, 0x1534: 0x4000, 0x1535: 0x4000, - 0x1537: 0x4000, 0x1538: 0x4000, 0x1539: 0x4000, 0x153a: 0x4000, 0x153b: 0x4000, - 0x153c: 0x4000, 0x153d: 0x4000, 0x153e: 0x4000, 0x153f: 0x4000, - // Block 0x55, offset 0x1540 - 0x1540: 0x4000, 0x1541: 0x4000, 0x1542: 0x4000, 0x1543: 0x4000, 0x1544: 0x4000, 0x1545: 0x4000, - 0x1546: 0x4000, 0x1547: 0x4000, 0x1548: 0x4000, 0x1549: 0x4000, 0x154a: 0x4000, 0x154b: 0x4000, - 0x154c: 0x4000, 0x154d: 0x4000, 0x154e: 0x4000, 0x154f: 0x4000, 0x1550: 0x4000, 0x1551: 0x4000, - 0x1552: 0x4000, 0x1553: 0x4000, 0x1554: 0x4000, 0x1555: 0x4000, 0x1556: 0x4000, 0x1557: 0x4000, - 0x1558: 0x4000, 0x1559: 0x4000, 0x155a: 0x4000, 0x155b: 0x4000, 0x155c: 0x4000, 0x155d: 0x4000, - 0x155e: 0x4000, 0x155f: 0x4000, 0x1560: 0x4000, 0x1561: 0x4000, 0x1562: 0x4000, 0x1563: 0x4000, - 0x1564: 0x4000, 0x1565: 0x4000, 0x1566: 0x4000, 0x1567: 0x4000, 0x1568: 0x4000, 0x1569: 0x4000, - 0x156a: 0x4000, 0x156b: 0x4000, 0x156c: 0x4000, 0x156d: 0x4000, 0x156e: 0x4000, 0x156f: 0x4000, - 0x1570: 0x4000, 0x1571: 0x4000, 0x1572: 0x4000, 0x1573: 0x4000, 0x1574: 0x4000, 0x1575: 0x4000, - 0x1576: 0x4000, 0x1577: 0x4000, 0x1578: 0x4000, 0x1579: 0x4000, 0x157a: 0x4000, 0x157b: 0x4000, - 0x157c: 0x4000, 0x157e: 0x4000, 0x157f: 0x4000, - // Block 0x56, offset 0x1580 - 0x1580: 0x4000, 0x1581: 0x4000, 0x1582: 0x4000, 0x1583: 0x4000, 0x1584: 0x4000, 0x1585: 0x4000, - 0x1586: 0x4000, 0x1587: 0x4000, 0x1588: 0x4000, 0x1589: 0x4000, 0x158a: 0x4000, 0x158b: 0x4000, - 0x158c: 0x4000, 0x158d: 0x4000, 0x158e: 0x4000, 0x158f: 0x4000, 0x1590: 0x4000, 0x1591: 0x4000, - 0x1592: 0x4000, 0x1593: 0x4000, - 0x15a0: 0x4000, 0x15a1: 0x4000, 0x15a2: 0x4000, 0x15a3: 0x4000, - 0x15a4: 0x4000, 0x15a5: 0x4000, 0x15a6: 0x4000, 0x15a7: 0x4000, 0x15a8: 0x4000, 0x15a9: 0x4000, - 0x15aa: 0x4000, 0x15ab: 0x4000, 0x15ac: 0x4000, 0x15ad: 0x4000, 0x15ae: 0x4000, 0x15af: 0x4000, - 0x15b0: 0x4000, 0x15b1: 0x4000, 0x15b2: 0x4000, 0x15b3: 0x4000, 0x15b4: 0x4000, 0x15b5: 0x4000, - 0x15b6: 0x4000, 0x15b7: 0x4000, 0x15b8: 0x4000, 0x15b9: 0x4000, 0x15ba: 0x4000, 0x15bb: 0x4000, - 0x15bc: 0x4000, 0x15bd: 0x4000, 0x15be: 0x4000, 0x15bf: 0x4000, - // Block 0x57, offset 0x15c0 - 0x15c0: 0x4000, 0x15c1: 0x4000, 0x15c2: 0x4000, 0x15c3: 0x4000, 0x15c4: 0x4000, 0x15c5: 0x4000, - 0x15c6: 0x4000, 0x15c7: 0x4000, 0x15c8: 0x4000, 0x15c9: 0x4000, 0x15ca: 0x4000, - 0x15cf: 0x4000, 0x15d0: 0x4000, 0x15d1: 0x4000, - 0x15d2: 0x4000, 0x15d3: 0x4000, - 0x15e0: 0x4000, 0x15e1: 0x4000, 0x15e2: 0x4000, 0x15e3: 0x4000, - 0x15e4: 0x4000, 0x15e5: 0x4000, 0x15e6: 0x4000, 0x15e7: 0x4000, 0x15e8: 0x4000, 0x15e9: 0x4000, - 0x15ea: 0x4000, 0x15eb: 0x4000, 0x15ec: 0x4000, 0x15ed: 0x4000, 0x15ee: 0x4000, 0x15ef: 0x4000, - 0x15f0: 0x4000, 0x15f4: 0x4000, - 0x15f8: 0x4000, 0x15f9: 0x4000, 0x15fa: 0x4000, 0x15fb: 0x4000, - 0x15fc: 0x4000, 0x15fd: 0x4000, 0x15fe: 0x4000, 0x15ff: 0x4000, - // Block 0x58, offset 0x1600 - 0x1600: 0x4000, 0x1602: 0x4000, 0x1603: 0x4000, 0x1604: 0x4000, 0x1605: 0x4000, - 0x1606: 0x4000, 0x1607: 0x4000, 0x1608: 0x4000, 0x1609: 0x4000, 0x160a: 0x4000, 0x160b: 0x4000, - 0x160c: 0x4000, 0x160d: 0x4000, 0x160e: 0x4000, 0x160f: 0x4000, 0x1610: 0x4000, 0x1611: 0x4000, - 0x1612: 0x4000, 0x1613: 0x4000, 0x1614: 0x4000, 0x1615: 0x4000, 0x1616: 0x4000, 0x1617: 0x4000, - 0x1618: 0x4000, 0x1619: 0x4000, 0x161a: 0x4000, 0x161b: 0x4000, 0x161c: 0x4000, 0x161d: 0x4000, - 0x161e: 0x4000, 0x161f: 0x4000, 0x1620: 0x4000, 0x1621: 0x4000, 0x1622: 0x4000, 0x1623: 0x4000, - 0x1624: 0x4000, 0x1625: 0x4000, 0x1626: 0x4000, 0x1627: 0x4000, 0x1628: 0x4000, 0x1629: 0x4000, - 0x162a: 0x4000, 0x162b: 0x4000, 0x162c: 0x4000, 0x162d: 0x4000, 0x162e: 0x4000, 0x162f: 0x4000, - 0x1630: 0x4000, 0x1631: 0x4000, 0x1632: 0x4000, 0x1633: 0x4000, 0x1634: 0x4000, 0x1635: 0x4000, - 0x1636: 0x4000, 0x1637: 0x4000, 0x1638: 0x4000, 0x1639: 0x4000, 0x163a: 0x4000, 0x163b: 0x4000, - 0x163c: 0x4000, 0x163d: 0x4000, 0x163e: 0x4000, 0x163f: 0x4000, - // Block 0x59, offset 0x1640 - 0x1640: 0x4000, 0x1641: 0x4000, 0x1642: 0x4000, 0x1643: 0x4000, 0x1644: 0x4000, 0x1645: 0x4000, - 0x1646: 0x4000, 0x1647: 0x4000, 0x1648: 0x4000, 0x1649: 0x4000, 0x164a: 0x4000, 0x164b: 0x4000, - 0x164c: 0x4000, 0x164d: 0x4000, 0x164e: 0x4000, 0x164f: 0x4000, 0x1650: 0x4000, 0x1651: 0x4000, - 0x1652: 0x4000, 0x1653: 0x4000, 0x1654: 0x4000, 0x1655: 0x4000, 0x1656: 0x4000, 0x1657: 0x4000, - 0x1658: 0x4000, 0x1659: 0x4000, 0x165a: 0x4000, 0x165b: 0x4000, 0x165c: 0x4000, 0x165d: 0x4000, - 0x165e: 0x4000, 0x165f: 0x4000, 0x1660: 0x4000, 0x1661: 0x4000, 0x1662: 0x4000, 0x1663: 0x4000, - 0x1664: 0x4000, 0x1665: 0x4000, 0x1666: 0x4000, 0x1667: 0x4000, 0x1668: 0x4000, 0x1669: 0x4000, - 0x166a: 0x4000, 0x166b: 0x4000, 0x166c: 0x4000, 0x166d: 0x4000, 0x166e: 0x4000, 0x166f: 0x4000, - 0x1670: 0x4000, 0x1671: 0x4000, 0x1672: 0x4000, 0x1673: 0x4000, 0x1674: 0x4000, 0x1675: 0x4000, - 0x1676: 0x4000, 0x1677: 0x4000, 0x1678: 0x4000, 0x1679: 0x4000, 0x167a: 0x4000, 0x167b: 0x4000, - 0x167c: 0x4000, 0x167f: 0x4000, - // Block 0x5a, offset 0x1680 - 0x1680: 0x4000, 0x1681: 0x4000, 0x1682: 0x4000, 0x1683: 0x4000, 0x1684: 0x4000, 0x1685: 0x4000, - 0x1686: 0x4000, 0x1687: 0x4000, 0x1688: 0x4000, 0x1689: 0x4000, 0x168a: 0x4000, 0x168b: 0x4000, - 0x168c: 0x4000, 0x168d: 0x4000, 0x168e: 0x4000, 0x168f: 0x4000, 0x1690: 0x4000, 0x1691: 0x4000, - 0x1692: 0x4000, 0x1693: 0x4000, 0x1694: 0x4000, 0x1695: 0x4000, 0x1696: 0x4000, 0x1697: 0x4000, - 0x1698: 0x4000, 0x1699: 0x4000, 0x169a: 0x4000, 0x169b: 0x4000, 0x169c: 0x4000, 0x169d: 0x4000, - 0x169e: 0x4000, 0x169f: 0x4000, 0x16a0: 0x4000, 0x16a1: 0x4000, 0x16a2: 0x4000, 0x16a3: 0x4000, - 0x16a4: 0x4000, 0x16a5: 0x4000, 0x16a6: 0x4000, 0x16a7: 0x4000, 0x16a8: 0x4000, 0x16a9: 0x4000, - 0x16aa: 0x4000, 0x16ab: 0x4000, 0x16ac: 0x4000, 0x16ad: 0x4000, 0x16ae: 0x4000, 0x16af: 0x4000, - 0x16b0: 0x4000, 0x16b1: 0x4000, 0x16b2: 0x4000, 0x16b3: 0x4000, 0x16b4: 0x4000, 0x16b5: 0x4000, - 0x16b6: 0x4000, 0x16b7: 0x4000, 0x16b8: 0x4000, 0x16b9: 0x4000, 0x16ba: 0x4000, 0x16bb: 0x4000, - 0x16bc: 0x4000, 0x16bd: 0x4000, - // Block 0x5b, offset 0x16c0 - 0x16cb: 0x4000, - 0x16cc: 0x4000, 0x16cd: 0x4000, 0x16ce: 0x4000, 0x16d0: 0x4000, 0x16d1: 0x4000, - 0x16d2: 0x4000, 0x16d3: 0x4000, 0x16d4: 0x4000, 0x16d5: 0x4000, 0x16d6: 0x4000, 0x16d7: 0x4000, - 0x16d8: 0x4000, 0x16d9: 0x4000, 0x16da: 0x4000, 0x16db: 0x4000, 0x16dc: 0x4000, 0x16dd: 0x4000, - 0x16de: 0x4000, 0x16df: 0x4000, 0x16e0: 0x4000, 0x16e1: 0x4000, 0x16e2: 0x4000, 0x16e3: 0x4000, - 0x16e4: 0x4000, 0x16e5: 0x4000, 0x16e6: 0x4000, 0x16e7: 0x4000, - 0x16fa: 0x4000, - // Block 0x5c, offset 0x1700 - 0x1715: 0x4000, 0x1716: 0x4000, - 0x1724: 0x4000, - // Block 0x5d, offset 0x1740 - 0x177b: 0x4000, - 0x177c: 0x4000, 0x177d: 0x4000, 0x177e: 0x4000, 0x177f: 0x4000, - // Block 0x5e, offset 0x1780 - 0x1780: 0x4000, 0x1781: 0x4000, 0x1782: 0x4000, 0x1783: 0x4000, 0x1784: 0x4000, 0x1785: 0x4000, - 0x1786: 0x4000, 0x1787: 0x4000, 0x1788: 0x4000, 0x1789: 0x4000, 0x178a: 0x4000, 0x178b: 0x4000, - 0x178c: 0x4000, 0x178d: 0x4000, 0x178e: 0x4000, 0x178f: 0x4000, - // Block 0x5f, offset 0x17c0 - 0x17c0: 0x4000, 0x17c1: 0x4000, 0x17c2: 0x4000, 0x17c3: 0x4000, 0x17c4: 0x4000, 0x17c5: 0x4000, - 0x17cc: 0x4000, 0x17d0: 0x4000, 0x17d1: 0x4000, - 0x17d2: 0x4000, - 0x17eb: 0x4000, 0x17ec: 0x4000, - 0x17f4: 0x4000, 0x17f5: 0x4000, - 0x17f6: 0x4000, 0x17f7: 0x4000, 0x17f8: 0x4000, 0x17f9: 0x4000, - // Block 0x60, offset 0x1800 - 0x1810: 0x4000, 0x1811: 0x4000, - 0x1812: 0x4000, 0x1813: 0x4000, 0x1814: 0x4000, 0x1815: 0x4000, 0x1816: 0x4000, 0x1817: 0x4000, - 0x1818: 0x4000, 0x1819: 0x4000, 0x181a: 0x4000, 0x181b: 0x4000, 0x181c: 0x4000, 0x181d: 0x4000, - 0x181e: 0x4000, 0x181f: 0x4000, 0x1820: 0x4000, 0x1821: 0x4000, 0x1822: 0x4000, 0x1823: 0x4000, - 0x1824: 0x4000, 0x1825: 0x4000, 0x1826: 0x4000, 0x1827: 0x4000, 0x1828: 0x4000, 0x1829: 0x4000, - 0x182a: 0x4000, 0x182b: 0x4000, 0x182c: 0x4000, 0x182d: 0x4000, 0x182e: 0x4000, 0x182f: 0x4000, - 0x1830: 0x4000, 0x1831: 0x4000, 0x1832: 0x4000, 0x1833: 0x4000, 0x1834: 0x4000, 0x1835: 0x4000, - 0x1836: 0x4000, 0x1837: 0x4000, 0x1838: 0x4000, 0x1839: 0x4000, 0x183a: 0x4000, 0x183b: 0x4000, - 0x183c: 0x4000, 0x183d: 0x4000, 0x183e: 0x4000, - // Block 0x61, offset 0x1840 - 0x1840: 0x4000, 0x1841: 0x4000, 0x1842: 0x4000, 0x1843: 0x4000, 0x1844: 0x4000, 0x1845: 0x4000, - 0x1846: 0x4000, 0x1847: 0x4000, 0x1848: 0x4000, 0x1849: 0x4000, 0x184a: 0x4000, 0x184b: 0x4000, - 0x184c: 0x4000, 0x184d: 0x4000, 0x184e: 0x4000, 0x184f: 0x4000, 0x1850: 0x4000, 0x1851: 0x4000, - 0x1852: 0x4000, 0x1853: 0x4000, 0x1854: 0x4000, 0x1855: 0x4000, 0x1856: 0x4000, 0x1857: 0x4000, - 0x1858: 0x4000, 0x1859: 0x4000, 0x185a: 0x4000, 0x185b: 0x4000, 0x185c: 0x4000, 0x185d: 0x4000, - 0x185e: 0x4000, 0x185f: 0x4000, 0x1860: 0x4000, 0x1861: 0x4000, 0x1862: 0x4000, 0x1863: 0x4000, - 0x1864: 0x4000, 0x1865: 0x4000, 0x1866: 0x4000, 0x1867: 0x4000, 0x1868: 0x4000, 0x1869: 0x4000, - 0x186a: 0x4000, 0x186b: 0x4000, 0x186c: 0x4000, 0x186d: 0x4000, 0x186e: 0x4000, 0x186f: 0x4000, - 0x1870: 0x4000, 0x1873: 0x4000, 0x1874: 0x4000, 0x1875: 0x4000, - 0x1876: 0x4000, 0x187a: 0x4000, - 0x187c: 0x4000, 0x187d: 0x4000, 0x187e: 0x4000, 0x187f: 0x4000, - // Block 0x62, offset 0x1880 - 0x1880: 0x4000, 0x1881: 0x4000, 0x1882: 0x4000, 0x1883: 0x4000, 0x1884: 0x4000, 0x1885: 0x4000, - 0x1886: 0x4000, 0x1887: 0x4000, 0x1888: 0x4000, 0x1889: 0x4000, 0x188a: 0x4000, 0x188b: 0x4000, - 0x188c: 0x4000, 0x188d: 0x4000, 0x188e: 0x4000, 0x188f: 0x4000, 0x1890: 0x4000, 0x1891: 0x4000, - 0x1892: 0x4000, 0x1893: 0x4000, 0x1894: 0x4000, 0x1895: 0x4000, 0x1896: 0x4000, 0x1897: 0x4000, - 0x1898: 0x4000, 0x1899: 0x4000, 0x189a: 0x4000, 0x189b: 0x4000, 0x189c: 0x4000, 0x189d: 0x4000, - 0x189e: 0x4000, 0x189f: 0x4000, 0x18a0: 0x4000, 0x18a1: 0x4000, 0x18a2: 0x4000, - 0x18b0: 0x4000, 0x18b1: 0x4000, 0x18b2: 0x4000, 0x18b3: 0x4000, 0x18b4: 0x4000, 0x18b5: 0x4000, - 0x18b6: 0x4000, 0x18b7: 0x4000, 0x18b8: 0x4000, 0x18b9: 0x4000, - // Block 0x63, offset 0x18c0 - 0x18c0: 0x4000, 0x18c1: 0x4000, 0x18c2: 0x4000, - 0x18d0: 0x4000, 0x18d1: 0x4000, - 0x18d2: 0x4000, 0x18d3: 0x4000, 0x18d4: 0x4000, 0x18d5: 0x4000, 0x18d6: 0x4000, 0x18d7: 0x4000, - 0x18d8: 0x4000, 0x18d9: 0x4000, 0x18da: 0x4000, 0x18db: 0x4000, 0x18dc: 0x4000, 0x18dd: 0x4000, - 0x18de: 0x4000, 0x18df: 0x4000, 0x18e0: 0x4000, 0x18e1: 0x4000, 0x18e2: 0x4000, 0x18e3: 0x4000, - 0x18e4: 0x4000, 0x18e5: 0x4000, 0x18e6: 0x4000, 0x18e7: 0x4000, 0x18e8: 0x4000, 0x18e9: 0x4000, - 0x18ea: 0x4000, 0x18eb: 0x4000, 0x18ec: 0x4000, 0x18ed: 0x4000, 0x18ee: 0x4000, 0x18ef: 0x4000, - 0x18f0: 0x4000, 0x18f1: 0x4000, 0x18f2: 0x4000, 0x18f3: 0x4000, 0x18f4: 0x4000, 0x18f5: 0x4000, - 0x18f6: 0x4000, 0x18f7: 0x4000, 0x18f8: 0x4000, 0x18f9: 0x4000, 0x18fa: 0x4000, 0x18fb: 0x4000, - 0x18fc: 0x4000, 0x18fd: 0x4000, 0x18fe: 0x4000, 0x18ff: 0x4000, - // Block 0x64, offset 0x1900 - 0x1900: 0x2000, 0x1901: 0x2000, 0x1902: 0x2000, 0x1903: 0x2000, 0x1904: 0x2000, 0x1905: 0x2000, - 0x1906: 0x2000, 0x1907: 0x2000, 0x1908: 0x2000, 0x1909: 0x2000, 0x190a: 0x2000, 0x190b: 0x2000, - 0x190c: 0x2000, 0x190d: 0x2000, 0x190e: 0x2000, 0x190f: 0x2000, 0x1910: 0x2000, 0x1911: 0x2000, - 0x1912: 0x2000, 0x1913: 0x2000, 0x1914: 0x2000, 0x1915: 0x2000, 0x1916: 0x2000, 0x1917: 0x2000, - 0x1918: 0x2000, 0x1919: 0x2000, 0x191a: 0x2000, 0x191b: 0x2000, 0x191c: 0x2000, 0x191d: 0x2000, - 0x191e: 0x2000, 0x191f: 0x2000, 0x1920: 0x2000, 0x1921: 0x2000, 0x1922: 0x2000, 0x1923: 0x2000, - 0x1924: 0x2000, 0x1925: 0x2000, 0x1926: 0x2000, 0x1927: 0x2000, 0x1928: 0x2000, 0x1929: 0x2000, - 0x192a: 0x2000, 0x192b: 0x2000, 0x192c: 0x2000, 0x192d: 0x2000, 0x192e: 0x2000, 0x192f: 0x2000, - 0x1930: 0x2000, 0x1931: 0x2000, 0x1932: 0x2000, 0x1933: 0x2000, 0x1934: 0x2000, 0x1935: 0x2000, - 0x1936: 0x2000, 0x1937: 0x2000, 0x1938: 0x2000, 0x1939: 0x2000, 0x193a: 0x2000, 0x193b: 0x2000, - 0x193c: 0x2000, 0x193d: 0x2000, -} - -// widthIndex: 22 blocks, 1408 entries, 1408 bytes -// Block 0 is the zero block. -var widthIndex = [1408]uint8{ - // Block 0x0, offset 0x0 - // Block 0x1, offset 0x40 - // Block 0x2, offset 0x80 - // Block 0x3, offset 0xc0 - 0xc2: 0x01, 0xc3: 0x02, 0xc4: 0x03, 0xc5: 0x04, 0xc7: 0x05, - 0xc9: 0x06, 0xcb: 0x07, 0xcc: 0x08, 0xcd: 0x09, 0xce: 0x0a, 0xcf: 0x0b, - 0xd0: 0x0c, 0xd1: 0x0d, - 0xe1: 0x02, 0xe2: 0x03, 0xe3: 0x04, 0xe4: 0x05, 0xe5: 0x06, 0xe6: 0x06, 0xe7: 0x06, - 0xe8: 0x06, 0xe9: 0x06, 0xea: 0x07, 0xeb: 0x06, 0xec: 0x06, 0xed: 0x08, 0xee: 0x09, 0xef: 0x0a, - 0xf0: 0x0f, 0xf3: 0x12, 0xf4: 0x13, - // Block 0x4, offset 0x100 - 0x104: 0x0e, 0x105: 0x0f, - // Block 0x5, offset 0x140 - 0x140: 0x10, 0x141: 0x11, 0x142: 0x12, 0x144: 0x13, 0x145: 0x14, 0x146: 0x15, 0x147: 0x16, - 0x148: 0x17, 0x149: 0x18, 0x14a: 0x19, 0x14c: 0x1a, 0x14f: 0x1b, - 0x151: 0x1c, 0x152: 0x08, 0x153: 0x1d, 0x154: 0x1e, 0x155: 0x1f, 0x156: 0x20, 0x157: 0x21, - 0x158: 0x22, 0x159: 0x23, 0x15a: 0x24, 0x15b: 0x25, 0x15c: 0x26, 0x15d: 0x27, 0x15e: 0x28, 0x15f: 0x29, - 0x166: 0x2a, - 0x16c: 0x2b, 0x16d: 0x2c, - 0x17a: 0x2d, 0x17b: 0x2e, 0x17c: 0x0e, 0x17d: 0x0e, 0x17e: 0x0e, 0x17f: 0x2f, - // Block 0x6, offset 0x180 - 0x180: 0x30, 0x181: 0x31, 0x182: 0x32, 0x183: 0x33, 0x184: 0x34, 0x185: 0x35, 0x186: 0x36, 0x187: 0x37, - 0x188: 0x38, 0x189: 0x39, 0x18a: 0x0e, 0x18b: 0x3a, 0x18c: 0x0e, 0x18d: 0x0e, 0x18e: 0x0e, 0x18f: 0x0e, - 0x190: 0x0e, 0x191: 0x0e, 0x192: 0x0e, 0x193: 0x0e, 0x194: 0x0e, 0x195: 0x0e, 0x196: 0x0e, 0x197: 0x0e, - 0x198: 0x0e, 0x199: 0x0e, 0x19a: 0x0e, 0x19b: 0x0e, 0x19c: 0x0e, 0x19d: 0x0e, 0x19e: 0x0e, 0x19f: 0x0e, - 0x1a0: 0x0e, 0x1a1: 0x0e, 0x1a2: 0x0e, 0x1a3: 0x0e, 0x1a4: 0x0e, 0x1a5: 0x0e, 0x1a6: 0x0e, 0x1a7: 0x0e, - 0x1a8: 0x0e, 0x1a9: 0x0e, 0x1aa: 0x0e, 0x1ab: 0x0e, 0x1ac: 0x0e, 0x1ad: 0x0e, 0x1ae: 0x0e, 0x1af: 0x0e, - 0x1b0: 0x0e, 0x1b1: 0x0e, 0x1b2: 0x0e, 0x1b3: 0x0e, 0x1b4: 0x0e, 0x1b5: 0x0e, 0x1b6: 0x0e, 0x1b7: 0x0e, - 0x1b8: 0x0e, 0x1b9: 0x0e, 0x1ba: 0x0e, 0x1bb: 0x0e, 0x1bc: 0x0e, 0x1bd: 0x0e, 0x1be: 0x0e, 0x1bf: 0x0e, - // Block 0x7, offset 0x1c0 - 0x1c0: 0x0e, 0x1c1: 0x0e, 0x1c2: 0x0e, 0x1c3: 0x0e, 0x1c4: 0x0e, 0x1c5: 0x0e, 0x1c6: 0x0e, 0x1c7: 0x0e, - 0x1c8: 0x0e, 0x1c9: 0x0e, 0x1ca: 0x0e, 0x1cb: 0x0e, 0x1cc: 0x0e, 0x1cd: 0x0e, 0x1ce: 0x0e, 0x1cf: 0x0e, - 0x1d0: 0x0e, 0x1d1: 0x0e, 0x1d2: 0x0e, 0x1d3: 0x0e, 0x1d4: 0x0e, 0x1d5: 0x0e, 0x1d6: 0x0e, 0x1d7: 0x0e, - 0x1d8: 0x0e, 0x1d9: 0x0e, 0x1da: 0x0e, 0x1db: 0x0e, 0x1dc: 0x0e, 0x1dd: 0x0e, 0x1de: 0x0e, 0x1df: 0x0e, - 0x1e0: 0x0e, 0x1e1: 0x0e, 0x1e2: 0x0e, 0x1e3: 0x0e, 0x1e4: 0x0e, 0x1e5: 0x0e, 0x1e6: 0x0e, 0x1e7: 0x0e, - 0x1e8: 0x0e, 0x1e9: 0x0e, 0x1ea: 0x0e, 0x1eb: 0x0e, 0x1ec: 0x0e, 0x1ed: 0x0e, 0x1ee: 0x0e, 0x1ef: 0x0e, - 0x1f0: 0x0e, 0x1f1: 0x0e, 0x1f2: 0x0e, 0x1f3: 0x0e, 0x1f4: 0x0e, 0x1f5: 0x0e, 0x1f6: 0x0e, - 0x1f8: 0x0e, 0x1f9: 0x0e, 0x1fa: 0x0e, 0x1fb: 0x0e, 0x1fc: 0x0e, 0x1fd: 0x0e, 0x1fe: 0x0e, 0x1ff: 0x0e, - // Block 0x8, offset 0x200 - 0x200: 0x0e, 0x201: 0x0e, 0x202: 0x0e, 0x203: 0x0e, 0x204: 0x0e, 0x205: 0x0e, 0x206: 0x0e, 0x207: 0x0e, - 0x208: 0x0e, 0x209: 0x0e, 0x20a: 0x0e, 0x20b: 0x0e, 0x20c: 0x0e, 0x20d: 0x0e, 0x20e: 0x0e, 0x20f: 0x0e, - 0x210: 0x0e, 0x211: 0x0e, 0x212: 0x0e, 0x213: 0x0e, 0x214: 0x0e, 0x215: 0x0e, 0x216: 0x0e, 0x217: 0x0e, - 0x218: 0x0e, 0x219: 0x0e, 0x21a: 0x0e, 0x21b: 0x0e, 0x21c: 0x0e, 0x21d: 0x0e, 0x21e: 0x0e, 0x21f: 0x0e, - 0x220: 0x0e, 0x221: 0x0e, 0x222: 0x0e, 0x223: 0x0e, 0x224: 0x0e, 0x225: 0x0e, 0x226: 0x0e, 0x227: 0x0e, - 0x228: 0x0e, 0x229: 0x0e, 0x22a: 0x0e, 0x22b: 0x0e, 0x22c: 0x0e, 0x22d: 0x0e, 0x22e: 0x0e, 0x22f: 0x0e, - 0x230: 0x0e, 0x231: 0x0e, 0x232: 0x0e, 0x233: 0x0e, 0x234: 0x0e, 0x235: 0x0e, 0x236: 0x0e, 0x237: 0x0e, - 0x238: 0x0e, 0x239: 0x0e, 0x23a: 0x0e, 0x23b: 0x0e, 0x23c: 0x0e, 0x23d: 0x0e, 0x23e: 0x0e, 0x23f: 0x0e, - // Block 0x9, offset 0x240 - 0x240: 0x0e, 0x241: 0x0e, 0x242: 0x0e, 0x243: 0x0e, 0x244: 0x0e, 0x245: 0x0e, 0x246: 0x0e, 0x247: 0x0e, - 0x248: 0x0e, 0x249: 0x0e, 0x24a: 0x0e, 0x24b: 0x0e, 0x24c: 0x0e, 0x24d: 0x0e, 0x24e: 0x0e, 0x24f: 0x0e, - 0x250: 0x0e, 0x251: 0x0e, 0x252: 0x3b, 0x253: 0x3c, - 0x265: 0x3d, - 0x270: 0x0e, 0x271: 0x0e, 0x272: 0x0e, 0x273: 0x0e, 0x274: 0x0e, 0x275: 0x0e, 0x276: 0x0e, 0x277: 0x0e, - 0x278: 0x0e, 0x279: 0x0e, 0x27a: 0x0e, 0x27b: 0x0e, 0x27c: 0x0e, 0x27d: 0x0e, 0x27e: 0x0e, 0x27f: 0x0e, - // Block 0xa, offset 0x280 - 0x280: 0x0e, 0x281: 0x0e, 0x282: 0x0e, 0x283: 0x0e, 0x284: 0x0e, 0x285: 0x0e, 0x286: 0x0e, 0x287: 0x0e, - 0x288: 0x0e, 0x289: 0x0e, 0x28a: 0x0e, 0x28b: 0x0e, 0x28c: 0x0e, 0x28d: 0x0e, 0x28e: 0x0e, 0x28f: 0x0e, - 0x290: 0x0e, 0x291: 0x0e, 0x292: 0x0e, 0x293: 0x0e, 0x294: 0x0e, 0x295: 0x0e, 0x296: 0x0e, 0x297: 0x0e, - 0x298: 0x0e, 0x299: 0x0e, 0x29a: 0x0e, 0x29b: 0x0e, 0x29c: 0x0e, 0x29d: 0x0e, 0x29e: 0x3e, - // Block 0xb, offset 0x2c0 - 0x2c0: 0x08, 0x2c1: 0x08, 0x2c2: 0x08, 0x2c3: 0x08, 0x2c4: 0x08, 0x2c5: 0x08, 0x2c6: 0x08, 0x2c7: 0x08, - 0x2c8: 0x08, 0x2c9: 0x08, 0x2ca: 0x08, 0x2cb: 0x08, 0x2cc: 0x08, 0x2cd: 0x08, 0x2ce: 0x08, 0x2cf: 0x08, - 0x2d0: 0x08, 0x2d1: 0x08, 0x2d2: 0x08, 0x2d3: 0x08, 0x2d4: 0x08, 0x2d5: 0x08, 0x2d6: 0x08, 0x2d7: 0x08, - 0x2d8: 0x08, 0x2d9: 0x08, 0x2da: 0x08, 0x2db: 0x08, 0x2dc: 0x08, 0x2dd: 0x08, 0x2de: 0x08, 0x2df: 0x08, - 0x2e0: 0x08, 0x2e1: 0x08, 0x2e2: 0x08, 0x2e3: 0x08, 0x2e4: 0x08, 0x2e5: 0x08, 0x2e6: 0x08, 0x2e7: 0x08, - 0x2e8: 0x08, 0x2e9: 0x08, 0x2ea: 0x08, 0x2eb: 0x08, 0x2ec: 0x08, 0x2ed: 0x08, 0x2ee: 0x08, 0x2ef: 0x08, - 0x2f0: 0x08, 0x2f1: 0x08, 0x2f2: 0x08, 0x2f3: 0x08, 0x2f4: 0x08, 0x2f5: 0x08, 0x2f6: 0x08, 0x2f7: 0x08, - 0x2f8: 0x08, 0x2f9: 0x08, 0x2fa: 0x08, 0x2fb: 0x08, 0x2fc: 0x08, 0x2fd: 0x08, 0x2fe: 0x08, 0x2ff: 0x08, - // Block 0xc, offset 0x300 - 0x300: 0x08, 0x301: 0x08, 0x302: 0x08, 0x303: 0x08, 0x304: 0x08, 0x305: 0x08, 0x306: 0x08, 0x307: 0x08, - 0x308: 0x08, 0x309: 0x08, 0x30a: 0x08, 0x30b: 0x08, 0x30c: 0x08, 0x30d: 0x08, 0x30e: 0x08, 0x30f: 0x08, - 0x310: 0x08, 0x311: 0x08, 0x312: 0x08, 0x313: 0x08, 0x314: 0x08, 0x315: 0x08, 0x316: 0x08, 0x317: 0x08, - 0x318: 0x08, 0x319: 0x08, 0x31a: 0x08, 0x31b: 0x08, 0x31c: 0x08, 0x31d: 0x08, 0x31e: 0x08, 0x31f: 0x08, - 0x320: 0x08, 0x321: 0x08, 0x322: 0x08, 0x323: 0x08, 0x324: 0x0e, 0x325: 0x0e, 0x326: 0x0e, 0x327: 0x0e, - 0x328: 0x0e, 0x329: 0x0e, 0x32a: 0x0e, 0x32b: 0x0e, - 0x338: 0x3f, 0x339: 0x40, 0x33c: 0x41, 0x33d: 0x42, 0x33e: 0x43, 0x33f: 0x44, - // Block 0xd, offset 0x340 - 0x37f: 0x45, - // Block 0xe, offset 0x380 - 0x380: 0x0e, 0x381: 0x0e, 0x382: 0x0e, 0x383: 0x0e, 0x384: 0x0e, 0x385: 0x0e, 0x386: 0x0e, 0x387: 0x0e, - 0x388: 0x0e, 0x389: 0x0e, 0x38a: 0x0e, 0x38b: 0x0e, 0x38c: 0x0e, 0x38d: 0x0e, 0x38e: 0x0e, 0x38f: 0x0e, - 0x390: 0x0e, 0x391: 0x0e, 0x392: 0x0e, 0x393: 0x0e, 0x394: 0x0e, 0x395: 0x0e, 0x396: 0x0e, 0x397: 0x0e, - 0x398: 0x0e, 0x399: 0x0e, 0x39a: 0x0e, 0x39b: 0x0e, 0x39c: 0x0e, 0x39d: 0x0e, 0x39e: 0x0e, 0x39f: 0x46, - 0x3a0: 0x0e, 0x3a1: 0x0e, 0x3a2: 0x0e, 0x3a3: 0x0e, 0x3a4: 0x0e, 0x3a5: 0x0e, 0x3a6: 0x0e, 0x3a7: 0x0e, - 0x3a8: 0x0e, 0x3a9: 0x0e, 0x3aa: 0x0e, 0x3ab: 0x47, - // Block 0xf, offset 0x3c0 - 0x3c0: 0x0e, 0x3c1: 0x0e, 0x3c2: 0x0e, 0x3c3: 0x0e, 0x3c4: 0x48, 0x3c5: 0x49, 0x3c6: 0x0e, 0x3c7: 0x0e, - 0x3c8: 0x0e, 0x3c9: 0x0e, 0x3ca: 0x0e, 0x3cb: 0x4a, - // Block 0x10, offset 0x400 - 0x400: 0x4b, 0x403: 0x4c, 0x404: 0x4d, 0x405: 0x4e, 0x406: 0x4f, - 0x408: 0x50, 0x409: 0x51, 0x40c: 0x52, 0x40d: 0x53, 0x40e: 0x54, 0x40f: 0x55, - 0x410: 0x3a, 0x411: 0x56, 0x412: 0x0e, 0x413: 0x57, 0x414: 0x58, 0x415: 0x59, 0x416: 0x5a, 0x417: 0x5b, - 0x418: 0x0e, 0x419: 0x5c, 0x41a: 0x0e, 0x41b: 0x5d, - 0x424: 0x5e, 0x425: 0x5f, 0x426: 0x60, 0x427: 0x61, - // Block 0x11, offset 0x440 - 0x456: 0x0b, 0x457: 0x06, - 0x458: 0x0c, 0x45b: 0x0d, 0x45f: 0x0e, - 0x460: 0x06, 0x461: 0x06, 0x462: 0x06, 0x463: 0x06, 0x464: 0x06, 0x465: 0x06, 0x466: 0x06, 0x467: 0x06, - 0x468: 0x06, 0x469: 0x06, 0x46a: 0x06, 0x46b: 0x06, 0x46c: 0x06, 0x46d: 0x06, 0x46e: 0x06, 0x46f: 0x06, - 0x470: 0x06, 0x471: 0x06, 0x472: 0x06, 0x473: 0x06, 0x474: 0x06, 0x475: 0x06, 0x476: 0x06, 0x477: 0x06, - 0x478: 0x06, 0x479: 0x06, 0x47a: 0x06, 0x47b: 0x06, 0x47c: 0x06, 0x47d: 0x06, 0x47e: 0x06, 0x47f: 0x06, - // Block 0x12, offset 0x480 - 0x484: 0x08, 0x485: 0x08, 0x486: 0x08, 0x487: 0x09, - // Block 0x13, offset 0x4c0 - 0x4c0: 0x08, 0x4c1: 0x08, 0x4c2: 0x08, 0x4c3: 0x08, 0x4c4: 0x08, 0x4c5: 0x08, 0x4c6: 0x08, 0x4c7: 0x08, - 0x4c8: 0x08, 0x4c9: 0x08, 0x4ca: 0x08, 0x4cb: 0x08, 0x4cc: 0x08, 0x4cd: 0x08, 0x4ce: 0x08, 0x4cf: 0x08, - 0x4d0: 0x08, 0x4d1: 0x08, 0x4d2: 0x08, 0x4d3: 0x08, 0x4d4: 0x08, 0x4d5: 0x08, 0x4d6: 0x08, 0x4d7: 0x08, - 0x4d8: 0x08, 0x4d9: 0x08, 0x4da: 0x08, 0x4db: 0x08, 0x4dc: 0x08, 0x4dd: 0x08, 0x4de: 0x08, 0x4df: 0x08, - 0x4e0: 0x08, 0x4e1: 0x08, 0x4e2: 0x08, 0x4e3: 0x08, 0x4e4: 0x08, 0x4e5: 0x08, 0x4e6: 0x08, 0x4e7: 0x08, - 0x4e8: 0x08, 0x4e9: 0x08, 0x4ea: 0x08, 0x4eb: 0x08, 0x4ec: 0x08, 0x4ed: 0x08, 0x4ee: 0x08, 0x4ef: 0x08, - 0x4f0: 0x08, 0x4f1: 0x08, 0x4f2: 0x08, 0x4f3: 0x08, 0x4f4: 0x08, 0x4f5: 0x08, 0x4f6: 0x08, 0x4f7: 0x08, - 0x4f8: 0x08, 0x4f9: 0x08, 0x4fa: 0x08, 0x4fb: 0x08, 0x4fc: 0x08, 0x4fd: 0x08, 0x4fe: 0x08, 0x4ff: 0x62, - // Block 0x14, offset 0x500 - 0x520: 0x10, - 0x530: 0x09, 0x531: 0x09, 0x532: 0x09, 0x533: 0x09, 0x534: 0x09, 0x535: 0x09, 0x536: 0x09, 0x537: 0x09, - 0x538: 0x09, 0x539: 0x09, 0x53a: 0x09, 0x53b: 0x09, 0x53c: 0x09, 0x53d: 0x09, 0x53e: 0x09, 0x53f: 0x11, - // Block 0x15, offset 0x540 - 0x540: 0x09, 0x541: 0x09, 0x542: 0x09, 0x543: 0x09, 0x544: 0x09, 0x545: 0x09, 0x546: 0x09, 0x547: 0x09, - 0x548: 0x09, 0x549: 0x09, 0x54a: 0x09, 0x54b: 0x09, 0x54c: 0x09, 0x54d: 0x09, 0x54e: 0x09, 0x54f: 0x11, -} - -// inverseData contains 4-byte entries of the following format: -// -// <0 padding> -// -// The last byte of the UTF-8-encoded rune is xor-ed with the last byte of the -// UTF-8 encoding of the original rune. Mappings often have the following -// pattern: -// -// Ａ -> A (U+FF21 -> U+0041) -// Ｂ -> B (U+FF22 -> U+0042) -// ... -// -// By xor-ing the last byte the same entry can be shared by many mappings. This -// reduces the total number of distinct entries by about two thirds. -// The resulting entry for the aforementioned mappings is -// -// { 0x01, 0xE0, 0x00, 0x00 } -// -// Using this entry to map U+FF21 (UTF-8 [EF BC A1]), we get -// -// E0 ^ A1 = 41. -// -// Similarly, for U+FF22 (UTF-8 [EF BC A2]), we get -// -// E0 ^ A2 = 42. -// -// Note that because of the xor-ing, the byte sequence stored in the entry is -// not valid UTF-8. -var inverseData = [150][4]byte{ - {0x00, 0x00, 0x00, 0x00}, - {0x03, 0xe3, 0x80, 0xa0}, - {0x03, 0xef, 0xbc, 0xa0}, - {0x03, 0xef, 0xbc, 0xe0}, - {0x03, 0xef, 0xbd, 0xe0}, - {0x03, 0xef, 0xbf, 0x02}, - {0x03, 0xef, 0xbf, 0x00}, - {0x03, 0xef, 0xbf, 0x0e}, - {0x03, 0xef, 0xbf, 0x0c}, - {0x03, 0xef, 0xbf, 0x0f}, - {0x03, 0xef, 0xbf, 0x39}, - {0x03, 0xef, 0xbf, 0x3b}, - {0x03, 0xef, 0xbf, 0x3f}, - {0x03, 0xef, 0xbf, 0x2a}, - {0x03, 0xef, 0xbf, 0x0d}, - {0x03, 0xef, 0xbf, 0x25}, - {0x03, 0xef, 0xbd, 0x1a}, - {0x03, 0xef, 0xbd, 0x26}, - {0x01, 0xa0, 0x00, 0x00}, - {0x03, 0xef, 0xbd, 0x25}, - {0x03, 0xef, 0xbd, 0x23}, - {0x03, 0xef, 0xbd, 0x2e}, - {0x03, 0xef, 0xbe, 0x07}, - {0x03, 0xef, 0xbe, 0x05}, - {0x03, 0xef, 0xbd, 0x06}, - {0x03, 0xef, 0xbd, 0x13}, - {0x03, 0xef, 0xbd, 0x0b}, - {0x03, 0xef, 0xbd, 0x16}, - {0x03, 0xef, 0xbd, 0x0c}, - {0x03, 0xef, 0xbd, 0x15}, - {0x03, 0xef, 0xbd, 0x0d}, - {0x03, 0xef, 0xbd, 0x1c}, - {0x03, 0xef, 0xbd, 0x02}, - {0x03, 0xef, 0xbd, 0x1f}, - {0x03, 0xef, 0xbd, 0x1d}, - {0x03, 0xef, 0xbd, 0x17}, - {0x03, 0xef, 0xbd, 0x08}, - {0x03, 0xef, 0xbd, 0x09}, - {0x03, 0xef, 0xbd, 0x0e}, - {0x03, 0xef, 0xbd, 0x04}, - {0x03, 0xef, 0xbd, 0x05}, - {0x03, 0xef, 0xbe, 0x3f}, - {0x03, 0xef, 0xbe, 0x00}, - {0x03, 0xef, 0xbd, 0x2c}, - {0x03, 0xef, 0xbe, 0x06}, - {0x03, 0xef, 0xbe, 0x0c}, - {0x03, 0xef, 0xbe, 0x0f}, - {0x03, 0xef, 0xbe, 0x0d}, - {0x03, 0xef, 0xbe, 0x0b}, - {0x03, 0xef, 0xbe, 0x19}, - {0x03, 0xef, 0xbe, 0x15}, - {0x03, 0xef, 0xbe, 0x11}, - {0x03, 0xef, 0xbe, 0x31}, - {0x03, 0xef, 0xbe, 0x33}, - {0x03, 0xef, 0xbd, 0x0f}, - {0x03, 0xef, 0xbe, 0x30}, - {0x03, 0xef, 0xbe, 0x3e}, - {0x03, 0xef, 0xbe, 0x32}, - {0x03, 0xef, 0xbe, 0x36}, - {0x03, 0xef, 0xbd, 0x14}, - {0x03, 0xef, 0xbe, 0x2e}, - {0x03, 0xef, 0xbd, 0x1e}, - {0x03, 0xef, 0xbe, 0x10}, - {0x03, 0xef, 0xbf, 0x13}, - {0x03, 0xef, 0xbf, 0x15}, - {0x03, 0xef, 0xbf, 0x17}, - {0x03, 0xef, 0xbf, 0x1f}, - {0x03, 0xef, 0xbf, 0x1d}, - {0x03, 0xef, 0xbf, 0x1b}, - {0x03, 0xef, 0xbf, 0x09}, - {0x03, 0xef, 0xbf, 0x0b}, - {0x03, 0xef, 0xbf, 0x37}, - {0x03, 0xef, 0xbe, 0x04}, - {0x01, 0xe0, 0x00, 0x00}, - {0x03, 0xe2, 0xa6, 0x1a}, - {0x03, 0xe2, 0xa6, 0x26}, - {0x03, 0xe3, 0x80, 0x23}, - {0x03, 0xe3, 0x80, 0x2e}, - {0x03, 0xe3, 0x80, 0x25}, - {0x03, 0xe3, 0x83, 0x1e}, - {0x03, 0xe3, 0x83, 0x14}, - {0x03, 0xe3, 0x82, 0x06}, - {0x03, 0xe3, 0x82, 0x0b}, - {0x03, 0xe3, 0x82, 0x0c}, - {0x03, 0xe3, 0x82, 0x0d}, - {0x03, 0xe3, 0x82, 0x02}, - {0x03, 0xe3, 0x83, 0x0f}, - {0x03, 0xe3, 0x83, 0x08}, - {0x03, 0xe3, 0x83, 0x09}, - {0x03, 0xe3, 0x83, 0x2c}, - {0x03, 0xe3, 0x83, 0x0c}, - {0x03, 0xe3, 0x82, 0x13}, - {0x03, 0xe3, 0x82, 0x16}, - {0x03, 0xe3, 0x82, 0x15}, - {0x03, 0xe3, 0x82, 0x1c}, - {0x03, 0xe3, 0x82, 0x1f}, - {0x03, 0xe3, 0x82, 0x1d}, - {0x03, 0xe3, 0x82, 0x1a}, - {0x03, 0xe3, 0x82, 0x17}, - {0x03, 0xe3, 0x82, 0x08}, - {0x03, 0xe3, 0x82, 0x09}, - {0x03, 0xe3, 0x82, 0x0e}, - {0x03, 0xe3, 0x82, 0x04}, - {0x03, 0xe3, 0x82, 0x05}, - {0x03, 0xe3, 0x82, 0x3f}, - {0x03, 0xe3, 0x83, 0x00}, - {0x03, 0xe3, 0x83, 0x06}, - {0x03, 0xe3, 0x83, 0x05}, - {0x03, 0xe3, 0x83, 0x0d}, - {0x03, 0xe3, 0x83, 0x0b}, - {0x03, 0xe3, 0x83, 0x07}, - {0x03, 0xe3, 0x83, 0x19}, - {0x03, 0xe3, 0x83, 0x15}, - {0x03, 0xe3, 0x83, 0x11}, - {0x03, 0xe3, 0x83, 0x31}, - {0x03, 0xe3, 0x83, 0x33}, - {0x03, 0xe3, 0x83, 0x30}, - {0x03, 0xe3, 0x83, 0x3e}, - {0x03, 0xe3, 0x83, 0x32}, - {0x03, 0xe3, 0x83, 0x36}, - {0x03, 0xe3, 0x83, 0x2e}, - {0x03, 0xe3, 0x82, 0x07}, - {0x03, 0xe3, 0x85, 0x04}, - {0x03, 0xe3, 0x84, 0x10}, - {0x03, 0xe3, 0x85, 0x30}, - {0x03, 0xe3, 0x85, 0x0d}, - {0x03, 0xe3, 0x85, 0x13}, - {0x03, 0xe3, 0x85, 0x15}, - {0x03, 0xe3, 0x85, 0x17}, - {0x03, 0xe3, 0x85, 0x1f}, - {0x03, 0xe3, 0x85, 0x1d}, - {0x03, 0xe3, 0x85, 0x1b}, - {0x03, 0xe3, 0x85, 0x09}, - {0x03, 0xe3, 0x85, 0x0f}, - {0x03, 0xe3, 0x85, 0x0b}, - {0x03, 0xe3, 0x85, 0x37}, - {0x03, 0xe3, 0x85, 0x3b}, - {0x03, 0xe3, 0x85, 0x39}, - {0x03, 0xe3, 0x85, 0x3f}, - {0x02, 0xc2, 0x02, 0x00}, - {0x02, 0xc2, 0x0e, 0x00}, - {0x02, 0xc2, 0x0c, 0x00}, - {0x02, 0xc2, 0x00, 0x00}, - {0x03, 0xe2, 0x82, 0x0f}, - {0x03, 0xe2, 0x94, 0x2a}, - {0x03, 0xe2, 0x86, 0x39}, - {0x03, 0xe2, 0x86, 0x3b}, - {0x03, 0xe2, 0x86, 0x3f}, - {0x03, 0xe2, 0x96, 0x0d}, - {0x03, 0xe2, 0x97, 0x25}, -} - -// Total table size 14936 bytes (14KiB) diff --git a/vendor/golang.org/x/text/width/tables12.0.0.go b/vendor/golang.org/x/text/width/tables12.0.0.go deleted file mode 100644 index 5c14ade6d9..0000000000 --- a/vendor/golang.org/x/text/width/tables12.0.0.go +++ /dev/null @@ -1,1361 +0,0 @@ -// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. - -//go:build go1.14 && !go1.16 -// +build go1.14,!go1.16 - -package width - -// UnicodeVersion is the Unicode version from which the tables in this package are derived. -const UnicodeVersion = "12.0.0" - -// lookup returns the trie value for the first UTF-8 encoding in s and -// the width in bytes of this encoding. The size will be 0 if s does not -// hold enough bytes to complete the encoding. len(s) must be greater than 0. -func (t *widthTrie) lookup(s []byte) (v uint16, sz int) { - c0 := s[0] - switch { - case c0 < 0x80: // is ASCII - return widthValues[c0], 1 - case c0 < 0xC2: - return 0, 1 // Illegal UTF-8: not a starter, not ASCII. - case c0 < 0xE0: // 2-byte UTF-8 - if len(s) < 2 { - return 0, 0 - } - i := widthIndex[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c1), 2 - case c0 < 0xF0: // 3-byte UTF-8 - if len(s) < 3 { - return 0, 0 - } - i := widthIndex[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - o := uint32(i)<<6 + uint32(c1) - i = widthIndex[o] - c2 := s[2] - if c2 < 0x80 || 0xC0 <= c2 { - return 0, 2 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c2), 3 - case c0 < 0xF8: // 4-byte UTF-8 - if len(s) < 4 { - return 0, 0 - } - i := widthIndex[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - o := uint32(i)<<6 + uint32(c1) - i = widthIndex[o] - c2 := s[2] - if c2 < 0x80 || 0xC0 <= c2 { - return 0, 2 // Illegal UTF-8: not a continuation byte. - } - o = uint32(i)<<6 + uint32(c2) - i = widthIndex[o] - c3 := s[3] - if c3 < 0x80 || 0xC0 <= c3 { - return 0, 3 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c3), 4 - } - // Illegal rune - return 0, 1 -} - -// lookupUnsafe returns the trie value for the first UTF-8 encoding in s. -// s must start with a full and valid UTF-8 encoded rune. -func (t *widthTrie) lookupUnsafe(s []byte) uint16 { - c0 := s[0] - if c0 < 0x80 { // is ASCII - return widthValues[c0] - } - i := widthIndex[c0] - if c0 < 0xE0 { // 2-byte UTF-8 - return t.lookupValue(uint32(i), s[1]) - } - i = widthIndex[uint32(i)<<6+uint32(s[1])] - if c0 < 0xF0 { // 3-byte UTF-8 - return t.lookupValue(uint32(i), s[2]) - } - i = widthIndex[uint32(i)<<6+uint32(s[2])] - if c0 < 0xF8 { // 4-byte UTF-8 - return t.lookupValue(uint32(i), s[3]) - } - return 0 -} - -// lookupString returns the trie value for the first UTF-8 encoding in s and -// the width in bytes of this encoding. The size will be 0 if s does not -// hold enough bytes to complete the encoding. len(s) must be greater than 0. -func (t *widthTrie) lookupString(s string) (v uint16, sz int) { - c0 := s[0] - switch { - case c0 < 0x80: // is ASCII - return widthValues[c0], 1 - case c0 < 0xC2: - return 0, 1 // Illegal UTF-8: not a starter, not ASCII. - case c0 < 0xE0: // 2-byte UTF-8 - if len(s) < 2 { - return 0, 0 - } - i := widthIndex[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c1), 2 - case c0 < 0xF0: // 3-byte UTF-8 - if len(s) < 3 { - return 0, 0 - } - i := widthIndex[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - o := uint32(i)<<6 + uint32(c1) - i = widthIndex[o] - c2 := s[2] - if c2 < 0x80 || 0xC0 <= c2 { - return 0, 2 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c2), 3 - case c0 < 0xF8: // 4-byte UTF-8 - if len(s) < 4 { - return 0, 0 - } - i := widthIndex[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - o := uint32(i)<<6 + uint32(c1) - i = widthIndex[o] - c2 := s[2] - if c2 < 0x80 || 0xC0 <= c2 { - return 0, 2 // Illegal UTF-8: not a continuation byte. - } - o = uint32(i)<<6 + uint32(c2) - i = widthIndex[o] - c3 := s[3] - if c3 < 0x80 || 0xC0 <= c3 { - return 0, 3 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c3), 4 - } - // Illegal rune - return 0, 1 -} - -// lookupStringUnsafe returns the trie value for the first UTF-8 encoding in s. -// s must start with a full and valid UTF-8 encoded rune. -func (t *widthTrie) lookupStringUnsafe(s string) uint16 { - c0 := s[0] - if c0 < 0x80 { // is ASCII - return widthValues[c0] - } - i := widthIndex[c0] - if c0 < 0xE0 { // 2-byte UTF-8 - return t.lookupValue(uint32(i), s[1]) - } - i = widthIndex[uint32(i)<<6+uint32(s[1])] - if c0 < 0xF0 { // 3-byte UTF-8 - return t.lookupValue(uint32(i), s[2]) - } - i = widthIndex[uint32(i)<<6+uint32(s[2])] - if c0 < 0xF8 { // 4-byte UTF-8 - return t.lookupValue(uint32(i), s[3]) - } - return 0 -} - -// widthTrie. Total size: 14720 bytes (14.38 KiB). Checksum: 3f4f2516ded5489b. -type widthTrie struct{} - -func newWidthTrie(i int) *widthTrie { - return &widthTrie{} -} - -// lookupValue determines the type of block n and looks up the value for b. -func (t *widthTrie) lookupValue(n uint32, b byte) uint16 { - switch { - default: - return uint16(widthValues[n<<6+uint32(b)]) - } -} - -// widthValues: 104 blocks, 6656 entries, 13312 bytes -// The third block is the zero block. -var widthValues = [6656]uint16{ - // Block 0x0, offset 0x0 - 0x20: 0x6001, 0x21: 0x6002, 0x22: 0x6002, 0x23: 0x6002, - 0x24: 0x6002, 0x25: 0x6002, 0x26: 0x6002, 0x27: 0x6002, 0x28: 0x6002, 0x29: 0x6002, - 0x2a: 0x6002, 0x2b: 0x6002, 0x2c: 0x6002, 0x2d: 0x6002, 0x2e: 0x6002, 0x2f: 0x6002, - 0x30: 0x6002, 0x31: 0x6002, 0x32: 0x6002, 0x33: 0x6002, 0x34: 0x6002, 0x35: 0x6002, - 0x36: 0x6002, 0x37: 0x6002, 0x38: 0x6002, 0x39: 0x6002, 0x3a: 0x6002, 0x3b: 0x6002, - 0x3c: 0x6002, 0x3d: 0x6002, 0x3e: 0x6002, 0x3f: 0x6002, - // Block 0x1, offset 0x40 - 0x40: 0x6003, 0x41: 0x6003, 0x42: 0x6003, 0x43: 0x6003, 0x44: 0x6003, 0x45: 0x6003, - 0x46: 0x6003, 0x47: 0x6003, 0x48: 0x6003, 0x49: 0x6003, 0x4a: 0x6003, 0x4b: 0x6003, - 0x4c: 0x6003, 0x4d: 0x6003, 0x4e: 0x6003, 0x4f: 0x6003, 0x50: 0x6003, 0x51: 0x6003, - 0x52: 0x6003, 0x53: 0x6003, 0x54: 0x6003, 0x55: 0x6003, 0x56: 0x6003, 0x57: 0x6003, - 0x58: 0x6003, 0x59: 0x6003, 0x5a: 0x6003, 0x5b: 0x6003, 0x5c: 0x6003, 0x5d: 0x6003, - 0x5e: 0x6003, 0x5f: 0x6003, 0x60: 0x6004, 0x61: 0x6004, 0x62: 0x6004, 0x63: 0x6004, - 0x64: 0x6004, 0x65: 0x6004, 0x66: 0x6004, 0x67: 0x6004, 0x68: 0x6004, 0x69: 0x6004, - 0x6a: 0x6004, 0x6b: 0x6004, 0x6c: 0x6004, 0x6d: 0x6004, 0x6e: 0x6004, 0x6f: 0x6004, - 0x70: 0x6004, 0x71: 0x6004, 0x72: 0x6004, 0x73: 0x6004, 0x74: 0x6004, 0x75: 0x6004, - 0x76: 0x6004, 0x77: 0x6004, 0x78: 0x6004, 0x79: 0x6004, 0x7a: 0x6004, 0x7b: 0x6004, - 0x7c: 0x6004, 0x7d: 0x6004, 0x7e: 0x6004, - // Block 0x2, offset 0x80 - // Block 0x3, offset 0xc0 - 0xe1: 0x2000, 0xe2: 0x6005, 0xe3: 0x6005, - 0xe4: 0x2000, 0xe5: 0x6006, 0xe6: 0x6005, 0xe7: 0x2000, 0xe8: 0x2000, - 0xea: 0x2000, 0xec: 0x6007, 0xed: 0x2000, 0xee: 0x2000, 0xef: 0x6008, - 0xf0: 0x2000, 0xf1: 0x2000, 0xf2: 0x2000, 0xf3: 0x2000, 0xf4: 0x2000, - 0xf6: 0x2000, 0xf7: 0x2000, 0xf8: 0x2000, 0xf9: 0x2000, 0xfa: 0x2000, - 0xfc: 0x2000, 0xfd: 0x2000, 0xfe: 0x2000, 0xff: 0x2000, - // Block 0x4, offset 0x100 - 0x106: 0x2000, - 0x110: 0x2000, - 0x117: 0x2000, - 0x118: 0x2000, - 0x11e: 0x2000, 0x11f: 0x2000, 0x120: 0x2000, 0x121: 0x2000, - 0x126: 0x2000, 0x128: 0x2000, 0x129: 0x2000, - 0x12a: 0x2000, 0x12c: 0x2000, 0x12d: 0x2000, - 0x130: 0x2000, 0x132: 0x2000, 0x133: 0x2000, - 0x137: 0x2000, 0x138: 0x2000, 0x139: 0x2000, 0x13a: 0x2000, - 0x13c: 0x2000, 0x13e: 0x2000, - // Block 0x5, offset 0x140 - 0x141: 0x2000, - 0x151: 0x2000, - 0x153: 0x2000, - 0x15b: 0x2000, - 0x166: 0x2000, 0x167: 0x2000, - 0x16b: 0x2000, - 0x171: 0x2000, 0x172: 0x2000, 0x173: 0x2000, - 0x178: 0x2000, - 0x17f: 0x2000, - // Block 0x6, offset 0x180 - 0x180: 0x2000, 0x181: 0x2000, 0x182: 0x2000, 0x184: 0x2000, - 0x188: 0x2000, 0x189: 0x2000, 0x18a: 0x2000, 0x18b: 0x2000, - 0x18d: 0x2000, - 0x192: 0x2000, 0x193: 0x2000, - 0x1a6: 0x2000, 0x1a7: 0x2000, - 0x1ab: 0x2000, - // Block 0x7, offset 0x1c0 - 0x1ce: 0x2000, 0x1d0: 0x2000, - 0x1d2: 0x2000, 0x1d4: 0x2000, 0x1d6: 0x2000, - 0x1d8: 0x2000, 0x1da: 0x2000, 0x1dc: 0x2000, - // Block 0x8, offset 0x200 - 0x211: 0x2000, - 0x221: 0x2000, - // Block 0x9, offset 0x240 - 0x244: 0x2000, - 0x247: 0x2000, 0x249: 0x2000, 0x24a: 0x2000, 0x24b: 0x2000, - 0x24d: 0x2000, 0x250: 0x2000, - 0x258: 0x2000, 0x259: 0x2000, 0x25a: 0x2000, 0x25b: 0x2000, 0x25d: 0x2000, - 0x25f: 0x2000, - // Block 0xa, offset 0x280 - 0x280: 0x2000, 0x281: 0x2000, 0x282: 0x2000, 0x283: 0x2000, 0x284: 0x2000, 0x285: 0x2000, - 0x286: 0x2000, 0x287: 0x2000, 0x288: 0x2000, 0x289: 0x2000, 0x28a: 0x2000, 0x28b: 0x2000, - 0x28c: 0x2000, 0x28d: 0x2000, 0x28e: 0x2000, 0x28f: 0x2000, 0x290: 0x2000, 0x291: 0x2000, - 0x292: 0x2000, 0x293: 0x2000, 0x294: 0x2000, 0x295: 0x2000, 0x296: 0x2000, 0x297: 0x2000, - 0x298: 0x2000, 0x299: 0x2000, 0x29a: 0x2000, 0x29b: 0x2000, 0x29c: 0x2000, 0x29d: 0x2000, - 0x29e: 0x2000, 0x29f: 0x2000, 0x2a0: 0x2000, 0x2a1: 0x2000, 0x2a2: 0x2000, 0x2a3: 0x2000, - 0x2a4: 0x2000, 0x2a5: 0x2000, 0x2a6: 0x2000, 0x2a7: 0x2000, 0x2a8: 0x2000, 0x2a9: 0x2000, - 0x2aa: 0x2000, 0x2ab: 0x2000, 0x2ac: 0x2000, 0x2ad: 0x2000, 0x2ae: 0x2000, 0x2af: 0x2000, - 0x2b0: 0x2000, 0x2b1: 0x2000, 0x2b2: 0x2000, 0x2b3: 0x2000, 0x2b4: 0x2000, 0x2b5: 0x2000, - 0x2b6: 0x2000, 0x2b7: 0x2000, 0x2b8: 0x2000, 0x2b9: 0x2000, 0x2ba: 0x2000, 0x2bb: 0x2000, - 0x2bc: 0x2000, 0x2bd: 0x2000, 0x2be: 0x2000, 0x2bf: 0x2000, - // Block 0xb, offset 0x2c0 - 0x2c0: 0x2000, 0x2c1: 0x2000, 0x2c2: 0x2000, 0x2c3: 0x2000, 0x2c4: 0x2000, 0x2c5: 0x2000, - 0x2c6: 0x2000, 0x2c7: 0x2000, 0x2c8: 0x2000, 0x2c9: 0x2000, 0x2ca: 0x2000, 0x2cb: 0x2000, - 0x2cc: 0x2000, 0x2cd: 0x2000, 0x2ce: 0x2000, 0x2cf: 0x2000, 0x2d0: 0x2000, 0x2d1: 0x2000, - 0x2d2: 0x2000, 0x2d3: 0x2000, 0x2d4: 0x2000, 0x2d5: 0x2000, 0x2d6: 0x2000, 0x2d7: 0x2000, - 0x2d8: 0x2000, 0x2d9: 0x2000, 0x2da: 0x2000, 0x2db: 0x2000, 0x2dc: 0x2000, 0x2dd: 0x2000, - 0x2de: 0x2000, 0x2df: 0x2000, 0x2e0: 0x2000, 0x2e1: 0x2000, 0x2e2: 0x2000, 0x2e3: 0x2000, - 0x2e4: 0x2000, 0x2e5: 0x2000, 0x2e6: 0x2000, 0x2e7: 0x2000, 0x2e8: 0x2000, 0x2e9: 0x2000, - 0x2ea: 0x2000, 0x2eb: 0x2000, 0x2ec: 0x2000, 0x2ed: 0x2000, 0x2ee: 0x2000, 0x2ef: 0x2000, - // Block 0xc, offset 0x300 - 0x311: 0x2000, - 0x312: 0x2000, 0x313: 0x2000, 0x314: 0x2000, 0x315: 0x2000, 0x316: 0x2000, 0x317: 0x2000, - 0x318: 0x2000, 0x319: 0x2000, 0x31a: 0x2000, 0x31b: 0x2000, 0x31c: 0x2000, 0x31d: 0x2000, - 0x31e: 0x2000, 0x31f: 0x2000, 0x320: 0x2000, 0x321: 0x2000, 0x323: 0x2000, - 0x324: 0x2000, 0x325: 0x2000, 0x326: 0x2000, 0x327: 0x2000, 0x328: 0x2000, 0x329: 0x2000, - 0x331: 0x2000, 0x332: 0x2000, 0x333: 0x2000, 0x334: 0x2000, 0x335: 0x2000, - 0x336: 0x2000, 0x337: 0x2000, 0x338: 0x2000, 0x339: 0x2000, 0x33a: 0x2000, 0x33b: 0x2000, - 0x33c: 0x2000, 0x33d: 0x2000, 0x33e: 0x2000, 0x33f: 0x2000, - // Block 0xd, offset 0x340 - 0x340: 0x2000, 0x341: 0x2000, 0x343: 0x2000, 0x344: 0x2000, 0x345: 0x2000, - 0x346: 0x2000, 0x347: 0x2000, 0x348: 0x2000, 0x349: 0x2000, - // Block 0xe, offset 0x380 - 0x381: 0x2000, - 0x390: 0x2000, 0x391: 0x2000, - 0x392: 0x2000, 0x393: 0x2000, 0x394: 0x2000, 0x395: 0x2000, 0x396: 0x2000, 0x397: 0x2000, - 0x398: 0x2000, 0x399: 0x2000, 0x39a: 0x2000, 0x39b: 0x2000, 0x39c: 0x2000, 0x39d: 0x2000, - 0x39e: 0x2000, 0x39f: 0x2000, 0x3a0: 0x2000, 0x3a1: 0x2000, 0x3a2: 0x2000, 0x3a3: 0x2000, - 0x3a4: 0x2000, 0x3a5: 0x2000, 0x3a6: 0x2000, 0x3a7: 0x2000, 0x3a8: 0x2000, 0x3a9: 0x2000, - 0x3aa: 0x2000, 0x3ab: 0x2000, 0x3ac: 0x2000, 0x3ad: 0x2000, 0x3ae: 0x2000, 0x3af: 0x2000, - 0x3b0: 0x2000, 0x3b1: 0x2000, 0x3b2: 0x2000, 0x3b3: 0x2000, 0x3b4: 0x2000, 0x3b5: 0x2000, - 0x3b6: 0x2000, 0x3b7: 0x2000, 0x3b8: 0x2000, 0x3b9: 0x2000, 0x3ba: 0x2000, 0x3bb: 0x2000, - 0x3bc: 0x2000, 0x3bd: 0x2000, 0x3be: 0x2000, 0x3bf: 0x2000, - // Block 0xf, offset 0x3c0 - 0x3c0: 0x2000, 0x3c1: 0x2000, 0x3c2: 0x2000, 0x3c3: 0x2000, 0x3c4: 0x2000, 0x3c5: 0x2000, - 0x3c6: 0x2000, 0x3c7: 0x2000, 0x3c8: 0x2000, 0x3c9: 0x2000, 0x3ca: 0x2000, 0x3cb: 0x2000, - 0x3cc: 0x2000, 0x3cd: 0x2000, 0x3ce: 0x2000, 0x3cf: 0x2000, 0x3d1: 0x2000, - // Block 0x10, offset 0x400 - 0x400: 0x4000, 0x401: 0x4000, 0x402: 0x4000, 0x403: 0x4000, 0x404: 0x4000, 0x405: 0x4000, - 0x406: 0x4000, 0x407: 0x4000, 0x408: 0x4000, 0x409: 0x4000, 0x40a: 0x4000, 0x40b: 0x4000, - 0x40c: 0x4000, 0x40d: 0x4000, 0x40e: 0x4000, 0x40f: 0x4000, 0x410: 0x4000, 0x411: 0x4000, - 0x412: 0x4000, 0x413: 0x4000, 0x414: 0x4000, 0x415: 0x4000, 0x416: 0x4000, 0x417: 0x4000, - 0x418: 0x4000, 0x419: 0x4000, 0x41a: 0x4000, 0x41b: 0x4000, 0x41c: 0x4000, 0x41d: 0x4000, - 0x41e: 0x4000, 0x41f: 0x4000, 0x420: 0x4000, 0x421: 0x4000, 0x422: 0x4000, 0x423: 0x4000, - 0x424: 0x4000, 0x425: 0x4000, 0x426: 0x4000, 0x427: 0x4000, 0x428: 0x4000, 0x429: 0x4000, - 0x42a: 0x4000, 0x42b: 0x4000, 0x42c: 0x4000, 0x42d: 0x4000, 0x42e: 0x4000, 0x42f: 0x4000, - 0x430: 0x4000, 0x431: 0x4000, 0x432: 0x4000, 0x433: 0x4000, 0x434: 0x4000, 0x435: 0x4000, - 0x436: 0x4000, 0x437: 0x4000, 0x438: 0x4000, 0x439: 0x4000, 0x43a: 0x4000, 0x43b: 0x4000, - 0x43c: 0x4000, 0x43d: 0x4000, 0x43e: 0x4000, 0x43f: 0x4000, - // Block 0x11, offset 0x440 - 0x440: 0x4000, 0x441: 0x4000, 0x442: 0x4000, 0x443: 0x4000, 0x444: 0x4000, 0x445: 0x4000, - 0x446: 0x4000, 0x447: 0x4000, 0x448: 0x4000, 0x449: 0x4000, 0x44a: 0x4000, 0x44b: 0x4000, - 0x44c: 0x4000, 0x44d: 0x4000, 0x44e: 0x4000, 0x44f: 0x4000, 0x450: 0x4000, 0x451: 0x4000, - 0x452: 0x4000, 0x453: 0x4000, 0x454: 0x4000, 0x455: 0x4000, 0x456: 0x4000, 0x457: 0x4000, - 0x458: 0x4000, 0x459: 0x4000, 0x45a: 0x4000, 0x45b: 0x4000, 0x45c: 0x4000, 0x45d: 0x4000, - 0x45e: 0x4000, 0x45f: 0x4000, - // Block 0x12, offset 0x480 - 0x490: 0x2000, - 0x493: 0x2000, 0x494: 0x2000, 0x495: 0x2000, 0x496: 0x2000, - 0x498: 0x2000, 0x499: 0x2000, 0x49c: 0x2000, 0x49d: 0x2000, - 0x4a0: 0x2000, 0x4a1: 0x2000, 0x4a2: 0x2000, - 0x4a4: 0x2000, 0x4a5: 0x2000, 0x4a6: 0x2000, 0x4a7: 0x2000, - 0x4b0: 0x2000, 0x4b2: 0x2000, 0x4b3: 0x2000, 0x4b5: 0x2000, - 0x4bb: 0x2000, - 0x4be: 0x2000, - // Block 0x13, offset 0x4c0 - 0x4f4: 0x2000, - 0x4ff: 0x2000, - // Block 0x14, offset 0x500 - 0x501: 0x2000, 0x502: 0x2000, 0x503: 0x2000, 0x504: 0x2000, - 0x529: 0xa009, - 0x52c: 0x2000, - // Block 0x15, offset 0x540 - 0x543: 0x2000, 0x545: 0x2000, - 0x549: 0x2000, - 0x553: 0x2000, 0x556: 0x2000, - 0x561: 0x2000, 0x562: 0x2000, - 0x566: 0x2000, - 0x56b: 0x2000, - // Block 0x16, offset 0x580 - 0x593: 0x2000, 0x594: 0x2000, - 0x59b: 0x2000, 0x59c: 0x2000, 0x59d: 0x2000, - 0x59e: 0x2000, 0x5a0: 0x2000, 0x5a1: 0x2000, 0x5a2: 0x2000, 0x5a3: 0x2000, - 0x5a4: 0x2000, 0x5a5: 0x2000, 0x5a6: 0x2000, 0x5a7: 0x2000, 0x5a8: 0x2000, 0x5a9: 0x2000, - 0x5aa: 0x2000, 0x5ab: 0x2000, - 0x5b0: 0x2000, 0x5b1: 0x2000, 0x5b2: 0x2000, 0x5b3: 0x2000, 0x5b4: 0x2000, 0x5b5: 0x2000, - 0x5b6: 0x2000, 0x5b7: 0x2000, 0x5b8: 0x2000, 0x5b9: 0x2000, - // Block 0x17, offset 0x5c0 - 0x5c9: 0x2000, - 0x5d0: 0x200a, 0x5d1: 0x200b, - 0x5d2: 0x200a, 0x5d3: 0x200c, 0x5d4: 0x2000, 0x5d5: 0x2000, 0x5d6: 0x2000, 0x5d7: 0x2000, - 0x5d8: 0x2000, 0x5d9: 0x2000, - 0x5f8: 0x2000, 0x5f9: 0x2000, - // Block 0x18, offset 0x600 - 0x612: 0x2000, 0x614: 0x2000, - 0x627: 0x2000, - // Block 0x19, offset 0x640 - 0x640: 0x2000, 0x642: 0x2000, 0x643: 0x2000, - 0x647: 0x2000, 0x648: 0x2000, 0x64b: 0x2000, - 0x64f: 0x2000, 0x651: 0x2000, - 0x655: 0x2000, - 0x65a: 0x2000, 0x65d: 0x2000, - 0x65e: 0x2000, 0x65f: 0x2000, 0x660: 0x2000, 0x663: 0x2000, - 0x665: 0x2000, 0x667: 0x2000, 0x668: 0x2000, 0x669: 0x2000, - 0x66a: 0x2000, 0x66b: 0x2000, 0x66c: 0x2000, 0x66e: 0x2000, - 0x674: 0x2000, 0x675: 0x2000, - 0x676: 0x2000, 0x677: 0x2000, - 0x67c: 0x2000, 0x67d: 0x2000, - // Block 0x1a, offset 0x680 - 0x688: 0x2000, - 0x68c: 0x2000, - 0x692: 0x2000, - 0x6a0: 0x2000, 0x6a1: 0x2000, - 0x6a4: 0x2000, 0x6a5: 0x2000, 0x6a6: 0x2000, 0x6a7: 0x2000, - 0x6aa: 0x2000, 0x6ab: 0x2000, 0x6ae: 0x2000, 0x6af: 0x2000, - // Block 0x1b, offset 0x6c0 - 0x6c2: 0x2000, 0x6c3: 0x2000, - 0x6c6: 0x2000, 0x6c7: 0x2000, - 0x6d5: 0x2000, - 0x6d9: 0x2000, - 0x6e5: 0x2000, - 0x6ff: 0x2000, - // Block 0x1c, offset 0x700 - 0x712: 0x2000, - 0x71a: 0x4000, 0x71b: 0x4000, - 0x729: 0x4000, - 0x72a: 0x4000, - // Block 0x1d, offset 0x740 - 0x769: 0x4000, - 0x76a: 0x4000, 0x76b: 0x4000, 0x76c: 0x4000, - 0x770: 0x4000, 0x773: 0x4000, - // Block 0x1e, offset 0x780 - 0x7a0: 0x2000, 0x7a1: 0x2000, 0x7a2: 0x2000, 0x7a3: 0x2000, - 0x7a4: 0x2000, 0x7a5: 0x2000, 0x7a6: 0x2000, 0x7a7: 0x2000, 0x7a8: 0x2000, 0x7a9: 0x2000, - 0x7aa: 0x2000, 0x7ab: 0x2000, 0x7ac: 0x2000, 0x7ad: 0x2000, 0x7ae: 0x2000, 0x7af: 0x2000, - 0x7b0: 0x2000, 0x7b1: 0x2000, 0x7b2: 0x2000, 0x7b3: 0x2000, 0x7b4: 0x2000, 0x7b5: 0x2000, - 0x7b6: 0x2000, 0x7b7: 0x2000, 0x7b8: 0x2000, 0x7b9: 0x2000, 0x7ba: 0x2000, 0x7bb: 0x2000, - 0x7bc: 0x2000, 0x7bd: 0x2000, 0x7be: 0x2000, 0x7bf: 0x2000, - // Block 0x1f, offset 0x7c0 - 0x7c0: 0x2000, 0x7c1: 0x2000, 0x7c2: 0x2000, 0x7c3: 0x2000, 0x7c4: 0x2000, 0x7c5: 0x2000, - 0x7c6: 0x2000, 0x7c7: 0x2000, 0x7c8: 0x2000, 0x7c9: 0x2000, 0x7ca: 0x2000, 0x7cb: 0x2000, - 0x7cc: 0x2000, 0x7cd: 0x2000, 0x7ce: 0x2000, 0x7cf: 0x2000, 0x7d0: 0x2000, 0x7d1: 0x2000, - 0x7d2: 0x2000, 0x7d3: 0x2000, 0x7d4: 0x2000, 0x7d5: 0x2000, 0x7d6: 0x2000, 0x7d7: 0x2000, - 0x7d8: 0x2000, 0x7d9: 0x2000, 0x7da: 0x2000, 0x7db: 0x2000, 0x7dc: 0x2000, 0x7dd: 0x2000, - 0x7de: 0x2000, 0x7df: 0x2000, 0x7e0: 0x2000, 0x7e1: 0x2000, 0x7e2: 0x2000, 0x7e3: 0x2000, - 0x7e4: 0x2000, 0x7e5: 0x2000, 0x7e6: 0x2000, 0x7e7: 0x2000, 0x7e8: 0x2000, 0x7e9: 0x2000, - 0x7eb: 0x2000, 0x7ec: 0x2000, 0x7ed: 0x2000, 0x7ee: 0x2000, 0x7ef: 0x2000, - 0x7f0: 0x2000, 0x7f1: 0x2000, 0x7f2: 0x2000, 0x7f3: 0x2000, 0x7f4: 0x2000, 0x7f5: 0x2000, - 0x7f6: 0x2000, 0x7f7: 0x2000, 0x7f8: 0x2000, 0x7f9: 0x2000, 0x7fa: 0x2000, 0x7fb: 0x2000, - 0x7fc: 0x2000, 0x7fd: 0x2000, 0x7fe: 0x2000, 0x7ff: 0x2000, - // Block 0x20, offset 0x800 - 0x800: 0x2000, 0x801: 0x2000, 0x802: 0x200d, 0x803: 0x2000, 0x804: 0x2000, 0x805: 0x2000, - 0x806: 0x2000, 0x807: 0x2000, 0x808: 0x2000, 0x809: 0x2000, 0x80a: 0x2000, 0x80b: 0x2000, - 0x80c: 0x2000, 0x80d: 0x2000, 0x80e: 0x2000, 0x80f: 0x2000, 0x810: 0x2000, 0x811: 0x2000, - 0x812: 0x2000, 0x813: 0x2000, 0x814: 0x2000, 0x815: 0x2000, 0x816: 0x2000, 0x817: 0x2000, - 0x818: 0x2000, 0x819: 0x2000, 0x81a: 0x2000, 0x81b: 0x2000, 0x81c: 0x2000, 0x81d: 0x2000, - 0x81e: 0x2000, 0x81f: 0x2000, 0x820: 0x2000, 0x821: 0x2000, 0x822: 0x2000, 0x823: 0x2000, - 0x824: 0x2000, 0x825: 0x2000, 0x826: 0x2000, 0x827: 0x2000, 0x828: 0x2000, 0x829: 0x2000, - 0x82a: 0x2000, 0x82b: 0x2000, 0x82c: 0x2000, 0x82d: 0x2000, 0x82e: 0x2000, 0x82f: 0x2000, - 0x830: 0x2000, 0x831: 0x2000, 0x832: 0x2000, 0x833: 0x2000, 0x834: 0x2000, 0x835: 0x2000, - 0x836: 0x2000, 0x837: 0x2000, 0x838: 0x2000, 0x839: 0x2000, 0x83a: 0x2000, 0x83b: 0x2000, - 0x83c: 0x2000, 0x83d: 0x2000, 0x83e: 0x2000, 0x83f: 0x2000, - // Block 0x21, offset 0x840 - 0x840: 0x2000, 0x841: 0x2000, 0x842: 0x2000, 0x843: 0x2000, 0x844: 0x2000, 0x845: 0x2000, - 0x846: 0x2000, 0x847: 0x2000, 0x848: 0x2000, 0x849: 0x2000, 0x84a: 0x2000, 0x84b: 0x2000, - 0x850: 0x2000, 0x851: 0x2000, - 0x852: 0x2000, 0x853: 0x2000, 0x854: 0x2000, 0x855: 0x2000, 0x856: 0x2000, 0x857: 0x2000, - 0x858: 0x2000, 0x859: 0x2000, 0x85a: 0x2000, 0x85b: 0x2000, 0x85c: 0x2000, 0x85d: 0x2000, - 0x85e: 0x2000, 0x85f: 0x2000, 0x860: 0x2000, 0x861: 0x2000, 0x862: 0x2000, 0x863: 0x2000, - 0x864: 0x2000, 0x865: 0x2000, 0x866: 0x2000, 0x867: 0x2000, 0x868: 0x2000, 0x869: 0x2000, - 0x86a: 0x2000, 0x86b: 0x2000, 0x86c: 0x2000, 0x86d: 0x2000, 0x86e: 0x2000, 0x86f: 0x2000, - 0x870: 0x2000, 0x871: 0x2000, 0x872: 0x2000, 0x873: 0x2000, - // Block 0x22, offset 0x880 - 0x880: 0x2000, 0x881: 0x2000, 0x882: 0x2000, 0x883: 0x2000, 0x884: 0x2000, 0x885: 0x2000, - 0x886: 0x2000, 0x887: 0x2000, 0x888: 0x2000, 0x889: 0x2000, 0x88a: 0x2000, 0x88b: 0x2000, - 0x88c: 0x2000, 0x88d: 0x2000, 0x88e: 0x2000, 0x88f: 0x2000, - 0x892: 0x2000, 0x893: 0x2000, 0x894: 0x2000, 0x895: 0x2000, - 0x8a0: 0x200e, 0x8a1: 0x2000, 0x8a3: 0x2000, - 0x8a4: 0x2000, 0x8a5: 0x2000, 0x8a6: 0x2000, 0x8a7: 0x2000, 0x8a8: 0x2000, 0x8a9: 0x2000, - 0x8b2: 0x2000, 0x8b3: 0x2000, - 0x8b6: 0x2000, 0x8b7: 0x2000, - 0x8bc: 0x2000, 0x8bd: 0x2000, - // Block 0x23, offset 0x8c0 - 0x8c0: 0x2000, 0x8c1: 0x2000, - 0x8c6: 0x2000, 0x8c7: 0x2000, 0x8c8: 0x2000, 0x8cb: 0x200f, - 0x8ce: 0x2000, 0x8cf: 0x2000, 0x8d0: 0x2000, 0x8d1: 0x2000, - 0x8e2: 0x2000, 0x8e3: 0x2000, - 0x8e4: 0x2000, 0x8e5: 0x2000, - 0x8ef: 0x2000, - 0x8fd: 0x4000, 0x8fe: 0x4000, - // Block 0x24, offset 0x900 - 0x905: 0x2000, - 0x906: 0x2000, 0x909: 0x2000, - 0x90e: 0x2000, 0x90f: 0x2000, - 0x914: 0x4000, 0x915: 0x4000, - 0x91c: 0x2000, - 0x91e: 0x2000, - // Block 0x25, offset 0x940 - 0x940: 0x2000, 0x942: 0x2000, - 0x948: 0x4000, 0x949: 0x4000, 0x94a: 0x4000, 0x94b: 0x4000, - 0x94c: 0x4000, 0x94d: 0x4000, 0x94e: 0x4000, 0x94f: 0x4000, 0x950: 0x4000, 0x951: 0x4000, - 0x952: 0x4000, 0x953: 0x4000, - 0x960: 0x2000, 0x961: 0x2000, 0x963: 0x2000, - 0x964: 0x2000, 0x965: 0x2000, 0x967: 0x2000, 0x968: 0x2000, 0x969: 0x2000, - 0x96a: 0x2000, 0x96c: 0x2000, 0x96d: 0x2000, 0x96f: 0x2000, - 0x97f: 0x4000, - // Block 0x26, offset 0x980 - 0x993: 0x4000, - 0x99e: 0x2000, 0x99f: 0x2000, 0x9a1: 0x4000, - 0x9aa: 0x4000, 0x9ab: 0x4000, - 0x9bd: 0x4000, 0x9be: 0x4000, 0x9bf: 0x2000, - // Block 0x27, offset 0x9c0 - 0x9c4: 0x4000, 0x9c5: 0x4000, - 0x9c6: 0x2000, 0x9c7: 0x2000, 0x9c8: 0x2000, 0x9c9: 0x2000, 0x9ca: 0x2000, 0x9cb: 0x2000, - 0x9cc: 0x2000, 0x9cd: 0x2000, 0x9ce: 0x4000, 0x9cf: 0x2000, 0x9d0: 0x2000, 0x9d1: 0x2000, - 0x9d2: 0x2000, 0x9d3: 0x2000, 0x9d4: 0x4000, 0x9d5: 0x2000, 0x9d6: 0x2000, 0x9d7: 0x2000, - 0x9d8: 0x2000, 0x9d9: 0x2000, 0x9da: 0x2000, 0x9db: 0x2000, 0x9dc: 0x2000, 0x9dd: 0x2000, - 0x9de: 0x2000, 0x9df: 0x2000, 0x9e0: 0x2000, 0x9e1: 0x2000, 0x9e3: 0x2000, - 0x9e8: 0x2000, 0x9e9: 0x2000, - 0x9ea: 0x4000, 0x9eb: 0x2000, 0x9ec: 0x2000, 0x9ed: 0x2000, 0x9ee: 0x2000, 0x9ef: 0x2000, - 0x9f0: 0x2000, 0x9f1: 0x2000, 0x9f2: 0x4000, 0x9f3: 0x4000, 0x9f4: 0x2000, 0x9f5: 0x4000, - 0x9f6: 0x2000, 0x9f7: 0x2000, 0x9f8: 0x2000, 0x9f9: 0x2000, 0x9fa: 0x4000, 0x9fb: 0x2000, - 0x9fc: 0x2000, 0x9fd: 0x4000, 0x9fe: 0x2000, 0x9ff: 0x2000, - // Block 0x28, offset 0xa00 - 0xa05: 0x4000, - 0xa0a: 0x4000, 0xa0b: 0x4000, - 0xa28: 0x4000, - 0xa3d: 0x2000, - // Block 0x29, offset 0xa40 - 0xa4c: 0x4000, 0xa4e: 0x4000, - 0xa53: 0x4000, 0xa54: 0x4000, 0xa55: 0x4000, 0xa57: 0x4000, - 0xa76: 0x2000, 0xa77: 0x2000, 0xa78: 0x2000, 0xa79: 0x2000, 0xa7a: 0x2000, 0xa7b: 0x2000, - 0xa7c: 0x2000, 0xa7d: 0x2000, 0xa7e: 0x2000, 0xa7f: 0x2000, - // Block 0x2a, offset 0xa80 - 0xa95: 0x4000, 0xa96: 0x4000, 0xa97: 0x4000, - 0xab0: 0x4000, - 0xabf: 0x4000, - // Block 0x2b, offset 0xac0 - 0xae6: 0x6000, 0xae7: 0x6000, 0xae8: 0x6000, 0xae9: 0x6000, - 0xaea: 0x6000, 0xaeb: 0x6000, 0xaec: 0x6000, 0xaed: 0x6000, - // Block 0x2c, offset 0xb00 - 0xb05: 0x6010, - 0xb06: 0x6011, - // Block 0x2d, offset 0xb40 - 0xb5b: 0x4000, 0xb5c: 0x4000, - // Block 0x2e, offset 0xb80 - 0xb90: 0x4000, - 0xb95: 0x4000, 0xb96: 0x2000, 0xb97: 0x2000, - 0xb98: 0x2000, 0xb99: 0x2000, - // Block 0x2f, offset 0xbc0 - 0xbc0: 0x4000, 0xbc1: 0x4000, 0xbc2: 0x4000, 0xbc3: 0x4000, 0xbc4: 0x4000, 0xbc5: 0x4000, - 0xbc6: 0x4000, 0xbc7: 0x4000, 0xbc8: 0x4000, 0xbc9: 0x4000, 0xbca: 0x4000, 0xbcb: 0x4000, - 0xbcc: 0x4000, 0xbcd: 0x4000, 0xbce: 0x4000, 0xbcf: 0x4000, 0xbd0: 0x4000, 0xbd1: 0x4000, - 0xbd2: 0x4000, 0xbd3: 0x4000, 0xbd4: 0x4000, 0xbd5: 0x4000, 0xbd6: 0x4000, 0xbd7: 0x4000, - 0xbd8: 0x4000, 0xbd9: 0x4000, 0xbdb: 0x4000, 0xbdc: 0x4000, 0xbdd: 0x4000, - 0xbde: 0x4000, 0xbdf: 0x4000, 0xbe0: 0x4000, 0xbe1: 0x4000, 0xbe2: 0x4000, 0xbe3: 0x4000, - 0xbe4: 0x4000, 0xbe5: 0x4000, 0xbe6: 0x4000, 0xbe7: 0x4000, 0xbe8: 0x4000, 0xbe9: 0x4000, - 0xbea: 0x4000, 0xbeb: 0x4000, 0xbec: 0x4000, 0xbed: 0x4000, 0xbee: 0x4000, 0xbef: 0x4000, - 0xbf0: 0x4000, 0xbf1: 0x4000, 0xbf2: 0x4000, 0xbf3: 0x4000, 0xbf4: 0x4000, 0xbf5: 0x4000, - 0xbf6: 0x4000, 0xbf7: 0x4000, 0xbf8: 0x4000, 0xbf9: 0x4000, 0xbfa: 0x4000, 0xbfb: 0x4000, - 0xbfc: 0x4000, 0xbfd: 0x4000, 0xbfe: 0x4000, 0xbff: 0x4000, - // Block 0x30, offset 0xc00 - 0xc00: 0x4000, 0xc01: 0x4000, 0xc02: 0x4000, 0xc03: 0x4000, 0xc04: 0x4000, 0xc05: 0x4000, - 0xc06: 0x4000, 0xc07: 0x4000, 0xc08: 0x4000, 0xc09: 0x4000, 0xc0a: 0x4000, 0xc0b: 0x4000, - 0xc0c: 0x4000, 0xc0d: 0x4000, 0xc0e: 0x4000, 0xc0f: 0x4000, 0xc10: 0x4000, 0xc11: 0x4000, - 0xc12: 0x4000, 0xc13: 0x4000, 0xc14: 0x4000, 0xc15: 0x4000, 0xc16: 0x4000, 0xc17: 0x4000, - 0xc18: 0x4000, 0xc19: 0x4000, 0xc1a: 0x4000, 0xc1b: 0x4000, 0xc1c: 0x4000, 0xc1d: 0x4000, - 0xc1e: 0x4000, 0xc1f: 0x4000, 0xc20: 0x4000, 0xc21: 0x4000, 0xc22: 0x4000, 0xc23: 0x4000, - 0xc24: 0x4000, 0xc25: 0x4000, 0xc26: 0x4000, 0xc27: 0x4000, 0xc28: 0x4000, 0xc29: 0x4000, - 0xc2a: 0x4000, 0xc2b: 0x4000, 0xc2c: 0x4000, 0xc2d: 0x4000, 0xc2e: 0x4000, 0xc2f: 0x4000, - 0xc30: 0x4000, 0xc31: 0x4000, 0xc32: 0x4000, 0xc33: 0x4000, - // Block 0x31, offset 0xc40 - 0xc40: 0x4000, 0xc41: 0x4000, 0xc42: 0x4000, 0xc43: 0x4000, 0xc44: 0x4000, 0xc45: 0x4000, - 0xc46: 0x4000, 0xc47: 0x4000, 0xc48: 0x4000, 0xc49: 0x4000, 0xc4a: 0x4000, 0xc4b: 0x4000, - 0xc4c: 0x4000, 0xc4d: 0x4000, 0xc4e: 0x4000, 0xc4f: 0x4000, 0xc50: 0x4000, 0xc51: 0x4000, - 0xc52: 0x4000, 0xc53: 0x4000, 0xc54: 0x4000, 0xc55: 0x4000, - 0xc70: 0x4000, 0xc71: 0x4000, 0xc72: 0x4000, 0xc73: 0x4000, 0xc74: 0x4000, 0xc75: 0x4000, - 0xc76: 0x4000, 0xc77: 0x4000, 0xc78: 0x4000, 0xc79: 0x4000, 0xc7a: 0x4000, 0xc7b: 0x4000, - // Block 0x32, offset 0xc80 - 0xc80: 0x9012, 0xc81: 0x4013, 0xc82: 0x4014, 0xc83: 0x4000, 0xc84: 0x4000, 0xc85: 0x4000, - 0xc86: 0x4000, 0xc87: 0x4000, 0xc88: 0x4000, 0xc89: 0x4000, 0xc8a: 0x4000, 0xc8b: 0x4000, - 0xc8c: 0x4015, 0xc8d: 0x4015, 0xc8e: 0x4000, 0xc8f: 0x4000, 0xc90: 0x4000, 0xc91: 0x4000, - 0xc92: 0x4000, 0xc93: 0x4000, 0xc94: 0x4000, 0xc95: 0x4000, 0xc96: 0x4000, 0xc97: 0x4000, - 0xc98: 0x4000, 0xc99: 0x4000, 0xc9a: 0x4000, 0xc9b: 0x4000, 0xc9c: 0x4000, 0xc9d: 0x4000, - 0xc9e: 0x4000, 0xc9f: 0x4000, 0xca0: 0x4000, 0xca1: 0x4000, 0xca2: 0x4000, 0xca3: 0x4000, - 0xca4: 0x4000, 0xca5: 0x4000, 0xca6: 0x4000, 0xca7: 0x4000, 0xca8: 0x4000, 0xca9: 0x4000, - 0xcaa: 0x4000, 0xcab: 0x4000, 0xcac: 0x4000, 0xcad: 0x4000, 0xcae: 0x4000, 0xcaf: 0x4000, - 0xcb0: 0x4000, 0xcb1: 0x4000, 0xcb2: 0x4000, 0xcb3: 0x4000, 0xcb4: 0x4000, 0xcb5: 0x4000, - 0xcb6: 0x4000, 0xcb7: 0x4000, 0xcb8: 0x4000, 0xcb9: 0x4000, 0xcba: 0x4000, 0xcbb: 0x4000, - 0xcbc: 0x4000, 0xcbd: 0x4000, 0xcbe: 0x4000, - // Block 0x33, offset 0xcc0 - 0xcc1: 0x4000, 0xcc2: 0x4000, 0xcc3: 0x4000, 0xcc4: 0x4000, 0xcc5: 0x4000, - 0xcc6: 0x4000, 0xcc7: 0x4000, 0xcc8: 0x4000, 0xcc9: 0x4000, 0xcca: 0x4000, 0xccb: 0x4000, - 0xccc: 0x4000, 0xccd: 0x4000, 0xcce: 0x4000, 0xccf: 0x4000, 0xcd0: 0x4000, 0xcd1: 0x4000, - 0xcd2: 0x4000, 0xcd3: 0x4000, 0xcd4: 0x4000, 0xcd5: 0x4000, 0xcd6: 0x4000, 0xcd7: 0x4000, - 0xcd8: 0x4000, 0xcd9: 0x4000, 0xcda: 0x4000, 0xcdb: 0x4000, 0xcdc: 0x4000, 0xcdd: 0x4000, - 0xcde: 0x4000, 0xcdf: 0x4000, 0xce0: 0x4000, 0xce1: 0x4000, 0xce2: 0x4000, 0xce3: 0x4000, - 0xce4: 0x4000, 0xce5: 0x4000, 0xce6: 0x4000, 0xce7: 0x4000, 0xce8: 0x4000, 0xce9: 0x4000, - 0xcea: 0x4000, 0xceb: 0x4000, 0xcec: 0x4000, 0xced: 0x4000, 0xcee: 0x4000, 0xcef: 0x4000, - 0xcf0: 0x4000, 0xcf1: 0x4000, 0xcf2: 0x4000, 0xcf3: 0x4000, 0xcf4: 0x4000, 0xcf5: 0x4000, - 0xcf6: 0x4000, 0xcf7: 0x4000, 0xcf8: 0x4000, 0xcf9: 0x4000, 0xcfa: 0x4000, 0xcfb: 0x4000, - 0xcfc: 0x4000, 0xcfd: 0x4000, 0xcfe: 0x4000, 0xcff: 0x4000, - // Block 0x34, offset 0xd00 - 0xd00: 0x4000, 0xd01: 0x4000, 0xd02: 0x4000, 0xd03: 0x4000, 0xd04: 0x4000, 0xd05: 0x4000, - 0xd06: 0x4000, 0xd07: 0x4000, 0xd08: 0x4000, 0xd09: 0x4000, 0xd0a: 0x4000, 0xd0b: 0x4000, - 0xd0c: 0x4000, 0xd0d: 0x4000, 0xd0e: 0x4000, 0xd0f: 0x4000, 0xd10: 0x4000, 0xd11: 0x4000, - 0xd12: 0x4000, 0xd13: 0x4000, 0xd14: 0x4000, 0xd15: 0x4000, 0xd16: 0x4000, - 0xd19: 0x4016, 0xd1a: 0x4017, 0xd1b: 0x4000, 0xd1c: 0x4000, 0xd1d: 0x4000, - 0xd1e: 0x4000, 0xd1f: 0x4000, 0xd20: 0x4000, 0xd21: 0x4018, 0xd22: 0x4019, 0xd23: 0x401a, - 0xd24: 0x401b, 0xd25: 0x401c, 0xd26: 0x401d, 0xd27: 0x401e, 0xd28: 0x401f, 0xd29: 0x4020, - 0xd2a: 0x4021, 0xd2b: 0x4022, 0xd2c: 0x4000, 0xd2d: 0x4010, 0xd2e: 0x4000, 0xd2f: 0x4023, - 0xd30: 0x4000, 0xd31: 0x4024, 0xd32: 0x4000, 0xd33: 0x4025, 0xd34: 0x4000, 0xd35: 0x4026, - 0xd36: 0x4000, 0xd37: 0x401a, 0xd38: 0x4000, 0xd39: 0x4027, 0xd3a: 0x4000, 0xd3b: 0x4028, - 0xd3c: 0x4000, 0xd3d: 0x4020, 0xd3e: 0x4000, 0xd3f: 0x4029, - // Block 0x35, offset 0xd40 - 0xd40: 0x4000, 0xd41: 0x402a, 0xd42: 0x4000, 0xd43: 0x402b, 0xd44: 0x402c, 0xd45: 0x4000, - 0xd46: 0x4017, 0xd47: 0x4000, 0xd48: 0x402d, 0xd49: 0x4000, 0xd4a: 0x402e, 0xd4b: 0x402f, - 0xd4c: 0x4030, 0xd4d: 0x4017, 0xd4e: 0x4016, 0xd4f: 0x4017, 0xd50: 0x4000, 0xd51: 0x4000, - 0xd52: 0x4031, 0xd53: 0x4000, 0xd54: 0x4000, 0xd55: 0x4031, 0xd56: 0x4000, 0xd57: 0x4000, - 0xd58: 0x4032, 0xd59: 0x4000, 0xd5a: 0x4000, 0xd5b: 0x4032, 0xd5c: 0x4000, 0xd5d: 0x4000, - 0xd5e: 0x4033, 0xd5f: 0x402e, 0xd60: 0x4034, 0xd61: 0x4035, 0xd62: 0x4034, 0xd63: 0x4036, - 0xd64: 0x4037, 0xd65: 0x4024, 0xd66: 0x4035, 0xd67: 0x4025, 0xd68: 0x4038, 0xd69: 0x4038, - 0xd6a: 0x4039, 0xd6b: 0x4039, 0xd6c: 0x403a, 0xd6d: 0x403a, 0xd6e: 0x4000, 0xd6f: 0x4035, - 0xd70: 0x4000, 0xd71: 0x4000, 0xd72: 0x403b, 0xd73: 0x403c, 0xd74: 0x4000, 0xd75: 0x4000, - 0xd76: 0x4000, 0xd77: 0x4000, 0xd78: 0x4000, 0xd79: 0x4000, 0xd7a: 0x4000, 0xd7b: 0x403d, - 0xd7c: 0x401c, 0xd7d: 0x4000, 0xd7e: 0x4000, 0xd7f: 0x4000, - // Block 0x36, offset 0xd80 - 0xd85: 0x4000, - 0xd86: 0x4000, 0xd87: 0x4000, 0xd88: 0x4000, 0xd89: 0x4000, 0xd8a: 0x4000, 0xd8b: 0x4000, - 0xd8c: 0x4000, 0xd8d: 0x4000, 0xd8e: 0x4000, 0xd8f: 0x4000, 0xd90: 0x4000, 0xd91: 0x4000, - 0xd92: 0x4000, 0xd93: 0x4000, 0xd94: 0x4000, 0xd95: 0x4000, 0xd96: 0x4000, 0xd97: 0x4000, - 0xd98: 0x4000, 0xd99: 0x4000, 0xd9a: 0x4000, 0xd9b: 0x4000, 0xd9c: 0x4000, 0xd9d: 0x4000, - 0xd9e: 0x4000, 0xd9f: 0x4000, 0xda0: 0x4000, 0xda1: 0x4000, 0xda2: 0x4000, 0xda3: 0x4000, - 0xda4: 0x4000, 0xda5: 0x4000, 0xda6: 0x4000, 0xda7: 0x4000, 0xda8: 0x4000, 0xda9: 0x4000, - 0xdaa: 0x4000, 0xdab: 0x4000, 0xdac: 0x4000, 0xdad: 0x4000, 0xdae: 0x4000, 0xdaf: 0x4000, - 0xdb1: 0x403e, 0xdb2: 0x403e, 0xdb3: 0x403e, 0xdb4: 0x403e, 0xdb5: 0x403e, - 0xdb6: 0x403e, 0xdb7: 0x403e, 0xdb8: 0x403e, 0xdb9: 0x403e, 0xdba: 0x403e, 0xdbb: 0x403e, - 0xdbc: 0x403e, 0xdbd: 0x403e, 0xdbe: 0x403e, 0xdbf: 0x403e, - // Block 0x37, offset 0xdc0 - 0xdc0: 0x4037, 0xdc1: 0x4037, 0xdc2: 0x4037, 0xdc3: 0x4037, 0xdc4: 0x4037, 0xdc5: 0x4037, - 0xdc6: 0x4037, 0xdc7: 0x4037, 0xdc8: 0x4037, 0xdc9: 0x4037, 0xdca: 0x4037, 0xdcb: 0x4037, - 0xdcc: 0x4037, 0xdcd: 0x4037, 0xdce: 0x4037, 0xdcf: 0x400e, 0xdd0: 0x403f, 0xdd1: 0x4040, - 0xdd2: 0x4041, 0xdd3: 0x4040, 0xdd4: 0x403f, 0xdd5: 0x4042, 0xdd6: 0x4043, 0xdd7: 0x4044, - 0xdd8: 0x4040, 0xdd9: 0x4041, 0xdda: 0x4040, 0xddb: 0x4045, 0xddc: 0x4009, 0xddd: 0x4045, - 0xdde: 0x4046, 0xddf: 0x4045, 0xde0: 0x4047, 0xde1: 0x400b, 0xde2: 0x400a, 0xde3: 0x400c, - 0xde4: 0x4048, 0xde5: 0x4000, 0xde6: 0x4000, 0xde7: 0x4000, 0xde8: 0x4000, 0xde9: 0x4000, - 0xdea: 0x4000, 0xdeb: 0x4000, 0xdec: 0x4000, 0xded: 0x4000, 0xdee: 0x4000, 0xdef: 0x4000, - 0xdf0: 0x4000, 0xdf1: 0x4000, 0xdf2: 0x4000, 0xdf3: 0x4000, 0xdf4: 0x4000, 0xdf5: 0x4000, - 0xdf6: 0x4000, 0xdf7: 0x4000, 0xdf8: 0x4000, 0xdf9: 0x4000, 0xdfa: 0x4000, 0xdfb: 0x4000, - 0xdfc: 0x4000, 0xdfd: 0x4000, 0xdfe: 0x4000, 0xdff: 0x4000, - // Block 0x38, offset 0xe00 - 0xe00: 0x4000, 0xe01: 0x4000, 0xe02: 0x4000, 0xe03: 0x4000, 0xe04: 0x4000, 0xe05: 0x4000, - 0xe06: 0x4000, 0xe07: 0x4000, 0xe08: 0x4000, 0xe09: 0x4000, 0xe0a: 0x4000, 0xe0b: 0x4000, - 0xe0c: 0x4000, 0xe0d: 0x4000, 0xe0e: 0x4000, 0xe10: 0x4000, 0xe11: 0x4000, - 0xe12: 0x4000, 0xe13: 0x4000, 0xe14: 0x4000, 0xe15: 0x4000, 0xe16: 0x4000, 0xe17: 0x4000, - 0xe18: 0x4000, 0xe19: 0x4000, 0xe1a: 0x4000, 0xe1b: 0x4000, 0xe1c: 0x4000, 0xe1d: 0x4000, - 0xe1e: 0x4000, 0xe1f: 0x4000, 0xe20: 0x4000, 0xe21: 0x4000, 0xe22: 0x4000, 0xe23: 0x4000, - 0xe24: 0x4000, 0xe25: 0x4000, 0xe26: 0x4000, 0xe27: 0x4000, 0xe28: 0x4000, 0xe29: 0x4000, - 0xe2a: 0x4000, 0xe2b: 0x4000, 0xe2c: 0x4000, 0xe2d: 0x4000, 0xe2e: 0x4000, 0xe2f: 0x4000, - 0xe30: 0x4000, 0xe31: 0x4000, 0xe32: 0x4000, 0xe33: 0x4000, 0xe34: 0x4000, 0xe35: 0x4000, - 0xe36: 0x4000, 0xe37: 0x4000, 0xe38: 0x4000, 0xe39: 0x4000, 0xe3a: 0x4000, - // Block 0x39, offset 0xe40 - 0xe40: 0x4000, 0xe41: 0x4000, 0xe42: 0x4000, 0xe43: 0x4000, 0xe44: 0x4000, 0xe45: 0x4000, - 0xe46: 0x4000, 0xe47: 0x4000, 0xe48: 0x4000, 0xe49: 0x4000, 0xe4a: 0x4000, 0xe4b: 0x4000, - 0xe4c: 0x4000, 0xe4d: 0x4000, 0xe4e: 0x4000, 0xe4f: 0x4000, 0xe50: 0x4000, 0xe51: 0x4000, - 0xe52: 0x4000, 0xe53: 0x4000, 0xe54: 0x4000, 0xe55: 0x4000, 0xe56: 0x4000, 0xe57: 0x4000, - 0xe58: 0x4000, 0xe59: 0x4000, 0xe5a: 0x4000, 0xe5b: 0x4000, 0xe5c: 0x4000, 0xe5d: 0x4000, - 0xe5e: 0x4000, 0xe5f: 0x4000, 0xe60: 0x4000, 0xe61: 0x4000, 0xe62: 0x4000, 0xe63: 0x4000, - 0xe70: 0x4000, 0xe71: 0x4000, 0xe72: 0x4000, 0xe73: 0x4000, 0xe74: 0x4000, 0xe75: 0x4000, - 0xe76: 0x4000, 0xe77: 0x4000, 0xe78: 0x4000, 0xe79: 0x4000, 0xe7a: 0x4000, 0xe7b: 0x4000, - 0xe7c: 0x4000, 0xe7d: 0x4000, 0xe7e: 0x4000, 0xe7f: 0x4000, - // Block 0x3a, offset 0xe80 - 0xe80: 0x4000, 0xe81: 0x4000, 0xe82: 0x4000, 0xe83: 0x4000, 0xe84: 0x4000, 0xe85: 0x4000, - 0xe86: 0x4000, 0xe87: 0x4000, 0xe88: 0x4000, 0xe89: 0x4000, 0xe8a: 0x4000, 0xe8b: 0x4000, - 0xe8c: 0x4000, 0xe8d: 0x4000, 0xe8e: 0x4000, 0xe8f: 0x4000, 0xe90: 0x4000, 0xe91: 0x4000, - 0xe92: 0x4000, 0xe93: 0x4000, 0xe94: 0x4000, 0xe95: 0x4000, 0xe96: 0x4000, 0xe97: 0x4000, - 0xe98: 0x4000, 0xe99: 0x4000, 0xe9a: 0x4000, 0xe9b: 0x4000, 0xe9c: 0x4000, 0xe9d: 0x4000, - 0xe9e: 0x4000, 0xea0: 0x4000, 0xea1: 0x4000, 0xea2: 0x4000, 0xea3: 0x4000, - 0xea4: 0x4000, 0xea5: 0x4000, 0xea6: 0x4000, 0xea7: 0x4000, 0xea8: 0x4000, 0xea9: 0x4000, - 0xeaa: 0x4000, 0xeab: 0x4000, 0xeac: 0x4000, 0xead: 0x4000, 0xeae: 0x4000, 0xeaf: 0x4000, - 0xeb0: 0x4000, 0xeb1: 0x4000, 0xeb2: 0x4000, 0xeb3: 0x4000, 0xeb4: 0x4000, 0xeb5: 0x4000, - 0xeb6: 0x4000, 0xeb7: 0x4000, 0xeb8: 0x4000, 0xeb9: 0x4000, 0xeba: 0x4000, 0xebb: 0x4000, - 0xebc: 0x4000, 0xebd: 0x4000, 0xebe: 0x4000, 0xebf: 0x4000, - // Block 0x3b, offset 0xec0 - 0xec0: 0x4000, 0xec1: 0x4000, 0xec2: 0x4000, 0xec3: 0x4000, 0xec4: 0x4000, 0xec5: 0x4000, - 0xec6: 0x4000, 0xec7: 0x4000, 0xec8: 0x2000, 0xec9: 0x2000, 0xeca: 0x2000, 0xecb: 0x2000, - 0xecc: 0x2000, 0xecd: 0x2000, 0xece: 0x2000, 0xecf: 0x2000, 0xed0: 0x4000, 0xed1: 0x4000, - 0xed2: 0x4000, 0xed3: 0x4000, 0xed4: 0x4000, 0xed5: 0x4000, 0xed6: 0x4000, 0xed7: 0x4000, - 0xed8: 0x4000, 0xed9: 0x4000, 0xeda: 0x4000, 0xedb: 0x4000, 0xedc: 0x4000, 0xedd: 0x4000, - 0xede: 0x4000, 0xedf: 0x4000, 0xee0: 0x4000, 0xee1: 0x4000, 0xee2: 0x4000, 0xee3: 0x4000, - 0xee4: 0x4000, 0xee5: 0x4000, 0xee6: 0x4000, 0xee7: 0x4000, 0xee8: 0x4000, 0xee9: 0x4000, - 0xeea: 0x4000, 0xeeb: 0x4000, 0xeec: 0x4000, 0xeed: 0x4000, 0xeee: 0x4000, 0xeef: 0x4000, - 0xef0: 0x4000, 0xef1: 0x4000, 0xef2: 0x4000, 0xef3: 0x4000, 0xef4: 0x4000, 0xef5: 0x4000, - 0xef6: 0x4000, 0xef7: 0x4000, 0xef8: 0x4000, 0xef9: 0x4000, 0xefa: 0x4000, 0xefb: 0x4000, - 0xefc: 0x4000, 0xefd: 0x4000, 0xefe: 0x4000, 0xeff: 0x4000, - // Block 0x3c, offset 0xf00 - 0xf00: 0x4000, 0xf01: 0x4000, 0xf02: 0x4000, 0xf03: 0x4000, 0xf04: 0x4000, 0xf05: 0x4000, - 0xf06: 0x4000, 0xf07: 0x4000, 0xf08: 0x4000, 0xf09: 0x4000, 0xf0a: 0x4000, 0xf0b: 0x4000, - 0xf0c: 0x4000, 0xf0d: 0x4000, 0xf0e: 0x4000, 0xf0f: 0x4000, 0xf10: 0x4000, 0xf11: 0x4000, - 0xf12: 0x4000, 0xf13: 0x4000, 0xf14: 0x4000, 0xf15: 0x4000, 0xf16: 0x4000, 0xf17: 0x4000, - 0xf18: 0x4000, 0xf19: 0x4000, 0xf1a: 0x4000, 0xf1b: 0x4000, 0xf1c: 0x4000, 0xf1d: 0x4000, - 0xf1e: 0x4000, 0xf1f: 0x4000, 0xf20: 0x4000, 0xf21: 0x4000, 0xf22: 0x4000, 0xf23: 0x4000, - 0xf24: 0x4000, 0xf25: 0x4000, 0xf26: 0x4000, 0xf27: 0x4000, 0xf28: 0x4000, 0xf29: 0x4000, - 0xf2a: 0x4000, 0xf2b: 0x4000, 0xf2c: 0x4000, 0xf2d: 0x4000, 0xf2e: 0x4000, 0xf2f: 0x4000, - 0xf30: 0x4000, 0xf31: 0x4000, 0xf32: 0x4000, 0xf33: 0x4000, 0xf34: 0x4000, 0xf35: 0x4000, - 0xf36: 0x4000, 0xf37: 0x4000, 0xf38: 0x4000, 0xf39: 0x4000, 0xf3a: 0x4000, 0xf3b: 0x4000, - 0xf3c: 0x4000, 0xf3d: 0x4000, 0xf3e: 0x4000, - // Block 0x3d, offset 0xf40 - 0xf40: 0x4000, 0xf41: 0x4000, 0xf42: 0x4000, 0xf43: 0x4000, 0xf44: 0x4000, 0xf45: 0x4000, - 0xf46: 0x4000, 0xf47: 0x4000, 0xf48: 0x4000, 0xf49: 0x4000, 0xf4a: 0x4000, 0xf4b: 0x4000, - 0xf4c: 0x4000, 0xf50: 0x4000, 0xf51: 0x4000, - 0xf52: 0x4000, 0xf53: 0x4000, 0xf54: 0x4000, 0xf55: 0x4000, 0xf56: 0x4000, 0xf57: 0x4000, - 0xf58: 0x4000, 0xf59: 0x4000, 0xf5a: 0x4000, 0xf5b: 0x4000, 0xf5c: 0x4000, 0xf5d: 0x4000, - 0xf5e: 0x4000, 0xf5f: 0x4000, 0xf60: 0x4000, 0xf61: 0x4000, 0xf62: 0x4000, 0xf63: 0x4000, - 0xf64: 0x4000, 0xf65: 0x4000, 0xf66: 0x4000, 0xf67: 0x4000, 0xf68: 0x4000, 0xf69: 0x4000, - 0xf6a: 0x4000, 0xf6b: 0x4000, 0xf6c: 0x4000, 0xf6d: 0x4000, 0xf6e: 0x4000, 0xf6f: 0x4000, - 0xf70: 0x4000, 0xf71: 0x4000, 0xf72: 0x4000, 0xf73: 0x4000, 0xf74: 0x4000, 0xf75: 0x4000, - 0xf76: 0x4000, 0xf77: 0x4000, 0xf78: 0x4000, 0xf79: 0x4000, 0xf7a: 0x4000, 0xf7b: 0x4000, - 0xf7c: 0x4000, 0xf7d: 0x4000, 0xf7e: 0x4000, 0xf7f: 0x4000, - // Block 0x3e, offset 0xf80 - 0xf80: 0x4000, 0xf81: 0x4000, 0xf82: 0x4000, 0xf83: 0x4000, 0xf84: 0x4000, 0xf85: 0x4000, - 0xf86: 0x4000, - // Block 0x3f, offset 0xfc0 - 0xfe0: 0x4000, 0xfe1: 0x4000, 0xfe2: 0x4000, 0xfe3: 0x4000, - 0xfe4: 0x4000, 0xfe5: 0x4000, 0xfe6: 0x4000, 0xfe7: 0x4000, 0xfe8: 0x4000, 0xfe9: 0x4000, - 0xfea: 0x4000, 0xfeb: 0x4000, 0xfec: 0x4000, 0xfed: 0x4000, 0xfee: 0x4000, 0xfef: 0x4000, - 0xff0: 0x4000, 0xff1: 0x4000, 0xff2: 0x4000, 0xff3: 0x4000, 0xff4: 0x4000, 0xff5: 0x4000, - 0xff6: 0x4000, 0xff7: 0x4000, 0xff8: 0x4000, 0xff9: 0x4000, 0xffa: 0x4000, 0xffb: 0x4000, - 0xffc: 0x4000, - // Block 0x40, offset 0x1000 - 0x1000: 0x4000, 0x1001: 0x4000, 0x1002: 0x4000, 0x1003: 0x4000, 0x1004: 0x4000, 0x1005: 0x4000, - 0x1006: 0x4000, 0x1007: 0x4000, 0x1008: 0x4000, 0x1009: 0x4000, 0x100a: 0x4000, 0x100b: 0x4000, - 0x100c: 0x4000, 0x100d: 0x4000, 0x100e: 0x4000, 0x100f: 0x4000, 0x1010: 0x4000, 0x1011: 0x4000, - 0x1012: 0x4000, 0x1013: 0x4000, 0x1014: 0x4000, 0x1015: 0x4000, 0x1016: 0x4000, 0x1017: 0x4000, - 0x1018: 0x4000, 0x1019: 0x4000, 0x101a: 0x4000, 0x101b: 0x4000, 0x101c: 0x4000, 0x101d: 0x4000, - 0x101e: 0x4000, 0x101f: 0x4000, 0x1020: 0x4000, 0x1021: 0x4000, 0x1022: 0x4000, 0x1023: 0x4000, - // Block 0x41, offset 0x1040 - 0x1040: 0x2000, 0x1041: 0x2000, 0x1042: 0x2000, 0x1043: 0x2000, 0x1044: 0x2000, 0x1045: 0x2000, - 0x1046: 0x2000, 0x1047: 0x2000, 0x1048: 0x2000, 0x1049: 0x2000, 0x104a: 0x2000, 0x104b: 0x2000, - 0x104c: 0x2000, 0x104d: 0x2000, 0x104e: 0x2000, 0x104f: 0x2000, 0x1050: 0x4000, 0x1051: 0x4000, - 0x1052: 0x4000, 0x1053: 0x4000, 0x1054: 0x4000, 0x1055: 0x4000, 0x1056: 0x4000, 0x1057: 0x4000, - 0x1058: 0x4000, 0x1059: 0x4000, - 0x1070: 0x4000, 0x1071: 0x4000, 0x1072: 0x4000, 0x1073: 0x4000, 0x1074: 0x4000, 0x1075: 0x4000, - 0x1076: 0x4000, 0x1077: 0x4000, 0x1078: 0x4000, 0x1079: 0x4000, 0x107a: 0x4000, 0x107b: 0x4000, - 0x107c: 0x4000, 0x107d: 0x4000, 0x107e: 0x4000, 0x107f: 0x4000, - // Block 0x42, offset 0x1080 - 0x1080: 0x4000, 0x1081: 0x4000, 0x1082: 0x4000, 0x1083: 0x4000, 0x1084: 0x4000, 0x1085: 0x4000, - 0x1086: 0x4000, 0x1087: 0x4000, 0x1088: 0x4000, 0x1089: 0x4000, 0x108a: 0x4000, 0x108b: 0x4000, - 0x108c: 0x4000, 0x108d: 0x4000, 0x108e: 0x4000, 0x108f: 0x4000, 0x1090: 0x4000, 0x1091: 0x4000, - 0x1092: 0x4000, 0x1094: 0x4000, 0x1095: 0x4000, 0x1096: 0x4000, 0x1097: 0x4000, - 0x1098: 0x4000, 0x1099: 0x4000, 0x109a: 0x4000, 0x109b: 0x4000, 0x109c: 0x4000, 0x109d: 0x4000, - 0x109e: 0x4000, 0x109f: 0x4000, 0x10a0: 0x4000, 0x10a1: 0x4000, 0x10a2: 0x4000, 0x10a3: 0x4000, - 0x10a4: 0x4000, 0x10a5: 0x4000, 0x10a6: 0x4000, 0x10a8: 0x4000, 0x10a9: 0x4000, - 0x10aa: 0x4000, 0x10ab: 0x4000, - // Block 0x43, offset 0x10c0 - 0x10c1: 0x9012, 0x10c2: 0x9012, 0x10c3: 0x9012, 0x10c4: 0x9012, 0x10c5: 0x9012, - 0x10c6: 0x9012, 0x10c7: 0x9012, 0x10c8: 0x9012, 0x10c9: 0x9012, 0x10ca: 0x9012, 0x10cb: 0x9012, - 0x10cc: 0x9012, 0x10cd: 0x9012, 0x10ce: 0x9012, 0x10cf: 0x9012, 0x10d0: 0x9012, 0x10d1: 0x9012, - 0x10d2: 0x9012, 0x10d3: 0x9012, 0x10d4: 0x9012, 0x10d5: 0x9012, 0x10d6: 0x9012, 0x10d7: 0x9012, - 0x10d8: 0x9012, 0x10d9: 0x9012, 0x10da: 0x9012, 0x10db: 0x9012, 0x10dc: 0x9012, 0x10dd: 0x9012, - 0x10de: 0x9012, 0x10df: 0x9012, 0x10e0: 0x9049, 0x10e1: 0x9049, 0x10e2: 0x9049, 0x10e3: 0x9049, - 0x10e4: 0x9049, 0x10e5: 0x9049, 0x10e6: 0x9049, 0x10e7: 0x9049, 0x10e8: 0x9049, 0x10e9: 0x9049, - 0x10ea: 0x9049, 0x10eb: 0x9049, 0x10ec: 0x9049, 0x10ed: 0x9049, 0x10ee: 0x9049, 0x10ef: 0x9049, - 0x10f0: 0x9049, 0x10f1: 0x9049, 0x10f2: 0x9049, 0x10f3: 0x9049, 0x10f4: 0x9049, 0x10f5: 0x9049, - 0x10f6: 0x9049, 0x10f7: 0x9049, 0x10f8: 0x9049, 0x10f9: 0x9049, 0x10fa: 0x9049, 0x10fb: 0x9049, - 0x10fc: 0x9049, 0x10fd: 0x9049, 0x10fe: 0x9049, 0x10ff: 0x9049, - // Block 0x44, offset 0x1100 - 0x1100: 0x9049, 0x1101: 0x9049, 0x1102: 0x9049, 0x1103: 0x9049, 0x1104: 0x9049, 0x1105: 0x9049, - 0x1106: 0x9049, 0x1107: 0x9049, 0x1108: 0x9049, 0x1109: 0x9049, 0x110a: 0x9049, 0x110b: 0x9049, - 0x110c: 0x9049, 0x110d: 0x9049, 0x110e: 0x9049, 0x110f: 0x9049, 0x1110: 0x9049, 0x1111: 0x9049, - 0x1112: 0x9049, 0x1113: 0x9049, 0x1114: 0x9049, 0x1115: 0x9049, 0x1116: 0x9049, 0x1117: 0x9049, - 0x1118: 0x9049, 0x1119: 0x9049, 0x111a: 0x9049, 0x111b: 0x9049, 0x111c: 0x9049, 0x111d: 0x9049, - 0x111e: 0x9049, 0x111f: 0x904a, 0x1120: 0x904b, 0x1121: 0xb04c, 0x1122: 0xb04d, 0x1123: 0xb04d, - 0x1124: 0xb04e, 0x1125: 0xb04f, 0x1126: 0xb050, 0x1127: 0xb051, 0x1128: 0xb052, 0x1129: 0xb053, - 0x112a: 0xb054, 0x112b: 0xb055, 0x112c: 0xb056, 0x112d: 0xb057, 0x112e: 0xb058, 0x112f: 0xb059, - 0x1130: 0xb05a, 0x1131: 0xb05b, 0x1132: 0xb05c, 0x1133: 0xb05d, 0x1134: 0xb05e, 0x1135: 0xb05f, - 0x1136: 0xb060, 0x1137: 0xb061, 0x1138: 0xb062, 0x1139: 0xb063, 0x113a: 0xb064, 0x113b: 0xb065, - 0x113c: 0xb052, 0x113d: 0xb066, 0x113e: 0xb067, 0x113f: 0xb055, - // Block 0x45, offset 0x1140 - 0x1140: 0xb068, 0x1141: 0xb069, 0x1142: 0xb06a, 0x1143: 0xb06b, 0x1144: 0xb05a, 0x1145: 0xb056, - 0x1146: 0xb06c, 0x1147: 0xb06d, 0x1148: 0xb06b, 0x1149: 0xb06e, 0x114a: 0xb06b, 0x114b: 0xb06f, - 0x114c: 0xb06f, 0x114d: 0xb070, 0x114e: 0xb070, 0x114f: 0xb071, 0x1150: 0xb056, 0x1151: 0xb072, - 0x1152: 0xb073, 0x1153: 0xb072, 0x1154: 0xb074, 0x1155: 0xb073, 0x1156: 0xb075, 0x1157: 0xb075, - 0x1158: 0xb076, 0x1159: 0xb076, 0x115a: 0xb077, 0x115b: 0xb077, 0x115c: 0xb073, 0x115d: 0xb078, - 0x115e: 0xb079, 0x115f: 0xb067, 0x1160: 0xb07a, 0x1161: 0xb07b, 0x1162: 0xb07b, 0x1163: 0xb07b, - 0x1164: 0xb07b, 0x1165: 0xb07b, 0x1166: 0xb07b, 0x1167: 0xb07b, 0x1168: 0xb07b, 0x1169: 0xb07b, - 0x116a: 0xb07b, 0x116b: 0xb07b, 0x116c: 0xb07b, 0x116d: 0xb07b, 0x116e: 0xb07b, 0x116f: 0xb07b, - 0x1170: 0xb07c, 0x1171: 0xb07c, 0x1172: 0xb07c, 0x1173: 0xb07c, 0x1174: 0xb07c, 0x1175: 0xb07c, - 0x1176: 0xb07c, 0x1177: 0xb07c, 0x1178: 0xb07c, 0x1179: 0xb07c, 0x117a: 0xb07c, 0x117b: 0xb07c, - 0x117c: 0xb07c, 0x117d: 0xb07c, 0x117e: 0xb07c, - // Block 0x46, offset 0x1180 - 0x1182: 0xb07d, 0x1183: 0xb07e, 0x1184: 0xb07f, 0x1185: 0xb080, - 0x1186: 0xb07f, 0x1187: 0xb07e, 0x118a: 0xb081, 0x118b: 0xb082, - 0x118c: 0xb083, 0x118d: 0xb07f, 0x118e: 0xb080, 0x118f: 0xb07f, - 0x1192: 0xb084, 0x1193: 0xb085, 0x1194: 0xb084, 0x1195: 0xb086, 0x1196: 0xb084, 0x1197: 0xb087, - 0x119a: 0xb088, 0x119b: 0xb089, 0x119c: 0xb08a, - 0x11a0: 0x908b, 0x11a1: 0x908b, 0x11a2: 0x908c, 0x11a3: 0x908d, - 0x11a4: 0x908b, 0x11a5: 0x908e, 0x11a6: 0x908f, 0x11a8: 0xb090, 0x11a9: 0xb091, - 0x11aa: 0xb092, 0x11ab: 0xb091, 0x11ac: 0xb093, 0x11ad: 0xb094, 0x11ae: 0xb095, - 0x11bd: 0x2000, - // Block 0x47, offset 0x11c0 - 0x11e0: 0x4000, 0x11e1: 0x4000, 0x11e2: 0x4000, 0x11e3: 0x4000, - // Block 0x48, offset 0x1200 - 0x1200: 0x4000, 0x1201: 0x4000, 0x1202: 0x4000, 0x1203: 0x4000, 0x1204: 0x4000, 0x1205: 0x4000, - 0x1206: 0x4000, 0x1207: 0x4000, 0x1208: 0x4000, 0x1209: 0x4000, 0x120a: 0x4000, 0x120b: 0x4000, - 0x120c: 0x4000, 0x120d: 0x4000, 0x120e: 0x4000, 0x120f: 0x4000, 0x1210: 0x4000, 0x1211: 0x4000, - 0x1212: 0x4000, 0x1213: 0x4000, 0x1214: 0x4000, 0x1215: 0x4000, 0x1216: 0x4000, 0x1217: 0x4000, - 0x1218: 0x4000, 0x1219: 0x4000, 0x121a: 0x4000, 0x121b: 0x4000, 0x121c: 0x4000, 0x121d: 0x4000, - 0x121e: 0x4000, 0x121f: 0x4000, 0x1220: 0x4000, 0x1221: 0x4000, 0x1222: 0x4000, 0x1223: 0x4000, - 0x1224: 0x4000, 0x1225: 0x4000, 0x1226: 0x4000, 0x1227: 0x4000, 0x1228: 0x4000, 0x1229: 0x4000, - 0x122a: 0x4000, 0x122b: 0x4000, 0x122c: 0x4000, 0x122d: 0x4000, 0x122e: 0x4000, 0x122f: 0x4000, - 0x1230: 0x4000, 0x1231: 0x4000, 0x1232: 0x4000, 0x1233: 0x4000, 0x1234: 0x4000, 0x1235: 0x4000, - 0x1236: 0x4000, 0x1237: 0x4000, - // Block 0x49, offset 0x1240 - 0x1240: 0x4000, 0x1241: 0x4000, 0x1242: 0x4000, 0x1243: 0x4000, 0x1244: 0x4000, 0x1245: 0x4000, - 0x1246: 0x4000, 0x1247: 0x4000, 0x1248: 0x4000, 0x1249: 0x4000, 0x124a: 0x4000, 0x124b: 0x4000, - 0x124c: 0x4000, 0x124d: 0x4000, 0x124e: 0x4000, 0x124f: 0x4000, 0x1250: 0x4000, 0x1251: 0x4000, - 0x1252: 0x4000, 0x1253: 0x4000, 0x1254: 0x4000, 0x1255: 0x4000, 0x1256: 0x4000, 0x1257: 0x4000, - 0x1258: 0x4000, 0x1259: 0x4000, 0x125a: 0x4000, 0x125b: 0x4000, 0x125c: 0x4000, 0x125d: 0x4000, - 0x125e: 0x4000, 0x125f: 0x4000, 0x1260: 0x4000, 0x1261: 0x4000, 0x1262: 0x4000, 0x1263: 0x4000, - 0x1264: 0x4000, 0x1265: 0x4000, 0x1266: 0x4000, 0x1267: 0x4000, 0x1268: 0x4000, 0x1269: 0x4000, - 0x126a: 0x4000, 0x126b: 0x4000, 0x126c: 0x4000, 0x126d: 0x4000, 0x126e: 0x4000, 0x126f: 0x4000, - 0x1270: 0x4000, 0x1271: 0x4000, 0x1272: 0x4000, - // Block 0x4a, offset 0x1280 - 0x1280: 0x4000, 0x1281: 0x4000, 0x1282: 0x4000, 0x1283: 0x4000, 0x1284: 0x4000, 0x1285: 0x4000, - 0x1286: 0x4000, 0x1287: 0x4000, 0x1288: 0x4000, 0x1289: 0x4000, 0x128a: 0x4000, 0x128b: 0x4000, - 0x128c: 0x4000, 0x128d: 0x4000, 0x128e: 0x4000, 0x128f: 0x4000, 0x1290: 0x4000, 0x1291: 0x4000, - 0x1292: 0x4000, 0x1293: 0x4000, 0x1294: 0x4000, 0x1295: 0x4000, 0x1296: 0x4000, 0x1297: 0x4000, - 0x1298: 0x4000, 0x1299: 0x4000, 0x129a: 0x4000, 0x129b: 0x4000, 0x129c: 0x4000, 0x129d: 0x4000, - 0x129e: 0x4000, - // Block 0x4b, offset 0x12c0 - 0x12d0: 0x4000, 0x12d1: 0x4000, - 0x12d2: 0x4000, - 0x12e4: 0x4000, 0x12e5: 0x4000, 0x12e6: 0x4000, 0x12e7: 0x4000, - 0x12f0: 0x4000, 0x12f1: 0x4000, 0x12f2: 0x4000, 0x12f3: 0x4000, 0x12f4: 0x4000, 0x12f5: 0x4000, - 0x12f6: 0x4000, 0x12f7: 0x4000, 0x12f8: 0x4000, 0x12f9: 0x4000, 0x12fa: 0x4000, 0x12fb: 0x4000, - 0x12fc: 0x4000, 0x12fd: 0x4000, 0x12fe: 0x4000, 0x12ff: 0x4000, - // Block 0x4c, offset 0x1300 - 0x1300: 0x4000, 0x1301: 0x4000, 0x1302: 0x4000, 0x1303: 0x4000, 0x1304: 0x4000, 0x1305: 0x4000, - 0x1306: 0x4000, 0x1307: 0x4000, 0x1308: 0x4000, 0x1309: 0x4000, 0x130a: 0x4000, 0x130b: 0x4000, - 0x130c: 0x4000, 0x130d: 0x4000, 0x130e: 0x4000, 0x130f: 0x4000, 0x1310: 0x4000, 0x1311: 0x4000, - 0x1312: 0x4000, 0x1313: 0x4000, 0x1314: 0x4000, 0x1315: 0x4000, 0x1316: 0x4000, 0x1317: 0x4000, - 0x1318: 0x4000, 0x1319: 0x4000, 0x131a: 0x4000, 0x131b: 0x4000, 0x131c: 0x4000, 0x131d: 0x4000, - 0x131e: 0x4000, 0x131f: 0x4000, 0x1320: 0x4000, 0x1321: 0x4000, 0x1322: 0x4000, 0x1323: 0x4000, - 0x1324: 0x4000, 0x1325: 0x4000, 0x1326: 0x4000, 0x1327: 0x4000, 0x1328: 0x4000, 0x1329: 0x4000, - 0x132a: 0x4000, 0x132b: 0x4000, 0x132c: 0x4000, 0x132d: 0x4000, 0x132e: 0x4000, 0x132f: 0x4000, - 0x1330: 0x4000, 0x1331: 0x4000, 0x1332: 0x4000, 0x1333: 0x4000, 0x1334: 0x4000, 0x1335: 0x4000, - 0x1336: 0x4000, 0x1337: 0x4000, 0x1338: 0x4000, 0x1339: 0x4000, 0x133a: 0x4000, 0x133b: 0x4000, - // Block 0x4d, offset 0x1340 - 0x1344: 0x4000, - // Block 0x4e, offset 0x1380 - 0x138f: 0x4000, - // Block 0x4f, offset 0x13c0 - 0x13c0: 0x2000, 0x13c1: 0x2000, 0x13c2: 0x2000, 0x13c3: 0x2000, 0x13c4: 0x2000, 0x13c5: 0x2000, - 0x13c6: 0x2000, 0x13c7: 0x2000, 0x13c8: 0x2000, 0x13c9: 0x2000, 0x13ca: 0x2000, - 0x13d0: 0x2000, 0x13d1: 0x2000, - 0x13d2: 0x2000, 0x13d3: 0x2000, 0x13d4: 0x2000, 0x13d5: 0x2000, 0x13d6: 0x2000, 0x13d7: 0x2000, - 0x13d8: 0x2000, 0x13d9: 0x2000, 0x13da: 0x2000, 0x13db: 0x2000, 0x13dc: 0x2000, 0x13dd: 0x2000, - 0x13de: 0x2000, 0x13df: 0x2000, 0x13e0: 0x2000, 0x13e1: 0x2000, 0x13e2: 0x2000, 0x13e3: 0x2000, - 0x13e4: 0x2000, 0x13e5: 0x2000, 0x13e6: 0x2000, 0x13e7: 0x2000, 0x13e8: 0x2000, 0x13e9: 0x2000, - 0x13ea: 0x2000, 0x13eb: 0x2000, 0x13ec: 0x2000, 0x13ed: 0x2000, - 0x13f0: 0x2000, 0x13f1: 0x2000, 0x13f2: 0x2000, 0x13f3: 0x2000, 0x13f4: 0x2000, 0x13f5: 0x2000, - 0x13f6: 0x2000, 0x13f7: 0x2000, 0x13f8: 0x2000, 0x13f9: 0x2000, 0x13fa: 0x2000, 0x13fb: 0x2000, - 0x13fc: 0x2000, 0x13fd: 0x2000, 0x13fe: 0x2000, 0x13ff: 0x2000, - // Block 0x50, offset 0x1400 - 0x1400: 0x2000, 0x1401: 0x2000, 0x1402: 0x2000, 0x1403: 0x2000, 0x1404: 0x2000, 0x1405: 0x2000, - 0x1406: 0x2000, 0x1407: 0x2000, 0x1408: 0x2000, 0x1409: 0x2000, 0x140a: 0x2000, 0x140b: 0x2000, - 0x140c: 0x2000, 0x140d: 0x2000, 0x140e: 0x2000, 0x140f: 0x2000, 0x1410: 0x2000, 0x1411: 0x2000, - 0x1412: 0x2000, 0x1413: 0x2000, 0x1414: 0x2000, 0x1415: 0x2000, 0x1416: 0x2000, 0x1417: 0x2000, - 0x1418: 0x2000, 0x1419: 0x2000, 0x141a: 0x2000, 0x141b: 0x2000, 0x141c: 0x2000, 0x141d: 0x2000, - 0x141e: 0x2000, 0x141f: 0x2000, 0x1420: 0x2000, 0x1421: 0x2000, 0x1422: 0x2000, 0x1423: 0x2000, - 0x1424: 0x2000, 0x1425: 0x2000, 0x1426: 0x2000, 0x1427: 0x2000, 0x1428: 0x2000, 0x1429: 0x2000, - 0x1430: 0x2000, 0x1431: 0x2000, 0x1432: 0x2000, 0x1433: 0x2000, 0x1434: 0x2000, 0x1435: 0x2000, - 0x1436: 0x2000, 0x1437: 0x2000, 0x1438: 0x2000, 0x1439: 0x2000, 0x143a: 0x2000, 0x143b: 0x2000, - 0x143c: 0x2000, 0x143d: 0x2000, 0x143e: 0x2000, 0x143f: 0x2000, - // Block 0x51, offset 0x1440 - 0x1440: 0x2000, 0x1441: 0x2000, 0x1442: 0x2000, 0x1443: 0x2000, 0x1444: 0x2000, 0x1445: 0x2000, - 0x1446: 0x2000, 0x1447: 0x2000, 0x1448: 0x2000, 0x1449: 0x2000, 0x144a: 0x2000, 0x144b: 0x2000, - 0x144c: 0x2000, 0x144d: 0x2000, 0x144e: 0x4000, 0x144f: 0x2000, 0x1450: 0x2000, 0x1451: 0x4000, - 0x1452: 0x4000, 0x1453: 0x4000, 0x1454: 0x4000, 0x1455: 0x4000, 0x1456: 0x4000, 0x1457: 0x4000, - 0x1458: 0x4000, 0x1459: 0x4000, 0x145a: 0x4000, 0x145b: 0x2000, 0x145c: 0x2000, 0x145d: 0x2000, - 0x145e: 0x2000, 0x145f: 0x2000, 0x1460: 0x2000, 0x1461: 0x2000, 0x1462: 0x2000, 0x1463: 0x2000, - 0x1464: 0x2000, 0x1465: 0x2000, 0x1466: 0x2000, 0x1467: 0x2000, 0x1468: 0x2000, 0x1469: 0x2000, - 0x146a: 0x2000, 0x146b: 0x2000, 0x146c: 0x2000, - // Block 0x52, offset 0x1480 - 0x1480: 0x4000, 0x1481: 0x4000, 0x1482: 0x4000, - 0x1490: 0x4000, 0x1491: 0x4000, - 0x1492: 0x4000, 0x1493: 0x4000, 0x1494: 0x4000, 0x1495: 0x4000, 0x1496: 0x4000, 0x1497: 0x4000, - 0x1498: 0x4000, 0x1499: 0x4000, 0x149a: 0x4000, 0x149b: 0x4000, 0x149c: 0x4000, 0x149d: 0x4000, - 0x149e: 0x4000, 0x149f: 0x4000, 0x14a0: 0x4000, 0x14a1: 0x4000, 0x14a2: 0x4000, 0x14a3: 0x4000, - 0x14a4: 0x4000, 0x14a5: 0x4000, 0x14a6: 0x4000, 0x14a7: 0x4000, 0x14a8: 0x4000, 0x14a9: 0x4000, - 0x14aa: 0x4000, 0x14ab: 0x4000, 0x14ac: 0x4000, 0x14ad: 0x4000, 0x14ae: 0x4000, 0x14af: 0x4000, - 0x14b0: 0x4000, 0x14b1: 0x4000, 0x14b2: 0x4000, 0x14b3: 0x4000, 0x14b4: 0x4000, 0x14b5: 0x4000, - 0x14b6: 0x4000, 0x14b7: 0x4000, 0x14b8: 0x4000, 0x14b9: 0x4000, 0x14ba: 0x4000, 0x14bb: 0x4000, - // Block 0x53, offset 0x14c0 - 0x14c0: 0x4000, 0x14c1: 0x4000, 0x14c2: 0x4000, 0x14c3: 0x4000, 0x14c4: 0x4000, 0x14c5: 0x4000, - 0x14c6: 0x4000, 0x14c7: 0x4000, 0x14c8: 0x4000, - 0x14d0: 0x4000, 0x14d1: 0x4000, - 0x14e0: 0x4000, 0x14e1: 0x4000, 0x14e2: 0x4000, 0x14e3: 0x4000, - 0x14e4: 0x4000, 0x14e5: 0x4000, - // Block 0x54, offset 0x1500 - 0x1500: 0x4000, 0x1501: 0x4000, 0x1502: 0x4000, 0x1503: 0x4000, 0x1504: 0x4000, 0x1505: 0x4000, - 0x1506: 0x4000, 0x1507: 0x4000, 0x1508: 0x4000, 0x1509: 0x4000, 0x150a: 0x4000, 0x150b: 0x4000, - 0x150c: 0x4000, 0x150d: 0x4000, 0x150e: 0x4000, 0x150f: 0x4000, 0x1510: 0x4000, 0x1511: 0x4000, - 0x1512: 0x4000, 0x1513: 0x4000, 0x1514: 0x4000, 0x1515: 0x4000, 0x1516: 0x4000, 0x1517: 0x4000, - 0x1518: 0x4000, 0x1519: 0x4000, 0x151a: 0x4000, 0x151b: 0x4000, 0x151c: 0x4000, 0x151d: 0x4000, - 0x151e: 0x4000, 0x151f: 0x4000, 0x1520: 0x4000, - 0x152d: 0x4000, 0x152e: 0x4000, 0x152f: 0x4000, - 0x1530: 0x4000, 0x1531: 0x4000, 0x1532: 0x4000, 0x1533: 0x4000, 0x1534: 0x4000, 0x1535: 0x4000, - 0x1537: 0x4000, 0x1538: 0x4000, 0x1539: 0x4000, 0x153a: 0x4000, 0x153b: 0x4000, - 0x153c: 0x4000, 0x153d: 0x4000, 0x153e: 0x4000, 0x153f: 0x4000, - // Block 0x55, offset 0x1540 - 0x1540: 0x4000, 0x1541: 0x4000, 0x1542: 0x4000, 0x1543: 0x4000, 0x1544: 0x4000, 0x1545: 0x4000, - 0x1546: 0x4000, 0x1547: 0x4000, 0x1548: 0x4000, 0x1549: 0x4000, 0x154a: 0x4000, 0x154b: 0x4000, - 0x154c: 0x4000, 0x154d: 0x4000, 0x154e: 0x4000, 0x154f: 0x4000, 0x1550: 0x4000, 0x1551: 0x4000, - 0x1552: 0x4000, 0x1553: 0x4000, 0x1554: 0x4000, 0x1555: 0x4000, 0x1556: 0x4000, 0x1557: 0x4000, - 0x1558: 0x4000, 0x1559: 0x4000, 0x155a: 0x4000, 0x155b: 0x4000, 0x155c: 0x4000, 0x155d: 0x4000, - 0x155e: 0x4000, 0x155f: 0x4000, 0x1560: 0x4000, 0x1561: 0x4000, 0x1562: 0x4000, 0x1563: 0x4000, - 0x1564: 0x4000, 0x1565: 0x4000, 0x1566: 0x4000, 0x1567: 0x4000, 0x1568: 0x4000, 0x1569: 0x4000, - 0x156a: 0x4000, 0x156b: 0x4000, 0x156c: 0x4000, 0x156d: 0x4000, 0x156e: 0x4000, 0x156f: 0x4000, - 0x1570: 0x4000, 0x1571: 0x4000, 0x1572: 0x4000, 0x1573: 0x4000, 0x1574: 0x4000, 0x1575: 0x4000, - 0x1576: 0x4000, 0x1577: 0x4000, 0x1578: 0x4000, 0x1579: 0x4000, 0x157a: 0x4000, 0x157b: 0x4000, - 0x157c: 0x4000, 0x157e: 0x4000, 0x157f: 0x4000, - // Block 0x56, offset 0x1580 - 0x1580: 0x4000, 0x1581: 0x4000, 0x1582: 0x4000, 0x1583: 0x4000, 0x1584: 0x4000, 0x1585: 0x4000, - 0x1586: 0x4000, 0x1587: 0x4000, 0x1588: 0x4000, 0x1589: 0x4000, 0x158a: 0x4000, 0x158b: 0x4000, - 0x158c: 0x4000, 0x158d: 0x4000, 0x158e: 0x4000, 0x158f: 0x4000, 0x1590: 0x4000, 0x1591: 0x4000, - 0x1592: 0x4000, 0x1593: 0x4000, - 0x15a0: 0x4000, 0x15a1: 0x4000, 0x15a2: 0x4000, 0x15a3: 0x4000, - 0x15a4: 0x4000, 0x15a5: 0x4000, 0x15a6: 0x4000, 0x15a7: 0x4000, 0x15a8: 0x4000, 0x15a9: 0x4000, - 0x15aa: 0x4000, 0x15ab: 0x4000, 0x15ac: 0x4000, 0x15ad: 0x4000, 0x15ae: 0x4000, 0x15af: 0x4000, - 0x15b0: 0x4000, 0x15b1: 0x4000, 0x15b2: 0x4000, 0x15b3: 0x4000, 0x15b4: 0x4000, 0x15b5: 0x4000, - 0x15b6: 0x4000, 0x15b7: 0x4000, 0x15b8: 0x4000, 0x15b9: 0x4000, 0x15ba: 0x4000, 0x15bb: 0x4000, - 0x15bc: 0x4000, 0x15bd: 0x4000, 0x15be: 0x4000, 0x15bf: 0x4000, - // Block 0x57, offset 0x15c0 - 0x15c0: 0x4000, 0x15c1: 0x4000, 0x15c2: 0x4000, 0x15c3: 0x4000, 0x15c4: 0x4000, 0x15c5: 0x4000, - 0x15c6: 0x4000, 0x15c7: 0x4000, 0x15c8: 0x4000, 0x15c9: 0x4000, 0x15ca: 0x4000, - 0x15cf: 0x4000, 0x15d0: 0x4000, 0x15d1: 0x4000, - 0x15d2: 0x4000, 0x15d3: 0x4000, - 0x15e0: 0x4000, 0x15e1: 0x4000, 0x15e2: 0x4000, 0x15e3: 0x4000, - 0x15e4: 0x4000, 0x15e5: 0x4000, 0x15e6: 0x4000, 0x15e7: 0x4000, 0x15e8: 0x4000, 0x15e9: 0x4000, - 0x15ea: 0x4000, 0x15eb: 0x4000, 0x15ec: 0x4000, 0x15ed: 0x4000, 0x15ee: 0x4000, 0x15ef: 0x4000, - 0x15f0: 0x4000, 0x15f4: 0x4000, - 0x15f8: 0x4000, 0x15f9: 0x4000, 0x15fa: 0x4000, 0x15fb: 0x4000, - 0x15fc: 0x4000, 0x15fd: 0x4000, 0x15fe: 0x4000, 0x15ff: 0x4000, - // Block 0x58, offset 0x1600 - 0x1600: 0x4000, 0x1602: 0x4000, 0x1603: 0x4000, 0x1604: 0x4000, 0x1605: 0x4000, - 0x1606: 0x4000, 0x1607: 0x4000, 0x1608: 0x4000, 0x1609: 0x4000, 0x160a: 0x4000, 0x160b: 0x4000, - 0x160c: 0x4000, 0x160d: 0x4000, 0x160e: 0x4000, 0x160f: 0x4000, 0x1610: 0x4000, 0x1611: 0x4000, - 0x1612: 0x4000, 0x1613: 0x4000, 0x1614: 0x4000, 0x1615: 0x4000, 0x1616: 0x4000, 0x1617: 0x4000, - 0x1618: 0x4000, 0x1619: 0x4000, 0x161a: 0x4000, 0x161b: 0x4000, 0x161c: 0x4000, 0x161d: 0x4000, - 0x161e: 0x4000, 0x161f: 0x4000, 0x1620: 0x4000, 0x1621: 0x4000, 0x1622: 0x4000, 0x1623: 0x4000, - 0x1624: 0x4000, 0x1625: 0x4000, 0x1626: 0x4000, 0x1627: 0x4000, 0x1628: 0x4000, 0x1629: 0x4000, - 0x162a: 0x4000, 0x162b: 0x4000, 0x162c: 0x4000, 0x162d: 0x4000, 0x162e: 0x4000, 0x162f: 0x4000, - 0x1630: 0x4000, 0x1631: 0x4000, 0x1632: 0x4000, 0x1633: 0x4000, 0x1634: 0x4000, 0x1635: 0x4000, - 0x1636: 0x4000, 0x1637: 0x4000, 0x1638: 0x4000, 0x1639: 0x4000, 0x163a: 0x4000, 0x163b: 0x4000, - 0x163c: 0x4000, 0x163d: 0x4000, 0x163e: 0x4000, 0x163f: 0x4000, - // Block 0x59, offset 0x1640 - 0x1640: 0x4000, 0x1641: 0x4000, 0x1642: 0x4000, 0x1643: 0x4000, 0x1644: 0x4000, 0x1645: 0x4000, - 0x1646: 0x4000, 0x1647: 0x4000, 0x1648: 0x4000, 0x1649: 0x4000, 0x164a: 0x4000, 0x164b: 0x4000, - 0x164c: 0x4000, 0x164d: 0x4000, 0x164e: 0x4000, 0x164f: 0x4000, 0x1650: 0x4000, 0x1651: 0x4000, - 0x1652: 0x4000, 0x1653: 0x4000, 0x1654: 0x4000, 0x1655: 0x4000, 0x1656: 0x4000, 0x1657: 0x4000, - 0x1658: 0x4000, 0x1659: 0x4000, 0x165a: 0x4000, 0x165b: 0x4000, 0x165c: 0x4000, 0x165d: 0x4000, - 0x165e: 0x4000, 0x165f: 0x4000, 0x1660: 0x4000, 0x1661: 0x4000, 0x1662: 0x4000, 0x1663: 0x4000, - 0x1664: 0x4000, 0x1665: 0x4000, 0x1666: 0x4000, 0x1667: 0x4000, 0x1668: 0x4000, 0x1669: 0x4000, - 0x166a: 0x4000, 0x166b: 0x4000, 0x166c: 0x4000, 0x166d: 0x4000, 0x166e: 0x4000, 0x166f: 0x4000, - 0x1670: 0x4000, 0x1671: 0x4000, 0x1672: 0x4000, 0x1673: 0x4000, 0x1674: 0x4000, 0x1675: 0x4000, - 0x1676: 0x4000, 0x1677: 0x4000, 0x1678: 0x4000, 0x1679: 0x4000, 0x167a: 0x4000, 0x167b: 0x4000, - 0x167c: 0x4000, 0x167f: 0x4000, - // Block 0x5a, offset 0x1680 - 0x1680: 0x4000, 0x1681: 0x4000, 0x1682: 0x4000, 0x1683: 0x4000, 0x1684: 0x4000, 0x1685: 0x4000, - 0x1686: 0x4000, 0x1687: 0x4000, 0x1688: 0x4000, 0x1689: 0x4000, 0x168a: 0x4000, 0x168b: 0x4000, - 0x168c: 0x4000, 0x168d: 0x4000, 0x168e: 0x4000, 0x168f: 0x4000, 0x1690: 0x4000, 0x1691: 0x4000, - 0x1692: 0x4000, 0x1693: 0x4000, 0x1694: 0x4000, 0x1695: 0x4000, 0x1696: 0x4000, 0x1697: 0x4000, - 0x1698: 0x4000, 0x1699: 0x4000, 0x169a: 0x4000, 0x169b: 0x4000, 0x169c: 0x4000, 0x169d: 0x4000, - 0x169e: 0x4000, 0x169f: 0x4000, 0x16a0: 0x4000, 0x16a1: 0x4000, 0x16a2: 0x4000, 0x16a3: 0x4000, - 0x16a4: 0x4000, 0x16a5: 0x4000, 0x16a6: 0x4000, 0x16a7: 0x4000, 0x16a8: 0x4000, 0x16a9: 0x4000, - 0x16aa: 0x4000, 0x16ab: 0x4000, 0x16ac: 0x4000, 0x16ad: 0x4000, 0x16ae: 0x4000, 0x16af: 0x4000, - 0x16b0: 0x4000, 0x16b1: 0x4000, 0x16b2: 0x4000, 0x16b3: 0x4000, 0x16b4: 0x4000, 0x16b5: 0x4000, - 0x16b6: 0x4000, 0x16b7: 0x4000, 0x16b8: 0x4000, 0x16b9: 0x4000, 0x16ba: 0x4000, 0x16bb: 0x4000, - 0x16bc: 0x4000, 0x16bd: 0x4000, - // Block 0x5b, offset 0x16c0 - 0x16cb: 0x4000, - 0x16cc: 0x4000, 0x16cd: 0x4000, 0x16ce: 0x4000, 0x16d0: 0x4000, 0x16d1: 0x4000, - 0x16d2: 0x4000, 0x16d3: 0x4000, 0x16d4: 0x4000, 0x16d5: 0x4000, 0x16d6: 0x4000, 0x16d7: 0x4000, - 0x16d8: 0x4000, 0x16d9: 0x4000, 0x16da: 0x4000, 0x16db: 0x4000, 0x16dc: 0x4000, 0x16dd: 0x4000, - 0x16de: 0x4000, 0x16df: 0x4000, 0x16e0: 0x4000, 0x16e1: 0x4000, 0x16e2: 0x4000, 0x16e3: 0x4000, - 0x16e4: 0x4000, 0x16e5: 0x4000, 0x16e6: 0x4000, 0x16e7: 0x4000, - 0x16fa: 0x4000, - // Block 0x5c, offset 0x1700 - 0x1715: 0x4000, 0x1716: 0x4000, - 0x1724: 0x4000, - // Block 0x5d, offset 0x1740 - 0x177b: 0x4000, - 0x177c: 0x4000, 0x177d: 0x4000, 0x177e: 0x4000, 0x177f: 0x4000, - // Block 0x5e, offset 0x1780 - 0x1780: 0x4000, 0x1781: 0x4000, 0x1782: 0x4000, 0x1783: 0x4000, 0x1784: 0x4000, 0x1785: 0x4000, - 0x1786: 0x4000, 0x1787: 0x4000, 0x1788: 0x4000, 0x1789: 0x4000, 0x178a: 0x4000, 0x178b: 0x4000, - 0x178c: 0x4000, 0x178d: 0x4000, 0x178e: 0x4000, 0x178f: 0x4000, - // Block 0x5f, offset 0x17c0 - 0x17c0: 0x4000, 0x17c1: 0x4000, 0x17c2: 0x4000, 0x17c3: 0x4000, 0x17c4: 0x4000, 0x17c5: 0x4000, - 0x17cc: 0x4000, 0x17d0: 0x4000, 0x17d1: 0x4000, - 0x17d2: 0x4000, 0x17d5: 0x4000, - 0x17eb: 0x4000, 0x17ec: 0x4000, - 0x17f4: 0x4000, 0x17f5: 0x4000, - 0x17f6: 0x4000, 0x17f7: 0x4000, 0x17f8: 0x4000, 0x17f9: 0x4000, 0x17fa: 0x4000, - // Block 0x60, offset 0x1800 - 0x1820: 0x4000, 0x1821: 0x4000, 0x1822: 0x4000, 0x1823: 0x4000, - 0x1824: 0x4000, 0x1825: 0x4000, 0x1826: 0x4000, 0x1827: 0x4000, 0x1828: 0x4000, 0x1829: 0x4000, - 0x182a: 0x4000, 0x182b: 0x4000, - // Block 0x61, offset 0x1840 - 0x184d: 0x4000, 0x184e: 0x4000, 0x184f: 0x4000, 0x1850: 0x4000, 0x1851: 0x4000, - 0x1852: 0x4000, 0x1853: 0x4000, 0x1854: 0x4000, 0x1855: 0x4000, 0x1856: 0x4000, 0x1857: 0x4000, - 0x1858: 0x4000, 0x1859: 0x4000, 0x185a: 0x4000, 0x185b: 0x4000, 0x185c: 0x4000, 0x185d: 0x4000, - 0x185e: 0x4000, 0x185f: 0x4000, 0x1860: 0x4000, 0x1861: 0x4000, 0x1862: 0x4000, 0x1863: 0x4000, - 0x1864: 0x4000, 0x1865: 0x4000, 0x1866: 0x4000, 0x1867: 0x4000, 0x1868: 0x4000, 0x1869: 0x4000, - 0x186a: 0x4000, 0x186b: 0x4000, 0x186c: 0x4000, 0x186d: 0x4000, 0x186e: 0x4000, 0x186f: 0x4000, - 0x1870: 0x4000, 0x1871: 0x4000, 0x1872: 0x4000, 0x1873: 0x4000, 0x1874: 0x4000, 0x1875: 0x4000, - 0x1876: 0x4000, 0x1877: 0x4000, 0x1878: 0x4000, 0x1879: 0x4000, 0x187a: 0x4000, 0x187b: 0x4000, - 0x187c: 0x4000, 0x187d: 0x4000, 0x187e: 0x4000, 0x187f: 0x4000, - // Block 0x62, offset 0x1880 - 0x1880: 0x4000, 0x1881: 0x4000, 0x1882: 0x4000, 0x1883: 0x4000, 0x1884: 0x4000, 0x1885: 0x4000, - 0x1886: 0x4000, 0x1887: 0x4000, 0x1888: 0x4000, 0x1889: 0x4000, 0x188a: 0x4000, 0x188b: 0x4000, - 0x188c: 0x4000, 0x188d: 0x4000, 0x188e: 0x4000, 0x188f: 0x4000, 0x1890: 0x4000, 0x1891: 0x4000, - 0x1892: 0x4000, 0x1893: 0x4000, 0x1894: 0x4000, 0x1895: 0x4000, 0x1896: 0x4000, 0x1897: 0x4000, - 0x1898: 0x4000, 0x1899: 0x4000, 0x189a: 0x4000, 0x189b: 0x4000, 0x189c: 0x4000, 0x189d: 0x4000, - 0x189e: 0x4000, 0x189f: 0x4000, 0x18a0: 0x4000, 0x18a1: 0x4000, 0x18a2: 0x4000, 0x18a3: 0x4000, - 0x18a4: 0x4000, 0x18a5: 0x4000, 0x18a6: 0x4000, 0x18a7: 0x4000, 0x18a8: 0x4000, 0x18a9: 0x4000, - 0x18aa: 0x4000, 0x18ab: 0x4000, 0x18ac: 0x4000, 0x18ad: 0x4000, 0x18ae: 0x4000, 0x18af: 0x4000, - 0x18b0: 0x4000, 0x18b1: 0x4000, 0x18b3: 0x4000, 0x18b4: 0x4000, 0x18b5: 0x4000, - 0x18b6: 0x4000, 0x18ba: 0x4000, 0x18bb: 0x4000, - 0x18bc: 0x4000, 0x18bd: 0x4000, 0x18be: 0x4000, 0x18bf: 0x4000, - // Block 0x63, offset 0x18c0 - 0x18c0: 0x4000, 0x18c1: 0x4000, 0x18c2: 0x4000, 0x18c3: 0x4000, 0x18c4: 0x4000, 0x18c5: 0x4000, - 0x18c6: 0x4000, 0x18c7: 0x4000, 0x18c8: 0x4000, 0x18c9: 0x4000, 0x18ca: 0x4000, 0x18cb: 0x4000, - 0x18cc: 0x4000, 0x18cd: 0x4000, 0x18ce: 0x4000, 0x18cf: 0x4000, 0x18d0: 0x4000, 0x18d1: 0x4000, - 0x18d2: 0x4000, 0x18d3: 0x4000, 0x18d4: 0x4000, 0x18d5: 0x4000, 0x18d6: 0x4000, 0x18d7: 0x4000, - 0x18d8: 0x4000, 0x18d9: 0x4000, 0x18da: 0x4000, 0x18db: 0x4000, 0x18dc: 0x4000, 0x18dd: 0x4000, - 0x18de: 0x4000, 0x18df: 0x4000, 0x18e0: 0x4000, 0x18e1: 0x4000, 0x18e2: 0x4000, - 0x18e5: 0x4000, 0x18e6: 0x4000, 0x18e7: 0x4000, 0x18e8: 0x4000, 0x18e9: 0x4000, - 0x18ea: 0x4000, 0x18ee: 0x4000, 0x18ef: 0x4000, - 0x18f0: 0x4000, 0x18f1: 0x4000, 0x18f2: 0x4000, 0x18f3: 0x4000, 0x18f4: 0x4000, 0x18f5: 0x4000, - 0x18f6: 0x4000, 0x18f7: 0x4000, 0x18f8: 0x4000, 0x18f9: 0x4000, 0x18fa: 0x4000, 0x18fb: 0x4000, - 0x18fc: 0x4000, 0x18fd: 0x4000, 0x18fe: 0x4000, 0x18ff: 0x4000, - // Block 0x64, offset 0x1900 - 0x1900: 0x4000, 0x1901: 0x4000, 0x1902: 0x4000, 0x1903: 0x4000, 0x1904: 0x4000, 0x1905: 0x4000, - 0x1906: 0x4000, 0x1907: 0x4000, 0x1908: 0x4000, 0x1909: 0x4000, 0x190a: 0x4000, - 0x190d: 0x4000, 0x190e: 0x4000, 0x190f: 0x4000, 0x1910: 0x4000, 0x1911: 0x4000, - 0x1912: 0x4000, 0x1913: 0x4000, 0x1914: 0x4000, 0x1915: 0x4000, 0x1916: 0x4000, 0x1917: 0x4000, - 0x1918: 0x4000, 0x1919: 0x4000, 0x191a: 0x4000, 0x191b: 0x4000, 0x191c: 0x4000, 0x191d: 0x4000, - 0x191e: 0x4000, 0x191f: 0x4000, 0x1920: 0x4000, 0x1921: 0x4000, 0x1922: 0x4000, 0x1923: 0x4000, - 0x1924: 0x4000, 0x1925: 0x4000, 0x1926: 0x4000, 0x1927: 0x4000, 0x1928: 0x4000, 0x1929: 0x4000, - 0x192a: 0x4000, 0x192b: 0x4000, 0x192c: 0x4000, 0x192d: 0x4000, 0x192e: 0x4000, 0x192f: 0x4000, - 0x1930: 0x4000, 0x1931: 0x4000, 0x1932: 0x4000, 0x1933: 0x4000, 0x1934: 0x4000, 0x1935: 0x4000, - 0x1936: 0x4000, 0x1937: 0x4000, 0x1938: 0x4000, 0x1939: 0x4000, 0x193a: 0x4000, 0x193b: 0x4000, - 0x193c: 0x4000, 0x193d: 0x4000, 0x193e: 0x4000, 0x193f: 0x4000, - // Block 0x65, offset 0x1940 - 0x1970: 0x4000, 0x1971: 0x4000, 0x1972: 0x4000, 0x1973: 0x4000, - 0x1978: 0x4000, 0x1979: 0x4000, 0x197a: 0x4000, - // Block 0x66, offset 0x1980 - 0x1980: 0x4000, 0x1981: 0x4000, 0x1982: 0x4000, - 0x1990: 0x4000, 0x1991: 0x4000, - 0x1992: 0x4000, 0x1993: 0x4000, 0x1994: 0x4000, 0x1995: 0x4000, - // Block 0x67, offset 0x19c0 - 0x19c0: 0x2000, 0x19c1: 0x2000, 0x19c2: 0x2000, 0x19c3: 0x2000, 0x19c4: 0x2000, 0x19c5: 0x2000, - 0x19c6: 0x2000, 0x19c7: 0x2000, 0x19c8: 0x2000, 0x19c9: 0x2000, 0x19ca: 0x2000, 0x19cb: 0x2000, - 0x19cc: 0x2000, 0x19cd: 0x2000, 0x19ce: 0x2000, 0x19cf: 0x2000, 0x19d0: 0x2000, 0x19d1: 0x2000, - 0x19d2: 0x2000, 0x19d3: 0x2000, 0x19d4: 0x2000, 0x19d5: 0x2000, 0x19d6: 0x2000, 0x19d7: 0x2000, - 0x19d8: 0x2000, 0x19d9: 0x2000, 0x19da: 0x2000, 0x19db: 0x2000, 0x19dc: 0x2000, 0x19dd: 0x2000, - 0x19de: 0x2000, 0x19df: 0x2000, 0x19e0: 0x2000, 0x19e1: 0x2000, 0x19e2: 0x2000, 0x19e3: 0x2000, - 0x19e4: 0x2000, 0x19e5: 0x2000, 0x19e6: 0x2000, 0x19e7: 0x2000, 0x19e8: 0x2000, 0x19e9: 0x2000, - 0x19ea: 0x2000, 0x19eb: 0x2000, 0x19ec: 0x2000, 0x19ed: 0x2000, 0x19ee: 0x2000, 0x19ef: 0x2000, - 0x19f0: 0x2000, 0x19f1: 0x2000, 0x19f2: 0x2000, 0x19f3: 0x2000, 0x19f4: 0x2000, 0x19f5: 0x2000, - 0x19f6: 0x2000, 0x19f7: 0x2000, 0x19f8: 0x2000, 0x19f9: 0x2000, 0x19fa: 0x2000, 0x19fb: 0x2000, - 0x19fc: 0x2000, 0x19fd: 0x2000, -} - -// widthIndex: 22 blocks, 1408 entries, 1408 bytes -// Block 0 is the zero block. -var widthIndex = [1408]uint8{ - // Block 0x0, offset 0x0 - // Block 0x1, offset 0x40 - // Block 0x2, offset 0x80 - // Block 0x3, offset 0xc0 - 0xc2: 0x01, 0xc3: 0x02, 0xc4: 0x03, 0xc5: 0x04, 0xc7: 0x05, - 0xc9: 0x06, 0xcb: 0x07, 0xcc: 0x08, 0xcd: 0x09, 0xce: 0x0a, 0xcf: 0x0b, - 0xd0: 0x0c, 0xd1: 0x0d, - 0xe1: 0x02, 0xe2: 0x03, 0xe3: 0x04, 0xe4: 0x05, 0xe5: 0x06, 0xe6: 0x06, 0xe7: 0x06, - 0xe8: 0x06, 0xe9: 0x06, 0xea: 0x07, 0xeb: 0x06, 0xec: 0x06, 0xed: 0x08, 0xee: 0x09, 0xef: 0x0a, - 0xf0: 0x0f, 0xf3: 0x12, 0xf4: 0x13, - // Block 0x4, offset 0x100 - 0x104: 0x0e, 0x105: 0x0f, - // Block 0x5, offset 0x140 - 0x140: 0x10, 0x141: 0x11, 0x142: 0x12, 0x144: 0x13, 0x145: 0x14, 0x146: 0x15, 0x147: 0x16, - 0x148: 0x17, 0x149: 0x18, 0x14a: 0x19, 0x14c: 0x1a, 0x14f: 0x1b, - 0x151: 0x1c, 0x152: 0x08, 0x153: 0x1d, 0x154: 0x1e, 0x155: 0x1f, 0x156: 0x20, 0x157: 0x21, - 0x158: 0x22, 0x159: 0x23, 0x15a: 0x24, 0x15b: 0x25, 0x15c: 0x26, 0x15d: 0x27, 0x15e: 0x28, 0x15f: 0x29, - 0x166: 0x2a, - 0x16c: 0x2b, 0x16d: 0x2c, - 0x17a: 0x2d, 0x17b: 0x2e, 0x17c: 0x0e, 0x17d: 0x0e, 0x17e: 0x0e, 0x17f: 0x2f, - // Block 0x6, offset 0x180 - 0x180: 0x30, 0x181: 0x31, 0x182: 0x32, 0x183: 0x33, 0x184: 0x34, 0x185: 0x35, 0x186: 0x36, 0x187: 0x37, - 0x188: 0x38, 0x189: 0x39, 0x18a: 0x0e, 0x18b: 0x3a, 0x18c: 0x0e, 0x18d: 0x0e, 0x18e: 0x0e, 0x18f: 0x0e, - 0x190: 0x0e, 0x191: 0x0e, 0x192: 0x0e, 0x193: 0x0e, 0x194: 0x0e, 0x195: 0x0e, 0x196: 0x0e, 0x197: 0x0e, - 0x198: 0x0e, 0x199: 0x0e, 0x19a: 0x0e, 0x19b: 0x0e, 0x19c: 0x0e, 0x19d: 0x0e, 0x19e: 0x0e, 0x19f: 0x0e, - 0x1a0: 0x0e, 0x1a1: 0x0e, 0x1a2: 0x0e, 0x1a3: 0x0e, 0x1a4: 0x0e, 0x1a5: 0x0e, 0x1a6: 0x0e, 0x1a7: 0x0e, - 0x1a8: 0x0e, 0x1a9: 0x0e, 0x1aa: 0x0e, 0x1ab: 0x0e, 0x1ac: 0x0e, 0x1ad: 0x0e, 0x1ae: 0x0e, 0x1af: 0x0e, - 0x1b0: 0x0e, 0x1b1: 0x0e, 0x1b2: 0x0e, 0x1b3: 0x0e, 0x1b4: 0x0e, 0x1b5: 0x0e, 0x1b6: 0x0e, 0x1b7: 0x0e, - 0x1b8: 0x0e, 0x1b9: 0x0e, 0x1ba: 0x0e, 0x1bb: 0x0e, 0x1bc: 0x0e, 0x1bd: 0x0e, 0x1be: 0x0e, 0x1bf: 0x0e, - // Block 0x7, offset 0x1c0 - 0x1c0: 0x0e, 0x1c1: 0x0e, 0x1c2: 0x0e, 0x1c3: 0x0e, 0x1c4: 0x0e, 0x1c5: 0x0e, 0x1c6: 0x0e, 0x1c7: 0x0e, - 0x1c8: 0x0e, 0x1c9: 0x0e, 0x1ca: 0x0e, 0x1cb: 0x0e, 0x1cc: 0x0e, 0x1cd: 0x0e, 0x1ce: 0x0e, 0x1cf: 0x0e, - 0x1d0: 0x0e, 0x1d1: 0x0e, 0x1d2: 0x0e, 0x1d3: 0x0e, 0x1d4: 0x0e, 0x1d5: 0x0e, 0x1d6: 0x0e, 0x1d7: 0x0e, - 0x1d8: 0x0e, 0x1d9: 0x0e, 0x1da: 0x0e, 0x1db: 0x0e, 0x1dc: 0x0e, 0x1dd: 0x0e, 0x1de: 0x0e, 0x1df: 0x0e, - 0x1e0: 0x0e, 0x1e1: 0x0e, 0x1e2: 0x0e, 0x1e3: 0x0e, 0x1e4: 0x0e, 0x1e5: 0x0e, 0x1e6: 0x0e, 0x1e7: 0x0e, - 0x1e8: 0x0e, 0x1e9: 0x0e, 0x1ea: 0x0e, 0x1eb: 0x0e, 0x1ec: 0x0e, 0x1ed: 0x0e, 0x1ee: 0x0e, 0x1ef: 0x0e, - 0x1f0: 0x0e, 0x1f1: 0x0e, 0x1f2: 0x0e, 0x1f3: 0x0e, 0x1f4: 0x0e, 0x1f5: 0x0e, 0x1f6: 0x0e, - 0x1f8: 0x0e, 0x1f9: 0x0e, 0x1fa: 0x0e, 0x1fb: 0x0e, 0x1fc: 0x0e, 0x1fd: 0x0e, 0x1fe: 0x0e, 0x1ff: 0x0e, - // Block 0x8, offset 0x200 - 0x200: 0x0e, 0x201: 0x0e, 0x202: 0x0e, 0x203: 0x0e, 0x204: 0x0e, 0x205: 0x0e, 0x206: 0x0e, 0x207: 0x0e, - 0x208: 0x0e, 0x209: 0x0e, 0x20a: 0x0e, 0x20b: 0x0e, 0x20c: 0x0e, 0x20d: 0x0e, 0x20e: 0x0e, 0x20f: 0x0e, - 0x210: 0x0e, 0x211: 0x0e, 0x212: 0x0e, 0x213: 0x0e, 0x214: 0x0e, 0x215: 0x0e, 0x216: 0x0e, 0x217: 0x0e, - 0x218: 0x0e, 0x219: 0x0e, 0x21a: 0x0e, 0x21b: 0x0e, 0x21c: 0x0e, 0x21d: 0x0e, 0x21e: 0x0e, 0x21f: 0x0e, - 0x220: 0x0e, 0x221: 0x0e, 0x222: 0x0e, 0x223: 0x0e, 0x224: 0x0e, 0x225: 0x0e, 0x226: 0x0e, 0x227: 0x0e, - 0x228: 0x0e, 0x229: 0x0e, 0x22a: 0x0e, 0x22b: 0x0e, 0x22c: 0x0e, 0x22d: 0x0e, 0x22e: 0x0e, 0x22f: 0x0e, - 0x230: 0x0e, 0x231: 0x0e, 0x232: 0x0e, 0x233: 0x0e, 0x234: 0x0e, 0x235: 0x0e, 0x236: 0x0e, 0x237: 0x0e, - 0x238: 0x0e, 0x239: 0x0e, 0x23a: 0x0e, 0x23b: 0x0e, 0x23c: 0x0e, 0x23d: 0x0e, 0x23e: 0x0e, 0x23f: 0x0e, - // Block 0x9, offset 0x240 - 0x240: 0x0e, 0x241: 0x0e, 0x242: 0x0e, 0x243: 0x0e, 0x244: 0x0e, 0x245: 0x0e, 0x246: 0x0e, 0x247: 0x0e, - 0x248: 0x0e, 0x249: 0x0e, 0x24a: 0x0e, 0x24b: 0x0e, 0x24c: 0x0e, 0x24d: 0x0e, 0x24e: 0x0e, 0x24f: 0x0e, - 0x250: 0x0e, 0x251: 0x0e, 0x252: 0x3b, 0x253: 0x3c, - 0x265: 0x3d, - 0x270: 0x0e, 0x271: 0x0e, 0x272: 0x0e, 0x273: 0x0e, 0x274: 0x0e, 0x275: 0x0e, 0x276: 0x0e, 0x277: 0x0e, - 0x278: 0x0e, 0x279: 0x0e, 0x27a: 0x0e, 0x27b: 0x0e, 0x27c: 0x0e, 0x27d: 0x0e, 0x27e: 0x0e, 0x27f: 0x0e, - // Block 0xa, offset 0x280 - 0x280: 0x0e, 0x281: 0x0e, 0x282: 0x0e, 0x283: 0x0e, 0x284: 0x0e, 0x285: 0x0e, 0x286: 0x0e, 0x287: 0x0e, - 0x288: 0x0e, 0x289: 0x0e, 0x28a: 0x0e, 0x28b: 0x0e, 0x28c: 0x0e, 0x28d: 0x0e, 0x28e: 0x0e, 0x28f: 0x0e, - 0x290: 0x0e, 0x291: 0x0e, 0x292: 0x0e, 0x293: 0x0e, 0x294: 0x0e, 0x295: 0x0e, 0x296: 0x0e, 0x297: 0x0e, - 0x298: 0x0e, 0x299: 0x0e, 0x29a: 0x0e, 0x29b: 0x0e, 0x29c: 0x0e, 0x29d: 0x0e, 0x29e: 0x3e, - // Block 0xb, offset 0x2c0 - 0x2c0: 0x08, 0x2c1: 0x08, 0x2c2: 0x08, 0x2c3: 0x08, 0x2c4: 0x08, 0x2c5: 0x08, 0x2c6: 0x08, 0x2c7: 0x08, - 0x2c8: 0x08, 0x2c9: 0x08, 0x2ca: 0x08, 0x2cb: 0x08, 0x2cc: 0x08, 0x2cd: 0x08, 0x2ce: 0x08, 0x2cf: 0x08, - 0x2d0: 0x08, 0x2d1: 0x08, 0x2d2: 0x08, 0x2d3: 0x08, 0x2d4: 0x08, 0x2d5: 0x08, 0x2d6: 0x08, 0x2d7: 0x08, - 0x2d8: 0x08, 0x2d9: 0x08, 0x2da: 0x08, 0x2db: 0x08, 0x2dc: 0x08, 0x2dd: 0x08, 0x2de: 0x08, 0x2df: 0x08, - 0x2e0: 0x08, 0x2e1: 0x08, 0x2e2: 0x08, 0x2e3: 0x08, 0x2e4: 0x08, 0x2e5: 0x08, 0x2e6: 0x08, 0x2e7: 0x08, - 0x2e8: 0x08, 0x2e9: 0x08, 0x2ea: 0x08, 0x2eb: 0x08, 0x2ec: 0x08, 0x2ed: 0x08, 0x2ee: 0x08, 0x2ef: 0x08, - 0x2f0: 0x08, 0x2f1: 0x08, 0x2f2: 0x08, 0x2f3: 0x08, 0x2f4: 0x08, 0x2f5: 0x08, 0x2f6: 0x08, 0x2f7: 0x08, - 0x2f8: 0x08, 0x2f9: 0x08, 0x2fa: 0x08, 0x2fb: 0x08, 0x2fc: 0x08, 0x2fd: 0x08, 0x2fe: 0x08, 0x2ff: 0x08, - // Block 0xc, offset 0x300 - 0x300: 0x08, 0x301: 0x08, 0x302: 0x08, 0x303: 0x08, 0x304: 0x08, 0x305: 0x08, 0x306: 0x08, 0x307: 0x08, - 0x308: 0x08, 0x309: 0x08, 0x30a: 0x08, 0x30b: 0x08, 0x30c: 0x08, 0x30d: 0x08, 0x30e: 0x08, 0x30f: 0x08, - 0x310: 0x08, 0x311: 0x08, 0x312: 0x08, 0x313: 0x08, 0x314: 0x08, 0x315: 0x08, 0x316: 0x08, 0x317: 0x08, - 0x318: 0x08, 0x319: 0x08, 0x31a: 0x08, 0x31b: 0x08, 0x31c: 0x08, 0x31d: 0x08, 0x31e: 0x08, 0x31f: 0x08, - 0x320: 0x08, 0x321: 0x08, 0x322: 0x08, 0x323: 0x08, 0x324: 0x0e, 0x325: 0x0e, 0x326: 0x0e, 0x327: 0x0e, - 0x328: 0x0e, 0x329: 0x0e, 0x32a: 0x0e, 0x32b: 0x0e, - 0x338: 0x3f, 0x339: 0x40, 0x33c: 0x41, 0x33d: 0x42, 0x33e: 0x43, 0x33f: 0x44, - // Block 0xd, offset 0x340 - 0x37f: 0x45, - // Block 0xe, offset 0x380 - 0x380: 0x0e, 0x381: 0x0e, 0x382: 0x0e, 0x383: 0x0e, 0x384: 0x0e, 0x385: 0x0e, 0x386: 0x0e, 0x387: 0x0e, - 0x388: 0x0e, 0x389: 0x0e, 0x38a: 0x0e, 0x38b: 0x0e, 0x38c: 0x0e, 0x38d: 0x0e, 0x38e: 0x0e, 0x38f: 0x0e, - 0x390: 0x0e, 0x391: 0x0e, 0x392: 0x0e, 0x393: 0x0e, 0x394: 0x0e, 0x395: 0x0e, 0x396: 0x0e, 0x397: 0x0e, - 0x398: 0x0e, 0x399: 0x0e, 0x39a: 0x0e, 0x39b: 0x0e, 0x39c: 0x0e, 0x39d: 0x0e, 0x39e: 0x0e, 0x39f: 0x46, - 0x3a0: 0x0e, 0x3a1: 0x0e, 0x3a2: 0x0e, 0x3a3: 0x0e, 0x3a4: 0x0e, 0x3a5: 0x0e, 0x3a6: 0x0e, 0x3a7: 0x0e, - 0x3a8: 0x0e, 0x3a9: 0x0e, 0x3aa: 0x0e, 0x3ab: 0x47, - // Block 0xf, offset 0x3c0 - 0x3c0: 0x0e, 0x3c1: 0x0e, 0x3c2: 0x0e, 0x3c3: 0x0e, 0x3c4: 0x48, 0x3c5: 0x49, 0x3c6: 0x0e, 0x3c7: 0x0e, - 0x3c8: 0x0e, 0x3c9: 0x0e, 0x3ca: 0x0e, 0x3cb: 0x4a, - // Block 0x10, offset 0x400 - 0x400: 0x4b, 0x403: 0x4c, 0x404: 0x4d, 0x405: 0x4e, 0x406: 0x4f, - 0x408: 0x50, 0x409: 0x51, 0x40c: 0x52, 0x40d: 0x53, 0x40e: 0x54, 0x40f: 0x55, - 0x410: 0x3a, 0x411: 0x56, 0x412: 0x0e, 0x413: 0x57, 0x414: 0x58, 0x415: 0x59, 0x416: 0x5a, 0x417: 0x5b, - 0x418: 0x0e, 0x419: 0x5c, 0x41a: 0x0e, 0x41b: 0x5d, 0x41f: 0x5e, - 0x424: 0x5f, 0x425: 0x60, 0x426: 0x61, 0x427: 0x62, - 0x429: 0x63, 0x42a: 0x64, - // Block 0x11, offset 0x440 - 0x456: 0x0b, 0x457: 0x06, - 0x458: 0x0c, 0x45b: 0x0d, 0x45f: 0x0e, - 0x460: 0x06, 0x461: 0x06, 0x462: 0x06, 0x463: 0x06, 0x464: 0x06, 0x465: 0x06, 0x466: 0x06, 0x467: 0x06, - 0x468: 0x06, 0x469: 0x06, 0x46a: 0x06, 0x46b: 0x06, 0x46c: 0x06, 0x46d: 0x06, 0x46e: 0x06, 0x46f: 0x06, - 0x470: 0x06, 0x471: 0x06, 0x472: 0x06, 0x473: 0x06, 0x474: 0x06, 0x475: 0x06, 0x476: 0x06, 0x477: 0x06, - 0x478: 0x06, 0x479: 0x06, 0x47a: 0x06, 0x47b: 0x06, 0x47c: 0x06, 0x47d: 0x06, 0x47e: 0x06, 0x47f: 0x06, - // Block 0x12, offset 0x480 - 0x484: 0x08, 0x485: 0x08, 0x486: 0x08, 0x487: 0x09, - // Block 0x13, offset 0x4c0 - 0x4c0: 0x08, 0x4c1: 0x08, 0x4c2: 0x08, 0x4c3: 0x08, 0x4c4: 0x08, 0x4c5: 0x08, 0x4c6: 0x08, 0x4c7: 0x08, - 0x4c8: 0x08, 0x4c9: 0x08, 0x4ca: 0x08, 0x4cb: 0x08, 0x4cc: 0x08, 0x4cd: 0x08, 0x4ce: 0x08, 0x4cf: 0x08, - 0x4d0: 0x08, 0x4d1: 0x08, 0x4d2: 0x08, 0x4d3: 0x08, 0x4d4: 0x08, 0x4d5: 0x08, 0x4d6: 0x08, 0x4d7: 0x08, - 0x4d8: 0x08, 0x4d9: 0x08, 0x4da: 0x08, 0x4db: 0x08, 0x4dc: 0x08, 0x4dd: 0x08, 0x4de: 0x08, 0x4df: 0x08, - 0x4e0: 0x08, 0x4e1: 0x08, 0x4e2: 0x08, 0x4e3: 0x08, 0x4e4: 0x08, 0x4e5: 0x08, 0x4e6: 0x08, 0x4e7: 0x08, - 0x4e8: 0x08, 0x4e9: 0x08, 0x4ea: 0x08, 0x4eb: 0x08, 0x4ec: 0x08, 0x4ed: 0x08, 0x4ee: 0x08, 0x4ef: 0x08, - 0x4f0: 0x08, 0x4f1: 0x08, 0x4f2: 0x08, 0x4f3: 0x08, 0x4f4: 0x08, 0x4f5: 0x08, 0x4f6: 0x08, 0x4f7: 0x08, - 0x4f8: 0x08, 0x4f9: 0x08, 0x4fa: 0x08, 0x4fb: 0x08, 0x4fc: 0x08, 0x4fd: 0x08, 0x4fe: 0x08, 0x4ff: 0x65, - // Block 0x14, offset 0x500 - 0x520: 0x10, - 0x530: 0x09, 0x531: 0x09, 0x532: 0x09, 0x533: 0x09, 0x534: 0x09, 0x535: 0x09, 0x536: 0x09, 0x537: 0x09, - 0x538: 0x09, 0x539: 0x09, 0x53a: 0x09, 0x53b: 0x09, 0x53c: 0x09, 0x53d: 0x09, 0x53e: 0x09, 0x53f: 0x11, - // Block 0x15, offset 0x540 - 0x540: 0x09, 0x541: 0x09, 0x542: 0x09, 0x543: 0x09, 0x544: 0x09, 0x545: 0x09, 0x546: 0x09, 0x547: 0x09, - 0x548: 0x09, 0x549: 0x09, 0x54a: 0x09, 0x54b: 0x09, 0x54c: 0x09, 0x54d: 0x09, 0x54e: 0x09, 0x54f: 0x11, -} - -// inverseData contains 4-byte entries of the following format: -// -// <0 padding> -// -// The last byte of the UTF-8-encoded rune is xor-ed with the last byte of the -// UTF-8 encoding of the original rune. Mappings often have the following -// pattern: -// -// Ａ -> A (U+FF21 -> U+0041) -// Ｂ -> B (U+FF22 -> U+0042) -// ... -// -// By xor-ing the last byte the same entry can be shared by many mappings. This -// reduces the total number of distinct entries by about two thirds. -// The resulting entry for the aforementioned mappings is -// -// { 0x01, 0xE0, 0x00, 0x00 } -// -// Using this entry to map U+FF21 (UTF-8 [EF BC A1]), we get -// -// E0 ^ A1 = 41. -// -// Similarly, for U+FF22 (UTF-8 [EF BC A2]), we get -// -// E0 ^ A2 = 42. -// -// Note that because of the xor-ing, the byte sequence stored in the entry is -// not valid UTF-8. -var inverseData = [150][4]byte{ - {0x00, 0x00, 0x00, 0x00}, - {0x03, 0xe3, 0x80, 0xa0}, - {0x03, 0xef, 0xbc, 0xa0}, - {0x03, 0xef, 0xbc, 0xe0}, - {0x03, 0xef, 0xbd, 0xe0}, - {0x03, 0xef, 0xbf, 0x02}, - {0x03, 0xef, 0xbf, 0x00}, - {0x03, 0xef, 0xbf, 0x0e}, - {0x03, 0xef, 0xbf, 0x0c}, - {0x03, 0xef, 0xbf, 0x0f}, - {0x03, 0xef, 0xbf, 0x39}, - {0x03, 0xef, 0xbf, 0x3b}, - {0x03, 0xef, 0xbf, 0x3f}, - {0x03, 0xef, 0xbf, 0x2a}, - {0x03, 0xef, 0xbf, 0x0d}, - {0x03, 0xef, 0xbf, 0x25}, - {0x03, 0xef, 0xbd, 0x1a}, - {0x03, 0xef, 0xbd, 0x26}, - {0x01, 0xa0, 0x00, 0x00}, - {0x03, 0xef, 0xbd, 0x25}, - {0x03, 0xef, 0xbd, 0x23}, - {0x03, 0xef, 0xbd, 0x2e}, - {0x03, 0xef, 0xbe, 0x07}, - {0x03, 0xef, 0xbe, 0x05}, - {0x03, 0xef, 0xbd, 0x06}, - {0x03, 0xef, 0xbd, 0x13}, - {0x03, 0xef, 0xbd, 0x0b}, - {0x03, 0xef, 0xbd, 0x16}, - {0x03, 0xef, 0xbd, 0x0c}, - {0x03, 0xef, 0xbd, 0x15}, - {0x03, 0xef, 0xbd, 0x0d}, - {0x03, 0xef, 0xbd, 0x1c}, - {0x03, 0xef, 0xbd, 0x02}, - {0x03, 0xef, 0xbd, 0x1f}, - {0x03, 0xef, 0xbd, 0x1d}, - {0x03, 0xef, 0xbd, 0x17}, - {0x03, 0xef, 0xbd, 0x08}, - {0x03, 0xef, 0xbd, 0x09}, - {0x03, 0xef, 0xbd, 0x0e}, - {0x03, 0xef, 0xbd, 0x04}, - {0x03, 0xef, 0xbd, 0x05}, - {0x03, 0xef, 0xbe, 0x3f}, - {0x03, 0xef, 0xbe, 0x00}, - {0x03, 0xef, 0xbd, 0x2c}, - {0x03, 0xef, 0xbe, 0x06}, - {0x03, 0xef, 0xbe, 0x0c}, - {0x03, 0xef, 0xbe, 0x0f}, - {0x03, 0xef, 0xbe, 0x0d}, - {0x03, 0xef, 0xbe, 0x0b}, - {0x03, 0xef, 0xbe, 0x19}, - {0x03, 0xef, 0xbe, 0x15}, - {0x03, 0xef, 0xbe, 0x11}, - {0x03, 0xef, 0xbe, 0x31}, - {0x03, 0xef, 0xbe, 0x33}, - {0x03, 0xef, 0xbd, 0x0f}, - {0x03, 0xef, 0xbe, 0x30}, - {0x03, 0xef, 0xbe, 0x3e}, - {0x03, 0xef, 0xbe, 0x32}, - {0x03, 0xef, 0xbe, 0x36}, - {0x03, 0xef, 0xbd, 0x14}, - {0x03, 0xef, 0xbe, 0x2e}, - {0x03, 0xef, 0xbd, 0x1e}, - {0x03, 0xef, 0xbe, 0x10}, - {0x03, 0xef, 0xbf, 0x13}, - {0x03, 0xef, 0xbf, 0x15}, - {0x03, 0xef, 0xbf, 0x17}, - {0x03, 0xef, 0xbf, 0x1f}, - {0x03, 0xef, 0xbf, 0x1d}, - {0x03, 0xef, 0xbf, 0x1b}, - {0x03, 0xef, 0xbf, 0x09}, - {0x03, 0xef, 0xbf, 0x0b}, - {0x03, 0xef, 0xbf, 0x37}, - {0x03, 0xef, 0xbe, 0x04}, - {0x01, 0xe0, 0x00, 0x00}, - {0x03, 0xe2, 0xa6, 0x1a}, - {0x03, 0xe2, 0xa6, 0x26}, - {0x03, 0xe3, 0x80, 0x23}, - {0x03, 0xe3, 0x80, 0x2e}, - {0x03, 0xe3, 0x80, 0x25}, - {0x03, 0xe3, 0x83, 0x1e}, - {0x03, 0xe3, 0x83, 0x14}, - {0x03, 0xe3, 0x82, 0x06}, - {0x03, 0xe3, 0x82, 0x0b}, - {0x03, 0xe3, 0x82, 0x0c}, - {0x03, 0xe3, 0x82, 0x0d}, - {0x03, 0xe3, 0x82, 0x02}, - {0x03, 0xe3, 0x83, 0x0f}, - {0x03, 0xe3, 0x83, 0x08}, - {0x03, 0xe3, 0x83, 0x09}, - {0x03, 0xe3, 0x83, 0x2c}, - {0x03, 0xe3, 0x83, 0x0c}, - {0x03, 0xe3, 0x82, 0x13}, - {0x03, 0xe3, 0x82, 0x16}, - {0x03, 0xe3, 0x82, 0x15}, - {0x03, 0xe3, 0x82, 0x1c}, - {0x03, 0xe3, 0x82, 0x1f}, - {0x03, 0xe3, 0x82, 0x1d}, - {0x03, 0xe3, 0x82, 0x1a}, - {0x03, 0xe3, 0x82, 0x17}, - {0x03, 0xe3, 0x82, 0x08}, - {0x03, 0xe3, 0x82, 0x09}, - {0x03, 0xe3, 0x82, 0x0e}, - {0x03, 0xe3, 0x82, 0x04}, - {0x03, 0xe3, 0x82, 0x05}, - {0x03, 0xe3, 0x82, 0x3f}, - {0x03, 0xe3, 0x83, 0x00}, - {0x03, 0xe3, 0x83, 0x06}, - {0x03, 0xe3, 0x83, 0x05}, - {0x03, 0xe3, 0x83, 0x0d}, - {0x03, 0xe3, 0x83, 0x0b}, - {0x03, 0xe3, 0x83, 0x07}, - {0x03, 0xe3, 0x83, 0x19}, - {0x03, 0xe3, 0x83, 0x15}, - {0x03, 0xe3, 0x83, 0x11}, - {0x03, 0xe3, 0x83, 0x31}, - {0x03, 0xe3, 0x83, 0x33}, - {0x03, 0xe3, 0x83, 0x30}, - {0x03, 0xe3, 0x83, 0x3e}, - {0x03, 0xe3, 0x83, 0x32}, - {0x03, 0xe3, 0x83, 0x36}, - {0x03, 0xe3, 0x83, 0x2e}, - {0x03, 0xe3, 0x82, 0x07}, - {0x03, 0xe3, 0x85, 0x04}, - {0x03, 0xe3, 0x84, 0x10}, - {0x03, 0xe3, 0x85, 0x30}, - {0x03, 0xe3, 0x85, 0x0d}, - {0x03, 0xe3, 0x85, 0x13}, - {0x03, 0xe3, 0x85, 0x15}, - {0x03, 0xe3, 0x85, 0x17}, - {0x03, 0xe3, 0x85, 0x1f}, - {0x03, 0xe3, 0x85, 0x1d}, - {0x03, 0xe3, 0x85, 0x1b}, - {0x03, 0xe3, 0x85, 0x09}, - {0x03, 0xe3, 0x85, 0x0f}, - {0x03, 0xe3, 0x85, 0x0b}, - {0x03, 0xe3, 0x85, 0x37}, - {0x03, 0xe3, 0x85, 0x3b}, - {0x03, 0xe3, 0x85, 0x39}, - {0x03, 0xe3, 0x85, 0x3f}, - {0x02, 0xc2, 0x02, 0x00}, - {0x02, 0xc2, 0x0e, 0x00}, - {0x02, 0xc2, 0x0c, 0x00}, - {0x02, 0xc2, 0x00, 0x00}, - {0x03, 0xe2, 0x82, 0x0f}, - {0x03, 0xe2, 0x94, 0x2a}, - {0x03, 0xe2, 0x86, 0x39}, - {0x03, 0xe2, 0x86, 0x3b}, - {0x03, 0xe2, 0x86, 0x3f}, - {0x03, 0xe2, 0x96, 0x0d}, - {0x03, 0xe2, 0x97, 0x25}, -} - -// Total table size 15320 bytes (14KiB) diff --git a/vendor/golang.org/x/text/width/tables13.0.0.go b/vendor/golang.org/x/text/width/tables13.0.0.go deleted file mode 100644 index b1fcb522cb..0000000000 --- a/vendor/golang.org/x/text/width/tables13.0.0.go +++ /dev/null @@ -1,1362 +0,0 @@ -// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. - -//go:build go1.16 && !go1.21 -// +build go1.16,!go1.21 - -package width - -// UnicodeVersion is the Unicode version from which the tables in this package are derived. -const UnicodeVersion = "13.0.0" - -// lookup returns the trie value for the first UTF-8 encoding in s and -// the width in bytes of this encoding. The size will be 0 if s does not -// hold enough bytes to complete the encoding. len(s) must be greater than 0. -func (t *widthTrie) lookup(s []byte) (v uint16, sz int) { - c0 := s[0] - switch { - case c0 < 0x80: // is ASCII - return widthValues[c0], 1 - case c0 < 0xC2: - return 0, 1 // Illegal UTF-8: not a starter, not ASCII. - case c0 < 0xE0: // 2-byte UTF-8 - if len(s) < 2 { - return 0, 0 - } - i := widthIndex[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c1), 2 - case c0 < 0xF0: // 3-byte UTF-8 - if len(s) < 3 { - return 0, 0 - } - i := widthIndex[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - o := uint32(i)<<6 + uint32(c1) - i = widthIndex[o] - c2 := s[2] - if c2 < 0x80 || 0xC0 <= c2 { - return 0, 2 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c2), 3 - case c0 < 0xF8: // 4-byte UTF-8 - if len(s) < 4 { - return 0, 0 - } - i := widthIndex[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - o := uint32(i)<<6 + uint32(c1) - i = widthIndex[o] - c2 := s[2] - if c2 < 0x80 || 0xC0 <= c2 { - return 0, 2 // Illegal UTF-8: not a continuation byte. - } - o = uint32(i)<<6 + uint32(c2) - i = widthIndex[o] - c3 := s[3] - if c3 < 0x80 || 0xC0 <= c3 { - return 0, 3 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c3), 4 - } - // Illegal rune - return 0, 1 -} - -// lookupUnsafe returns the trie value for the first UTF-8 encoding in s. -// s must start with a full and valid UTF-8 encoded rune. -func (t *widthTrie) lookupUnsafe(s []byte) uint16 { - c0 := s[0] - if c0 < 0x80 { // is ASCII - return widthValues[c0] - } - i := widthIndex[c0] - if c0 < 0xE0 { // 2-byte UTF-8 - return t.lookupValue(uint32(i), s[1]) - } - i = widthIndex[uint32(i)<<6+uint32(s[1])] - if c0 < 0xF0 { // 3-byte UTF-8 - return t.lookupValue(uint32(i), s[2]) - } - i = widthIndex[uint32(i)<<6+uint32(s[2])] - if c0 < 0xF8 { // 4-byte UTF-8 - return t.lookupValue(uint32(i), s[3]) - } - return 0 -} - -// lookupString returns the trie value for the first UTF-8 encoding in s and -// the width in bytes of this encoding. The size will be 0 if s does not -// hold enough bytes to complete the encoding. len(s) must be greater than 0. -func (t *widthTrie) lookupString(s string) (v uint16, sz int) { - c0 := s[0] - switch { - case c0 < 0x80: // is ASCII - return widthValues[c0], 1 - case c0 < 0xC2: - return 0, 1 // Illegal UTF-8: not a starter, not ASCII. - case c0 < 0xE0: // 2-byte UTF-8 - if len(s) < 2 { - return 0, 0 - } - i := widthIndex[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c1), 2 - case c0 < 0xF0: // 3-byte UTF-8 - if len(s) < 3 { - return 0, 0 - } - i := widthIndex[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - o := uint32(i)<<6 + uint32(c1) - i = widthIndex[o] - c2 := s[2] - if c2 < 0x80 || 0xC0 <= c2 { - return 0, 2 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c2), 3 - case c0 < 0xF8: // 4-byte UTF-8 - if len(s) < 4 { - return 0, 0 - } - i := widthIndex[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - o := uint32(i)<<6 + uint32(c1) - i = widthIndex[o] - c2 := s[2] - if c2 < 0x80 || 0xC0 <= c2 { - return 0, 2 // Illegal UTF-8: not a continuation byte. - } - o = uint32(i)<<6 + uint32(c2) - i = widthIndex[o] - c3 := s[3] - if c3 < 0x80 || 0xC0 <= c3 { - return 0, 3 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c3), 4 - } - // Illegal rune - return 0, 1 -} - -// lookupStringUnsafe returns the trie value for the first UTF-8 encoding in s. -// s must start with a full and valid UTF-8 encoded rune. -func (t *widthTrie) lookupStringUnsafe(s string) uint16 { - c0 := s[0] - if c0 < 0x80 { // is ASCII - return widthValues[c0] - } - i := widthIndex[c0] - if c0 < 0xE0 { // 2-byte UTF-8 - return t.lookupValue(uint32(i), s[1]) - } - i = widthIndex[uint32(i)<<6+uint32(s[1])] - if c0 < 0xF0 { // 3-byte UTF-8 - return t.lookupValue(uint32(i), s[2]) - } - i = widthIndex[uint32(i)<<6+uint32(s[2])] - if c0 < 0xF8 { // 4-byte UTF-8 - return t.lookupValue(uint32(i), s[3]) - } - return 0 -} - -// widthTrie. Total size: 14848 bytes (14.50 KiB). Checksum: 17e24343536472f6. -type widthTrie struct{} - -func newWidthTrie(i int) *widthTrie { - return &widthTrie{} -} - -// lookupValue determines the type of block n and looks up the value for b. -func (t *widthTrie) lookupValue(n uint32, b byte) uint16 { - switch { - default: - return uint16(widthValues[n<<6+uint32(b)]) - } -} - -// widthValues: 105 blocks, 6720 entries, 13440 bytes -// The third block is the zero block. -var widthValues = [6720]uint16{ - // Block 0x0, offset 0x0 - 0x20: 0x6001, 0x21: 0x6002, 0x22: 0x6002, 0x23: 0x6002, - 0x24: 0x6002, 0x25: 0x6002, 0x26: 0x6002, 0x27: 0x6002, 0x28: 0x6002, 0x29: 0x6002, - 0x2a: 0x6002, 0x2b: 0x6002, 0x2c: 0x6002, 0x2d: 0x6002, 0x2e: 0x6002, 0x2f: 0x6002, - 0x30: 0x6002, 0x31: 0x6002, 0x32: 0x6002, 0x33: 0x6002, 0x34: 0x6002, 0x35: 0x6002, - 0x36: 0x6002, 0x37: 0x6002, 0x38: 0x6002, 0x39: 0x6002, 0x3a: 0x6002, 0x3b: 0x6002, - 0x3c: 0x6002, 0x3d: 0x6002, 0x3e: 0x6002, 0x3f: 0x6002, - // Block 0x1, offset 0x40 - 0x40: 0x6003, 0x41: 0x6003, 0x42: 0x6003, 0x43: 0x6003, 0x44: 0x6003, 0x45: 0x6003, - 0x46: 0x6003, 0x47: 0x6003, 0x48: 0x6003, 0x49: 0x6003, 0x4a: 0x6003, 0x4b: 0x6003, - 0x4c: 0x6003, 0x4d: 0x6003, 0x4e: 0x6003, 0x4f: 0x6003, 0x50: 0x6003, 0x51: 0x6003, - 0x52: 0x6003, 0x53: 0x6003, 0x54: 0x6003, 0x55: 0x6003, 0x56: 0x6003, 0x57: 0x6003, - 0x58: 0x6003, 0x59: 0x6003, 0x5a: 0x6003, 0x5b: 0x6003, 0x5c: 0x6003, 0x5d: 0x6003, - 0x5e: 0x6003, 0x5f: 0x6003, 0x60: 0x6004, 0x61: 0x6004, 0x62: 0x6004, 0x63: 0x6004, - 0x64: 0x6004, 0x65: 0x6004, 0x66: 0x6004, 0x67: 0x6004, 0x68: 0x6004, 0x69: 0x6004, - 0x6a: 0x6004, 0x6b: 0x6004, 0x6c: 0x6004, 0x6d: 0x6004, 0x6e: 0x6004, 0x6f: 0x6004, - 0x70: 0x6004, 0x71: 0x6004, 0x72: 0x6004, 0x73: 0x6004, 0x74: 0x6004, 0x75: 0x6004, - 0x76: 0x6004, 0x77: 0x6004, 0x78: 0x6004, 0x79: 0x6004, 0x7a: 0x6004, 0x7b: 0x6004, - 0x7c: 0x6004, 0x7d: 0x6004, 0x7e: 0x6004, - // Block 0x2, offset 0x80 - // Block 0x3, offset 0xc0 - 0xe1: 0x2000, 0xe2: 0x6005, 0xe3: 0x6005, - 0xe4: 0x2000, 0xe5: 0x6006, 0xe6: 0x6005, 0xe7: 0x2000, 0xe8: 0x2000, - 0xea: 0x2000, 0xec: 0x6007, 0xed: 0x2000, 0xee: 0x2000, 0xef: 0x6008, - 0xf0: 0x2000, 0xf1: 0x2000, 0xf2: 0x2000, 0xf3: 0x2000, 0xf4: 0x2000, - 0xf6: 0x2000, 0xf7: 0x2000, 0xf8: 0x2000, 0xf9: 0x2000, 0xfa: 0x2000, - 0xfc: 0x2000, 0xfd: 0x2000, 0xfe: 0x2000, 0xff: 0x2000, - // Block 0x4, offset 0x100 - 0x106: 0x2000, - 0x110: 0x2000, - 0x117: 0x2000, - 0x118: 0x2000, - 0x11e: 0x2000, 0x11f: 0x2000, 0x120: 0x2000, 0x121: 0x2000, - 0x126: 0x2000, 0x128: 0x2000, 0x129: 0x2000, - 0x12a: 0x2000, 0x12c: 0x2000, 0x12d: 0x2000, - 0x130: 0x2000, 0x132: 0x2000, 0x133: 0x2000, - 0x137: 0x2000, 0x138: 0x2000, 0x139: 0x2000, 0x13a: 0x2000, - 0x13c: 0x2000, 0x13e: 0x2000, - // Block 0x5, offset 0x140 - 0x141: 0x2000, - 0x151: 0x2000, - 0x153: 0x2000, - 0x15b: 0x2000, - 0x166: 0x2000, 0x167: 0x2000, - 0x16b: 0x2000, - 0x171: 0x2000, 0x172: 0x2000, 0x173: 0x2000, - 0x178: 0x2000, - 0x17f: 0x2000, - // Block 0x6, offset 0x180 - 0x180: 0x2000, 0x181: 0x2000, 0x182: 0x2000, 0x184: 0x2000, - 0x188: 0x2000, 0x189: 0x2000, 0x18a: 0x2000, 0x18b: 0x2000, - 0x18d: 0x2000, - 0x192: 0x2000, 0x193: 0x2000, - 0x1a6: 0x2000, 0x1a7: 0x2000, - 0x1ab: 0x2000, - // Block 0x7, offset 0x1c0 - 0x1ce: 0x2000, 0x1d0: 0x2000, - 0x1d2: 0x2000, 0x1d4: 0x2000, 0x1d6: 0x2000, - 0x1d8: 0x2000, 0x1da: 0x2000, 0x1dc: 0x2000, - // Block 0x8, offset 0x200 - 0x211: 0x2000, - 0x221: 0x2000, - // Block 0x9, offset 0x240 - 0x244: 0x2000, - 0x247: 0x2000, 0x249: 0x2000, 0x24a: 0x2000, 0x24b: 0x2000, - 0x24d: 0x2000, 0x250: 0x2000, - 0x258: 0x2000, 0x259: 0x2000, 0x25a: 0x2000, 0x25b: 0x2000, 0x25d: 0x2000, - 0x25f: 0x2000, - // Block 0xa, offset 0x280 - 0x280: 0x2000, 0x281: 0x2000, 0x282: 0x2000, 0x283: 0x2000, 0x284: 0x2000, 0x285: 0x2000, - 0x286: 0x2000, 0x287: 0x2000, 0x288: 0x2000, 0x289: 0x2000, 0x28a: 0x2000, 0x28b: 0x2000, - 0x28c: 0x2000, 0x28d: 0x2000, 0x28e: 0x2000, 0x28f: 0x2000, 0x290: 0x2000, 0x291: 0x2000, - 0x292: 0x2000, 0x293: 0x2000, 0x294: 0x2000, 0x295: 0x2000, 0x296: 0x2000, 0x297: 0x2000, - 0x298: 0x2000, 0x299: 0x2000, 0x29a: 0x2000, 0x29b: 0x2000, 0x29c: 0x2000, 0x29d: 0x2000, - 0x29e: 0x2000, 0x29f: 0x2000, 0x2a0: 0x2000, 0x2a1: 0x2000, 0x2a2: 0x2000, 0x2a3: 0x2000, - 0x2a4: 0x2000, 0x2a5: 0x2000, 0x2a6: 0x2000, 0x2a7: 0x2000, 0x2a8: 0x2000, 0x2a9: 0x2000, - 0x2aa: 0x2000, 0x2ab: 0x2000, 0x2ac: 0x2000, 0x2ad: 0x2000, 0x2ae: 0x2000, 0x2af: 0x2000, - 0x2b0: 0x2000, 0x2b1: 0x2000, 0x2b2: 0x2000, 0x2b3: 0x2000, 0x2b4: 0x2000, 0x2b5: 0x2000, - 0x2b6: 0x2000, 0x2b7: 0x2000, 0x2b8: 0x2000, 0x2b9: 0x2000, 0x2ba: 0x2000, 0x2bb: 0x2000, - 0x2bc: 0x2000, 0x2bd: 0x2000, 0x2be: 0x2000, 0x2bf: 0x2000, - // Block 0xb, offset 0x2c0 - 0x2c0: 0x2000, 0x2c1: 0x2000, 0x2c2: 0x2000, 0x2c3: 0x2000, 0x2c4: 0x2000, 0x2c5: 0x2000, - 0x2c6: 0x2000, 0x2c7: 0x2000, 0x2c8: 0x2000, 0x2c9: 0x2000, 0x2ca: 0x2000, 0x2cb: 0x2000, - 0x2cc: 0x2000, 0x2cd: 0x2000, 0x2ce: 0x2000, 0x2cf: 0x2000, 0x2d0: 0x2000, 0x2d1: 0x2000, - 0x2d2: 0x2000, 0x2d3: 0x2000, 0x2d4: 0x2000, 0x2d5: 0x2000, 0x2d6: 0x2000, 0x2d7: 0x2000, - 0x2d8: 0x2000, 0x2d9: 0x2000, 0x2da: 0x2000, 0x2db: 0x2000, 0x2dc: 0x2000, 0x2dd: 0x2000, - 0x2de: 0x2000, 0x2df: 0x2000, 0x2e0: 0x2000, 0x2e1: 0x2000, 0x2e2: 0x2000, 0x2e3: 0x2000, - 0x2e4: 0x2000, 0x2e5: 0x2000, 0x2e6: 0x2000, 0x2e7: 0x2000, 0x2e8: 0x2000, 0x2e9: 0x2000, - 0x2ea: 0x2000, 0x2eb: 0x2000, 0x2ec: 0x2000, 0x2ed: 0x2000, 0x2ee: 0x2000, 0x2ef: 0x2000, - // Block 0xc, offset 0x300 - 0x311: 0x2000, - 0x312: 0x2000, 0x313: 0x2000, 0x314: 0x2000, 0x315: 0x2000, 0x316: 0x2000, 0x317: 0x2000, - 0x318: 0x2000, 0x319: 0x2000, 0x31a: 0x2000, 0x31b: 0x2000, 0x31c: 0x2000, 0x31d: 0x2000, - 0x31e: 0x2000, 0x31f: 0x2000, 0x320: 0x2000, 0x321: 0x2000, 0x323: 0x2000, - 0x324: 0x2000, 0x325: 0x2000, 0x326: 0x2000, 0x327: 0x2000, 0x328: 0x2000, 0x329: 0x2000, - 0x331: 0x2000, 0x332: 0x2000, 0x333: 0x2000, 0x334: 0x2000, 0x335: 0x2000, - 0x336: 0x2000, 0x337: 0x2000, 0x338: 0x2000, 0x339: 0x2000, 0x33a: 0x2000, 0x33b: 0x2000, - 0x33c: 0x2000, 0x33d: 0x2000, 0x33e: 0x2000, 0x33f: 0x2000, - // Block 0xd, offset 0x340 - 0x340: 0x2000, 0x341: 0x2000, 0x343: 0x2000, 0x344: 0x2000, 0x345: 0x2000, - 0x346: 0x2000, 0x347: 0x2000, 0x348: 0x2000, 0x349: 0x2000, - // Block 0xe, offset 0x380 - 0x381: 0x2000, - 0x390: 0x2000, 0x391: 0x2000, - 0x392: 0x2000, 0x393: 0x2000, 0x394: 0x2000, 0x395: 0x2000, 0x396: 0x2000, 0x397: 0x2000, - 0x398: 0x2000, 0x399: 0x2000, 0x39a: 0x2000, 0x39b: 0x2000, 0x39c: 0x2000, 0x39d: 0x2000, - 0x39e: 0x2000, 0x39f: 0x2000, 0x3a0: 0x2000, 0x3a1: 0x2000, 0x3a2: 0x2000, 0x3a3: 0x2000, - 0x3a4: 0x2000, 0x3a5: 0x2000, 0x3a6: 0x2000, 0x3a7: 0x2000, 0x3a8: 0x2000, 0x3a9: 0x2000, - 0x3aa: 0x2000, 0x3ab: 0x2000, 0x3ac: 0x2000, 0x3ad: 0x2000, 0x3ae: 0x2000, 0x3af: 0x2000, - 0x3b0: 0x2000, 0x3b1: 0x2000, 0x3b2: 0x2000, 0x3b3: 0x2000, 0x3b4: 0x2000, 0x3b5: 0x2000, - 0x3b6: 0x2000, 0x3b7: 0x2000, 0x3b8: 0x2000, 0x3b9: 0x2000, 0x3ba: 0x2000, 0x3bb: 0x2000, - 0x3bc: 0x2000, 0x3bd: 0x2000, 0x3be: 0x2000, 0x3bf: 0x2000, - // Block 0xf, offset 0x3c0 - 0x3c0: 0x2000, 0x3c1: 0x2000, 0x3c2: 0x2000, 0x3c3: 0x2000, 0x3c4: 0x2000, 0x3c5: 0x2000, - 0x3c6: 0x2000, 0x3c7: 0x2000, 0x3c8: 0x2000, 0x3c9: 0x2000, 0x3ca: 0x2000, 0x3cb: 0x2000, - 0x3cc: 0x2000, 0x3cd: 0x2000, 0x3ce: 0x2000, 0x3cf: 0x2000, 0x3d1: 0x2000, - // Block 0x10, offset 0x400 - 0x400: 0x4000, 0x401: 0x4000, 0x402: 0x4000, 0x403: 0x4000, 0x404: 0x4000, 0x405: 0x4000, - 0x406: 0x4000, 0x407: 0x4000, 0x408: 0x4000, 0x409: 0x4000, 0x40a: 0x4000, 0x40b: 0x4000, - 0x40c: 0x4000, 0x40d: 0x4000, 0x40e: 0x4000, 0x40f: 0x4000, 0x410: 0x4000, 0x411: 0x4000, - 0x412: 0x4000, 0x413: 0x4000, 0x414: 0x4000, 0x415: 0x4000, 0x416: 0x4000, 0x417: 0x4000, - 0x418: 0x4000, 0x419: 0x4000, 0x41a: 0x4000, 0x41b: 0x4000, 0x41c: 0x4000, 0x41d: 0x4000, - 0x41e: 0x4000, 0x41f: 0x4000, 0x420: 0x4000, 0x421: 0x4000, 0x422: 0x4000, 0x423: 0x4000, - 0x424: 0x4000, 0x425: 0x4000, 0x426: 0x4000, 0x427: 0x4000, 0x428: 0x4000, 0x429: 0x4000, - 0x42a: 0x4000, 0x42b: 0x4000, 0x42c: 0x4000, 0x42d: 0x4000, 0x42e: 0x4000, 0x42f: 0x4000, - 0x430: 0x4000, 0x431: 0x4000, 0x432: 0x4000, 0x433: 0x4000, 0x434: 0x4000, 0x435: 0x4000, - 0x436: 0x4000, 0x437: 0x4000, 0x438: 0x4000, 0x439: 0x4000, 0x43a: 0x4000, 0x43b: 0x4000, - 0x43c: 0x4000, 0x43d: 0x4000, 0x43e: 0x4000, 0x43f: 0x4000, - // Block 0x11, offset 0x440 - 0x440: 0x4000, 0x441: 0x4000, 0x442: 0x4000, 0x443: 0x4000, 0x444: 0x4000, 0x445: 0x4000, - 0x446: 0x4000, 0x447: 0x4000, 0x448: 0x4000, 0x449: 0x4000, 0x44a: 0x4000, 0x44b: 0x4000, - 0x44c: 0x4000, 0x44d: 0x4000, 0x44e: 0x4000, 0x44f: 0x4000, 0x450: 0x4000, 0x451: 0x4000, - 0x452: 0x4000, 0x453: 0x4000, 0x454: 0x4000, 0x455: 0x4000, 0x456: 0x4000, 0x457: 0x4000, - 0x458: 0x4000, 0x459: 0x4000, 0x45a: 0x4000, 0x45b: 0x4000, 0x45c: 0x4000, 0x45d: 0x4000, - 0x45e: 0x4000, 0x45f: 0x4000, - // Block 0x12, offset 0x480 - 0x490: 0x2000, - 0x493: 0x2000, 0x494: 0x2000, 0x495: 0x2000, 0x496: 0x2000, - 0x498: 0x2000, 0x499: 0x2000, 0x49c: 0x2000, 0x49d: 0x2000, - 0x4a0: 0x2000, 0x4a1: 0x2000, 0x4a2: 0x2000, - 0x4a4: 0x2000, 0x4a5: 0x2000, 0x4a6: 0x2000, 0x4a7: 0x2000, - 0x4b0: 0x2000, 0x4b2: 0x2000, 0x4b3: 0x2000, 0x4b5: 0x2000, - 0x4bb: 0x2000, - 0x4be: 0x2000, - // Block 0x13, offset 0x4c0 - 0x4f4: 0x2000, - 0x4ff: 0x2000, - // Block 0x14, offset 0x500 - 0x501: 0x2000, 0x502: 0x2000, 0x503: 0x2000, 0x504: 0x2000, - 0x529: 0xa009, - 0x52c: 0x2000, - // Block 0x15, offset 0x540 - 0x543: 0x2000, 0x545: 0x2000, - 0x549: 0x2000, - 0x553: 0x2000, 0x556: 0x2000, - 0x561: 0x2000, 0x562: 0x2000, - 0x566: 0x2000, - 0x56b: 0x2000, - // Block 0x16, offset 0x580 - 0x593: 0x2000, 0x594: 0x2000, - 0x59b: 0x2000, 0x59c: 0x2000, 0x59d: 0x2000, - 0x59e: 0x2000, 0x5a0: 0x2000, 0x5a1: 0x2000, 0x5a2: 0x2000, 0x5a3: 0x2000, - 0x5a4: 0x2000, 0x5a5: 0x2000, 0x5a6: 0x2000, 0x5a7: 0x2000, 0x5a8: 0x2000, 0x5a9: 0x2000, - 0x5aa: 0x2000, 0x5ab: 0x2000, - 0x5b0: 0x2000, 0x5b1: 0x2000, 0x5b2: 0x2000, 0x5b3: 0x2000, 0x5b4: 0x2000, 0x5b5: 0x2000, - 0x5b6: 0x2000, 0x5b7: 0x2000, 0x5b8: 0x2000, 0x5b9: 0x2000, - // Block 0x17, offset 0x5c0 - 0x5c9: 0x2000, - 0x5d0: 0x200a, 0x5d1: 0x200b, - 0x5d2: 0x200a, 0x5d3: 0x200c, 0x5d4: 0x2000, 0x5d5: 0x2000, 0x5d6: 0x2000, 0x5d7: 0x2000, - 0x5d8: 0x2000, 0x5d9: 0x2000, - 0x5f8: 0x2000, 0x5f9: 0x2000, - // Block 0x18, offset 0x600 - 0x612: 0x2000, 0x614: 0x2000, - 0x627: 0x2000, - // Block 0x19, offset 0x640 - 0x640: 0x2000, 0x642: 0x2000, 0x643: 0x2000, - 0x647: 0x2000, 0x648: 0x2000, 0x64b: 0x2000, - 0x64f: 0x2000, 0x651: 0x2000, - 0x655: 0x2000, - 0x65a: 0x2000, 0x65d: 0x2000, - 0x65e: 0x2000, 0x65f: 0x2000, 0x660: 0x2000, 0x663: 0x2000, - 0x665: 0x2000, 0x667: 0x2000, 0x668: 0x2000, 0x669: 0x2000, - 0x66a: 0x2000, 0x66b: 0x2000, 0x66c: 0x2000, 0x66e: 0x2000, - 0x674: 0x2000, 0x675: 0x2000, - 0x676: 0x2000, 0x677: 0x2000, - 0x67c: 0x2000, 0x67d: 0x2000, - // Block 0x1a, offset 0x680 - 0x688: 0x2000, - 0x68c: 0x2000, - 0x692: 0x2000, - 0x6a0: 0x2000, 0x6a1: 0x2000, - 0x6a4: 0x2000, 0x6a5: 0x2000, 0x6a6: 0x2000, 0x6a7: 0x2000, - 0x6aa: 0x2000, 0x6ab: 0x2000, 0x6ae: 0x2000, 0x6af: 0x2000, - // Block 0x1b, offset 0x6c0 - 0x6c2: 0x2000, 0x6c3: 0x2000, - 0x6c6: 0x2000, 0x6c7: 0x2000, - 0x6d5: 0x2000, - 0x6d9: 0x2000, - 0x6e5: 0x2000, - 0x6ff: 0x2000, - // Block 0x1c, offset 0x700 - 0x712: 0x2000, - 0x71a: 0x4000, 0x71b: 0x4000, - 0x729: 0x4000, - 0x72a: 0x4000, - // Block 0x1d, offset 0x740 - 0x769: 0x4000, - 0x76a: 0x4000, 0x76b: 0x4000, 0x76c: 0x4000, - 0x770: 0x4000, 0x773: 0x4000, - // Block 0x1e, offset 0x780 - 0x7a0: 0x2000, 0x7a1: 0x2000, 0x7a2: 0x2000, 0x7a3: 0x2000, - 0x7a4: 0x2000, 0x7a5: 0x2000, 0x7a6: 0x2000, 0x7a7: 0x2000, 0x7a8: 0x2000, 0x7a9: 0x2000, - 0x7aa: 0x2000, 0x7ab: 0x2000, 0x7ac: 0x2000, 0x7ad: 0x2000, 0x7ae: 0x2000, 0x7af: 0x2000, - 0x7b0: 0x2000, 0x7b1: 0x2000, 0x7b2: 0x2000, 0x7b3: 0x2000, 0x7b4: 0x2000, 0x7b5: 0x2000, - 0x7b6: 0x2000, 0x7b7: 0x2000, 0x7b8: 0x2000, 0x7b9: 0x2000, 0x7ba: 0x2000, 0x7bb: 0x2000, - 0x7bc: 0x2000, 0x7bd: 0x2000, 0x7be: 0x2000, 0x7bf: 0x2000, - // Block 0x1f, offset 0x7c0 - 0x7c0: 0x2000, 0x7c1: 0x2000, 0x7c2: 0x2000, 0x7c3: 0x2000, 0x7c4: 0x2000, 0x7c5: 0x2000, - 0x7c6: 0x2000, 0x7c7: 0x2000, 0x7c8: 0x2000, 0x7c9: 0x2000, 0x7ca: 0x2000, 0x7cb: 0x2000, - 0x7cc: 0x2000, 0x7cd: 0x2000, 0x7ce: 0x2000, 0x7cf: 0x2000, 0x7d0: 0x2000, 0x7d1: 0x2000, - 0x7d2: 0x2000, 0x7d3: 0x2000, 0x7d4: 0x2000, 0x7d5: 0x2000, 0x7d6: 0x2000, 0x7d7: 0x2000, - 0x7d8: 0x2000, 0x7d9: 0x2000, 0x7da: 0x2000, 0x7db: 0x2000, 0x7dc: 0x2000, 0x7dd: 0x2000, - 0x7de: 0x2000, 0x7df: 0x2000, 0x7e0: 0x2000, 0x7e1: 0x2000, 0x7e2: 0x2000, 0x7e3: 0x2000, - 0x7e4: 0x2000, 0x7e5: 0x2000, 0x7e6: 0x2000, 0x7e7: 0x2000, 0x7e8: 0x2000, 0x7e9: 0x2000, - 0x7eb: 0x2000, 0x7ec: 0x2000, 0x7ed: 0x2000, 0x7ee: 0x2000, 0x7ef: 0x2000, - 0x7f0: 0x2000, 0x7f1: 0x2000, 0x7f2: 0x2000, 0x7f3: 0x2000, 0x7f4: 0x2000, 0x7f5: 0x2000, - 0x7f6: 0x2000, 0x7f7: 0x2000, 0x7f8: 0x2000, 0x7f9: 0x2000, 0x7fa: 0x2000, 0x7fb: 0x2000, - 0x7fc: 0x2000, 0x7fd: 0x2000, 0x7fe: 0x2000, 0x7ff: 0x2000, - // Block 0x20, offset 0x800 - 0x800: 0x2000, 0x801: 0x2000, 0x802: 0x200d, 0x803: 0x2000, 0x804: 0x2000, 0x805: 0x2000, - 0x806: 0x2000, 0x807: 0x2000, 0x808: 0x2000, 0x809: 0x2000, 0x80a: 0x2000, 0x80b: 0x2000, - 0x80c: 0x2000, 0x80d: 0x2000, 0x80e: 0x2000, 0x80f: 0x2000, 0x810: 0x2000, 0x811: 0x2000, - 0x812: 0x2000, 0x813: 0x2000, 0x814: 0x2000, 0x815: 0x2000, 0x816: 0x2000, 0x817: 0x2000, - 0x818: 0x2000, 0x819: 0x2000, 0x81a: 0x2000, 0x81b: 0x2000, 0x81c: 0x2000, 0x81d: 0x2000, - 0x81e: 0x2000, 0x81f: 0x2000, 0x820: 0x2000, 0x821: 0x2000, 0x822: 0x2000, 0x823: 0x2000, - 0x824: 0x2000, 0x825: 0x2000, 0x826: 0x2000, 0x827: 0x2000, 0x828: 0x2000, 0x829: 0x2000, - 0x82a: 0x2000, 0x82b: 0x2000, 0x82c: 0x2000, 0x82d: 0x2000, 0x82e: 0x2000, 0x82f: 0x2000, - 0x830: 0x2000, 0x831: 0x2000, 0x832: 0x2000, 0x833: 0x2000, 0x834: 0x2000, 0x835: 0x2000, - 0x836: 0x2000, 0x837: 0x2000, 0x838: 0x2000, 0x839: 0x2000, 0x83a: 0x2000, 0x83b: 0x2000, - 0x83c: 0x2000, 0x83d: 0x2000, 0x83e: 0x2000, 0x83f: 0x2000, - // Block 0x21, offset 0x840 - 0x840: 0x2000, 0x841: 0x2000, 0x842: 0x2000, 0x843: 0x2000, 0x844: 0x2000, 0x845: 0x2000, - 0x846: 0x2000, 0x847: 0x2000, 0x848: 0x2000, 0x849: 0x2000, 0x84a: 0x2000, 0x84b: 0x2000, - 0x850: 0x2000, 0x851: 0x2000, - 0x852: 0x2000, 0x853: 0x2000, 0x854: 0x2000, 0x855: 0x2000, 0x856: 0x2000, 0x857: 0x2000, - 0x858: 0x2000, 0x859: 0x2000, 0x85a: 0x2000, 0x85b: 0x2000, 0x85c: 0x2000, 0x85d: 0x2000, - 0x85e: 0x2000, 0x85f: 0x2000, 0x860: 0x2000, 0x861: 0x2000, 0x862: 0x2000, 0x863: 0x2000, - 0x864: 0x2000, 0x865: 0x2000, 0x866: 0x2000, 0x867: 0x2000, 0x868: 0x2000, 0x869: 0x2000, - 0x86a: 0x2000, 0x86b: 0x2000, 0x86c: 0x2000, 0x86d: 0x2000, 0x86e: 0x2000, 0x86f: 0x2000, - 0x870: 0x2000, 0x871: 0x2000, 0x872: 0x2000, 0x873: 0x2000, - // Block 0x22, offset 0x880 - 0x880: 0x2000, 0x881: 0x2000, 0x882: 0x2000, 0x883: 0x2000, 0x884: 0x2000, 0x885: 0x2000, - 0x886: 0x2000, 0x887: 0x2000, 0x888: 0x2000, 0x889: 0x2000, 0x88a: 0x2000, 0x88b: 0x2000, - 0x88c: 0x2000, 0x88d: 0x2000, 0x88e: 0x2000, 0x88f: 0x2000, - 0x892: 0x2000, 0x893: 0x2000, 0x894: 0x2000, 0x895: 0x2000, - 0x8a0: 0x200e, 0x8a1: 0x2000, 0x8a3: 0x2000, - 0x8a4: 0x2000, 0x8a5: 0x2000, 0x8a6: 0x2000, 0x8a7: 0x2000, 0x8a8: 0x2000, 0x8a9: 0x2000, - 0x8b2: 0x2000, 0x8b3: 0x2000, - 0x8b6: 0x2000, 0x8b7: 0x2000, - 0x8bc: 0x2000, 0x8bd: 0x2000, - // Block 0x23, offset 0x8c0 - 0x8c0: 0x2000, 0x8c1: 0x2000, - 0x8c6: 0x2000, 0x8c7: 0x2000, 0x8c8: 0x2000, 0x8cb: 0x200f, - 0x8ce: 0x2000, 0x8cf: 0x2000, 0x8d0: 0x2000, 0x8d1: 0x2000, - 0x8e2: 0x2000, 0x8e3: 0x2000, - 0x8e4: 0x2000, 0x8e5: 0x2000, - 0x8ef: 0x2000, - 0x8fd: 0x4000, 0x8fe: 0x4000, - // Block 0x24, offset 0x900 - 0x905: 0x2000, - 0x906: 0x2000, 0x909: 0x2000, - 0x90e: 0x2000, 0x90f: 0x2000, - 0x914: 0x4000, 0x915: 0x4000, - 0x91c: 0x2000, - 0x91e: 0x2000, - // Block 0x25, offset 0x940 - 0x940: 0x2000, 0x942: 0x2000, - 0x948: 0x4000, 0x949: 0x4000, 0x94a: 0x4000, 0x94b: 0x4000, - 0x94c: 0x4000, 0x94d: 0x4000, 0x94e: 0x4000, 0x94f: 0x4000, 0x950: 0x4000, 0x951: 0x4000, - 0x952: 0x4000, 0x953: 0x4000, - 0x960: 0x2000, 0x961: 0x2000, 0x963: 0x2000, - 0x964: 0x2000, 0x965: 0x2000, 0x967: 0x2000, 0x968: 0x2000, 0x969: 0x2000, - 0x96a: 0x2000, 0x96c: 0x2000, 0x96d: 0x2000, 0x96f: 0x2000, - 0x97f: 0x4000, - // Block 0x26, offset 0x980 - 0x993: 0x4000, - 0x99e: 0x2000, 0x99f: 0x2000, 0x9a1: 0x4000, - 0x9aa: 0x4000, 0x9ab: 0x4000, - 0x9bd: 0x4000, 0x9be: 0x4000, 0x9bf: 0x2000, - // Block 0x27, offset 0x9c0 - 0x9c4: 0x4000, 0x9c5: 0x4000, - 0x9c6: 0x2000, 0x9c7: 0x2000, 0x9c8: 0x2000, 0x9c9: 0x2000, 0x9ca: 0x2000, 0x9cb: 0x2000, - 0x9cc: 0x2000, 0x9cd: 0x2000, 0x9ce: 0x4000, 0x9cf: 0x2000, 0x9d0: 0x2000, 0x9d1: 0x2000, - 0x9d2: 0x2000, 0x9d3: 0x2000, 0x9d4: 0x4000, 0x9d5: 0x2000, 0x9d6: 0x2000, 0x9d7: 0x2000, - 0x9d8: 0x2000, 0x9d9: 0x2000, 0x9da: 0x2000, 0x9db: 0x2000, 0x9dc: 0x2000, 0x9dd: 0x2000, - 0x9de: 0x2000, 0x9df: 0x2000, 0x9e0: 0x2000, 0x9e1: 0x2000, 0x9e3: 0x2000, - 0x9e8: 0x2000, 0x9e9: 0x2000, - 0x9ea: 0x4000, 0x9eb: 0x2000, 0x9ec: 0x2000, 0x9ed: 0x2000, 0x9ee: 0x2000, 0x9ef: 0x2000, - 0x9f0: 0x2000, 0x9f1: 0x2000, 0x9f2: 0x4000, 0x9f3: 0x4000, 0x9f4: 0x2000, 0x9f5: 0x4000, - 0x9f6: 0x2000, 0x9f7: 0x2000, 0x9f8: 0x2000, 0x9f9: 0x2000, 0x9fa: 0x4000, 0x9fb: 0x2000, - 0x9fc: 0x2000, 0x9fd: 0x4000, 0x9fe: 0x2000, 0x9ff: 0x2000, - // Block 0x28, offset 0xa00 - 0xa05: 0x4000, - 0xa0a: 0x4000, 0xa0b: 0x4000, - 0xa28: 0x4000, - 0xa3d: 0x2000, - // Block 0x29, offset 0xa40 - 0xa4c: 0x4000, 0xa4e: 0x4000, - 0xa53: 0x4000, 0xa54: 0x4000, 0xa55: 0x4000, 0xa57: 0x4000, - 0xa76: 0x2000, 0xa77: 0x2000, 0xa78: 0x2000, 0xa79: 0x2000, 0xa7a: 0x2000, 0xa7b: 0x2000, - 0xa7c: 0x2000, 0xa7d: 0x2000, 0xa7e: 0x2000, 0xa7f: 0x2000, - // Block 0x2a, offset 0xa80 - 0xa95: 0x4000, 0xa96: 0x4000, 0xa97: 0x4000, - 0xab0: 0x4000, - 0xabf: 0x4000, - // Block 0x2b, offset 0xac0 - 0xae6: 0x6000, 0xae7: 0x6000, 0xae8: 0x6000, 0xae9: 0x6000, - 0xaea: 0x6000, 0xaeb: 0x6000, 0xaec: 0x6000, 0xaed: 0x6000, - // Block 0x2c, offset 0xb00 - 0xb05: 0x6010, - 0xb06: 0x6011, - // Block 0x2d, offset 0xb40 - 0xb5b: 0x4000, 0xb5c: 0x4000, - // Block 0x2e, offset 0xb80 - 0xb90: 0x4000, - 0xb95: 0x4000, 0xb96: 0x2000, 0xb97: 0x2000, - 0xb98: 0x2000, 0xb99: 0x2000, - // Block 0x2f, offset 0xbc0 - 0xbc0: 0x4000, 0xbc1: 0x4000, 0xbc2: 0x4000, 0xbc3: 0x4000, 0xbc4: 0x4000, 0xbc5: 0x4000, - 0xbc6: 0x4000, 0xbc7: 0x4000, 0xbc8: 0x4000, 0xbc9: 0x4000, 0xbca: 0x4000, 0xbcb: 0x4000, - 0xbcc: 0x4000, 0xbcd: 0x4000, 0xbce: 0x4000, 0xbcf: 0x4000, 0xbd0: 0x4000, 0xbd1: 0x4000, - 0xbd2: 0x4000, 0xbd3: 0x4000, 0xbd4: 0x4000, 0xbd5: 0x4000, 0xbd6: 0x4000, 0xbd7: 0x4000, - 0xbd8: 0x4000, 0xbd9: 0x4000, 0xbdb: 0x4000, 0xbdc: 0x4000, 0xbdd: 0x4000, - 0xbde: 0x4000, 0xbdf: 0x4000, 0xbe0: 0x4000, 0xbe1: 0x4000, 0xbe2: 0x4000, 0xbe3: 0x4000, - 0xbe4: 0x4000, 0xbe5: 0x4000, 0xbe6: 0x4000, 0xbe7: 0x4000, 0xbe8: 0x4000, 0xbe9: 0x4000, - 0xbea: 0x4000, 0xbeb: 0x4000, 0xbec: 0x4000, 0xbed: 0x4000, 0xbee: 0x4000, 0xbef: 0x4000, - 0xbf0: 0x4000, 0xbf1: 0x4000, 0xbf2: 0x4000, 0xbf3: 0x4000, 0xbf4: 0x4000, 0xbf5: 0x4000, - 0xbf6: 0x4000, 0xbf7: 0x4000, 0xbf8: 0x4000, 0xbf9: 0x4000, 0xbfa: 0x4000, 0xbfb: 0x4000, - 0xbfc: 0x4000, 0xbfd: 0x4000, 0xbfe: 0x4000, 0xbff: 0x4000, - // Block 0x30, offset 0xc00 - 0xc00: 0x4000, 0xc01: 0x4000, 0xc02: 0x4000, 0xc03: 0x4000, 0xc04: 0x4000, 0xc05: 0x4000, - 0xc06: 0x4000, 0xc07: 0x4000, 0xc08: 0x4000, 0xc09: 0x4000, 0xc0a: 0x4000, 0xc0b: 0x4000, - 0xc0c: 0x4000, 0xc0d: 0x4000, 0xc0e: 0x4000, 0xc0f: 0x4000, 0xc10: 0x4000, 0xc11: 0x4000, - 0xc12: 0x4000, 0xc13: 0x4000, 0xc14: 0x4000, 0xc15: 0x4000, 0xc16: 0x4000, 0xc17: 0x4000, - 0xc18: 0x4000, 0xc19: 0x4000, 0xc1a: 0x4000, 0xc1b: 0x4000, 0xc1c: 0x4000, 0xc1d: 0x4000, - 0xc1e: 0x4000, 0xc1f: 0x4000, 0xc20: 0x4000, 0xc21: 0x4000, 0xc22: 0x4000, 0xc23: 0x4000, - 0xc24: 0x4000, 0xc25: 0x4000, 0xc26: 0x4000, 0xc27: 0x4000, 0xc28: 0x4000, 0xc29: 0x4000, - 0xc2a: 0x4000, 0xc2b: 0x4000, 0xc2c: 0x4000, 0xc2d: 0x4000, 0xc2e: 0x4000, 0xc2f: 0x4000, - 0xc30: 0x4000, 0xc31: 0x4000, 0xc32: 0x4000, 0xc33: 0x4000, - // Block 0x31, offset 0xc40 - 0xc40: 0x4000, 0xc41: 0x4000, 0xc42: 0x4000, 0xc43: 0x4000, 0xc44: 0x4000, 0xc45: 0x4000, - 0xc46: 0x4000, 0xc47: 0x4000, 0xc48: 0x4000, 0xc49: 0x4000, 0xc4a: 0x4000, 0xc4b: 0x4000, - 0xc4c: 0x4000, 0xc4d: 0x4000, 0xc4e: 0x4000, 0xc4f: 0x4000, 0xc50: 0x4000, 0xc51: 0x4000, - 0xc52: 0x4000, 0xc53: 0x4000, 0xc54: 0x4000, 0xc55: 0x4000, - 0xc70: 0x4000, 0xc71: 0x4000, 0xc72: 0x4000, 0xc73: 0x4000, 0xc74: 0x4000, 0xc75: 0x4000, - 0xc76: 0x4000, 0xc77: 0x4000, 0xc78: 0x4000, 0xc79: 0x4000, 0xc7a: 0x4000, 0xc7b: 0x4000, - // Block 0x32, offset 0xc80 - 0xc80: 0x9012, 0xc81: 0x4013, 0xc82: 0x4014, 0xc83: 0x4000, 0xc84: 0x4000, 0xc85: 0x4000, - 0xc86: 0x4000, 0xc87: 0x4000, 0xc88: 0x4000, 0xc89: 0x4000, 0xc8a: 0x4000, 0xc8b: 0x4000, - 0xc8c: 0x4015, 0xc8d: 0x4015, 0xc8e: 0x4000, 0xc8f: 0x4000, 0xc90: 0x4000, 0xc91: 0x4000, - 0xc92: 0x4000, 0xc93: 0x4000, 0xc94: 0x4000, 0xc95: 0x4000, 0xc96: 0x4000, 0xc97: 0x4000, - 0xc98: 0x4000, 0xc99: 0x4000, 0xc9a: 0x4000, 0xc9b: 0x4000, 0xc9c: 0x4000, 0xc9d: 0x4000, - 0xc9e: 0x4000, 0xc9f: 0x4000, 0xca0: 0x4000, 0xca1: 0x4000, 0xca2: 0x4000, 0xca3: 0x4000, - 0xca4: 0x4000, 0xca5: 0x4000, 0xca6: 0x4000, 0xca7: 0x4000, 0xca8: 0x4000, 0xca9: 0x4000, - 0xcaa: 0x4000, 0xcab: 0x4000, 0xcac: 0x4000, 0xcad: 0x4000, 0xcae: 0x4000, 0xcaf: 0x4000, - 0xcb0: 0x4000, 0xcb1: 0x4000, 0xcb2: 0x4000, 0xcb3: 0x4000, 0xcb4: 0x4000, 0xcb5: 0x4000, - 0xcb6: 0x4000, 0xcb7: 0x4000, 0xcb8: 0x4000, 0xcb9: 0x4000, 0xcba: 0x4000, 0xcbb: 0x4000, - 0xcbc: 0x4000, 0xcbd: 0x4000, 0xcbe: 0x4000, - // Block 0x33, offset 0xcc0 - 0xcc1: 0x4000, 0xcc2: 0x4000, 0xcc3: 0x4000, 0xcc4: 0x4000, 0xcc5: 0x4000, - 0xcc6: 0x4000, 0xcc7: 0x4000, 0xcc8: 0x4000, 0xcc9: 0x4000, 0xcca: 0x4000, 0xccb: 0x4000, - 0xccc: 0x4000, 0xccd: 0x4000, 0xcce: 0x4000, 0xccf: 0x4000, 0xcd0: 0x4000, 0xcd1: 0x4000, - 0xcd2: 0x4000, 0xcd3: 0x4000, 0xcd4: 0x4000, 0xcd5: 0x4000, 0xcd6: 0x4000, 0xcd7: 0x4000, - 0xcd8: 0x4000, 0xcd9: 0x4000, 0xcda: 0x4000, 0xcdb: 0x4000, 0xcdc: 0x4000, 0xcdd: 0x4000, - 0xcde: 0x4000, 0xcdf: 0x4000, 0xce0: 0x4000, 0xce1: 0x4000, 0xce2: 0x4000, 0xce3: 0x4000, - 0xce4: 0x4000, 0xce5: 0x4000, 0xce6: 0x4000, 0xce7: 0x4000, 0xce8: 0x4000, 0xce9: 0x4000, - 0xcea: 0x4000, 0xceb: 0x4000, 0xcec: 0x4000, 0xced: 0x4000, 0xcee: 0x4000, 0xcef: 0x4000, - 0xcf0: 0x4000, 0xcf1: 0x4000, 0xcf2: 0x4000, 0xcf3: 0x4000, 0xcf4: 0x4000, 0xcf5: 0x4000, - 0xcf6: 0x4000, 0xcf7: 0x4000, 0xcf8: 0x4000, 0xcf9: 0x4000, 0xcfa: 0x4000, 0xcfb: 0x4000, - 0xcfc: 0x4000, 0xcfd: 0x4000, 0xcfe: 0x4000, 0xcff: 0x4000, - // Block 0x34, offset 0xd00 - 0xd00: 0x4000, 0xd01: 0x4000, 0xd02: 0x4000, 0xd03: 0x4000, 0xd04: 0x4000, 0xd05: 0x4000, - 0xd06: 0x4000, 0xd07: 0x4000, 0xd08: 0x4000, 0xd09: 0x4000, 0xd0a: 0x4000, 0xd0b: 0x4000, - 0xd0c: 0x4000, 0xd0d: 0x4000, 0xd0e: 0x4000, 0xd0f: 0x4000, 0xd10: 0x4000, 0xd11: 0x4000, - 0xd12: 0x4000, 0xd13: 0x4000, 0xd14: 0x4000, 0xd15: 0x4000, 0xd16: 0x4000, - 0xd19: 0x4016, 0xd1a: 0x4017, 0xd1b: 0x4000, 0xd1c: 0x4000, 0xd1d: 0x4000, - 0xd1e: 0x4000, 0xd1f: 0x4000, 0xd20: 0x4000, 0xd21: 0x4018, 0xd22: 0x4019, 0xd23: 0x401a, - 0xd24: 0x401b, 0xd25: 0x401c, 0xd26: 0x401d, 0xd27: 0x401e, 0xd28: 0x401f, 0xd29: 0x4020, - 0xd2a: 0x4021, 0xd2b: 0x4022, 0xd2c: 0x4000, 0xd2d: 0x4010, 0xd2e: 0x4000, 0xd2f: 0x4023, - 0xd30: 0x4000, 0xd31: 0x4024, 0xd32: 0x4000, 0xd33: 0x4025, 0xd34: 0x4000, 0xd35: 0x4026, - 0xd36: 0x4000, 0xd37: 0x401a, 0xd38: 0x4000, 0xd39: 0x4027, 0xd3a: 0x4000, 0xd3b: 0x4028, - 0xd3c: 0x4000, 0xd3d: 0x4020, 0xd3e: 0x4000, 0xd3f: 0x4029, - // Block 0x35, offset 0xd40 - 0xd40: 0x4000, 0xd41: 0x402a, 0xd42: 0x4000, 0xd43: 0x402b, 0xd44: 0x402c, 0xd45: 0x4000, - 0xd46: 0x4017, 0xd47: 0x4000, 0xd48: 0x402d, 0xd49: 0x4000, 0xd4a: 0x402e, 0xd4b: 0x402f, - 0xd4c: 0x4030, 0xd4d: 0x4017, 0xd4e: 0x4016, 0xd4f: 0x4017, 0xd50: 0x4000, 0xd51: 0x4000, - 0xd52: 0x4031, 0xd53: 0x4000, 0xd54: 0x4000, 0xd55: 0x4031, 0xd56: 0x4000, 0xd57: 0x4000, - 0xd58: 0x4032, 0xd59: 0x4000, 0xd5a: 0x4000, 0xd5b: 0x4032, 0xd5c: 0x4000, 0xd5d: 0x4000, - 0xd5e: 0x4033, 0xd5f: 0x402e, 0xd60: 0x4034, 0xd61: 0x4035, 0xd62: 0x4034, 0xd63: 0x4036, - 0xd64: 0x4037, 0xd65: 0x4024, 0xd66: 0x4035, 0xd67: 0x4025, 0xd68: 0x4038, 0xd69: 0x4038, - 0xd6a: 0x4039, 0xd6b: 0x4039, 0xd6c: 0x403a, 0xd6d: 0x403a, 0xd6e: 0x4000, 0xd6f: 0x4035, - 0xd70: 0x4000, 0xd71: 0x4000, 0xd72: 0x403b, 0xd73: 0x403c, 0xd74: 0x4000, 0xd75: 0x4000, - 0xd76: 0x4000, 0xd77: 0x4000, 0xd78: 0x4000, 0xd79: 0x4000, 0xd7a: 0x4000, 0xd7b: 0x403d, - 0xd7c: 0x401c, 0xd7d: 0x4000, 0xd7e: 0x4000, 0xd7f: 0x4000, - // Block 0x36, offset 0xd80 - 0xd85: 0x4000, - 0xd86: 0x4000, 0xd87: 0x4000, 0xd88: 0x4000, 0xd89: 0x4000, 0xd8a: 0x4000, 0xd8b: 0x4000, - 0xd8c: 0x4000, 0xd8d: 0x4000, 0xd8e: 0x4000, 0xd8f: 0x4000, 0xd90: 0x4000, 0xd91: 0x4000, - 0xd92: 0x4000, 0xd93: 0x4000, 0xd94: 0x4000, 0xd95: 0x4000, 0xd96: 0x4000, 0xd97: 0x4000, - 0xd98: 0x4000, 0xd99: 0x4000, 0xd9a: 0x4000, 0xd9b: 0x4000, 0xd9c: 0x4000, 0xd9d: 0x4000, - 0xd9e: 0x4000, 0xd9f: 0x4000, 0xda0: 0x4000, 0xda1: 0x4000, 0xda2: 0x4000, 0xda3: 0x4000, - 0xda4: 0x4000, 0xda5: 0x4000, 0xda6: 0x4000, 0xda7: 0x4000, 0xda8: 0x4000, 0xda9: 0x4000, - 0xdaa: 0x4000, 0xdab: 0x4000, 0xdac: 0x4000, 0xdad: 0x4000, 0xdae: 0x4000, 0xdaf: 0x4000, - 0xdb1: 0x403e, 0xdb2: 0x403e, 0xdb3: 0x403e, 0xdb4: 0x403e, 0xdb5: 0x403e, - 0xdb6: 0x403e, 0xdb7: 0x403e, 0xdb8: 0x403e, 0xdb9: 0x403e, 0xdba: 0x403e, 0xdbb: 0x403e, - 0xdbc: 0x403e, 0xdbd: 0x403e, 0xdbe: 0x403e, 0xdbf: 0x403e, - // Block 0x37, offset 0xdc0 - 0xdc0: 0x4037, 0xdc1: 0x4037, 0xdc2: 0x4037, 0xdc3: 0x4037, 0xdc4: 0x4037, 0xdc5: 0x4037, - 0xdc6: 0x4037, 0xdc7: 0x4037, 0xdc8: 0x4037, 0xdc9: 0x4037, 0xdca: 0x4037, 0xdcb: 0x4037, - 0xdcc: 0x4037, 0xdcd: 0x4037, 0xdce: 0x4037, 0xdcf: 0x400e, 0xdd0: 0x403f, 0xdd1: 0x4040, - 0xdd2: 0x4041, 0xdd3: 0x4040, 0xdd4: 0x403f, 0xdd5: 0x4042, 0xdd6: 0x4043, 0xdd7: 0x4044, - 0xdd8: 0x4040, 0xdd9: 0x4041, 0xdda: 0x4040, 0xddb: 0x4045, 0xddc: 0x4009, 0xddd: 0x4045, - 0xdde: 0x4046, 0xddf: 0x4045, 0xde0: 0x4047, 0xde1: 0x400b, 0xde2: 0x400a, 0xde3: 0x400c, - 0xde4: 0x4048, 0xde5: 0x4000, 0xde6: 0x4000, 0xde7: 0x4000, 0xde8: 0x4000, 0xde9: 0x4000, - 0xdea: 0x4000, 0xdeb: 0x4000, 0xdec: 0x4000, 0xded: 0x4000, 0xdee: 0x4000, 0xdef: 0x4000, - 0xdf0: 0x4000, 0xdf1: 0x4000, 0xdf2: 0x4000, 0xdf3: 0x4000, 0xdf4: 0x4000, 0xdf5: 0x4000, - 0xdf6: 0x4000, 0xdf7: 0x4000, 0xdf8: 0x4000, 0xdf9: 0x4000, 0xdfa: 0x4000, 0xdfb: 0x4000, - 0xdfc: 0x4000, 0xdfd: 0x4000, 0xdfe: 0x4000, 0xdff: 0x4000, - // Block 0x38, offset 0xe00 - 0xe00: 0x4000, 0xe01: 0x4000, 0xe02: 0x4000, 0xe03: 0x4000, 0xe04: 0x4000, 0xe05: 0x4000, - 0xe06: 0x4000, 0xe07: 0x4000, 0xe08: 0x4000, 0xe09: 0x4000, 0xe0a: 0x4000, 0xe0b: 0x4000, - 0xe0c: 0x4000, 0xe0d: 0x4000, 0xe0e: 0x4000, 0xe10: 0x4000, 0xe11: 0x4000, - 0xe12: 0x4000, 0xe13: 0x4000, 0xe14: 0x4000, 0xe15: 0x4000, 0xe16: 0x4000, 0xe17: 0x4000, - 0xe18: 0x4000, 0xe19: 0x4000, 0xe1a: 0x4000, 0xe1b: 0x4000, 0xe1c: 0x4000, 0xe1d: 0x4000, - 0xe1e: 0x4000, 0xe1f: 0x4000, 0xe20: 0x4000, 0xe21: 0x4000, 0xe22: 0x4000, 0xe23: 0x4000, - 0xe24: 0x4000, 0xe25: 0x4000, 0xe26: 0x4000, 0xe27: 0x4000, 0xe28: 0x4000, 0xe29: 0x4000, - 0xe2a: 0x4000, 0xe2b: 0x4000, 0xe2c: 0x4000, 0xe2d: 0x4000, 0xe2e: 0x4000, 0xe2f: 0x4000, - 0xe30: 0x4000, 0xe31: 0x4000, 0xe32: 0x4000, 0xe33: 0x4000, 0xe34: 0x4000, 0xe35: 0x4000, - 0xe36: 0x4000, 0xe37: 0x4000, 0xe38: 0x4000, 0xe39: 0x4000, 0xe3a: 0x4000, 0xe3b: 0x4000, - 0xe3c: 0x4000, 0xe3d: 0x4000, 0xe3e: 0x4000, 0xe3f: 0x4000, - // Block 0x39, offset 0xe40 - 0xe40: 0x4000, 0xe41: 0x4000, 0xe42: 0x4000, 0xe43: 0x4000, 0xe44: 0x4000, 0xe45: 0x4000, - 0xe46: 0x4000, 0xe47: 0x4000, 0xe48: 0x4000, 0xe49: 0x4000, 0xe4a: 0x4000, 0xe4b: 0x4000, - 0xe4c: 0x4000, 0xe4d: 0x4000, 0xe4e: 0x4000, 0xe4f: 0x4000, 0xe50: 0x4000, 0xe51: 0x4000, - 0xe52: 0x4000, 0xe53: 0x4000, 0xe54: 0x4000, 0xe55: 0x4000, 0xe56: 0x4000, 0xe57: 0x4000, - 0xe58: 0x4000, 0xe59: 0x4000, 0xe5a: 0x4000, 0xe5b: 0x4000, 0xe5c: 0x4000, 0xe5d: 0x4000, - 0xe5e: 0x4000, 0xe5f: 0x4000, 0xe60: 0x4000, 0xe61: 0x4000, 0xe62: 0x4000, 0xe63: 0x4000, - 0xe70: 0x4000, 0xe71: 0x4000, 0xe72: 0x4000, 0xe73: 0x4000, 0xe74: 0x4000, 0xe75: 0x4000, - 0xe76: 0x4000, 0xe77: 0x4000, 0xe78: 0x4000, 0xe79: 0x4000, 0xe7a: 0x4000, 0xe7b: 0x4000, - 0xe7c: 0x4000, 0xe7d: 0x4000, 0xe7e: 0x4000, 0xe7f: 0x4000, - // Block 0x3a, offset 0xe80 - 0xe80: 0x4000, 0xe81: 0x4000, 0xe82: 0x4000, 0xe83: 0x4000, 0xe84: 0x4000, 0xe85: 0x4000, - 0xe86: 0x4000, 0xe87: 0x4000, 0xe88: 0x4000, 0xe89: 0x4000, 0xe8a: 0x4000, 0xe8b: 0x4000, - 0xe8c: 0x4000, 0xe8d: 0x4000, 0xe8e: 0x4000, 0xe8f: 0x4000, 0xe90: 0x4000, 0xe91: 0x4000, - 0xe92: 0x4000, 0xe93: 0x4000, 0xe94: 0x4000, 0xe95: 0x4000, 0xe96: 0x4000, 0xe97: 0x4000, - 0xe98: 0x4000, 0xe99: 0x4000, 0xe9a: 0x4000, 0xe9b: 0x4000, 0xe9c: 0x4000, 0xe9d: 0x4000, - 0xe9e: 0x4000, 0xea0: 0x4000, 0xea1: 0x4000, 0xea2: 0x4000, 0xea3: 0x4000, - 0xea4: 0x4000, 0xea5: 0x4000, 0xea6: 0x4000, 0xea7: 0x4000, 0xea8: 0x4000, 0xea9: 0x4000, - 0xeaa: 0x4000, 0xeab: 0x4000, 0xeac: 0x4000, 0xead: 0x4000, 0xeae: 0x4000, 0xeaf: 0x4000, - 0xeb0: 0x4000, 0xeb1: 0x4000, 0xeb2: 0x4000, 0xeb3: 0x4000, 0xeb4: 0x4000, 0xeb5: 0x4000, - 0xeb6: 0x4000, 0xeb7: 0x4000, 0xeb8: 0x4000, 0xeb9: 0x4000, 0xeba: 0x4000, 0xebb: 0x4000, - 0xebc: 0x4000, 0xebd: 0x4000, 0xebe: 0x4000, 0xebf: 0x4000, - // Block 0x3b, offset 0xec0 - 0xec0: 0x4000, 0xec1: 0x4000, 0xec2: 0x4000, 0xec3: 0x4000, 0xec4: 0x4000, 0xec5: 0x4000, - 0xec6: 0x4000, 0xec7: 0x4000, 0xec8: 0x2000, 0xec9: 0x2000, 0xeca: 0x2000, 0xecb: 0x2000, - 0xecc: 0x2000, 0xecd: 0x2000, 0xece: 0x2000, 0xecf: 0x2000, 0xed0: 0x4000, 0xed1: 0x4000, - 0xed2: 0x4000, 0xed3: 0x4000, 0xed4: 0x4000, 0xed5: 0x4000, 0xed6: 0x4000, 0xed7: 0x4000, - 0xed8: 0x4000, 0xed9: 0x4000, 0xeda: 0x4000, 0xedb: 0x4000, 0xedc: 0x4000, 0xedd: 0x4000, - 0xede: 0x4000, 0xedf: 0x4000, 0xee0: 0x4000, 0xee1: 0x4000, 0xee2: 0x4000, 0xee3: 0x4000, - 0xee4: 0x4000, 0xee5: 0x4000, 0xee6: 0x4000, 0xee7: 0x4000, 0xee8: 0x4000, 0xee9: 0x4000, - 0xeea: 0x4000, 0xeeb: 0x4000, 0xeec: 0x4000, 0xeed: 0x4000, 0xeee: 0x4000, 0xeef: 0x4000, - 0xef0: 0x4000, 0xef1: 0x4000, 0xef2: 0x4000, 0xef3: 0x4000, 0xef4: 0x4000, 0xef5: 0x4000, - 0xef6: 0x4000, 0xef7: 0x4000, 0xef8: 0x4000, 0xef9: 0x4000, 0xefa: 0x4000, 0xefb: 0x4000, - 0xefc: 0x4000, 0xefd: 0x4000, 0xefe: 0x4000, 0xeff: 0x4000, - // Block 0x3c, offset 0xf00 - 0xf00: 0x4000, 0xf01: 0x4000, 0xf02: 0x4000, 0xf03: 0x4000, 0xf04: 0x4000, 0xf05: 0x4000, - 0xf06: 0x4000, 0xf07: 0x4000, 0xf08: 0x4000, 0xf09: 0x4000, 0xf0a: 0x4000, 0xf0b: 0x4000, - 0xf0c: 0x4000, 0xf10: 0x4000, 0xf11: 0x4000, - 0xf12: 0x4000, 0xf13: 0x4000, 0xf14: 0x4000, 0xf15: 0x4000, 0xf16: 0x4000, 0xf17: 0x4000, - 0xf18: 0x4000, 0xf19: 0x4000, 0xf1a: 0x4000, 0xf1b: 0x4000, 0xf1c: 0x4000, 0xf1d: 0x4000, - 0xf1e: 0x4000, 0xf1f: 0x4000, 0xf20: 0x4000, 0xf21: 0x4000, 0xf22: 0x4000, 0xf23: 0x4000, - 0xf24: 0x4000, 0xf25: 0x4000, 0xf26: 0x4000, 0xf27: 0x4000, 0xf28: 0x4000, 0xf29: 0x4000, - 0xf2a: 0x4000, 0xf2b: 0x4000, 0xf2c: 0x4000, 0xf2d: 0x4000, 0xf2e: 0x4000, 0xf2f: 0x4000, - 0xf30: 0x4000, 0xf31: 0x4000, 0xf32: 0x4000, 0xf33: 0x4000, 0xf34: 0x4000, 0xf35: 0x4000, - 0xf36: 0x4000, 0xf37: 0x4000, 0xf38: 0x4000, 0xf39: 0x4000, 0xf3a: 0x4000, 0xf3b: 0x4000, - 0xf3c: 0x4000, 0xf3d: 0x4000, 0xf3e: 0x4000, 0xf3f: 0x4000, - // Block 0x3d, offset 0xf40 - 0xf40: 0x4000, 0xf41: 0x4000, 0xf42: 0x4000, 0xf43: 0x4000, 0xf44: 0x4000, 0xf45: 0x4000, - 0xf46: 0x4000, - // Block 0x3e, offset 0xf80 - 0xfa0: 0x4000, 0xfa1: 0x4000, 0xfa2: 0x4000, 0xfa3: 0x4000, - 0xfa4: 0x4000, 0xfa5: 0x4000, 0xfa6: 0x4000, 0xfa7: 0x4000, 0xfa8: 0x4000, 0xfa9: 0x4000, - 0xfaa: 0x4000, 0xfab: 0x4000, 0xfac: 0x4000, 0xfad: 0x4000, 0xfae: 0x4000, 0xfaf: 0x4000, - 0xfb0: 0x4000, 0xfb1: 0x4000, 0xfb2: 0x4000, 0xfb3: 0x4000, 0xfb4: 0x4000, 0xfb5: 0x4000, - 0xfb6: 0x4000, 0xfb7: 0x4000, 0xfb8: 0x4000, 0xfb9: 0x4000, 0xfba: 0x4000, 0xfbb: 0x4000, - 0xfbc: 0x4000, - // Block 0x3f, offset 0xfc0 - 0xfc0: 0x4000, 0xfc1: 0x4000, 0xfc2: 0x4000, 0xfc3: 0x4000, 0xfc4: 0x4000, 0xfc5: 0x4000, - 0xfc6: 0x4000, 0xfc7: 0x4000, 0xfc8: 0x4000, 0xfc9: 0x4000, 0xfca: 0x4000, 0xfcb: 0x4000, - 0xfcc: 0x4000, 0xfcd: 0x4000, 0xfce: 0x4000, 0xfcf: 0x4000, 0xfd0: 0x4000, 0xfd1: 0x4000, - 0xfd2: 0x4000, 0xfd3: 0x4000, 0xfd4: 0x4000, 0xfd5: 0x4000, 0xfd6: 0x4000, 0xfd7: 0x4000, - 0xfd8: 0x4000, 0xfd9: 0x4000, 0xfda: 0x4000, 0xfdb: 0x4000, 0xfdc: 0x4000, 0xfdd: 0x4000, - 0xfde: 0x4000, 0xfdf: 0x4000, 0xfe0: 0x4000, 0xfe1: 0x4000, 0xfe2: 0x4000, 0xfe3: 0x4000, - // Block 0x40, offset 0x1000 - 0x1000: 0x2000, 0x1001: 0x2000, 0x1002: 0x2000, 0x1003: 0x2000, 0x1004: 0x2000, 0x1005: 0x2000, - 0x1006: 0x2000, 0x1007: 0x2000, 0x1008: 0x2000, 0x1009: 0x2000, 0x100a: 0x2000, 0x100b: 0x2000, - 0x100c: 0x2000, 0x100d: 0x2000, 0x100e: 0x2000, 0x100f: 0x2000, 0x1010: 0x4000, 0x1011: 0x4000, - 0x1012: 0x4000, 0x1013: 0x4000, 0x1014: 0x4000, 0x1015: 0x4000, 0x1016: 0x4000, 0x1017: 0x4000, - 0x1018: 0x4000, 0x1019: 0x4000, - 0x1030: 0x4000, 0x1031: 0x4000, 0x1032: 0x4000, 0x1033: 0x4000, 0x1034: 0x4000, 0x1035: 0x4000, - 0x1036: 0x4000, 0x1037: 0x4000, 0x1038: 0x4000, 0x1039: 0x4000, 0x103a: 0x4000, 0x103b: 0x4000, - 0x103c: 0x4000, 0x103d: 0x4000, 0x103e: 0x4000, 0x103f: 0x4000, - // Block 0x41, offset 0x1040 - 0x1040: 0x4000, 0x1041: 0x4000, 0x1042: 0x4000, 0x1043: 0x4000, 0x1044: 0x4000, 0x1045: 0x4000, - 0x1046: 0x4000, 0x1047: 0x4000, 0x1048: 0x4000, 0x1049: 0x4000, 0x104a: 0x4000, 0x104b: 0x4000, - 0x104c: 0x4000, 0x104d: 0x4000, 0x104e: 0x4000, 0x104f: 0x4000, 0x1050: 0x4000, 0x1051: 0x4000, - 0x1052: 0x4000, 0x1054: 0x4000, 0x1055: 0x4000, 0x1056: 0x4000, 0x1057: 0x4000, - 0x1058: 0x4000, 0x1059: 0x4000, 0x105a: 0x4000, 0x105b: 0x4000, 0x105c: 0x4000, 0x105d: 0x4000, - 0x105e: 0x4000, 0x105f: 0x4000, 0x1060: 0x4000, 0x1061: 0x4000, 0x1062: 0x4000, 0x1063: 0x4000, - 0x1064: 0x4000, 0x1065: 0x4000, 0x1066: 0x4000, 0x1068: 0x4000, 0x1069: 0x4000, - 0x106a: 0x4000, 0x106b: 0x4000, - // Block 0x42, offset 0x1080 - 0x1081: 0x9012, 0x1082: 0x9012, 0x1083: 0x9012, 0x1084: 0x9012, 0x1085: 0x9012, - 0x1086: 0x9012, 0x1087: 0x9012, 0x1088: 0x9012, 0x1089: 0x9012, 0x108a: 0x9012, 0x108b: 0x9012, - 0x108c: 0x9012, 0x108d: 0x9012, 0x108e: 0x9012, 0x108f: 0x9012, 0x1090: 0x9012, 0x1091: 0x9012, - 0x1092: 0x9012, 0x1093: 0x9012, 0x1094: 0x9012, 0x1095: 0x9012, 0x1096: 0x9012, 0x1097: 0x9012, - 0x1098: 0x9012, 0x1099: 0x9012, 0x109a: 0x9012, 0x109b: 0x9012, 0x109c: 0x9012, 0x109d: 0x9012, - 0x109e: 0x9012, 0x109f: 0x9012, 0x10a0: 0x9049, 0x10a1: 0x9049, 0x10a2: 0x9049, 0x10a3: 0x9049, - 0x10a4: 0x9049, 0x10a5: 0x9049, 0x10a6: 0x9049, 0x10a7: 0x9049, 0x10a8: 0x9049, 0x10a9: 0x9049, - 0x10aa: 0x9049, 0x10ab: 0x9049, 0x10ac: 0x9049, 0x10ad: 0x9049, 0x10ae: 0x9049, 0x10af: 0x9049, - 0x10b0: 0x9049, 0x10b1: 0x9049, 0x10b2: 0x9049, 0x10b3: 0x9049, 0x10b4: 0x9049, 0x10b5: 0x9049, - 0x10b6: 0x9049, 0x10b7: 0x9049, 0x10b8: 0x9049, 0x10b9: 0x9049, 0x10ba: 0x9049, 0x10bb: 0x9049, - 0x10bc: 0x9049, 0x10bd: 0x9049, 0x10be: 0x9049, 0x10bf: 0x9049, - // Block 0x43, offset 0x10c0 - 0x10c0: 0x9049, 0x10c1: 0x9049, 0x10c2: 0x9049, 0x10c3: 0x9049, 0x10c4: 0x9049, 0x10c5: 0x9049, - 0x10c6: 0x9049, 0x10c7: 0x9049, 0x10c8: 0x9049, 0x10c9: 0x9049, 0x10ca: 0x9049, 0x10cb: 0x9049, - 0x10cc: 0x9049, 0x10cd: 0x9049, 0x10ce: 0x9049, 0x10cf: 0x9049, 0x10d0: 0x9049, 0x10d1: 0x9049, - 0x10d2: 0x9049, 0x10d3: 0x9049, 0x10d4: 0x9049, 0x10d5: 0x9049, 0x10d6: 0x9049, 0x10d7: 0x9049, - 0x10d8: 0x9049, 0x10d9: 0x9049, 0x10da: 0x9049, 0x10db: 0x9049, 0x10dc: 0x9049, 0x10dd: 0x9049, - 0x10de: 0x9049, 0x10df: 0x904a, 0x10e0: 0x904b, 0x10e1: 0xb04c, 0x10e2: 0xb04d, 0x10e3: 0xb04d, - 0x10e4: 0xb04e, 0x10e5: 0xb04f, 0x10e6: 0xb050, 0x10e7: 0xb051, 0x10e8: 0xb052, 0x10e9: 0xb053, - 0x10ea: 0xb054, 0x10eb: 0xb055, 0x10ec: 0xb056, 0x10ed: 0xb057, 0x10ee: 0xb058, 0x10ef: 0xb059, - 0x10f0: 0xb05a, 0x10f1: 0xb05b, 0x10f2: 0xb05c, 0x10f3: 0xb05d, 0x10f4: 0xb05e, 0x10f5: 0xb05f, - 0x10f6: 0xb060, 0x10f7: 0xb061, 0x10f8: 0xb062, 0x10f9: 0xb063, 0x10fa: 0xb064, 0x10fb: 0xb065, - 0x10fc: 0xb052, 0x10fd: 0xb066, 0x10fe: 0xb067, 0x10ff: 0xb055, - // Block 0x44, offset 0x1100 - 0x1100: 0xb068, 0x1101: 0xb069, 0x1102: 0xb06a, 0x1103: 0xb06b, 0x1104: 0xb05a, 0x1105: 0xb056, - 0x1106: 0xb06c, 0x1107: 0xb06d, 0x1108: 0xb06b, 0x1109: 0xb06e, 0x110a: 0xb06b, 0x110b: 0xb06f, - 0x110c: 0xb06f, 0x110d: 0xb070, 0x110e: 0xb070, 0x110f: 0xb071, 0x1110: 0xb056, 0x1111: 0xb072, - 0x1112: 0xb073, 0x1113: 0xb072, 0x1114: 0xb074, 0x1115: 0xb073, 0x1116: 0xb075, 0x1117: 0xb075, - 0x1118: 0xb076, 0x1119: 0xb076, 0x111a: 0xb077, 0x111b: 0xb077, 0x111c: 0xb073, 0x111d: 0xb078, - 0x111e: 0xb079, 0x111f: 0xb067, 0x1120: 0xb07a, 0x1121: 0xb07b, 0x1122: 0xb07b, 0x1123: 0xb07b, - 0x1124: 0xb07b, 0x1125: 0xb07b, 0x1126: 0xb07b, 0x1127: 0xb07b, 0x1128: 0xb07b, 0x1129: 0xb07b, - 0x112a: 0xb07b, 0x112b: 0xb07b, 0x112c: 0xb07b, 0x112d: 0xb07b, 0x112e: 0xb07b, 0x112f: 0xb07b, - 0x1130: 0xb07c, 0x1131: 0xb07c, 0x1132: 0xb07c, 0x1133: 0xb07c, 0x1134: 0xb07c, 0x1135: 0xb07c, - 0x1136: 0xb07c, 0x1137: 0xb07c, 0x1138: 0xb07c, 0x1139: 0xb07c, 0x113a: 0xb07c, 0x113b: 0xb07c, - 0x113c: 0xb07c, 0x113d: 0xb07c, 0x113e: 0xb07c, - // Block 0x45, offset 0x1140 - 0x1142: 0xb07d, 0x1143: 0xb07e, 0x1144: 0xb07f, 0x1145: 0xb080, - 0x1146: 0xb07f, 0x1147: 0xb07e, 0x114a: 0xb081, 0x114b: 0xb082, - 0x114c: 0xb083, 0x114d: 0xb07f, 0x114e: 0xb080, 0x114f: 0xb07f, - 0x1152: 0xb084, 0x1153: 0xb085, 0x1154: 0xb084, 0x1155: 0xb086, 0x1156: 0xb084, 0x1157: 0xb087, - 0x115a: 0xb088, 0x115b: 0xb089, 0x115c: 0xb08a, - 0x1160: 0x908b, 0x1161: 0x908b, 0x1162: 0x908c, 0x1163: 0x908d, - 0x1164: 0x908b, 0x1165: 0x908e, 0x1166: 0x908f, 0x1168: 0xb090, 0x1169: 0xb091, - 0x116a: 0xb092, 0x116b: 0xb091, 0x116c: 0xb093, 0x116d: 0xb094, 0x116e: 0xb095, - 0x117d: 0x2000, - // Block 0x46, offset 0x1180 - 0x11a0: 0x4000, 0x11a1: 0x4000, 0x11a2: 0x4000, 0x11a3: 0x4000, - 0x11a4: 0x4000, - 0x11b0: 0x4000, 0x11b1: 0x4000, - // Block 0x47, offset 0x11c0 - 0x11c0: 0x4000, 0x11c1: 0x4000, 0x11c2: 0x4000, 0x11c3: 0x4000, 0x11c4: 0x4000, 0x11c5: 0x4000, - 0x11c6: 0x4000, 0x11c7: 0x4000, 0x11c8: 0x4000, 0x11c9: 0x4000, 0x11ca: 0x4000, 0x11cb: 0x4000, - 0x11cc: 0x4000, 0x11cd: 0x4000, 0x11ce: 0x4000, 0x11cf: 0x4000, 0x11d0: 0x4000, 0x11d1: 0x4000, - 0x11d2: 0x4000, 0x11d3: 0x4000, 0x11d4: 0x4000, 0x11d5: 0x4000, 0x11d6: 0x4000, 0x11d7: 0x4000, - 0x11d8: 0x4000, 0x11d9: 0x4000, 0x11da: 0x4000, 0x11db: 0x4000, 0x11dc: 0x4000, 0x11dd: 0x4000, - 0x11de: 0x4000, 0x11df: 0x4000, 0x11e0: 0x4000, 0x11e1: 0x4000, 0x11e2: 0x4000, 0x11e3: 0x4000, - 0x11e4: 0x4000, 0x11e5: 0x4000, 0x11e6: 0x4000, 0x11e7: 0x4000, 0x11e8: 0x4000, 0x11e9: 0x4000, - 0x11ea: 0x4000, 0x11eb: 0x4000, 0x11ec: 0x4000, 0x11ed: 0x4000, 0x11ee: 0x4000, 0x11ef: 0x4000, - 0x11f0: 0x4000, 0x11f1: 0x4000, 0x11f2: 0x4000, 0x11f3: 0x4000, 0x11f4: 0x4000, 0x11f5: 0x4000, - 0x11f6: 0x4000, 0x11f7: 0x4000, - // Block 0x48, offset 0x1200 - 0x1200: 0x4000, 0x1201: 0x4000, 0x1202: 0x4000, 0x1203: 0x4000, 0x1204: 0x4000, 0x1205: 0x4000, - 0x1206: 0x4000, 0x1207: 0x4000, 0x1208: 0x4000, 0x1209: 0x4000, 0x120a: 0x4000, 0x120b: 0x4000, - 0x120c: 0x4000, 0x120d: 0x4000, 0x120e: 0x4000, 0x120f: 0x4000, 0x1210: 0x4000, 0x1211: 0x4000, - 0x1212: 0x4000, 0x1213: 0x4000, 0x1214: 0x4000, 0x1215: 0x4000, - // Block 0x49, offset 0x1240 - 0x1240: 0x4000, 0x1241: 0x4000, 0x1242: 0x4000, 0x1243: 0x4000, 0x1244: 0x4000, 0x1245: 0x4000, - 0x1246: 0x4000, 0x1247: 0x4000, 0x1248: 0x4000, - // Block 0x4a, offset 0x1280 - 0x1280: 0x4000, 0x1281: 0x4000, 0x1282: 0x4000, 0x1283: 0x4000, 0x1284: 0x4000, 0x1285: 0x4000, - 0x1286: 0x4000, 0x1287: 0x4000, 0x1288: 0x4000, 0x1289: 0x4000, 0x128a: 0x4000, 0x128b: 0x4000, - 0x128c: 0x4000, 0x128d: 0x4000, 0x128e: 0x4000, 0x128f: 0x4000, 0x1290: 0x4000, 0x1291: 0x4000, - 0x1292: 0x4000, 0x1293: 0x4000, 0x1294: 0x4000, 0x1295: 0x4000, 0x1296: 0x4000, 0x1297: 0x4000, - 0x1298: 0x4000, 0x1299: 0x4000, 0x129a: 0x4000, 0x129b: 0x4000, 0x129c: 0x4000, 0x129d: 0x4000, - 0x129e: 0x4000, - // Block 0x4b, offset 0x12c0 - 0x12d0: 0x4000, 0x12d1: 0x4000, - 0x12d2: 0x4000, - 0x12e4: 0x4000, 0x12e5: 0x4000, 0x12e6: 0x4000, 0x12e7: 0x4000, - 0x12f0: 0x4000, 0x12f1: 0x4000, 0x12f2: 0x4000, 0x12f3: 0x4000, 0x12f4: 0x4000, 0x12f5: 0x4000, - 0x12f6: 0x4000, 0x12f7: 0x4000, 0x12f8: 0x4000, 0x12f9: 0x4000, 0x12fa: 0x4000, 0x12fb: 0x4000, - 0x12fc: 0x4000, 0x12fd: 0x4000, 0x12fe: 0x4000, 0x12ff: 0x4000, - // Block 0x4c, offset 0x1300 - 0x1300: 0x4000, 0x1301: 0x4000, 0x1302: 0x4000, 0x1303: 0x4000, 0x1304: 0x4000, 0x1305: 0x4000, - 0x1306: 0x4000, 0x1307: 0x4000, 0x1308: 0x4000, 0x1309: 0x4000, 0x130a: 0x4000, 0x130b: 0x4000, - 0x130c: 0x4000, 0x130d: 0x4000, 0x130e: 0x4000, 0x130f: 0x4000, 0x1310: 0x4000, 0x1311: 0x4000, - 0x1312: 0x4000, 0x1313: 0x4000, 0x1314: 0x4000, 0x1315: 0x4000, 0x1316: 0x4000, 0x1317: 0x4000, - 0x1318: 0x4000, 0x1319: 0x4000, 0x131a: 0x4000, 0x131b: 0x4000, 0x131c: 0x4000, 0x131d: 0x4000, - 0x131e: 0x4000, 0x131f: 0x4000, 0x1320: 0x4000, 0x1321: 0x4000, 0x1322: 0x4000, 0x1323: 0x4000, - 0x1324: 0x4000, 0x1325: 0x4000, 0x1326: 0x4000, 0x1327: 0x4000, 0x1328: 0x4000, 0x1329: 0x4000, - 0x132a: 0x4000, 0x132b: 0x4000, 0x132c: 0x4000, 0x132d: 0x4000, 0x132e: 0x4000, 0x132f: 0x4000, - 0x1330: 0x4000, 0x1331: 0x4000, 0x1332: 0x4000, 0x1333: 0x4000, 0x1334: 0x4000, 0x1335: 0x4000, - 0x1336: 0x4000, 0x1337: 0x4000, 0x1338: 0x4000, 0x1339: 0x4000, 0x133a: 0x4000, 0x133b: 0x4000, - // Block 0x4d, offset 0x1340 - 0x1344: 0x4000, - // Block 0x4e, offset 0x1380 - 0x138f: 0x4000, - // Block 0x4f, offset 0x13c0 - 0x13c0: 0x2000, 0x13c1: 0x2000, 0x13c2: 0x2000, 0x13c3: 0x2000, 0x13c4: 0x2000, 0x13c5: 0x2000, - 0x13c6: 0x2000, 0x13c7: 0x2000, 0x13c8: 0x2000, 0x13c9: 0x2000, 0x13ca: 0x2000, - 0x13d0: 0x2000, 0x13d1: 0x2000, - 0x13d2: 0x2000, 0x13d3: 0x2000, 0x13d4: 0x2000, 0x13d5: 0x2000, 0x13d6: 0x2000, 0x13d7: 0x2000, - 0x13d8: 0x2000, 0x13d9: 0x2000, 0x13da: 0x2000, 0x13db: 0x2000, 0x13dc: 0x2000, 0x13dd: 0x2000, - 0x13de: 0x2000, 0x13df: 0x2000, 0x13e0: 0x2000, 0x13e1: 0x2000, 0x13e2: 0x2000, 0x13e3: 0x2000, - 0x13e4: 0x2000, 0x13e5: 0x2000, 0x13e6: 0x2000, 0x13e7: 0x2000, 0x13e8: 0x2000, 0x13e9: 0x2000, - 0x13ea: 0x2000, 0x13eb: 0x2000, 0x13ec: 0x2000, 0x13ed: 0x2000, - 0x13f0: 0x2000, 0x13f1: 0x2000, 0x13f2: 0x2000, 0x13f3: 0x2000, 0x13f4: 0x2000, 0x13f5: 0x2000, - 0x13f6: 0x2000, 0x13f7: 0x2000, 0x13f8: 0x2000, 0x13f9: 0x2000, 0x13fa: 0x2000, 0x13fb: 0x2000, - 0x13fc: 0x2000, 0x13fd: 0x2000, 0x13fe: 0x2000, 0x13ff: 0x2000, - // Block 0x50, offset 0x1400 - 0x1400: 0x2000, 0x1401: 0x2000, 0x1402: 0x2000, 0x1403: 0x2000, 0x1404: 0x2000, 0x1405: 0x2000, - 0x1406: 0x2000, 0x1407: 0x2000, 0x1408: 0x2000, 0x1409: 0x2000, 0x140a: 0x2000, 0x140b: 0x2000, - 0x140c: 0x2000, 0x140d: 0x2000, 0x140e: 0x2000, 0x140f: 0x2000, 0x1410: 0x2000, 0x1411: 0x2000, - 0x1412: 0x2000, 0x1413: 0x2000, 0x1414: 0x2000, 0x1415: 0x2000, 0x1416: 0x2000, 0x1417: 0x2000, - 0x1418: 0x2000, 0x1419: 0x2000, 0x141a: 0x2000, 0x141b: 0x2000, 0x141c: 0x2000, 0x141d: 0x2000, - 0x141e: 0x2000, 0x141f: 0x2000, 0x1420: 0x2000, 0x1421: 0x2000, 0x1422: 0x2000, 0x1423: 0x2000, - 0x1424: 0x2000, 0x1425: 0x2000, 0x1426: 0x2000, 0x1427: 0x2000, 0x1428: 0x2000, 0x1429: 0x2000, - 0x1430: 0x2000, 0x1431: 0x2000, 0x1432: 0x2000, 0x1433: 0x2000, 0x1434: 0x2000, 0x1435: 0x2000, - 0x1436: 0x2000, 0x1437: 0x2000, 0x1438: 0x2000, 0x1439: 0x2000, 0x143a: 0x2000, 0x143b: 0x2000, - 0x143c: 0x2000, 0x143d: 0x2000, 0x143e: 0x2000, 0x143f: 0x2000, - // Block 0x51, offset 0x1440 - 0x1440: 0x2000, 0x1441: 0x2000, 0x1442: 0x2000, 0x1443: 0x2000, 0x1444: 0x2000, 0x1445: 0x2000, - 0x1446: 0x2000, 0x1447: 0x2000, 0x1448: 0x2000, 0x1449: 0x2000, 0x144a: 0x2000, 0x144b: 0x2000, - 0x144c: 0x2000, 0x144d: 0x2000, 0x144e: 0x4000, 0x144f: 0x2000, 0x1450: 0x2000, 0x1451: 0x4000, - 0x1452: 0x4000, 0x1453: 0x4000, 0x1454: 0x4000, 0x1455: 0x4000, 0x1456: 0x4000, 0x1457: 0x4000, - 0x1458: 0x4000, 0x1459: 0x4000, 0x145a: 0x4000, 0x145b: 0x2000, 0x145c: 0x2000, 0x145d: 0x2000, - 0x145e: 0x2000, 0x145f: 0x2000, 0x1460: 0x2000, 0x1461: 0x2000, 0x1462: 0x2000, 0x1463: 0x2000, - 0x1464: 0x2000, 0x1465: 0x2000, 0x1466: 0x2000, 0x1467: 0x2000, 0x1468: 0x2000, 0x1469: 0x2000, - 0x146a: 0x2000, 0x146b: 0x2000, 0x146c: 0x2000, - // Block 0x52, offset 0x1480 - 0x1480: 0x4000, 0x1481: 0x4000, 0x1482: 0x4000, - 0x1490: 0x4000, 0x1491: 0x4000, - 0x1492: 0x4000, 0x1493: 0x4000, 0x1494: 0x4000, 0x1495: 0x4000, 0x1496: 0x4000, 0x1497: 0x4000, - 0x1498: 0x4000, 0x1499: 0x4000, 0x149a: 0x4000, 0x149b: 0x4000, 0x149c: 0x4000, 0x149d: 0x4000, - 0x149e: 0x4000, 0x149f: 0x4000, 0x14a0: 0x4000, 0x14a1: 0x4000, 0x14a2: 0x4000, 0x14a3: 0x4000, - 0x14a4: 0x4000, 0x14a5: 0x4000, 0x14a6: 0x4000, 0x14a7: 0x4000, 0x14a8: 0x4000, 0x14a9: 0x4000, - 0x14aa: 0x4000, 0x14ab: 0x4000, 0x14ac: 0x4000, 0x14ad: 0x4000, 0x14ae: 0x4000, 0x14af: 0x4000, - 0x14b0: 0x4000, 0x14b1: 0x4000, 0x14b2: 0x4000, 0x14b3: 0x4000, 0x14b4: 0x4000, 0x14b5: 0x4000, - 0x14b6: 0x4000, 0x14b7: 0x4000, 0x14b8: 0x4000, 0x14b9: 0x4000, 0x14ba: 0x4000, 0x14bb: 0x4000, - // Block 0x53, offset 0x14c0 - 0x14c0: 0x4000, 0x14c1: 0x4000, 0x14c2: 0x4000, 0x14c3: 0x4000, 0x14c4: 0x4000, 0x14c5: 0x4000, - 0x14c6: 0x4000, 0x14c7: 0x4000, 0x14c8: 0x4000, - 0x14d0: 0x4000, 0x14d1: 0x4000, - 0x14e0: 0x4000, 0x14e1: 0x4000, 0x14e2: 0x4000, 0x14e3: 0x4000, - 0x14e4: 0x4000, 0x14e5: 0x4000, - // Block 0x54, offset 0x1500 - 0x1500: 0x4000, 0x1501: 0x4000, 0x1502: 0x4000, 0x1503: 0x4000, 0x1504: 0x4000, 0x1505: 0x4000, - 0x1506: 0x4000, 0x1507: 0x4000, 0x1508: 0x4000, 0x1509: 0x4000, 0x150a: 0x4000, 0x150b: 0x4000, - 0x150c: 0x4000, 0x150d: 0x4000, 0x150e: 0x4000, 0x150f: 0x4000, 0x1510: 0x4000, 0x1511: 0x4000, - 0x1512: 0x4000, 0x1513: 0x4000, 0x1514: 0x4000, 0x1515: 0x4000, 0x1516: 0x4000, 0x1517: 0x4000, - 0x1518: 0x4000, 0x1519: 0x4000, 0x151a: 0x4000, 0x151b: 0x4000, 0x151c: 0x4000, 0x151d: 0x4000, - 0x151e: 0x4000, 0x151f: 0x4000, 0x1520: 0x4000, - 0x152d: 0x4000, 0x152e: 0x4000, 0x152f: 0x4000, - 0x1530: 0x4000, 0x1531: 0x4000, 0x1532: 0x4000, 0x1533: 0x4000, 0x1534: 0x4000, 0x1535: 0x4000, - 0x1537: 0x4000, 0x1538: 0x4000, 0x1539: 0x4000, 0x153a: 0x4000, 0x153b: 0x4000, - 0x153c: 0x4000, 0x153d: 0x4000, 0x153e: 0x4000, 0x153f: 0x4000, - // Block 0x55, offset 0x1540 - 0x1540: 0x4000, 0x1541: 0x4000, 0x1542: 0x4000, 0x1543: 0x4000, 0x1544: 0x4000, 0x1545: 0x4000, - 0x1546: 0x4000, 0x1547: 0x4000, 0x1548: 0x4000, 0x1549: 0x4000, 0x154a: 0x4000, 0x154b: 0x4000, - 0x154c: 0x4000, 0x154d: 0x4000, 0x154e: 0x4000, 0x154f: 0x4000, 0x1550: 0x4000, 0x1551: 0x4000, - 0x1552: 0x4000, 0x1553: 0x4000, 0x1554: 0x4000, 0x1555: 0x4000, 0x1556: 0x4000, 0x1557: 0x4000, - 0x1558: 0x4000, 0x1559: 0x4000, 0x155a: 0x4000, 0x155b: 0x4000, 0x155c: 0x4000, 0x155d: 0x4000, - 0x155e: 0x4000, 0x155f: 0x4000, 0x1560: 0x4000, 0x1561: 0x4000, 0x1562: 0x4000, 0x1563: 0x4000, - 0x1564: 0x4000, 0x1565: 0x4000, 0x1566: 0x4000, 0x1567: 0x4000, 0x1568: 0x4000, 0x1569: 0x4000, - 0x156a: 0x4000, 0x156b: 0x4000, 0x156c: 0x4000, 0x156d: 0x4000, 0x156e: 0x4000, 0x156f: 0x4000, - 0x1570: 0x4000, 0x1571: 0x4000, 0x1572: 0x4000, 0x1573: 0x4000, 0x1574: 0x4000, 0x1575: 0x4000, - 0x1576: 0x4000, 0x1577: 0x4000, 0x1578: 0x4000, 0x1579: 0x4000, 0x157a: 0x4000, 0x157b: 0x4000, - 0x157c: 0x4000, 0x157e: 0x4000, 0x157f: 0x4000, - // Block 0x56, offset 0x1580 - 0x1580: 0x4000, 0x1581: 0x4000, 0x1582: 0x4000, 0x1583: 0x4000, 0x1584: 0x4000, 0x1585: 0x4000, - 0x1586: 0x4000, 0x1587: 0x4000, 0x1588: 0x4000, 0x1589: 0x4000, 0x158a: 0x4000, 0x158b: 0x4000, - 0x158c: 0x4000, 0x158d: 0x4000, 0x158e: 0x4000, 0x158f: 0x4000, 0x1590: 0x4000, 0x1591: 0x4000, - 0x1592: 0x4000, 0x1593: 0x4000, - 0x15a0: 0x4000, 0x15a1: 0x4000, 0x15a2: 0x4000, 0x15a3: 0x4000, - 0x15a4: 0x4000, 0x15a5: 0x4000, 0x15a6: 0x4000, 0x15a7: 0x4000, 0x15a8: 0x4000, 0x15a9: 0x4000, - 0x15aa: 0x4000, 0x15ab: 0x4000, 0x15ac: 0x4000, 0x15ad: 0x4000, 0x15ae: 0x4000, 0x15af: 0x4000, - 0x15b0: 0x4000, 0x15b1: 0x4000, 0x15b2: 0x4000, 0x15b3: 0x4000, 0x15b4: 0x4000, 0x15b5: 0x4000, - 0x15b6: 0x4000, 0x15b7: 0x4000, 0x15b8: 0x4000, 0x15b9: 0x4000, 0x15ba: 0x4000, 0x15bb: 0x4000, - 0x15bc: 0x4000, 0x15bd: 0x4000, 0x15be: 0x4000, 0x15bf: 0x4000, - // Block 0x57, offset 0x15c0 - 0x15c0: 0x4000, 0x15c1: 0x4000, 0x15c2: 0x4000, 0x15c3: 0x4000, 0x15c4: 0x4000, 0x15c5: 0x4000, - 0x15c6: 0x4000, 0x15c7: 0x4000, 0x15c8: 0x4000, 0x15c9: 0x4000, 0x15ca: 0x4000, - 0x15cf: 0x4000, 0x15d0: 0x4000, 0x15d1: 0x4000, - 0x15d2: 0x4000, 0x15d3: 0x4000, - 0x15e0: 0x4000, 0x15e1: 0x4000, 0x15e2: 0x4000, 0x15e3: 0x4000, - 0x15e4: 0x4000, 0x15e5: 0x4000, 0x15e6: 0x4000, 0x15e7: 0x4000, 0x15e8: 0x4000, 0x15e9: 0x4000, - 0x15ea: 0x4000, 0x15eb: 0x4000, 0x15ec: 0x4000, 0x15ed: 0x4000, 0x15ee: 0x4000, 0x15ef: 0x4000, - 0x15f0: 0x4000, 0x15f4: 0x4000, - 0x15f8: 0x4000, 0x15f9: 0x4000, 0x15fa: 0x4000, 0x15fb: 0x4000, - 0x15fc: 0x4000, 0x15fd: 0x4000, 0x15fe: 0x4000, 0x15ff: 0x4000, - // Block 0x58, offset 0x1600 - 0x1600: 0x4000, 0x1601: 0x4000, 0x1602: 0x4000, 0x1603: 0x4000, 0x1604: 0x4000, 0x1605: 0x4000, - 0x1606: 0x4000, 0x1607: 0x4000, 0x1608: 0x4000, 0x1609: 0x4000, 0x160a: 0x4000, 0x160b: 0x4000, - 0x160c: 0x4000, 0x160d: 0x4000, 0x160e: 0x4000, 0x160f: 0x4000, 0x1610: 0x4000, 0x1611: 0x4000, - 0x1612: 0x4000, 0x1613: 0x4000, 0x1614: 0x4000, 0x1615: 0x4000, 0x1616: 0x4000, 0x1617: 0x4000, - 0x1618: 0x4000, 0x1619: 0x4000, 0x161a: 0x4000, 0x161b: 0x4000, 0x161c: 0x4000, 0x161d: 0x4000, - 0x161e: 0x4000, 0x161f: 0x4000, 0x1620: 0x4000, 0x1621: 0x4000, 0x1622: 0x4000, 0x1623: 0x4000, - 0x1624: 0x4000, 0x1625: 0x4000, 0x1626: 0x4000, 0x1627: 0x4000, 0x1628: 0x4000, 0x1629: 0x4000, - 0x162a: 0x4000, 0x162b: 0x4000, 0x162c: 0x4000, 0x162d: 0x4000, 0x162e: 0x4000, 0x162f: 0x4000, - 0x1630: 0x4000, 0x1631: 0x4000, 0x1632: 0x4000, 0x1633: 0x4000, 0x1634: 0x4000, 0x1635: 0x4000, - 0x1636: 0x4000, 0x1637: 0x4000, 0x1638: 0x4000, 0x1639: 0x4000, 0x163a: 0x4000, 0x163b: 0x4000, - 0x163c: 0x4000, 0x163d: 0x4000, 0x163e: 0x4000, - // Block 0x59, offset 0x1640 - 0x1640: 0x4000, 0x1642: 0x4000, 0x1643: 0x4000, 0x1644: 0x4000, 0x1645: 0x4000, - 0x1646: 0x4000, 0x1647: 0x4000, 0x1648: 0x4000, 0x1649: 0x4000, 0x164a: 0x4000, 0x164b: 0x4000, - 0x164c: 0x4000, 0x164d: 0x4000, 0x164e: 0x4000, 0x164f: 0x4000, 0x1650: 0x4000, 0x1651: 0x4000, - 0x1652: 0x4000, 0x1653: 0x4000, 0x1654: 0x4000, 0x1655: 0x4000, 0x1656: 0x4000, 0x1657: 0x4000, - 0x1658: 0x4000, 0x1659: 0x4000, 0x165a: 0x4000, 0x165b: 0x4000, 0x165c: 0x4000, 0x165d: 0x4000, - 0x165e: 0x4000, 0x165f: 0x4000, 0x1660: 0x4000, 0x1661: 0x4000, 0x1662: 0x4000, 0x1663: 0x4000, - 0x1664: 0x4000, 0x1665: 0x4000, 0x1666: 0x4000, 0x1667: 0x4000, 0x1668: 0x4000, 0x1669: 0x4000, - 0x166a: 0x4000, 0x166b: 0x4000, 0x166c: 0x4000, 0x166d: 0x4000, 0x166e: 0x4000, 0x166f: 0x4000, - 0x1670: 0x4000, 0x1671: 0x4000, 0x1672: 0x4000, 0x1673: 0x4000, 0x1674: 0x4000, 0x1675: 0x4000, - 0x1676: 0x4000, 0x1677: 0x4000, 0x1678: 0x4000, 0x1679: 0x4000, 0x167a: 0x4000, 0x167b: 0x4000, - 0x167c: 0x4000, 0x167d: 0x4000, 0x167e: 0x4000, 0x167f: 0x4000, - // Block 0x5a, offset 0x1680 - 0x1680: 0x4000, 0x1681: 0x4000, 0x1682: 0x4000, 0x1683: 0x4000, 0x1684: 0x4000, 0x1685: 0x4000, - 0x1686: 0x4000, 0x1687: 0x4000, 0x1688: 0x4000, 0x1689: 0x4000, 0x168a: 0x4000, 0x168b: 0x4000, - 0x168c: 0x4000, 0x168d: 0x4000, 0x168e: 0x4000, 0x168f: 0x4000, 0x1690: 0x4000, 0x1691: 0x4000, - 0x1692: 0x4000, 0x1693: 0x4000, 0x1694: 0x4000, 0x1695: 0x4000, 0x1696: 0x4000, 0x1697: 0x4000, - 0x1698: 0x4000, 0x1699: 0x4000, 0x169a: 0x4000, 0x169b: 0x4000, 0x169c: 0x4000, 0x169d: 0x4000, - 0x169e: 0x4000, 0x169f: 0x4000, 0x16a0: 0x4000, 0x16a1: 0x4000, 0x16a2: 0x4000, 0x16a3: 0x4000, - 0x16a4: 0x4000, 0x16a5: 0x4000, 0x16a6: 0x4000, 0x16a7: 0x4000, 0x16a8: 0x4000, 0x16a9: 0x4000, - 0x16aa: 0x4000, 0x16ab: 0x4000, 0x16ac: 0x4000, 0x16ad: 0x4000, 0x16ae: 0x4000, 0x16af: 0x4000, - 0x16b0: 0x4000, 0x16b1: 0x4000, 0x16b2: 0x4000, 0x16b3: 0x4000, 0x16b4: 0x4000, 0x16b5: 0x4000, - 0x16b6: 0x4000, 0x16b7: 0x4000, 0x16b8: 0x4000, 0x16b9: 0x4000, 0x16ba: 0x4000, 0x16bb: 0x4000, - 0x16bc: 0x4000, 0x16bf: 0x4000, - // Block 0x5b, offset 0x16c0 - 0x16c0: 0x4000, 0x16c1: 0x4000, 0x16c2: 0x4000, 0x16c3: 0x4000, 0x16c4: 0x4000, 0x16c5: 0x4000, - 0x16c6: 0x4000, 0x16c7: 0x4000, 0x16c8: 0x4000, 0x16c9: 0x4000, 0x16ca: 0x4000, 0x16cb: 0x4000, - 0x16cc: 0x4000, 0x16cd: 0x4000, 0x16ce: 0x4000, 0x16cf: 0x4000, 0x16d0: 0x4000, 0x16d1: 0x4000, - 0x16d2: 0x4000, 0x16d3: 0x4000, 0x16d4: 0x4000, 0x16d5: 0x4000, 0x16d6: 0x4000, 0x16d7: 0x4000, - 0x16d8: 0x4000, 0x16d9: 0x4000, 0x16da: 0x4000, 0x16db: 0x4000, 0x16dc: 0x4000, 0x16dd: 0x4000, - 0x16de: 0x4000, 0x16df: 0x4000, 0x16e0: 0x4000, 0x16e1: 0x4000, 0x16e2: 0x4000, 0x16e3: 0x4000, - 0x16e4: 0x4000, 0x16e5: 0x4000, 0x16e6: 0x4000, 0x16e7: 0x4000, 0x16e8: 0x4000, 0x16e9: 0x4000, - 0x16ea: 0x4000, 0x16eb: 0x4000, 0x16ec: 0x4000, 0x16ed: 0x4000, 0x16ee: 0x4000, 0x16ef: 0x4000, - 0x16f0: 0x4000, 0x16f1: 0x4000, 0x16f2: 0x4000, 0x16f3: 0x4000, 0x16f4: 0x4000, 0x16f5: 0x4000, - 0x16f6: 0x4000, 0x16f7: 0x4000, 0x16f8: 0x4000, 0x16f9: 0x4000, 0x16fa: 0x4000, 0x16fb: 0x4000, - 0x16fc: 0x4000, 0x16fd: 0x4000, - // Block 0x5c, offset 0x1700 - 0x170b: 0x4000, - 0x170c: 0x4000, 0x170d: 0x4000, 0x170e: 0x4000, 0x1710: 0x4000, 0x1711: 0x4000, - 0x1712: 0x4000, 0x1713: 0x4000, 0x1714: 0x4000, 0x1715: 0x4000, 0x1716: 0x4000, 0x1717: 0x4000, - 0x1718: 0x4000, 0x1719: 0x4000, 0x171a: 0x4000, 0x171b: 0x4000, 0x171c: 0x4000, 0x171d: 0x4000, - 0x171e: 0x4000, 0x171f: 0x4000, 0x1720: 0x4000, 0x1721: 0x4000, 0x1722: 0x4000, 0x1723: 0x4000, - 0x1724: 0x4000, 0x1725: 0x4000, 0x1726: 0x4000, 0x1727: 0x4000, - 0x173a: 0x4000, - // Block 0x5d, offset 0x1740 - 0x1755: 0x4000, 0x1756: 0x4000, - 0x1764: 0x4000, - // Block 0x5e, offset 0x1780 - 0x17bb: 0x4000, - 0x17bc: 0x4000, 0x17bd: 0x4000, 0x17be: 0x4000, 0x17bf: 0x4000, - // Block 0x5f, offset 0x17c0 - 0x17c0: 0x4000, 0x17c1: 0x4000, 0x17c2: 0x4000, 0x17c3: 0x4000, 0x17c4: 0x4000, 0x17c5: 0x4000, - 0x17c6: 0x4000, 0x17c7: 0x4000, 0x17c8: 0x4000, 0x17c9: 0x4000, 0x17ca: 0x4000, 0x17cb: 0x4000, - 0x17cc: 0x4000, 0x17cd: 0x4000, 0x17ce: 0x4000, 0x17cf: 0x4000, - // Block 0x60, offset 0x1800 - 0x1800: 0x4000, 0x1801: 0x4000, 0x1802: 0x4000, 0x1803: 0x4000, 0x1804: 0x4000, 0x1805: 0x4000, - 0x180c: 0x4000, 0x1810: 0x4000, 0x1811: 0x4000, - 0x1812: 0x4000, 0x1815: 0x4000, 0x1816: 0x4000, 0x1817: 0x4000, - 0x182b: 0x4000, 0x182c: 0x4000, - 0x1834: 0x4000, 0x1835: 0x4000, - 0x1836: 0x4000, 0x1837: 0x4000, 0x1838: 0x4000, 0x1839: 0x4000, 0x183a: 0x4000, 0x183b: 0x4000, - 0x183c: 0x4000, - // Block 0x61, offset 0x1840 - 0x1860: 0x4000, 0x1861: 0x4000, 0x1862: 0x4000, 0x1863: 0x4000, - 0x1864: 0x4000, 0x1865: 0x4000, 0x1866: 0x4000, 0x1867: 0x4000, 0x1868: 0x4000, 0x1869: 0x4000, - 0x186a: 0x4000, 0x186b: 0x4000, - // Block 0x62, offset 0x1880 - 0x188c: 0x4000, 0x188d: 0x4000, 0x188e: 0x4000, 0x188f: 0x4000, 0x1890: 0x4000, 0x1891: 0x4000, - 0x1892: 0x4000, 0x1893: 0x4000, 0x1894: 0x4000, 0x1895: 0x4000, 0x1896: 0x4000, 0x1897: 0x4000, - 0x1898: 0x4000, 0x1899: 0x4000, 0x189a: 0x4000, 0x189b: 0x4000, 0x189c: 0x4000, 0x189d: 0x4000, - 0x189e: 0x4000, 0x189f: 0x4000, 0x18a0: 0x4000, 0x18a1: 0x4000, 0x18a2: 0x4000, 0x18a3: 0x4000, - 0x18a4: 0x4000, 0x18a5: 0x4000, 0x18a6: 0x4000, 0x18a7: 0x4000, 0x18a8: 0x4000, 0x18a9: 0x4000, - 0x18aa: 0x4000, 0x18ab: 0x4000, 0x18ac: 0x4000, 0x18ad: 0x4000, 0x18ae: 0x4000, 0x18af: 0x4000, - 0x18b0: 0x4000, 0x18b1: 0x4000, 0x18b2: 0x4000, 0x18b3: 0x4000, 0x18b4: 0x4000, 0x18b5: 0x4000, - 0x18b6: 0x4000, 0x18b7: 0x4000, 0x18b8: 0x4000, 0x18b9: 0x4000, 0x18ba: 0x4000, - 0x18bc: 0x4000, 0x18bd: 0x4000, 0x18be: 0x4000, 0x18bf: 0x4000, - // Block 0x63, offset 0x18c0 - 0x18c0: 0x4000, 0x18c1: 0x4000, 0x18c2: 0x4000, 0x18c3: 0x4000, 0x18c4: 0x4000, 0x18c5: 0x4000, - 0x18c7: 0x4000, 0x18c8: 0x4000, 0x18c9: 0x4000, 0x18ca: 0x4000, 0x18cb: 0x4000, - 0x18cc: 0x4000, 0x18cd: 0x4000, 0x18ce: 0x4000, 0x18cf: 0x4000, 0x18d0: 0x4000, 0x18d1: 0x4000, - 0x18d2: 0x4000, 0x18d3: 0x4000, 0x18d4: 0x4000, 0x18d5: 0x4000, 0x18d6: 0x4000, 0x18d7: 0x4000, - 0x18d8: 0x4000, 0x18d9: 0x4000, 0x18da: 0x4000, 0x18db: 0x4000, 0x18dc: 0x4000, 0x18dd: 0x4000, - 0x18de: 0x4000, 0x18df: 0x4000, 0x18e0: 0x4000, 0x18e1: 0x4000, 0x18e2: 0x4000, 0x18e3: 0x4000, - 0x18e4: 0x4000, 0x18e5: 0x4000, 0x18e6: 0x4000, 0x18e7: 0x4000, 0x18e8: 0x4000, 0x18e9: 0x4000, - 0x18ea: 0x4000, 0x18eb: 0x4000, 0x18ec: 0x4000, 0x18ed: 0x4000, 0x18ee: 0x4000, 0x18ef: 0x4000, - 0x18f0: 0x4000, 0x18f1: 0x4000, 0x18f2: 0x4000, 0x18f3: 0x4000, 0x18f4: 0x4000, 0x18f5: 0x4000, - 0x18f6: 0x4000, 0x18f7: 0x4000, 0x18f8: 0x4000, 0x18fa: 0x4000, 0x18fb: 0x4000, - 0x18fc: 0x4000, 0x18fd: 0x4000, 0x18fe: 0x4000, 0x18ff: 0x4000, - // Block 0x64, offset 0x1900 - 0x1900: 0x4000, 0x1901: 0x4000, 0x1902: 0x4000, 0x1903: 0x4000, 0x1904: 0x4000, 0x1905: 0x4000, - 0x1906: 0x4000, 0x1907: 0x4000, 0x1908: 0x4000, 0x1909: 0x4000, 0x190a: 0x4000, 0x190b: 0x4000, - 0x190d: 0x4000, 0x190e: 0x4000, 0x190f: 0x4000, 0x1910: 0x4000, 0x1911: 0x4000, - 0x1912: 0x4000, 0x1913: 0x4000, 0x1914: 0x4000, 0x1915: 0x4000, 0x1916: 0x4000, 0x1917: 0x4000, - 0x1918: 0x4000, 0x1919: 0x4000, 0x191a: 0x4000, 0x191b: 0x4000, 0x191c: 0x4000, 0x191d: 0x4000, - 0x191e: 0x4000, 0x191f: 0x4000, 0x1920: 0x4000, 0x1921: 0x4000, 0x1922: 0x4000, 0x1923: 0x4000, - 0x1924: 0x4000, 0x1925: 0x4000, 0x1926: 0x4000, 0x1927: 0x4000, 0x1928: 0x4000, 0x1929: 0x4000, - 0x192a: 0x4000, 0x192b: 0x4000, 0x192c: 0x4000, 0x192d: 0x4000, 0x192e: 0x4000, 0x192f: 0x4000, - 0x1930: 0x4000, 0x1931: 0x4000, 0x1932: 0x4000, 0x1933: 0x4000, 0x1934: 0x4000, 0x1935: 0x4000, - 0x1936: 0x4000, 0x1937: 0x4000, 0x1938: 0x4000, 0x1939: 0x4000, 0x193a: 0x4000, 0x193b: 0x4000, - 0x193c: 0x4000, 0x193d: 0x4000, 0x193e: 0x4000, 0x193f: 0x4000, - // Block 0x65, offset 0x1940 - 0x1970: 0x4000, 0x1971: 0x4000, 0x1972: 0x4000, 0x1973: 0x4000, 0x1974: 0x4000, - 0x1978: 0x4000, 0x1979: 0x4000, 0x197a: 0x4000, - // Block 0x66, offset 0x1980 - 0x1980: 0x4000, 0x1981: 0x4000, 0x1982: 0x4000, 0x1983: 0x4000, 0x1984: 0x4000, 0x1985: 0x4000, - 0x1986: 0x4000, - 0x1990: 0x4000, 0x1991: 0x4000, - 0x1992: 0x4000, 0x1993: 0x4000, 0x1994: 0x4000, 0x1995: 0x4000, 0x1996: 0x4000, 0x1997: 0x4000, - 0x1998: 0x4000, 0x1999: 0x4000, 0x199a: 0x4000, 0x199b: 0x4000, 0x199c: 0x4000, 0x199d: 0x4000, - 0x199e: 0x4000, 0x199f: 0x4000, 0x19a0: 0x4000, 0x19a1: 0x4000, 0x19a2: 0x4000, 0x19a3: 0x4000, - 0x19a4: 0x4000, 0x19a5: 0x4000, 0x19a6: 0x4000, 0x19a7: 0x4000, 0x19a8: 0x4000, - 0x19b0: 0x4000, 0x19b1: 0x4000, 0x19b2: 0x4000, 0x19b3: 0x4000, 0x19b4: 0x4000, 0x19b5: 0x4000, - 0x19b6: 0x4000, - // Block 0x67, offset 0x19c0 - 0x19c0: 0x4000, 0x19c1: 0x4000, 0x19c2: 0x4000, - 0x19d0: 0x4000, 0x19d1: 0x4000, - 0x19d2: 0x4000, 0x19d3: 0x4000, 0x19d4: 0x4000, 0x19d5: 0x4000, 0x19d6: 0x4000, - // Block 0x68, offset 0x1a00 - 0x1a00: 0x2000, 0x1a01: 0x2000, 0x1a02: 0x2000, 0x1a03: 0x2000, 0x1a04: 0x2000, 0x1a05: 0x2000, - 0x1a06: 0x2000, 0x1a07: 0x2000, 0x1a08: 0x2000, 0x1a09: 0x2000, 0x1a0a: 0x2000, 0x1a0b: 0x2000, - 0x1a0c: 0x2000, 0x1a0d: 0x2000, 0x1a0e: 0x2000, 0x1a0f: 0x2000, 0x1a10: 0x2000, 0x1a11: 0x2000, - 0x1a12: 0x2000, 0x1a13: 0x2000, 0x1a14: 0x2000, 0x1a15: 0x2000, 0x1a16: 0x2000, 0x1a17: 0x2000, - 0x1a18: 0x2000, 0x1a19: 0x2000, 0x1a1a: 0x2000, 0x1a1b: 0x2000, 0x1a1c: 0x2000, 0x1a1d: 0x2000, - 0x1a1e: 0x2000, 0x1a1f: 0x2000, 0x1a20: 0x2000, 0x1a21: 0x2000, 0x1a22: 0x2000, 0x1a23: 0x2000, - 0x1a24: 0x2000, 0x1a25: 0x2000, 0x1a26: 0x2000, 0x1a27: 0x2000, 0x1a28: 0x2000, 0x1a29: 0x2000, - 0x1a2a: 0x2000, 0x1a2b: 0x2000, 0x1a2c: 0x2000, 0x1a2d: 0x2000, 0x1a2e: 0x2000, 0x1a2f: 0x2000, - 0x1a30: 0x2000, 0x1a31: 0x2000, 0x1a32: 0x2000, 0x1a33: 0x2000, 0x1a34: 0x2000, 0x1a35: 0x2000, - 0x1a36: 0x2000, 0x1a37: 0x2000, 0x1a38: 0x2000, 0x1a39: 0x2000, 0x1a3a: 0x2000, 0x1a3b: 0x2000, - 0x1a3c: 0x2000, 0x1a3d: 0x2000, -} - -// widthIndex: 22 blocks, 1408 entries, 1408 bytes -// Block 0 is the zero block. -var widthIndex = [1408]uint8{ - // Block 0x0, offset 0x0 - // Block 0x1, offset 0x40 - // Block 0x2, offset 0x80 - // Block 0x3, offset 0xc0 - 0xc2: 0x01, 0xc3: 0x02, 0xc4: 0x03, 0xc5: 0x04, 0xc7: 0x05, - 0xc9: 0x06, 0xcb: 0x07, 0xcc: 0x08, 0xcd: 0x09, 0xce: 0x0a, 0xcf: 0x0b, - 0xd0: 0x0c, 0xd1: 0x0d, - 0xe1: 0x02, 0xe2: 0x03, 0xe3: 0x04, 0xe4: 0x05, 0xe5: 0x06, 0xe6: 0x06, 0xe7: 0x06, - 0xe8: 0x06, 0xe9: 0x06, 0xea: 0x07, 0xeb: 0x06, 0xec: 0x06, 0xed: 0x08, 0xee: 0x09, 0xef: 0x0a, - 0xf0: 0x0f, 0xf3: 0x12, 0xf4: 0x13, - // Block 0x4, offset 0x100 - 0x104: 0x0e, 0x105: 0x0f, - // Block 0x5, offset 0x140 - 0x140: 0x10, 0x141: 0x11, 0x142: 0x12, 0x144: 0x13, 0x145: 0x14, 0x146: 0x15, 0x147: 0x16, - 0x148: 0x17, 0x149: 0x18, 0x14a: 0x19, 0x14c: 0x1a, 0x14f: 0x1b, - 0x151: 0x1c, 0x152: 0x08, 0x153: 0x1d, 0x154: 0x1e, 0x155: 0x1f, 0x156: 0x20, 0x157: 0x21, - 0x158: 0x22, 0x159: 0x23, 0x15a: 0x24, 0x15b: 0x25, 0x15c: 0x26, 0x15d: 0x27, 0x15e: 0x28, 0x15f: 0x29, - 0x166: 0x2a, - 0x16c: 0x2b, 0x16d: 0x2c, - 0x17a: 0x2d, 0x17b: 0x2e, 0x17c: 0x0e, 0x17d: 0x0e, 0x17e: 0x0e, 0x17f: 0x2f, - // Block 0x6, offset 0x180 - 0x180: 0x30, 0x181: 0x31, 0x182: 0x32, 0x183: 0x33, 0x184: 0x34, 0x185: 0x35, 0x186: 0x36, 0x187: 0x37, - 0x188: 0x38, 0x189: 0x39, 0x18a: 0x0e, 0x18b: 0x0e, 0x18c: 0x0e, 0x18d: 0x0e, 0x18e: 0x0e, 0x18f: 0x0e, - 0x190: 0x0e, 0x191: 0x0e, 0x192: 0x0e, 0x193: 0x0e, 0x194: 0x0e, 0x195: 0x0e, 0x196: 0x0e, 0x197: 0x0e, - 0x198: 0x0e, 0x199: 0x0e, 0x19a: 0x0e, 0x19b: 0x0e, 0x19c: 0x0e, 0x19d: 0x0e, 0x19e: 0x0e, 0x19f: 0x0e, - 0x1a0: 0x0e, 0x1a1: 0x0e, 0x1a2: 0x0e, 0x1a3: 0x0e, 0x1a4: 0x0e, 0x1a5: 0x0e, 0x1a6: 0x0e, 0x1a7: 0x0e, - 0x1a8: 0x0e, 0x1a9: 0x0e, 0x1aa: 0x0e, 0x1ab: 0x0e, 0x1ac: 0x0e, 0x1ad: 0x0e, 0x1ae: 0x0e, 0x1af: 0x0e, - 0x1b0: 0x0e, 0x1b1: 0x0e, 0x1b2: 0x0e, 0x1b3: 0x0e, 0x1b4: 0x0e, 0x1b5: 0x0e, 0x1b6: 0x0e, 0x1b7: 0x0e, - 0x1b8: 0x0e, 0x1b9: 0x0e, 0x1ba: 0x0e, 0x1bb: 0x0e, 0x1bc: 0x0e, 0x1bd: 0x0e, 0x1be: 0x0e, 0x1bf: 0x0e, - // Block 0x7, offset 0x1c0 - 0x1c0: 0x0e, 0x1c1: 0x0e, 0x1c2: 0x0e, 0x1c3: 0x0e, 0x1c4: 0x0e, 0x1c5: 0x0e, 0x1c6: 0x0e, 0x1c7: 0x0e, - 0x1c8: 0x0e, 0x1c9: 0x0e, 0x1ca: 0x0e, 0x1cb: 0x0e, 0x1cc: 0x0e, 0x1cd: 0x0e, 0x1ce: 0x0e, 0x1cf: 0x0e, - 0x1d0: 0x0e, 0x1d1: 0x0e, 0x1d2: 0x0e, 0x1d3: 0x0e, 0x1d4: 0x0e, 0x1d5: 0x0e, 0x1d6: 0x0e, 0x1d7: 0x0e, - 0x1d8: 0x0e, 0x1d9: 0x0e, 0x1da: 0x0e, 0x1db: 0x0e, 0x1dc: 0x0e, 0x1dd: 0x0e, 0x1de: 0x0e, 0x1df: 0x0e, - 0x1e0: 0x0e, 0x1e1: 0x0e, 0x1e2: 0x0e, 0x1e3: 0x0e, 0x1e4: 0x0e, 0x1e5: 0x0e, 0x1e6: 0x0e, 0x1e7: 0x0e, - 0x1e8: 0x0e, 0x1e9: 0x0e, 0x1ea: 0x0e, 0x1eb: 0x0e, 0x1ec: 0x0e, 0x1ed: 0x0e, 0x1ee: 0x0e, 0x1ef: 0x0e, - 0x1f0: 0x0e, 0x1f1: 0x0e, 0x1f2: 0x0e, 0x1f3: 0x0e, 0x1f4: 0x0e, 0x1f5: 0x0e, 0x1f6: 0x0e, - 0x1f8: 0x0e, 0x1f9: 0x0e, 0x1fa: 0x0e, 0x1fb: 0x0e, 0x1fc: 0x0e, 0x1fd: 0x0e, 0x1fe: 0x0e, 0x1ff: 0x0e, - // Block 0x8, offset 0x200 - 0x200: 0x0e, 0x201: 0x0e, 0x202: 0x0e, 0x203: 0x0e, 0x204: 0x0e, 0x205: 0x0e, 0x206: 0x0e, 0x207: 0x0e, - 0x208: 0x0e, 0x209: 0x0e, 0x20a: 0x0e, 0x20b: 0x0e, 0x20c: 0x0e, 0x20d: 0x0e, 0x20e: 0x0e, 0x20f: 0x0e, - 0x210: 0x0e, 0x211: 0x0e, 0x212: 0x0e, 0x213: 0x0e, 0x214: 0x0e, 0x215: 0x0e, 0x216: 0x0e, 0x217: 0x0e, - 0x218: 0x0e, 0x219: 0x0e, 0x21a: 0x0e, 0x21b: 0x0e, 0x21c: 0x0e, 0x21d: 0x0e, 0x21e: 0x0e, 0x21f: 0x0e, - 0x220: 0x0e, 0x221: 0x0e, 0x222: 0x0e, 0x223: 0x0e, 0x224: 0x0e, 0x225: 0x0e, 0x226: 0x0e, 0x227: 0x0e, - 0x228: 0x0e, 0x229: 0x0e, 0x22a: 0x0e, 0x22b: 0x0e, 0x22c: 0x0e, 0x22d: 0x0e, 0x22e: 0x0e, 0x22f: 0x0e, - 0x230: 0x0e, 0x231: 0x0e, 0x232: 0x0e, 0x233: 0x0e, 0x234: 0x0e, 0x235: 0x0e, 0x236: 0x0e, 0x237: 0x0e, - 0x238: 0x0e, 0x239: 0x0e, 0x23a: 0x0e, 0x23b: 0x0e, 0x23c: 0x0e, 0x23d: 0x0e, 0x23e: 0x0e, 0x23f: 0x0e, - // Block 0x9, offset 0x240 - 0x240: 0x0e, 0x241: 0x0e, 0x242: 0x0e, 0x243: 0x0e, 0x244: 0x0e, 0x245: 0x0e, 0x246: 0x0e, 0x247: 0x0e, - 0x248: 0x0e, 0x249: 0x0e, 0x24a: 0x0e, 0x24b: 0x0e, 0x24c: 0x0e, 0x24d: 0x0e, 0x24e: 0x0e, 0x24f: 0x0e, - 0x250: 0x0e, 0x251: 0x0e, 0x252: 0x3a, 0x253: 0x3b, - 0x265: 0x3c, - 0x270: 0x0e, 0x271: 0x0e, 0x272: 0x0e, 0x273: 0x0e, 0x274: 0x0e, 0x275: 0x0e, 0x276: 0x0e, 0x277: 0x0e, - 0x278: 0x0e, 0x279: 0x0e, 0x27a: 0x0e, 0x27b: 0x0e, 0x27c: 0x0e, 0x27d: 0x0e, 0x27e: 0x0e, 0x27f: 0x0e, - // Block 0xa, offset 0x280 - 0x280: 0x0e, 0x281: 0x0e, 0x282: 0x0e, 0x283: 0x0e, 0x284: 0x0e, 0x285: 0x0e, 0x286: 0x0e, 0x287: 0x0e, - 0x288: 0x0e, 0x289: 0x0e, 0x28a: 0x0e, 0x28b: 0x0e, 0x28c: 0x0e, 0x28d: 0x0e, 0x28e: 0x0e, 0x28f: 0x0e, - 0x290: 0x0e, 0x291: 0x0e, 0x292: 0x0e, 0x293: 0x0e, 0x294: 0x0e, 0x295: 0x0e, 0x296: 0x0e, 0x297: 0x0e, - 0x298: 0x0e, 0x299: 0x0e, 0x29a: 0x0e, 0x29b: 0x0e, 0x29c: 0x0e, 0x29d: 0x0e, 0x29e: 0x3d, - // Block 0xb, offset 0x2c0 - 0x2c0: 0x08, 0x2c1: 0x08, 0x2c2: 0x08, 0x2c3: 0x08, 0x2c4: 0x08, 0x2c5: 0x08, 0x2c6: 0x08, 0x2c7: 0x08, - 0x2c8: 0x08, 0x2c9: 0x08, 0x2ca: 0x08, 0x2cb: 0x08, 0x2cc: 0x08, 0x2cd: 0x08, 0x2ce: 0x08, 0x2cf: 0x08, - 0x2d0: 0x08, 0x2d1: 0x08, 0x2d2: 0x08, 0x2d3: 0x08, 0x2d4: 0x08, 0x2d5: 0x08, 0x2d6: 0x08, 0x2d7: 0x08, - 0x2d8: 0x08, 0x2d9: 0x08, 0x2da: 0x08, 0x2db: 0x08, 0x2dc: 0x08, 0x2dd: 0x08, 0x2de: 0x08, 0x2df: 0x08, - 0x2e0: 0x08, 0x2e1: 0x08, 0x2e2: 0x08, 0x2e3: 0x08, 0x2e4: 0x08, 0x2e5: 0x08, 0x2e6: 0x08, 0x2e7: 0x08, - 0x2e8: 0x08, 0x2e9: 0x08, 0x2ea: 0x08, 0x2eb: 0x08, 0x2ec: 0x08, 0x2ed: 0x08, 0x2ee: 0x08, 0x2ef: 0x08, - 0x2f0: 0x08, 0x2f1: 0x08, 0x2f2: 0x08, 0x2f3: 0x08, 0x2f4: 0x08, 0x2f5: 0x08, 0x2f6: 0x08, 0x2f7: 0x08, - 0x2f8: 0x08, 0x2f9: 0x08, 0x2fa: 0x08, 0x2fb: 0x08, 0x2fc: 0x08, 0x2fd: 0x08, 0x2fe: 0x08, 0x2ff: 0x08, - // Block 0xc, offset 0x300 - 0x300: 0x08, 0x301: 0x08, 0x302: 0x08, 0x303: 0x08, 0x304: 0x08, 0x305: 0x08, 0x306: 0x08, 0x307: 0x08, - 0x308: 0x08, 0x309: 0x08, 0x30a: 0x08, 0x30b: 0x08, 0x30c: 0x08, 0x30d: 0x08, 0x30e: 0x08, 0x30f: 0x08, - 0x310: 0x08, 0x311: 0x08, 0x312: 0x08, 0x313: 0x08, 0x314: 0x08, 0x315: 0x08, 0x316: 0x08, 0x317: 0x08, - 0x318: 0x08, 0x319: 0x08, 0x31a: 0x08, 0x31b: 0x08, 0x31c: 0x08, 0x31d: 0x08, 0x31e: 0x08, 0x31f: 0x08, - 0x320: 0x08, 0x321: 0x08, 0x322: 0x08, 0x323: 0x08, 0x324: 0x0e, 0x325: 0x0e, 0x326: 0x0e, 0x327: 0x0e, - 0x328: 0x0e, 0x329: 0x0e, 0x32a: 0x0e, 0x32b: 0x0e, - 0x338: 0x3e, 0x339: 0x3f, 0x33c: 0x40, 0x33d: 0x41, 0x33e: 0x42, 0x33f: 0x43, - // Block 0xd, offset 0x340 - 0x37f: 0x44, - // Block 0xe, offset 0x380 - 0x380: 0x0e, 0x381: 0x0e, 0x382: 0x0e, 0x383: 0x0e, 0x384: 0x0e, 0x385: 0x0e, 0x386: 0x0e, 0x387: 0x0e, - 0x388: 0x0e, 0x389: 0x0e, 0x38a: 0x0e, 0x38b: 0x0e, 0x38c: 0x0e, 0x38d: 0x0e, 0x38e: 0x0e, 0x38f: 0x0e, - 0x390: 0x0e, 0x391: 0x0e, 0x392: 0x0e, 0x393: 0x0e, 0x394: 0x0e, 0x395: 0x0e, 0x396: 0x0e, 0x397: 0x0e, - 0x398: 0x0e, 0x399: 0x0e, 0x39a: 0x0e, 0x39b: 0x0e, 0x39c: 0x0e, 0x39d: 0x0e, 0x39e: 0x0e, 0x39f: 0x45, - 0x3a0: 0x0e, 0x3a1: 0x0e, 0x3a2: 0x0e, 0x3a3: 0x0e, 0x3a4: 0x0e, 0x3a5: 0x0e, 0x3a6: 0x0e, 0x3a7: 0x0e, - 0x3a8: 0x0e, 0x3a9: 0x0e, 0x3aa: 0x0e, 0x3ab: 0x0e, 0x3ac: 0x0e, 0x3ad: 0x0e, 0x3ae: 0x0e, 0x3af: 0x0e, - 0x3b0: 0x0e, 0x3b1: 0x0e, 0x3b2: 0x0e, 0x3b3: 0x46, 0x3b4: 0x47, - // Block 0xf, offset 0x3c0 - 0x3c0: 0x0e, 0x3c1: 0x0e, 0x3c2: 0x0e, 0x3c3: 0x0e, 0x3c4: 0x48, 0x3c5: 0x49, 0x3c6: 0x0e, 0x3c7: 0x0e, - 0x3c8: 0x0e, 0x3c9: 0x0e, 0x3ca: 0x0e, 0x3cb: 0x4a, - // Block 0x10, offset 0x400 - 0x400: 0x4b, 0x403: 0x4c, 0x404: 0x4d, 0x405: 0x4e, 0x406: 0x4f, - 0x408: 0x50, 0x409: 0x51, 0x40c: 0x52, 0x40d: 0x53, 0x40e: 0x54, 0x40f: 0x55, - 0x410: 0x56, 0x411: 0x57, 0x412: 0x0e, 0x413: 0x58, 0x414: 0x59, 0x415: 0x5a, 0x416: 0x5b, 0x417: 0x5c, - 0x418: 0x0e, 0x419: 0x5d, 0x41a: 0x0e, 0x41b: 0x5e, 0x41f: 0x5f, - 0x424: 0x60, 0x425: 0x61, 0x426: 0x0e, 0x427: 0x62, - 0x429: 0x63, 0x42a: 0x64, 0x42b: 0x65, - // Block 0x11, offset 0x440 - 0x456: 0x0b, 0x457: 0x06, - 0x458: 0x0c, 0x45b: 0x0d, 0x45f: 0x0e, - 0x460: 0x06, 0x461: 0x06, 0x462: 0x06, 0x463: 0x06, 0x464: 0x06, 0x465: 0x06, 0x466: 0x06, 0x467: 0x06, - 0x468: 0x06, 0x469: 0x06, 0x46a: 0x06, 0x46b: 0x06, 0x46c: 0x06, 0x46d: 0x06, 0x46e: 0x06, 0x46f: 0x06, - 0x470: 0x06, 0x471: 0x06, 0x472: 0x06, 0x473: 0x06, 0x474: 0x06, 0x475: 0x06, 0x476: 0x06, 0x477: 0x06, - 0x478: 0x06, 0x479: 0x06, 0x47a: 0x06, 0x47b: 0x06, 0x47c: 0x06, 0x47d: 0x06, 0x47e: 0x06, 0x47f: 0x06, - // Block 0x12, offset 0x480 - 0x484: 0x08, 0x485: 0x08, 0x486: 0x08, 0x487: 0x09, - // Block 0x13, offset 0x4c0 - 0x4c0: 0x08, 0x4c1: 0x08, 0x4c2: 0x08, 0x4c3: 0x08, 0x4c4: 0x08, 0x4c5: 0x08, 0x4c6: 0x08, 0x4c7: 0x08, - 0x4c8: 0x08, 0x4c9: 0x08, 0x4ca: 0x08, 0x4cb: 0x08, 0x4cc: 0x08, 0x4cd: 0x08, 0x4ce: 0x08, 0x4cf: 0x08, - 0x4d0: 0x08, 0x4d1: 0x08, 0x4d2: 0x08, 0x4d3: 0x08, 0x4d4: 0x08, 0x4d5: 0x08, 0x4d6: 0x08, 0x4d7: 0x08, - 0x4d8: 0x08, 0x4d9: 0x08, 0x4da: 0x08, 0x4db: 0x08, 0x4dc: 0x08, 0x4dd: 0x08, 0x4de: 0x08, 0x4df: 0x08, - 0x4e0: 0x08, 0x4e1: 0x08, 0x4e2: 0x08, 0x4e3: 0x08, 0x4e4: 0x08, 0x4e5: 0x08, 0x4e6: 0x08, 0x4e7: 0x08, - 0x4e8: 0x08, 0x4e9: 0x08, 0x4ea: 0x08, 0x4eb: 0x08, 0x4ec: 0x08, 0x4ed: 0x08, 0x4ee: 0x08, 0x4ef: 0x08, - 0x4f0: 0x08, 0x4f1: 0x08, 0x4f2: 0x08, 0x4f3: 0x08, 0x4f4: 0x08, 0x4f5: 0x08, 0x4f6: 0x08, 0x4f7: 0x08, - 0x4f8: 0x08, 0x4f9: 0x08, 0x4fa: 0x08, 0x4fb: 0x08, 0x4fc: 0x08, 0x4fd: 0x08, 0x4fe: 0x08, 0x4ff: 0x66, - // Block 0x14, offset 0x500 - 0x520: 0x10, - 0x530: 0x09, 0x531: 0x09, 0x532: 0x09, 0x533: 0x09, 0x534: 0x09, 0x535: 0x09, 0x536: 0x09, 0x537: 0x09, - 0x538: 0x09, 0x539: 0x09, 0x53a: 0x09, 0x53b: 0x09, 0x53c: 0x09, 0x53d: 0x09, 0x53e: 0x09, 0x53f: 0x11, - // Block 0x15, offset 0x540 - 0x540: 0x09, 0x541: 0x09, 0x542: 0x09, 0x543: 0x09, 0x544: 0x09, 0x545: 0x09, 0x546: 0x09, 0x547: 0x09, - 0x548: 0x09, 0x549: 0x09, 0x54a: 0x09, 0x54b: 0x09, 0x54c: 0x09, 0x54d: 0x09, 0x54e: 0x09, 0x54f: 0x11, -} - -// inverseData contains 4-byte entries of the following format: -// -// <0 padding> -// -// The last byte of the UTF-8-encoded rune is xor-ed with the last byte of the -// UTF-8 encoding of the original rune. Mappings often have the following -// pattern: -// -// Ａ -> A (U+FF21 -> U+0041) -// Ｂ -> B (U+FF22 -> U+0042) -// ... -// -// By xor-ing the last byte the same entry can be shared by many mappings. This -// reduces the total number of distinct entries by about two thirds. -// The resulting entry for the aforementioned mappings is -// -// { 0x01, 0xE0, 0x00, 0x00 } -// -// Using this entry to map U+FF21 (UTF-8 [EF BC A1]), we get -// -// E0 ^ A1 = 41. -// -// Similarly, for U+FF22 (UTF-8 [EF BC A2]), we get -// -// E0 ^ A2 = 42. -// -// Note that because of the xor-ing, the byte sequence stored in the entry is -// not valid UTF-8. -var inverseData = [150][4]byte{ - {0x00, 0x00, 0x00, 0x00}, - {0x03, 0xe3, 0x80, 0xa0}, - {0x03, 0xef, 0xbc, 0xa0}, - {0x03, 0xef, 0xbc, 0xe0}, - {0x03, 0xef, 0xbd, 0xe0}, - {0x03, 0xef, 0xbf, 0x02}, - {0x03, 0xef, 0xbf, 0x00}, - {0x03, 0xef, 0xbf, 0x0e}, - {0x03, 0xef, 0xbf, 0x0c}, - {0x03, 0xef, 0xbf, 0x0f}, - {0x03, 0xef, 0xbf, 0x39}, - {0x03, 0xef, 0xbf, 0x3b}, - {0x03, 0xef, 0xbf, 0x3f}, - {0x03, 0xef, 0xbf, 0x2a}, - {0x03, 0xef, 0xbf, 0x0d}, - {0x03, 0xef, 0xbf, 0x25}, - {0x03, 0xef, 0xbd, 0x1a}, - {0x03, 0xef, 0xbd, 0x26}, - {0x01, 0xa0, 0x00, 0x00}, - {0x03, 0xef, 0xbd, 0x25}, - {0x03, 0xef, 0xbd, 0x23}, - {0x03, 0xef, 0xbd, 0x2e}, - {0x03, 0xef, 0xbe, 0x07}, - {0x03, 0xef, 0xbe, 0x05}, - {0x03, 0xef, 0xbd, 0x06}, - {0x03, 0xef, 0xbd, 0x13}, - {0x03, 0xef, 0xbd, 0x0b}, - {0x03, 0xef, 0xbd, 0x16}, - {0x03, 0xef, 0xbd, 0x0c}, - {0x03, 0xef, 0xbd, 0x15}, - {0x03, 0xef, 0xbd, 0x0d}, - {0x03, 0xef, 0xbd, 0x1c}, - {0x03, 0xef, 0xbd, 0x02}, - {0x03, 0xef, 0xbd, 0x1f}, - {0x03, 0xef, 0xbd, 0x1d}, - {0x03, 0xef, 0xbd, 0x17}, - {0x03, 0xef, 0xbd, 0x08}, - {0x03, 0xef, 0xbd, 0x09}, - {0x03, 0xef, 0xbd, 0x0e}, - {0x03, 0xef, 0xbd, 0x04}, - {0x03, 0xef, 0xbd, 0x05}, - {0x03, 0xef, 0xbe, 0x3f}, - {0x03, 0xef, 0xbe, 0x00}, - {0x03, 0xef, 0xbd, 0x2c}, - {0x03, 0xef, 0xbe, 0x06}, - {0x03, 0xef, 0xbe, 0x0c}, - {0x03, 0xef, 0xbe, 0x0f}, - {0x03, 0xef, 0xbe, 0x0d}, - {0x03, 0xef, 0xbe, 0x0b}, - {0x03, 0xef, 0xbe, 0x19}, - {0x03, 0xef, 0xbe, 0x15}, - {0x03, 0xef, 0xbe, 0x11}, - {0x03, 0xef, 0xbe, 0x31}, - {0x03, 0xef, 0xbe, 0x33}, - {0x03, 0xef, 0xbd, 0x0f}, - {0x03, 0xef, 0xbe, 0x30}, - {0x03, 0xef, 0xbe, 0x3e}, - {0x03, 0xef, 0xbe, 0x32}, - {0x03, 0xef, 0xbe, 0x36}, - {0x03, 0xef, 0xbd, 0x14}, - {0x03, 0xef, 0xbe, 0x2e}, - {0x03, 0xef, 0xbd, 0x1e}, - {0x03, 0xef, 0xbe, 0x10}, - {0x03, 0xef, 0xbf, 0x13}, - {0x03, 0xef, 0xbf, 0x15}, - {0x03, 0xef, 0xbf, 0x17}, - {0x03, 0xef, 0xbf, 0x1f}, - {0x03, 0xef, 0xbf, 0x1d}, - {0x03, 0xef, 0xbf, 0x1b}, - {0x03, 0xef, 0xbf, 0x09}, - {0x03, 0xef, 0xbf, 0x0b}, - {0x03, 0xef, 0xbf, 0x37}, - {0x03, 0xef, 0xbe, 0x04}, - {0x01, 0xe0, 0x00, 0x00}, - {0x03, 0xe2, 0xa6, 0x1a}, - {0x03, 0xe2, 0xa6, 0x26}, - {0x03, 0xe3, 0x80, 0x23}, - {0x03, 0xe3, 0x80, 0x2e}, - {0x03, 0xe3, 0x80, 0x25}, - {0x03, 0xe3, 0x83, 0x1e}, - {0x03, 0xe3, 0x83, 0x14}, - {0x03, 0xe3, 0x82, 0x06}, - {0x03, 0xe3, 0x82, 0x0b}, - {0x03, 0xe3, 0x82, 0x0c}, - {0x03, 0xe3, 0x82, 0x0d}, - {0x03, 0xe3, 0x82, 0x02}, - {0x03, 0xe3, 0x83, 0x0f}, - {0x03, 0xe3, 0x83, 0x08}, - {0x03, 0xe3, 0x83, 0x09}, - {0x03, 0xe3, 0x83, 0x2c}, - {0x03, 0xe3, 0x83, 0x0c}, - {0x03, 0xe3, 0x82, 0x13}, - {0x03, 0xe3, 0x82, 0x16}, - {0x03, 0xe3, 0x82, 0x15}, - {0x03, 0xe3, 0x82, 0x1c}, - {0x03, 0xe3, 0x82, 0x1f}, - {0x03, 0xe3, 0x82, 0x1d}, - {0x03, 0xe3, 0x82, 0x1a}, - {0x03, 0xe3, 0x82, 0x17}, - {0x03, 0xe3, 0x82, 0x08}, - {0x03, 0xe3, 0x82, 0x09}, - {0x03, 0xe3, 0x82, 0x0e}, - {0x03, 0xe3, 0x82, 0x04}, - {0x03, 0xe3, 0x82, 0x05}, - {0x03, 0xe3, 0x82, 0x3f}, - {0x03, 0xe3, 0x83, 0x00}, - {0x03, 0xe3, 0x83, 0x06}, - {0x03, 0xe3, 0x83, 0x05}, - {0x03, 0xe3, 0x83, 0x0d}, - {0x03, 0xe3, 0x83, 0x0b}, - {0x03, 0xe3, 0x83, 0x07}, - {0x03, 0xe3, 0x83, 0x19}, - {0x03, 0xe3, 0x83, 0x15}, - {0x03, 0xe3, 0x83, 0x11}, - {0x03, 0xe3, 0x83, 0x31}, - {0x03, 0xe3, 0x83, 0x33}, - {0x03, 0xe3, 0x83, 0x30}, - {0x03, 0xe3, 0x83, 0x3e}, - {0x03, 0xe3, 0x83, 0x32}, - {0x03, 0xe3, 0x83, 0x36}, - {0x03, 0xe3, 0x83, 0x2e}, - {0x03, 0xe3, 0x82, 0x07}, - {0x03, 0xe3, 0x85, 0x04}, - {0x03, 0xe3, 0x84, 0x10}, - {0x03, 0xe3, 0x85, 0x30}, - {0x03, 0xe3, 0x85, 0x0d}, - {0x03, 0xe3, 0x85, 0x13}, - {0x03, 0xe3, 0x85, 0x15}, - {0x03, 0xe3, 0x85, 0x17}, - {0x03, 0xe3, 0x85, 0x1f}, - {0x03, 0xe3, 0x85, 0x1d}, - {0x03, 0xe3, 0x85, 0x1b}, - {0x03, 0xe3, 0x85, 0x09}, - {0x03, 0xe3, 0x85, 0x0f}, - {0x03, 0xe3, 0x85, 0x0b}, - {0x03, 0xe3, 0x85, 0x37}, - {0x03, 0xe3, 0x85, 0x3b}, - {0x03, 0xe3, 0x85, 0x39}, - {0x03, 0xe3, 0x85, 0x3f}, - {0x02, 0xc2, 0x02, 0x00}, - {0x02, 0xc2, 0x0e, 0x00}, - {0x02, 0xc2, 0x0c, 0x00}, - {0x02, 0xc2, 0x00, 0x00}, - {0x03, 0xe2, 0x82, 0x0f}, - {0x03, 0xe2, 0x94, 0x2a}, - {0x03, 0xe2, 0x86, 0x39}, - {0x03, 0xe2, 0x86, 0x3b}, - {0x03, 0xe2, 0x86, 0x3f}, - {0x03, 0xe2, 0x96, 0x0d}, - {0x03, 0xe2, 0x97, 0x25}, -} - -// Total table size 15448 bytes (15KiB) diff --git a/vendor/golang.org/x/text/width/tables15.0.0.go b/vendor/golang.org/x/text/width/tables15.0.0.go deleted file mode 100644 index 4b91e3384d..0000000000 --- a/vendor/golang.org/x/text/width/tables15.0.0.go +++ /dev/null @@ -1,1368 +0,0 @@ -// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. - -//go:build go1.21 -// +build go1.21 - -package width - -// UnicodeVersion is the Unicode version from which the tables in this package are derived. -const UnicodeVersion = "15.0.0" - -// lookup returns the trie value for the first UTF-8 encoding in s and -// the width in bytes of this encoding. The size will be 0 if s does not -// hold enough bytes to complete the encoding. len(s) must be greater than 0. -func (t *widthTrie) lookup(s []byte) (v uint16, sz int) { - c0 := s[0] - switch { - case c0 < 0x80: // is ASCII - return widthValues[c0], 1 - case c0 < 0xC2: - return 0, 1 // Illegal UTF-8: not a starter, not ASCII. - case c0 < 0xE0: // 2-byte UTF-8 - if len(s) < 2 { - return 0, 0 - } - i := widthIndex[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c1), 2 - case c0 < 0xF0: // 3-byte UTF-8 - if len(s) < 3 { - return 0, 0 - } - i := widthIndex[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - o := uint32(i)<<6 + uint32(c1) - i = widthIndex[o] - c2 := s[2] - if c2 < 0x80 || 0xC0 <= c2 { - return 0, 2 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c2), 3 - case c0 < 0xF8: // 4-byte UTF-8 - if len(s) < 4 { - return 0, 0 - } - i := widthIndex[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - o := uint32(i)<<6 + uint32(c1) - i = widthIndex[o] - c2 := s[2] - if c2 < 0x80 || 0xC0 <= c2 { - return 0, 2 // Illegal UTF-8: not a continuation byte. - } - o = uint32(i)<<6 + uint32(c2) - i = widthIndex[o] - c3 := s[3] - if c3 < 0x80 || 0xC0 <= c3 { - return 0, 3 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c3), 4 - } - // Illegal rune - return 0, 1 -} - -// lookupUnsafe returns the trie value for the first UTF-8 encoding in s. -// s must start with a full and valid UTF-8 encoded rune. -func (t *widthTrie) lookupUnsafe(s []byte) uint16 { - c0 := s[0] - if c0 < 0x80 { // is ASCII - return widthValues[c0] - } - i := widthIndex[c0] - if c0 < 0xE0 { // 2-byte UTF-8 - return t.lookupValue(uint32(i), s[1]) - } - i = widthIndex[uint32(i)<<6+uint32(s[1])] - if c0 < 0xF0 { // 3-byte UTF-8 - return t.lookupValue(uint32(i), s[2]) - } - i = widthIndex[uint32(i)<<6+uint32(s[2])] - if c0 < 0xF8 { // 4-byte UTF-8 - return t.lookupValue(uint32(i), s[3]) - } - return 0 -} - -// lookupString returns the trie value for the first UTF-8 encoding in s and -// the width in bytes of this encoding. The size will be 0 if s does not -// hold enough bytes to complete the encoding. len(s) must be greater than 0. -func (t *widthTrie) lookupString(s string) (v uint16, sz int) { - c0 := s[0] - switch { - case c0 < 0x80: // is ASCII - return widthValues[c0], 1 - case c0 < 0xC2: - return 0, 1 // Illegal UTF-8: not a starter, not ASCII. - case c0 < 0xE0: // 2-byte UTF-8 - if len(s) < 2 { - return 0, 0 - } - i := widthIndex[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c1), 2 - case c0 < 0xF0: // 3-byte UTF-8 - if len(s) < 3 { - return 0, 0 - } - i := widthIndex[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - o := uint32(i)<<6 + uint32(c1) - i = widthIndex[o] - c2 := s[2] - if c2 < 0x80 || 0xC0 <= c2 { - return 0, 2 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c2), 3 - case c0 < 0xF8: // 4-byte UTF-8 - if len(s) < 4 { - return 0, 0 - } - i := widthIndex[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - o := uint32(i)<<6 + uint32(c1) - i = widthIndex[o] - c2 := s[2] - if c2 < 0x80 || 0xC0 <= c2 { - return 0, 2 // Illegal UTF-8: not a continuation byte. - } - o = uint32(i)<<6 + uint32(c2) - i = widthIndex[o] - c3 := s[3] - if c3 < 0x80 || 0xC0 <= c3 { - return 0, 3 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c3), 4 - } - // Illegal rune - return 0, 1 -} - -// lookupStringUnsafe returns the trie value for the first UTF-8 encoding in s. -// s must start with a full and valid UTF-8 encoded rune. -func (t *widthTrie) lookupStringUnsafe(s string) uint16 { - c0 := s[0] - if c0 < 0x80 { // is ASCII - return widthValues[c0] - } - i := widthIndex[c0] - if c0 < 0xE0 { // 2-byte UTF-8 - return t.lookupValue(uint32(i), s[1]) - } - i = widthIndex[uint32(i)<<6+uint32(s[1])] - if c0 < 0xF0 { // 3-byte UTF-8 - return t.lookupValue(uint32(i), s[2]) - } - i = widthIndex[uint32(i)<<6+uint32(s[2])] - if c0 < 0xF8 { // 4-byte UTF-8 - return t.lookupValue(uint32(i), s[3]) - } - return 0 -} - -// widthTrie. Total size: 14912 bytes (14.56 KiB). Checksum: 4468b6cd178303d2. -type widthTrie struct{} - -func newWidthTrie(i int) *widthTrie { - return &widthTrie{} -} - -// lookupValue determines the type of block n and looks up the value for b. -func (t *widthTrie) lookupValue(n uint32, b byte) uint16 { - switch { - default: - return uint16(widthValues[n<<6+uint32(b)]) - } -} - -// widthValues: 105 blocks, 6720 entries, 13440 bytes -// The third block is the zero block. -var widthValues = [6720]uint16{ - // Block 0x0, offset 0x0 - 0x20: 0x6001, 0x21: 0x6002, 0x22: 0x6002, 0x23: 0x6002, - 0x24: 0x6002, 0x25: 0x6002, 0x26: 0x6002, 0x27: 0x6002, 0x28: 0x6002, 0x29: 0x6002, - 0x2a: 0x6002, 0x2b: 0x6002, 0x2c: 0x6002, 0x2d: 0x6002, 0x2e: 0x6002, 0x2f: 0x6002, - 0x30: 0x6002, 0x31: 0x6002, 0x32: 0x6002, 0x33: 0x6002, 0x34: 0x6002, 0x35: 0x6002, - 0x36: 0x6002, 0x37: 0x6002, 0x38: 0x6002, 0x39: 0x6002, 0x3a: 0x6002, 0x3b: 0x6002, - 0x3c: 0x6002, 0x3d: 0x6002, 0x3e: 0x6002, 0x3f: 0x6002, - // Block 0x1, offset 0x40 - 0x40: 0x6003, 0x41: 0x6003, 0x42: 0x6003, 0x43: 0x6003, 0x44: 0x6003, 0x45: 0x6003, - 0x46: 0x6003, 0x47: 0x6003, 0x48: 0x6003, 0x49: 0x6003, 0x4a: 0x6003, 0x4b: 0x6003, - 0x4c: 0x6003, 0x4d: 0x6003, 0x4e: 0x6003, 0x4f: 0x6003, 0x50: 0x6003, 0x51: 0x6003, - 0x52: 0x6003, 0x53: 0x6003, 0x54: 0x6003, 0x55: 0x6003, 0x56: 0x6003, 0x57: 0x6003, - 0x58: 0x6003, 0x59: 0x6003, 0x5a: 0x6003, 0x5b: 0x6003, 0x5c: 0x6003, 0x5d: 0x6003, - 0x5e: 0x6003, 0x5f: 0x6003, 0x60: 0x6004, 0x61: 0x6004, 0x62: 0x6004, 0x63: 0x6004, - 0x64: 0x6004, 0x65: 0x6004, 0x66: 0x6004, 0x67: 0x6004, 0x68: 0x6004, 0x69: 0x6004, - 0x6a: 0x6004, 0x6b: 0x6004, 0x6c: 0x6004, 0x6d: 0x6004, 0x6e: 0x6004, 0x6f: 0x6004, - 0x70: 0x6004, 0x71: 0x6004, 0x72: 0x6004, 0x73: 0x6004, 0x74: 0x6004, 0x75: 0x6004, - 0x76: 0x6004, 0x77: 0x6004, 0x78: 0x6004, 0x79: 0x6004, 0x7a: 0x6004, 0x7b: 0x6004, - 0x7c: 0x6004, 0x7d: 0x6004, 0x7e: 0x6004, - // Block 0x2, offset 0x80 - // Block 0x3, offset 0xc0 - 0xe1: 0x2000, 0xe2: 0x6005, 0xe3: 0x6005, - 0xe4: 0x2000, 0xe5: 0x6006, 0xe6: 0x6005, 0xe7: 0x2000, 0xe8: 0x2000, - 0xea: 0x2000, 0xec: 0x6007, 0xed: 0x2000, 0xee: 0x2000, 0xef: 0x6008, - 0xf0: 0x2000, 0xf1: 0x2000, 0xf2: 0x2000, 0xf3: 0x2000, 0xf4: 0x2000, - 0xf6: 0x2000, 0xf7: 0x2000, 0xf8: 0x2000, 0xf9: 0x2000, 0xfa: 0x2000, - 0xfc: 0x2000, 0xfd: 0x2000, 0xfe: 0x2000, 0xff: 0x2000, - // Block 0x4, offset 0x100 - 0x106: 0x2000, - 0x110: 0x2000, - 0x117: 0x2000, - 0x118: 0x2000, - 0x11e: 0x2000, 0x11f: 0x2000, 0x120: 0x2000, 0x121: 0x2000, - 0x126: 0x2000, 0x128: 0x2000, 0x129: 0x2000, - 0x12a: 0x2000, 0x12c: 0x2000, 0x12d: 0x2000, - 0x130: 0x2000, 0x132: 0x2000, 0x133: 0x2000, - 0x137: 0x2000, 0x138: 0x2000, 0x139: 0x2000, 0x13a: 0x2000, - 0x13c: 0x2000, 0x13e: 0x2000, - // Block 0x5, offset 0x140 - 0x141: 0x2000, - 0x151: 0x2000, - 0x153: 0x2000, - 0x15b: 0x2000, - 0x166: 0x2000, 0x167: 0x2000, - 0x16b: 0x2000, - 0x171: 0x2000, 0x172: 0x2000, 0x173: 0x2000, - 0x178: 0x2000, - 0x17f: 0x2000, - // Block 0x6, offset 0x180 - 0x180: 0x2000, 0x181: 0x2000, 0x182: 0x2000, 0x184: 0x2000, - 0x188: 0x2000, 0x189: 0x2000, 0x18a: 0x2000, 0x18b: 0x2000, - 0x18d: 0x2000, - 0x192: 0x2000, 0x193: 0x2000, - 0x1a6: 0x2000, 0x1a7: 0x2000, - 0x1ab: 0x2000, - // Block 0x7, offset 0x1c0 - 0x1ce: 0x2000, 0x1d0: 0x2000, - 0x1d2: 0x2000, 0x1d4: 0x2000, 0x1d6: 0x2000, - 0x1d8: 0x2000, 0x1da: 0x2000, 0x1dc: 0x2000, - // Block 0x8, offset 0x200 - 0x211: 0x2000, - 0x221: 0x2000, - // Block 0x9, offset 0x240 - 0x244: 0x2000, - 0x247: 0x2000, 0x249: 0x2000, 0x24a: 0x2000, 0x24b: 0x2000, - 0x24d: 0x2000, 0x250: 0x2000, - 0x258: 0x2000, 0x259: 0x2000, 0x25a: 0x2000, 0x25b: 0x2000, 0x25d: 0x2000, - 0x25f: 0x2000, - // Block 0xa, offset 0x280 - 0x280: 0x2000, 0x281: 0x2000, 0x282: 0x2000, 0x283: 0x2000, 0x284: 0x2000, 0x285: 0x2000, - 0x286: 0x2000, 0x287: 0x2000, 0x288: 0x2000, 0x289: 0x2000, 0x28a: 0x2000, 0x28b: 0x2000, - 0x28c: 0x2000, 0x28d: 0x2000, 0x28e: 0x2000, 0x28f: 0x2000, 0x290: 0x2000, 0x291: 0x2000, - 0x292: 0x2000, 0x293: 0x2000, 0x294: 0x2000, 0x295: 0x2000, 0x296: 0x2000, 0x297: 0x2000, - 0x298: 0x2000, 0x299: 0x2000, 0x29a: 0x2000, 0x29b: 0x2000, 0x29c: 0x2000, 0x29d: 0x2000, - 0x29e: 0x2000, 0x29f: 0x2000, 0x2a0: 0x2000, 0x2a1: 0x2000, 0x2a2: 0x2000, 0x2a3: 0x2000, - 0x2a4: 0x2000, 0x2a5: 0x2000, 0x2a6: 0x2000, 0x2a7: 0x2000, 0x2a8: 0x2000, 0x2a9: 0x2000, - 0x2aa: 0x2000, 0x2ab: 0x2000, 0x2ac: 0x2000, 0x2ad: 0x2000, 0x2ae: 0x2000, 0x2af: 0x2000, - 0x2b0: 0x2000, 0x2b1: 0x2000, 0x2b2: 0x2000, 0x2b3: 0x2000, 0x2b4: 0x2000, 0x2b5: 0x2000, - 0x2b6: 0x2000, 0x2b7: 0x2000, 0x2b8: 0x2000, 0x2b9: 0x2000, 0x2ba: 0x2000, 0x2bb: 0x2000, - 0x2bc: 0x2000, 0x2bd: 0x2000, 0x2be: 0x2000, 0x2bf: 0x2000, - // Block 0xb, offset 0x2c0 - 0x2c0: 0x2000, 0x2c1: 0x2000, 0x2c2: 0x2000, 0x2c3: 0x2000, 0x2c4: 0x2000, 0x2c5: 0x2000, - 0x2c6: 0x2000, 0x2c7: 0x2000, 0x2c8: 0x2000, 0x2c9: 0x2000, 0x2ca: 0x2000, 0x2cb: 0x2000, - 0x2cc: 0x2000, 0x2cd: 0x2000, 0x2ce: 0x2000, 0x2cf: 0x2000, 0x2d0: 0x2000, 0x2d1: 0x2000, - 0x2d2: 0x2000, 0x2d3: 0x2000, 0x2d4: 0x2000, 0x2d5: 0x2000, 0x2d6: 0x2000, 0x2d7: 0x2000, - 0x2d8: 0x2000, 0x2d9: 0x2000, 0x2da: 0x2000, 0x2db: 0x2000, 0x2dc: 0x2000, 0x2dd: 0x2000, - 0x2de: 0x2000, 0x2df: 0x2000, 0x2e0: 0x2000, 0x2e1: 0x2000, 0x2e2: 0x2000, 0x2e3: 0x2000, - 0x2e4: 0x2000, 0x2e5: 0x2000, 0x2e6: 0x2000, 0x2e7: 0x2000, 0x2e8: 0x2000, 0x2e9: 0x2000, - 0x2ea: 0x2000, 0x2eb: 0x2000, 0x2ec: 0x2000, 0x2ed: 0x2000, 0x2ee: 0x2000, 0x2ef: 0x2000, - // Block 0xc, offset 0x300 - 0x311: 0x2000, - 0x312: 0x2000, 0x313: 0x2000, 0x314: 0x2000, 0x315: 0x2000, 0x316: 0x2000, 0x317: 0x2000, - 0x318: 0x2000, 0x319: 0x2000, 0x31a: 0x2000, 0x31b: 0x2000, 0x31c: 0x2000, 0x31d: 0x2000, - 0x31e: 0x2000, 0x31f: 0x2000, 0x320: 0x2000, 0x321: 0x2000, 0x323: 0x2000, - 0x324: 0x2000, 0x325: 0x2000, 0x326: 0x2000, 0x327: 0x2000, 0x328: 0x2000, 0x329: 0x2000, - 0x331: 0x2000, 0x332: 0x2000, 0x333: 0x2000, 0x334: 0x2000, 0x335: 0x2000, - 0x336: 0x2000, 0x337: 0x2000, 0x338: 0x2000, 0x339: 0x2000, 0x33a: 0x2000, 0x33b: 0x2000, - 0x33c: 0x2000, 0x33d: 0x2000, 0x33e: 0x2000, 0x33f: 0x2000, - // Block 0xd, offset 0x340 - 0x340: 0x2000, 0x341: 0x2000, 0x343: 0x2000, 0x344: 0x2000, 0x345: 0x2000, - 0x346: 0x2000, 0x347: 0x2000, 0x348: 0x2000, 0x349: 0x2000, - // Block 0xe, offset 0x380 - 0x381: 0x2000, - 0x390: 0x2000, 0x391: 0x2000, - 0x392: 0x2000, 0x393: 0x2000, 0x394: 0x2000, 0x395: 0x2000, 0x396: 0x2000, 0x397: 0x2000, - 0x398: 0x2000, 0x399: 0x2000, 0x39a: 0x2000, 0x39b: 0x2000, 0x39c: 0x2000, 0x39d: 0x2000, - 0x39e: 0x2000, 0x39f: 0x2000, 0x3a0: 0x2000, 0x3a1: 0x2000, 0x3a2: 0x2000, 0x3a3: 0x2000, - 0x3a4: 0x2000, 0x3a5: 0x2000, 0x3a6: 0x2000, 0x3a7: 0x2000, 0x3a8: 0x2000, 0x3a9: 0x2000, - 0x3aa: 0x2000, 0x3ab: 0x2000, 0x3ac: 0x2000, 0x3ad: 0x2000, 0x3ae: 0x2000, 0x3af: 0x2000, - 0x3b0: 0x2000, 0x3b1: 0x2000, 0x3b2: 0x2000, 0x3b3: 0x2000, 0x3b4: 0x2000, 0x3b5: 0x2000, - 0x3b6: 0x2000, 0x3b7: 0x2000, 0x3b8: 0x2000, 0x3b9: 0x2000, 0x3ba: 0x2000, 0x3bb: 0x2000, - 0x3bc: 0x2000, 0x3bd: 0x2000, 0x3be: 0x2000, 0x3bf: 0x2000, - // Block 0xf, offset 0x3c0 - 0x3c0: 0x2000, 0x3c1: 0x2000, 0x3c2: 0x2000, 0x3c3: 0x2000, 0x3c4: 0x2000, 0x3c5: 0x2000, - 0x3c6: 0x2000, 0x3c7: 0x2000, 0x3c8: 0x2000, 0x3c9: 0x2000, 0x3ca: 0x2000, 0x3cb: 0x2000, - 0x3cc: 0x2000, 0x3cd: 0x2000, 0x3ce: 0x2000, 0x3cf: 0x2000, 0x3d1: 0x2000, - // Block 0x10, offset 0x400 - 0x400: 0x4000, 0x401: 0x4000, 0x402: 0x4000, 0x403: 0x4000, 0x404: 0x4000, 0x405: 0x4000, - 0x406: 0x4000, 0x407: 0x4000, 0x408: 0x4000, 0x409: 0x4000, 0x40a: 0x4000, 0x40b: 0x4000, - 0x40c: 0x4000, 0x40d: 0x4000, 0x40e: 0x4000, 0x40f: 0x4000, 0x410: 0x4000, 0x411: 0x4000, - 0x412: 0x4000, 0x413: 0x4000, 0x414: 0x4000, 0x415: 0x4000, 0x416: 0x4000, 0x417: 0x4000, - 0x418: 0x4000, 0x419: 0x4000, 0x41a: 0x4000, 0x41b: 0x4000, 0x41c: 0x4000, 0x41d: 0x4000, - 0x41e: 0x4000, 0x41f: 0x4000, 0x420: 0x4000, 0x421: 0x4000, 0x422: 0x4000, 0x423: 0x4000, - 0x424: 0x4000, 0x425: 0x4000, 0x426: 0x4000, 0x427: 0x4000, 0x428: 0x4000, 0x429: 0x4000, - 0x42a: 0x4000, 0x42b: 0x4000, 0x42c: 0x4000, 0x42d: 0x4000, 0x42e: 0x4000, 0x42f: 0x4000, - 0x430: 0x4000, 0x431: 0x4000, 0x432: 0x4000, 0x433: 0x4000, 0x434: 0x4000, 0x435: 0x4000, - 0x436: 0x4000, 0x437: 0x4000, 0x438: 0x4000, 0x439: 0x4000, 0x43a: 0x4000, 0x43b: 0x4000, - 0x43c: 0x4000, 0x43d: 0x4000, 0x43e: 0x4000, 0x43f: 0x4000, - // Block 0x11, offset 0x440 - 0x440: 0x4000, 0x441: 0x4000, 0x442: 0x4000, 0x443: 0x4000, 0x444: 0x4000, 0x445: 0x4000, - 0x446: 0x4000, 0x447: 0x4000, 0x448: 0x4000, 0x449: 0x4000, 0x44a: 0x4000, 0x44b: 0x4000, - 0x44c: 0x4000, 0x44d: 0x4000, 0x44e: 0x4000, 0x44f: 0x4000, 0x450: 0x4000, 0x451: 0x4000, - 0x452: 0x4000, 0x453: 0x4000, 0x454: 0x4000, 0x455: 0x4000, 0x456: 0x4000, 0x457: 0x4000, - 0x458: 0x4000, 0x459: 0x4000, 0x45a: 0x4000, 0x45b: 0x4000, 0x45c: 0x4000, 0x45d: 0x4000, - 0x45e: 0x4000, 0x45f: 0x4000, - // Block 0x12, offset 0x480 - 0x490: 0x2000, - 0x493: 0x2000, 0x494: 0x2000, 0x495: 0x2000, 0x496: 0x2000, - 0x498: 0x2000, 0x499: 0x2000, 0x49c: 0x2000, 0x49d: 0x2000, - 0x4a0: 0x2000, 0x4a1: 0x2000, 0x4a2: 0x2000, - 0x4a4: 0x2000, 0x4a5: 0x2000, 0x4a6: 0x2000, 0x4a7: 0x2000, - 0x4b0: 0x2000, 0x4b2: 0x2000, 0x4b3: 0x2000, 0x4b5: 0x2000, - 0x4bb: 0x2000, - 0x4be: 0x2000, - // Block 0x13, offset 0x4c0 - 0x4f4: 0x2000, - 0x4ff: 0x2000, - // Block 0x14, offset 0x500 - 0x501: 0x2000, 0x502: 0x2000, 0x503: 0x2000, 0x504: 0x2000, - 0x529: 0xa009, - 0x52c: 0x2000, - // Block 0x15, offset 0x540 - 0x543: 0x2000, 0x545: 0x2000, - 0x549: 0x2000, - 0x553: 0x2000, 0x556: 0x2000, - 0x561: 0x2000, 0x562: 0x2000, - 0x566: 0x2000, - 0x56b: 0x2000, - // Block 0x16, offset 0x580 - 0x593: 0x2000, 0x594: 0x2000, - 0x59b: 0x2000, 0x59c: 0x2000, 0x59d: 0x2000, - 0x59e: 0x2000, 0x5a0: 0x2000, 0x5a1: 0x2000, 0x5a2: 0x2000, 0x5a3: 0x2000, - 0x5a4: 0x2000, 0x5a5: 0x2000, 0x5a6: 0x2000, 0x5a7: 0x2000, 0x5a8: 0x2000, 0x5a9: 0x2000, - 0x5aa: 0x2000, 0x5ab: 0x2000, - 0x5b0: 0x2000, 0x5b1: 0x2000, 0x5b2: 0x2000, 0x5b3: 0x2000, 0x5b4: 0x2000, 0x5b5: 0x2000, - 0x5b6: 0x2000, 0x5b7: 0x2000, 0x5b8: 0x2000, 0x5b9: 0x2000, - // Block 0x17, offset 0x5c0 - 0x5c9: 0x2000, - 0x5d0: 0x200a, 0x5d1: 0x200b, - 0x5d2: 0x200a, 0x5d3: 0x200c, 0x5d4: 0x2000, 0x5d5: 0x2000, 0x5d6: 0x2000, 0x5d7: 0x2000, - 0x5d8: 0x2000, 0x5d9: 0x2000, - 0x5f8: 0x2000, 0x5f9: 0x2000, - // Block 0x18, offset 0x600 - 0x612: 0x2000, 0x614: 0x2000, - 0x627: 0x2000, - // Block 0x19, offset 0x640 - 0x640: 0x2000, 0x642: 0x2000, 0x643: 0x2000, - 0x647: 0x2000, 0x648: 0x2000, 0x64b: 0x2000, - 0x64f: 0x2000, 0x651: 0x2000, - 0x655: 0x2000, - 0x65a: 0x2000, 0x65d: 0x2000, - 0x65e: 0x2000, 0x65f: 0x2000, 0x660: 0x2000, 0x663: 0x2000, - 0x665: 0x2000, 0x667: 0x2000, 0x668: 0x2000, 0x669: 0x2000, - 0x66a: 0x2000, 0x66b: 0x2000, 0x66c: 0x2000, 0x66e: 0x2000, - 0x674: 0x2000, 0x675: 0x2000, - 0x676: 0x2000, 0x677: 0x2000, - 0x67c: 0x2000, 0x67d: 0x2000, - // Block 0x1a, offset 0x680 - 0x688: 0x2000, - 0x68c: 0x2000, - 0x692: 0x2000, - 0x6a0: 0x2000, 0x6a1: 0x2000, - 0x6a4: 0x2000, 0x6a5: 0x2000, 0x6a6: 0x2000, 0x6a7: 0x2000, - 0x6aa: 0x2000, 0x6ab: 0x2000, 0x6ae: 0x2000, 0x6af: 0x2000, - // Block 0x1b, offset 0x6c0 - 0x6c2: 0x2000, 0x6c3: 0x2000, - 0x6c6: 0x2000, 0x6c7: 0x2000, - 0x6d5: 0x2000, - 0x6d9: 0x2000, - 0x6e5: 0x2000, - 0x6ff: 0x2000, - // Block 0x1c, offset 0x700 - 0x712: 0x2000, - 0x71a: 0x4000, 0x71b: 0x4000, - 0x729: 0x4000, - 0x72a: 0x4000, - // Block 0x1d, offset 0x740 - 0x769: 0x4000, - 0x76a: 0x4000, 0x76b: 0x4000, 0x76c: 0x4000, - 0x770: 0x4000, 0x773: 0x4000, - // Block 0x1e, offset 0x780 - 0x7a0: 0x2000, 0x7a1: 0x2000, 0x7a2: 0x2000, 0x7a3: 0x2000, - 0x7a4: 0x2000, 0x7a5: 0x2000, 0x7a6: 0x2000, 0x7a7: 0x2000, 0x7a8: 0x2000, 0x7a9: 0x2000, - 0x7aa: 0x2000, 0x7ab: 0x2000, 0x7ac: 0x2000, 0x7ad: 0x2000, 0x7ae: 0x2000, 0x7af: 0x2000, - 0x7b0: 0x2000, 0x7b1: 0x2000, 0x7b2: 0x2000, 0x7b3: 0x2000, 0x7b4: 0x2000, 0x7b5: 0x2000, - 0x7b6: 0x2000, 0x7b7: 0x2000, 0x7b8: 0x2000, 0x7b9: 0x2000, 0x7ba: 0x2000, 0x7bb: 0x2000, - 0x7bc: 0x2000, 0x7bd: 0x2000, 0x7be: 0x2000, 0x7bf: 0x2000, - // Block 0x1f, offset 0x7c0 - 0x7c0: 0x2000, 0x7c1: 0x2000, 0x7c2: 0x2000, 0x7c3: 0x2000, 0x7c4: 0x2000, 0x7c5: 0x2000, - 0x7c6: 0x2000, 0x7c7: 0x2000, 0x7c8: 0x2000, 0x7c9: 0x2000, 0x7ca: 0x2000, 0x7cb: 0x2000, - 0x7cc: 0x2000, 0x7cd: 0x2000, 0x7ce: 0x2000, 0x7cf: 0x2000, 0x7d0: 0x2000, 0x7d1: 0x2000, - 0x7d2: 0x2000, 0x7d3: 0x2000, 0x7d4: 0x2000, 0x7d5: 0x2000, 0x7d6: 0x2000, 0x7d7: 0x2000, - 0x7d8: 0x2000, 0x7d9: 0x2000, 0x7da: 0x2000, 0x7db: 0x2000, 0x7dc: 0x2000, 0x7dd: 0x2000, - 0x7de: 0x2000, 0x7df: 0x2000, 0x7e0: 0x2000, 0x7e1: 0x2000, 0x7e2: 0x2000, 0x7e3: 0x2000, - 0x7e4: 0x2000, 0x7e5: 0x2000, 0x7e6: 0x2000, 0x7e7: 0x2000, 0x7e8: 0x2000, 0x7e9: 0x2000, - 0x7eb: 0x2000, 0x7ec: 0x2000, 0x7ed: 0x2000, 0x7ee: 0x2000, 0x7ef: 0x2000, - 0x7f0: 0x2000, 0x7f1: 0x2000, 0x7f2: 0x2000, 0x7f3: 0x2000, 0x7f4: 0x2000, 0x7f5: 0x2000, - 0x7f6: 0x2000, 0x7f7: 0x2000, 0x7f8: 0x2000, 0x7f9: 0x2000, 0x7fa: 0x2000, 0x7fb: 0x2000, - 0x7fc: 0x2000, 0x7fd: 0x2000, 0x7fe: 0x2000, 0x7ff: 0x2000, - // Block 0x20, offset 0x800 - 0x800: 0x2000, 0x801: 0x2000, 0x802: 0x200d, 0x803: 0x2000, 0x804: 0x2000, 0x805: 0x2000, - 0x806: 0x2000, 0x807: 0x2000, 0x808: 0x2000, 0x809: 0x2000, 0x80a: 0x2000, 0x80b: 0x2000, - 0x80c: 0x2000, 0x80d: 0x2000, 0x80e: 0x2000, 0x80f: 0x2000, 0x810: 0x2000, 0x811: 0x2000, - 0x812: 0x2000, 0x813: 0x2000, 0x814: 0x2000, 0x815: 0x2000, 0x816: 0x2000, 0x817: 0x2000, - 0x818: 0x2000, 0x819: 0x2000, 0x81a: 0x2000, 0x81b: 0x2000, 0x81c: 0x2000, 0x81d: 0x2000, - 0x81e: 0x2000, 0x81f: 0x2000, 0x820: 0x2000, 0x821: 0x2000, 0x822: 0x2000, 0x823: 0x2000, - 0x824: 0x2000, 0x825: 0x2000, 0x826: 0x2000, 0x827: 0x2000, 0x828: 0x2000, 0x829: 0x2000, - 0x82a: 0x2000, 0x82b: 0x2000, 0x82c: 0x2000, 0x82d: 0x2000, 0x82e: 0x2000, 0x82f: 0x2000, - 0x830: 0x2000, 0x831: 0x2000, 0x832: 0x2000, 0x833: 0x2000, 0x834: 0x2000, 0x835: 0x2000, - 0x836: 0x2000, 0x837: 0x2000, 0x838: 0x2000, 0x839: 0x2000, 0x83a: 0x2000, 0x83b: 0x2000, - 0x83c: 0x2000, 0x83d: 0x2000, 0x83e: 0x2000, 0x83f: 0x2000, - // Block 0x21, offset 0x840 - 0x840: 0x2000, 0x841: 0x2000, 0x842: 0x2000, 0x843: 0x2000, 0x844: 0x2000, 0x845: 0x2000, - 0x846: 0x2000, 0x847: 0x2000, 0x848: 0x2000, 0x849: 0x2000, 0x84a: 0x2000, 0x84b: 0x2000, - 0x850: 0x2000, 0x851: 0x2000, - 0x852: 0x2000, 0x853: 0x2000, 0x854: 0x2000, 0x855: 0x2000, 0x856: 0x2000, 0x857: 0x2000, - 0x858: 0x2000, 0x859: 0x2000, 0x85a: 0x2000, 0x85b: 0x2000, 0x85c: 0x2000, 0x85d: 0x2000, - 0x85e: 0x2000, 0x85f: 0x2000, 0x860: 0x2000, 0x861: 0x2000, 0x862: 0x2000, 0x863: 0x2000, - 0x864: 0x2000, 0x865: 0x2000, 0x866: 0x2000, 0x867: 0x2000, 0x868: 0x2000, 0x869: 0x2000, - 0x86a: 0x2000, 0x86b: 0x2000, 0x86c: 0x2000, 0x86d: 0x2000, 0x86e: 0x2000, 0x86f: 0x2000, - 0x870: 0x2000, 0x871: 0x2000, 0x872: 0x2000, 0x873: 0x2000, - // Block 0x22, offset 0x880 - 0x880: 0x2000, 0x881: 0x2000, 0x882: 0x2000, 0x883: 0x2000, 0x884: 0x2000, 0x885: 0x2000, - 0x886: 0x2000, 0x887: 0x2000, 0x888: 0x2000, 0x889: 0x2000, 0x88a: 0x2000, 0x88b: 0x2000, - 0x88c: 0x2000, 0x88d: 0x2000, 0x88e: 0x2000, 0x88f: 0x2000, - 0x892: 0x2000, 0x893: 0x2000, 0x894: 0x2000, 0x895: 0x2000, - 0x8a0: 0x200e, 0x8a1: 0x2000, 0x8a3: 0x2000, - 0x8a4: 0x2000, 0x8a5: 0x2000, 0x8a6: 0x2000, 0x8a7: 0x2000, 0x8a8: 0x2000, 0x8a9: 0x2000, - 0x8b2: 0x2000, 0x8b3: 0x2000, - 0x8b6: 0x2000, 0x8b7: 0x2000, - 0x8bc: 0x2000, 0x8bd: 0x2000, - // Block 0x23, offset 0x8c0 - 0x8c0: 0x2000, 0x8c1: 0x2000, - 0x8c6: 0x2000, 0x8c7: 0x2000, 0x8c8: 0x2000, 0x8cb: 0x200f, - 0x8ce: 0x2000, 0x8cf: 0x2000, 0x8d0: 0x2000, 0x8d1: 0x2000, - 0x8e2: 0x2000, 0x8e3: 0x2000, - 0x8e4: 0x2000, 0x8e5: 0x2000, - 0x8ef: 0x2000, - 0x8fd: 0x4000, 0x8fe: 0x4000, - // Block 0x24, offset 0x900 - 0x905: 0x2000, - 0x906: 0x2000, 0x909: 0x2000, - 0x90e: 0x2000, 0x90f: 0x2000, - 0x914: 0x4000, 0x915: 0x4000, - 0x91c: 0x2000, - 0x91e: 0x2000, - // Block 0x25, offset 0x940 - 0x940: 0x2000, 0x942: 0x2000, - 0x948: 0x4000, 0x949: 0x4000, 0x94a: 0x4000, 0x94b: 0x4000, - 0x94c: 0x4000, 0x94d: 0x4000, 0x94e: 0x4000, 0x94f: 0x4000, 0x950: 0x4000, 0x951: 0x4000, - 0x952: 0x4000, 0x953: 0x4000, - 0x960: 0x2000, 0x961: 0x2000, 0x963: 0x2000, - 0x964: 0x2000, 0x965: 0x2000, 0x967: 0x2000, 0x968: 0x2000, 0x969: 0x2000, - 0x96a: 0x2000, 0x96c: 0x2000, 0x96d: 0x2000, 0x96f: 0x2000, - 0x97f: 0x4000, - // Block 0x26, offset 0x980 - 0x993: 0x4000, - 0x99e: 0x2000, 0x99f: 0x2000, 0x9a1: 0x4000, - 0x9aa: 0x4000, 0x9ab: 0x4000, - 0x9bd: 0x4000, 0x9be: 0x4000, 0x9bf: 0x2000, - // Block 0x27, offset 0x9c0 - 0x9c4: 0x4000, 0x9c5: 0x4000, - 0x9c6: 0x2000, 0x9c7: 0x2000, 0x9c8: 0x2000, 0x9c9: 0x2000, 0x9ca: 0x2000, 0x9cb: 0x2000, - 0x9cc: 0x2000, 0x9cd: 0x2000, 0x9ce: 0x4000, 0x9cf: 0x2000, 0x9d0: 0x2000, 0x9d1: 0x2000, - 0x9d2: 0x2000, 0x9d3: 0x2000, 0x9d4: 0x4000, 0x9d5: 0x2000, 0x9d6: 0x2000, 0x9d7: 0x2000, - 0x9d8: 0x2000, 0x9d9: 0x2000, 0x9da: 0x2000, 0x9db: 0x2000, 0x9dc: 0x2000, 0x9dd: 0x2000, - 0x9de: 0x2000, 0x9df: 0x2000, 0x9e0: 0x2000, 0x9e1: 0x2000, 0x9e3: 0x2000, - 0x9e8: 0x2000, 0x9e9: 0x2000, - 0x9ea: 0x4000, 0x9eb: 0x2000, 0x9ec: 0x2000, 0x9ed: 0x2000, 0x9ee: 0x2000, 0x9ef: 0x2000, - 0x9f0: 0x2000, 0x9f1: 0x2000, 0x9f2: 0x4000, 0x9f3: 0x4000, 0x9f4: 0x2000, 0x9f5: 0x4000, - 0x9f6: 0x2000, 0x9f7: 0x2000, 0x9f8: 0x2000, 0x9f9: 0x2000, 0x9fa: 0x4000, 0x9fb: 0x2000, - 0x9fc: 0x2000, 0x9fd: 0x4000, 0x9fe: 0x2000, 0x9ff: 0x2000, - // Block 0x28, offset 0xa00 - 0xa05: 0x4000, - 0xa0a: 0x4000, 0xa0b: 0x4000, - 0xa28: 0x4000, - 0xa3d: 0x2000, - // Block 0x29, offset 0xa40 - 0xa4c: 0x4000, 0xa4e: 0x4000, - 0xa53: 0x4000, 0xa54: 0x4000, 0xa55: 0x4000, 0xa57: 0x4000, - 0xa76: 0x2000, 0xa77: 0x2000, 0xa78: 0x2000, 0xa79: 0x2000, 0xa7a: 0x2000, 0xa7b: 0x2000, - 0xa7c: 0x2000, 0xa7d: 0x2000, 0xa7e: 0x2000, 0xa7f: 0x2000, - // Block 0x2a, offset 0xa80 - 0xa95: 0x4000, 0xa96: 0x4000, 0xa97: 0x4000, - 0xab0: 0x4000, - 0xabf: 0x4000, - // Block 0x2b, offset 0xac0 - 0xae6: 0x6000, 0xae7: 0x6000, 0xae8: 0x6000, 0xae9: 0x6000, - 0xaea: 0x6000, 0xaeb: 0x6000, 0xaec: 0x6000, 0xaed: 0x6000, - // Block 0x2c, offset 0xb00 - 0xb05: 0x6010, - 0xb06: 0x6011, - // Block 0x2d, offset 0xb40 - 0xb5b: 0x4000, 0xb5c: 0x4000, - // Block 0x2e, offset 0xb80 - 0xb90: 0x4000, - 0xb95: 0x4000, 0xb96: 0x2000, 0xb97: 0x2000, - 0xb98: 0x2000, 0xb99: 0x2000, - // Block 0x2f, offset 0xbc0 - 0xbc0: 0x4000, 0xbc1: 0x4000, 0xbc2: 0x4000, 0xbc3: 0x4000, 0xbc4: 0x4000, 0xbc5: 0x4000, - 0xbc6: 0x4000, 0xbc7: 0x4000, 0xbc8: 0x4000, 0xbc9: 0x4000, 0xbca: 0x4000, 0xbcb: 0x4000, - 0xbcc: 0x4000, 0xbcd: 0x4000, 0xbce: 0x4000, 0xbcf: 0x4000, 0xbd0: 0x4000, 0xbd1: 0x4000, - 0xbd2: 0x4000, 0xbd3: 0x4000, 0xbd4: 0x4000, 0xbd5: 0x4000, 0xbd6: 0x4000, 0xbd7: 0x4000, - 0xbd8: 0x4000, 0xbd9: 0x4000, 0xbdb: 0x4000, 0xbdc: 0x4000, 0xbdd: 0x4000, - 0xbde: 0x4000, 0xbdf: 0x4000, 0xbe0: 0x4000, 0xbe1: 0x4000, 0xbe2: 0x4000, 0xbe3: 0x4000, - 0xbe4: 0x4000, 0xbe5: 0x4000, 0xbe6: 0x4000, 0xbe7: 0x4000, 0xbe8: 0x4000, 0xbe9: 0x4000, - 0xbea: 0x4000, 0xbeb: 0x4000, 0xbec: 0x4000, 0xbed: 0x4000, 0xbee: 0x4000, 0xbef: 0x4000, - 0xbf0: 0x4000, 0xbf1: 0x4000, 0xbf2: 0x4000, 0xbf3: 0x4000, 0xbf4: 0x4000, 0xbf5: 0x4000, - 0xbf6: 0x4000, 0xbf7: 0x4000, 0xbf8: 0x4000, 0xbf9: 0x4000, 0xbfa: 0x4000, 0xbfb: 0x4000, - 0xbfc: 0x4000, 0xbfd: 0x4000, 0xbfe: 0x4000, 0xbff: 0x4000, - // Block 0x30, offset 0xc00 - 0xc00: 0x4000, 0xc01: 0x4000, 0xc02: 0x4000, 0xc03: 0x4000, 0xc04: 0x4000, 0xc05: 0x4000, - 0xc06: 0x4000, 0xc07: 0x4000, 0xc08: 0x4000, 0xc09: 0x4000, 0xc0a: 0x4000, 0xc0b: 0x4000, - 0xc0c: 0x4000, 0xc0d: 0x4000, 0xc0e: 0x4000, 0xc0f: 0x4000, 0xc10: 0x4000, 0xc11: 0x4000, - 0xc12: 0x4000, 0xc13: 0x4000, 0xc14: 0x4000, 0xc15: 0x4000, 0xc16: 0x4000, 0xc17: 0x4000, - 0xc18: 0x4000, 0xc19: 0x4000, 0xc1a: 0x4000, 0xc1b: 0x4000, 0xc1c: 0x4000, 0xc1d: 0x4000, - 0xc1e: 0x4000, 0xc1f: 0x4000, 0xc20: 0x4000, 0xc21: 0x4000, 0xc22: 0x4000, 0xc23: 0x4000, - 0xc24: 0x4000, 0xc25: 0x4000, 0xc26: 0x4000, 0xc27: 0x4000, 0xc28: 0x4000, 0xc29: 0x4000, - 0xc2a: 0x4000, 0xc2b: 0x4000, 0xc2c: 0x4000, 0xc2d: 0x4000, 0xc2e: 0x4000, 0xc2f: 0x4000, - 0xc30: 0x4000, 0xc31: 0x4000, 0xc32: 0x4000, 0xc33: 0x4000, - // Block 0x31, offset 0xc40 - 0xc40: 0x4000, 0xc41: 0x4000, 0xc42: 0x4000, 0xc43: 0x4000, 0xc44: 0x4000, 0xc45: 0x4000, - 0xc46: 0x4000, 0xc47: 0x4000, 0xc48: 0x4000, 0xc49: 0x4000, 0xc4a: 0x4000, 0xc4b: 0x4000, - 0xc4c: 0x4000, 0xc4d: 0x4000, 0xc4e: 0x4000, 0xc4f: 0x4000, 0xc50: 0x4000, 0xc51: 0x4000, - 0xc52: 0x4000, 0xc53: 0x4000, 0xc54: 0x4000, 0xc55: 0x4000, - 0xc70: 0x4000, 0xc71: 0x4000, 0xc72: 0x4000, 0xc73: 0x4000, 0xc74: 0x4000, 0xc75: 0x4000, - 0xc76: 0x4000, 0xc77: 0x4000, 0xc78: 0x4000, 0xc79: 0x4000, 0xc7a: 0x4000, 0xc7b: 0x4000, - // Block 0x32, offset 0xc80 - 0xc80: 0x9012, 0xc81: 0x4013, 0xc82: 0x4014, 0xc83: 0x4000, 0xc84: 0x4000, 0xc85: 0x4000, - 0xc86: 0x4000, 0xc87: 0x4000, 0xc88: 0x4000, 0xc89: 0x4000, 0xc8a: 0x4000, 0xc8b: 0x4000, - 0xc8c: 0x4015, 0xc8d: 0x4015, 0xc8e: 0x4000, 0xc8f: 0x4000, 0xc90: 0x4000, 0xc91: 0x4000, - 0xc92: 0x4000, 0xc93: 0x4000, 0xc94: 0x4000, 0xc95: 0x4000, 0xc96: 0x4000, 0xc97: 0x4000, - 0xc98: 0x4000, 0xc99: 0x4000, 0xc9a: 0x4000, 0xc9b: 0x4000, 0xc9c: 0x4000, 0xc9d: 0x4000, - 0xc9e: 0x4000, 0xc9f: 0x4000, 0xca0: 0x4000, 0xca1: 0x4000, 0xca2: 0x4000, 0xca3: 0x4000, - 0xca4: 0x4000, 0xca5: 0x4000, 0xca6: 0x4000, 0xca7: 0x4000, 0xca8: 0x4000, 0xca9: 0x4000, - 0xcaa: 0x4000, 0xcab: 0x4000, 0xcac: 0x4000, 0xcad: 0x4000, 0xcae: 0x4000, 0xcaf: 0x4000, - 0xcb0: 0x4000, 0xcb1: 0x4000, 0xcb2: 0x4000, 0xcb3: 0x4000, 0xcb4: 0x4000, 0xcb5: 0x4000, - 0xcb6: 0x4000, 0xcb7: 0x4000, 0xcb8: 0x4000, 0xcb9: 0x4000, 0xcba: 0x4000, 0xcbb: 0x4000, - 0xcbc: 0x4000, 0xcbd: 0x4000, 0xcbe: 0x4000, - // Block 0x33, offset 0xcc0 - 0xcc1: 0x4000, 0xcc2: 0x4000, 0xcc3: 0x4000, 0xcc4: 0x4000, 0xcc5: 0x4000, - 0xcc6: 0x4000, 0xcc7: 0x4000, 0xcc8: 0x4000, 0xcc9: 0x4000, 0xcca: 0x4000, 0xccb: 0x4000, - 0xccc: 0x4000, 0xccd: 0x4000, 0xcce: 0x4000, 0xccf: 0x4000, 0xcd0: 0x4000, 0xcd1: 0x4000, - 0xcd2: 0x4000, 0xcd3: 0x4000, 0xcd4: 0x4000, 0xcd5: 0x4000, 0xcd6: 0x4000, 0xcd7: 0x4000, - 0xcd8: 0x4000, 0xcd9: 0x4000, 0xcda: 0x4000, 0xcdb: 0x4000, 0xcdc: 0x4000, 0xcdd: 0x4000, - 0xcde: 0x4000, 0xcdf: 0x4000, 0xce0: 0x4000, 0xce1: 0x4000, 0xce2: 0x4000, 0xce3: 0x4000, - 0xce4: 0x4000, 0xce5: 0x4000, 0xce6: 0x4000, 0xce7: 0x4000, 0xce8: 0x4000, 0xce9: 0x4000, - 0xcea: 0x4000, 0xceb: 0x4000, 0xcec: 0x4000, 0xced: 0x4000, 0xcee: 0x4000, 0xcef: 0x4000, - 0xcf0: 0x4000, 0xcf1: 0x4000, 0xcf2: 0x4000, 0xcf3: 0x4000, 0xcf4: 0x4000, 0xcf5: 0x4000, - 0xcf6: 0x4000, 0xcf7: 0x4000, 0xcf8: 0x4000, 0xcf9: 0x4000, 0xcfa: 0x4000, 0xcfb: 0x4000, - 0xcfc: 0x4000, 0xcfd: 0x4000, 0xcfe: 0x4000, 0xcff: 0x4000, - // Block 0x34, offset 0xd00 - 0xd00: 0x4000, 0xd01: 0x4000, 0xd02: 0x4000, 0xd03: 0x4000, 0xd04: 0x4000, 0xd05: 0x4000, - 0xd06: 0x4000, 0xd07: 0x4000, 0xd08: 0x4000, 0xd09: 0x4000, 0xd0a: 0x4000, 0xd0b: 0x4000, - 0xd0c: 0x4000, 0xd0d: 0x4000, 0xd0e: 0x4000, 0xd0f: 0x4000, 0xd10: 0x4000, 0xd11: 0x4000, - 0xd12: 0x4000, 0xd13: 0x4000, 0xd14: 0x4000, 0xd15: 0x4000, 0xd16: 0x4000, - 0xd19: 0x4016, 0xd1a: 0x4017, 0xd1b: 0x4000, 0xd1c: 0x4000, 0xd1d: 0x4000, - 0xd1e: 0x4000, 0xd1f: 0x4000, 0xd20: 0x4000, 0xd21: 0x4018, 0xd22: 0x4019, 0xd23: 0x401a, - 0xd24: 0x401b, 0xd25: 0x401c, 0xd26: 0x401d, 0xd27: 0x401e, 0xd28: 0x401f, 0xd29: 0x4020, - 0xd2a: 0x4021, 0xd2b: 0x4022, 0xd2c: 0x4000, 0xd2d: 0x4010, 0xd2e: 0x4000, 0xd2f: 0x4023, - 0xd30: 0x4000, 0xd31: 0x4024, 0xd32: 0x4000, 0xd33: 0x4025, 0xd34: 0x4000, 0xd35: 0x4026, - 0xd36: 0x4000, 0xd37: 0x401a, 0xd38: 0x4000, 0xd39: 0x4027, 0xd3a: 0x4000, 0xd3b: 0x4028, - 0xd3c: 0x4000, 0xd3d: 0x4020, 0xd3e: 0x4000, 0xd3f: 0x4029, - // Block 0x35, offset 0xd40 - 0xd40: 0x4000, 0xd41: 0x402a, 0xd42: 0x4000, 0xd43: 0x402b, 0xd44: 0x402c, 0xd45: 0x4000, - 0xd46: 0x4017, 0xd47: 0x4000, 0xd48: 0x402d, 0xd49: 0x4000, 0xd4a: 0x402e, 0xd4b: 0x402f, - 0xd4c: 0x4030, 0xd4d: 0x4017, 0xd4e: 0x4016, 0xd4f: 0x4017, 0xd50: 0x4000, 0xd51: 0x4000, - 0xd52: 0x4031, 0xd53: 0x4000, 0xd54: 0x4000, 0xd55: 0x4031, 0xd56: 0x4000, 0xd57: 0x4000, - 0xd58: 0x4032, 0xd59: 0x4000, 0xd5a: 0x4000, 0xd5b: 0x4032, 0xd5c: 0x4000, 0xd5d: 0x4000, - 0xd5e: 0x4033, 0xd5f: 0x402e, 0xd60: 0x4034, 0xd61: 0x4035, 0xd62: 0x4034, 0xd63: 0x4036, - 0xd64: 0x4037, 0xd65: 0x4024, 0xd66: 0x4035, 0xd67: 0x4025, 0xd68: 0x4038, 0xd69: 0x4038, - 0xd6a: 0x4039, 0xd6b: 0x4039, 0xd6c: 0x403a, 0xd6d: 0x403a, 0xd6e: 0x4000, 0xd6f: 0x4035, - 0xd70: 0x4000, 0xd71: 0x4000, 0xd72: 0x403b, 0xd73: 0x403c, 0xd74: 0x4000, 0xd75: 0x4000, - 0xd76: 0x4000, 0xd77: 0x4000, 0xd78: 0x4000, 0xd79: 0x4000, 0xd7a: 0x4000, 0xd7b: 0x403d, - 0xd7c: 0x401c, 0xd7d: 0x4000, 0xd7e: 0x4000, 0xd7f: 0x4000, - // Block 0x36, offset 0xd80 - 0xd85: 0x4000, - 0xd86: 0x4000, 0xd87: 0x4000, 0xd88: 0x4000, 0xd89: 0x4000, 0xd8a: 0x4000, 0xd8b: 0x4000, - 0xd8c: 0x4000, 0xd8d: 0x4000, 0xd8e: 0x4000, 0xd8f: 0x4000, 0xd90: 0x4000, 0xd91: 0x4000, - 0xd92: 0x4000, 0xd93: 0x4000, 0xd94: 0x4000, 0xd95: 0x4000, 0xd96: 0x4000, 0xd97: 0x4000, - 0xd98: 0x4000, 0xd99: 0x4000, 0xd9a: 0x4000, 0xd9b: 0x4000, 0xd9c: 0x4000, 0xd9d: 0x4000, - 0xd9e: 0x4000, 0xd9f: 0x4000, 0xda0: 0x4000, 0xda1: 0x4000, 0xda2: 0x4000, 0xda3: 0x4000, - 0xda4: 0x4000, 0xda5: 0x4000, 0xda6: 0x4000, 0xda7: 0x4000, 0xda8: 0x4000, 0xda9: 0x4000, - 0xdaa: 0x4000, 0xdab: 0x4000, 0xdac: 0x4000, 0xdad: 0x4000, 0xdae: 0x4000, 0xdaf: 0x4000, - 0xdb1: 0x403e, 0xdb2: 0x403e, 0xdb3: 0x403e, 0xdb4: 0x403e, 0xdb5: 0x403e, - 0xdb6: 0x403e, 0xdb7: 0x403e, 0xdb8: 0x403e, 0xdb9: 0x403e, 0xdba: 0x403e, 0xdbb: 0x403e, - 0xdbc: 0x403e, 0xdbd: 0x403e, 0xdbe: 0x403e, 0xdbf: 0x403e, - // Block 0x37, offset 0xdc0 - 0xdc0: 0x4037, 0xdc1: 0x4037, 0xdc2: 0x4037, 0xdc3: 0x4037, 0xdc4: 0x4037, 0xdc5: 0x4037, - 0xdc6: 0x4037, 0xdc7: 0x4037, 0xdc8: 0x4037, 0xdc9: 0x4037, 0xdca: 0x4037, 0xdcb: 0x4037, - 0xdcc: 0x4037, 0xdcd: 0x4037, 0xdce: 0x4037, 0xdcf: 0x400e, 0xdd0: 0x403f, 0xdd1: 0x4040, - 0xdd2: 0x4041, 0xdd3: 0x4040, 0xdd4: 0x403f, 0xdd5: 0x4042, 0xdd6: 0x4043, 0xdd7: 0x4044, - 0xdd8: 0x4040, 0xdd9: 0x4041, 0xdda: 0x4040, 0xddb: 0x4045, 0xddc: 0x4009, 0xddd: 0x4045, - 0xdde: 0x4046, 0xddf: 0x4045, 0xde0: 0x4047, 0xde1: 0x400b, 0xde2: 0x400a, 0xde3: 0x400c, - 0xde4: 0x4048, 0xde5: 0x4000, 0xde6: 0x4000, 0xde7: 0x4000, 0xde8: 0x4000, 0xde9: 0x4000, - 0xdea: 0x4000, 0xdeb: 0x4000, 0xdec: 0x4000, 0xded: 0x4000, 0xdee: 0x4000, 0xdef: 0x4000, - 0xdf0: 0x4000, 0xdf1: 0x4000, 0xdf2: 0x4000, 0xdf3: 0x4000, 0xdf4: 0x4000, 0xdf5: 0x4000, - 0xdf6: 0x4000, 0xdf7: 0x4000, 0xdf8: 0x4000, 0xdf9: 0x4000, 0xdfa: 0x4000, 0xdfb: 0x4000, - 0xdfc: 0x4000, 0xdfd: 0x4000, 0xdfe: 0x4000, 0xdff: 0x4000, - // Block 0x38, offset 0xe00 - 0xe00: 0x4000, 0xe01: 0x4000, 0xe02: 0x4000, 0xe03: 0x4000, 0xe04: 0x4000, 0xe05: 0x4000, - 0xe06: 0x4000, 0xe07: 0x4000, 0xe08: 0x4000, 0xe09: 0x4000, 0xe0a: 0x4000, 0xe0b: 0x4000, - 0xe0c: 0x4000, 0xe0d: 0x4000, 0xe0e: 0x4000, 0xe10: 0x4000, 0xe11: 0x4000, - 0xe12: 0x4000, 0xe13: 0x4000, 0xe14: 0x4000, 0xe15: 0x4000, 0xe16: 0x4000, 0xe17: 0x4000, - 0xe18: 0x4000, 0xe19: 0x4000, 0xe1a: 0x4000, 0xe1b: 0x4000, 0xe1c: 0x4000, 0xe1d: 0x4000, - 0xe1e: 0x4000, 0xe1f: 0x4000, 0xe20: 0x4000, 0xe21: 0x4000, 0xe22: 0x4000, 0xe23: 0x4000, - 0xe24: 0x4000, 0xe25: 0x4000, 0xe26: 0x4000, 0xe27: 0x4000, 0xe28: 0x4000, 0xe29: 0x4000, - 0xe2a: 0x4000, 0xe2b: 0x4000, 0xe2c: 0x4000, 0xe2d: 0x4000, 0xe2e: 0x4000, 0xe2f: 0x4000, - 0xe30: 0x4000, 0xe31: 0x4000, 0xe32: 0x4000, 0xe33: 0x4000, 0xe34: 0x4000, 0xe35: 0x4000, - 0xe36: 0x4000, 0xe37: 0x4000, 0xe38: 0x4000, 0xe39: 0x4000, 0xe3a: 0x4000, 0xe3b: 0x4000, - 0xe3c: 0x4000, 0xe3d: 0x4000, 0xe3e: 0x4000, 0xe3f: 0x4000, - // Block 0x39, offset 0xe40 - 0xe40: 0x4000, 0xe41: 0x4000, 0xe42: 0x4000, 0xe43: 0x4000, 0xe44: 0x4000, 0xe45: 0x4000, - 0xe46: 0x4000, 0xe47: 0x4000, 0xe48: 0x4000, 0xe49: 0x4000, 0xe4a: 0x4000, 0xe4b: 0x4000, - 0xe4c: 0x4000, 0xe4d: 0x4000, 0xe4e: 0x4000, 0xe4f: 0x4000, 0xe50: 0x4000, 0xe51: 0x4000, - 0xe52: 0x4000, 0xe53: 0x4000, 0xe54: 0x4000, 0xe55: 0x4000, 0xe56: 0x4000, 0xe57: 0x4000, - 0xe58: 0x4000, 0xe59: 0x4000, 0xe5a: 0x4000, 0xe5b: 0x4000, 0xe5c: 0x4000, 0xe5d: 0x4000, - 0xe5e: 0x4000, 0xe5f: 0x4000, 0xe60: 0x4000, 0xe61: 0x4000, 0xe62: 0x4000, 0xe63: 0x4000, - 0xe70: 0x4000, 0xe71: 0x4000, 0xe72: 0x4000, 0xe73: 0x4000, 0xe74: 0x4000, 0xe75: 0x4000, - 0xe76: 0x4000, 0xe77: 0x4000, 0xe78: 0x4000, 0xe79: 0x4000, 0xe7a: 0x4000, 0xe7b: 0x4000, - 0xe7c: 0x4000, 0xe7d: 0x4000, 0xe7e: 0x4000, 0xe7f: 0x4000, - // Block 0x3a, offset 0xe80 - 0xe80: 0x4000, 0xe81: 0x4000, 0xe82: 0x4000, 0xe83: 0x4000, 0xe84: 0x4000, 0xe85: 0x4000, - 0xe86: 0x4000, 0xe87: 0x4000, 0xe88: 0x4000, 0xe89: 0x4000, 0xe8a: 0x4000, 0xe8b: 0x4000, - 0xe8c: 0x4000, 0xe8d: 0x4000, 0xe8e: 0x4000, 0xe8f: 0x4000, 0xe90: 0x4000, 0xe91: 0x4000, - 0xe92: 0x4000, 0xe93: 0x4000, 0xe94: 0x4000, 0xe95: 0x4000, 0xe96: 0x4000, 0xe97: 0x4000, - 0xe98: 0x4000, 0xe99: 0x4000, 0xe9a: 0x4000, 0xe9b: 0x4000, 0xe9c: 0x4000, 0xe9d: 0x4000, - 0xe9e: 0x4000, 0xea0: 0x4000, 0xea1: 0x4000, 0xea2: 0x4000, 0xea3: 0x4000, - 0xea4: 0x4000, 0xea5: 0x4000, 0xea6: 0x4000, 0xea7: 0x4000, 0xea8: 0x4000, 0xea9: 0x4000, - 0xeaa: 0x4000, 0xeab: 0x4000, 0xeac: 0x4000, 0xead: 0x4000, 0xeae: 0x4000, 0xeaf: 0x4000, - 0xeb0: 0x4000, 0xeb1: 0x4000, 0xeb2: 0x4000, 0xeb3: 0x4000, 0xeb4: 0x4000, 0xeb5: 0x4000, - 0xeb6: 0x4000, 0xeb7: 0x4000, 0xeb8: 0x4000, 0xeb9: 0x4000, 0xeba: 0x4000, 0xebb: 0x4000, - 0xebc: 0x4000, 0xebd: 0x4000, 0xebe: 0x4000, 0xebf: 0x4000, - // Block 0x3b, offset 0xec0 - 0xec0: 0x4000, 0xec1: 0x4000, 0xec2: 0x4000, 0xec3: 0x4000, 0xec4: 0x4000, 0xec5: 0x4000, - 0xec6: 0x4000, 0xec7: 0x4000, 0xec8: 0x2000, 0xec9: 0x2000, 0xeca: 0x2000, 0xecb: 0x2000, - 0xecc: 0x2000, 0xecd: 0x2000, 0xece: 0x2000, 0xecf: 0x2000, 0xed0: 0x4000, 0xed1: 0x4000, - 0xed2: 0x4000, 0xed3: 0x4000, 0xed4: 0x4000, 0xed5: 0x4000, 0xed6: 0x4000, 0xed7: 0x4000, - 0xed8: 0x4000, 0xed9: 0x4000, 0xeda: 0x4000, 0xedb: 0x4000, 0xedc: 0x4000, 0xedd: 0x4000, - 0xede: 0x4000, 0xedf: 0x4000, 0xee0: 0x4000, 0xee1: 0x4000, 0xee2: 0x4000, 0xee3: 0x4000, - 0xee4: 0x4000, 0xee5: 0x4000, 0xee6: 0x4000, 0xee7: 0x4000, 0xee8: 0x4000, 0xee9: 0x4000, - 0xeea: 0x4000, 0xeeb: 0x4000, 0xeec: 0x4000, 0xeed: 0x4000, 0xeee: 0x4000, 0xeef: 0x4000, - 0xef0: 0x4000, 0xef1: 0x4000, 0xef2: 0x4000, 0xef3: 0x4000, 0xef4: 0x4000, 0xef5: 0x4000, - 0xef6: 0x4000, 0xef7: 0x4000, 0xef8: 0x4000, 0xef9: 0x4000, 0xefa: 0x4000, 0xefb: 0x4000, - 0xefc: 0x4000, 0xefd: 0x4000, 0xefe: 0x4000, 0xeff: 0x4000, - // Block 0x3c, offset 0xf00 - 0xf00: 0x4000, 0xf01: 0x4000, 0xf02: 0x4000, 0xf03: 0x4000, 0xf04: 0x4000, 0xf05: 0x4000, - 0xf06: 0x4000, 0xf07: 0x4000, 0xf08: 0x4000, 0xf09: 0x4000, 0xf0a: 0x4000, 0xf0b: 0x4000, - 0xf0c: 0x4000, 0xf10: 0x4000, 0xf11: 0x4000, - 0xf12: 0x4000, 0xf13: 0x4000, 0xf14: 0x4000, 0xf15: 0x4000, 0xf16: 0x4000, 0xf17: 0x4000, - 0xf18: 0x4000, 0xf19: 0x4000, 0xf1a: 0x4000, 0xf1b: 0x4000, 0xf1c: 0x4000, 0xf1d: 0x4000, - 0xf1e: 0x4000, 0xf1f: 0x4000, 0xf20: 0x4000, 0xf21: 0x4000, 0xf22: 0x4000, 0xf23: 0x4000, - 0xf24: 0x4000, 0xf25: 0x4000, 0xf26: 0x4000, 0xf27: 0x4000, 0xf28: 0x4000, 0xf29: 0x4000, - 0xf2a: 0x4000, 0xf2b: 0x4000, 0xf2c: 0x4000, 0xf2d: 0x4000, 0xf2e: 0x4000, 0xf2f: 0x4000, - 0xf30: 0x4000, 0xf31: 0x4000, 0xf32: 0x4000, 0xf33: 0x4000, 0xf34: 0x4000, 0xf35: 0x4000, - 0xf36: 0x4000, 0xf37: 0x4000, 0xf38: 0x4000, 0xf39: 0x4000, 0xf3a: 0x4000, 0xf3b: 0x4000, - 0xf3c: 0x4000, 0xf3d: 0x4000, 0xf3e: 0x4000, 0xf3f: 0x4000, - // Block 0x3d, offset 0xf40 - 0xf40: 0x4000, 0xf41: 0x4000, 0xf42: 0x4000, 0xf43: 0x4000, 0xf44: 0x4000, 0xf45: 0x4000, - 0xf46: 0x4000, - // Block 0x3e, offset 0xf80 - 0xfa0: 0x4000, 0xfa1: 0x4000, 0xfa2: 0x4000, 0xfa3: 0x4000, - 0xfa4: 0x4000, 0xfa5: 0x4000, 0xfa6: 0x4000, 0xfa7: 0x4000, 0xfa8: 0x4000, 0xfa9: 0x4000, - 0xfaa: 0x4000, 0xfab: 0x4000, 0xfac: 0x4000, 0xfad: 0x4000, 0xfae: 0x4000, 0xfaf: 0x4000, - 0xfb0: 0x4000, 0xfb1: 0x4000, 0xfb2: 0x4000, 0xfb3: 0x4000, 0xfb4: 0x4000, 0xfb5: 0x4000, - 0xfb6: 0x4000, 0xfb7: 0x4000, 0xfb8: 0x4000, 0xfb9: 0x4000, 0xfba: 0x4000, 0xfbb: 0x4000, - 0xfbc: 0x4000, - // Block 0x3f, offset 0xfc0 - 0xfc0: 0x4000, 0xfc1: 0x4000, 0xfc2: 0x4000, 0xfc3: 0x4000, 0xfc4: 0x4000, 0xfc5: 0x4000, - 0xfc6: 0x4000, 0xfc7: 0x4000, 0xfc8: 0x4000, 0xfc9: 0x4000, 0xfca: 0x4000, 0xfcb: 0x4000, - 0xfcc: 0x4000, 0xfcd: 0x4000, 0xfce: 0x4000, 0xfcf: 0x4000, 0xfd0: 0x4000, 0xfd1: 0x4000, - 0xfd2: 0x4000, 0xfd3: 0x4000, 0xfd4: 0x4000, 0xfd5: 0x4000, 0xfd6: 0x4000, 0xfd7: 0x4000, - 0xfd8: 0x4000, 0xfd9: 0x4000, 0xfda: 0x4000, 0xfdb: 0x4000, 0xfdc: 0x4000, 0xfdd: 0x4000, - 0xfde: 0x4000, 0xfdf: 0x4000, 0xfe0: 0x4000, 0xfe1: 0x4000, 0xfe2: 0x4000, 0xfe3: 0x4000, - // Block 0x40, offset 0x1000 - 0x1000: 0x2000, 0x1001: 0x2000, 0x1002: 0x2000, 0x1003: 0x2000, 0x1004: 0x2000, 0x1005: 0x2000, - 0x1006: 0x2000, 0x1007: 0x2000, 0x1008: 0x2000, 0x1009: 0x2000, 0x100a: 0x2000, 0x100b: 0x2000, - 0x100c: 0x2000, 0x100d: 0x2000, 0x100e: 0x2000, 0x100f: 0x2000, 0x1010: 0x4000, 0x1011: 0x4000, - 0x1012: 0x4000, 0x1013: 0x4000, 0x1014: 0x4000, 0x1015: 0x4000, 0x1016: 0x4000, 0x1017: 0x4000, - 0x1018: 0x4000, 0x1019: 0x4000, - 0x1030: 0x4000, 0x1031: 0x4000, 0x1032: 0x4000, 0x1033: 0x4000, 0x1034: 0x4000, 0x1035: 0x4000, - 0x1036: 0x4000, 0x1037: 0x4000, 0x1038: 0x4000, 0x1039: 0x4000, 0x103a: 0x4000, 0x103b: 0x4000, - 0x103c: 0x4000, 0x103d: 0x4000, 0x103e: 0x4000, 0x103f: 0x4000, - // Block 0x41, offset 0x1040 - 0x1040: 0x4000, 0x1041: 0x4000, 0x1042: 0x4000, 0x1043: 0x4000, 0x1044: 0x4000, 0x1045: 0x4000, - 0x1046: 0x4000, 0x1047: 0x4000, 0x1048: 0x4000, 0x1049: 0x4000, 0x104a: 0x4000, 0x104b: 0x4000, - 0x104c: 0x4000, 0x104d: 0x4000, 0x104e: 0x4000, 0x104f: 0x4000, 0x1050: 0x4000, 0x1051: 0x4000, - 0x1052: 0x4000, 0x1054: 0x4000, 0x1055: 0x4000, 0x1056: 0x4000, 0x1057: 0x4000, - 0x1058: 0x4000, 0x1059: 0x4000, 0x105a: 0x4000, 0x105b: 0x4000, 0x105c: 0x4000, 0x105d: 0x4000, - 0x105e: 0x4000, 0x105f: 0x4000, 0x1060: 0x4000, 0x1061: 0x4000, 0x1062: 0x4000, 0x1063: 0x4000, - 0x1064: 0x4000, 0x1065: 0x4000, 0x1066: 0x4000, 0x1068: 0x4000, 0x1069: 0x4000, - 0x106a: 0x4000, 0x106b: 0x4000, - // Block 0x42, offset 0x1080 - 0x1081: 0x9012, 0x1082: 0x9012, 0x1083: 0x9012, 0x1084: 0x9012, 0x1085: 0x9012, - 0x1086: 0x9012, 0x1087: 0x9012, 0x1088: 0x9012, 0x1089: 0x9012, 0x108a: 0x9012, 0x108b: 0x9012, - 0x108c: 0x9012, 0x108d: 0x9012, 0x108e: 0x9012, 0x108f: 0x9012, 0x1090: 0x9012, 0x1091: 0x9012, - 0x1092: 0x9012, 0x1093: 0x9012, 0x1094: 0x9012, 0x1095: 0x9012, 0x1096: 0x9012, 0x1097: 0x9012, - 0x1098: 0x9012, 0x1099: 0x9012, 0x109a: 0x9012, 0x109b: 0x9012, 0x109c: 0x9012, 0x109d: 0x9012, - 0x109e: 0x9012, 0x109f: 0x9012, 0x10a0: 0x9049, 0x10a1: 0x9049, 0x10a2: 0x9049, 0x10a3: 0x9049, - 0x10a4: 0x9049, 0x10a5: 0x9049, 0x10a6: 0x9049, 0x10a7: 0x9049, 0x10a8: 0x9049, 0x10a9: 0x9049, - 0x10aa: 0x9049, 0x10ab: 0x9049, 0x10ac: 0x9049, 0x10ad: 0x9049, 0x10ae: 0x9049, 0x10af: 0x9049, - 0x10b0: 0x9049, 0x10b1: 0x9049, 0x10b2: 0x9049, 0x10b3: 0x9049, 0x10b4: 0x9049, 0x10b5: 0x9049, - 0x10b6: 0x9049, 0x10b7: 0x9049, 0x10b8: 0x9049, 0x10b9: 0x9049, 0x10ba: 0x9049, 0x10bb: 0x9049, - 0x10bc: 0x9049, 0x10bd: 0x9049, 0x10be: 0x9049, 0x10bf: 0x9049, - // Block 0x43, offset 0x10c0 - 0x10c0: 0x9049, 0x10c1: 0x9049, 0x10c2: 0x9049, 0x10c3: 0x9049, 0x10c4: 0x9049, 0x10c5: 0x9049, - 0x10c6: 0x9049, 0x10c7: 0x9049, 0x10c8: 0x9049, 0x10c9: 0x9049, 0x10ca: 0x9049, 0x10cb: 0x9049, - 0x10cc: 0x9049, 0x10cd: 0x9049, 0x10ce: 0x9049, 0x10cf: 0x9049, 0x10d0: 0x9049, 0x10d1: 0x9049, - 0x10d2: 0x9049, 0x10d3: 0x9049, 0x10d4: 0x9049, 0x10d5: 0x9049, 0x10d6: 0x9049, 0x10d7: 0x9049, - 0x10d8: 0x9049, 0x10d9: 0x9049, 0x10da: 0x9049, 0x10db: 0x9049, 0x10dc: 0x9049, 0x10dd: 0x9049, - 0x10de: 0x9049, 0x10df: 0x904a, 0x10e0: 0x904b, 0x10e1: 0xb04c, 0x10e2: 0xb04d, 0x10e3: 0xb04d, - 0x10e4: 0xb04e, 0x10e5: 0xb04f, 0x10e6: 0xb050, 0x10e7: 0xb051, 0x10e8: 0xb052, 0x10e9: 0xb053, - 0x10ea: 0xb054, 0x10eb: 0xb055, 0x10ec: 0xb056, 0x10ed: 0xb057, 0x10ee: 0xb058, 0x10ef: 0xb059, - 0x10f0: 0xb05a, 0x10f1: 0xb05b, 0x10f2: 0xb05c, 0x10f3: 0xb05d, 0x10f4: 0xb05e, 0x10f5: 0xb05f, - 0x10f6: 0xb060, 0x10f7: 0xb061, 0x10f8: 0xb062, 0x10f9: 0xb063, 0x10fa: 0xb064, 0x10fb: 0xb065, - 0x10fc: 0xb052, 0x10fd: 0xb066, 0x10fe: 0xb067, 0x10ff: 0xb055, - // Block 0x44, offset 0x1100 - 0x1100: 0xb068, 0x1101: 0xb069, 0x1102: 0xb06a, 0x1103: 0xb06b, 0x1104: 0xb05a, 0x1105: 0xb056, - 0x1106: 0xb06c, 0x1107: 0xb06d, 0x1108: 0xb06b, 0x1109: 0xb06e, 0x110a: 0xb06b, 0x110b: 0xb06f, - 0x110c: 0xb06f, 0x110d: 0xb070, 0x110e: 0xb070, 0x110f: 0xb071, 0x1110: 0xb056, 0x1111: 0xb072, - 0x1112: 0xb073, 0x1113: 0xb072, 0x1114: 0xb074, 0x1115: 0xb073, 0x1116: 0xb075, 0x1117: 0xb075, - 0x1118: 0xb076, 0x1119: 0xb076, 0x111a: 0xb077, 0x111b: 0xb077, 0x111c: 0xb073, 0x111d: 0xb078, - 0x111e: 0xb079, 0x111f: 0xb067, 0x1120: 0xb07a, 0x1121: 0xb07b, 0x1122: 0xb07b, 0x1123: 0xb07b, - 0x1124: 0xb07b, 0x1125: 0xb07b, 0x1126: 0xb07b, 0x1127: 0xb07b, 0x1128: 0xb07b, 0x1129: 0xb07b, - 0x112a: 0xb07b, 0x112b: 0xb07b, 0x112c: 0xb07b, 0x112d: 0xb07b, 0x112e: 0xb07b, 0x112f: 0xb07b, - 0x1130: 0xb07c, 0x1131: 0xb07c, 0x1132: 0xb07c, 0x1133: 0xb07c, 0x1134: 0xb07c, 0x1135: 0xb07c, - 0x1136: 0xb07c, 0x1137: 0xb07c, 0x1138: 0xb07c, 0x1139: 0xb07c, 0x113a: 0xb07c, 0x113b: 0xb07c, - 0x113c: 0xb07c, 0x113d: 0xb07c, 0x113e: 0xb07c, - // Block 0x45, offset 0x1140 - 0x1142: 0xb07d, 0x1143: 0xb07e, 0x1144: 0xb07f, 0x1145: 0xb080, - 0x1146: 0xb07f, 0x1147: 0xb07e, 0x114a: 0xb081, 0x114b: 0xb082, - 0x114c: 0xb083, 0x114d: 0xb07f, 0x114e: 0xb080, 0x114f: 0xb07f, - 0x1152: 0xb084, 0x1153: 0xb085, 0x1154: 0xb084, 0x1155: 0xb086, 0x1156: 0xb084, 0x1157: 0xb087, - 0x115a: 0xb088, 0x115b: 0xb089, 0x115c: 0xb08a, - 0x1160: 0x908b, 0x1161: 0x908b, 0x1162: 0x908c, 0x1163: 0x908d, - 0x1164: 0x908b, 0x1165: 0x908e, 0x1166: 0x908f, 0x1168: 0xb090, 0x1169: 0xb091, - 0x116a: 0xb092, 0x116b: 0xb091, 0x116c: 0xb093, 0x116d: 0xb094, 0x116e: 0xb095, - 0x117d: 0x2000, - // Block 0x46, offset 0x1180 - 0x11a0: 0x4000, 0x11a1: 0x4000, 0x11a2: 0x4000, 0x11a3: 0x4000, - 0x11a4: 0x4000, - 0x11b0: 0x4000, 0x11b1: 0x4000, - // Block 0x47, offset 0x11c0 - 0x11c0: 0x4000, 0x11c1: 0x4000, 0x11c2: 0x4000, 0x11c3: 0x4000, 0x11c4: 0x4000, 0x11c5: 0x4000, - 0x11c6: 0x4000, 0x11c7: 0x4000, 0x11c8: 0x4000, 0x11c9: 0x4000, 0x11ca: 0x4000, 0x11cb: 0x4000, - 0x11cc: 0x4000, 0x11cd: 0x4000, 0x11ce: 0x4000, 0x11cf: 0x4000, 0x11d0: 0x4000, 0x11d1: 0x4000, - 0x11d2: 0x4000, 0x11d3: 0x4000, 0x11d4: 0x4000, 0x11d5: 0x4000, 0x11d6: 0x4000, 0x11d7: 0x4000, - 0x11d8: 0x4000, 0x11d9: 0x4000, 0x11da: 0x4000, 0x11db: 0x4000, 0x11dc: 0x4000, 0x11dd: 0x4000, - 0x11de: 0x4000, 0x11df: 0x4000, 0x11e0: 0x4000, 0x11e1: 0x4000, 0x11e2: 0x4000, 0x11e3: 0x4000, - 0x11e4: 0x4000, 0x11e5: 0x4000, 0x11e6: 0x4000, 0x11e7: 0x4000, 0x11e8: 0x4000, 0x11e9: 0x4000, - 0x11ea: 0x4000, 0x11eb: 0x4000, 0x11ec: 0x4000, 0x11ed: 0x4000, 0x11ee: 0x4000, 0x11ef: 0x4000, - 0x11f0: 0x4000, 0x11f1: 0x4000, 0x11f2: 0x4000, 0x11f3: 0x4000, 0x11f4: 0x4000, 0x11f5: 0x4000, - 0x11f6: 0x4000, 0x11f7: 0x4000, - // Block 0x48, offset 0x1200 - 0x1200: 0x4000, 0x1201: 0x4000, 0x1202: 0x4000, 0x1203: 0x4000, 0x1204: 0x4000, 0x1205: 0x4000, - 0x1206: 0x4000, 0x1207: 0x4000, 0x1208: 0x4000, 0x1209: 0x4000, 0x120a: 0x4000, 0x120b: 0x4000, - 0x120c: 0x4000, 0x120d: 0x4000, 0x120e: 0x4000, 0x120f: 0x4000, 0x1210: 0x4000, 0x1211: 0x4000, - 0x1212: 0x4000, 0x1213: 0x4000, 0x1214: 0x4000, 0x1215: 0x4000, - // Block 0x49, offset 0x1240 - 0x1240: 0x4000, 0x1241: 0x4000, 0x1242: 0x4000, 0x1243: 0x4000, 0x1244: 0x4000, 0x1245: 0x4000, - 0x1246: 0x4000, 0x1247: 0x4000, 0x1248: 0x4000, - // Block 0x4a, offset 0x1280 - 0x12b0: 0x4000, 0x12b1: 0x4000, 0x12b2: 0x4000, 0x12b3: 0x4000, 0x12b5: 0x4000, - 0x12b6: 0x4000, 0x12b7: 0x4000, 0x12b8: 0x4000, 0x12b9: 0x4000, 0x12ba: 0x4000, 0x12bb: 0x4000, - 0x12bd: 0x4000, 0x12be: 0x4000, - // Block 0x4b, offset 0x12c0 - 0x12c0: 0x4000, 0x12c1: 0x4000, 0x12c2: 0x4000, 0x12c3: 0x4000, 0x12c4: 0x4000, 0x12c5: 0x4000, - 0x12c6: 0x4000, 0x12c7: 0x4000, 0x12c8: 0x4000, 0x12c9: 0x4000, 0x12ca: 0x4000, 0x12cb: 0x4000, - 0x12cc: 0x4000, 0x12cd: 0x4000, 0x12ce: 0x4000, 0x12cf: 0x4000, 0x12d0: 0x4000, 0x12d1: 0x4000, - 0x12d2: 0x4000, 0x12d3: 0x4000, 0x12d4: 0x4000, 0x12d5: 0x4000, 0x12d6: 0x4000, 0x12d7: 0x4000, - 0x12d8: 0x4000, 0x12d9: 0x4000, 0x12da: 0x4000, 0x12db: 0x4000, 0x12dc: 0x4000, 0x12dd: 0x4000, - 0x12de: 0x4000, 0x12df: 0x4000, 0x12e0: 0x4000, 0x12e1: 0x4000, 0x12e2: 0x4000, - 0x12f2: 0x4000, - // Block 0x4c, offset 0x1300 - 0x1310: 0x4000, 0x1311: 0x4000, - 0x1312: 0x4000, 0x1315: 0x4000, - 0x1324: 0x4000, 0x1325: 0x4000, 0x1326: 0x4000, 0x1327: 0x4000, - 0x1330: 0x4000, 0x1331: 0x4000, 0x1332: 0x4000, 0x1333: 0x4000, 0x1334: 0x4000, 0x1335: 0x4000, - 0x1336: 0x4000, 0x1337: 0x4000, 0x1338: 0x4000, 0x1339: 0x4000, 0x133a: 0x4000, 0x133b: 0x4000, - 0x133c: 0x4000, 0x133d: 0x4000, 0x133e: 0x4000, 0x133f: 0x4000, - // Block 0x4d, offset 0x1340 - 0x1340: 0x4000, 0x1341: 0x4000, 0x1342: 0x4000, 0x1343: 0x4000, 0x1344: 0x4000, 0x1345: 0x4000, - 0x1346: 0x4000, 0x1347: 0x4000, 0x1348: 0x4000, 0x1349: 0x4000, 0x134a: 0x4000, 0x134b: 0x4000, - 0x134c: 0x4000, 0x134d: 0x4000, 0x134e: 0x4000, 0x134f: 0x4000, 0x1350: 0x4000, 0x1351: 0x4000, - 0x1352: 0x4000, 0x1353: 0x4000, 0x1354: 0x4000, 0x1355: 0x4000, 0x1356: 0x4000, 0x1357: 0x4000, - 0x1358: 0x4000, 0x1359: 0x4000, 0x135a: 0x4000, 0x135b: 0x4000, 0x135c: 0x4000, 0x135d: 0x4000, - 0x135e: 0x4000, 0x135f: 0x4000, 0x1360: 0x4000, 0x1361: 0x4000, 0x1362: 0x4000, 0x1363: 0x4000, - 0x1364: 0x4000, 0x1365: 0x4000, 0x1366: 0x4000, 0x1367: 0x4000, 0x1368: 0x4000, 0x1369: 0x4000, - 0x136a: 0x4000, 0x136b: 0x4000, 0x136c: 0x4000, 0x136d: 0x4000, 0x136e: 0x4000, 0x136f: 0x4000, - 0x1370: 0x4000, 0x1371: 0x4000, 0x1372: 0x4000, 0x1373: 0x4000, 0x1374: 0x4000, 0x1375: 0x4000, - 0x1376: 0x4000, 0x1377: 0x4000, 0x1378: 0x4000, 0x1379: 0x4000, 0x137a: 0x4000, 0x137b: 0x4000, - // Block 0x4e, offset 0x1380 - 0x1384: 0x4000, - // Block 0x4f, offset 0x13c0 - 0x13cf: 0x4000, - // Block 0x50, offset 0x1400 - 0x1400: 0x2000, 0x1401: 0x2000, 0x1402: 0x2000, 0x1403: 0x2000, 0x1404: 0x2000, 0x1405: 0x2000, - 0x1406: 0x2000, 0x1407: 0x2000, 0x1408: 0x2000, 0x1409: 0x2000, 0x140a: 0x2000, - 0x1410: 0x2000, 0x1411: 0x2000, - 0x1412: 0x2000, 0x1413: 0x2000, 0x1414: 0x2000, 0x1415: 0x2000, 0x1416: 0x2000, 0x1417: 0x2000, - 0x1418: 0x2000, 0x1419: 0x2000, 0x141a: 0x2000, 0x141b: 0x2000, 0x141c: 0x2000, 0x141d: 0x2000, - 0x141e: 0x2000, 0x141f: 0x2000, 0x1420: 0x2000, 0x1421: 0x2000, 0x1422: 0x2000, 0x1423: 0x2000, - 0x1424: 0x2000, 0x1425: 0x2000, 0x1426: 0x2000, 0x1427: 0x2000, 0x1428: 0x2000, 0x1429: 0x2000, - 0x142a: 0x2000, 0x142b: 0x2000, 0x142c: 0x2000, 0x142d: 0x2000, - 0x1430: 0x2000, 0x1431: 0x2000, 0x1432: 0x2000, 0x1433: 0x2000, 0x1434: 0x2000, 0x1435: 0x2000, - 0x1436: 0x2000, 0x1437: 0x2000, 0x1438: 0x2000, 0x1439: 0x2000, 0x143a: 0x2000, 0x143b: 0x2000, - 0x143c: 0x2000, 0x143d: 0x2000, 0x143e: 0x2000, 0x143f: 0x2000, - // Block 0x51, offset 0x1440 - 0x1440: 0x2000, 0x1441: 0x2000, 0x1442: 0x2000, 0x1443: 0x2000, 0x1444: 0x2000, 0x1445: 0x2000, - 0x1446: 0x2000, 0x1447: 0x2000, 0x1448: 0x2000, 0x1449: 0x2000, 0x144a: 0x2000, 0x144b: 0x2000, - 0x144c: 0x2000, 0x144d: 0x2000, 0x144e: 0x2000, 0x144f: 0x2000, 0x1450: 0x2000, 0x1451: 0x2000, - 0x1452: 0x2000, 0x1453: 0x2000, 0x1454: 0x2000, 0x1455: 0x2000, 0x1456: 0x2000, 0x1457: 0x2000, - 0x1458: 0x2000, 0x1459: 0x2000, 0x145a: 0x2000, 0x145b: 0x2000, 0x145c: 0x2000, 0x145d: 0x2000, - 0x145e: 0x2000, 0x145f: 0x2000, 0x1460: 0x2000, 0x1461: 0x2000, 0x1462: 0x2000, 0x1463: 0x2000, - 0x1464: 0x2000, 0x1465: 0x2000, 0x1466: 0x2000, 0x1467: 0x2000, 0x1468: 0x2000, 0x1469: 0x2000, - 0x1470: 0x2000, 0x1471: 0x2000, 0x1472: 0x2000, 0x1473: 0x2000, 0x1474: 0x2000, 0x1475: 0x2000, - 0x1476: 0x2000, 0x1477: 0x2000, 0x1478: 0x2000, 0x1479: 0x2000, 0x147a: 0x2000, 0x147b: 0x2000, - 0x147c: 0x2000, 0x147d: 0x2000, 0x147e: 0x2000, 0x147f: 0x2000, - // Block 0x52, offset 0x1480 - 0x1480: 0x2000, 0x1481: 0x2000, 0x1482: 0x2000, 0x1483: 0x2000, 0x1484: 0x2000, 0x1485: 0x2000, - 0x1486: 0x2000, 0x1487: 0x2000, 0x1488: 0x2000, 0x1489: 0x2000, 0x148a: 0x2000, 0x148b: 0x2000, - 0x148c: 0x2000, 0x148d: 0x2000, 0x148e: 0x4000, 0x148f: 0x2000, 0x1490: 0x2000, 0x1491: 0x4000, - 0x1492: 0x4000, 0x1493: 0x4000, 0x1494: 0x4000, 0x1495: 0x4000, 0x1496: 0x4000, 0x1497: 0x4000, - 0x1498: 0x4000, 0x1499: 0x4000, 0x149a: 0x4000, 0x149b: 0x2000, 0x149c: 0x2000, 0x149d: 0x2000, - 0x149e: 0x2000, 0x149f: 0x2000, 0x14a0: 0x2000, 0x14a1: 0x2000, 0x14a2: 0x2000, 0x14a3: 0x2000, - 0x14a4: 0x2000, 0x14a5: 0x2000, 0x14a6: 0x2000, 0x14a7: 0x2000, 0x14a8: 0x2000, 0x14a9: 0x2000, - 0x14aa: 0x2000, 0x14ab: 0x2000, 0x14ac: 0x2000, - // Block 0x53, offset 0x14c0 - 0x14c0: 0x4000, 0x14c1: 0x4000, 0x14c2: 0x4000, - 0x14d0: 0x4000, 0x14d1: 0x4000, - 0x14d2: 0x4000, 0x14d3: 0x4000, 0x14d4: 0x4000, 0x14d5: 0x4000, 0x14d6: 0x4000, 0x14d7: 0x4000, - 0x14d8: 0x4000, 0x14d9: 0x4000, 0x14da: 0x4000, 0x14db: 0x4000, 0x14dc: 0x4000, 0x14dd: 0x4000, - 0x14de: 0x4000, 0x14df: 0x4000, 0x14e0: 0x4000, 0x14e1: 0x4000, 0x14e2: 0x4000, 0x14e3: 0x4000, - 0x14e4: 0x4000, 0x14e5: 0x4000, 0x14e6: 0x4000, 0x14e7: 0x4000, 0x14e8: 0x4000, 0x14e9: 0x4000, - 0x14ea: 0x4000, 0x14eb: 0x4000, 0x14ec: 0x4000, 0x14ed: 0x4000, 0x14ee: 0x4000, 0x14ef: 0x4000, - 0x14f0: 0x4000, 0x14f1: 0x4000, 0x14f2: 0x4000, 0x14f3: 0x4000, 0x14f4: 0x4000, 0x14f5: 0x4000, - 0x14f6: 0x4000, 0x14f7: 0x4000, 0x14f8: 0x4000, 0x14f9: 0x4000, 0x14fa: 0x4000, 0x14fb: 0x4000, - // Block 0x54, offset 0x1500 - 0x1500: 0x4000, 0x1501: 0x4000, 0x1502: 0x4000, 0x1503: 0x4000, 0x1504: 0x4000, 0x1505: 0x4000, - 0x1506: 0x4000, 0x1507: 0x4000, 0x1508: 0x4000, - 0x1510: 0x4000, 0x1511: 0x4000, - 0x1520: 0x4000, 0x1521: 0x4000, 0x1522: 0x4000, 0x1523: 0x4000, - 0x1524: 0x4000, 0x1525: 0x4000, - // Block 0x55, offset 0x1540 - 0x1540: 0x4000, 0x1541: 0x4000, 0x1542: 0x4000, 0x1543: 0x4000, 0x1544: 0x4000, 0x1545: 0x4000, - 0x1546: 0x4000, 0x1547: 0x4000, 0x1548: 0x4000, 0x1549: 0x4000, 0x154a: 0x4000, 0x154b: 0x4000, - 0x154c: 0x4000, 0x154d: 0x4000, 0x154e: 0x4000, 0x154f: 0x4000, 0x1550: 0x4000, 0x1551: 0x4000, - 0x1552: 0x4000, 0x1553: 0x4000, 0x1554: 0x4000, 0x1555: 0x4000, 0x1556: 0x4000, 0x1557: 0x4000, - 0x1558: 0x4000, 0x1559: 0x4000, 0x155a: 0x4000, 0x155b: 0x4000, 0x155c: 0x4000, 0x155d: 0x4000, - 0x155e: 0x4000, 0x155f: 0x4000, 0x1560: 0x4000, - 0x156d: 0x4000, 0x156e: 0x4000, 0x156f: 0x4000, - 0x1570: 0x4000, 0x1571: 0x4000, 0x1572: 0x4000, 0x1573: 0x4000, 0x1574: 0x4000, 0x1575: 0x4000, - 0x1577: 0x4000, 0x1578: 0x4000, 0x1579: 0x4000, 0x157a: 0x4000, 0x157b: 0x4000, - 0x157c: 0x4000, 0x157d: 0x4000, 0x157e: 0x4000, 0x157f: 0x4000, - // Block 0x56, offset 0x1580 - 0x1580: 0x4000, 0x1581: 0x4000, 0x1582: 0x4000, 0x1583: 0x4000, 0x1584: 0x4000, 0x1585: 0x4000, - 0x1586: 0x4000, 0x1587: 0x4000, 0x1588: 0x4000, 0x1589: 0x4000, 0x158a: 0x4000, 0x158b: 0x4000, - 0x158c: 0x4000, 0x158d: 0x4000, 0x158e: 0x4000, 0x158f: 0x4000, 0x1590: 0x4000, 0x1591: 0x4000, - 0x1592: 0x4000, 0x1593: 0x4000, 0x1594: 0x4000, 0x1595: 0x4000, 0x1596: 0x4000, 0x1597: 0x4000, - 0x1598: 0x4000, 0x1599: 0x4000, 0x159a: 0x4000, 0x159b: 0x4000, 0x159c: 0x4000, 0x159d: 0x4000, - 0x159e: 0x4000, 0x159f: 0x4000, 0x15a0: 0x4000, 0x15a1: 0x4000, 0x15a2: 0x4000, 0x15a3: 0x4000, - 0x15a4: 0x4000, 0x15a5: 0x4000, 0x15a6: 0x4000, 0x15a7: 0x4000, 0x15a8: 0x4000, 0x15a9: 0x4000, - 0x15aa: 0x4000, 0x15ab: 0x4000, 0x15ac: 0x4000, 0x15ad: 0x4000, 0x15ae: 0x4000, 0x15af: 0x4000, - 0x15b0: 0x4000, 0x15b1: 0x4000, 0x15b2: 0x4000, 0x15b3: 0x4000, 0x15b4: 0x4000, 0x15b5: 0x4000, - 0x15b6: 0x4000, 0x15b7: 0x4000, 0x15b8: 0x4000, 0x15b9: 0x4000, 0x15ba: 0x4000, 0x15bb: 0x4000, - 0x15bc: 0x4000, 0x15be: 0x4000, 0x15bf: 0x4000, - // Block 0x57, offset 0x15c0 - 0x15c0: 0x4000, 0x15c1: 0x4000, 0x15c2: 0x4000, 0x15c3: 0x4000, 0x15c4: 0x4000, 0x15c5: 0x4000, - 0x15c6: 0x4000, 0x15c7: 0x4000, 0x15c8: 0x4000, 0x15c9: 0x4000, 0x15ca: 0x4000, 0x15cb: 0x4000, - 0x15cc: 0x4000, 0x15cd: 0x4000, 0x15ce: 0x4000, 0x15cf: 0x4000, 0x15d0: 0x4000, 0x15d1: 0x4000, - 0x15d2: 0x4000, 0x15d3: 0x4000, - 0x15e0: 0x4000, 0x15e1: 0x4000, 0x15e2: 0x4000, 0x15e3: 0x4000, - 0x15e4: 0x4000, 0x15e5: 0x4000, 0x15e6: 0x4000, 0x15e7: 0x4000, 0x15e8: 0x4000, 0x15e9: 0x4000, - 0x15ea: 0x4000, 0x15eb: 0x4000, 0x15ec: 0x4000, 0x15ed: 0x4000, 0x15ee: 0x4000, 0x15ef: 0x4000, - 0x15f0: 0x4000, 0x15f1: 0x4000, 0x15f2: 0x4000, 0x15f3: 0x4000, 0x15f4: 0x4000, 0x15f5: 0x4000, - 0x15f6: 0x4000, 0x15f7: 0x4000, 0x15f8: 0x4000, 0x15f9: 0x4000, 0x15fa: 0x4000, 0x15fb: 0x4000, - 0x15fc: 0x4000, 0x15fd: 0x4000, 0x15fe: 0x4000, 0x15ff: 0x4000, - // Block 0x58, offset 0x1600 - 0x1600: 0x4000, 0x1601: 0x4000, 0x1602: 0x4000, 0x1603: 0x4000, 0x1604: 0x4000, 0x1605: 0x4000, - 0x1606: 0x4000, 0x1607: 0x4000, 0x1608: 0x4000, 0x1609: 0x4000, 0x160a: 0x4000, - 0x160f: 0x4000, 0x1610: 0x4000, 0x1611: 0x4000, - 0x1612: 0x4000, 0x1613: 0x4000, - 0x1620: 0x4000, 0x1621: 0x4000, 0x1622: 0x4000, 0x1623: 0x4000, - 0x1624: 0x4000, 0x1625: 0x4000, 0x1626: 0x4000, 0x1627: 0x4000, 0x1628: 0x4000, 0x1629: 0x4000, - 0x162a: 0x4000, 0x162b: 0x4000, 0x162c: 0x4000, 0x162d: 0x4000, 0x162e: 0x4000, 0x162f: 0x4000, - 0x1630: 0x4000, 0x1634: 0x4000, - 0x1638: 0x4000, 0x1639: 0x4000, 0x163a: 0x4000, 0x163b: 0x4000, - 0x163c: 0x4000, 0x163d: 0x4000, 0x163e: 0x4000, 0x163f: 0x4000, - // Block 0x59, offset 0x1640 - 0x1640: 0x4000, 0x1641: 0x4000, 0x1642: 0x4000, 0x1643: 0x4000, 0x1644: 0x4000, 0x1645: 0x4000, - 0x1646: 0x4000, 0x1647: 0x4000, 0x1648: 0x4000, 0x1649: 0x4000, 0x164a: 0x4000, 0x164b: 0x4000, - 0x164c: 0x4000, 0x164d: 0x4000, 0x164e: 0x4000, 0x164f: 0x4000, 0x1650: 0x4000, 0x1651: 0x4000, - 0x1652: 0x4000, 0x1653: 0x4000, 0x1654: 0x4000, 0x1655: 0x4000, 0x1656: 0x4000, 0x1657: 0x4000, - 0x1658: 0x4000, 0x1659: 0x4000, 0x165a: 0x4000, 0x165b: 0x4000, 0x165c: 0x4000, 0x165d: 0x4000, - 0x165e: 0x4000, 0x165f: 0x4000, 0x1660: 0x4000, 0x1661: 0x4000, 0x1662: 0x4000, 0x1663: 0x4000, - 0x1664: 0x4000, 0x1665: 0x4000, 0x1666: 0x4000, 0x1667: 0x4000, 0x1668: 0x4000, 0x1669: 0x4000, - 0x166a: 0x4000, 0x166b: 0x4000, 0x166c: 0x4000, 0x166d: 0x4000, 0x166e: 0x4000, 0x166f: 0x4000, - 0x1670: 0x4000, 0x1671: 0x4000, 0x1672: 0x4000, 0x1673: 0x4000, 0x1674: 0x4000, 0x1675: 0x4000, - 0x1676: 0x4000, 0x1677: 0x4000, 0x1678: 0x4000, 0x1679: 0x4000, 0x167a: 0x4000, 0x167b: 0x4000, - 0x167c: 0x4000, 0x167d: 0x4000, 0x167e: 0x4000, - // Block 0x5a, offset 0x1680 - 0x1680: 0x4000, 0x1682: 0x4000, 0x1683: 0x4000, 0x1684: 0x4000, 0x1685: 0x4000, - 0x1686: 0x4000, 0x1687: 0x4000, 0x1688: 0x4000, 0x1689: 0x4000, 0x168a: 0x4000, 0x168b: 0x4000, - 0x168c: 0x4000, 0x168d: 0x4000, 0x168e: 0x4000, 0x168f: 0x4000, 0x1690: 0x4000, 0x1691: 0x4000, - 0x1692: 0x4000, 0x1693: 0x4000, 0x1694: 0x4000, 0x1695: 0x4000, 0x1696: 0x4000, 0x1697: 0x4000, - 0x1698: 0x4000, 0x1699: 0x4000, 0x169a: 0x4000, 0x169b: 0x4000, 0x169c: 0x4000, 0x169d: 0x4000, - 0x169e: 0x4000, 0x169f: 0x4000, 0x16a0: 0x4000, 0x16a1: 0x4000, 0x16a2: 0x4000, 0x16a3: 0x4000, - 0x16a4: 0x4000, 0x16a5: 0x4000, 0x16a6: 0x4000, 0x16a7: 0x4000, 0x16a8: 0x4000, 0x16a9: 0x4000, - 0x16aa: 0x4000, 0x16ab: 0x4000, 0x16ac: 0x4000, 0x16ad: 0x4000, 0x16ae: 0x4000, 0x16af: 0x4000, - 0x16b0: 0x4000, 0x16b1: 0x4000, 0x16b2: 0x4000, 0x16b3: 0x4000, 0x16b4: 0x4000, 0x16b5: 0x4000, - 0x16b6: 0x4000, 0x16b7: 0x4000, 0x16b8: 0x4000, 0x16b9: 0x4000, 0x16ba: 0x4000, 0x16bb: 0x4000, - 0x16bc: 0x4000, 0x16bd: 0x4000, 0x16be: 0x4000, 0x16bf: 0x4000, - // Block 0x5b, offset 0x16c0 - 0x16c0: 0x4000, 0x16c1: 0x4000, 0x16c2: 0x4000, 0x16c3: 0x4000, 0x16c4: 0x4000, 0x16c5: 0x4000, - 0x16c6: 0x4000, 0x16c7: 0x4000, 0x16c8: 0x4000, 0x16c9: 0x4000, 0x16ca: 0x4000, 0x16cb: 0x4000, - 0x16cc: 0x4000, 0x16cd: 0x4000, 0x16ce: 0x4000, 0x16cf: 0x4000, 0x16d0: 0x4000, 0x16d1: 0x4000, - 0x16d2: 0x4000, 0x16d3: 0x4000, 0x16d4: 0x4000, 0x16d5: 0x4000, 0x16d6: 0x4000, 0x16d7: 0x4000, - 0x16d8: 0x4000, 0x16d9: 0x4000, 0x16da: 0x4000, 0x16db: 0x4000, 0x16dc: 0x4000, 0x16dd: 0x4000, - 0x16de: 0x4000, 0x16df: 0x4000, 0x16e0: 0x4000, 0x16e1: 0x4000, 0x16e2: 0x4000, 0x16e3: 0x4000, - 0x16e4: 0x4000, 0x16e5: 0x4000, 0x16e6: 0x4000, 0x16e7: 0x4000, 0x16e8: 0x4000, 0x16e9: 0x4000, - 0x16ea: 0x4000, 0x16eb: 0x4000, 0x16ec: 0x4000, 0x16ed: 0x4000, 0x16ee: 0x4000, 0x16ef: 0x4000, - 0x16f0: 0x4000, 0x16f1: 0x4000, 0x16f2: 0x4000, 0x16f3: 0x4000, 0x16f4: 0x4000, 0x16f5: 0x4000, - 0x16f6: 0x4000, 0x16f7: 0x4000, 0x16f8: 0x4000, 0x16f9: 0x4000, 0x16fa: 0x4000, 0x16fb: 0x4000, - 0x16fc: 0x4000, 0x16ff: 0x4000, - // Block 0x5c, offset 0x1700 - 0x1700: 0x4000, 0x1701: 0x4000, 0x1702: 0x4000, 0x1703: 0x4000, 0x1704: 0x4000, 0x1705: 0x4000, - 0x1706: 0x4000, 0x1707: 0x4000, 0x1708: 0x4000, 0x1709: 0x4000, 0x170a: 0x4000, 0x170b: 0x4000, - 0x170c: 0x4000, 0x170d: 0x4000, 0x170e: 0x4000, 0x170f: 0x4000, 0x1710: 0x4000, 0x1711: 0x4000, - 0x1712: 0x4000, 0x1713: 0x4000, 0x1714: 0x4000, 0x1715: 0x4000, 0x1716: 0x4000, 0x1717: 0x4000, - 0x1718: 0x4000, 0x1719: 0x4000, 0x171a: 0x4000, 0x171b: 0x4000, 0x171c: 0x4000, 0x171d: 0x4000, - 0x171e: 0x4000, 0x171f: 0x4000, 0x1720: 0x4000, 0x1721: 0x4000, 0x1722: 0x4000, 0x1723: 0x4000, - 0x1724: 0x4000, 0x1725: 0x4000, 0x1726: 0x4000, 0x1727: 0x4000, 0x1728: 0x4000, 0x1729: 0x4000, - 0x172a: 0x4000, 0x172b: 0x4000, 0x172c: 0x4000, 0x172d: 0x4000, 0x172e: 0x4000, 0x172f: 0x4000, - 0x1730: 0x4000, 0x1731: 0x4000, 0x1732: 0x4000, 0x1733: 0x4000, 0x1734: 0x4000, 0x1735: 0x4000, - 0x1736: 0x4000, 0x1737: 0x4000, 0x1738: 0x4000, 0x1739: 0x4000, 0x173a: 0x4000, 0x173b: 0x4000, - 0x173c: 0x4000, 0x173d: 0x4000, - // Block 0x5d, offset 0x1740 - 0x174b: 0x4000, - 0x174c: 0x4000, 0x174d: 0x4000, 0x174e: 0x4000, 0x1750: 0x4000, 0x1751: 0x4000, - 0x1752: 0x4000, 0x1753: 0x4000, 0x1754: 0x4000, 0x1755: 0x4000, 0x1756: 0x4000, 0x1757: 0x4000, - 0x1758: 0x4000, 0x1759: 0x4000, 0x175a: 0x4000, 0x175b: 0x4000, 0x175c: 0x4000, 0x175d: 0x4000, - 0x175e: 0x4000, 0x175f: 0x4000, 0x1760: 0x4000, 0x1761: 0x4000, 0x1762: 0x4000, 0x1763: 0x4000, - 0x1764: 0x4000, 0x1765: 0x4000, 0x1766: 0x4000, 0x1767: 0x4000, - 0x177a: 0x4000, - // Block 0x5e, offset 0x1780 - 0x1795: 0x4000, 0x1796: 0x4000, - 0x17a4: 0x4000, - // Block 0x5f, offset 0x17c0 - 0x17fb: 0x4000, - 0x17fc: 0x4000, 0x17fd: 0x4000, 0x17fe: 0x4000, 0x17ff: 0x4000, - // Block 0x60, offset 0x1800 - 0x1800: 0x4000, 0x1801: 0x4000, 0x1802: 0x4000, 0x1803: 0x4000, 0x1804: 0x4000, 0x1805: 0x4000, - 0x1806: 0x4000, 0x1807: 0x4000, 0x1808: 0x4000, 0x1809: 0x4000, 0x180a: 0x4000, 0x180b: 0x4000, - 0x180c: 0x4000, 0x180d: 0x4000, 0x180e: 0x4000, 0x180f: 0x4000, - // Block 0x61, offset 0x1840 - 0x1840: 0x4000, 0x1841: 0x4000, 0x1842: 0x4000, 0x1843: 0x4000, 0x1844: 0x4000, 0x1845: 0x4000, - 0x184c: 0x4000, 0x1850: 0x4000, 0x1851: 0x4000, - 0x1852: 0x4000, 0x1855: 0x4000, 0x1856: 0x4000, 0x1857: 0x4000, - 0x185c: 0x4000, 0x185d: 0x4000, - 0x185e: 0x4000, 0x185f: 0x4000, - 0x186b: 0x4000, 0x186c: 0x4000, - 0x1874: 0x4000, 0x1875: 0x4000, - 0x1876: 0x4000, 0x1877: 0x4000, 0x1878: 0x4000, 0x1879: 0x4000, 0x187a: 0x4000, 0x187b: 0x4000, - 0x187c: 0x4000, - // Block 0x62, offset 0x1880 - 0x18a0: 0x4000, 0x18a1: 0x4000, 0x18a2: 0x4000, 0x18a3: 0x4000, - 0x18a4: 0x4000, 0x18a5: 0x4000, 0x18a6: 0x4000, 0x18a7: 0x4000, 0x18a8: 0x4000, 0x18a9: 0x4000, - 0x18aa: 0x4000, 0x18ab: 0x4000, - 0x18b0: 0x4000, - // Block 0x63, offset 0x18c0 - 0x18cc: 0x4000, 0x18cd: 0x4000, 0x18ce: 0x4000, 0x18cf: 0x4000, 0x18d0: 0x4000, 0x18d1: 0x4000, - 0x18d2: 0x4000, 0x18d3: 0x4000, 0x18d4: 0x4000, 0x18d5: 0x4000, 0x18d6: 0x4000, 0x18d7: 0x4000, - 0x18d8: 0x4000, 0x18d9: 0x4000, 0x18da: 0x4000, 0x18db: 0x4000, 0x18dc: 0x4000, 0x18dd: 0x4000, - 0x18de: 0x4000, 0x18df: 0x4000, 0x18e0: 0x4000, 0x18e1: 0x4000, 0x18e2: 0x4000, 0x18e3: 0x4000, - 0x18e4: 0x4000, 0x18e5: 0x4000, 0x18e6: 0x4000, 0x18e7: 0x4000, 0x18e8: 0x4000, 0x18e9: 0x4000, - 0x18ea: 0x4000, 0x18eb: 0x4000, 0x18ec: 0x4000, 0x18ed: 0x4000, 0x18ee: 0x4000, 0x18ef: 0x4000, - 0x18f0: 0x4000, 0x18f1: 0x4000, 0x18f2: 0x4000, 0x18f3: 0x4000, 0x18f4: 0x4000, 0x18f5: 0x4000, - 0x18f6: 0x4000, 0x18f7: 0x4000, 0x18f8: 0x4000, 0x18f9: 0x4000, 0x18fa: 0x4000, - 0x18fc: 0x4000, 0x18fd: 0x4000, 0x18fe: 0x4000, 0x18ff: 0x4000, - // Block 0x64, offset 0x1900 - 0x1900: 0x4000, 0x1901: 0x4000, 0x1902: 0x4000, 0x1903: 0x4000, 0x1904: 0x4000, 0x1905: 0x4000, - 0x1907: 0x4000, 0x1908: 0x4000, 0x1909: 0x4000, 0x190a: 0x4000, 0x190b: 0x4000, - 0x190c: 0x4000, 0x190d: 0x4000, 0x190e: 0x4000, 0x190f: 0x4000, 0x1910: 0x4000, 0x1911: 0x4000, - 0x1912: 0x4000, 0x1913: 0x4000, 0x1914: 0x4000, 0x1915: 0x4000, 0x1916: 0x4000, 0x1917: 0x4000, - 0x1918: 0x4000, 0x1919: 0x4000, 0x191a: 0x4000, 0x191b: 0x4000, 0x191c: 0x4000, 0x191d: 0x4000, - 0x191e: 0x4000, 0x191f: 0x4000, 0x1920: 0x4000, 0x1921: 0x4000, 0x1922: 0x4000, 0x1923: 0x4000, - 0x1924: 0x4000, 0x1925: 0x4000, 0x1926: 0x4000, 0x1927: 0x4000, 0x1928: 0x4000, 0x1929: 0x4000, - 0x192a: 0x4000, 0x192b: 0x4000, 0x192c: 0x4000, 0x192d: 0x4000, 0x192e: 0x4000, 0x192f: 0x4000, - 0x1930: 0x4000, 0x1931: 0x4000, 0x1932: 0x4000, 0x1933: 0x4000, 0x1934: 0x4000, 0x1935: 0x4000, - 0x1936: 0x4000, 0x1937: 0x4000, 0x1938: 0x4000, 0x1939: 0x4000, 0x193a: 0x4000, 0x193b: 0x4000, - 0x193c: 0x4000, 0x193d: 0x4000, 0x193e: 0x4000, 0x193f: 0x4000, - // Block 0x65, offset 0x1940 - 0x1970: 0x4000, 0x1971: 0x4000, 0x1972: 0x4000, 0x1973: 0x4000, 0x1974: 0x4000, 0x1975: 0x4000, - 0x1976: 0x4000, 0x1977: 0x4000, 0x1978: 0x4000, 0x1979: 0x4000, 0x197a: 0x4000, 0x197b: 0x4000, - 0x197c: 0x4000, - // Block 0x66, offset 0x1980 - 0x1980: 0x4000, 0x1981: 0x4000, 0x1982: 0x4000, 0x1983: 0x4000, 0x1984: 0x4000, 0x1985: 0x4000, - 0x1986: 0x4000, 0x1987: 0x4000, 0x1988: 0x4000, - 0x1990: 0x4000, 0x1991: 0x4000, - 0x1992: 0x4000, 0x1993: 0x4000, 0x1994: 0x4000, 0x1995: 0x4000, 0x1996: 0x4000, 0x1997: 0x4000, - 0x1998: 0x4000, 0x1999: 0x4000, 0x199a: 0x4000, 0x199b: 0x4000, 0x199c: 0x4000, 0x199d: 0x4000, - 0x199e: 0x4000, 0x199f: 0x4000, 0x19a0: 0x4000, 0x19a1: 0x4000, 0x19a2: 0x4000, 0x19a3: 0x4000, - 0x19a4: 0x4000, 0x19a5: 0x4000, 0x19a6: 0x4000, 0x19a7: 0x4000, 0x19a8: 0x4000, 0x19a9: 0x4000, - 0x19aa: 0x4000, 0x19ab: 0x4000, 0x19ac: 0x4000, 0x19ad: 0x4000, 0x19ae: 0x4000, 0x19af: 0x4000, - 0x19b0: 0x4000, 0x19b1: 0x4000, 0x19b2: 0x4000, 0x19b3: 0x4000, 0x19b4: 0x4000, 0x19b5: 0x4000, - 0x19b6: 0x4000, 0x19b7: 0x4000, 0x19b8: 0x4000, 0x19b9: 0x4000, 0x19ba: 0x4000, 0x19bb: 0x4000, - 0x19bc: 0x4000, 0x19bd: 0x4000, 0x19bf: 0x4000, - // Block 0x67, offset 0x19c0 - 0x19c0: 0x4000, 0x19c1: 0x4000, 0x19c2: 0x4000, 0x19c3: 0x4000, 0x19c4: 0x4000, 0x19c5: 0x4000, - 0x19ce: 0x4000, 0x19cf: 0x4000, 0x19d0: 0x4000, 0x19d1: 0x4000, - 0x19d2: 0x4000, 0x19d3: 0x4000, 0x19d4: 0x4000, 0x19d5: 0x4000, 0x19d6: 0x4000, 0x19d7: 0x4000, - 0x19d8: 0x4000, 0x19d9: 0x4000, 0x19da: 0x4000, 0x19db: 0x4000, - 0x19e0: 0x4000, 0x19e1: 0x4000, 0x19e2: 0x4000, 0x19e3: 0x4000, - 0x19e4: 0x4000, 0x19e5: 0x4000, 0x19e6: 0x4000, 0x19e7: 0x4000, 0x19e8: 0x4000, - 0x19f0: 0x4000, 0x19f1: 0x4000, 0x19f2: 0x4000, 0x19f3: 0x4000, 0x19f4: 0x4000, 0x19f5: 0x4000, - 0x19f6: 0x4000, 0x19f7: 0x4000, 0x19f8: 0x4000, - // Block 0x68, offset 0x1a00 - 0x1a00: 0x2000, 0x1a01: 0x2000, 0x1a02: 0x2000, 0x1a03: 0x2000, 0x1a04: 0x2000, 0x1a05: 0x2000, - 0x1a06: 0x2000, 0x1a07: 0x2000, 0x1a08: 0x2000, 0x1a09: 0x2000, 0x1a0a: 0x2000, 0x1a0b: 0x2000, - 0x1a0c: 0x2000, 0x1a0d: 0x2000, 0x1a0e: 0x2000, 0x1a0f: 0x2000, 0x1a10: 0x2000, 0x1a11: 0x2000, - 0x1a12: 0x2000, 0x1a13: 0x2000, 0x1a14: 0x2000, 0x1a15: 0x2000, 0x1a16: 0x2000, 0x1a17: 0x2000, - 0x1a18: 0x2000, 0x1a19: 0x2000, 0x1a1a: 0x2000, 0x1a1b: 0x2000, 0x1a1c: 0x2000, 0x1a1d: 0x2000, - 0x1a1e: 0x2000, 0x1a1f: 0x2000, 0x1a20: 0x2000, 0x1a21: 0x2000, 0x1a22: 0x2000, 0x1a23: 0x2000, - 0x1a24: 0x2000, 0x1a25: 0x2000, 0x1a26: 0x2000, 0x1a27: 0x2000, 0x1a28: 0x2000, 0x1a29: 0x2000, - 0x1a2a: 0x2000, 0x1a2b: 0x2000, 0x1a2c: 0x2000, 0x1a2d: 0x2000, 0x1a2e: 0x2000, 0x1a2f: 0x2000, - 0x1a30: 0x2000, 0x1a31: 0x2000, 0x1a32: 0x2000, 0x1a33: 0x2000, 0x1a34: 0x2000, 0x1a35: 0x2000, - 0x1a36: 0x2000, 0x1a37: 0x2000, 0x1a38: 0x2000, 0x1a39: 0x2000, 0x1a3a: 0x2000, 0x1a3b: 0x2000, - 0x1a3c: 0x2000, 0x1a3d: 0x2000, -} - -// widthIndex: 23 blocks, 1472 entries, 1472 bytes -// Block 0 is the zero block. -var widthIndex = [1472]uint8{ - // Block 0x0, offset 0x0 - // Block 0x1, offset 0x40 - // Block 0x2, offset 0x80 - // Block 0x3, offset 0xc0 - 0xc2: 0x01, 0xc3: 0x02, 0xc4: 0x03, 0xc5: 0x04, 0xc7: 0x05, - 0xc9: 0x06, 0xcb: 0x07, 0xcc: 0x08, 0xcd: 0x09, 0xce: 0x0a, 0xcf: 0x0b, - 0xd0: 0x0c, 0xd1: 0x0d, - 0xe1: 0x02, 0xe2: 0x03, 0xe3: 0x04, 0xe4: 0x05, 0xe5: 0x06, 0xe6: 0x06, 0xe7: 0x06, - 0xe8: 0x06, 0xe9: 0x06, 0xea: 0x07, 0xeb: 0x06, 0xec: 0x06, 0xed: 0x08, 0xee: 0x09, 0xef: 0x0a, - 0xf0: 0x10, 0xf3: 0x13, 0xf4: 0x14, - // Block 0x4, offset 0x100 - 0x104: 0x0e, 0x105: 0x0f, - // Block 0x5, offset 0x140 - 0x140: 0x10, 0x141: 0x11, 0x142: 0x12, 0x144: 0x13, 0x145: 0x14, 0x146: 0x15, 0x147: 0x16, - 0x148: 0x17, 0x149: 0x18, 0x14a: 0x19, 0x14c: 0x1a, 0x14f: 0x1b, - 0x151: 0x1c, 0x152: 0x08, 0x153: 0x1d, 0x154: 0x1e, 0x155: 0x1f, 0x156: 0x20, 0x157: 0x21, - 0x158: 0x22, 0x159: 0x23, 0x15a: 0x24, 0x15b: 0x25, 0x15c: 0x26, 0x15d: 0x27, 0x15e: 0x28, 0x15f: 0x29, - 0x166: 0x2a, - 0x16c: 0x2b, 0x16d: 0x2c, - 0x17a: 0x2d, 0x17b: 0x2e, 0x17c: 0x0e, 0x17d: 0x0e, 0x17e: 0x0e, 0x17f: 0x2f, - // Block 0x6, offset 0x180 - 0x180: 0x30, 0x181: 0x31, 0x182: 0x32, 0x183: 0x33, 0x184: 0x34, 0x185: 0x35, 0x186: 0x36, 0x187: 0x37, - 0x188: 0x38, 0x189: 0x39, 0x18a: 0x0e, 0x18b: 0x0e, 0x18c: 0x0e, 0x18d: 0x0e, 0x18e: 0x0e, 0x18f: 0x0e, - 0x190: 0x0e, 0x191: 0x0e, 0x192: 0x0e, 0x193: 0x0e, 0x194: 0x0e, 0x195: 0x0e, 0x196: 0x0e, 0x197: 0x0e, - 0x198: 0x0e, 0x199: 0x0e, 0x19a: 0x0e, 0x19b: 0x0e, 0x19c: 0x0e, 0x19d: 0x0e, 0x19e: 0x0e, 0x19f: 0x0e, - 0x1a0: 0x0e, 0x1a1: 0x0e, 0x1a2: 0x0e, 0x1a3: 0x0e, 0x1a4: 0x0e, 0x1a5: 0x0e, 0x1a6: 0x0e, 0x1a7: 0x0e, - 0x1a8: 0x0e, 0x1a9: 0x0e, 0x1aa: 0x0e, 0x1ab: 0x0e, 0x1ac: 0x0e, 0x1ad: 0x0e, 0x1ae: 0x0e, 0x1af: 0x0e, - 0x1b0: 0x0e, 0x1b1: 0x0e, 0x1b2: 0x0e, 0x1b3: 0x0e, 0x1b4: 0x0e, 0x1b5: 0x0e, 0x1b6: 0x0e, 0x1b7: 0x0e, - 0x1b8: 0x0e, 0x1b9: 0x0e, 0x1ba: 0x0e, 0x1bb: 0x0e, 0x1bc: 0x0e, 0x1bd: 0x0e, 0x1be: 0x0e, 0x1bf: 0x0e, - // Block 0x7, offset 0x1c0 - 0x1c0: 0x0e, 0x1c1: 0x0e, 0x1c2: 0x0e, 0x1c3: 0x0e, 0x1c4: 0x0e, 0x1c5: 0x0e, 0x1c6: 0x0e, 0x1c7: 0x0e, - 0x1c8: 0x0e, 0x1c9: 0x0e, 0x1ca: 0x0e, 0x1cb: 0x0e, 0x1cc: 0x0e, 0x1cd: 0x0e, 0x1ce: 0x0e, 0x1cf: 0x0e, - 0x1d0: 0x0e, 0x1d1: 0x0e, 0x1d2: 0x0e, 0x1d3: 0x0e, 0x1d4: 0x0e, 0x1d5: 0x0e, 0x1d6: 0x0e, 0x1d7: 0x0e, - 0x1d8: 0x0e, 0x1d9: 0x0e, 0x1da: 0x0e, 0x1db: 0x0e, 0x1dc: 0x0e, 0x1dd: 0x0e, 0x1de: 0x0e, 0x1df: 0x0e, - 0x1e0: 0x0e, 0x1e1: 0x0e, 0x1e2: 0x0e, 0x1e3: 0x0e, 0x1e4: 0x0e, 0x1e5: 0x0e, 0x1e6: 0x0e, 0x1e7: 0x0e, - 0x1e8: 0x0e, 0x1e9: 0x0e, 0x1ea: 0x0e, 0x1eb: 0x0e, 0x1ec: 0x0e, 0x1ed: 0x0e, 0x1ee: 0x0e, 0x1ef: 0x0e, - 0x1f0: 0x0e, 0x1f1: 0x0e, 0x1f2: 0x0e, 0x1f3: 0x0e, 0x1f4: 0x0e, 0x1f5: 0x0e, 0x1f6: 0x0e, - 0x1f8: 0x0e, 0x1f9: 0x0e, 0x1fa: 0x0e, 0x1fb: 0x0e, 0x1fc: 0x0e, 0x1fd: 0x0e, 0x1fe: 0x0e, 0x1ff: 0x0e, - // Block 0x8, offset 0x200 - 0x200: 0x0e, 0x201: 0x0e, 0x202: 0x0e, 0x203: 0x0e, 0x204: 0x0e, 0x205: 0x0e, 0x206: 0x0e, 0x207: 0x0e, - 0x208: 0x0e, 0x209: 0x0e, 0x20a: 0x0e, 0x20b: 0x0e, 0x20c: 0x0e, 0x20d: 0x0e, 0x20e: 0x0e, 0x20f: 0x0e, - 0x210: 0x0e, 0x211: 0x0e, 0x212: 0x0e, 0x213: 0x0e, 0x214: 0x0e, 0x215: 0x0e, 0x216: 0x0e, 0x217: 0x0e, - 0x218: 0x0e, 0x219: 0x0e, 0x21a: 0x0e, 0x21b: 0x0e, 0x21c: 0x0e, 0x21d: 0x0e, 0x21e: 0x0e, 0x21f: 0x0e, - 0x220: 0x0e, 0x221: 0x0e, 0x222: 0x0e, 0x223: 0x0e, 0x224: 0x0e, 0x225: 0x0e, 0x226: 0x0e, 0x227: 0x0e, - 0x228: 0x0e, 0x229: 0x0e, 0x22a: 0x0e, 0x22b: 0x0e, 0x22c: 0x0e, 0x22d: 0x0e, 0x22e: 0x0e, 0x22f: 0x0e, - 0x230: 0x0e, 0x231: 0x0e, 0x232: 0x0e, 0x233: 0x0e, 0x234: 0x0e, 0x235: 0x0e, 0x236: 0x0e, 0x237: 0x0e, - 0x238: 0x0e, 0x239: 0x0e, 0x23a: 0x0e, 0x23b: 0x0e, 0x23c: 0x0e, 0x23d: 0x0e, 0x23e: 0x0e, 0x23f: 0x0e, - // Block 0x9, offset 0x240 - 0x240: 0x0e, 0x241: 0x0e, 0x242: 0x0e, 0x243: 0x0e, 0x244: 0x0e, 0x245: 0x0e, 0x246: 0x0e, 0x247: 0x0e, - 0x248: 0x0e, 0x249: 0x0e, 0x24a: 0x0e, 0x24b: 0x0e, 0x24c: 0x0e, 0x24d: 0x0e, 0x24e: 0x0e, 0x24f: 0x0e, - 0x250: 0x0e, 0x251: 0x0e, 0x252: 0x3a, 0x253: 0x3b, - 0x265: 0x3c, - 0x270: 0x0e, 0x271: 0x0e, 0x272: 0x0e, 0x273: 0x0e, 0x274: 0x0e, 0x275: 0x0e, 0x276: 0x0e, 0x277: 0x0e, - 0x278: 0x0e, 0x279: 0x0e, 0x27a: 0x0e, 0x27b: 0x0e, 0x27c: 0x0e, 0x27d: 0x0e, 0x27e: 0x0e, 0x27f: 0x0e, - // Block 0xa, offset 0x280 - 0x280: 0x0e, 0x281: 0x0e, 0x282: 0x0e, 0x283: 0x0e, 0x284: 0x0e, 0x285: 0x0e, 0x286: 0x0e, 0x287: 0x0e, - 0x288: 0x0e, 0x289: 0x0e, 0x28a: 0x0e, 0x28b: 0x0e, 0x28c: 0x0e, 0x28d: 0x0e, 0x28e: 0x0e, 0x28f: 0x0e, - 0x290: 0x0e, 0x291: 0x0e, 0x292: 0x0e, 0x293: 0x0e, 0x294: 0x0e, 0x295: 0x0e, 0x296: 0x0e, 0x297: 0x0e, - 0x298: 0x0e, 0x299: 0x0e, 0x29a: 0x0e, 0x29b: 0x0e, 0x29c: 0x0e, 0x29d: 0x0e, 0x29e: 0x3d, - // Block 0xb, offset 0x2c0 - 0x2c0: 0x08, 0x2c1: 0x08, 0x2c2: 0x08, 0x2c3: 0x08, 0x2c4: 0x08, 0x2c5: 0x08, 0x2c6: 0x08, 0x2c7: 0x08, - 0x2c8: 0x08, 0x2c9: 0x08, 0x2ca: 0x08, 0x2cb: 0x08, 0x2cc: 0x08, 0x2cd: 0x08, 0x2ce: 0x08, 0x2cf: 0x08, - 0x2d0: 0x08, 0x2d1: 0x08, 0x2d2: 0x08, 0x2d3: 0x08, 0x2d4: 0x08, 0x2d5: 0x08, 0x2d6: 0x08, 0x2d7: 0x08, - 0x2d8: 0x08, 0x2d9: 0x08, 0x2da: 0x08, 0x2db: 0x08, 0x2dc: 0x08, 0x2dd: 0x08, 0x2de: 0x08, 0x2df: 0x08, - 0x2e0: 0x08, 0x2e1: 0x08, 0x2e2: 0x08, 0x2e3: 0x08, 0x2e4: 0x08, 0x2e5: 0x08, 0x2e6: 0x08, 0x2e7: 0x08, - 0x2e8: 0x08, 0x2e9: 0x08, 0x2ea: 0x08, 0x2eb: 0x08, 0x2ec: 0x08, 0x2ed: 0x08, 0x2ee: 0x08, 0x2ef: 0x08, - 0x2f0: 0x08, 0x2f1: 0x08, 0x2f2: 0x08, 0x2f3: 0x08, 0x2f4: 0x08, 0x2f5: 0x08, 0x2f6: 0x08, 0x2f7: 0x08, - 0x2f8: 0x08, 0x2f9: 0x08, 0x2fa: 0x08, 0x2fb: 0x08, 0x2fc: 0x08, 0x2fd: 0x08, 0x2fe: 0x08, 0x2ff: 0x08, - // Block 0xc, offset 0x300 - 0x300: 0x08, 0x301: 0x08, 0x302: 0x08, 0x303: 0x08, 0x304: 0x08, 0x305: 0x08, 0x306: 0x08, 0x307: 0x08, - 0x308: 0x08, 0x309: 0x08, 0x30a: 0x08, 0x30b: 0x08, 0x30c: 0x08, 0x30d: 0x08, 0x30e: 0x08, 0x30f: 0x08, - 0x310: 0x08, 0x311: 0x08, 0x312: 0x08, 0x313: 0x08, 0x314: 0x08, 0x315: 0x08, 0x316: 0x08, 0x317: 0x08, - 0x318: 0x08, 0x319: 0x08, 0x31a: 0x08, 0x31b: 0x08, 0x31c: 0x08, 0x31d: 0x08, 0x31e: 0x08, 0x31f: 0x08, - 0x320: 0x08, 0x321: 0x08, 0x322: 0x08, 0x323: 0x08, 0x324: 0x0e, 0x325: 0x0e, 0x326: 0x0e, 0x327: 0x0e, - 0x328: 0x0e, 0x329: 0x0e, 0x32a: 0x0e, 0x32b: 0x0e, - 0x338: 0x3e, 0x339: 0x3f, 0x33c: 0x40, 0x33d: 0x41, 0x33e: 0x42, 0x33f: 0x43, - // Block 0xd, offset 0x340 - 0x37f: 0x44, - // Block 0xe, offset 0x380 - 0x380: 0x0e, 0x381: 0x0e, 0x382: 0x0e, 0x383: 0x0e, 0x384: 0x0e, 0x385: 0x0e, 0x386: 0x0e, 0x387: 0x0e, - 0x388: 0x0e, 0x389: 0x0e, 0x38a: 0x0e, 0x38b: 0x0e, 0x38c: 0x0e, 0x38d: 0x0e, 0x38e: 0x0e, 0x38f: 0x0e, - 0x390: 0x0e, 0x391: 0x0e, 0x392: 0x0e, 0x393: 0x0e, 0x394: 0x0e, 0x395: 0x0e, 0x396: 0x0e, 0x397: 0x0e, - 0x398: 0x0e, 0x399: 0x0e, 0x39a: 0x0e, 0x39b: 0x0e, 0x39c: 0x0e, 0x39d: 0x0e, 0x39e: 0x0e, 0x39f: 0x45, - 0x3a0: 0x0e, 0x3a1: 0x0e, 0x3a2: 0x0e, 0x3a3: 0x0e, 0x3a4: 0x0e, 0x3a5: 0x0e, 0x3a6: 0x0e, 0x3a7: 0x0e, - 0x3a8: 0x0e, 0x3a9: 0x0e, 0x3aa: 0x0e, 0x3ab: 0x0e, 0x3ac: 0x0e, 0x3ad: 0x0e, 0x3ae: 0x0e, 0x3af: 0x0e, - 0x3b0: 0x0e, 0x3b1: 0x0e, 0x3b2: 0x0e, 0x3b3: 0x46, 0x3b4: 0x47, - // Block 0xf, offset 0x3c0 - 0x3ff: 0x48, - // Block 0x10, offset 0x400 - 0x400: 0x0e, 0x401: 0x0e, 0x402: 0x0e, 0x403: 0x0e, 0x404: 0x49, 0x405: 0x4a, 0x406: 0x0e, 0x407: 0x0e, - 0x408: 0x0e, 0x409: 0x0e, 0x40a: 0x0e, 0x40b: 0x4b, - // Block 0x11, offset 0x440 - 0x440: 0x4c, 0x443: 0x4d, 0x444: 0x4e, 0x445: 0x4f, 0x446: 0x50, - 0x448: 0x51, 0x449: 0x52, 0x44c: 0x53, 0x44d: 0x54, 0x44e: 0x55, 0x44f: 0x56, - 0x450: 0x57, 0x451: 0x58, 0x452: 0x0e, 0x453: 0x59, 0x454: 0x5a, 0x455: 0x5b, 0x456: 0x5c, 0x457: 0x5d, - 0x458: 0x0e, 0x459: 0x5e, 0x45a: 0x0e, 0x45b: 0x5f, 0x45f: 0x60, - 0x464: 0x61, 0x465: 0x62, 0x466: 0x0e, 0x467: 0x0e, - 0x469: 0x63, 0x46a: 0x64, 0x46b: 0x65, - // Block 0x12, offset 0x480 - 0x496: 0x0b, 0x497: 0x06, - 0x498: 0x0c, 0x49a: 0x0d, 0x49b: 0x0e, 0x49f: 0x0f, - 0x4a0: 0x06, 0x4a1: 0x06, 0x4a2: 0x06, 0x4a3: 0x06, 0x4a4: 0x06, 0x4a5: 0x06, 0x4a6: 0x06, 0x4a7: 0x06, - 0x4a8: 0x06, 0x4a9: 0x06, 0x4aa: 0x06, 0x4ab: 0x06, 0x4ac: 0x06, 0x4ad: 0x06, 0x4ae: 0x06, 0x4af: 0x06, - 0x4b0: 0x06, 0x4b1: 0x06, 0x4b2: 0x06, 0x4b3: 0x06, 0x4b4: 0x06, 0x4b5: 0x06, 0x4b6: 0x06, 0x4b7: 0x06, - 0x4b8: 0x06, 0x4b9: 0x06, 0x4ba: 0x06, 0x4bb: 0x06, 0x4bc: 0x06, 0x4bd: 0x06, 0x4be: 0x06, 0x4bf: 0x06, - // Block 0x13, offset 0x4c0 - 0x4c4: 0x08, 0x4c5: 0x08, 0x4c6: 0x08, 0x4c7: 0x09, - // Block 0x14, offset 0x500 - 0x500: 0x08, 0x501: 0x08, 0x502: 0x08, 0x503: 0x08, 0x504: 0x08, 0x505: 0x08, 0x506: 0x08, 0x507: 0x08, - 0x508: 0x08, 0x509: 0x08, 0x50a: 0x08, 0x50b: 0x08, 0x50c: 0x08, 0x50d: 0x08, 0x50e: 0x08, 0x50f: 0x08, - 0x510: 0x08, 0x511: 0x08, 0x512: 0x08, 0x513: 0x08, 0x514: 0x08, 0x515: 0x08, 0x516: 0x08, 0x517: 0x08, - 0x518: 0x08, 0x519: 0x08, 0x51a: 0x08, 0x51b: 0x08, 0x51c: 0x08, 0x51d: 0x08, 0x51e: 0x08, 0x51f: 0x08, - 0x520: 0x08, 0x521: 0x08, 0x522: 0x08, 0x523: 0x08, 0x524: 0x08, 0x525: 0x08, 0x526: 0x08, 0x527: 0x08, - 0x528: 0x08, 0x529: 0x08, 0x52a: 0x08, 0x52b: 0x08, 0x52c: 0x08, 0x52d: 0x08, 0x52e: 0x08, 0x52f: 0x08, - 0x530: 0x08, 0x531: 0x08, 0x532: 0x08, 0x533: 0x08, 0x534: 0x08, 0x535: 0x08, 0x536: 0x08, 0x537: 0x08, - 0x538: 0x08, 0x539: 0x08, 0x53a: 0x08, 0x53b: 0x08, 0x53c: 0x08, 0x53d: 0x08, 0x53e: 0x08, 0x53f: 0x66, - // Block 0x15, offset 0x540 - 0x560: 0x11, - 0x570: 0x09, 0x571: 0x09, 0x572: 0x09, 0x573: 0x09, 0x574: 0x09, 0x575: 0x09, 0x576: 0x09, 0x577: 0x09, - 0x578: 0x09, 0x579: 0x09, 0x57a: 0x09, 0x57b: 0x09, 0x57c: 0x09, 0x57d: 0x09, 0x57e: 0x09, 0x57f: 0x12, - // Block 0x16, offset 0x580 - 0x580: 0x09, 0x581: 0x09, 0x582: 0x09, 0x583: 0x09, 0x584: 0x09, 0x585: 0x09, 0x586: 0x09, 0x587: 0x09, - 0x588: 0x09, 0x589: 0x09, 0x58a: 0x09, 0x58b: 0x09, 0x58c: 0x09, 0x58d: 0x09, 0x58e: 0x09, 0x58f: 0x12, -} - -// inverseData contains 4-byte entries of the following format: -// -// <0 padding> -// -// The last byte of the UTF-8-encoded rune is xor-ed with the last byte of the -// UTF-8 encoding of the original rune. Mappings often have the following -// pattern: -// -// Ａ -> A (U+FF21 -> U+0041) -// Ｂ -> B (U+FF22 -> U+0042) -// ... -// -// By xor-ing the last byte the same entry can be shared by many mappings. This -// reduces the total number of distinct entries by about two thirds. -// The resulting entry for the aforementioned mappings is -// -// { 0x01, 0xE0, 0x00, 0x00 } -// -// Using this entry to map U+FF21 (UTF-8 [EF BC A1]), we get -// -// E0 ^ A1 = 41. -// -// Similarly, for U+FF22 (UTF-8 [EF BC A2]), we get -// -// E0 ^ A2 = 42. -// -// Note that because of the xor-ing, the byte sequence stored in the entry is -// not valid UTF-8. -var inverseData = [150][4]byte{ - {0x00, 0x00, 0x00, 0x00}, - {0x03, 0xe3, 0x80, 0xa0}, - {0x03, 0xef, 0xbc, 0xa0}, - {0x03, 0xef, 0xbc, 0xe0}, - {0x03, 0xef, 0xbd, 0xe0}, - {0x03, 0xef, 0xbf, 0x02}, - {0x03, 0xef, 0xbf, 0x00}, - {0x03, 0xef, 0xbf, 0x0e}, - {0x03, 0xef, 0xbf, 0x0c}, - {0x03, 0xef, 0xbf, 0x0f}, - {0x03, 0xef, 0xbf, 0x39}, - {0x03, 0xef, 0xbf, 0x3b}, - {0x03, 0xef, 0xbf, 0x3f}, - {0x03, 0xef, 0xbf, 0x2a}, - {0x03, 0xef, 0xbf, 0x0d}, - {0x03, 0xef, 0xbf, 0x25}, - {0x03, 0xef, 0xbd, 0x1a}, - {0x03, 0xef, 0xbd, 0x26}, - {0x01, 0xa0, 0x00, 0x00}, - {0x03, 0xef, 0xbd, 0x25}, - {0x03, 0xef, 0xbd, 0x23}, - {0x03, 0xef, 0xbd, 0x2e}, - {0x03, 0xef, 0xbe, 0x07}, - {0x03, 0xef, 0xbe, 0x05}, - {0x03, 0xef, 0xbd, 0x06}, - {0x03, 0xef, 0xbd, 0x13}, - {0x03, 0xef, 0xbd, 0x0b}, - {0x03, 0xef, 0xbd, 0x16}, - {0x03, 0xef, 0xbd, 0x0c}, - {0x03, 0xef, 0xbd, 0x15}, - {0x03, 0xef, 0xbd, 0x0d}, - {0x03, 0xef, 0xbd, 0x1c}, - {0x03, 0xef, 0xbd, 0x02}, - {0x03, 0xef, 0xbd, 0x1f}, - {0x03, 0xef, 0xbd, 0x1d}, - {0x03, 0xef, 0xbd, 0x17}, - {0x03, 0xef, 0xbd, 0x08}, - {0x03, 0xef, 0xbd, 0x09}, - {0x03, 0xef, 0xbd, 0x0e}, - {0x03, 0xef, 0xbd, 0x04}, - {0x03, 0xef, 0xbd, 0x05}, - {0x03, 0xef, 0xbe, 0x3f}, - {0x03, 0xef, 0xbe, 0x00}, - {0x03, 0xef, 0xbd, 0x2c}, - {0x03, 0xef, 0xbe, 0x06}, - {0x03, 0xef, 0xbe, 0x0c}, - {0x03, 0xef, 0xbe, 0x0f}, - {0x03, 0xef, 0xbe, 0x0d}, - {0x03, 0xef, 0xbe, 0x0b}, - {0x03, 0xef, 0xbe, 0x19}, - {0x03, 0xef, 0xbe, 0x15}, - {0x03, 0xef, 0xbe, 0x11}, - {0x03, 0xef, 0xbe, 0x31}, - {0x03, 0xef, 0xbe, 0x33}, - {0x03, 0xef, 0xbd, 0x0f}, - {0x03, 0xef, 0xbe, 0x30}, - {0x03, 0xef, 0xbe, 0x3e}, - {0x03, 0xef, 0xbe, 0x32}, - {0x03, 0xef, 0xbe, 0x36}, - {0x03, 0xef, 0xbd, 0x14}, - {0x03, 0xef, 0xbe, 0x2e}, - {0x03, 0xef, 0xbd, 0x1e}, - {0x03, 0xef, 0xbe, 0x10}, - {0x03, 0xef, 0xbf, 0x13}, - {0x03, 0xef, 0xbf, 0x15}, - {0x03, 0xef, 0xbf, 0x17}, - {0x03, 0xef, 0xbf, 0x1f}, - {0x03, 0xef, 0xbf, 0x1d}, - {0x03, 0xef, 0xbf, 0x1b}, - {0x03, 0xef, 0xbf, 0x09}, - {0x03, 0xef, 0xbf, 0x0b}, - {0x03, 0xef, 0xbf, 0x37}, - {0x03, 0xef, 0xbe, 0x04}, - {0x01, 0xe0, 0x00, 0x00}, - {0x03, 0xe2, 0xa6, 0x1a}, - {0x03, 0xe2, 0xa6, 0x26}, - {0x03, 0xe3, 0x80, 0x23}, - {0x03, 0xe3, 0x80, 0x2e}, - {0x03, 0xe3, 0x80, 0x25}, - {0x03, 0xe3, 0x83, 0x1e}, - {0x03, 0xe3, 0x83, 0x14}, - {0x03, 0xe3, 0x82, 0x06}, - {0x03, 0xe3, 0x82, 0x0b}, - {0x03, 0xe3, 0x82, 0x0c}, - {0x03, 0xe3, 0x82, 0x0d}, - {0x03, 0xe3, 0x82, 0x02}, - {0x03, 0xe3, 0x83, 0x0f}, - {0x03, 0xe3, 0x83, 0x08}, - {0x03, 0xe3, 0x83, 0x09}, - {0x03, 0xe3, 0x83, 0x2c}, - {0x03, 0xe3, 0x83, 0x0c}, - {0x03, 0xe3, 0x82, 0x13}, - {0x03, 0xe3, 0x82, 0x16}, - {0x03, 0xe3, 0x82, 0x15}, - {0x03, 0xe3, 0x82, 0x1c}, - {0x03, 0xe3, 0x82, 0x1f}, - {0x03, 0xe3, 0x82, 0x1d}, - {0x03, 0xe3, 0x82, 0x1a}, - {0x03, 0xe3, 0x82, 0x17}, - {0x03, 0xe3, 0x82, 0x08}, - {0x03, 0xe3, 0x82, 0x09}, - {0x03, 0xe3, 0x82, 0x0e}, - {0x03, 0xe3, 0x82, 0x04}, - {0x03, 0xe3, 0x82, 0x05}, - {0x03, 0xe3, 0x82, 0x3f}, - {0x03, 0xe3, 0x83, 0x00}, - {0x03, 0xe3, 0x83, 0x06}, - {0x03, 0xe3, 0x83, 0x05}, - {0x03, 0xe3, 0x83, 0x0d}, - {0x03, 0xe3, 0x83, 0x0b}, - {0x03, 0xe3, 0x83, 0x07}, - {0x03, 0xe3, 0x83, 0x19}, - {0x03, 0xe3, 0x83, 0x15}, - {0x03, 0xe3, 0x83, 0x11}, - {0x03, 0xe3, 0x83, 0x31}, - {0x03, 0xe3, 0x83, 0x33}, - {0x03, 0xe3, 0x83, 0x30}, - {0x03, 0xe3, 0x83, 0x3e}, - {0x03, 0xe3, 0x83, 0x32}, - {0x03, 0xe3, 0x83, 0x36}, - {0x03, 0xe3, 0x83, 0x2e}, - {0x03, 0xe3, 0x82, 0x07}, - {0x03, 0xe3, 0x85, 0x04}, - {0x03, 0xe3, 0x84, 0x10}, - {0x03, 0xe3, 0x85, 0x30}, - {0x03, 0xe3, 0x85, 0x0d}, - {0x03, 0xe3, 0x85, 0x13}, - {0x03, 0xe3, 0x85, 0x15}, - {0x03, 0xe3, 0x85, 0x17}, - {0x03, 0xe3, 0x85, 0x1f}, - {0x03, 0xe3, 0x85, 0x1d}, - {0x03, 0xe3, 0x85, 0x1b}, - {0x03, 0xe3, 0x85, 0x09}, - {0x03, 0xe3, 0x85, 0x0f}, - {0x03, 0xe3, 0x85, 0x0b}, - {0x03, 0xe3, 0x85, 0x37}, - {0x03, 0xe3, 0x85, 0x3b}, - {0x03, 0xe3, 0x85, 0x39}, - {0x03, 0xe3, 0x85, 0x3f}, - {0x02, 0xc2, 0x02, 0x00}, - {0x02, 0xc2, 0x0e, 0x00}, - {0x02, 0xc2, 0x0c, 0x00}, - {0x02, 0xc2, 0x00, 0x00}, - {0x03, 0xe2, 0x82, 0x0f}, - {0x03, 0xe2, 0x94, 0x2a}, - {0x03, 0xe2, 0x86, 0x39}, - {0x03, 0xe2, 0x86, 0x3b}, - {0x03, 0xe2, 0x86, 0x3f}, - {0x03, 0xe2, 0x96, 0x0d}, - {0x03, 0xe2, 0x97, 0x25}, -} - -// Total table size 15512 bytes (15KiB) diff --git a/vendor/golang.org/x/text/width/tables9.0.0.go b/vendor/golang.org/x/text/width/tables9.0.0.go deleted file mode 100644 index 6781f3d960..0000000000 --- a/vendor/golang.org/x/text/width/tables9.0.0.go +++ /dev/null @@ -1,1297 +0,0 @@ -// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. - -//go:build !go1.10 -// +build !go1.10 - -package width - -// UnicodeVersion is the Unicode version from which the tables in this package are derived. -const UnicodeVersion = "9.0.0" - -// lookup returns the trie value for the first UTF-8 encoding in s and -// the width in bytes of this encoding. The size will be 0 if s does not -// hold enough bytes to complete the encoding. len(s) must be greater than 0. -func (t *widthTrie) lookup(s []byte) (v uint16, sz int) { - c0 := s[0] - switch { - case c0 < 0x80: // is ASCII - return widthValues[c0], 1 - case c0 < 0xC2: - return 0, 1 // Illegal UTF-8: not a starter, not ASCII. - case c0 < 0xE0: // 2-byte UTF-8 - if len(s) < 2 { - return 0, 0 - } - i := widthIndex[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c1), 2 - case c0 < 0xF0: // 3-byte UTF-8 - if len(s) < 3 { - return 0, 0 - } - i := widthIndex[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - o := uint32(i)<<6 + uint32(c1) - i = widthIndex[o] - c2 := s[2] - if c2 < 0x80 || 0xC0 <= c2 { - return 0, 2 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c2), 3 - case c0 < 0xF8: // 4-byte UTF-8 - if len(s) < 4 { - return 0, 0 - } - i := widthIndex[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - o := uint32(i)<<6 + uint32(c1) - i = widthIndex[o] - c2 := s[2] - if c2 < 0x80 || 0xC0 <= c2 { - return 0, 2 // Illegal UTF-8: not a continuation byte. - } - o = uint32(i)<<6 + uint32(c2) - i = widthIndex[o] - c3 := s[3] - if c3 < 0x80 || 0xC0 <= c3 { - return 0, 3 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c3), 4 - } - // Illegal rune - return 0, 1 -} - -// lookupUnsafe returns the trie value for the first UTF-8 encoding in s. -// s must start with a full and valid UTF-8 encoded rune. -func (t *widthTrie) lookupUnsafe(s []byte) uint16 { - c0 := s[0] - if c0 < 0x80 { // is ASCII - return widthValues[c0] - } - i := widthIndex[c0] - if c0 < 0xE0 { // 2-byte UTF-8 - return t.lookupValue(uint32(i), s[1]) - } - i = widthIndex[uint32(i)<<6+uint32(s[1])] - if c0 < 0xF0 { // 3-byte UTF-8 - return t.lookupValue(uint32(i), s[2]) - } - i = widthIndex[uint32(i)<<6+uint32(s[2])] - if c0 < 0xF8 { // 4-byte UTF-8 - return t.lookupValue(uint32(i), s[3]) - } - return 0 -} - -// lookupString returns the trie value for the first UTF-8 encoding in s and -// the width in bytes of this encoding. The size will be 0 if s does not -// hold enough bytes to complete the encoding. len(s) must be greater than 0. -func (t *widthTrie) lookupString(s string) (v uint16, sz int) { - c0 := s[0] - switch { - case c0 < 0x80: // is ASCII - return widthValues[c0], 1 - case c0 < 0xC2: - return 0, 1 // Illegal UTF-8: not a starter, not ASCII. - case c0 < 0xE0: // 2-byte UTF-8 - if len(s) < 2 { - return 0, 0 - } - i := widthIndex[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c1), 2 - case c0 < 0xF0: // 3-byte UTF-8 - if len(s) < 3 { - return 0, 0 - } - i := widthIndex[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - o := uint32(i)<<6 + uint32(c1) - i = widthIndex[o] - c2 := s[2] - if c2 < 0x80 || 0xC0 <= c2 { - return 0, 2 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c2), 3 - case c0 < 0xF8: // 4-byte UTF-8 - if len(s) < 4 { - return 0, 0 - } - i := widthIndex[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - o := uint32(i)<<6 + uint32(c1) - i = widthIndex[o] - c2 := s[2] - if c2 < 0x80 || 0xC0 <= c2 { - return 0, 2 // Illegal UTF-8: not a continuation byte. - } - o = uint32(i)<<6 + uint32(c2) - i = widthIndex[o] - c3 := s[3] - if c3 < 0x80 || 0xC0 <= c3 { - return 0, 3 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c3), 4 - } - // Illegal rune - return 0, 1 -} - -// lookupStringUnsafe returns the trie value for the first UTF-8 encoding in s. -// s must start with a full and valid UTF-8 encoded rune. -func (t *widthTrie) lookupStringUnsafe(s string) uint16 { - c0 := s[0] - if c0 < 0x80 { // is ASCII - return widthValues[c0] - } - i := widthIndex[c0] - if c0 < 0xE0 { // 2-byte UTF-8 - return t.lookupValue(uint32(i), s[1]) - } - i = widthIndex[uint32(i)<<6+uint32(s[1])] - if c0 < 0xF0 { // 3-byte UTF-8 - return t.lookupValue(uint32(i), s[2]) - } - i = widthIndex[uint32(i)<<6+uint32(s[2])] - if c0 < 0xF8 { // 4-byte UTF-8 - return t.lookupValue(uint32(i), s[3]) - } - return 0 -} - -// widthTrie. Total size: 14080 bytes (13.75 KiB). Checksum: 3b8aeb3dc03667a3. -type widthTrie struct{} - -func newWidthTrie(i int) *widthTrie { - return &widthTrie{} -} - -// lookupValue determines the type of block n and looks up the value for b. -func (t *widthTrie) lookupValue(n uint32, b byte) uint16 { - switch { - default: - return uint16(widthValues[n<<6+uint32(b)]) - } -} - -// widthValues: 99 blocks, 6336 entries, 12672 bytes -// The third block is the zero block. -var widthValues = [6336]uint16{ - // Block 0x0, offset 0x0 - 0x20: 0x6001, 0x21: 0x6002, 0x22: 0x6002, 0x23: 0x6002, - 0x24: 0x6002, 0x25: 0x6002, 0x26: 0x6002, 0x27: 0x6002, 0x28: 0x6002, 0x29: 0x6002, - 0x2a: 0x6002, 0x2b: 0x6002, 0x2c: 0x6002, 0x2d: 0x6002, 0x2e: 0x6002, 0x2f: 0x6002, - 0x30: 0x6002, 0x31: 0x6002, 0x32: 0x6002, 0x33: 0x6002, 0x34: 0x6002, 0x35: 0x6002, - 0x36: 0x6002, 0x37: 0x6002, 0x38: 0x6002, 0x39: 0x6002, 0x3a: 0x6002, 0x3b: 0x6002, - 0x3c: 0x6002, 0x3d: 0x6002, 0x3e: 0x6002, 0x3f: 0x6002, - // Block 0x1, offset 0x40 - 0x40: 0x6003, 0x41: 0x6003, 0x42: 0x6003, 0x43: 0x6003, 0x44: 0x6003, 0x45: 0x6003, - 0x46: 0x6003, 0x47: 0x6003, 0x48: 0x6003, 0x49: 0x6003, 0x4a: 0x6003, 0x4b: 0x6003, - 0x4c: 0x6003, 0x4d: 0x6003, 0x4e: 0x6003, 0x4f: 0x6003, 0x50: 0x6003, 0x51: 0x6003, - 0x52: 0x6003, 0x53: 0x6003, 0x54: 0x6003, 0x55: 0x6003, 0x56: 0x6003, 0x57: 0x6003, - 0x58: 0x6003, 0x59: 0x6003, 0x5a: 0x6003, 0x5b: 0x6003, 0x5c: 0x6003, 0x5d: 0x6003, - 0x5e: 0x6003, 0x5f: 0x6003, 0x60: 0x6004, 0x61: 0x6004, 0x62: 0x6004, 0x63: 0x6004, - 0x64: 0x6004, 0x65: 0x6004, 0x66: 0x6004, 0x67: 0x6004, 0x68: 0x6004, 0x69: 0x6004, - 0x6a: 0x6004, 0x6b: 0x6004, 0x6c: 0x6004, 0x6d: 0x6004, 0x6e: 0x6004, 0x6f: 0x6004, - 0x70: 0x6004, 0x71: 0x6004, 0x72: 0x6004, 0x73: 0x6004, 0x74: 0x6004, 0x75: 0x6004, - 0x76: 0x6004, 0x77: 0x6004, 0x78: 0x6004, 0x79: 0x6004, 0x7a: 0x6004, 0x7b: 0x6004, - 0x7c: 0x6004, 0x7d: 0x6004, 0x7e: 0x6004, - // Block 0x2, offset 0x80 - // Block 0x3, offset 0xc0 - 0xe1: 0x2000, 0xe2: 0x6005, 0xe3: 0x6005, - 0xe4: 0x2000, 0xe5: 0x6006, 0xe6: 0x6005, 0xe7: 0x2000, 0xe8: 0x2000, - 0xea: 0x2000, 0xec: 0x6007, 0xed: 0x2000, 0xee: 0x2000, 0xef: 0x6008, - 0xf0: 0x2000, 0xf1: 0x2000, 0xf2: 0x2000, 0xf3: 0x2000, 0xf4: 0x2000, - 0xf6: 0x2000, 0xf7: 0x2000, 0xf8: 0x2000, 0xf9: 0x2000, 0xfa: 0x2000, - 0xfc: 0x2000, 0xfd: 0x2000, 0xfe: 0x2000, 0xff: 0x2000, - // Block 0x4, offset 0x100 - 0x106: 0x2000, - 0x110: 0x2000, - 0x117: 0x2000, - 0x118: 0x2000, - 0x11e: 0x2000, 0x11f: 0x2000, 0x120: 0x2000, 0x121: 0x2000, - 0x126: 0x2000, 0x128: 0x2000, 0x129: 0x2000, - 0x12a: 0x2000, 0x12c: 0x2000, 0x12d: 0x2000, - 0x130: 0x2000, 0x132: 0x2000, 0x133: 0x2000, - 0x137: 0x2000, 0x138: 0x2000, 0x139: 0x2000, 0x13a: 0x2000, - 0x13c: 0x2000, 0x13e: 0x2000, - // Block 0x5, offset 0x140 - 0x141: 0x2000, - 0x151: 0x2000, - 0x153: 0x2000, - 0x15b: 0x2000, - 0x166: 0x2000, 0x167: 0x2000, - 0x16b: 0x2000, - 0x171: 0x2000, 0x172: 0x2000, 0x173: 0x2000, - 0x178: 0x2000, - 0x17f: 0x2000, - // Block 0x6, offset 0x180 - 0x180: 0x2000, 0x181: 0x2000, 0x182: 0x2000, 0x184: 0x2000, - 0x188: 0x2000, 0x189: 0x2000, 0x18a: 0x2000, 0x18b: 0x2000, - 0x18d: 0x2000, - 0x192: 0x2000, 0x193: 0x2000, - 0x1a6: 0x2000, 0x1a7: 0x2000, - 0x1ab: 0x2000, - // Block 0x7, offset 0x1c0 - 0x1ce: 0x2000, 0x1d0: 0x2000, - 0x1d2: 0x2000, 0x1d4: 0x2000, 0x1d6: 0x2000, - 0x1d8: 0x2000, 0x1da: 0x2000, 0x1dc: 0x2000, - // Block 0x8, offset 0x200 - 0x211: 0x2000, - 0x221: 0x2000, - // Block 0x9, offset 0x240 - 0x244: 0x2000, - 0x247: 0x2000, 0x249: 0x2000, 0x24a: 0x2000, 0x24b: 0x2000, - 0x24d: 0x2000, 0x250: 0x2000, - 0x258: 0x2000, 0x259: 0x2000, 0x25a: 0x2000, 0x25b: 0x2000, 0x25d: 0x2000, - 0x25f: 0x2000, - // Block 0xa, offset 0x280 - 0x280: 0x2000, 0x281: 0x2000, 0x282: 0x2000, 0x283: 0x2000, 0x284: 0x2000, 0x285: 0x2000, - 0x286: 0x2000, 0x287: 0x2000, 0x288: 0x2000, 0x289: 0x2000, 0x28a: 0x2000, 0x28b: 0x2000, - 0x28c: 0x2000, 0x28d: 0x2000, 0x28e: 0x2000, 0x28f: 0x2000, 0x290: 0x2000, 0x291: 0x2000, - 0x292: 0x2000, 0x293: 0x2000, 0x294: 0x2000, 0x295: 0x2000, 0x296: 0x2000, 0x297: 0x2000, - 0x298: 0x2000, 0x299: 0x2000, 0x29a: 0x2000, 0x29b: 0x2000, 0x29c: 0x2000, 0x29d: 0x2000, - 0x29e: 0x2000, 0x29f: 0x2000, 0x2a0: 0x2000, 0x2a1: 0x2000, 0x2a2: 0x2000, 0x2a3: 0x2000, - 0x2a4: 0x2000, 0x2a5: 0x2000, 0x2a6: 0x2000, 0x2a7: 0x2000, 0x2a8: 0x2000, 0x2a9: 0x2000, - 0x2aa: 0x2000, 0x2ab: 0x2000, 0x2ac: 0x2000, 0x2ad: 0x2000, 0x2ae: 0x2000, 0x2af: 0x2000, - 0x2b0: 0x2000, 0x2b1: 0x2000, 0x2b2: 0x2000, 0x2b3: 0x2000, 0x2b4: 0x2000, 0x2b5: 0x2000, - 0x2b6: 0x2000, 0x2b7: 0x2000, 0x2b8: 0x2000, 0x2b9: 0x2000, 0x2ba: 0x2000, 0x2bb: 0x2000, - 0x2bc: 0x2000, 0x2bd: 0x2000, 0x2be: 0x2000, 0x2bf: 0x2000, - // Block 0xb, offset 0x2c0 - 0x2c0: 0x2000, 0x2c1: 0x2000, 0x2c2: 0x2000, 0x2c3: 0x2000, 0x2c4: 0x2000, 0x2c5: 0x2000, - 0x2c6: 0x2000, 0x2c7: 0x2000, 0x2c8: 0x2000, 0x2c9: 0x2000, 0x2ca: 0x2000, 0x2cb: 0x2000, - 0x2cc: 0x2000, 0x2cd: 0x2000, 0x2ce: 0x2000, 0x2cf: 0x2000, 0x2d0: 0x2000, 0x2d1: 0x2000, - 0x2d2: 0x2000, 0x2d3: 0x2000, 0x2d4: 0x2000, 0x2d5: 0x2000, 0x2d6: 0x2000, 0x2d7: 0x2000, - 0x2d8: 0x2000, 0x2d9: 0x2000, 0x2da: 0x2000, 0x2db: 0x2000, 0x2dc: 0x2000, 0x2dd: 0x2000, - 0x2de: 0x2000, 0x2df: 0x2000, 0x2e0: 0x2000, 0x2e1: 0x2000, 0x2e2: 0x2000, 0x2e3: 0x2000, - 0x2e4: 0x2000, 0x2e5: 0x2000, 0x2e6: 0x2000, 0x2e7: 0x2000, 0x2e8: 0x2000, 0x2e9: 0x2000, - 0x2ea: 0x2000, 0x2eb: 0x2000, 0x2ec: 0x2000, 0x2ed: 0x2000, 0x2ee: 0x2000, 0x2ef: 0x2000, - // Block 0xc, offset 0x300 - 0x311: 0x2000, - 0x312: 0x2000, 0x313: 0x2000, 0x314: 0x2000, 0x315: 0x2000, 0x316: 0x2000, 0x317: 0x2000, - 0x318: 0x2000, 0x319: 0x2000, 0x31a: 0x2000, 0x31b: 0x2000, 0x31c: 0x2000, 0x31d: 0x2000, - 0x31e: 0x2000, 0x31f: 0x2000, 0x320: 0x2000, 0x321: 0x2000, 0x323: 0x2000, - 0x324: 0x2000, 0x325: 0x2000, 0x326: 0x2000, 0x327: 0x2000, 0x328: 0x2000, 0x329: 0x2000, - 0x331: 0x2000, 0x332: 0x2000, 0x333: 0x2000, 0x334: 0x2000, 0x335: 0x2000, - 0x336: 0x2000, 0x337: 0x2000, 0x338: 0x2000, 0x339: 0x2000, 0x33a: 0x2000, 0x33b: 0x2000, - 0x33c: 0x2000, 0x33d: 0x2000, 0x33e: 0x2000, 0x33f: 0x2000, - // Block 0xd, offset 0x340 - 0x340: 0x2000, 0x341: 0x2000, 0x343: 0x2000, 0x344: 0x2000, 0x345: 0x2000, - 0x346: 0x2000, 0x347: 0x2000, 0x348: 0x2000, 0x349: 0x2000, - // Block 0xe, offset 0x380 - 0x381: 0x2000, - 0x390: 0x2000, 0x391: 0x2000, - 0x392: 0x2000, 0x393: 0x2000, 0x394: 0x2000, 0x395: 0x2000, 0x396: 0x2000, 0x397: 0x2000, - 0x398: 0x2000, 0x399: 0x2000, 0x39a: 0x2000, 0x39b: 0x2000, 0x39c: 0x2000, 0x39d: 0x2000, - 0x39e: 0x2000, 0x39f: 0x2000, 0x3a0: 0x2000, 0x3a1: 0x2000, 0x3a2: 0x2000, 0x3a3: 0x2000, - 0x3a4: 0x2000, 0x3a5: 0x2000, 0x3a6: 0x2000, 0x3a7: 0x2000, 0x3a8: 0x2000, 0x3a9: 0x2000, - 0x3aa: 0x2000, 0x3ab: 0x2000, 0x3ac: 0x2000, 0x3ad: 0x2000, 0x3ae: 0x2000, 0x3af: 0x2000, - 0x3b0: 0x2000, 0x3b1: 0x2000, 0x3b2: 0x2000, 0x3b3: 0x2000, 0x3b4: 0x2000, 0x3b5: 0x2000, - 0x3b6: 0x2000, 0x3b7: 0x2000, 0x3b8: 0x2000, 0x3b9: 0x2000, 0x3ba: 0x2000, 0x3bb: 0x2000, - 0x3bc: 0x2000, 0x3bd: 0x2000, 0x3be: 0x2000, 0x3bf: 0x2000, - // Block 0xf, offset 0x3c0 - 0x3c0: 0x2000, 0x3c1: 0x2000, 0x3c2: 0x2000, 0x3c3: 0x2000, 0x3c4: 0x2000, 0x3c5: 0x2000, - 0x3c6: 0x2000, 0x3c7: 0x2000, 0x3c8: 0x2000, 0x3c9: 0x2000, 0x3ca: 0x2000, 0x3cb: 0x2000, - 0x3cc: 0x2000, 0x3cd: 0x2000, 0x3ce: 0x2000, 0x3cf: 0x2000, 0x3d1: 0x2000, - // Block 0x10, offset 0x400 - 0x400: 0x4000, 0x401: 0x4000, 0x402: 0x4000, 0x403: 0x4000, 0x404: 0x4000, 0x405: 0x4000, - 0x406: 0x4000, 0x407: 0x4000, 0x408: 0x4000, 0x409: 0x4000, 0x40a: 0x4000, 0x40b: 0x4000, - 0x40c: 0x4000, 0x40d: 0x4000, 0x40e: 0x4000, 0x40f: 0x4000, 0x410: 0x4000, 0x411: 0x4000, - 0x412: 0x4000, 0x413: 0x4000, 0x414: 0x4000, 0x415: 0x4000, 0x416: 0x4000, 0x417: 0x4000, - 0x418: 0x4000, 0x419: 0x4000, 0x41a: 0x4000, 0x41b: 0x4000, 0x41c: 0x4000, 0x41d: 0x4000, - 0x41e: 0x4000, 0x41f: 0x4000, 0x420: 0x4000, 0x421: 0x4000, 0x422: 0x4000, 0x423: 0x4000, - 0x424: 0x4000, 0x425: 0x4000, 0x426: 0x4000, 0x427: 0x4000, 0x428: 0x4000, 0x429: 0x4000, - 0x42a: 0x4000, 0x42b: 0x4000, 0x42c: 0x4000, 0x42d: 0x4000, 0x42e: 0x4000, 0x42f: 0x4000, - 0x430: 0x4000, 0x431: 0x4000, 0x432: 0x4000, 0x433: 0x4000, 0x434: 0x4000, 0x435: 0x4000, - 0x436: 0x4000, 0x437: 0x4000, 0x438: 0x4000, 0x439: 0x4000, 0x43a: 0x4000, 0x43b: 0x4000, - 0x43c: 0x4000, 0x43d: 0x4000, 0x43e: 0x4000, 0x43f: 0x4000, - // Block 0x11, offset 0x440 - 0x440: 0x4000, 0x441: 0x4000, 0x442: 0x4000, 0x443: 0x4000, 0x444: 0x4000, 0x445: 0x4000, - 0x446: 0x4000, 0x447: 0x4000, 0x448: 0x4000, 0x449: 0x4000, 0x44a: 0x4000, 0x44b: 0x4000, - 0x44c: 0x4000, 0x44d: 0x4000, 0x44e: 0x4000, 0x44f: 0x4000, 0x450: 0x4000, 0x451: 0x4000, - 0x452: 0x4000, 0x453: 0x4000, 0x454: 0x4000, 0x455: 0x4000, 0x456: 0x4000, 0x457: 0x4000, - 0x458: 0x4000, 0x459: 0x4000, 0x45a: 0x4000, 0x45b: 0x4000, 0x45c: 0x4000, 0x45d: 0x4000, - 0x45e: 0x4000, 0x45f: 0x4000, - // Block 0x12, offset 0x480 - 0x490: 0x2000, - 0x493: 0x2000, 0x494: 0x2000, 0x495: 0x2000, 0x496: 0x2000, - 0x498: 0x2000, 0x499: 0x2000, 0x49c: 0x2000, 0x49d: 0x2000, - 0x4a0: 0x2000, 0x4a1: 0x2000, 0x4a2: 0x2000, - 0x4a4: 0x2000, 0x4a5: 0x2000, 0x4a6: 0x2000, 0x4a7: 0x2000, - 0x4b0: 0x2000, 0x4b2: 0x2000, 0x4b3: 0x2000, 0x4b5: 0x2000, - 0x4bb: 0x2000, - 0x4be: 0x2000, - // Block 0x13, offset 0x4c0 - 0x4f4: 0x2000, - 0x4ff: 0x2000, - // Block 0x14, offset 0x500 - 0x501: 0x2000, 0x502: 0x2000, 0x503: 0x2000, 0x504: 0x2000, - 0x529: 0xa009, - 0x52c: 0x2000, - // Block 0x15, offset 0x540 - 0x543: 0x2000, 0x545: 0x2000, - 0x549: 0x2000, - 0x553: 0x2000, 0x556: 0x2000, - 0x561: 0x2000, 0x562: 0x2000, - 0x566: 0x2000, - 0x56b: 0x2000, - // Block 0x16, offset 0x580 - 0x593: 0x2000, 0x594: 0x2000, - 0x59b: 0x2000, 0x59c: 0x2000, 0x59d: 0x2000, - 0x59e: 0x2000, 0x5a0: 0x2000, 0x5a1: 0x2000, 0x5a2: 0x2000, 0x5a3: 0x2000, - 0x5a4: 0x2000, 0x5a5: 0x2000, 0x5a6: 0x2000, 0x5a7: 0x2000, 0x5a8: 0x2000, 0x5a9: 0x2000, - 0x5aa: 0x2000, 0x5ab: 0x2000, - 0x5b0: 0x2000, 0x5b1: 0x2000, 0x5b2: 0x2000, 0x5b3: 0x2000, 0x5b4: 0x2000, 0x5b5: 0x2000, - 0x5b6: 0x2000, 0x5b7: 0x2000, 0x5b8: 0x2000, 0x5b9: 0x2000, - // Block 0x17, offset 0x5c0 - 0x5c9: 0x2000, - 0x5d0: 0x200a, 0x5d1: 0x200b, - 0x5d2: 0x200a, 0x5d3: 0x200c, 0x5d4: 0x2000, 0x5d5: 0x2000, 0x5d6: 0x2000, 0x5d7: 0x2000, - 0x5d8: 0x2000, 0x5d9: 0x2000, - 0x5f8: 0x2000, 0x5f9: 0x2000, - // Block 0x18, offset 0x600 - 0x612: 0x2000, 0x614: 0x2000, - 0x627: 0x2000, - // Block 0x19, offset 0x640 - 0x640: 0x2000, 0x642: 0x2000, 0x643: 0x2000, - 0x647: 0x2000, 0x648: 0x2000, 0x64b: 0x2000, - 0x64f: 0x2000, 0x651: 0x2000, - 0x655: 0x2000, - 0x65a: 0x2000, 0x65d: 0x2000, - 0x65e: 0x2000, 0x65f: 0x2000, 0x660: 0x2000, 0x663: 0x2000, - 0x665: 0x2000, 0x667: 0x2000, 0x668: 0x2000, 0x669: 0x2000, - 0x66a: 0x2000, 0x66b: 0x2000, 0x66c: 0x2000, 0x66e: 0x2000, - 0x674: 0x2000, 0x675: 0x2000, - 0x676: 0x2000, 0x677: 0x2000, - 0x67c: 0x2000, 0x67d: 0x2000, - // Block 0x1a, offset 0x680 - 0x688: 0x2000, - 0x68c: 0x2000, - 0x692: 0x2000, - 0x6a0: 0x2000, 0x6a1: 0x2000, - 0x6a4: 0x2000, 0x6a5: 0x2000, 0x6a6: 0x2000, 0x6a7: 0x2000, - 0x6aa: 0x2000, 0x6ab: 0x2000, 0x6ae: 0x2000, 0x6af: 0x2000, - // Block 0x1b, offset 0x6c0 - 0x6c2: 0x2000, 0x6c3: 0x2000, - 0x6c6: 0x2000, 0x6c7: 0x2000, - 0x6d5: 0x2000, - 0x6d9: 0x2000, - 0x6e5: 0x2000, - 0x6ff: 0x2000, - // Block 0x1c, offset 0x700 - 0x712: 0x2000, - 0x71a: 0x4000, 0x71b: 0x4000, - 0x729: 0x4000, - 0x72a: 0x4000, - // Block 0x1d, offset 0x740 - 0x769: 0x4000, - 0x76a: 0x4000, 0x76b: 0x4000, 0x76c: 0x4000, - 0x770: 0x4000, 0x773: 0x4000, - // Block 0x1e, offset 0x780 - 0x7a0: 0x2000, 0x7a1: 0x2000, 0x7a2: 0x2000, 0x7a3: 0x2000, - 0x7a4: 0x2000, 0x7a5: 0x2000, 0x7a6: 0x2000, 0x7a7: 0x2000, 0x7a8: 0x2000, 0x7a9: 0x2000, - 0x7aa: 0x2000, 0x7ab: 0x2000, 0x7ac: 0x2000, 0x7ad: 0x2000, 0x7ae: 0x2000, 0x7af: 0x2000, - 0x7b0: 0x2000, 0x7b1: 0x2000, 0x7b2: 0x2000, 0x7b3: 0x2000, 0x7b4: 0x2000, 0x7b5: 0x2000, - 0x7b6: 0x2000, 0x7b7: 0x2000, 0x7b8: 0x2000, 0x7b9: 0x2000, 0x7ba: 0x2000, 0x7bb: 0x2000, - 0x7bc: 0x2000, 0x7bd: 0x2000, 0x7be: 0x2000, 0x7bf: 0x2000, - // Block 0x1f, offset 0x7c0 - 0x7c0: 0x2000, 0x7c1: 0x2000, 0x7c2: 0x2000, 0x7c3: 0x2000, 0x7c4: 0x2000, 0x7c5: 0x2000, - 0x7c6: 0x2000, 0x7c7: 0x2000, 0x7c8: 0x2000, 0x7c9: 0x2000, 0x7ca: 0x2000, 0x7cb: 0x2000, - 0x7cc: 0x2000, 0x7cd: 0x2000, 0x7ce: 0x2000, 0x7cf: 0x2000, 0x7d0: 0x2000, 0x7d1: 0x2000, - 0x7d2: 0x2000, 0x7d3: 0x2000, 0x7d4: 0x2000, 0x7d5: 0x2000, 0x7d6: 0x2000, 0x7d7: 0x2000, - 0x7d8: 0x2000, 0x7d9: 0x2000, 0x7da: 0x2000, 0x7db: 0x2000, 0x7dc: 0x2000, 0x7dd: 0x2000, - 0x7de: 0x2000, 0x7df: 0x2000, 0x7e0: 0x2000, 0x7e1: 0x2000, 0x7e2: 0x2000, 0x7e3: 0x2000, - 0x7e4: 0x2000, 0x7e5: 0x2000, 0x7e6: 0x2000, 0x7e7: 0x2000, 0x7e8: 0x2000, 0x7e9: 0x2000, - 0x7eb: 0x2000, 0x7ec: 0x2000, 0x7ed: 0x2000, 0x7ee: 0x2000, 0x7ef: 0x2000, - 0x7f0: 0x2000, 0x7f1: 0x2000, 0x7f2: 0x2000, 0x7f3: 0x2000, 0x7f4: 0x2000, 0x7f5: 0x2000, - 0x7f6: 0x2000, 0x7f7: 0x2000, 0x7f8: 0x2000, 0x7f9: 0x2000, 0x7fa: 0x2000, 0x7fb: 0x2000, - 0x7fc: 0x2000, 0x7fd: 0x2000, 0x7fe: 0x2000, 0x7ff: 0x2000, - // Block 0x20, offset 0x800 - 0x800: 0x2000, 0x801: 0x2000, 0x802: 0x200d, 0x803: 0x2000, 0x804: 0x2000, 0x805: 0x2000, - 0x806: 0x2000, 0x807: 0x2000, 0x808: 0x2000, 0x809: 0x2000, 0x80a: 0x2000, 0x80b: 0x2000, - 0x80c: 0x2000, 0x80d: 0x2000, 0x80e: 0x2000, 0x80f: 0x2000, 0x810: 0x2000, 0x811: 0x2000, - 0x812: 0x2000, 0x813: 0x2000, 0x814: 0x2000, 0x815: 0x2000, 0x816: 0x2000, 0x817: 0x2000, - 0x818: 0x2000, 0x819: 0x2000, 0x81a: 0x2000, 0x81b: 0x2000, 0x81c: 0x2000, 0x81d: 0x2000, - 0x81e: 0x2000, 0x81f: 0x2000, 0x820: 0x2000, 0x821: 0x2000, 0x822: 0x2000, 0x823: 0x2000, - 0x824: 0x2000, 0x825: 0x2000, 0x826: 0x2000, 0x827: 0x2000, 0x828: 0x2000, 0x829: 0x2000, - 0x82a: 0x2000, 0x82b: 0x2000, 0x82c: 0x2000, 0x82d: 0x2000, 0x82e: 0x2000, 0x82f: 0x2000, - 0x830: 0x2000, 0x831: 0x2000, 0x832: 0x2000, 0x833: 0x2000, 0x834: 0x2000, 0x835: 0x2000, - 0x836: 0x2000, 0x837: 0x2000, 0x838: 0x2000, 0x839: 0x2000, 0x83a: 0x2000, 0x83b: 0x2000, - 0x83c: 0x2000, 0x83d: 0x2000, 0x83e: 0x2000, 0x83f: 0x2000, - // Block 0x21, offset 0x840 - 0x840: 0x2000, 0x841: 0x2000, 0x842: 0x2000, 0x843: 0x2000, 0x844: 0x2000, 0x845: 0x2000, - 0x846: 0x2000, 0x847: 0x2000, 0x848: 0x2000, 0x849: 0x2000, 0x84a: 0x2000, 0x84b: 0x2000, - 0x850: 0x2000, 0x851: 0x2000, - 0x852: 0x2000, 0x853: 0x2000, 0x854: 0x2000, 0x855: 0x2000, 0x856: 0x2000, 0x857: 0x2000, - 0x858: 0x2000, 0x859: 0x2000, 0x85a: 0x2000, 0x85b: 0x2000, 0x85c: 0x2000, 0x85d: 0x2000, - 0x85e: 0x2000, 0x85f: 0x2000, 0x860: 0x2000, 0x861: 0x2000, 0x862: 0x2000, 0x863: 0x2000, - 0x864: 0x2000, 0x865: 0x2000, 0x866: 0x2000, 0x867: 0x2000, 0x868: 0x2000, 0x869: 0x2000, - 0x86a: 0x2000, 0x86b: 0x2000, 0x86c: 0x2000, 0x86d: 0x2000, 0x86e: 0x2000, 0x86f: 0x2000, - 0x870: 0x2000, 0x871: 0x2000, 0x872: 0x2000, 0x873: 0x2000, - // Block 0x22, offset 0x880 - 0x880: 0x2000, 0x881: 0x2000, 0x882: 0x2000, 0x883: 0x2000, 0x884: 0x2000, 0x885: 0x2000, - 0x886: 0x2000, 0x887: 0x2000, 0x888: 0x2000, 0x889: 0x2000, 0x88a: 0x2000, 0x88b: 0x2000, - 0x88c: 0x2000, 0x88d: 0x2000, 0x88e: 0x2000, 0x88f: 0x2000, - 0x892: 0x2000, 0x893: 0x2000, 0x894: 0x2000, 0x895: 0x2000, - 0x8a0: 0x200e, 0x8a1: 0x2000, 0x8a3: 0x2000, - 0x8a4: 0x2000, 0x8a5: 0x2000, 0x8a6: 0x2000, 0x8a7: 0x2000, 0x8a8: 0x2000, 0x8a9: 0x2000, - 0x8b2: 0x2000, 0x8b3: 0x2000, - 0x8b6: 0x2000, 0x8b7: 0x2000, - 0x8bc: 0x2000, 0x8bd: 0x2000, - // Block 0x23, offset 0x8c0 - 0x8c0: 0x2000, 0x8c1: 0x2000, - 0x8c6: 0x2000, 0x8c7: 0x2000, 0x8c8: 0x2000, 0x8cb: 0x200f, - 0x8ce: 0x2000, 0x8cf: 0x2000, 0x8d0: 0x2000, 0x8d1: 0x2000, - 0x8e2: 0x2000, 0x8e3: 0x2000, - 0x8e4: 0x2000, 0x8e5: 0x2000, - 0x8ef: 0x2000, - 0x8fd: 0x4000, 0x8fe: 0x4000, - // Block 0x24, offset 0x900 - 0x905: 0x2000, - 0x906: 0x2000, 0x909: 0x2000, - 0x90e: 0x2000, 0x90f: 0x2000, - 0x914: 0x4000, 0x915: 0x4000, - 0x91c: 0x2000, - 0x91e: 0x2000, - // Block 0x25, offset 0x940 - 0x940: 0x2000, 0x942: 0x2000, - 0x948: 0x4000, 0x949: 0x4000, 0x94a: 0x4000, 0x94b: 0x4000, - 0x94c: 0x4000, 0x94d: 0x4000, 0x94e: 0x4000, 0x94f: 0x4000, 0x950: 0x4000, 0x951: 0x4000, - 0x952: 0x4000, 0x953: 0x4000, - 0x960: 0x2000, 0x961: 0x2000, 0x963: 0x2000, - 0x964: 0x2000, 0x965: 0x2000, 0x967: 0x2000, 0x968: 0x2000, 0x969: 0x2000, - 0x96a: 0x2000, 0x96c: 0x2000, 0x96d: 0x2000, 0x96f: 0x2000, - 0x97f: 0x4000, - // Block 0x26, offset 0x980 - 0x993: 0x4000, - 0x99e: 0x2000, 0x99f: 0x2000, 0x9a1: 0x4000, - 0x9aa: 0x4000, 0x9ab: 0x4000, - 0x9bd: 0x4000, 0x9be: 0x4000, 0x9bf: 0x2000, - // Block 0x27, offset 0x9c0 - 0x9c4: 0x4000, 0x9c5: 0x4000, - 0x9c6: 0x2000, 0x9c7: 0x2000, 0x9c8: 0x2000, 0x9c9: 0x2000, 0x9ca: 0x2000, 0x9cb: 0x2000, - 0x9cc: 0x2000, 0x9cd: 0x2000, 0x9ce: 0x4000, 0x9cf: 0x2000, 0x9d0: 0x2000, 0x9d1: 0x2000, - 0x9d2: 0x2000, 0x9d3: 0x2000, 0x9d4: 0x4000, 0x9d5: 0x2000, 0x9d6: 0x2000, 0x9d7: 0x2000, - 0x9d8: 0x2000, 0x9d9: 0x2000, 0x9da: 0x2000, 0x9db: 0x2000, 0x9dc: 0x2000, 0x9dd: 0x2000, - 0x9de: 0x2000, 0x9df: 0x2000, 0x9e0: 0x2000, 0x9e1: 0x2000, 0x9e3: 0x2000, - 0x9e8: 0x2000, 0x9e9: 0x2000, - 0x9ea: 0x4000, 0x9eb: 0x2000, 0x9ec: 0x2000, 0x9ed: 0x2000, 0x9ee: 0x2000, 0x9ef: 0x2000, - 0x9f0: 0x2000, 0x9f1: 0x2000, 0x9f2: 0x4000, 0x9f3: 0x4000, 0x9f4: 0x2000, 0x9f5: 0x4000, - 0x9f6: 0x2000, 0x9f7: 0x2000, 0x9f8: 0x2000, 0x9f9: 0x2000, 0x9fa: 0x4000, 0x9fb: 0x2000, - 0x9fc: 0x2000, 0x9fd: 0x4000, 0x9fe: 0x2000, 0x9ff: 0x2000, - // Block 0x28, offset 0xa00 - 0xa05: 0x4000, - 0xa0a: 0x4000, 0xa0b: 0x4000, - 0xa28: 0x4000, - 0xa3d: 0x2000, - // Block 0x29, offset 0xa40 - 0xa4c: 0x4000, 0xa4e: 0x4000, - 0xa53: 0x4000, 0xa54: 0x4000, 0xa55: 0x4000, 0xa57: 0x4000, - 0xa76: 0x2000, 0xa77: 0x2000, 0xa78: 0x2000, 0xa79: 0x2000, 0xa7a: 0x2000, 0xa7b: 0x2000, - 0xa7c: 0x2000, 0xa7d: 0x2000, 0xa7e: 0x2000, 0xa7f: 0x2000, - // Block 0x2a, offset 0xa80 - 0xa95: 0x4000, 0xa96: 0x4000, 0xa97: 0x4000, - 0xab0: 0x4000, - 0xabf: 0x4000, - // Block 0x2b, offset 0xac0 - 0xae6: 0x6000, 0xae7: 0x6000, 0xae8: 0x6000, 0xae9: 0x6000, - 0xaea: 0x6000, 0xaeb: 0x6000, 0xaec: 0x6000, 0xaed: 0x6000, - // Block 0x2c, offset 0xb00 - 0xb05: 0x6010, - 0xb06: 0x6011, - // Block 0x2d, offset 0xb40 - 0xb5b: 0x4000, 0xb5c: 0x4000, - // Block 0x2e, offset 0xb80 - 0xb90: 0x4000, - 0xb95: 0x4000, 0xb96: 0x2000, 0xb97: 0x2000, - 0xb98: 0x2000, 0xb99: 0x2000, - // Block 0x2f, offset 0xbc0 - 0xbc0: 0x4000, 0xbc1: 0x4000, 0xbc2: 0x4000, 0xbc3: 0x4000, 0xbc4: 0x4000, 0xbc5: 0x4000, - 0xbc6: 0x4000, 0xbc7: 0x4000, 0xbc8: 0x4000, 0xbc9: 0x4000, 0xbca: 0x4000, 0xbcb: 0x4000, - 0xbcc: 0x4000, 0xbcd: 0x4000, 0xbce: 0x4000, 0xbcf: 0x4000, 0xbd0: 0x4000, 0xbd1: 0x4000, - 0xbd2: 0x4000, 0xbd3: 0x4000, 0xbd4: 0x4000, 0xbd5: 0x4000, 0xbd6: 0x4000, 0xbd7: 0x4000, - 0xbd8: 0x4000, 0xbd9: 0x4000, 0xbdb: 0x4000, 0xbdc: 0x4000, 0xbdd: 0x4000, - 0xbde: 0x4000, 0xbdf: 0x4000, 0xbe0: 0x4000, 0xbe1: 0x4000, 0xbe2: 0x4000, 0xbe3: 0x4000, - 0xbe4: 0x4000, 0xbe5: 0x4000, 0xbe6: 0x4000, 0xbe7: 0x4000, 0xbe8: 0x4000, 0xbe9: 0x4000, - 0xbea: 0x4000, 0xbeb: 0x4000, 0xbec: 0x4000, 0xbed: 0x4000, 0xbee: 0x4000, 0xbef: 0x4000, - 0xbf0: 0x4000, 0xbf1: 0x4000, 0xbf2: 0x4000, 0xbf3: 0x4000, 0xbf4: 0x4000, 0xbf5: 0x4000, - 0xbf6: 0x4000, 0xbf7: 0x4000, 0xbf8: 0x4000, 0xbf9: 0x4000, 0xbfa: 0x4000, 0xbfb: 0x4000, - 0xbfc: 0x4000, 0xbfd: 0x4000, 0xbfe: 0x4000, 0xbff: 0x4000, - // Block 0x30, offset 0xc00 - 0xc00: 0x4000, 0xc01: 0x4000, 0xc02: 0x4000, 0xc03: 0x4000, 0xc04: 0x4000, 0xc05: 0x4000, - 0xc06: 0x4000, 0xc07: 0x4000, 0xc08: 0x4000, 0xc09: 0x4000, 0xc0a: 0x4000, 0xc0b: 0x4000, - 0xc0c: 0x4000, 0xc0d: 0x4000, 0xc0e: 0x4000, 0xc0f: 0x4000, 0xc10: 0x4000, 0xc11: 0x4000, - 0xc12: 0x4000, 0xc13: 0x4000, 0xc14: 0x4000, 0xc15: 0x4000, 0xc16: 0x4000, 0xc17: 0x4000, - 0xc18: 0x4000, 0xc19: 0x4000, 0xc1a: 0x4000, 0xc1b: 0x4000, 0xc1c: 0x4000, 0xc1d: 0x4000, - 0xc1e: 0x4000, 0xc1f: 0x4000, 0xc20: 0x4000, 0xc21: 0x4000, 0xc22: 0x4000, 0xc23: 0x4000, - 0xc24: 0x4000, 0xc25: 0x4000, 0xc26: 0x4000, 0xc27: 0x4000, 0xc28: 0x4000, 0xc29: 0x4000, - 0xc2a: 0x4000, 0xc2b: 0x4000, 0xc2c: 0x4000, 0xc2d: 0x4000, 0xc2e: 0x4000, 0xc2f: 0x4000, - 0xc30: 0x4000, 0xc31: 0x4000, 0xc32: 0x4000, 0xc33: 0x4000, - // Block 0x31, offset 0xc40 - 0xc40: 0x4000, 0xc41: 0x4000, 0xc42: 0x4000, 0xc43: 0x4000, 0xc44: 0x4000, 0xc45: 0x4000, - 0xc46: 0x4000, 0xc47: 0x4000, 0xc48: 0x4000, 0xc49: 0x4000, 0xc4a: 0x4000, 0xc4b: 0x4000, - 0xc4c: 0x4000, 0xc4d: 0x4000, 0xc4e: 0x4000, 0xc4f: 0x4000, 0xc50: 0x4000, 0xc51: 0x4000, - 0xc52: 0x4000, 0xc53: 0x4000, 0xc54: 0x4000, 0xc55: 0x4000, - 0xc70: 0x4000, 0xc71: 0x4000, 0xc72: 0x4000, 0xc73: 0x4000, 0xc74: 0x4000, 0xc75: 0x4000, - 0xc76: 0x4000, 0xc77: 0x4000, 0xc78: 0x4000, 0xc79: 0x4000, 0xc7a: 0x4000, 0xc7b: 0x4000, - // Block 0x32, offset 0xc80 - 0xc80: 0x9012, 0xc81: 0x4013, 0xc82: 0x4014, 0xc83: 0x4000, 0xc84: 0x4000, 0xc85: 0x4000, - 0xc86: 0x4000, 0xc87: 0x4000, 0xc88: 0x4000, 0xc89: 0x4000, 0xc8a: 0x4000, 0xc8b: 0x4000, - 0xc8c: 0x4015, 0xc8d: 0x4015, 0xc8e: 0x4000, 0xc8f: 0x4000, 0xc90: 0x4000, 0xc91: 0x4000, - 0xc92: 0x4000, 0xc93: 0x4000, 0xc94: 0x4000, 0xc95: 0x4000, 0xc96: 0x4000, 0xc97: 0x4000, - 0xc98: 0x4000, 0xc99: 0x4000, 0xc9a: 0x4000, 0xc9b: 0x4000, 0xc9c: 0x4000, 0xc9d: 0x4000, - 0xc9e: 0x4000, 0xc9f: 0x4000, 0xca0: 0x4000, 0xca1: 0x4000, 0xca2: 0x4000, 0xca3: 0x4000, - 0xca4: 0x4000, 0xca5: 0x4000, 0xca6: 0x4000, 0xca7: 0x4000, 0xca8: 0x4000, 0xca9: 0x4000, - 0xcaa: 0x4000, 0xcab: 0x4000, 0xcac: 0x4000, 0xcad: 0x4000, 0xcae: 0x4000, 0xcaf: 0x4000, - 0xcb0: 0x4000, 0xcb1: 0x4000, 0xcb2: 0x4000, 0xcb3: 0x4000, 0xcb4: 0x4000, 0xcb5: 0x4000, - 0xcb6: 0x4000, 0xcb7: 0x4000, 0xcb8: 0x4000, 0xcb9: 0x4000, 0xcba: 0x4000, 0xcbb: 0x4000, - 0xcbc: 0x4000, 0xcbd: 0x4000, 0xcbe: 0x4000, - // Block 0x33, offset 0xcc0 - 0xcc1: 0x4000, 0xcc2: 0x4000, 0xcc3: 0x4000, 0xcc4: 0x4000, 0xcc5: 0x4000, - 0xcc6: 0x4000, 0xcc7: 0x4000, 0xcc8: 0x4000, 0xcc9: 0x4000, 0xcca: 0x4000, 0xccb: 0x4000, - 0xccc: 0x4000, 0xccd: 0x4000, 0xcce: 0x4000, 0xccf: 0x4000, 0xcd0: 0x4000, 0xcd1: 0x4000, - 0xcd2: 0x4000, 0xcd3: 0x4000, 0xcd4: 0x4000, 0xcd5: 0x4000, 0xcd6: 0x4000, 0xcd7: 0x4000, - 0xcd8: 0x4000, 0xcd9: 0x4000, 0xcda: 0x4000, 0xcdb: 0x4000, 0xcdc: 0x4000, 0xcdd: 0x4000, - 0xcde: 0x4000, 0xcdf: 0x4000, 0xce0: 0x4000, 0xce1: 0x4000, 0xce2: 0x4000, 0xce3: 0x4000, - 0xce4: 0x4000, 0xce5: 0x4000, 0xce6: 0x4000, 0xce7: 0x4000, 0xce8: 0x4000, 0xce9: 0x4000, - 0xcea: 0x4000, 0xceb: 0x4000, 0xcec: 0x4000, 0xced: 0x4000, 0xcee: 0x4000, 0xcef: 0x4000, - 0xcf0: 0x4000, 0xcf1: 0x4000, 0xcf2: 0x4000, 0xcf3: 0x4000, 0xcf4: 0x4000, 0xcf5: 0x4000, - 0xcf6: 0x4000, 0xcf7: 0x4000, 0xcf8: 0x4000, 0xcf9: 0x4000, 0xcfa: 0x4000, 0xcfb: 0x4000, - 0xcfc: 0x4000, 0xcfd: 0x4000, 0xcfe: 0x4000, 0xcff: 0x4000, - // Block 0x34, offset 0xd00 - 0xd00: 0x4000, 0xd01: 0x4000, 0xd02: 0x4000, 0xd03: 0x4000, 0xd04: 0x4000, 0xd05: 0x4000, - 0xd06: 0x4000, 0xd07: 0x4000, 0xd08: 0x4000, 0xd09: 0x4000, 0xd0a: 0x4000, 0xd0b: 0x4000, - 0xd0c: 0x4000, 0xd0d: 0x4000, 0xd0e: 0x4000, 0xd0f: 0x4000, 0xd10: 0x4000, 0xd11: 0x4000, - 0xd12: 0x4000, 0xd13: 0x4000, 0xd14: 0x4000, 0xd15: 0x4000, 0xd16: 0x4000, - 0xd19: 0x4016, 0xd1a: 0x4017, 0xd1b: 0x4000, 0xd1c: 0x4000, 0xd1d: 0x4000, - 0xd1e: 0x4000, 0xd1f: 0x4000, 0xd20: 0x4000, 0xd21: 0x4018, 0xd22: 0x4019, 0xd23: 0x401a, - 0xd24: 0x401b, 0xd25: 0x401c, 0xd26: 0x401d, 0xd27: 0x401e, 0xd28: 0x401f, 0xd29: 0x4020, - 0xd2a: 0x4021, 0xd2b: 0x4022, 0xd2c: 0x4000, 0xd2d: 0x4010, 0xd2e: 0x4000, 0xd2f: 0x4023, - 0xd30: 0x4000, 0xd31: 0x4024, 0xd32: 0x4000, 0xd33: 0x4025, 0xd34: 0x4000, 0xd35: 0x4026, - 0xd36: 0x4000, 0xd37: 0x401a, 0xd38: 0x4000, 0xd39: 0x4027, 0xd3a: 0x4000, 0xd3b: 0x4028, - 0xd3c: 0x4000, 0xd3d: 0x4020, 0xd3e: 0x4000, 0xd3f: 0x4029, - // Block 0x35, offset 0xd40 - 0xd40: 0x4000, 0xd41: 0x402a, 0xd42: 0x4000, 0xd43: 0x402b, 0xd44: 0x402c, 0xd45: 0x4000, - 0xd46: 0x4017, 0xd47: 0x4000, 0xd48: 0x402d, 0xd49: 0x4000, 0xd4a: 0x402e, 0xd4b: 0x402f, - 0xd4c: 0x4030, 0xd4d: 0x4017, 0xd4e: 0x4016, 0xd4f: 0x4017, 0xd50: 0x4000, 0xd51: 0x4000, - 0xd52: 0x4031, 0xd53: 0x4000, 0xd54: 0x4000, 0xd55: 0x4031, 0xd56: 0x4000, 0xd57: 0x4000, - 0xd58: 0x4032, 0xd59: 0x4000, 0xd5a: 0x4000, 0xd5b: 0x4032, 0xd5c: 0x4000, 0xd5d: 0x4000, - 0xd5e: 0x4033, 0xd5f: 0x402e, 0xd60: 0x4034, 0xd61: 0x4035, 0xd62: 0x4034, 0xd63: 0x4036, - 0xd64: 0x4037, 0xd65: 0x4024, 0xd66: 0x4035, 0xd67: 0x4025, 0xd68: 0x4038, 0xd69: 0x4038, - 0xd6a: 0x4039, 0xd6b: 0x4039, 0xd6c: 0x403a, 0xd6d: 0x403a, 0xd6e: 0x4000, 0xd6f: 0x4035, - 0xd70: 0x4000, 0xd71: 0x4000, 0xd72: 0x403b, 0xd73: 0x403c, 0xd74: 0x4000, 0xd75: 0x4000, - 0xd76: 0x4000, 0xd77: 0x4000, 0xd78: 0x4000, 0xd79: 0x4000, 0xd7a: 0x4000, 0xd7b: 0x403d, - 0xd7c: 0x401c, 0xd7d: 0x4000, 0xd7e: 0x4000, 0xd7f: 0x4000, - // Block 0x36, offset 0xd80 - 0xd85: 0x4000, - 0xd86: 0x4000, 0xd87: 0x4000, 0xd88: 0x4000, 0xd89: 0x4000, 0xd8a: 0x4000, 0xd8b: 0x4000, - 0xd8c: 0x4000, 0xd8d: 0x4000, 0xd8e: 0x4000, 0xd8f: 0x4000, 0xd90: 0x4000, 0xd91: 0x4000, - 0xd92: 0x4000, 0xd93: 0x4000, 0xd94: 0x4000, 0xd95: 0x4000, 0xd96: 0x4000, 0xd97: 0x4000, - 0xd98: 0x4000, 0xd99: 0x4000, 0xd9a: 0x4000, 0xd9b: 0x4000, 0xd9c: 0x4000, 0xd9d: 0x4000, - 0xd9e: 0x4000, 0xd9f: 0x4000, 0xda0: 0x4000, 0xda1: 0x4000, 0xda2: 0x4000, 0xda3: 0x4000, - 0xda4: 0x4000, 0xda5: 0x4000, 0xda6: 0x4000, 0xda7: 0x4000, 0xda8: 0x4000, 0xda9: 0x4000, - 0xdaa: 0x4000, 0xdab: 0x4000, 0xdac: 0x4000, 0xdad: 0x4000, - 0xdb1: 0x403e, 0xdb2: 0x403e, 0xdb3: 0x403e, 0xdb4: 0x403e, 0xdb5: 0x403e, - 0xdb6: 0x403e, 0xdb7: 0x403e, 0xdb8: 0x403e, 0xdb9: 0x403e, 0xdba: 0x403e, 0xdbb: 0x403e, - 0xdbc: 0x403e, 0xdbd: 0x403e, 0xdbe: 0x403e, 0xdbf: 0x403e, - // Block 0x37, offset 0xdc0 - 0xdc0: 0x4037, 0xdc1: 0x4037, 0xdc2: 0x4037, 0xdc3: 0x4037, 0xdc4: 0x4037, 0xdc5: 0x4037, - 0xdc6: 0x4037, 0xdc7: 0x4037, 0xdc8: 0x4037, 0xdc9: 0x4037, 0xdca: 0x4037, 0xdcb: 0x4037, - 0xdcc: 0x4037, 0xdcd: 0x4037, 0xdce: 0x4037, 0xdcf: 0x400e, 0xdd0: 0x403f, 0xdd1: 0x4040, - 0xdd2: 0x4041, 0xdd3: 0x4040, 0xdd4: 0x403f, 0xdd5: 0x4042, 0xdd6: 0x4043, 0xdd7: 0x4044, - 0xdd8: 0x4040, 0xdd9: 0x4041, 0xdda: 0x4040, 0xddb: 0x4045, 0xddc: 0x4009, 0xddd: 0x4045, - 0xdde: 0x4046, 0xddf: 0x4045, 0xde0: 0x4047, 0xde1: 0x400b, 0xde2: 0x400a, 0xde3: 0x400c, - 0xde4: 0x4048, 0xde5: 0x4000, 0xde6: 0x4000, 0xde7: 0x4000, 0xde8: 0x4000, 0xde9: 0x4000, - 0xdea: 0x4000, 0xdeb: 0x4000, 0xdec: 0x4000, 0xded: 0x4000, 0xdee: 0x4000, 0xdef: 0x4000, - 0xdf0: 0x4000, 0xdf1: 0x4000, 0xdf2: 0x4000, 0xdf3: 0x4000, 0xdf4: 0x4000, 0xdf5: 0x4000, - 0xdf6: 0x4000, 0xdf7: 0x4000, 0xdf8: 0x4000, 0xdf9: 0x4000, 0xdfa: 0x4000, 0xdfb: 0x4000, - 0xdfc: 0x4000, 0xdfd: 0x4000, 0xdfe: 0x4000, 0xdff: 0x4000, - // Block 0x38, offset 0xe00 - 0xe00: 0x4000, 0xe01: 0x4000, 0xe02: 0x4000, 0xe03: 0x4000, 0xe04: 0x4000, 0xe05: 0x4000, - 0xe06: 0x4000, 0xe07: 0x4000, 0xe08: 0x4000, 0xe09: 0x4000, 0xe0a: 0x4000, 0xe0b: 0x4000, - 0xe0c: 0x4000, 0xe0d: 0x4000, 0xe0e: 0x4000, 0xe10: 0x4000, 0xe11: 0x4000, - 0xe12: 0x4000, 0xe13: 0x4000, 0xe14: 0x4000, 0xe15: 0x4000, 0xe16: 0x4000, 0xe17: 0x4000, - 0xe18: 0x4000, 0xe19: 0x4000, 0xe1a: 0x4000, 0xe1b: 0x4000, 0xe1c: 0x4000, 0xe1d: 0x4000, - 0xe1e: 0x4000, 0xe1f: 0x4000, 0xe20: 0x4000, 0xe21: 0x4000, 0xe22: 0x4000, 0xe23: 0x4000, - 0xe24: 0x4000, 0xe25: 0x4000, 0xe26: 0x4000, 0xe27: 0x4000, 0xe28: 0x4000, 0xe29: 0x4000, - 0xe2a: 0x4000, 0xe2b: 0x4000, 0xe2c: 0x4000, 0xe2d: 0x4000, 0xe2e: 0x4000, 0xe2f: 0x4000, - 0xe30: 0x4000, 0xe31: 0x4000, 0xe32: 0x4000, 0xe33: 0x4000, 0xe34: 0x4000, 0xe35: 0x4000, - 0xe36: 0x4000, 0xe37: 0x4000, 0xe38: 0x4000, 0xe39: 0x4000, 0xe3a: 0x4000, - // Block 0x39, offset 0xe40 - 0xe40: 0x4000, 0xe41: 0x4000, 0xe42: 0x4000, 0xe43: 0x4000, 0xe44: 0x4000, 0xe45: 0x4000, - 0xe46: 0x4000, 0xe47: 0x4000, 0xe48: 0x4000, 0xe49: 0x4000, 0xe4a: 0x4000, 0xe4b: 0x4000, - 0xe4c: 0x4000, 0xe4d: 0x4000, 0xe4e: 0x4000, 0xe4f: 0x4000, 0xe50: 0x4000, 0xe51: 0x4000, - 0xe52: 0x4000, 0xe53: 0x4000, 0xe54: 0x4000, 0xe55: 0x4000, 0xe56: 0x4000, 0xe57: 0x4000, - 0xe58: 0x4000, 0xe59: 0x4000, 0xe5a: 0x4000, 0xe5b: 0x4000, 0xe5c: 0x4000, 0xe5d: 0x4000, - 0xe5e: 0x4000, 0xe5f: 0x4000, 0xe60: 0x4000, 0xe61: 0x4000, 0xe62: 0x4000, 0xe63: 0x4000, - 0xe70: 0x4000, 0xe71: 0x4000, 0xe72: 0x4000, 0xe73: 0x4000, 0xe74: 0x4000, 0xe75: 0x4000, - 0xe76: 0x4000, 0xe77: 0x4000, 0xe78: 0x4000, 0xe79: 0x4000, 0xe7a: 0x4000, 0xe7b: 0x4000, - 0xe7c: 0x4000, 0xe7d: 0x4000, 0xe7e: 0x4000, 0xe7f: 0x4000, - // Block 0x3a, offset 0xe80 - 0xe80: 0x4000, 0xe81: 0x4000, 0xe82: 0x4000, 0xe83: 0x4000, 0xe84: 0x4000, 0xe85: 0x4000, - 0xe86: 0x4000, 0xe87: 0x4000, 0xe88: 0x4000, 0xe89: 0x4000, 0xe8a: 0x4000, 0xe8b: 0x4000, - 0xe8c: 0x4000, 0xe8d: 0x4000, 0xe8e: 0x4000, 0xe8f: 0x4000, 0xe90: 0x4000, 0xe91: 0x4000, - 0xe92: 0x4000, 0xe93: 0x4000, 0xe94: 0x4000, 0xe95: 0x4000, 0xe96: 0x4000, 0xe97: 0x4000, - 0xe98: 0x4000, 0xe99: 0x4000, 0xe9a: 0x4000, 0xe9b: 0x4000, 0xe9c: 0x4000, 0xe9d: 0x4000, - 0xe9e: 0x4000, 0xea0: 0x4000, 0xea1: 0x4000, 0xea2: 0x4000, 0xea3: 0x4000, - 0xea4: 0x4000, 0xea5: 0x4000, 0xea6: 0x4000, 0xea7: 0x4000, 0xea8: 0x4000, 0xea9: 0x4000, - 0xeaa: 0x4000, 0xeab: 0x4000, 0xeac: 0x4000, 0xead: 0x4000, 0xeae: 0x4000, 0xeaf: 0x4000, - 0xeb0: 0x4000, 0xeb1: 0x4000, 0xeb2: 0x4000, 0xeb3: 0x4000, 0xeb4: 0x4000, 0xeb5: 0x4000, - 0xeb6: 0x4000, 0xeb7: 0x4000, 0xeb8: 0x4000, 0xeb9: 0x4000, 0xeba: 0x4000, 0xebb: 0x4000, - 0xebc: 0x4000, 0xebd: 0x4000, 0xebe: 0x4000, 0xebf: 0x4000, - // Block 0x3b, offset 0xec0 - 0xec0: 0x4000, 0xec1: 0x4000, 0xec2: 0x4000, 0xec3: 0x4000, 0xec4: 0x4000, 0xec5: 0x4000, - 0xec6: 0x4000, 0xec7: 0x4000, 0xec8: 0x2000, 0xec9: 0x2000, 0xeca: 0x2000, 0xecb: 0x2000, - 0xecc: 0x2000, 0xecd: 0x2000, 0xece: 0x2000, 0xecf: 0x2000, 0xed0: 0x4000, 0xed1: 0x4000, - 0xed2: 0x4000, 0xed3: 0x4000, 0xed4: 0x4000, 0xed5: 0x4000, 0xed6: 0x4000, 0xed7: 0x4000, - 0xed8: 0x4000, 0xed9: 0x4000, 0xeda: 0x4000, 0xedb: 0x4000, 0xedc: 0x4000, 0xedd: 0x4000, - 0xede: 0x4000, 0xedf: 0x4000, 0xee0: 0x4000, 0xee1: 0x4000, 0xee2: 0x4000, 0xee3: 0x4000, - 0xee4: 0x4000, 0xee5: 0x4000, 0xee6: 0x4000, 0xee7: 0x4000, 0xee8: 0x4000, 0xee9: 0x4000, - 0xeea: 0x4000, 0xeeb: 0x4000, 0xeec: 0x4000, 0xeed: 0x4000, 0xeee: 0x4000, 0xeef: 0x4000, - 0xef0: 0x4000, 0xef1: 0x4000, 0xef2: 0x4000, 0xef3: 0x4000, 0xef4: 0x4000, 0xef5: 0x4000, - 0xef6: 0x4000, 0xef7: 0x4000, 0xef8: 0x4000, 0xef9: 0x4000, 0xefa: 0x4000, 0xefb: 0x4000, - 0xefc: 0x4000, 0xefd: 0x4000, 0xefe: 0x4000, 0xeff: 0x4000, - // Block 0x3c, offset 0xf00 - 0xf00: 0x4000, 0xf01: 0x4000, 0xf02: 0x4000, 0xf03: 0x4000, 0xf04: 0x4000, 0xf05: 0x4000, - 0xf06: 0x4000, 0xf07: 0x4000, 0xf08: 0x4000, 0xf09: 0x4000, 0xf0a: 0x4000, 0xf0b: 0x4000, - 0xf0c: 0x4000, 0xf0d: 0x4000, 0xf0e: 0x4000, 0xf0f: 0x4000, 0xf10: 0x4000, 0xf11: 0x4000, - 0xf12: 0x4000, 0xf13: 0x4000, 0xf14: 0x4000, 0xf15: 0x4000, 0xf16: 0x4000, 0xf17: 0x4000, - 0xf18: 0x4000, 0xf19: 0x4000, 0xf1a: 0x4000, 0xf1b: 0x4000, 0xf1c: 0x4000, 0xf1d: 0x4000, - 0xf1e: 0x4000, 0xf1f: 0x4000, 0xf20: 0x4000, 0xf21: 0x4000, 0xf22: 0x4000, 0xf23: 0x4000, - 0xf24: 0x4000, 0xf25: 0x4000, 0xf26: 0x4000, 0xf27: 0x4000, 0xf28: 0x4000, 0xf29: 0x4000, - 0xf2a: 0x4000, 0xf2b: 0x4000, 0xf2c: 0x4000, 0xf2d: 0x4000, 0xf2e: 0x4000, 0xf2f: 0x4000, - 0xf30: 0x4000, 0xf31: 0x4000, 0xf32: 0x4000, 0xf33: 0x4000, 0xf34: 0x4000, 0xf35: 0x4000, - 0xf36: 0x4000, 0xf37: 0x4000, 0xf38: 0x4000, 0xf39: 0x4000, 0xf3a: 0x4000, 0xf3b: 0x4000, - 0xf3c: 0x4000, 0xf3d: 0x4000, 0xf3e: 0x4000, - // Block 0x3d, offset 0xf40 - 0xf40: 0x4000, 0xf41: 0x4000, 0xf42: 0x4000, 0xf43: 0x4000, 0xf44: 0x4000, 0xf45: 0x4000, - 0xf46: 0x4000, 0xf47: 0x4000, 0xf48: 0x4000, 0xf49: 0x4000, 0xf4a: 0x4000, 0xf4b: 0x4000, - 0xf4c: 0x4000, 0xf50: 0x4000, 0xf51: 0x4000, - 0xf52: 0x4000, 0xf53: 0x4000, 0xf54: 0x4000, 0xf55: 0x4000, 0xf56: 0x4000, 0xf57: 0x4000, - 0xf58: 0x4000, 0xf59: 0x4000, 0xf5a: 0x4000, 0xf5b: 0x4000, 0xf5c: 0x4000, 0xf5d: 0x4000, - 0xf5e: 0x4000, 0xf5f: 0x4000, 0xf60: 0x4000, 0xf61: 0x4000, 0xf62: 0x4000, 0xf63: 0x4000, - 0xf64: 0x4000, 0xf65: 0x4000, 0xf66: 0x4000, 0xf67: 0x4000, 0xf68: 0x4000, 0xf69: 0x4000, - 0xf6a: 0x4000, 0xf6b: 0x4000, 0xf6c: 0x4000, 0xf6d: 0x4000, 0xf6e: 0x4000, 0xf6f: 0x4000, - 0xf70: 0x4000, 0xf71: 0x4000, 0xf72: 0x4000, 0xf73: 0x4000, 0xf74: 0x4000, 0xf75: 0x4000, - 0xf76: 0x4000, 0xf77: 0x4000, 0xf78: 0x4000, 0xf79: 0x4000, 0xf7a: 0x4000, 0xf7b: 0x4000, - 0xf7c: 0x4000, 0xf7d: 0x4000, 0xf7e: 0x4000, 0xf7f: 0x4000, - // Block 0x3e, offset 0xf80 - 0xf80: 0x4000, 0xf81: 0x4000, 0xf82: 0x4000, 0xf83: 0x4000, 0xf84: 0x4000, 0xf85: 0x4000, - 0xf86: 0x4000, - // Block 0x3f, offset 0xfc0 - 0xfe0: 0x4000, 0xfe1: 0x4000, 0xfe2: 0x4000, 0xfe3: 0x4000, - 0xfe4: 0x4000, 0xfe5: 0x4000, 0xfe6: 0x4000, 0xfe7: 0x4000, 0xfe8: 0x4000, 0xfe9: 0x4000, - 0xfea: 0x4000, 0xfeb: 0x4000, 0xfec: 0x4000, 0xfed: 0x4000, 0xfee: 0x4000, 0xfef: 0x4000, - 0xff0: 0x4000, 0xff1: 0x4000, 0xff2: 0x4000, 0xff3: 0x4000, 0xff4: 0x4000, 0xff5: 0x4000, - 0xff6: 0x4000, 0xff7: 0x4000, 0xff8: 0x4000, 0xff9: 0x4000, 0xffa: 0x4000, 0xffb: 0x4000, - 0xffc: 0x4000, - // Block 0x40, offset 0x1000 - 0x1000: 0x4000, 0x1001: 0x4000, 0x1002: 0x4000, 0x1003: 0x4000, 0x1004: 0x4000, 0x1005: 0x4000, - 0x1006: 0x4000, 0x1007: 0x4000, 0x1008: 0x4000, 0x1009: 0x4000, 0x100a: 0x4000, 0x100b: 0x4000, - 0x100c: 0x4000, 0x100d: 0x4000, 0x100e: 0x4000, 0x100f: 0x4000, 0x1010: 0x4000, 0x1011: 0x4000, - 0x1012: 0x4000, 0x1013: 0x4000, 0x1014: 0x4000, 0x1015: 0x4000, 0x1016: 0x4000, 0x1017: 0x4000, - 0x1018: 0x4000, 0x1019: 0x4000, 0x101a: 0x4000, 0x101b: 0x4000, 0x101c: 0x4000, 0x101d: 0x4000, - 0x101e: 0x4000, 0x101f: 0x4000, 0x1020: 0x4000, 0x1021: 0x4000, 0x1022: 0x4000, 0x1023: 0x4000, - // Block 0x41, offset 0x1040 - 0x1040: 0x2000, 0x1041: 0x2000, 0x1042: 0x2000, 0x1043: 0x2000, 0x1044: 0x2000, 0x1045: 0x2000, - 0x1046: 0x2000, 0x1047: 0x2000, 0x1048: 0x2000, 0x1049: 0x2000, 0x104a: 0x2000, 0x104b: 0x2000, - 0x104c: 0x2000, 0x104d: 0x2000, 0x104e: 0x2000, 0x104f: 0x2000, 0x1050: 0x4000, 0x1051: 0x4000, - 0x1052: 0x4000, 0x1053: 0x4000, 0x1054: 0x4000, 0x1055: 0x4000, 0x1056: 0x4000, 0x1057: 0x4000, - 0x1058: 0x4000, 0x1059: 0x4000, - 0x1070: 0x4000, 0x1071: 0x4000, 0x1072: 0x4000, 0x1073: 0x4000, 0x1074: 0x4000, 0x1075: 0x4000, - 0x1076: 0x4000, 0x1077: 0x4000, 0x1078: 0x4000, 0x1079: 0x4000, 0x107a: 0x4000, 0x107b: 0x4000, - 0x107c: 0x4000, 0x107d: 0x4000, 0x107e: 0x4000, 0x107f: 0x4000, - // Block 0x42, offset 0x1080 - 0x1080: 0x4000, 0x1081: 0x4000, 0x1082: 0x4000, 0x1083: 0x4000, 0x1084: 0x4000, 0x1085: 0x4000, - 0x1086: 0x4000, 0x1087: 0x4000, 0x1088: 0x4000, 0x1089: 0x4000, 0x108a: 0x4000, 0x108b: 0x4000, - 0x108c: 0x4000, 0x108d: 0x4000, 0x108e: 0x4000, 0x108f: 0x4000, 0x1090: 0x4000, 0x1091: 0x4000, - 0x1092: 0x4000, 0x1094: 0x4000, 0x1095: 0x4000, 0x1096: 0x4000, 0x1097: 0x4000, - 0x1098: 0x4000, 0x1099: 0x4000, 0x109a: 0x4000, 0x109b: 0x4000, 0x109c: 0x4000, 0x109d: 0x4000, - 0x109e: 0x4000, 0x109f: 0x4000, 0x10a0: 0x4000, 0x10a1: 0x4000, 0x10a2: 0x4000, 0x10a3: 0x4000, - 0x10a4: 0x4000, 0x10a5: 0x4000, 0x10a6: 0x4000, 0x10a8: 0x4000, 0x10a9: 0x4000, - 0x10aa: 0x4000, 0x10ab: 0x4000, - // Block 0x43, offset 0x10c0 - 0x10c1: 0x9012, 0x10c2: 0x9012, 0x10c3: 0x9012, 0x10c4: 0x9012, 0x10c5: 0x9012, - 0x10c6: 0x9012, 0x10c7: 0x9012, 0x10c8: 0x9012, 0x10c9: 0x9012, 0x10ca: 0x9012, 0x10cb: 0x9012, - 0x10cc: 0x9012, 0x10cd: 0x9012, 0x10ce: 0x9012, 0x10cf: 0x9012, 0x10d0: 0x9012, 0x10d1: 0x9012, - 0x10d2: 0x9012, 0x10d3: 0x9012, 0x10d4: 0x9012, 0x10d5: 0x9012, 0x10d6: 0x9012, 0x10d7: 0x9012, - 0x10d8: 0x9012, 0x10d9: 0x9012, 0x10da: 0x9012, 0x10db: 0x9012, 0x10dc: 0x9012, 0x10dd: 0x9012, - 0x10de: 0x9012, 0x10df: 0x9012, 0x10e0: 0x9049, 0x10e1: 0x9049, 0x10e2: 0x9049, 0x10e3: 0x9049, - 0x10e4: 0x9049, 0x10e5: 0x9049, 0x10e6: 0x9049, 0x10e7: 0x9049, 0x10e8: 0x9049, 0x10e9: 0x9049, - 0x10ea: 0x9049, 0x10eb: 0x9049, 0x10ec: 0x9049, 0x10ed: 0x9049, 0x10ee: 0x9049, 0x10ef: 0x9049, - 0x10f0: 0x9049, 0x10f1: 0x9049, 0x10f2: 0x9049, 0x10f3: 0x9049, 0x10f4: 0x9049, 0x10f5: 0x9049, - 0x10f6: 0x9049, 0x10f7: 0x9049, 0x10f8: 0x9049, 0x10f9: 0x9049, 0x10fa: 0x9049, 0x10fb: 0x9049, - 0x10fc: 0x9049, 0x10fd: 0x9049, 0x10fe: 0x9049, 0x10ff: 0x9049, - // Block 0x44, offset 0x1100 - 0x1100: 0x9049, 0x1101: 0x9049, 0x1102: 0x9049, 0x1103: 0x9049, 0x1104: 0x9049, 0x1105: 0x9049, - 0x1106: 0x9049, 0x1107: 0x9049, 0x1108: 0x9049, 0x1109: 0x9049, 0x110a: 0x9049, 0x110b: 0x9049, - 0x110c: 0x9049, 0x110d: 0x9049, 0x110e: 0x9049, 0x110f: 0x9049, 0x1110: 0x9049, 0x1111: 0x9049, - 0x1112: 0x9049, 0x1113: 0x9049, 0x1114: 0x9049, 0x1115: 0x9049, 0x1116: 0x9049, 0x1117: 0x9049, - 0x1118: 0x9049, 0x1119: 0x9049, 0x111a: 0x9049, 0x111b: 0x9049, 0x111c: 0x9049, 0x111d: 0x9049, - 0x111e: 0x9049, 0x111f: 0x904a, 0x1120: 0x904b, 0x1121: 0xb04c, 0x1122: 0xb04d, 0x1123: 0xb04d, - 0x1124: 0xb04e, 0x1125: 0xb04f, 0x1126: 0xb050, 0x1127: 0xb051, 0x1128: 0xb052, 0x1129: 0xb053, - 0x112a: 0xb054, 0x112b: 0xb055, 0x112c: 0xb056, 0x112d: 0xb057, 0x112e: 0xb058, 0x112f: 0xb059, - 0x1130: 0xb05a, 0x1131: 0xb05b, 0x1132: 0xb05c, 0x1133: 0xb05d, 0x1134: 0xb05e, 0x1135: 0xb05f, - 0x1136: 0xb060, 0x1137: 0xb061, 0x1138: 0xb062, 0x1139: 0xb063, 0x113a: 0xb064, 0x113b: 0xb065, - 0x113c: 0xb052, 0x113d: 0xb066, 0x113e: 0xb067, 0x113f: 0xb055, - // Block 0x45, offset 0x1140 - 0x1140: 0xb068, 0x1141: 0xb069, 0x1142: 0xb06a, 0x1143: 0xb06b, 0x1144: 0xb05a, 0x1145: 0xb056, - 0x1146: 0xb06c, 0x1147: 0xb06d, 0x1148: 0xb06b, 0x1149: 0xb06e, 0x114a: 0xb06b, 0x114b: 0xb06f, - 0x114c: 0xb06f, 0x114d: 0xb070, 0x114e: 0xb070, 0x114f: 0xb071, 0x1150: 0xb056, 0x1151: 0xb072, - 0x1152: 0xb073, 0x1153: 0xb072, 0x1154: 0xb074, 0x1155: 0xb073, 0x1156: 0xb075, 0x1157: 0xb075, - 0x1158: 0xb076, 0x1159: 0xb076, 0x115a: 0xb077, 0x115b: 0xb077, 0x115c: 0xb073, 0x115d: 0xb078, - 0x115e: 0xb079, 0x115f: 0xb067, 0x1160: 0xb07a, 0x1161: 0xb07b, 0x1162: 0xb07b, 0x1163: 0xb07b, - 0x1164: 0xb07b, 0x1165: 0xb07b, 0x1166: 0xb07b, 0x1167: 0xb07b, 0x1168: 0xb07b, 0x1169: 0xb07b, - 0x116a: 0xb07b, 0x116b: 0xb07b, 0x116c: 0xb07b, 0x116d: 0xb07b, 0x116e: 0xb07b, 0x116f: 0xb07b, - 0x1170: 0xb07c, 0x1171: 0xb07c, 0x1172: 0xb07c, 0x1173: 0xb07c, 0x1174: 0xb07c, 0x1175: 0xb07c, - 0x1176: 0xb07c, 0x1177: 0xb07c, 0x1178: 0xb07c, 0x1179: 0xb07c, 0x117a: 0xb07c, 0x117b: 0xb07c, - 0x117c: 0xb07c, 0x117d: 0xb07c, 0x117e: 0xb07c, - // Block 0x46, offset 0x1180 - 0x1182: 0xb07d, 0x1183: 0xb07e, 0x1184: 0xb07f, 0x1185: 0xb080, - 0x1186: 0xb07f, 0x1187: 0xb07e, 0x118a: 0xb081, 0x118b: 0xb082, - 0x118c: 0xb083, 0x118d: 0xb07f, 0x118e: 0xb080, 0x118f: 0xb07f, - 0x1192: 0xb084, 0x1193: 0xb085, 0x1194: 0xb084, 0x1195: 0xb086, 0x1196: 0xb084, 0x1197: 0xb087, - 0x119a: 0xb088, 0x119b: 0xb089, 0x119c: 0xb08a, - 0x11a0: 0x908b, 0x11a1: 0x908b, 0x11a2: 0x908c, 0x11a3: 0x908d, - 0x11a4: 0x908b, 0x11a5: 0x908e, 0x11a6: 0x908f, 0x11a8: 0xb090, 0x11a9: 0xb091, - 0x11aa: 0xb092, 0x11ab: 0xb091, 0x11ac: 0xb093, 0x11ad: 0xb094, 0x11ae: 0xb095, - 0x11bd: 0x2000, - // Block 0x47, offset 0x11c0 - 0x11e0: 0x4000, - // Block 0x48, offset 0x1200 - 0x1200: 0x4000, 0x1201: 0x4000, 0x1202: 0x4000, 0x1203: 0x4000, 0x1204: 0x4000, 0x1205: 0x4000, - 0x1206: 0x4000, 0x1207: 0x4000, 0x1208: 0x4000, 0x1209: 0x4000, 0x120a: 0x4000, 0x120b: 0x4000, - 0x120c: 0x4000, 0x120d: 0x4000, 0x120e: 0x4000, 0x120f: 0x4000, 0x1210: 0x4000, 0x1211: 0x4000, - 0x1212: 0x4000, 0x1213: 0x4000, 0x1214: 0x4000, 0x1215: 0x4000, 0x1216: 0x4000, 0x1217: 0x4000, - 0x1218: 0x4000, 0x1219: 0x4000, 0x121a: 0x4000, 0x121b: 0x4000, 0x121c: 0x4000, 0x121d: 0x4000, - 0x121e: 0x4000, 0x121f: 0x4000, 0x1220: 0x4000, 0x1221: 0x4000, 0x1222: 0x4000, 0x1223: 0x4000, - 0x1224: 0x4000, 0x1225: 0x4000, 0x1226: 0x4000, 0x1227: 0x4000, 0x1228: 0x4000, 0x1229: 0x4000, - 0x122a: 0x4000, 0x122b: 0x4000, 0x122c: 0x4000, - // Block 0x49, offset 0x1240 - 0x1240: 0x4000, 0x1241: 0x4000, 0x1242: 0x4000, 0x1243: 0x4000, 0x1244: 0x4000, 0x1245: 0x4000, - 0x1246: 0x4000, 0x1247: 0x4000, 0x1248: 0x4000, 0x1249: 0x4000, 0x124a: 0x4000, 0x124b: 0x4000, - 0x124c: 0x4000, 0x124d: 0x4000, 0x124e: 0x4000, 0x124f: 0x4000, 0x1250: 0x4000, 0x1251: 0x4000, - 0x1252: 0x4000, 0x1253: 0x4000, 0x1254: 0x4000, 0x1255: 0x4000, 0x1256: 0x4000, 0x1257: 0x4000, - 0x1258: 0x4000, 0x1259: 0x4000, 0x125a: 0x4000, 0x125b: 0x4000, 0x125c: 0x4000, 0x125d: 0x4000, - 0x125e: 0x4000, 0x125f: 0x4000, 0x1260: 0x4000, 0x1261: 0x4000, 0x1262: 0x4000, 0x1263: 0x4000, - 0x1264: 0x4000, 0x1265: 0x4000, 0x1266: 0x4000, 0x1267: 0x4000, 0x1268: 0x4000, 0x1269: 0x4000, - 0x126a: 0x4000, 0x126b: 0x4000, 0x126c: 0x4000, 0x126d: 0x4000, 0x126e: 0x4000, 0x126f: 0x4000, - 0x1270: 0x4000, 0x1271: 0x4000, 0x1272: 0x4000, - // Block 0x4a, offset 0x1280 - 0x1280: 0x4000, 0x1281: 0x4000, - // Block 0x4b, offset 0x12c0 - 0x12c4: 0x4000, - // Block 0x4c, offset 0x1300 - 0x130f: 0x4000, - // Block 0x4d, offset 0x1340 - 0x1340: 0x2000, 0x1341: 0x2000, 0x1342: 0x2000, 0x1343: 0x2000, 0x1344: 0x2000, 0x1345: 0x2000, - 0x1346: 0x2000, 0x1347: 0x2000, 0x1348: 0x2000, 0x1349: 0x2000, 0x134a: 0x2000, - 0x1350: 0x2000, 0x1351: 0x2000, - 0x1352: 0x2000, 0x1353: 0x2000, 0x1354: 0x2000, 0x1355: 0x2000, 0x1356: 0x2000, 0x1357: 0x2000, - 0x1358: 0x2000, 0x1359: 0x2000, 0x135a: 0x2000, 0x135b: 0x2000, 0x135c: 0x2000, 0x135d: 0x2000, - 0x135e: 0x2000, 0x135f: 0x2000, 0x1360: 0x2000, 0x1361: 0x2000, 0x1362: 0x2000, 0x1363: 0x2000, - 0x1364: 0x2000, 0x1365: 0x2000, 0x1366: 0x2000, 0x1367: 0x2000, 0x1368: 0x2000, 0x1369: 0x2000, - 0x136a: 0x2000, 0x136b: 0x2000, 0x136c: 0x2000, 0x136d: 0x2000, - 0x1370: 0x2000, 0x1371: 0x2000, 0x1372: 0x2000, 0x1373: 0x2000, 0x1374: 0x2000, 0x1375: 0x2000, - 0x1376: 0x2000, 0x1377: 0x2000, 0x1378: 0x2000, 0x1379: 0x2000, 0x137a: 0x2000, 0x137b: 0x2000, - 0x137c: 0x2000, 0x137d: 0x2000, 0x137e: 0x2000, 0x137f: 0x2000, - // Block 0x4e, offset 0x1380 - 0x1380: 0x2000, 0x1381: 0x2000, 0x1382: 0x2000, 0x1383: 0x2000, 0x1384: 0x2000, 0x1385: 0x2000, - 0x1386: 0x2000, 0x1387: 0x2000, 0x1388: 0x2000, 0x1389: 0x2000, 0x138a: 0x2000, 0x138b: 0x2000, - 0x138c: 0x2000, 0x138d: 0x2000, 0x138e: 0x2000, 0x138f: 0x2000, 0x1390: 0x2000, 0x1391: 0x2000, - 0x1392: 0x2000, 0x1393: 0x2000, 0x1394: 0x2000, 0x1395: 0x2000, 0x1396: 0x2000, 0x1397: 0x2000, - 0x1398: 0x2000, 0x1399: 0x2000, 0x139a: 0x2000, 0x139b: 0x2000, 0x139c: 0x2000, 0x139d: 0x2000, - 0x139e: 0x2000, 0x139f: 0x2000, 0x13a0: 0x2000, 0x13a1: 0x2000, 0x13a2: 0x2000, 0x13a3: 0x2000, - 0x13a4: 0x2000, 0x13a5: 0x2000, 0x13a6: 0x2000, 0x13a7: 0x2000, 0x13a8: 0x2000, 0x13a9: 0x2000, - 0x13b0: 0x2000, 0x13b1: 0x2000, 0x13b2: 0x2000, 0x13b3: 0x2000, 0x13b4: 0x2000, 0x13b5: 0x2000, - 0x13b6: 0x2000, 0x13b7: 0x2000, 0x13b8: 0x2000, 0x13b9: 0x2000, 0x13ba: 0x2000, 0x13bb: 0x2000, - 0x13bc: 0x2000, 0x13bd: 0x2000, 0x13be: 0x2000, 0x13bf: 0x2000, - // Block 0x4f, offset 0x13c0 - 0x13c0: 0x2000, 0x13c1: 0x2000, 0x13c2: 0x2000, 0x13c3: 0x2000, 0x13c4: 0x2000, 0x13c5: 0x2000, - 0x13c6: 0x2000, 0x13c7: 0x2000, 0x13c8: 0x2000, 0x13c9: 0x2000, 0x13ca: 0x2000, 0x13cb: 0x2000, - 0x13cc: 0x2000, 0x13cd: 0x2000, 0x13ce: 0x4000, 0x13cf: 0x2000, 0x13d0: 0x2000, 0x13d1: 0x4000, - 0x13d2: 0x4000, 0x13d3: 0x4000, 0x13d4: 0x4000, 0x13d5: 0x4000, 0x13d6: 0x4000, 0x13d7: 0x4000, - 0x13d8: 0x4000, 0x13d9: 0x4000, 0x13da: 0x4000, 0x13db: 0x2000, 0x13dc: 0x2000, 0x13dd: 0x2000, - 0x13de: 0x2000, 0x13df: 0x2000, 0x13e0: 0x2000, 0x13e1: 0x2000, 0x13e2: 0x2000, 0x13e3: 0x2000, - 0x13e4: 0x2000, 0x13e5: 0x2000, 0x13e6: 0x2000, 0x13e7: 0x2000, 0x13e8: 0x2000, 0x13e9: 0x2000, - 0x13ea: 0x2000, 0x13eb: 0x2000, 0x13ec: 0x2000, - // Block 0x50, offset 0x1400 - 0x1400: 0x4000, 0x1401: 0x4000, 0x1402: 0x4000, - 0x1410: 0x4000, 0x1411: 0x4000, - 0x1412: 0x4000, 0x1413: 0x4000, 0x1414: 0x4000, 0x1415: 0x4000, 0x1416: 0x4000, 0x1417: 0x4000, - 0x1418: 0x4000, 0x1419: 0x4000, 0x141a: 0x4000, 0x141b: 0x4000, 0x141c: 0x4000, 0x141d: 0x4000, - 0x141e: 0x4000, 0x141f: 0x4000, 0x1420: 0x4000, 0x1421: 0x4000, 0x1422: 0x4000, 0x1423: 0x4000, - 0x1424: 0x4000, 0x1425: 0x4000, 0x1426: 0x4000, 0x1427: 0x4000, 0x1428: 0x4000, 0x1429: 0x4000, - 0x142a: 0x4000, 0x142b: 0x4000, 0x142c: 0x4000, 0x142d: 0x4000, 0x142e: 0x4000, 0x142f: 0x4000, - 0x1430: 0x4000, 0x1431: 0x4000, 0x1432: 0x4000, 0x1433: 0x4000, 0x1434: 0x4000, 0x1435: 0x4000, - 0x1436: 0x4000, 0x1437: 0x4000, 0x1438: 0x4000, 0x1439: 0x4000, 0x143a: 0x4000, 0x143b: 0x4000, - // Block 0x51, offset 0x1440 - 0x1440: 0x4000, 0x1441: 0x4000, 0x1442: 0x4000, 0x1443: 0x4000, 0x1444: 0x4000, 0x1445: 0x4000, - 0x1446: 0x4000, 0x1447: 0x4000, 0x1448: 0x4000, - 0x1450: 0x4000, 0x1451: 0x4000, - // Block 0x52, offset 0x1480 - 0x1480: 0x4000, 0x1481: 0x4000, 0x1482: 0x4000, 0x1483: 0x4000, 0x1484: 0x4000, 0x1485: 0x4000, - 0x1486: 0x4000, 0x1487: 0x4000, 0x1488: 0x4000, 0x1489: 0x4000, 0x148a: 0x4000, 0x148b: 0x4000, - 0x148c: 0x4000, 0x148d: 0x4000, 0x148e: 0x4000, 0x148f: 0x4000, 0x1490: 0x4000, 0x1491: 0x4000, - 0x1492: 0x4000, 0x1493: 0x4000, 0x1494: 0x4000, 0x1495: 0x4000, 0x1496: 0x4000, 0x1497: 0x4000, - 0x1498: 0x4000, 0x1499: 0x4000, 0x149a: 0x4000, 0x149b: 0x4000, 0x149c: 0x4000, 0x149d: 0x4000, - 0x149e: 0x4000, 0x149f: 0x4000, 0x14a0: 0x4000, - 0x14ad: 0x4000, 0x14ae: 0x4000, 0x14af: 0x4000, - 0x14b0: 0x4000, 0x14b1: 0x4000, 0x14b2: 0x4000, 0x14b3: 0x4000, 0x14b4: 0x4000, 0x14b5: 0x4000, - 0x14b7: 0x4000, 0x14b8: 0x4000, 0x14b9: 0x4000, 0x14ba: 0x4000, 0x14bb: 0x4000, - 0x14bc: 0x4000, 0x14bd: 0x4000, 0x14be: 0x4000, 0x14bf: 0x4000, - // Block 0x53, offset 0x14c0 - 0x14c0: 0x4000, 0x14c1: 0x4000, 0x14c2: 0x4000, 0x14c3: 0x4000, 0x14c4: 0x4000, 0x14c5: 0x4000, - 0x14c6: 0x4000, 0x14c7: 0x4000, 0x14c8: 0x4000, 0x14c9: 0x4000, 0x14ca: 0x4000, 0x14cb: 0x4000, - 0x14cc: 0x4000, 0x14cd: 0x4000, 0x14ce: 0x4000, 0x14cf: 0x4000, 0x14d0: 0x4000, 0x14d1: 0x4000, - 0x14d2: 0x4000, 0x14d3: 0x4000, 0x14d4: 0x4000, 0x14d5: 0x4000, 0x14d6: 0x4000, 0x14d7: 0x4000, - 0x14d8: 0x4000, 0x14d9: 0x4000, 0x14da: 0x4000, 0x14db: 0x4000, 0x14dc: 0x4000, 0x14dd: 0x4000, - 0x14de: 0x4000, 0x14df: 0x4000, 0x14e0: 0x4000, 0x14e1: 0x4000, 0x14e2: 0x4000, 0x14e3: 0x4000, - 0x14e4: 0x4000, 0x14e5: 0x4000, 0x14e6: 0x4000, 0x14e7: 0x4000, 0x14e8: 0x4000, 0x14e9: 0x4000, - 0x14ea: 0x4000, 0x14eb: 0x4000, 0x14ec: 0x4000, 0x14ed: 0x4000, 0x14ee: 0x4000, 0x14ef: 0x4000, - 0x14f0: 0x4000, 0x14f1: 0x4000, 0x14f2: 0x4000, 0x14f3: 0x4000, 0x14f4: 0x4000, 0x14f5: 0x4000, - 0x14f6: 0x4000, 0x14f7: 0x4000, 0x14f8: 0x4000, 0x14f9: 0x4000, 0x14fa: 0x4000, 0x14fb: 0x4000, - 0x14fc: 0x4000, 0x14fe: 0x4000, 0x14ff: 0x4000, - // Block 0x54, offset 0x1500 - 0x1500: 0x4000, 0x1501: 0x4000, 0x1502: 0x4000, 0x1503: 0x4000, 0x1504: 0x4000, 0x1505: 0x4000, - 0x1506: 0x4000, 0x1507: 0x4000, 0x1508: 0x4000, 0x1509: 0x4000, 0x150a: 0x4000, 0x150b: 0x4000, - 0x150c: 0x4000, 0x150d: 0x4000, 0x150e: 0x4000, 0x150f: 0x4000, 0x1510: 0x4000, 0x1511: 0x4000, - 0x1512: 0x4000, 0x1513: 0x4000, - 0x1520: 0x4000, 0x1521: 0x4000, 0x1522: 0x4000, 0x1523: 0x4000, - 0x1524: 0x4000, 0x1525: 0x4000, 0x1526: 0x4000, 0x1527: 0x4000, 0x1528: 0x4000, 0x1529: 0x4000, - 0x152a: 0x4000, 0x152b: 0x4000, 0x152c: 0x4000, 0x152d: 0x4000, 0x152e: 0x4000, 0x152f: 0x4000, - 0x1530: 0x4000, 0x1531: 0x4000, 0x1532: 0x4000, 0x1533: 0x4000, 0x1534: 0x4000, 0x1535: 0x4000, - 0x1536: 0x4000, 0x1537: 0x4000, 0x1538: 0x4000, 0x1539: 0x4000, 0x153a: 0x4000, 0x153b: 0x4000, - 0x153c: 0x4000, 0x153d: 0x4000, 0x153e: 0x4000, 0x153f: 0x4000, - // Block 0x55, offset 0x1540 - 0x1540: 0x4000, 0x1541: 0x4000, 0x1542: 0x4000, 0x1543: 0x4000, 0x1544: 0x4000, 0x1545: 0x4000, - 0x1546: 0x4000, 0x1547: 0x4000, 0x1548: 0x4000, 0x1549: 0x4000, 0x154a: 0x4000, - 0x154f: 0x4000, 0x1550: 0x4000, 0x1551: 0x4000, - 0x1552: 0x4000, 0x1553: 0x4000, - 0x1560: 0x4000, 0x1561: 0x4000, 0x1562: 0x4000, 0x1563: 0x4000, - 0x1564: 0x4000, 0x1565: 0x4000, 0x1566: 0x4000, 0x1567: 0x4000, 0x1568: 0x4000, 0x1569: 0x4000, - 0x156a: 0x4000, 0x156b: 0x4000, 0x156c: 0x4000, 0x156d: 0x4000, 0x156e: 0x4000, 0x156f: 0x4000, - 0x1570: 0x4000, 0x1574: 0x4000, - 0x1578: 0x4000, 0x1579: 0x4000, 0x157a: 0x4000, 0x157b: 0x4000, - 0x157c: 0x4000, 0x157d: 0x4000, 0x157e: 0x4000, 0x157f: 0x4000, - // Block 0x56, offset 0x1580 - 0x1580: 0x4000, 0x1582: 0x4000, 0x1583: 0x4000, 0x1584: 0x4000, 0x1585: 0x4000, - 0x1586: 0x4000, 0x1587: 0x4000, 0x1588: 0x4000, 0x1589: 0x4000, 0x158a: 0x4000, 0x158b: 0x4000, - 0x158c: 0x4000, 0x158d: 0x4000, 0x158e: 0x4000, 0x158f: 0x4000, 0x1590: 0x4000, 0x1591: 0x4000, - 0x1592: 0x4000, 0x1593: 0x4000, 0x1594: 0x4000, 0x1595: 0x4000, 0x1596: 0x4000, 0x1597: 0x4000, - 0x1598: 0x4000, 0x1599: 0x4000, 0x159a: 0x4000, 0x159b: 0x4000, 0x159c: 0x4000, 0x159d: 0x4000, - 0x159e: 0x4000, 0x159f: 0x4000, 0x15a0: 0x4000, 0x15a1: 0x4000, 0x15a2: 0x4000, 0x15a3: 0x4000, - 0x15a4: 0x4000, 0x15a5: 0x4000, 0x15a6: 0x4000, 0x15a7: 0x4000, 0x15a8: 0x4000, 0x15a9: 0x4000, - 0x15aa: 0x4000, 0x15ab: 0x4000, 0x15ac: 0x4000, 0x15ad: 0x4000, 0x15ae: 0x4000, 0x15af: 0x4000, - 0x15b0: 0x4000, 0x15b1: 0x4000, 0x15b2: 0x4000, 0x15b3: 0x4000, 0x15b4: 0x4000, 0x15b5: 0x4000, - 0x15b6: 0x4000, 0x15b7: 0x4000, 0x15b8: 0x4000, 0x15b9: 0x4000, 0x15ba: 0x4000, 0x15bb: 0x4000, - 0x15bc: 0x4000, 0x15bd: 0x4000, 0x15be: 0x4000, 0x15bf: 0x4000, - // Block 0x57, offset 0x15c0 - 0x15c0: 0x4000, 0x15c1: 0x4000, 0x15c2: 0x4000, 0x15c3: 0x4000, 0x15c4: 0x4000, 0x15c5: 0x4000, - 0x15c6: 0x4000, 0x15c7: 0x4000, 0x15c8: 0x4000, 0x15c9: 0x4000, 0x15ca: 0x4000, 0x15cb: 0x4000, - 0x15cc: 0x4000, 0x15cd: 0x4000, 0x15ce: 0x4000, 0x15cf: 0x4000, 0x15d0: 0x4000, 0x15d1: 0x4000, - 0x15d2: 0x4000, 0x15d3: 0x4000, 0x15d4: 0x4000, 0x15d5: 0x4000, 0x15d6: 0x4000, 0x15d7: 0x4000, - 0x15d8: 0x4000, 0x15d9: 0x4000, 0x15da: 0x4000, 0x15db: 0x4000, 0x15dc: 0x4000, 0x15dd: 0x4000, - 0x15de: 0x4000, 0x15df: 0x4000, 0x15e0: 0x4000, 0x15e1: 0x4000, 0x15e2: 0x4000, 0x15e3: 0x4000, - 0x15e4: 0x4000, 0x15e5: 0x4000, 0x15e6: 0x4000, 0x15e7: 0x4000, 0x15e8: 0x4000, 0x15e9: 0x4000, - 0x15ea: 0x4000, 0x15eb: 0x4000, 0x15ec: 0x4000, 0x15ed: 0x4000, 0x15ee: 0x4000, 0x15ef: 0x4000, - 0x15f0: 0x4000, 0x15f1: 0x4000, 0x15f2: 0x4000, 0x15f3: 0x4000, 0x15f4: 0x4000, 0x15f5: 0x4000, - 0x15f6: 0x4000, 0x15f7: 0x4000, 0x15f8: 0x4000, 0x15f9: 0x4000, 0x15fa: 0x4000, 0x15fb: 0x4000, - 0x15fc: 0x4000, 0x15ff: 0x4000, - // Block 0x58, offset 0x1600 - 0x1600: 0x4000, 0x1601: 0x4000, 0x1602: 0x4000, 0x1603: 0x4000, 0x1604: 0x4000, 0x1605: 0x4000, - 0x1606: 0x4000, 0x1607: 0x4000, 0x1608: 0x4000, 0x1609: 0x4000, 0x160a: 0x4000, 0x160b: 0x4000, - 0x160c: 0x4000, 0x160d: 0x4000, 0x160e: 0x4000, 0x160f: 0x4000, 0x1610: 0x4000, 0x1611: 0x4000, - 0x1612: 0x4000, 0x1613: 0x4000, 0x1614: 0x4000, 0x1615: 0x4000, 0x1616: 0x4000, 0x1617: 0x4000, - 0x1618: 0x4000, 0x1619: 0x4000, 0x161a: 0x4000, 0x161b: 0x4000, 0x161c: 0x4000, 0x161d: 0x4000, - 0x161e: 0x4000, 0x161f: 0x4000, 0x1620: 0x4000, 0x1621: 0x4000, 0x1622: 0x4000, 0x1623: 0x4000, - 0x1624: 0x4000, 0x1625: 0x4000, 0x1626: 0x4000, 0x1627: 0x4000, 0x1628: 0x4000, 0x1629: 0x4000, - 0x162a: 0x4000, 0x162b: 0x4000, 0x162c: 0x4000, 0x162d: 0x4000, 0x162e: 0x4000, 0x162f: 0x4000, - 0x1630: 0x4000, 0x1631: 0x4000, 0x1632: 0x4000, 0x1633: 0x4000, 0x1634: 0x4000, 0x1635: 0x4000, - 0x1636: 0x4000, 0x1637: 0x4000, 0x1638: 0x4000, 0x1639: 0x4000, 0x163a: 0x4000, 0x163b: 0x4000, - 0x163c: 0x4000, 0x163d: 0x4000, - // Block 0x59, offset 0x1640 - 0x164b: 0x4000, - 0x164c: 0x4000, 0x164d: 0x4000, 0x164e: 0x4000, 0x1650: 0x4000, 0x1651: 0x4000, - 0x1652: 0x4000, 0x1653: 0x4000, 0x1654: 0x4000, 0x1655: 0x4000, 0x1656: 0x4000, 0x1657: 0x4000, - 0x1658: 0x4000, 0x1659: 0x4000, 0x165a: 0x4000, 0x165b: 0x4000, 0x165c: 0x4000, 0x165d: 0x4000, - 0x165e: 0x4000, 0x165f: 0x4000, 0x1660: 0x4000, 0x1661: 0x4000, 0x1662: 0x4000, 0x1663: 0x4000, - 0x1664: 0x4000, 0x1665: 0x4000, 0x1666: 0x4000, 0x1667: 0x4000, - 0x167a: 0x4000, - // Block 0x5a, offset 0x1680 - 0x1695: 0x4000, 0x1696: 0x4000, - 0x16a4: 0x4000, - // Block 0x5b, offset 0x16c0 - 0x16fb: 0x4000, - 0x16fc: 0x4000, 0x16fd: 0x4000, 0x16fe: 0x4000, 0x16ff: 0x4000, - // Block 0x5c, offset 0x1700 - 0x1700: 0x4000, 0x1701: 0x4000, 0x1702: 0x4000, 0x1703: 0x4000, 0x1704: 0x4000, 0x1705: 0x4000, - 0x1706: 0x4000, 0x1707: 0x4000, 0x1708: 0x4000, 0x1709: 0x4000, 0x170a: 0x4000, 0x170b: 0x4000, - 0x170c: 0x4000, 0x170d: 0x4000, 0x170e: 0x4000, 0x170f: 0x4000, - // Block 0x5d, offset 0x1740 - 0x1740: 0x4000, 0x1741: 0x4000, 0x1742: 0x4000, 0x1743: 0x4000, 0x1744: 0x4000, 0x1745: 0x4000, - 0x174c: 0x4000, 0x1750: 0x4000, 0x1751: 0x4000, - 0x1752: 0x4000, - 0x176b: 0x4000, 0x176c: 0x4000, - 0x1774: 0x4000, 0x1775: 0x4000, - 0x1776: 0x4000, - // Block 0x5e, offset 0x1780 - 0x1790: 0x4000, 0x1791: 0x4000, - 0x1792: 0x4000, 0x1793: 0x4000, 0x1794: 0x4000, 0x1795: 0x4000, 0x1796: 0x4000, 0x1797: 0x4000, - 0x1798: 0x4000, 0x1799: 0x4000, 0x179a: 0x4000, 0x179b: 0x4000, 0x179c: 0x4000, 0x179d: 0x4000, - 0x179e: 0x4000, 0x17a0: 0x4000, 0x17a1: 0x4000, 0x17a2: 0x4000, 0x17a3: 0x4000, - 0x17a4: 0x4000, 0x17a5: 0x4000, 0x17a6: 0x4000, 0x17a7: 0x4000, - 0x17b0: 0x4000, 0x17b3: 0x4000, 0x17b4: 0x4000, 0x17b5: 0x4000, - 0x17b6: 0x4000, 0x17b7: 0x4000, 0x17b8: 0x4000, 0x17b9: 0x4000, 0x17ba: 0x4000, 0x17bb: 0x4000, - 0x17bc: 0x4000, 0x17bd: 0x4000, 0x17be: 0x4000, - // Block 0x5f, offset 0x17c0 - 0x17c0: 0x4000, 0x17c1: 0x4000, 0x17c2: 0x4000, 0x17c3: 0x4000, 0x17c4: 0x4000, 0x17c5: 0x4000, - 0x17c6: 0x4000, 0x17c7: 0x4000, 0x17c8: 0x4000, 0x17c9: 0x4000, 0x17ca: 0x4000, 0x17cb: 0x4000, - 0x17d0: 0x4000, 0x17d1: 0x4000, - 0x17d2: 0x4000, 0x17d3: 0x4000, 0x17d4: 0x4000, 0x17d5: 0x4000, 0x17d6: 0x4000, 0x17d7: 0x4000, - 0x17d8: 0x4000, 0x17d9: 0x4000, 0x17da: 0x4000, 0x17db: 0x4000, 0x17dc: 0x4000, 0x17dd: 0x4000, - 0x17de: 0x4000, - // Block 0x60, offset 0x1800 - 0x1800: 0x4000, 0x1801: 0x4000, 0x1802: 0x4000, 0x1803: 0x4000, 0x1804: 0x4000, 0x1805: 0x4000, - 0x1806: 0x4000, 0x1807: 0x4000, 0x1808: 0x4000, 0x1809: 0x4000, 0x180a: 0x4000, 0x180b: 0x4000, - 0x180c: 0x4000, 0x180d: 0x4000, 0x180e: 0x4000, 0x180f: 0x4000, 0x1810: 0x4000, 0x1811: 0x4000, - // Block 0x61, offset 0x1840 - 0x1840: 0x4000, - // Block 0x62, offset 0x1880 - 0x1880: 0x2000, 0x1881: 0x2000, 0x1882: 0x2000, 0x1883: 0x2000, 0x1884: 0x2000, 0x1885: 0x2000, - 0x1886: 0x2000, 0x1887: 0x2000, 0x1888: 0x2000, 0x1889: 0x2000, 0x188a: 0x2000, 0x188b: 0x2000, - 0x188c: 0x2000, 0x188d: 0x2000, 0x188e: 0x2000, 0x188f: 0x2000, 0x1890: 0x2000, 0x1891: 0x2000, - 0x1892: 0x2000, 0x1893: 0x2000, 0x1894: 0x2000, 0x1895: 0x2000, 0x1896: 0x2000, 0x1897: 0x2000, - 0x1898: 0x2000, 0x1899: 0x2000, 0x189a: 0x2000, 0x189b: 0x2000, 0x189c: 0x2000, 0x189d: 0x2000, - 0x189e: 0x2000, 0x189f: 0x2000, 0x18a0: 0x2000, 0x18a1: 0x2000, 0x18a2: 0x2000, 0x18a3: 0x2000, - 0x18a4: 0x2000, 0x18a5: 0x2000, 0x18a6: 0x2000, 0x18a7: 0x2000, 0x18a8: 0x2000, 0x18a9: 0x2000, - 0x18aa: 0x2000, 0x18ab: 0x2000, 0x18ac: 0x2000, 0x18ad: 0x2000, 0x18ae: 0x2000, 0x18af: 0x2000, - 0x18b0: 0x2000, 0x18b1: 0x2000, 0x18b2: 0x2000, 0x18b3: 0x2000, 0x18b4: 0x2000, 0x18b5: 0x2000, - 0x18b6: 0x2000, 0x18b7: 0x2000, 0x18b8: 0x2000, 0x18b9: 0x2000, 0x18ba: 0x2000, 0x18bb: 0x2000, - 0x18bc: 0x2000, 0x18bd: 0x2000, -} - -// widthIndex: 22 blocks, 1408 entries, 1408 bytes -// Block 0 is the zero block. -var widthIndex = [1408]uint8{ - // Block 0x0, offset 0x0 - // Block 0x1, offset 0x40 - // Block 0x2, offset 0x80 - // Block 0x3, offset 0xc0 - 0xc2: 0x01, 0xc3: 0x02, 0xc4: 0x03, 0xc5: 0x04, 0xc7: 0x05, - 0xc9: 0x06, 0xcb: 0x07, 0xcc: 0x08, 0xcd: 0x09, 0xce: 0x0a, 0xcf: 0x0b, - 0xd0: 0x0c, 0xd1: 0x0d, - 0xe1: 0x02, 0xe2: 0x03, 0xe3: 0x04, 0xe4: 0x05, 0xe5: 0x06, 0xe6: 0x06, 0xe7: 0x06, - 0xe8: 0x06, 0xe9: 0x06, 0xea: 0x07, 0xeb: 0x06, 0xec: 0x06, 0xed: 0x08, 0xee: 0x09, 0xef: 0x0a, - 0xf0: 0x0f, 0xf3: 0x12, 0xf4: 0x13, - // Block 0x4, offset 0x100 - 0x104: 0x0e, 0x105: 0x0f, - // Block 0x5, offset 0x140 - 0x140: 0x10, 0x141: 0x11, 0x142: 0x12, 0x144: 0x13, 0x145: 0x14, 0x146: 0x15, 0x147: 0x16, - 0x148: 0x17, 0x149: 0x18, 0x14a: 0x19, 0x14c: 0x1a, 0x14f: 0x1b, - 0x151: 0x1c, 0x152: 0x08, 0x153: 0x1d, 0x154: 0x1e, 0x155: 0x1f, 0x156: 0x20, 0x157: 0x21, - 0x158: 0x22, 0x159: 0x23, 0x15a: 0x24, 0x15b: 0x25, 0x15c: 0x26, 0x15d: 0x27, 0x15e: 0x28, 0x15f: 0x29, - 0x166: 0x2a, - 0x16c: 0x2b, 0x16d: 0x2c, - 0x17a: 0x2d, 0x17b: 0x2e, 0x17c: 0x0e, 0x17d: 0x0e, 0x17e: 0x0e, 0x17f: 0x2f, - // Block 0x6, offset 0x180 - 0x180: 0x30, 0x181: 0x31, 0x182: 0x32, 0x183: 0x33, 0x184: 0x34, 0x185: 0x35, 0x186: 0x36, 0x187: 0x37, - 0x188: 0x38, 0x189: 0x39, 0x18a: 0x0e, 0x18b: 0x3a, 0x18c: 0x0e, 0x18d: 0x0e, 0x18e: 0x0e, 0x18f: 0x0e, - 0x190: 0x0e, 0x191: 0x0e, 0x192: 0x0e, 0x193: 0x0e, 0x194: 0x0e, 0x195: 0x0e, 0x196: 0x0e, 0x197: 0x0e, - 0x198: 0x0e, 0x199: 0x0e, 0x19a: 0x0e, 0x19b: 0x0e, 0x19c: 0x0e, 0x19d: 0x0e, 0x19e: 0x0e, 0x19f: 0x0e, - 0x1a0: 0x0e, 0x1a1: 0x0e, 0x1a2: 0x0e, 0x1a3: 0x0e, 0x1a4: 0x0e, 0x1a5: 0x0e, 0x1a6: 0x0e, 0x1a7: 0x0e, - 0x1a8: 0x0e, 0x1a9: 0x0e, 0x1aa: 0x0e, 0x1ab: 0x0e, 0x1ac: 0x0e, 0x1ad: 0x0e, 0x1ae: 0x0e, 0x1af: 0x0e, - 0x1b0: 0x0e, 0x1b1: 0x0e, 0x1b2: 0x0e, 0x1b3: 0x0e, 0x1b4: 0x0e, 0x1b5: 0x0e, 0x1b6: 0x0e, 0x1b7: 0x0e, - 0x1b8: 0x0e, 0x1b9: 0x0e, 0x1ba: 0x0e, 0x1bb: 0x0e, 0x1bc: 0x0e, 0x1bd: 0x0e, 0x1be: 0x0e, 0x1bf: 0x0e, - // Block 0x7, offset 0x1c0 - 0x1c0: 0x0e, 0x1c1: 0x0e, 0x1c2: 0x0e, 0x1c3: 0x0e, 0x1c4: 0x0e, 0x1c5: 0x0e, 0x1c6: 0x0e, 0x1c7: 0x0e, - 0x1c8: 0x0e, 0x1c9: 0x0e, 0x1ca: 0x0e, 0x1cb: 0x0e, 0x1cc: 0x0e, 0x1cd: 0x0e, 0x1ce: 0x0e, 0x1cf: 0x0e, - 0x1d0: 0x0e, 0x1d1: 0x0e, 0x1d2: 0x0e, 0x1d3: 0x0e, 0x1d4: 0x0e, 0x1d5: 0x0e, 0x1d6: 0x0e, 0x1d7: 0x0e, - 0x1d8: 0x0e, 0x1d9: 0x0e, 0x1da: 0x0e, 0x1db: 0x0e, 0x1dc: 0x0e, 0x1dd: 0x0e, 0x1de: 0x0e, 0x1df: 0x0e, - 0x1e0: 0x0e, 0x1e1: 0x0e, 0x1e2: 0x0e, 0x1e3: 0x0e, 0x1e4: 0x0e, 0x1e5: 0x0e, 0x1e6: 0x0e, 0x1e7: 0x0e, - 0x1e8: 0x0e, 0x1e9: 0x0e, 0x1ea: 0x0e, 0x1eb: 0x0e, 0x1ec: 0x0e, 0x1ed: 0x0e, 0x1ee: 0x0e, 0x1ef: 0x0e, - 0x1f0: 0x0e, 0x1f1: 0x0e, 0x1f2: 0x0e, 0x1f3: 0x0e, 0x1f4: 0x0e, 0x1f5: 0x0e, 0x1f6: 0x0e, - 0x1f8: 0x0e, 0x1f9: 0x0e, 0x1fa: 0x0e, 0x1fb: 0x0e, 0x1fc: 0x0e, 0x1fd: 0x0e, 0x1fe: 0x0e, 0x1ff: 0x0e, - // Block 0x8, offset 0x200 - 0x200: 0x0e, 0x201: 0x0e, 0x202: 0x0e, 0x203: 0x0e, 0x204: 0x0e, 0x205: 0x0e, 0x206: 0x0e, 0x207: 0x0e, - 0x208: 0x0e, 0x209: 0x0e, 0x20a: 0x0e, 0x20b: 0x0e, 0x20c: 0x0e, 0x20d: 0x0e, 0x20e: 0x0e, 0x20f: 0x0e, - 0x210: 0x0e, 0x211: 0x0e, 0x212: 0x0e, 0x213: 0x0e, 0x214: 0x0e, 0x215: 0x0e, 0x216: 0x0e, 0x217: 0x0e, - 0x218: 0x0e, 0x219: 0x0e, 0x21a: 0x0e, 0x21b: 0x0e, 0x21c: 0x0e, 0x21d: 0x0e, 0x21e: 0x0e, 0x21f: 0x0e, - 0x220: 0x0e, 0x221: 0x0e, 0x222: 0x0e, 0x223: 0x0e, 0x224: 0x0e, 0x225: 0x0e, 0x226: 0x0e, 0x227: 0x0e, - 0x228: 0x0e, 0x229: 0x0e, 0x22a: 0x0e, 0x22b: 0x0e, 0x22c: 0x0e, 0x22d: 0x0e, 0x22e: 0x0e, 0x22f: 0x0e, - 0x230: 0x0e, 0x231: 0x0e, 0x232: 0x0e, 0x233: 0x0e, 0x234: 0x0e, 0x235: 0x0e, 0x236: 0x0e, 0x237: 0x0e, - 0x238: 0x0e, 0x239: 0x0e, 0x23a: 0x0e, 0x23b: 0x0e, 0x23c: 0x0e, 0x23d: 0x0e, 0x23e: 0x0e, 0x23f: 0x0e, - // Block 0x9, offset 0x240 - 0x240: 0x0e, 0x241: 0x0e, 0x242: 0x0e, 0x243: 0x0e, 0x244: 0x0e, 0x245: 0x0e, 0x246: 0x0e, 0x247: 0x0e, - 0x248: 0x0e, 0x249: 0x0e, 0x24a: 0x0e, 0x24b: 0x0e, 0x24c: 0x0e, 0x24d: 0x0e, 0x24e: 0x0e, 0x24f: 0x0e, - 0x250: 0x0e, 0x251: 0x0e, 0x252: 0x3b, 0x253: 0x3c, - 0x265: 0x3d, - 0x270: 0x0e, 0x271: 0x0e, 0x272: 0x0e, 0x273: 0x0e, 0x274: 0x0e, 0x275: 0x0e, 0x276: 0x0e, 0x277: 0x0e, - 0x278: 0x0e, 0x279: 0x0e, 0x27a: 0x0e, 0x27b: 0x0e, 0x27c: 0x0e, 0x27d: 0x0e, 0x27e: 0x0e, 0x27f: 0x0e, - // Block 0xa, offset 0x280 - 0x280: 0x0e, 0x281: 0x0e, 0x282: 0x0e, 0x283: 0x0e, 0x284: 0x0e, 0x285: 0x0e, 0x286: 0x0e, 0x287: 0x0e, - 0x288: 0x0e, 0x289: 0x0e, 0x28a: 0x0e, 0x28b: 0x0e, 0x28c: 0x0e, 0x28d: 0x0e, 0x28e: 0x0e, 0x28f: 0x0e, - 0x290: 0x0e, 0x291: 0x0e, 0x292: 0x0e, 0x293: 0x0e, 0x294: 0x0e, 0x295: 0x0e, 0x296: 0x0e, 0x297: 0x0e, - 0x298: 0x0e, 0x299: 0x0e, 0x29a: 0x0e, 0x29b: 0x0e, 0x29c: 0x0e, 0x29d: 0x0e, 0x29e: 0x3e, - // Block 0xb, offset 0x2c0 - 0x2c0: 0x08, 0x2c1: 0x08, 0x2c2: 0x08, 0x2c3: 0x08, 0x2c4: 0x08, 0x2c5: 0x08, 0x2c6: 0x08, 0x2c7: 0x08, - 0x2c8: 0x08, 0x2c9: 0x08, 0x2ca: 0x08, 0x2cb: 0x08, 0x2cc: 0x08, 0x2cd: 0x08, 0x2ce: 0x08, 0x2cf: 0x08, - 0x2d0: 0x08, 0x2d1: 0x08, 0x2d2: 0x08, 0x2d3: 0x08, 0x2d4: 0x08, 0x2d5: 0x08, 0x2d6: 0x08, 0x2d7: 0x08, - 0x2d8: 0x08, 0x2d9: 0x08, 0x2da: 0x08, 0x2db: 0x08, 0x2dc: 0x08, 0x2dd: 0x08, 0x2de: 0x08, 0x2df: 0x08, - 0x2e0: 0x08, 0x2e1: 0x08, 0x2e2: 0x08, 0x2e3: 0x08, 0x2e4: 0x08, 0x2e5: 0x08, 0x2e6: 0x08, 0x2e7: 0x08, - 0x2e8: 0x08, 0x2e9: 0x08, 0x2ea: 0x08, 0x2eb: 0x08, 0x2ec: 0x08, 0x2ed: 0x08, 0x2ee: 0x08, 0x2ef: 0x08, - 0x2f0: 0x08, 0x2f1: 0x08, 0x2f2: 0x08, 0x2f3: 0x08, 0x2f4: 0x08, 0x2f5: 0x08, 0x2f6: 0x08, 0x2f7: 0x08, - 0x2f8: 0x08, 0x2f9: 0x08, 0x2fa: 0x08, 0x2fb: 0x08, 0x2fc: 0x08, 0x2fd: 0x08, 0x2fe: 0x08, 0x2ff: 0x08, - // Block 0xc, offset 0x300 - 0x300: 0x08, 0x301: 0x08, 0x302: 0x08, 0x303: 0x08, 0x304: 0x08, 0x305: 0x08, 0x306: 0x08, 0x307: 0x08, - 0x308: 0x08, 0x309: 0x08, 0x30a: 0x08, 0x30b: 0x08, 0x30c: 0x08, 0x30d: 0x08, 0x30e: 0x08, 0x30f: 0x08, - 0x310: 0x08, 0x311: 0x08, 0x312: 0x08, 0x313: 0x08, 0x314: 0x08, 0x315: 0x08, 0x316: 0x08, 0x317: 0x08, - 0x318: 0x08, 0x319: 0x08, 0x31a: 0x08, 0x31b: 0x08, 0x31c: 0x08, 0x31d: 0x08, 0x31e: 0x08, 0x31f: 0x08, - 0x320: 0x08, 0x321: 0x08, 0x322: 0x08, 0x323: 0x08, 0x324: 0x0e, 0x325: 0x0e, 0x326: 0x0e, 0x327: 0x0e, - 0x328: 0x0e, 0x329: 0x0e, 0x32a: 0x0e, 0x32b: 0x0e, - 0x338: 0x3f, 0x339: 0x40, 0x33c: 0x41, 0x33d: 0x42, 0x33e: 0x43, 0x33f: 0x44, - // Block 0xd, offset 0x340 - 0x37f: 0x45, - // Block 0xe, offset 0x380 - 0x380: 0x0e, 0x381: 0x0e, 0x382: 0x0e, 0x383: 0x0e, 0x384: 0x0e, 0x385: 0x0e, 0x386: 0x0e, 0x387: 0x0e, - 0x388: 0x0e, 0x389: 0x0e, 0x38a: 0x0e, 0x38b: 0x0e, 0x38c: 0x0e, 0x38d: 0x0e, 0x38e: 0x0e, 0x38f: 0x0e, - 0x390: 0x0e, 0x391: 0x0e, 0x392: 0x0e, 0x393: 0x0e, 0x394: 0x0e, 0x395: 0x0e, 0x396: 0x0e, 0x397: 0x0e, - 0x398: 0x0e, 0x399: 0x0e, 0x39a: 0x0e, 0x39b: 0x0e, 0x39c: 0x0e, 0x39d: 0x0e, 0x39e: 0x0e, 0x39f: 0x46, - 0x3a0: 0x0e, 0x3a1: 0x0e, 0x3a2: 0x0e, 0x3a3: 0x0e, 0x3a4: 0x0e, 0x3a5: 0x0e, 0x3a6: 0x0e, 0x3a7: 0x0e, - 0x3a8: 0x0e, 0x3a9: 0x0e, 0x3aa: 0x0e, 0x3ab: 0x47, - // Block 0xf, offset 0x3c0 - 0x3c0: 0x48, - // Block 0x10, offset 0x400 - 0x400: 0x49, 0x403: 0x4a, 0x404: 0x4b, 0x405: 0x4c, 0x406: 0x4d, - 0x408: 0x4e, 0x409: 0x4f, 0x40c: 0x50, 0x40d: 0x51, 0x40e: 0x52, 0x40f: 0x53, - 0x410: 0x3a, 0x411: 0x54, 0x412: 0x0e, 0x413: 0x55, 0x414: 0x56, 0x415: 0x57, 0x416: 0x58, 0x417: 0x59, - 0x418: 0x0e, 0x419: 0x5a, 0x41a: 0x0e, 0x41b: 0x5b, - 0x424: 0x5c, 0x425: 0x5d, 0x426: 0x5e, 0x427: 0x5f, - // Block 0x11, offset 0x440 - 0x456: 0x0b, 0x457: 0x06, - 0x458: 0x0c, 0x45b: 0x0d, 0x45f: 0x0e, - 0x460: 0x06, 0x461: 0x06, 0x462: 0x06, 0x463: 0x06, 0x464: 0x06, 0x465: 0x06, 0x466: 0x06, 0x467: 0x06, - 0x468: 0x06, 0x469: 0x06, 0x46a: 0x06, 0x46b: 0x06, 0x46c: 0x06, 0x46d: 0x06, 0x46e: 0x06, 0x46f: 0x06, - 0x470: 0x06, 0x471: 0x06, 0x472: 0x06, 0x473: 0x06, 0x474: 0x06, 0x475: 0x06, 0x476: 0x06, 0x477: 0x06, - 0x478: 0x06, 0x479: 0x06, 0x47a: 0x06, 0x47b: 0x06, 0x47c: 0x06, 0x47d: 0x06, 0x47e: 0x06, 0x47f: 0x06, - // Block 0x12, offset 0x480 - 0x484: 0x08, 0x485: 0x08, 0x486: 0x08, 0x487: 0x09, - // Block 0x13, offset 0x4c0 - 0x4c0: 0x08, 0x4c1: 0x08, 0x4c2: 0x08, 0x4c3: 0x08, 0x4c4: 0x08, 0x4c5: 0x08, 0x4c6: 0x08, 0x4c7: 0x08, - 0x4c8: 0x08, 0x4c9: 0x08, 0x4ca: 0x08, 0x4cb: 0x08, 0x4cc: 0x08, 0x4cd: 0x08, 0x4ce: 0x08, 0x4cf: 0x08, - 0x4d0: 0x08, 0x4d1: 0x08, 0x4d2: 0x08, 0x4d3: 0x08, 0x4d4: 0x08, 0x4d5: 0x08, 0x4d6: 0x08, 0x4d7: 0x08, - 0x4d8: 0x08, 0x4d9: 0x08, 0x4da: 0x08, 0x4db: 0x08, 0x4dc: 0x08, 0x4dd: 0x08, 0x4de: 0x08, 0x4df: 0x08, - 0x4e0: 0x08, 0x4e1: 0x08, 0x4e2: 0x08, 0x4e3: 0x08, 0x4e4: 0x08, 0x4e5: 0x08, 0x4e6: 0x08, 0x4e7: 0x08, - 0x4e8: 0x08, 0x4e9: 0x08, 0x4ea: 0x08, 0x4eb: 0x08, 0x4ec: 0x08, 0x4ed: 0x08, 0x4ee: 0x08, 0x4ef: 0x08, - 0x4f0: 0x08, 0x4f1: 0x08, 0x4f2: 0x08, 0x4f3: 0x08, 0x4f4: 0x08, 0x4f5: 0x08, 0x4f6: 0x08, 0x4f7: 0x08, - 0x4f8: 0x08, 0x4f9: 0x08, 0x4fa: 0x08, 0x4fb: 0x08, 0x4fc: 0x08, 0x4fd: 0x08, 0x4fe: 0x08, 0x4ff: 0x60, - // Block 0x14, offset 0x500 - 0x520: 0x10, - 0x530: 0x09, 0x531: 0x09, 0x532: 0x09, 0x533: 0x09, 0x534: 0x09, 0x535: 0x09, 0x536: 0x09, 0x537: 0x09, - 0x538: 0x09, 0x539: 0x09, 0x53a: 0x09, 0x53b: 0x09, 0x53c: 0x09, 0x53d: 0x09, 0x53e: 0x09, 0x53f: 0x11, - // Block 0x15, offset 0x540 - 0x540: 0x09, 0x541: 0x09, 0x542: 0x09, 0x543: 0x09, 0x544: 0x09, 0x545: 0x09, 0x546: 0x09, 0x547: 0x09, - 0x548: 0x09, 0x549: 0x09, 0x54a: 0x09, 0x54b: 0x09, 0x54c: 0x09, 0x54d: 0x09, 0x54e: 0x09, 0x54f: 0x11, -} - -// inverseData contains 4-byte entries of the following format: -// -// <0 padding> -// -// The last byte of the UTF-8-encoded rune is xor-ed with the last byte of the -// UTF-8 encoding of the original rune. Mappings often have the following -// pattern: -// -// Ａ -> A (U+FF21 -> U+0041) -// Ｂ -> B (U+FF22 -> U+0042) -// ... -// -// By xor-ing the last byte the same entry can be shared by many mappings. This -// reduces the total number of distinct entries by about two thirds. -// The resulting entry for the aforementioned mappings is -// -// { 0x01, 0xE0, 0x00, 0x00 } -// -// Using this entry to map U+FF21 (UTF-8 [EF BC A1]), we get -// -// E0 ^ A1 = 41. -// -// Similarly, for U+FF22 (UTF-8 [EF BC A2]), we get -// -// E0 ^ A2 = 42. -// -// Note that because of the xor-ing, the byte sequence stored in the entry is -// not valid UTF-8. -var inverseData = [150][4]byte{ - {0x00, 0x00, 0x00, 0x00}, - {0x03, 0xe3, 0x80, 0xa0}, - {0x03, 0xef, 0xbc, 0xa0}, - {0x03, 0xef, 0xbc, 0xe0}, - {0x03, 0xef, 0xbd, 0xe0}, - {0x03, 0xef, 0xbf, 0x02}, - {0x03, 0xef, 0xbf, 0x00}, - {0x03, 0xef, 0xbf, 0x0e}, - {0x03, 0xef, 0xbf, 0x0c}, - {0x03, 0xef, 0xbf, 0x0f}, - {0x03, 0xef, 0xbf, 0x39}, - {0x03, 0xef, 0xbf, 0x3b}, - {0x03, 0xef, 0xbf, 0x3f}, - {0x03, 0xef, 0xbf, 0x2a}, - {0x03, 0xef, 0xbf, 0x0d}, - {0x03, 0xef, 0xbf, 0x25}, - {0x03, 0xef, 0xbd, 0x1a}, - {0x03, 0xef, 0xbd, 0x26}, - {0x01, 0xa0, 0x00, 0x00}, - {0x03, 0xef, 0xbd, 0x25}, - {0x03, 0xef, 0xbd, 0x23}, - {0x03, 0xef, 0xbd, 0x2e}, - {0x03, 0xef, 0xbe, 0x07}, - {0x03, 0xef, 0xbe, 0x05}, - {0x03, 0xef, 0xbd, 0x06}, - {0x03, 0xef, 0xbd, 0x13}, - {0x03, 0xef, 0xbd, 0x0b}, - {0x03, 0xef, 0xbd, 0x16}, - {0x03, 0xef, 0xbd, 0x0c}, - {0x03, 0xef, 0xbd, 0x15}, - {0x03, 0xef, 0xbd, 0x0d}, - {0x03, 0xef, 0xbd, 0x1c}, - {0x03, 0xef, 0xbd, 0x02}, - {0x03, 0xef, 0xbd, 0x1f}, - {0x03, 0xef, 0xbd, 0x1d}, - {0x03, 0xef, 0xbd, 0x17}, - {0x03, 0xef, 0xbd, 0x08}, - {0x03, 0xef, 0xbd, 0x09}, - {0x03, 0xef, 0xbd, 0x0e}, - {0x03, 0xef, 0xbd, 0x04}, - {0x03, 0xef, 0xbd, 0x05}, - {0x03, 0xef, 0xbe, 0x3f}, - {0x03, 0xef, 0xbe, 0x00}, - {0x03, 0xef, 0xbd, 0x2c}, - {0x03, 0xef, 0xbe, 0x06}, - {0x03, 0xef, 0xbe, 0x0c}, - {0x03, 0xef, 0xbe, 0x0f}, - {0x03, 0xef, 0xbe, 0x0d}, - {0x03, 0xef, 0xbe, 0x0b}, - {0x03, 0xef, 0xbe, 0x19}, - {0x03, 0xef, 0xbe, 0x15}, - {0x03, 0xef, 0xbe, 0x11}, - {0x03, 0xef, 0xbe, 0x31}, - {0x03, 0xef, 0xbe, 0x33}, - {0x03, 0xef, 0xbd, 0x0f}, - {0x03, 0xef, 0xbe, 0x30}, - {0x03, 0xef, 0xbe, 0x3e}, - {0x03, 0xef, 0xbe, 0x32}, - {0x03, 0xef, 0xbe, 0x36}, - {0x03, 0xef, 0xbd, 0x14}, - {0x03, 0xef, 0xbe, 0x2e}, - {0x03, 0xef, 0xbd, 0x1e}, - {0x03, 0xef, 0xbe, 0x10}, - {0x03, 0xef, 0xbf, 0x13}, - {0x03, 0xef, 0xbf, 0x15}, - {0x03, 0xef, 0xbf, 0x17}, - {0x03, 0xef, 0xbf, 0x1f}, - {0x03, 0xef, 0xbf, 0x1d}, - {0x03, 0xef, 0xbf, 0x1b}, - {0x03, 0xef, 0xbf, 0x09}, - {0x03, 0xef, 0xbf, 0x0b}, - {0x03, 0xef, 0xbf, 0x37}, - {0x03, 0xef, 0xbe, 0x04}, - {0x01, 0xe0, 0x00, 0x00}, - {0x03, 0xe2, 0xa6, 0x1a}, - {0x03, 0xe2, 0xa6, 0x26}, - {0x03, 0xe3, 0x80, 0x23}, - {0x03, 0xe3, 0x80, 0x2e}, - {0x03, 0xe3, 0x80, 0x25}, - {0x03, 0xe3, 0x83, 0x1e}, - {0x03, 0xe3, 0x83, 0x14}, - {0x03, 0xe3, 0x82, 0x06}, - {0x03, 0xe3, 0x82, 0x0b}, - {0x03, 0xe3, 0x82, 0x0c}, - {0x03, 0xe3, 0x82, 0x0d}, - {0x03, 0xe3, 0x82, 0x02}, - {0x03, 0xe3, 0x83, 0x0f}, - {0x03, 0xe3, 0x83, 0x08}, - {0x03, 0xe3, 0x83, 0x09}, - {0x03, 0xe3, 0x83, 0x2c}, - {0x03, 0xe3, 0x83, 0x0c}, - {0x03, 0xe3, 0x82, 0x13}, - {0x03, 0xe3, 0x82, 0x16}, - {0x03, 0xe3, 0x82, 0x15}, - {0x03, 0xe3, 0x82, 0x1c}, - {0x03, 0xe3, 0x82, 0x1f}, - {0x03, 0xe3, 0x82, 0x1d}, - {0x03, 0xe3, 0x82, 0x1a}, - {0x03, 0xe3, 0x82, 0x17}, - {0x03, 0xe3, 0x82, 0x08}, - {0x03, 0xe3, 0x82, 0x09}, - {0x03, 0xe3, 0x82, 0x0e}, - {0x03, 0xe3, 0x82, 0x04}, - {0x03, 0xe3, 0x82, 0x05}, - {0x03, 0xe3, 0x82, 0x3f}, - {0x03, 0xe3, 0x83, 0x00}, - {0x03, 0xe3, 0x83, 0x06}, - {0x03, 0xe3, 0x83, 0x05}, - {0x03, 0xe3, 0x83, 0x0d}, - {0x03, 0xe3, 0x83, 0x0b}, - {0x03, 0xe3, 0x83, 0x07}, - {0x03, 0xe3, 0x83, 0x19}, - {0x03, 0xe3, 0x83, 0x15}, - {0x03, 0xe3, 0x83, 0x11}, - {0x03, 0xe3, 0x83, 0x31}, - {0x03, 0xe3, 0x83, 0x33}, - {0x03, 0xe3, 0x83, 0x30}, - {0x03, 0xe3, 0x83, 0x3e}, - {0x03, 0xe3, 0x83, 0x32}, - {0x03, 0xe3, 0x83, 0x36}, - {0x03, 0xe3, 0x83, 0x2e}, - {0x03, 0xe3, 0x82, 0x07}, - {0x03, 0xe3, 0x85, 0x04}, - {0x03, 0xe3, 0x84, 0x10}, - {0x03, 0xe3, 0x85, 0x30}, - {0x03, 0xe3, 0x85, 0x0d}, - {0x03, 0xe3, 0x85, 0x13}, - {0x03, 0xe3, 0x85, 0x15}, - {0x03, 0xe3, 0x85, 0x17}, - {0x03, 0xe3, 0x85, 0x1f}, - {0x03, 0xe3, 0x85, 0x1d}, - {0x03, 0xe3, 0x85, 0x1b}, - {0x03, 0xe3, 0x85, 0x09}, - {0x03, 0xe3, 0x85, 0x0f}, - {0x03, 0xe3, 0x85, 0x0b}, - {0x03, 0xe3, 0x85, 0x37}, - {0x03, 0xe3, 0x85, 0x3b}, - {0x03, 0xe3, 0x85, 0x39}, - {0x03, 0xe3, 0x85, 0x3f}, - {0x02, 0xc2, 0x02, 0x00}, - {0x02, 0xc2, 0x0e, 0x00}, - {0x02, 0xc2, 0x0c, 0x00}, - {0x02, 0xc2, 0x00, 0x00}, - {0x03, 0xe2, 0x82, 0x0f}, - {0x03, 0xe2, 0x94, 0x2a}, - {0x03, 0xe2, 0x86, 0x39}, - {0x03, 0xe2, 0x86, 0x3b}, - {0x03, 0xe2, 0x86, 0x3f}, - {0x03, 0xe2, 0x96, 0x0d}, - {0x03, 0xe2, 0x97, 0x25}, -} - -// Total table size 14680 bytes (14KiB) diff --git a/vendor/golang.org/x/text/width/transform.go b/vendor/golang.org/x/text/width/transform.go deleted file mode 100644 index 0049f700a2..0000000000 --- a/vendor/golang.org/x/text/width/transform.go +++ /dev/null @@ -1,239 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package width - -import ( - "unicode/utf8" - - "golang.org/x/text/transform" -) - -type foldTransform struct { - transform.NopResetter -} - -func (foldTransform) Span(src []byte, atEOF bool) (n int, err error) { - for n < len(src) { - if src[n] < utf8.RuneSelf { - // ASCII fast path. - for n++; n < len(src) && src[n] < utf8.RuneSelf; n++ { - } - continue - } - v, size := trie.lookup(src[n:]) - if size == 0 { // incomplete UTF-8 encoding - if !atEOF { - err = transform.ErrShortSrc - } else { - n = len(src) - } - break - } - if elem(v)&tagNeedsFold != 0 { - err = transform.ErrEndOfSpan - break - } - n += size - } - return n, err -} - -func (foldTransform) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { - for nSrc < len(src) { - if src[nSrc] < utf8.RuneSelf { - // ASCII fast path. - start, end := nSrc, len(src) - if d := len(dst) - nDst; d < end-start { - end = nSrc + d - } - for nSrc++; nSrc < end && src[nSrc] < utf8.RuneSelf; nSrc++ { - } - n := copy(dst[nDst:], src[start:nSrc]) - if nDst += n; nDst == len(dst) { - nSrc = start + n - if nSrc == len(src) { - return nDst, nSrc, nil - } - if src[nSrc] < utf8.RuneSelf { - return nDst, nSrc, transform.ErrShortDst - } - } - continue - } - v, size := trie.lookup(src[nSrc:]) - if size == 0 { // incomplete UTF-8 encoding - if !atEOF { - return nDst, nSrc, transform.ErrShortSrc - } - size = 1 // gobble 1 byte - } - if elem(v)&tagNeedsFold == 0 { - if size != copy(dst[nDst:], src[nSrc:nSrc+size]) { - return nDst, nSrc, transform.ErrShortDst - } - nDst += size - } else { - data := inverseData[byte(v)] - if len(dst)-nDst < int(data[0]) { - return nDst, nSrc, transform.ErrShortDst - } - i := 1 - for end := int(data[0]); i < end; i++ { - dst[nDst] = data[i] - nDst++ - } - dst[nDst] = data[i] ^ src[nSrc+size-1] - nDst++ - } - nSrc += size - } - return nDst, nSrc, nil -} - -type narrowTransform struct { - transform.NopResetter -} - -func (narrowTransform) Span(src []byte, atEOF bool) (n int, err error) { - for n < len(src) { - if src[n] < utf8.RuneSelf { - // ASCII fast path. - for n++; n < len(src) && src[n] < utf8.RuneSelf; n++ { - } - continue - } - v, size := trie.lookup(src[n:]) - if size == 0 { // incomplete UTF-8 encoding - if !atEOF { - err = transform.ErrShortSrc - } else { - n = len(src) - } - break - } - if k := elem(v).kind(); byte(v) == 0 || k != EastAsianFullwidth && k != EastAsianWide && k != EastAsianAmbiguous { - } else { - err = transform.ErrEndOfSpan - break - } - n += size - } - return n, err -} - -func (narrowTransform) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { - for nSrc < len(src) { - if src[nSrc] < utf8.RuneSelf { - // ASCII fast path. - start, end := nSrc, len(src) - if d := len(dst) - nDst; d < end-start { - end = nSrc + d - } - for nSrc++; nSrc < end && src[nSrc] < utf8.RuneSelf; nSrc++ { - } - n := copy(dst[nDst:], src[start:nSrc]) - if nDst += n; nDst == len(dst) { - nSrc = start + n - if nSrc == len(src) { - return nDst, nSrc, nil - } - if src[nSrc] < utf8.RuneSelf { - return nDst, nSrc, transform.ErrShortDst - } - } - continue - } - v, size := trie.lookup(src[nSrc:]) - if size == 0 { // incomplete UTF-8 encoding - if !atEOF { - return nDst, nSrc, transform.ErrShortSrc - } - size = 1 // gobble 1 byte - } - if k := elem(v).kind(); byte(v) == 0 || k != EastAsianFullwidth && k != EastAsianWide && k != EastAsianAmbiguous { - if size != copy(dst[nDst:], src[nSrc:nSrc+size]) { - return nDst, nSrc, transform.ErrShortDst - } - nDst += size - } else { - data := inverseData[byte(v)] - if len(dst)-nDst < int(data[0]) { - return nDst, nSrc, transform.ErrShortDst - } - i := 1 - for end := int(data[0]); i < end; i++ { - dst[nDst] = data[i] - nDst++ - } - dst[nDst] = data[i] ^ src[nSrc+size-1] - nDst++ - } - nSrc += size - } - return nDst, nSrc, nil -} - -type wideTransform struct { - transform.NopResetter -} - -func (wideTransform) Span(src []byte, atEOF bool) (n int, err error) { - for n < len(src) { - // TODO: Consider ASCII fast path. Special-casing ASCII handling can - // reduce the ns/op of BenchmarkWideASCII by about 30%. This is probably - // not enough to warrant the extra code and complexity. - v, size := trie.lookup(src[n:]) - if size == 0 { // incomplete UTF-8 encoding - if !atEOF { - err = transform.ErrShortSrc - } else { - n = len(src) - } - break - } - if k := elem(v).kind(); byte(v) == 0 || k != EastAsianHalfwidth && k != EastAsianNarrow { - } else { - err = transform.ErrEndOfSpan - break - } - n += size - } - return n, err -} - -func (wideTransform) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { - for nSrc < len(src) { - // TODO: Consider ASCII fast path. Special-casing ASCII handling can - // reduce the ns/op of BenchmarkWideASCII by about 30%. This is probably - // not enough to warrant the extra code and complexity. - v, size := trie.lookup(src[nSrc:]) - if size == 0 { // incomplete UTF-8 encoding - if !atEOF { - return nDst, nSrc, transform.ErrShortSrc - } - size = 1 // gobble 1 byte - } - if k := elem(v).kind(); byte(v) == 0 || k != EastAsianHalfwidth && k != EastAsianNarrow { - if size != copy(dst[nDst:], src[nSrc:nSrc+size]) { - return nDst, nSrc, transform.ErrShortDst - } - nDst += size - } else { - data := inverseData[byte(v)] - if len(dst)-nDst < int(data[0]) { - return nDst, nSrc, transform.ErrShortDst - } - i := 1 - for end := int(data[0]); i < end; i++ { - dst[nDst] = data[i] - nDst++ - } - dst[nDst] = data[i] ^ src[nSrc+size-1] - nDst++ - } - nSrc += size - } - return nDst, nSrc, nil -} diff --git a/vendor/golang.org/x/text/width/trieval.go b/vendor/golang.org/x/text/width/trieval.go deleted file mode 100644 index ca8e45fd19..0000000000 --- a/vendor/golang.org/x/text/width/trieval.go +++ /dev/null @@ -1,30 +0,0 @@ -// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. - -package width - -// elem is an entry of the width trie. The high byte is used to encode the type -// of the rune. The low byte is used to store the index to a mapping entry in -// the inverseData array. -type elem uint16 - -const ( - tagNeutral elem = iota << typeShift - tagAmbiguous - tagWide - tagNarrow - tagFullwidth - tagHalfwidth -) - -const ( - numTypeBits = 3 - typeShift = 16 - numTypeBits - - // tagNeedsFold is true for all fullwidth and halfwidth runes except for - // the Won sign U+20A9. - tagNeedsFold = 0x1000 - - // The Korean Won sign is halfwidth, but SHOULD NOT be mapped to a wide - // variant. - wonSign rune = 0x20A9 -) diff --git a/vendor/golang.org/x/text/width/width.go b/vendor/golang.org/x/text/width/width.go deleted file mode 100644 index 29c7509be7..0000000000 --- a/vendor/golang.org/x/text/width/width.go +++ /dev/null @@ -1,206 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:generate stringer -type=Kind -//go:generate go run gen.go gen_common.go gen_trieval.go - -// Package width provides functionality for handling different widths in text. -// -// Wide characters behave like ideographs; they tend to allow line breaks after -// each character and remain upright in vertical text layout. Narrow characters -// are kept together in words or runs that are rotated sideways in vertical text -// layout. -// -// For more information, see https://unicode.org/reports/tr11/. -package width // import "golang.org/x/text/width" - -import ( - "unicode/utf8" - - "golang.org/x/text/transform" -) - -// TODO -// 1) Reduce table size by compressing blocks. -// 2) API proposition for computing display length -// (approximation, fixed pitch only). -// 3) Implement display length. - -// Kind indicates the type of width property as defined in https://unicode.org/reports/tr11/. -type Kind int - -const ( - // Neutral characters do not occur in legacy East Asian character sets. - Neutral Kind = iota - - // EastAsianAmbiguous characters that can be sometimes wide and sometimes - // narrow and require additional information not contained in the character - // code to further resolve their width. - EastAsianAmbiguous - - // EastAsianWide characters are wide in its usual form. They occur only in - // the context of East Asian typography. These runes may have explicit - // halfwidth counterparts. - EastAsianWide - - // EastAsianNarrow characters are narrow in its usual form. They often have - // fullwidth counterparts. - EastAsianNarrow - - // Note: there exist Narrow runes that do not have fullwidth or wide - // counterparts, despite what the definition says (e.g. U+27E6). - - // EastAsianFullwidth characters have a compatibility decompositions of type - // wide that map to a narrow counterpart. - EastAsianFullwidth - - // EastAsianHalfwidth characters have a compatibility decomposition of type - // narrow that map to a wide or ambiguous counterpart, plus U+20A9 ₩ WON - // SIGN. - EastAsianHalfwidth - - // Note: there exist runes that have a halfwidth counterparts but that are - // classified as Ambiguous, rather than wide (e.g. U+2190). -) - -// TODO: the generated tries need to return size 1 for invalid runes for the -// width to be computed correctly (each byte should render width 1) - -var trie = newWidthTrie(0) - -// Lookup reports the Properties of the first rune in b and the number of bytes -// of its UTF-8 encoding. -func Lookup(b []byte) (p Properties, size int) { - v, sz := trie.lookup(b) - return Properties{elem(v), b[sz-1]}, sz -} - -// LookupString reports the Properties of the first rune in s and the number of -// bytes of its UTF-8 encoding. -func LookupString(s string) (p Properties, size int) { - v, sz := trie.lookupString(s) - return Properties{elem(v), s[sz-1]}, sz -} - -// LookupRune reports the Properties of rune r. -func LookupRune(r rune) Properties { - var buf [4]byte - n := utf8.EncodeRune(buf[:], r) - v, _ := trie.lookup(buf[:n]) - last := byte(r) - if r >= utf8.RuneSelf { - last = 0x80 + byte(r&0x3f) - } - return Properties{elem(v), last} -} - -// Properties provides access to width properties of a rune. -type Properties struct { - elem elem - last byte -} - -func (e elem) kind() Kind { - return Kind(e >> typeShift) -} - -// Kind returns the Kind of a rune as defined in Unicode TR #11. -// See https://unicode.org/reports/tr11/ for more details. -func (p Properties) Kind() Kind { - return p.elem.kind() -} - -// Folded returns the folded variant of a rune or 0 if the rune is canonical. -func (p Properties) Folded() rune { - if p.elem&tagNeedsFold != 0 { - buf := inverseData[byte(p.elem)] - buf[buf[0]] ^= p.last - r, _ := utf8.DecodeRune(buf[1 : 1+buf[0]]) - return r - } - return 0 -} - -// Narrow returns the narrow variant of a rune or 0 if the rune is already -// narrow or doesn't have a narrow variant. -func (p Properties) Narrow() rune { - if k := p.elem.kind(); byte(p.elem) != 0 && (k == EastAsianFullwidth || k == EastAsianWide || k == EastAsianAmbiguous) { - buf := inverseData[byte(p.elem)] - buf[buf[0]] ^= p.last - r, _ := utf8.DecodeRune(buf[1 : 1+buf[0]]) - return r - } - return 0 -} - -// Wide returns the wide variant of a rune or 0 if the rune is already -// wide or doesn't have a wide variant. -func (p Properties) Wide() rune { - if k := p.elem.kind(); byte(p.elem) != 0 && (k == EastAsianHalfwidth || k == EastAsianNarrow) { - buf := inverseData[byte(p.elem)] - buf[buf[0]] ^= p.last - r, _ := utf8.DecodeRune(buf[1 : 1+buf[0]]) - return r - } - return 0 -} - -// TODO for Properties: -// - Add Fullwidth/Halfwidth or Inverted methods for computing variants -// mapping. -// - Add width information (including information on non-spacing runes). - -// Transformer implements the transform.Transformer interface. -type Transformer struct { - t transform.SpanningTransformer -} - -// Reset implements the transform.Transformer interface. -func (t Transformer) Reset() { t.t.Reset() } - -// Transform implements the transform.Transformer interface. -func (t Transformer) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { - return t.t.Transform(dst, src, atEOF) -} - -// Span implements the transform.SpanningTransformer interface. -func (t Transformer) Span(src []byte, atEOF bool) (n int, err error) { - return t.t.Span(src, atEOF) -} - -// Bytes returns a new byte slice with the result of applying t to b. -func (t Transformer) Bytes(b []byte) []byte { - b, _, _ = transform.Bytes(t, b) - return b -} - -// String returns a string with the result of applying t to s. -func (t Transformer) String(s string) string { - s, _, _ = transform.String(t, s) - return s -} - -var ( - // Fold is a transform that maps all runes to their canonical width. - // - // Note that the NFKC and NFKD transforms in golang.org/x/text/unicode/norm - // provide a more generic folding mechanism. - Fold Transformer = Transformer{foldTransform{}} - - // Widen is a transform that maps runes to their wide variant, if - // available. - Widen Transformer = Transformer{wideTransform{}} - - // Narrow is a transform that maps runes to their narrow variant, if - // available. - Narrow Transformer = Transformer{narrowTransform{}} -) - -// TODO: Consider the following options: -// - Treat Ambiguous runes that have a halfwidth counterpart as wide, or some -// generalized variant of this. -// - Consider a wide Won character to be the default width (or some generalized -// variant of this). -// - Filter the set of characters that gets converted (the preferred approach is -// to allow applying filters to transforms). diff --git a/vendor/modules.txt b/vendor/modules.txt index 9cdd24d955..6f0e7047bc 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -355,14 +355,17 @@ github.com/kubernetes-csi/csi-proxy/client/groups/disk/v1 github.com/kubernetes-csi/csi-proxy/client/groups/filesystem/v1 github.com/kubernetes-csi/csi-proxy/client/groups/system/v1alpha1 github.com/kubernetes-csi/csi-proxy/client/groups/volume/v1 -# github.com/kubernetes-csi/external-snapshotter/client/v6 v6.1.0 -## explicit; go 1.18 -github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1 -github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned -github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/fake -github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/scheme -github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1 -github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1/fake +# github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0 +## explicit; go 1.15 +github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1 +github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1beta1 +github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned +github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/fake +github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/scheme +github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1 +github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1/fake +github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1 +github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/typed/volumesnapshot/v1beta1/fake # github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de ## explicit github.com/liggitt/tabwriter