feat: Migrate container registries from etcd to postgreSQL

.gitignore

@@ -142,3 +142,5 @@ docs/manager/rest-reference/openapi.json
 
 /DIST-INFO
 /INSTALL-INFO
+
+etcd_container_registries_backup.json

changes/1917.breaking.md

@@ -0,0 +1 @@
+Update `ContainerRegistry` graphql API argument from `hostname` to `id`

changes/1917.feature.md

@@ -0,0 +1 @@
+Migrate container registry schema from `etcd` to `postgreSQL`

fixtures/manager/example-container-registries-backendai.json

@@ -0,0 +1,25 @@
+{
+    "container_registries": [
+        {
+            "id": "fe878f09-06cc-4b91-9242-4c71015cce04",
+            "registry_name": "cr.backend.ai",
+            "url": "https://cr.backend.ai",
+            "type": "harbor2",
+            "project": "stable"
+        },
+        {
+            "id": "f9617204-12c8-4a44-b650-98201a2fb406",
+            "registry_name": "cr.backend.ai",
+            "url": "https://cr.backend.ai",
+            "type": "harbor2",
+            "project": "community"
+        },
+        {
+            "id": "d53a6b55-78dd-4412-8786-585777300870",
+            "registry_name": "cr.backend.ai",
+            "url": "https://cr.backend.ai",
+            "type": "harbor2",
+            "project": "multiarch"
+        }
+    ]
+}

fixtures/manager/example-container-registries-dockerhub.json

@@ -0,0 +1,11 @@
+{
+    "container_registries": [
+        {
+            "id": "abc42a05-4471-41fa-8772-10bf6452c7d1",
+            "registry_name": "index.docker.io",
+            "url": "https://registry-1.docker.io",
+            "type": "docker",
+            "username": "lablup"
+        }
+    ]
+}

python.lock

@@ -936,48 +936,48 @@
           "artifacts": [
             {
               "algorithm": "sha256",
-              "hash": "992e994c7e481a5d3259c699574882b79d631a46f7c369bea350b7ccb0651317",
-              "url": "https://files.pythonhosted.org/packages/60/15/9aac35742c47578eb9a1aee4519fecebba5faec51eeb09f093eca9938567/boto3-1.34.61-py3-none-any.whl"
+              "hash": "8c6fbd3d45399a4e4685010117fb2dc52fc6afdab5a9460957d463ae0c2cc55d",
+              "url": "https://files.pythonhosted.org/packages/26/fb/b89de0e5e585e6fdecbf93fe1e05df8582d558cd92f8e3162fa88277fd78/boto3-1.34.64-py3-none-any.whl"
             },
             {
               "algorithm": "sha256",
-              "hash": "4b40bf2c8494647c9e88c180537dc9fc0c1047a9fffbb1e5b0da6596f1e59b7b",
-              "url": "https://files.pythonhosted.org/packages/0b/c1/5f220936875b62ba4dfc04eb24ab8d2e5e925e9baea3e174632739f9fa3a/boto3-1.34.61.tar.gz"
+              "hash": "e5d681f443645e6953ed0727bf756bf16d85efefcb69cf051d04a070ce65e545",
+              "url": "https://files.pythonhosted.org/packages/56/6a/be50db7805679f79de57f3b4228e762f8028c2d3a9789ce65c2260ed85c3/boto3-1.34.64.tar.gz"
             }
           ],
           "project_name": "boto3",
           "requires_dists": [
-            "botocore<1.35.0,>=1.34.61",
+            "botocore<1.35.0,>=1.34.64",
             "botocore[crt]<2.0a0,>=1.21.0; extra == \"crt\"",
             "jmespath<2.0.0,>=0.7.1",
             "s3transfer<0.11.0,>=0.10.0"
           ],
           "requires_python": ">=3.8",
-          "version": "1.34.61"
+          "version": "1.34.64"
         },
         {
           "artifacts": [
             {
               "algorithm": "sha256",
-              "hash": "079f3288d38f97fd5656c25c44a94bea0e7090b938abfdeea463eaadb210c4a0",
-              "url": "https://files.pythonhosted.org/packages/1d/4f/79dc11e13879bec49b577319a073f346ff340b0d00ebbd669ace4325bbbd/botocore-1.34.61-py3-none-any.whl"
+              "hash": "0ab760908749fe82325698591c49755a5bb20307d85a419aca9cc74e783b9407",
+              "url": "https://files.pythonhosted.org/packages/22/7e/7eea3d93fbc0d7ec8c322d2a6aef9be330a97f3fc4cffdcdc54c0455457d/botocore-1.34.64-py3-none-any.whl"
             },
             {
               "algorithm": "sha256",
-              "hash": "72df4af7e4e6392552c882d48c74e4be9bf7be4cd8d829711b312fbae13d7034",
-              "url": "https://files.pythonhosted.org/packages/11/3b/71b9b5cba2f8c90f40500c2827ce2b0229dd3c6b86a49065dc40b8e4a059/botocore-1.34.61.tar.gz"
+              "hash": "084f8c45216d62dc1add2350e236a2d5283526aacd0681e9818b37a6a5e5438b",
+              "url": "https://files.pythonhosted.org/packages/35/6e/9c59955f5e73e69729216f7ae76eb6843ce78da65d184d161eeb0f481a3a/botocore-1.34.64.tar.gz"
             }
           ],
           "project_name": "botocore",
           "requires_dists": [
             "awscrt==0.19.19; extra == \"crt\"",
             "jmespath<2.0.0,>=0.7.1",
             "python-dateutil<3.0.0,>=2.1",
-            "urllib3<1.27,>=1.25.4; python_version < \"3.10\"",
-            "urllib3<2.1,>=1.25.4; python_version >= \"3.10\""
+            "urllib3!=2.2.0,<3,>=1.25.4; python_version >= \"3.10\"",
+            "urllib3<1.27,>=1.25.4; python_version < \"3.10\""
           ],
           "requires_python": ">=3.8",
-          "version": "1.34.61"
+          "version": "1.34.64"
         },
         {
           "artifacts": [
@@ -3378,21 +3378,21 @@
           "artifacts": [
             {
               "algorithm": "sha256",
-              "hash": "76ec784a5dd2afac3b7da8003329834cdd9824294c260027f8c8d2e4d0a78f43",
-              "url": "https://files.pythonhosted.org/packages/cd/14/730280df294e52e395a70111f4d9b07be94f5ba7a69db7eba3c324f113b2/readchar-4.0.5-py3-none-any.whl"
+              "hash": "b4b31dd35de4897be738f27e8f9f62426b5fedb54b648364987e30ae534b71bc",
+              "url": "https://files.pythonhosted.org/packages/86/db/aca9e5e6a53a499d61cbd78b3594d700f1e48a50ab6970a92a4d1251f8db/readchar-4.0.6-py3-none-any.whl"
             },
             {
               "algorithm": "sha256",
-              "hash": "08a456c2d7c1888cde3f4688b542621b676eb38cd6cfed7eb6cb2e2905ddc826",
-              "url": "https://files.pythonhosted.org/packages/a1/57/439aaa28659e66265518232bf4291ae5568aa01cd9e0e0f6f8fe3b300e9e/readchar-4.0.5.tar.gz"
+              "hash": "e0dae942d3a746f8d5423f83dbad67efe704004baafe31b626477929faaee472",
+              "url": "https://files.pythonhosted.org/packages/ec/85/35c1a04aa52c432ec604b2816570fb0ab721cb7403191130b9c068c672c3/readchar-4.0.6.tar.gz"
             }
           ],
           "project_name": "readchar",
           "requires_dists": [
             "setuptools>=41.0"
           ],
-          "requires_python": ">=3.7",
-          "version": "4.0.5"
+          "requires_python": ">=3.8",
+          "version": "4.0.6"
         },
         {
           "artifacts": [
@@ -3514,13 +3514,13 @@
           "artifacts": [
             {
               "algorithm": "sha256",
-              "hash": "3cdb40f5cfa6966e812209d0994f2a4709b561c88e90cf00c2696d2df4e56b2e",
-              "url": "https://files.pythonhosted.org/packages/12/bb/7e7912e18cd558e7880d9b58ffc57300b2c28ffba9882b3a54ba5ce3ebc4/s3transfer-0.10.0-py3-none-any.whl"
+              "hash": "ceb252b11bcf87080fb7850a224fb6e05c8a776bab8f2b64b7f25b969464839d",
+              "url": "https://files.pythonhosted.org/packages/83/37/395cdb6ee92925fa211e55d8f07b9f93cf93f60d7d4ce5e66fd73f1ea986/s3transfer-0.10.1-py3-none-any.whl"
             },
             {
               "algorithm": "sha256",
-              "hash": "d0c8bbf672d5eebbe4e57945e23b972d963f07d82f661cabf678a5c88831595b",
-              "url": "https://files.pythonhosted.org/packages/a0/b5/4c570b08cb85fdcc65037b5229e00412583bb38d974efecb7ec3495f40ba/s3transfer-0.10.0.tar.gz"
+              "hash": "5683916b4c724f799e600f41dd9e10a9ff19871bf87623cc8f491cb4f5fa0a19",
+              "url": "https://files.pythonhosted.org/packages/83/bc/fb0c1f76517e3380eb142af8a9d6b969c150cfca1324cea7d965d8c66571/s3transfer-0.10.1.tar.gz"
             }
           ],
           "project_name": "s3transfer",
@@ -3529,7 +3529,7 @@
             "botocore[crt]<2.0a.0,>=1.33.2; extra == \"crt\""
           ],
           "requires_python": ">=3.8",
-          "version": "0.10.0"
+          "version": "0.10.1"
         },
         {
           "artifacts": [
@@ -3600,13 +3600,13 @@
           "artifacts": [
             {
               "algorithm": "sha256",
-              "hash": "02fa291a0471b3a18b2b2481ed902af520c69e8ae0919c13da936542754b4c56",
-              "url": "https://files.pythonhosted.org/packages/c0/7a/3da654f49c95d0cc6e9549a855b5818e66a917e852ec608e77550c8dc08b/setuptools-69.1.1-py3-none-any.whl"
+              "hash": "c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c",
+              "url": "https://files.pythonhosted.org/packages/92/e1/1c8bb3420105e70bdf357d57dd5567202b4ef8d27f810e98bb962d950834/setuptools-69.2.0-py3-none-any.whl"
             },
             {
               "algorithm": "sha256",
-              "hash": "5c0806c7d9af348e6dd3777b4f4dbb42c7ad85b190104837488eab9a7c945cf8",
-              "url": "https://files.pythonhosted.org/packages/c8/1f/e026746e5885a83e1af99002ae63650b7c577af5c424d4c27edcf729ab44/setuptools-69.1.1.tar.gz"
+              "hash": "0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e",
+              "url": "https://files.pythonhosted.org/packages/4d/5b/dc575711b6b8f2f866131a40d053e30e962e633b332acf7cd2c24843d83d/setuptools-69.2.0.tar.gz"
             }
           ],
           "project_name": "setuptools",
@@ -3615,8 +3615,8 @@
             "build[virtualenv]>=1.0.3; extra == \"testing-integration\"",
             "filelock>=3.4.0; extra == \"testing\"",
             "filelock>=3.4.0; extra == \"testing-integration\"",
-            "flake8-2020; extra == \"testing\"",
             "furo; extra == \"docs\"",
+            "importlib-metadata; extra == \"testing\"",
             "ini2toml[lite]>=0.9; extra == \"testing\"",
             "jaraco.develop>=7.21; (python_version >= \"3.9\" and sys_platform != \"cygwin\") and extra == \"testing\"",
             "jaraco.envs>=2.2; extra == \"testing\"",
@@ -3625,6 +3625,7 @@
             "jaraco.path>=3.2.0; extra == \"testing\"",
             "jaraco.path>=3.2.0; extra == \"testing-integration\"",
             "jaraco.tidelift>=1.4; extra == \"docs\"",
+            "mypy==1.9; extra == \"testing\"",
             "packaging>=23.2; extra == \"testing\"",
             "packaging>=23.2; extra == \"testing-integration\"",
             "pip>=19.1; extra == \"testing\"",
@@ -3638,8 +3639,8 @@
             "pytest-perf; sys_platform != \"cygwin\" and extra == \"testing\"",
             "pytest-ruff>=0.2.1; sys_platform != \"cygwin\" and extra == \"testing\"",
             "pytest-timeout; extra == \"testing\"",
-            "pytest-xdist; extra == \"testing\"",
             "pytest-xdist; extra == \"testing-integration\"",
+            "pytest-xdist>=3; extra == \"testing\"",
             "pytest; extra == \"testing-integration\"",
             "pytest>=6; extra == \"testing\"",
             "rst.linker>=1.9; extra == \"docs\"",
@@ -3652,14 +3653,15 @@
             "sphinx>=3.5; extra == \"docs\"",
             "sphinxcontrib-towncrier; extra == \"docs\"",
             "tomli-w>=1.0.0; extra == \"testing\"",
+            "tomli; extra == \"testing\"",
             "tomli; extra == \"testing-integration\"",
             "virtualenv>=13.0.0; extra == \"testing\"",
             "virtualenv>=13.0.0; extra == \"testing-integration\"",
             "wheel; extra == \"testing\"",
             "wheel; extra == \"testing-integration\""
           ],
           "requires_python": ">=3.8",
-          "version": "69.1.1"
+          "version": "69.2.0"
         },
         {
           "artifacts": [
@@ -4155,19 +4157,19 @@
           "artifacts": [
             {
               "algorithm": "sha256",
-              "hash": "ef813da0809aca76472ca88807addbeea98b19339aebe56159ae2f4b4f70857a",
-              "url": "https://files.pythonhosted.org/packages/e8/19/e11b8098f5d7864a9950385760f1d8a68abd1e0b8ba89656d94c45cf9f93/types_python_dateutil-2.8.19.20240311-py3-none-any.whl"
+              "hash": "6b8cb66d960771ce5ff974e9dd45e38facb81718cc1e208b10b1baccbfdbee3b",
+              "url": "https://files.pythonhosted.org/packages/c7/1b/af4f4c4f3f7339a4b7eb3c0ab13416db98f8ac09de3399129ee5fdfa282b/types_python_dateutil-2.9.0.20240316-py3-none-any.whl"
             },
             {
               "algorithm": "sha256",
-              "hash": "51178227bbd4cbec35dc9adffbf59d832f20e09842d7dcb8c73b169b8780b7cb",
-              "url": "https://files.pythonhosted.org/packages/fb/90/e37d402a07f5a93791fc2837ee14b6947989aed6dc7895c420eb93354aea/types-python-dateutil-2.8.19.20240311.tar.gz"
+              "hash": "5d2f2e240b86905e40944dd787db6da9263f0deabef1076ddaed797351ec0202",
+              "url": "https://files.pythonhosted.org/packages/61/c5/c3a4d72ffa8efc2e78f7897b1c69ec760553246b67d3ce8c4431fac5d4e3/types-python-dateutil-2.9.0.20240316.tar.gz"
             }
           ],
           "project_name": "types-python-dateutil",
           "requires_dists": [],
           "requires_python": ">=3.8",
-          "version": "2.8.19.20240311"
+          "version": "2.9.0.20240316"
         },
         {
           "artifacts": [
@@ -4212,19 +4214,19 @@
           "artifacts": [
             {
               "algorithm": "sha256",
-              "hash": "7801245ecaf371d24f1154924c8f1f0efdc53977339bf79886b5b10890af6478",
-              "url": "https://files.pythonhosted.org/packages/a6/bf/2c0de809f83077f4095b89e3dad36807d466c6bd951d2e24bf8a08c968dc/types_setuptools-69.1.0.20240310-py3-none-any.whl"
+              "hash": "cf91ff7c87ab7bf0625c3f0d4d90427c9da68561f3b0feab77977aaf0bbf7531",
+              "url": "https://files.pythonhosted.org/packages/1f/22/904934a3344fa5f332ecab887003f3f033c1272432a4af877007b75b0bd3/types_setuptools-69.2.0.20240317-py3-none-any.whl"
             },
             {
               "algorithm": "sha256",
-              "hash": "fc0e1082f55c974611bce844b1e5beb2d1a895501f4a464e48305592a4268100",
-              "url": "https://files.pythonhosted.org/packages/e6/5f/a48e5273ffded6e85749866edec3d8d283dc0c7ed8cd8fd6f1b98d280cfa/types-setuptools-69.1.0.20240310.tar.gz"
+              "hash": "b607c4c48842ef3ee49dc0c7fe9c1bad75700b071e1018bb4d7e3ac492d47048",
+              "url": "https://files.pythonhosted.org/packages/2d/06/0de7b539346aaa8758b3c80375c4841dc2764ef92c5e743f1ebe9789da54/types-setuptools-69.2.0.20240317.tar.gz"
             }
           ],
           "project_name": "types-setuptools",
           "requires_dists": [],
           "requires_python": ">=3.8",
-          "version": "69.1.0.20240310"
+          "version": "69.2.0.20240317"
         },
         {
           "artifacts": [
@@ -4328,29 +4330,25 @@
           "artifacts": [
             {
               "algorithm": "sha256",
-              "hash": "fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e",
-              "url": "https://files.pythonhosted.org/packages/d2/b2/b157855192a68541a91ba7b2bbcb91f1b4faa51f8bae38d8005c034be524/urllib3-2.0.7-py3-none-any.whl"
+              "hash": "450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d",
+              "url": "https://files.pythonhosted.org/packages/a2/73/a68704750a7679d0b6d3ad7aa8d4da8e14e151ae82e6fee774e6e0d05ec8/urllib3-2.2.1-py3-none-any.whl"
             },
             {
               "algorithm": "sha256",
-              "hash": "c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84",
-              "url": "https://files.pythonhosted.org/packages/af/47/b215df9f71b4fdba1025fc05a77db2ad243fa0926755a52c5e71659f4e3c/urllib3-2.0.7.tar.gz"
+              "hash": "d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19",
+              "url": "https://files.pythonhosted.org/packages/7a/50/7fd50a27caa0652cd4caf224aa87741ea41d3265ad13f010886167cfcc79/urllib3-2.2.1.tar.gz"
             }
           ],
           "project_name": "urllib3",
           "requires_dists": [
             "brotli>=1.0.9; platform_python_implementation == \"CPython\" and extra == \"brotli\"",
             "brotlicffi>=0.8.0; platform_python_implementation != \"CPython\" and extra == \"brotli\"",
-            "certifi; extra == \"secure\"",
-            "cryptography>=1.9; extra == \"secure\"",
-            "idna>=2.0.0; extra == \"secure\"",
-            "pyopenssl>=17.1.0; extra == \"secure\"",
+            "h2<5,>=4; extra == \"h2\"",
             "pysocks!=1.5.7,<2.0,>=1.5.6; extra == \"socks\"",
-            "urllib3-secure-extra; extra == \"secure\"",
             "zstandard>=0.18.0; extra == \"zstd\""
           ],
-          "requires_python": ">=3.7",
-          "version": "2.0.7"
+          "requires_python": ">=3.8",
+          "version": "2.2.1"
         },
         {
           "artifacts": [

scripts/install-dev.sh

@@ -955,21 +955,13 @@ configure_backendai() {
   # initialize the DB schema
   show_info "Setting up databases..."
   ./backend.ai mgr schema oneshot
+
+  ./backend.ai mgr fixture populate fixtures/manager/example-container-registries-backendai.json
   ./backend.ai mgr fixture populate fixtures/manager/example-users.json
   ./backend.ai mgr fixture populate fixtures/manager/example-keypairs.json
   ./backend.ai mgr fixture populate fixtures/manager/example-set-user-main-access-keys.json
   ./backend.ai mgr fixture populate fixtures/manager/example-resource-presets.json
 
-  # Docker registry setup
-  show_info "Configuring the Lablup's official image registry..."
-  ./backend.ai mgr etcd put config/docker/registry/cr.backend.ai "https://cr.backend.ai"
-  ./backend.ai mgr etcd put config/docker/registry/cr.backend.ai/type "harbor2"
-  if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then
-    ./backend.ai mgr etcd put config/docker/registry/cr.backend.ai/project "stable,community,multiarch"
-  else
-    ./backend.ai mgr etcd put config/docker/registry/cr.backend.ai/project "stable,community"
-  fi
-
   # Scan the container image registry
   show_info "Scanning the image registry..."
   ./backend.ai mgr image rescan cr.backend.ai