Azure blob storage support (#30995)

This PR implemented object storages(LFS/Packages/Attachments and etc.)
for Azure Blob Storage. It depends on azure official golang SDK and can
support both the azure blob storage cloud service and azurite mock
server.

Replace #25458
Fix #22527 

- [x] CI Tests
- [x] integration test, MSSQL integration tests will now based on
azureblob
  - [x] unit test 
- [x] CLI Migrate Storage
- [x] Documentation for configuration added

------

TODO (other PRs):
- [ ] Improve performance of `blob download`.

---------

Co-authored-by: yp05327 <576951401@qq.com>

.devcontainer/devcontainer.json

@@ -26,7 +26,8 @@
         "ms-azuretools.vscode-docker",
         "vitest.explorer",
         "qwtel.sqlite-viewer",
-        "GitHub.vscode-pull-request-github"
+        "GitHub.vscode-pull-request-github",
+        "Azurite.azurite"
       ]
     }
   },

.github/workflows/pull-db-tests.yml

@@ -119,14 +119,18 @@ jobs:
           MINIO_SECRET_KEY: 12345678
         ports:
           - "9000:9000"
+      devstoreaccount1.azurite.local: # https://github.com/Azure/Azurite/issues/1583
+        image: mcr.microsoft.com/azure-storage/azurite:latest
+        ports:
+          - 10000:10000
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
           go-version-file: go.mod
           check-latest: true
       - name: Add hosts to /etc/hosts
-        run: '[ -e "/.dockerenv" ] || [ -e "/run/.containerenv" ] || echo "127.0.0.1 mysql elasticsearch meilisearch smtpimap" | sudo tee -a /etc/hosts'
+        run: '[ -e "/.dockerenv" ] || [ -e "/run/.containerenv" ] || echo "127.0.0.1 minio devstoreaccount1.azurite.local mysql elasticsearch meilisearch smtpimap" | sudo tee -a /etc/hosts'
       - run: make deps-backend
       - run: make backend
         env:
@@ -204,14 +208,18 @@ jobs:
           SA_PASSWORD: MwantsaSecurePassword1
         ports:
           - "1433:1433"
+      devstoreaccount1.azurite.local: # https://github.com/Azure/Azurite/issues/1583
+        image: mcr.microsoft.com/azure-storage/azurite:latest
+        ports:
+          - 10000:10000
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
           go-version-file: go.mod
           check-latest: true
       - name: Add hosts to /etc/hosts
-        run: '[ -e "/.dockerenv" ] || [ -e "/run/.containerenv" ] || echo "127.0.0.1 mssql" | sudo tee -a /etc/hosts'
+        run: '[ -e "/.dockerenv" ] || [ -e "/run/.containerenv" ] || echo "127.0.0.1 mssql devstoreaccount1.azurite.local" | sudo tee -a /etc/hosts'
       - run: make deps-backend
       - run: make backend
         env:

cmd/migrate_storage.go

@@ -40,14 +40,15 @@ var CmdMigrateStorage = &cli.Command{
 			Name:    "storage",
 			Aliases: []string{"s"},
 			Value:   "",
-			Usage:   "New storage type: local (default) or minio",
+			Usage:   "New storage type: local (default), minio or azureblob",
 		},
 		&cli.StringFlag{
 			Name:    "path",
 			Aliases: []string{"p"},
 			Value:   "",
 			Usage:   "New storage placement if store is local (leave blank for default)",
 		},
+		// Minio Storage special configurations
 		&cli.StringFlag{
 			Name:  "minio-endpoint",
 			Value: "",
@@ -96,6 +97,32 @@ var CmdMigrateStorage = &cli.Command{
 			Value: "",
 			Usage: "Minio bucket lookup type",
 		},
+		// Azure Blob Storage special configurations
+		&cli.StringFlag{
+			Name:  "azureblob-endpoint",
+			Value: "",
+			Usage: "Azure Blob storage endpoint",
+		},
+		&cli.StringFlag{
+			Name:  "azureblob-account-name",
+			Value: "",
+			Usage: "Azure Blob storage account name",
+		},
+		&cli.StringFlag{
+			Name:  "azureblob-account-key",
+			Value: "",
+			Usage: "Azure Blob storage account key",
+		},
+		&cli.StringFlag{
+			Name:  "azureblob-container",
+			Value: "",
+			Usage: "Azure Blob storage container",
+		},
+		&cli.StringFlag{
+			Name:  "azureblob-base-path",
+			Value: "",
+			Usage: "Azure Blob storage base path",
+		},
 	},
 }
 
@@ -228,6 +255,18 @@ func runMigrateStorage(ctx *cli.Context) error {
 					BucketLookUpType:   ctx.String("minio-bucket-lookup-type"),
 				},
 			})
+	case string(setting.AzureBlobStorageType):
+		dstStorage, err = storage.NewAzureBlobStorage(
+			stdCtx,
+			&setting.Storage{
+				AzureBlobConfig: setting.AzureBlobStorageConfig{
+					Endpoint:    ctx.String("azureblob-endpoint"),
+					AccountName: ctx.String("azureblob-account-name"),
+					AccountKey:  ctx.String("azureblob-account-key"),
+					Container:   ctx.String("azureblob-container"),
+					BasePath:    ctx.String("azureblob-base-path"),
+				},
+			})
 	default:
 		return fmt.Errorf("unsupported storage type: %s", ctx.String("storage"))
 	}