diff --git a/README.md b/README.md index e15ada3d..771533ec 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ SmythOS provides a complete **Operating System for Agentic AI**. Just as traditi ### Unified Resource Abstraction -SmythOS provides a **unified interface for all resources**, ensuring consistency and simplicity across your entire AI platform. Whether you're storing a file locally, on S3, or any other storage provider, you don't need to worry about the underlying implementation details. SmythOS offers a powerful abstraction layer where all providers expose the same functions and APIs. +SmythOS provides a **unified interface for all resources**, ensuring consistency and simplicity across your entire AI platform. Whether you're storing a file locally, on AWS S3, Azure Blob Storage or any other storage provider, you don't need to worry about the underlying implementation details. SmythOS offers a powerful abstraction layer where all providers expose the same functions and APIs. This principle applies to **all services** - not just storage. Whether you're working with VectorDBs, cache (Redis, RAM), LLMs (OpenAI, Anthropic), or any other resource, the interface remains consistent across providers. diff --git a/examples/06-Storage-no-agent/03-AzureBlobStorage.ts b/examples/06-Storage-no-agent/03-AzureBlobStorage.ts new file mode 100644 index 00000000..9ef3636b --- /dev/null +++ b/examples/06-Storage-no-agent/03-AzureBlobStorage.ts @@ -0,0 +1,56 @@ +import 'dotenv/config'; +import { Storage } from '@smythos/sdk'; + +/** + * Writes a test blob to Azure Blob Storage, reads it back, and verifies the content matches. + * + * Validates AZURE_STORAGE_ACCOUNT_NAME, AZURE_STORAGE_ACCESS_KEY and AZURE_BLOB_CONTAINER_NAME from environment variables. + * If credentials are missing, the function logs an error and returns early. + * + * @returns A promise that resolves when the example completes. + * @throws Error If the read content does not match the written content. + */ +async function main() { + // Ensure all required environment variables are loaded before proceeding. + if (!process.env.AZURE_STORAGE_ACCOUNT_NAME || !process.env.AZURE_STORAGE_ACCESS_KEY || !process.env.AZURE_BLOB_CONTAINER_NAME) { + console.error("Error: Missing Azure config in your .env file. Ensure AZURE_STORAGE_ACCOUNT_NAME, AZURE_STORAGE_ACCESS_KEY, and AZURE_BLOB_CONTAINER_NAME are set."); + return; + } + + // Initialize the Azure Blob Storage connector with credentials from the .env file + const azureStorage = Storage.AzureBlobStorage({ + storageAccountName: process.env.AZURE_STORAGE_ACCOUNT_NAME, + storageAccountAccessKey: process.env.AZURE_STORAGE_ACCESS_KEY, + blobContainerName: process.env.AZURE_BLOB_CONTAINER_NAME, + }); + + const resourceId = 'smythos-azure-test.txt'; + const content = 'Hello, world from Azure!'; + + console.log(`=== Running Azure Blob Storage Example ===`); + + // Write a file to your container. + console.log(`Writing "${content}" to "${resourceId}"...`); + await azureStorage.write(resourceId, content); + console.log('Write operation complete.'); + + // Read the file back. + console.log(`Reading "${resourceId}"...`); + const data = await azureStorage.read(resourceId); + console.log('Read operation complete.'); + + // Log the content to verify it's correct. + const dataAsString = data?.toString(); + console.log(`Content read from blob: "${dataAsString}"`); + + if (dataAsString !== content) { + throw new Error("Verification failed: Read content does not match written content!"); + } + + console.log('Verification successful!'); + console.log(`=== Example Finished ===`); +} + +main().catch(error => { + console.error("An error occurred:", error.message); +}); \ No newline at end of file diff --git a/packages/core/docs/connectors/storage.md b/packages/core/docs/connectors/storage.md index 84b2342e..369bcc50 100644 --- a/packages/core/docs/connectors/storage.md +++ b/packages/core/docs/connectors/storage.md @@ -81,3 +81,48 @@ SRE.init({ - Store credentials securely using environment variables or AWS Secrets Manager - Configure appropriate bucket policies and CORS settings - Enable encryption at rest and in transit for sensitive data + +----- + +### Azure Blob Storage + +**Role**: Microsoft Azure Blob Storage cloud connector +**Summary**: Provides highly scalable and secure cloud storage using Azure Blob Storage, ideal for production workloads integrated with the Microsoft Azure ecosystem. + +| Setting | Type | Required | Default | Description | +|:--------------------------|:-------|:---------|:--------|:---------------------------------------------| +| `storageAccountName` | string | Yes | - | The name of your Azure Storage Account | +| `storageAccountAccessKey` | string | Yes | - | Access key for the storage account | +| `blobContainerName` | string | Yes | - | The container name for storing blobs (files) | + + +**Example Configuration:** + +```typescript +import { SRE } from '@smythos/sre'; + +SRE.init({ + Storage: { + Connector: 'AzureBlobStorage', + Settings: { + storageAccountName: 'myappstorageaccount', + storageAccountAccessKey: process.env.AZURE_STORAGE_ACCESS_KEY, + blobContainerName: 'my-app-blobs', + }, + }, +}); +``` + +**Use Cases:** + + * Production environments, especially those hosted on Microsoft Azure + * Applications requiring durable, high-availability, and geo-redundant storage + * Integration with the Azure ecosystem (Azure Functions, Logic Apps, etc.) + * Storing large-scale unstructured data like images, videos, and documents + +**Security Notes:** + + * Use Managed Identities when running on Azure infrastructure for the most secure, keyless authentication. + * Store credentials securely using environment variables or Azure Key Vault. + * Configure container access policies and use Shared Access Signatures (SAS) for fine-grained, temporary access. + * Enable encryption at rest and in transit for sensitive data. \ No newline at end of file diff --git a/packages/core/package.json b/packages/core/package.json index d09776ab..9b1b11ce 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -60,6 +60,7 @@ "@aws-sdk/client-lambda": "^3.835.0", "@aws-sdk/client-s3": "^3.826.0", "@aws-sdk/client-secrets-manager": "^3.826.0", + "@azure/storage-blob": "^12.28.0", "@faker-js/faker": "^9.8.0", "@google-cloud/vertexai": "^1.7.0", "@google/genai": "^1.10.0", diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 81c3eab9..66365158 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -162,6 +162,7 @@ export * from './subsystems/IO/Router.service/connectors/ExpressRouter.class'; export * from './subsystems/IO/Router.service/connectors/NullRouter.class'; export * from './subsystems/IO/Storage.service/connectors/LocalStorage.class'; export * from './subsystems/IO/Storage.service/connectors/S3Storage.class'; +export * from './subsystems/IO/Storage.service/connectors/AzureBlobStorage.class' export * from './subsystems/IO/VectorDB.service/connectors/MilvusVectorDB.class'; export * from './subsystems/IO/VectorDB.service/connectors/PineconeVectorDB.class'; export * from './subsystems/IO/VectorDB.service/connectors/RAMVecrtorDB.class'; diff --git a/packages/core/src/subsystems/IO/Storage.service/connectors/AzureBlobStorage.class.ts b/packages/core/src/subsystems/IO/Storage.service/connectors/AzureBlobStorage.class.ts new file mode 100644 index 00000000..47ac1fed --- /dev/null +++ b/packages/core/src/subsystems/IO/Storage.service/connectors/AzureBlobStorage.class.ts @@ -0,0 +1,621 @@ +//==[ SRE: AzureBlobStorage ]====================== + +import { BlobServiceClient, ContainerClient, StorageSharedKeyCredential } from '@azure/storage-blob'; + +import { Logger } from '@sre/helpers/Log.helper'; +import { StorageConnector } from '@sre/IO/Storage.service/StorageConnector'; +import { ACL } from '@sre/Security/AccessControl/ACL.class'; +import { IAccessCandidate, IACL, TAccessLevel, TAccessRole } from '@sre/types/ACL.types'; +import { StorageData, StorageMetadata } from '@sre/types/Storage.types'; +import { AccessRequest } from '@sre/Security/AccessControl/AccessRequest.class'; +import { SecureConnector } from '@sre/Security/SecureConnector.class'; + +const console = Logger('AzureBlobStorage'); + +export type AzureBlobConfig = { + storageAccountName: string; + storageAccountAccessKey: string; + blobContainerName: string; +}; + +export class AzureBlobStorage extends StorageConnector { + public name = 'AzureBlobStorage'; + private blobServiceClient: BlobServiceClient; + private containerClient: ContainerClient; + private isInitialized: boolean = false; + private initializationPromise: Promise | null = null; + + constructor(protected _settings: AzureBlobConfig) { + super(_settings); + + // Validate essential configuration settings on instantiation. + // This follows the "Fail Fast" principle by throwing an error immediately if required settings are missing or empty. + if (!_settings.storageAccountName?.trim()) { + throw new Error('Configuration Error: "storageAccountName" is required and cannot be empty.'); + } + + if (!_settings.storageAccountAccessKey?.trim()) { + throw new Error('Configuration Error: "storageAccountAccessKey" is required and cannot be empty.'); + } + + if (!_settings.blobContainerName?.trim()) { + throw new Error('Configuration Error: "blobContainerName" is required and cannot be empty.'); + } + + const endpointUrl = `https://${_settings.storageAccountName}.blob.core.windows.net`; + const credential = new StorageSharedKeyCredential(_settings.storageAccountName, _settings.storageAccountAccessKey); + + this.blobServiceClient = new BlobServiceClient(endpointUrl, credential); + this.containerClient = this.blobServiceClient.getContainerClient(_settings.blobContainerName); + } + + private async ensureInitialized(): Promise { + if (this.isInitialized) { + return; + } + + if (this.initializationPromise) { + return this.initializationPromise; + } + + this.initializationPromise = this.initialize(); + return this.initializationPromise; + } + + private async initialize(): Promise { + if (this.isInitialized) { + return; + } + try{ + await this.containerClient.createIfNotExists(); + console.log(`Container "${this._settings.blobContainerName}" is ready.`); + this.isInitialized = true; + } + catch(error){ + console.error('Failed to initialize AzureBlobStorage:', error); + this.initializationPromise = null; + throw error; + } + } + + + /** + * Reads a blob's content from the Azure Storage container. + * This method fetches the entire blob content into a Buffer. If the specified + * blob does not exist, it gracefully returns undefined. + * + * @param {AccessRequest} acRequest - The access request object, handled by the @SecureConnector decorator to authorize the operation. + * @param {string} resourceId - The unique identifier (name) of the blob to be read from the container. + * @returns {Promise} A promise that resolves with the blob's content as a Buffer (`StorageData`), or undefined if the blob is not found. + * @throws Throws an error if any other issue occurs during the download process (e.g., network issues, permissions problems). + */ + @SecureConnector.AccessControl + public async read(acRequest: AccessRequest, resourceId: string): Promise { + await this.ensureInitialized(); + + const blockBlobClient = this.containerClient.getBlockBlobClient(resourceId); + + try { + // Directly attempt to download. This avoids a separate `exists()` check, + // reducing two potential network calls to just one. + return await blockBlobClient.downloadToBuffer(); + } catch (error) { + const status = (error as any)?.statusCode; + + // A 404 error is an expected outcome if the blob doesn't exist. + // In this case, we return undefined as per the method's contract. + if (status === 404) { + return undefined; + } + + // For any other error (e.g., network failure, credentials issue), + // log it and re-throw it to be handled by the application's upper layers. + console.error(`Failed to read blob "${resourceId}":`, (error as any)?.message ?? error); + throw error; + } + } + + + /** + * Writes or overwrites a blob in the Azure Storage container. + * + * This method uploads data to a specified blob. It automatically handles the SmythOS + * Access Control List (ACL) by ensuring the writer is made the owner of the object. + * Any provided metadata is merged with this ACL and stored with the blob. + * If a blob with the same `resourceId` already exists, this operation will completely replace its content and metadata. + * + * @param {AccessRequest} acRequest - The access request object, handled by the @SecureConnector decorator to authorize the operation. + * @param {string} resourceId - The unique identifier (name) of the blob to be written or overwritten. + * @param {StorageData} value - The content to be written to the blob, typically a Buffer. + * @param {IACL} [acl] - An optional Access Control List to apply to the blob. If not provided, a default ACL is generated. + * @param {StorageMetadata} [metadata] - Optional key-value metadata to associate with the blob, including properties like `ContentType`. + * @returns {Promise} A promise that resolves when the upload operation is complete. + * @throws Throws an error if the upload fails due to network issues, invalid credentials, or other storage-related problems. + */ + @SecureConnector.AccessControl + async write(acRequest: AccessRequest, resourceId: string, value: StorageData, acl?: IACL, metadata?: StorageMetadata): Promise { + await this.ensureInitialized(); + + const accessCandidate = acRequest.candidate; + + // Ensure the creator of the object is always granted ownership, merging with any provided ACL. + const aclData = ACL.from(acl).addAccess(accessCandidate.role, accessCandidate.id, TAccessLevel.Owner).ACL; + const azureMetadata = { ...metadata, 'azure-acl': aclData }; + + const blockBlobClient = this.containerClient.getBlockBlobClient(resourceId); + + try { + const uploadOptions = { + // Serialize the combined metadata for storage with the blob. + metadata: this._serializeAzureMetadata(azureMetadata), + // Set the blob's Content-Type for correct handling by browsers and clients. + blobHTTPHeaders: { blobContentType: metadata?.['ContentType'] || 'application/octet-stream' } + }; + + // Perform the upload. This is a single, efficient network operation. + await blockBlobClient.uploadData(value, uploadOptions); + + } catch (error) { + console.error(`Failed to write blob "${resourceId}":`, error.message); + throw error; + } + } + + + /** + * Deletes a blob from the Azure Storage container. + * + * This method permanently removes the specified blob. The operation is idempotent, + * meaning it will complete successfully without error even if the blob does not + * already exist. This is useful for ensuring a resource is gone without needing to + * check for its existence first. + * + * @param {AccessRequest} acRequest - The access request object, handled by the @SecureConnector decorator to authorize the operation. + * @param {string} resourceId - The unique identifier (name) of the blob to be deleted. + * @returns {Promise} A promise that resolves when the delete operation is complete. + * @throws Throws an error if the deletion fails for reasons other than the blob not existing (e.g., network issues, permissions problems). + */ + @SecureConnector.AccessControl + async delete(acRequest: AccessRequest, resourceId: string): Promise { + await this.ensureInitialized(); + const blockBlobClient = this.containerClient.getBlockBlobClient(resourceId); + + try { + // The Azure SDK for Blob Storage handles the "not found" case gracefully + // by design, so no preliminary `exists()` check is needed. + await blockBlobClient.delete(); + + } catch (error) { + // This catch block will only execute for unexpected errors, such as + // network failures or insufficient permissions, not for a 404 Not Found. + console.error(`Failed to delete blob "${resourceId}":`, error.message); + throw error; + } + } + + + /** + * Checks for the existence of a blob in the Azure Storage container. + * + * This method efficiently verifies if a blob with the specified `resourceId` is + * present without downloading its content. It's the most performant way to + * check for a blob's presence before performing other operations. + * + * @param {AccessRequest} acRequest - The access request object, handled by the @SecureConnector decorator to authorize the operation. + * @param {string} resourceId - The unique identifier (name) of the blob to check. + * @returns {Promise} A promise that resolves with `true` if the blob exists, and `false` otherwise. + * @throws Throws an error for any issue other than the blob not being found (e.g., network issues, invalid credentials). + */ + @SecureConnector.AccessControl + async exists(acRequest: AccessRequest, resourceId: string): Promise { + await this.ensureInitialized(); + const blockBlobClient = this.containerClient.getBlockBlobClient(resourceId); + + try { + // This is a single, efficient network call that returns a boolean. + return await blockBlobClient.exists(); + } catch (error) { + // This block will catch unexpected errors like invalid credentials or network failures. + console.error(`Failed to check existence for blob "${resourceId}":`, error.message); + throw error; + } + } + + /** + * Retrieves the user-defined metadata for a specific blob. + * + * This method fetches the key-value metadata associated with a blob without + * downloading the blob's content. It's an efficient way to read custom + * information stored with an object. If the blob is not found, it returns undefined. + * + * @param {AccessRequest} acRequest - The access request object, handled by the @SecureConnector decorator to authorize the operation. + * @param {string} resourceId - The unique identifier (name) of the blob whose metadata is to be retrieved. + * @returns {Promise} A promise that resolves with the blob's metadata object, or undefined if the blob does not exist. + * @throws Throws an error for any issue other than the blob not being found (e.g., network issues, invalid credentials). + */ + @SecureConnector.AccessControl + async getMetadata(acRequest: AccessRequest, resourceId: string): Promise { + await this.ensureInitialized(); + + try { + // This public method delegates to a private helper for the actual implementation. + // This is a clean design pattern that separates the public API from internal logic. + return await this._getAzureMetadata(resourceId); + } catch (error) { + // The catch block ensures any unexpected errors from the helper are logged + // with the context of the public operation. + console.error(`Failed to get metadata for blob "${resourceId}":`, error.message); + throw error; + } + } + + + /** + * Sets or updates the user-defined metadata for a specific blob. + * + * This method applies new metadata to an existing blob. It performs a "merge and + * update" operation: it first reads the blob's current metadata, then merges + * the provided metadata with it (new values overwrite existing ones). + * This operation does not affect the blob's content. + * + * @param {AccessRequest} acRequest - The access request object, handled by the @SecureConnector decorator to authorize the operation. + * @param {string} resourceId - The unique identifier (name) of the blob whose metadata is to be updated. + * @param {StorageMetadata} metadata - An object containing the key-value pairs to set. These will be merged with any existing metadata. + * @returns {Promise} A promise that resolves when the metadata has been successfully updated. + * @throws Throws an error if the blob does not exist, or if the update fails for other reasons (e.g., network issues). + */ + @SecureConnector.AccessControl + async setMetadata(acRequest: AccessRequest, resourceId: string, metadata: StorageMetadata): Promise { + await this.ensureInitialized(); + + try { + // First, fetch the existing metadata to ensure the blob exists. + const existingMetadata = await this._getAzureMetadata(resourceId); + + // Explicitly fail if the blob does not exist. Metadata can only be set on an existing blob. + if (existingMetadata === undefined) { + throw new Error(`Cannot set metadata for non-existent blob: "${resourceId}"`); + } + + // Merge new metadata with existing metadata. New keys will overwrite old ones. + const mergedMetadata = { ...existingMetadata, ...metadata }; + + // Delegate the actual SDK call to a private helper. + await this._setAzureMetadata(resourceId, mergedMetadata); + + } catch (error) { + console.error(`Failed to set metadata for blob "${resourceId}":`, error.message); + throw error; + } + } + + + /** + * Retrieves the Access Control List (ACL) for a specific blob. + * + * This method fetches the blob's metadata and specifically extracts the + * SmythOS-native ACL object. It provides a direct way to inspect the + * permissions of a blob without downloading its content. + * + * @param {AccessRequest} acRequest - The access request object, handled by the @SecureConnector decorator to authorize the operation. + * @param {string} resourceId - The unique identifier (name) of the blob whose ACL is to be retrieved. + * @returns {Promise} A promise that resolves with the `ACL` object for the blob. Returns undefined if the blob does not exist or has no ACL. + * @throws Throws an error for any issue other than the blob not being found (e.g., network issues, invalid credentials). + */ + @SecureConnector.AccessControl + async getACL(acRequest: AccessRequest, resourceId: string): Promise { + await this.ensureInitialized(); + + try { + // Reuse the private helper to efficiently fetch the blob's properties. + const azureMetadata = await this._getAzureMetadata(resourceId); + + // If the blob doesn't exist, the metadata will be undefined. + if (!azureMetadata) { + return undefined; + } + + const raw = azureMetadata['azure-acl'] as IACL | undefined; + + if (!raw) return undefined; + + // The ACL.from() utility safely constructs an ACL object from the raw metadata. + return ACL.from(raw); + } catch (error) { + // Catches any unexpected errors during the process and logs them with context. + console.error(`Failed to get ACL for blob "${resourceId}":`, error.message); + throw error; + } + } + + + /** + * Sets or updates the Access Control List (ACL) for a specific blob. + * + * This method applies a new SmythOS ACL to an existing blob. It reads the blob's + * full metadata, replaces the ACL portion, and writes the updated metadata back. + * A crucial security feature of this method is that it automatically ensures the + * user/agent making the request always retains ownership of the blob. + * + * @param {AccessRequest} acRequest - The access request object, used to identify the user/agent to ensure they retain ownership. + * @param {string} resourceId - The unique identifier (name) of the blob whose ACL is to be updated. + * @param {IACL} acl - The new Access Control List object to apply to the blob. + * @returns {Promise} A promise that resolves when the ACL has been successfully updated. + * @throws Throws an error if the blob does not exist or if the update fails for other reasons. + */ + @SecureConnector.AccessControl + async setACL(acRequest: AccessRequest, resourceId: string, acl: IACL): Promise { + await this.ensureInitialized(); + + try { + // Fetch the current metadata to ensure the blob exists. + const azureMetadata = await this._getAzureMetadata(resourceId); + + // Explicitly fail if the blob does not exist, as an ACL cannot be set on a non-existent resource. + if (!azureMetadata) { + throw new Error(`Cannot set ACL for non-existent resource: "${resourceId}"`); + } + + // Security check: Ensure the user performing the action retains ownership. + // This prevents a user from accidentally locking themselves out. + azureMetadata['azure-acl'] = ACL.from(acl) + .addAccess(acRequest.candidate.role, acRequest.candidate.id, TAccessLevel.Owner) + .ACL; + + // Delegate the actual SDK call to the private helper. + await this._setAzureMetadata(resourceId, azureMetadata); + + } catch (error) { + // Catches any unexpected errors and logs them with clear, actionable context. + console.error(`Failed to set ACL for blob "${resourceId}":`, error.message); + throw error; + } + } + + /** + * Determines the effective Access Control List (ACL) for a given resource. + * + * This crucial security method is called by the access control system to fetch a + * resource's ACL before making an authorization decision. If the resource does + * not exist (and thus has no ACL), this method dynamically generates a new ACL + * that grants 'Owner' access to the requesting candidate. This is the mechanism + * that allows authorized users to create new resources. + * + * @param {string} resourceId - The unique identifier (name) of the blob whose ACL is to be determined. + * @param {IAccessCandidate} candidate - The user or agent attempting to access the resource, used to grant ownership if the resource is new. + * @returns {Promise} A promise that resolves with the existing ACL object if the blob is found, or a new ACL object granting ownership if the blob does not exist. + * @throws Throws an error if there is an unexpected issue fetching the blob's metadata (e.g., network issues). + */ + public async getResourceACL(resourceId: string, candidate: IAccessCandidate): Promise { + await this.ensureInitialized(); + + try { + // Reuse the private helper to efficiently fetch the blob's properties. + const azureMetadata = await this._getAzureMetadata(resourceId); + + // If the blob does not exist, azureMetadata will be undefined. + if (!azureMetadata) { + // This is the "create" path: grant ownership of the non-existent resource + // to the candidate so they are allowed to write it for the first time. + return new ACL().addAccess(candidate.role, candidate.id, TAccessLevel.Owner); + } + + // If the blob exists, return its stored ACL. + return ACL.from(azureMetadata['azure-acl'] as IACL); + + } catch (error) { + // Catches any unexpected errors during the process and logs them with context. + console.error(`Failed to get resource ACL for blob "${resourceId}":`, error.message); + throw error; + } + } + + /** + * Schedules a blob for automatic deletion after a specified time-to-live (TTL). + * + * This method applies a Blob Index Tag ('deleteAfterDays') to an existing blob. + * It does not delete the blob immediately. Instead, it relies on a pre-configured + * Lifecycle Management rule on the Azure Storage Account to automatically delete the + * blob after the specified number of days. + * + * @param {AccessRequest} acRequest - The access request object, handled by the @SecureConnector decorator to authorize the operation. + * @param {string} resourceId - The unique identifier (name) of the blob to schedule for deletion. + * @param {number} ttl - The time-to-live for the blob, in seconds. Must be a positive number. + * @returns {Promise} A promise that resolves when the expiration tag has been successfully applied. + * @throws Throws an error if the blob does not exist, if the TTL is invalid, or if applying the tag fails. + */ + @SecureConnector.AccessControl + async expire(acRequest: AccessRequest, resourceId: string, ttl: number): Promise { + await this.ensureInitialized(); + const blockBlobClient = this.containerClient.getBlockBlobClient(resourceId); + + try { + // 1. Input Validation: Ensure the TTL is a valid positive number. + if (ttl <= 0) { + throw new Error('Time-to-live (ttl) must be a positive number of seconds.'); + } + + // 2. Existence Check: Ensure the blob exists before trying to tag it. + if (!(await blockBlobClient.exists())) { + throw new Error(`Cannot set expiration for non-existent blob: "${resourceId}"`); + } + + // 3. Calculation: Convert TTL in seconds to the nearest whole day. + const ttlInDays = Math.ceil(ttl / 86400); + + // 4. Set Tags: Apply the tag that the Azure Lifecycle Management rule will act upon. + await blockBlobClient.setTags({ + deleteAfterDays: ttlInDays.toString() + }); + + } catch (error) { + // Catches any errors from the process and logs them with clear, actionable context. + console.error(`Failed to set expiration for blob "${resourceId}":`, error.message); + throw error; + } + } + + + /** + * Migrates legacy ACL metadata to the new structured format. + * + * This internal helper function checks for an outdated ACL format where permissions + * were stored in separate keys like `userid`, `teamid`, or `agentid`. If found, + * it converts them into the modern, structured `azure-acl` object, granting full + * ownership. This ensures seamless backward compatibility with older data. + * + * @param {Record} metadata - The raw metadata object which may contain legacy ACL keys. + * @returns {Record} The metadata object, with legacy keys converted into a new `azure-acl` property. + * @private + */ + private _migrateMetadata(metadata: Record): Record { + // Use object destructuring to cleanly separate legacy keys from the rest of the metadata. + const { agentid, userid, teamid, ...rest } = metadata; + + // Fast path: If no legacy keys are present, return the original object immediately. + if (!agentid && !userid && !teamid) { + return metadata; + } + + const aclHelper = new ACL(); + const legacyAcls: { [key: string]: string } = { agentid, userid, teamid }; + + // A map provides a clean, maintainable way to link legacy keys to system roles. + const roleMap: Record = { + agentid: TAccessRole.Agent, + userid: TAccessRole.User, + teamid: TAccessRole.Team, + }; + + for (const key in roleMap) { + const principalId = legacyAcls[key]; + if (principalId) { + const role = roleMap[key]; + // Grant full ownership to the legacy principal. + aclHelper.addAccess(role, principalId, [TAccessLevel.Owner, TAccessLevel.Read, TAccessLevel.Write]); + } + } + + // Record migration status in metadata rather than mutating ACL instance. + rest['acl_migrated'] = 'true'; + + // Reconstruct the metadata object, combining the remaining properties with the new, structured 'azure-acl'. + return { + ...rest, + 'azure-acl': aclHelper.ACL, + }; + } + + + /** + * Serializes the internal metadata object into a format compatible with Azure Blob Storage. + * + * This internal helper prepares the metadata for upload. It converts the `ACL` object + * into its serialized string representation and stringifies any other non-string + * metadata values. The `ContentType` property is intentionally excluded as it's + * handled separately in the blob's HTTP headers. + * + * @param {Record} metadata - The internal metadata object, containing mixed-type values. + * @returns {Record} A flat key-value object where all values are strings. + * @private + */ + private _serializeAzureMetadata(metadata: Record): Record { + // Use destructuring to separate special keys from the rest of the metadata. + // This avoids mutating the original metadata object (a best practice). + const { 'azure-acl': aclData, ContentType, ...rest } = metadata; + + const serialized: Record = {}; + + // Handle the ACL serialization separately. + if (aclData) { + serialized['azure-acl'] = typeof aclData === 'string' + ? aclData + : ACL.from(aclData).serializedACL; + } + + // Iterate over the remaining properties and stringify them if necessary. + for (const key in rest) { + const value = rest[key]; + serialized[key] = typeof value === 'string' ? value : JSON.stringify(value); + } + + return serialized; + } + + + /** + * Deserializes a flat metadata object from Azure into the rich internal format. + * + * This internal helper processes the raw string-based metadata retrieved from a blob. + * It reconstructs the `ACL` object from its string representation and parses any + * other JSON-stringified values back into their original object/data types. It then + * calls the migration helper to ensure backward compatibility with legacy ACL formats. + * + * @param {Record} metadata - The flat, string-only key-value metadata from the Azure SDK. + * @returns {Record} The rich internal metadata object with complex data types restored. + * @private + */ + private _deserializeAzureMetadata(metadata: Record): Record { + // Use Object.entries and reduce for a modern, functional approach to object transformation. + const deserialized = Object.entries(metadata).reduce((acc, [key, value]) => { + if (key === 'azure-acl') { + // Reconstruct the ACL object from its serialized string form. + acc[key] = ACL.from(value).ACL; + } else { + // Try to parse the value as JSON, falling back to the raw string if it's not valid JSON. + try { + acc[key] = JSON.parse(value); + } catch { + acc[key] = value; + } + } + return acc; + }, {} as Record); + + // Finally, run the result through the migration helper to handle any legacy formats. + return this._migrateMetadata(deserialized); + } + + + /** + * Private helper to fetch and deserialize metadata from Azure. + * @private + */ + private async _getAzureMetadata(resourceId: string): Promise | undefined> { + const blockBlobClient = this.containerClient.getBlockBlobClient(resourceId); + + try { + // getProperties() is an efficient HEAD request. + const properties = await blockBlobClient.getProperties(); + + const customMetadata = properties.metadata || {}; + const deserialized = this._deserializeAzureMetadata(customMetadata); + deserialized['ContentType'] = properties.contentType || 'application/octet-stream'; + return deserialized; + } catch (error) { + const status = (error as any)?.statusCode; + if (status === 404) { + return undefined; + } + // For all other errors, re-throw to be caught by the public-facing method. + throw error; + } + } + + + /** + * Private helper to serialize and write metadata to Azure. + * @private + */ + private async _setAzureMetadata(resourceId: string, metadata: Record): Promise { + const blockBlobClient = this.containerClient.getBlockBlobClient(resourceId); + try { + const serialized = this._serializeAzureMetadata(metadata); + // This is an efficient HEAD request to update metadata without touching blob content. + await blockBlobClient.setMetadata(serialized); + } catch (error) { + // Re-throw to be caught and logged by the public-facing method. + throw error; + } + } +} \ No newline at end of file diff --git a/packages/core/src/subsystems/IO/Storage.service/index.ts b/packages/core/src/subsystems/IO/Storage.service/index.ts index 87b4b5d8..ecdf757c 100644 --- a/packages/core/src/subsystems/IO/Storage.service/index.ts +++ b/packages/core/src/subsystems/IO/Storage.service/index.ts @@ -4,10 +4,12 @@ import { ConnectorService, ConnectorServiceProvider } from '@sre/Core/Connectors import { TConnectorService } from '@sre/types/SRE.types'; import { S3Storage } from './connectors/S3Storage.class'; import { LocalStorage } from './connectors/LocalStorage.class'; +import { AzureBlobStorage} from './connectors/AzureBlobStorage.class' export class StorageService extends ConnectorServiceProvider { public register() { ConnectorService.register(TConnectorService.Storage, 'S3', S3Storage); ConnectorService.register(TConnectorService.Storage, 'LocalStorage', LocalStorage); + ConnectorService.register(TConnectorService.Storage, 'AzureBlobStorage', AzureBlobStorage); } } diff --git a/packages/sdk/docs/07-services.md b/packages/sdk/docs/07-services.md index 80a2d348..7faedb07 100644 --- a/packages/sdk/docs/07-services.md +++ b/packages/sdk/docs/07-services.md @@ -43,7 +43,7 @@ An agent provides access to its configured storage providers. ```typescript // 'agent' is an existing Agent instance -const storage = agent.storage.LocalStorage(); // Or agent.storage.S3() etc. +const storage = agent.storage.LocalStorage(); // Or agent.storage.S3(), agent.storage.AzureBlobStorage() const fileUri = await storage.write('my-file.txt', 'This is some important data.'); console.log(`File written to: ${fileUri}`); const content = await storage.read(fileUri); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 97c01216..c296ba60 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -310,6 +310,9 @@ importers: specifier: ^1.1.2 version: 1.1.2 devDependencies: + '@azure/storage-blob': + specifier: ^12.28.0 + version: 12.28.0 '@istanbuljs/nyc-config-typescript': specifier: ^1.0.2 version: 1.0.2(nyc@17.1.0) @@ -732,6 +735,58 @@ packages: resolution: {integrity: sha512-DIIotRnefVL6DiaHtO6/21DhJ4JZnnIwdNbpwiAhdt/AVbttcE4yw925gsjur0OGv5BTYXQXU3YnANBYnZjuQA==} engines: {node: '>=18.0.0'} + '@azure/abort-controller@2.1.2': + resolution: {integrity: sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==} + engines: {node: '>=18.0.0'} + + '@azure/core-auth@1.10.0': + resolution: {integrity: sha512-88Djs5vBvGbHQHf5ZZcaoNHo6Y8BKZkt3cw2iuJIQzLEgH4Ox6Tm4hjFhbqOxyYsgIG/eJbFEHpxRIfEEWv5Ow==} + engines: {node: '>=20.0.0'} + + '@azure/core-client@1.10.0': + resolution: {integrity: sha512-O4aP3CLFNodg8eTHXECaH3B3CjicfzkxVtnrfLkOq0XNP7TIECGfHpK/C6vADZkWP75wzmdBnsIA8ksuJMk18g==} + engines: {node: '>=20.0.0'} + + '@azure/core-http-compat@2.3.0': + resolution: {integrity: sha512-qLQujmUypBBG0gxHd0j6/Jdmul6ttl24c8WGiLXIk7IHXdBlfoBqW27hyz3Xn6xbfdyVSarl1Ttbk0AwnZBYCw==} + engines: {node: '>=18.0.0'} + + '@azure/core-lro@2.7.2': + resolution: {integrity: sha512-0YIpccoX8m/k00O7mDDMdJpbr6mf1yWo2dfmxt5A8XVZVVMz2SSKaEbMCeJRvgQ0IaSlqhjT47p4hVIRRy90xw==} + engines: {node: '>=18.0.0'} + + '@azure/core-paging@1.6.2': + resolution: {integrity: sha512-YKWi9YuCU04B55h25cnOYZHxXYtEvQEbKST5vqRga7hWY9ydd3FZHdeQF8pyh+acWZvppw13M/LMGx0LABUVMA==} + engines: {node: '>=18.0.0'} + + '@azure/core-rest-pipeline@1.22.0': + resolution: {integrity: sha512-OKHmb3/Kpm06HypvB3g6Q3zJuvyXcpxDpCS1PnU8OV6AJgSFaee/covXBcPbWc6XDDxtEPlbi3EMQ6nUiPaQtw==} + engines: {node: '>=20.0.0'} + + '@azure/core-tracing@1.3.0': + resolution: {integrity: sha512-+XvmZLLWPe67WXNZo9Oc9CrPj/Tm8QnHR92fFAFdnbzwNdCH1h+7UdpaQgRSBsMY+oW1kHXNUZQLdZ1gHX3ROw==} + engines: {node: '>=20.0.0'} + + '@azure/core-util@1.13.0': + resolution: {integrity: sha512-o0psW8QWQ58fq3i24Q1K2XfS/jYTxr7O1HRcyUE9bV9NttLU+kYOH82Ixj8DGlMTOWgxm1Sss2QAfKK5UkSPxw==} + engines: {node: '>=20.0.0'} + + '@azure/core-xml@1.5.0': + resolution: {integrity: sha512-D/sdlJBMJfx7gqoj66PKVmhDDaU6TKA49ptcolxdas29X7AfvLTmfAGLjAcIMBK7UZ2o4lygHIqVckOlQU3xWw==} + engines: {node: '>=20.0.0'} + + '@azure/logger@1.3.0': + resolution: {integrity: sha512-fCqPIfOcLE+CGqGPd66c8bZpwAji98tZ4JI9i/mlTNTlsIWslCfpg48s/ypyLxZTump5sypjrKn2/kY7q8oAbA==} + engines: {node: '>=20.0.0'} + + '@azure/storage-blob@12.28.0': + resolution: {integrity: sha512-VhQHITXXO03SURhDiGuHhvc/k/sD2WvJUS7hqhiVNbErVCuQoLtWql7r97fleBlIRKHJaa9R7DpBjfE0pfLYcA==} + engines: {node: '>=20.0.0'} + + '@azure/storage-common@12.0.0': + resolution: {integrity: sha512-QyEWXgi4kdRo0wc1rHum9/KnaWZKCdQGZK1BjU4fFL6Jtedp7KLbQihgTTVxldFy1z1ZPtuDPx8mQ5l3huPPbA==} + engines: {node: '>=20.0.0'} + '@babel/code-frame@7.27.1': resolution: {integrity: sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==} engines: {node: '>=6.9.0'} @@ -2038,6 +2093,10 @@ packages: '@types/yauzl@2.10.3': resolution: {integrity: sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==} + '@typespec/ts-http-runtime@0.3.0': + resolution: {integrity: sha512-sOx1PKSuFwnIl7z4RN0Ls7N9AQawmR9r66eI5rFCzLDIs8HTIYrIpH9QjYWoX0lkgGrkLxXhi4QnK7MizPRrIg==} + engines: {node: '>=20.0.0'} + '@vitest/coverage-v8@3.2.2': resolution: {integrity: sha512-RVAi5xnqedSKvaoQyCTWvncMk8eYZcTTOsLK7XmnfOEvdGP/O/upA0/MA8Ss+Qs++mj0GcSRi/whR0S5iBPpTQ==} peerDependencies: @@ -2830,6 +2889,10 @@ packages: resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} engines: {node: '>=6'} + events@3.3.0: + resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} + engines: {node: '>=0.8.x'} + eventsource-parser@3.0.2: resolution: {integrity: sha512-6RxOBZ/cYgd8usLwsEl+EC09Au/9BcmCKYF2/xbml6DNczf7nv0MQb+7BA2F+li6//I+28VNlQR37XfQtcAJuA==} engines: {node: '>=18.0.0'} @@ -2897,6 +2960,10 @@ packages: resolution: {integrity: sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==} hasBin: true + fast-xml-parser@5.2.5: + resolution: {integrity: sha512-pfX9uG9Ki0yekDHx2SiuRIyFdyAr1kMIMitPvb0YBo8SUfKvia7w7FIyd/l6av85pFYRhZscS75MwMnbvY+hcQ==} + hasBin: true + fastest-levenshtein@1.0.16: resolution: {integrity: sha512-eRnCtTTtGZFpQCwhJiUOuxPQWRXVKYDn0b2PeHfXL6/Zi53SLAzAHfVhVWK2AryC/WH05kGfxhFIPvTF0SXQzg==} engines: {node: '>= 4.9.1'} @@ -3220,6 +3287,10 @@ packages: resolution: {integrity: sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==} engines: {node: '>= 0.8'} + http-proxy-agent@7.0.2: + resolution: {integrity: sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==} + engines: {node: '>= 14'} + http2-wrapper@2.2.1: resolution: {integrity: sha512-V5nVw1PAOgfI3Lmeaj2Exmeg7fenjhRUgz1lPSezy1CuhPYbgQtbQj4jZfEAEMlaL+vupsvhjqCyjzob0yxsmQ==} engines: {node: '>=10.19.0'} @@ -4676,6 +4747,9 @@ packages: strnum@1.1.2: resolution: {integrity: sha512-vrN+B7DBIoTTZjnPNewwhx6cBA/H+IS7rfW68n7XxC1y7uoiGQBxaKzqucGUgavX15dJgiGztLJ8vxuEzwqBdA==} + strnum@2.1.1: + resolution: {integrity: sha512-7ZvoFTiCnGxBtDqJ//Cu6fWtZtc7Y3x+QOirG15wztbdngGSkht27o2pyGWrVy0b4WAy3jbKmnoK6g5VlVNUUw==} + strtok3@9.1.1: resolution: {integrity: sha512-FhwotcEqjr241ZbjFzjlIYg6c5/L/s4yBGWSMvJ9UoExiSqL+FnFA/CaeZx17WGaZMS/4SOZp8wH18jSS4R4lw==} engines: {node: '>=16'} @@ -6367,6 +6441,120 @@ snapshots: '@smithy/types': 4.3.1 tslib: 2.8.1 + '@azure/abort-controller@2.1.2': + dependencies: + tslib: 2.8.1 + + '@azure/core-auth@1.10.0': + dependencies: + '@azure/abort-controller': 2.1.2 + '@azure/core-util': 1.13.0 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + + '@azure/core-client@1.10.0': + dependencies: + '@azure/abort-controller': 2.1.2 + '@azure/core-auth': 1.10.0 + '@azure/core-rest-pipeline': 1.22.0 + '@azure/core-tracing': 1.3.0 + '@azure/core-util': 1.13.0 + '@azure/logger': 1.3.0 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + + '@azure/core-http-compat@2.3.0': + dependencies: + '@azure/abort-controller': 2.1.2 + '@azure/core-client': 1.10.0 + '@azure/core-rest-pipeline': 1.22.0 + transitivePeerDependencies: + - supports-color + + '@azure/core-lro@2.7.2': + dependencies: + '@azure/abort-controller': 2.1.2 + '@azure/core-util': 1.13.0 + '@azure/logger': 1.3.0 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + + '@azure/core-paging@1.6.2': + dependencies: + tslib: 2.8.1 + + '@azure/core-rest-pipeline@1.22.0': + dependencies: + '@azure/abort-controller': 2.1.2 + '@azure/core-auth': 1.10.0 + '@azure/core-tracing': 1.3.0 + '@azure/core-util': 1.13.0 + '@azure/logger': 1.3.0 + '@typespec/ts-http-runtime': 0.3.0 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + + '@azure/core-tracing@1.3.0': + dependencies: + tslib: 2.8.1 + + '@azure/core-util@1.13.0': + dependencies: + '@azure/abort-controller': 2.1.2 + '@typespec/ts-http-runtime': 0.3.0 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + + '@azure/core-xml@1.5.0': + dependencies: + fast-xml-parser: 5.2.5 + tslib: 2.8.1 + + '@azure/logger@1.3.0': + dependencies: + '@typespec/ts-http-runtime': 0.3.0 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + + '@azure/storage-blob@12.28.0': + dependencies: + '@azure/abort-controller': 2.1.2 + '@azure/core-auth': 1.10.0 + '@azure/core-client': 1.10.0 + '@azure/core-http-compat': 2.3.0 + '@azure/core-lro': 2.7.2 + '@azure/core-paging': 1.6.2 + '@azure/core-rest-pipeline': 1.22.0 + '@azure/core-tracing': 1.3.0 + '@azure/core-util': 1.13.0 + '@azure/core-xml': 1.5.0 + '@azure/logger': 1.3.0 + '@azure/storage-common': 12.0.0 + events: 3.3.0 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + + '@azure/storage-common@12.0.0': + dependencies: + '@azure/abort-controller': 2.1.2 + '@azure/core-auth': 1.10.0 + '@azure/core-http-compat': 2.3.0 + '@azure/core-rest-pipeline': 1.22.0 + '@azure/core-tracing': 1.3.0 + '@azure/core-util': 1.13.0 + '@azure/logger': 1.3.0 + events: 3.3.0 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + '@babel/code-frame@7.27.1': dependencies: '@babel/helper-validator-identifier': 7.27.1 @@ -7861,6 +8049,14 @@ snapshots: '@types/node': 20.19.0 optional: true + '@typespec/ts-http-runtime@0.3.0': + dependencies: + http-proxy-agent: 7.0.2 + https-proxy-agent: 7.0.6 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + '@vitest/coverage-v8@3.2.2(vitest@3.2.3(@types/node@22.15.31)(jiti@2.4.2)(terser@5.41.0)(tsx@4.19.4)(yaml@2.8.0))': dependencies: '@ampproject/remapping': 2.3.0 @@ -8722,6 +8918,8 @@ snapshots: event-target-shim@5.0.1: {} + events@3.3.0: {} + eventsource-parser@3.0.2: {} eventsource-parser@3.0.3: {} @@ -8858,6 +9056,10 @@ snapshots: dependencies: strnum: 1.1.2 + fast-xml-parser@5.2.5: + dependencies: + strnum: 2.1.1 + fastest-levenshtein@1.0.16: {} fastq@1.19.1: @@ -9238,6 +9440,13 @@ snapshots: statuses: 2.0.1 toidentifier: 1.0.1 + http-proxy-agent@7.0.2: + dependencies: + agent-base: 7.1.3 + debug: 4.4.1(supports-color@8.1.1) + transitivePeerDependencies: + - supports-color + http2-wrapper@2.2.1: dependencies: quick-lru: 5.1.1 @@ -10827,6 +11036,8 @@ snapshots: strnum@1.1.2: {} + strnum@2.1.1: {} + strtok3@9.1.1: dependencies: '@tokenizer/token': 0.3.0