DataLakeServiceClient.ts

// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

import "@azure/core-paging";

import { getDefaultProxySettings, isNode, TokenCredential } from "@azure/core-http";
import { PagedAsyncIterableIterator } from "@azure/core-paging";
import { BlobServiceClient } from "@azure/storage-blob";

import { AnonymousCredential } from "./credentials/AnonymousCredential";
import { StorageSharedKeyCredential } from "./credentials/StorageSharedKeyCredential";
import { DataLakeFileSystemClient } from "./DataLakeFileSystemClient";
import {
  FileSystemItem,
  FileSystemRenameResponse,
  ServiceGenerateAccountSasUrlOptions,
  ServiceListFileSystemsOptions,
  ServiceListFileSystemsSegmentResponse,
  ServiceRenameFileSystemOptions,
  ServiceUndeleteFileSystemOptions,
  FileSystemUndeleteResponse
} from "./models";
import { Pipeline, StoragePipelineOptions, newPipeline } from "./Pipeline";
import { StorageClient } from "./StorageClient";
import {
  appendToURLPath,
  appendToURLQuery,
  extractConnectionStringParts
} from "./utils/utils.common";
import { createSpan } from "./utils/tracing";
import { toDfsEndpointUrl, toFileSystemPagedAsyncIterableIterator } from "./transforms";
import { ServiceGetUserDelegationKeyOptions, ServiceGetUserDelegationKeyResponse } from "./models";
import { CanonicalCode } from "@opentelemetry/api";
import { AccountSASPermissions } from "./sas/AccountSASPermissions";
import { generateAccountSASQueryParameters } from "./sas/AccountSASSignatureValues";
import { AccountSASServices } from "./sas/AccountSASServices";

/**
 * DataLakeServiceClient allows you to manipulate Azure
 * Data Lake service resources and file systems. The storage account provides
 * the top-level namespace for the Data Lake service.
 *
 * @export
 * @class DataLakeServiceClient
 * @extends {StorageClient}
 */
export class DataLakeServiceClient extends StorageClient {
  // private serviceContext: Service;

  /**
   * blobServiceClient provided by @azure/storage-blob package.
   *
   * @private
   * @type {BlobServiceClient}
   * @memberof DataLakeServiceClient
   */
  private blobServiceClient: BlobServiceClient;

  /**
   *
   * Creates an instance of DataLakeServiceClient from connection string.
   *
   * @param {string} connectionString Account connection string or a SAS connection string of an Azure storage account.
   *                                  [ Note - Account connection string can only be used in NODE.JS runtime. ]
   *                                  Account connection string example -
   *                                  `DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=accountKey;EndpointSuffix=core.windows.net`
   *                                  SAS connection string example -
   *                                  `BlobEndpoint=https://myaccount.blob.core.windows.net/;QueueEndpoint=https://myaccount.queue.core.windows.net/;FileEndpoint=https://myaccount.file.core.windows.net/;TableEndpoint=https://myaccount.table.core.windows.net/;SharedAccessSignature=sasString`
   * @param {StoragePipelineOptions} [options] Optional. Options to configure the HTTP pipeline.
   * @memberof DataLakeServiceClient
   */
  public static fromConnectionString(connectionString: string, options?: StoragePipelineOptions) {
    options = options || {};
    const extractedCreds = extractConnectionStringParts(connectionString);
    if (extractedCreds.kind === "AccountConnString") {
      if (isNode) {
        const sharedKeyCredential = new StorageSharedKeyCredential(
          extractedCreds.accountName!,
          extractedCreds.accountKey
        );
        options.proxyOptions = getDefaultProxySettings(extractedCreds.proxyUri);
        const pipeline = newPipeline(sharedKeyCredential, options);
        return new DataLakeServiceClient(toDfsEndpointUrl(extractedCreds.url), pipeline);
      } else {
        throw new Error("Account connection string is only supported in Node.js environment");
      }
    } else if (extractedCreds.kind === "SASConnString") {
      const pipeline = newPipeline(new AnonymousCredential(), options);
      return new DataLakeServiceClient(
        toDfsEndpointUrl(extractedCreds.url) + "?" + extractedCreds.accountSas,
        pipeline
      );
    } else {
      throw new Error(
        "Connection string must be either an Account connection string or a SAS connection string"
      );
    }
  }

  /**
   * Creates an instance of DataLakeServiceClient from url.
   *
   * @param {string} url A Client string pointing to Azure Storage data lake service, such as
   *                     "https://myaccount.dfs.core.windows.net". You can append a SAS
   *                     if using AnonymousCredential, such as "https://myaccount.dfs.core.windows.net?sasString".
   * @param {(StorageSharedKeyCredential | AnonymousCredential | TokenCredential)} [credential] Such as AnonymousCredential, StorageSharedKeyCredential or any credential from the @azure/identity package to authenticate requests to the service. You can also provide an object that implements the TokenCredential interface. If not specified, AnonymousCredential is used.
   * @param {StoragePipelineOptions} [options] Optional. Options to configure the HTTP pipeline.
   * @memberof DataLakeServiceClient
   */
  public constructor(
    url: string,
    credential?: StorageSharedKeyCredential | AnonymousCredential | TokenCredential,
    options?: StoragePipelineOptions
  );

  /**
   * Creates an instance of DataLakeServiceClient from url and pipeline.
   *
   * @param {string} url A Client string pointing to Azure Storage data lake service, such as
   *                     "https://myaccount.dfs.core.windows.net". You can append a SAS
   *                     if using AnonymousCredential, such as "https://myaccount.dfs.core.windows.net?sasString".
   * @param {Pipeline} pipeline Call newPipeline() to create a default
   *                            pipeline, or provide a customized pipeline.
   * @memberof DataLakeServiceClient
   */
  public constructor(url: string, pipeline: Pipeline);

  public constructor(
    url: string,
    credentialOrPipeline?:
      | StorageSharedKeyCredential
      | AnonymousCredential
      | TokenCredential
      | Pipeline,
    options?: StoragePipelineOptions
  ) {
    if (credentialOrPipeline instanceof Pipeline) {
      super(url, credentialOrPipeline);
    } else {
      let credential;
      if (credentialOrPipeline === undefined) {
        credential = new AnonymousCredential();
      } else {
        credential = credentialOrPipeline;
      }

      const pipeline = newPipeline(credential, options);
      super(url, pipeline);
    }

    // this.serviceContext = new Service(this.storageClientContext);
    this.blobServiceClient = new BlobServiceClient(this.blobEndpointUrl, this.pipeline);
  }

  /**
   * Creates a {@link DataLakeFileSystemClient} object.
   *
   * @param {string} fileSystemName File system name.
   * @returns {DataLakeFileSystemClient}
   * @memberof DataLakeServiceClient
   */
  public getFileSystemClient(fileSystemName: string): DataLakeFileSystemClient {
    return new DataLakeFileSystemClient(
      appendToURLPath(this.url, encodeURIComponent(fileSystemName)),
      this.pipeline
    );
  }

  /**
   * ONLY AVAILABLE WHEN USING BEARER TOKEN AUTHENTICATION (TokenCredential).
   *
   * Retrieves a user delegation key for the Data Lake service. This is only a valid operation when using
   * bearer token authentication.
   *
   * @example
   * ```js
   * // Generate user delegation SAS for a file system
   * const userDelegationKey = await dataLakeServiceClient.getUserDelegationKey(startsOn, expiresOn);
   * const fileSystemSAS = generateDataLakeSASQueryParameters({
   *     fileSystemName, // Required
   *     permissions: FileSystemSASPermissions.parse("racwdl"), // Required
   *     startsOn, // Required. Date type
   *     expiresOn, // Optional. Date type
   *     ipRange: { start: "0.0.0.0", end: "255.255.255.255" }, // Optional
   *     protocol: SASProtocol.HttpsAndHttp, // Optional
   *     version: "2018-11-09" // Must >= 2018-11-09 to generate user delegation SAS
   *   },
   *   userDelegationKey, // UserDelegationKey
   *   accountName
   * ).toString();
   * ```
   * @see https://docs.microsoft.com/en-us/rest/api/storageservices/get-user-delegation-key
   *
   * @param {Date} startsOn The start time for the user delegation SAS. Must be within 7 days of the current time.
   * @param {Date} expiresOn The end time for the user delegation SAS. Must be within 7 days of the current time.
   * @param {ServiceGetUserDelegationKeyOptions} [options={}]
   * @returns {Promise<ServiceGetUserDelegationKeyResponse>}
   * @memberof DataLakeServiceClient
   */
  public async getUserDelegationKey(
    startsOn: Date,
    expiresOn: Date,
    options: ServiceGetUserDelegationKeyOptions = {}
  ): Promise<ServiceGetUserDelegationKeyResponse> {
    const { span, spanOptions } = createSpan(
      "DataLakeServiceClient-getUserDelegationKey",
      options.tracingOptions
    );
    try {
      return await this.blobServiceClient.getUserDelegationKey(startsOn, expiresOn, {
        ...options,
        tracingOptions: {
          ...options.tracingOptions,
          spanOptions
        }
      });
    } catch (e) {
      span.setStatus({
        code: CanonicalCode.UNKNOWN,
        message: e.message
      });
      throw e;
    } finally {
      span.end();
    }
  }

  /**
   * Returns an async iterable iterator to list all the file systems
   * under the specified account.
   *
   * .byPage() returns an async iterable iterator to list the file systems in pages.
   *
   * Example using `for await` syntax:
   *
   * ```js
   * let i = 1;
   * for await (const fileSystem of serviceClient.listFileSystems()) {
   *   console.log(`FileSystem ${i++}: ${fileSystem.name}`);
   * }
   * ```
   *
   * Example using `iter.next()`:
   *
   * ```js
   * let i = 1;
   * const iter = serviceClient.listFileSystems();
   * let fileSystemItem = await iter.next();
   * while (!fileSystemItem.done) {
   *   console.log(`FileSystem ${i++}: ${fileSystemItem.value.name}`);
   *   fileSystemItem = await iter.next();
   * }
   * ```
   *
   * Example using `byPage()`:
   *
   * ```js
   * // passing optional maxPageSize in the page settings
   * let i = 1;
   * for await (const response of serviceClient.listFileSystems().byPage({ maxPageSize: 20 })) {
   *   if (response.fileSystemItems) {
   *     for (const fileSystem of response.fileSystemItems) {
   *       console.log(`FileSystem ${i++}: ${fileSystem.name}`);
   *     }
   *   }
   * }
   * ```
   *
   * Example using paging with a marker:
   *
   * ```js
   * let i = 1;
   * let iterator = serviceClient.listFileSystems().byPage({ maxPageSize: 2 });
   * let response = (await iterator.next()).value;
   *
   * // Prints 2 file system names
   * if (response.fileSystemItems) {
   *   for (const fileSystem of response.fileSystemItems) {
   *     console.log(`FileSystem ${i++}: ${fileSystem.name}`);
   *   }
   * }
   *
   * // Gets next marker
   * let marker = response.continuationToken;
   * // Passing next marker as continuationToken
   * iterator = serviceClient
   *   .listContainers()
   *   .byPage({ continuationToken: marker, maxPageSize: 10 });
   * response = (await iterator.next()).value;
   *
   * // Prints 10 file system names
   * if (response.fileSystemItems) {
   *   for (const fileSystem of response.fileSystemItems) {
   *      console.log(`FileSystem ${i++}: ${fileSystem.name}`);
   *   }
   * }
   * ```
   *
   * @see https://docs.microsoft.com/en-us/rest/api/storageservices/list-containers2
   *
   * @param {ServiceListFileSystemsOptions} [options={}]
   * @returns {PagedAsyncIterableIterator<FileSystemItem, ServiceListFileSystemsSegmentResponse>}
   * @memberof DataLakeServiceClient
   */
  public listFileSystems(
    options: ServiceListFileSystemsOptions = {}
  ): PagedAsyncIterableIterator<FileSystemItem, ServiceListFileSystemsSegmentResponse> {
    return toFileSystemPagedAsyncIterableIterator(this.blobServiceClient.listContainers(options));
  }

  // public async createFileSystem(): Promise<DataLakeFileSystemClient> {
  //   throw Error("NotImplemented");
  // }

  // public async deleteFileSystem(fileSystem: string): Promise<ServiceDeleteFileSystemResponse> {
  //   throw Error("NotImplemented");
  // }

  /**
   * Only available for DataLakeServiceClient constructed with a shared key credential.
   *
   * Generates an account Shared Access Signature (SAS) URI based on the client properties
   * and parameters passed in. The SAS is signed by the shared key credential of the client.
   *
   * @see https://docs.microsoft.com/en-us/rest/api/storageservices/create-account-sas
   *
   * @param {Date} expiresOn Optional. The time at which the shared access signature becomes invalid. Default to an hour later if not specified.
   * @param {AccountSASPermissions} [permissions=AccountSASPermissions.parse("r")] Specifies the list of permissions to be associated with the SAS.
   * @param {string} [resourceTypes="sco"] Specifies the resource types associated with the shared access signature.
   * @param {ServiceGenerateAccountSasUrlOptions} [options={}] Optional parameters.
   * @returns {string} An account SAS URI consisting of the URI to the resource represented by this client, followed by the generated SAS token.
   * @memberof DataLakeServiceClient
   */
  public generateAccountSasUrl(
    expiresOn?: Date,
    permissions: AccountSASPermissions = AccountSASPermissions.parse("r"),
    resourceTypes: string = "sco",
    options: ServiceGenerateAccountSasUrlOptions = {}
  ): string {
    if (!(this.credential instanceof StorageSharedKeyCredential)) {
      throw RangeError(
        "Can only generate the account SAS when the client is initialized with a shared key credential"
      );
    }

    if (expiresOn === undefined) {
      const now = new Date();
      expiresOn = new Date(now.getTime() + 3600 * 1000);
    }

    const sas = generateAccountSASQueryParameters(
      {
        permissions,
        expiresOn,
        resourceTypes,
        services: AccountSASServices.parse("b").toString(),
        ...options
      },
      this.credential
    ).toString();

    return appendToURLQuery(this.url, sas);
  }

  /**
   * Renames an existing File System.
   *
   * @param {string} sourceFileSystemName The name of the source File System.
   * @param {string} destinationContainerName The new name of the File System.
   * @param {ServiceRenameFileSystemOptions} [options] Options to configure File System Rename operation.
   * @memberof DataLakeServiceClient
   */
  // @ts-ignore Need to hide this interface for now. Make it public and turn on the live tests for it when the service is ready.
  private async renameFileSystem(
    sourceFileSystemName: string,
    destinationFileSystemName: string,
    options: ServiceRenameFileSystemOptions = {}
  ): Promise<{
    fileSystemClient: DataLakeFileSystemClient;
    fileSystemRenameResponse: FileSystemRenameResponse;
  }> {
    const { span, spanOptions } = createSpan(
      "DataLakeServiceClient-renameFileSystem",
      options.tracingOptions
    );
    try {
      // const res = await this.blobServiceClient.renameContainer(
      const res = await this.blobServiceClient["renameContainer"](
        sourceFileSystemName,
        destinationFileSystemName,
        {
          ...options,
          tracingOptions: { ...options.tracingOptions, spanOptions }
        }
      );

      const fileSystemClient = this.getFileSystemClient(destinationFileSystemName);
      return {
        fileSystemClient,
        fileSystemRenameResponse: res.containerRenameResponse
      };
    } catch (e) {
      span.setStatus({
        code: CanonicalCode.UNKNOWN,
        message: e.message
      });
      throw e;
    } finally {
      span.end();
    }
  }

  /**
   * Restore a previously deleted File System.
   * This API is only functional if Container Soft Delete is enabled for the storage account.
   *
   * @param {string} deletedFileSystemName The name of the source File System.
   * @param {string} deleteFileSystemVersion The new name of the File System.
   * @param {ServiceRenameFileSystemOptions} [options] Options to configure File System Restore operation.
   * @memberof DataLakeServiceClient
   */
  public async undeleteFileSystem(
    deletedFileSystemName: string,
    deleteFileSystemVersion: string,
    options: ServiceUndeleteFileSystemOptions = {}
  ): Promise<{
    fileSystemClient: DataLakeFileSystemClient;
    fileSystemUndeleteResponse: FileSystemUndeleteResponse;
  }> {
    const { span, spanOptions } = createSpan(
      "DataLakeServiceClient-undeleteFileSystem",
      options.tracingOptions
    );
    try {
      const res = await this.blobServiceClient.undeleteContainer(
        deletedFileSystemName,
        deleteFileSystemVersion,
        {
          ...options,
          destinationContainerName: options.destinationFileSystemName,
          tracingOptions: { ...options.tracingOptions, spanOptions }
        }
      );

      const fileSystemClient = this.getFileSystemClient(
        options.destinationFileSystemName || deletedFileSystemName
      );
      return {
        fileSystemClient,
        fileSystemUndeleteResponse: res.containerUndeleteResponse
      };
    } catch (e) {
      span.setStatus({
        code: CanonicalCode.UNKNOWN,
        message: e.message
      });
      throw e;
    } finally {
      span.end();
    }
  }
}