Work on fixing up hacks around the NETStandard SDK not having sync me…

…thods and converting them to proper Async methods.

FSharp.Azure.StorageTypeProvider.sln

@@ -1,6 +1,6 @@
 Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio 14
-VisualStudioVersion = 14.0.23107.0
+# Visual Studio 15
+VisualStudioVersion = 15.0.28010.2046
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".paket", ".paket", "{94066CAA-4417-4C43-A286-40182E920E43}"
 	ProjectSection(SolutionItems) = preProject
@@ -16,7 +16,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 		RELEASE_NOTES.md = RELEASE_NOTES.md
 	EndProjectSection
 EndProject
-Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "FSharp.Azure.StorageTypeProvider", "src\FSharp.Azure.StorageTypeProvider\FSharp.Azure.StorageTypeProvider.fsproj", "{FB7CA8F3-C158-49E9-B816-501741E2921F}"
+Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Azure.StorageTypeProvider", "src\FSharp.Azure.StorageTypeProvider\FSharp.Azure.StorageTypeProvider.fsproj", "{FB7CA8F3-C158-49E9-B816-501741E2921F}"
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "docs", "docs", "{7CAF8D38-6250-41CB-A65B-ABD760B09281}"
 	ProjectSection(SolutionItems) = preProject
@@ -52,4 +52,7 @@ Global
 	GlobalSection(NestedProjects) = preSolution
 		{15350981-90B9-44A0-B356-776735ACC7F1} = {7CAF8D38-6250-41CB-A65B-ABD760B09281}
 	EndGlobalSection
-EndGlobal
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {C68560AB-A7F2-49A3-BEED-B56A8D7074D2}
+	EndGlobalSection
+EndGlobal

paket.dependencies

@@ -5,7 +5,7 @@ framework: netstandard2.0, netcoreapp2.0
 nuget FAKE
 nuget Deedle
 nuget Expecto
-nuget WindowsAzure.Storage 9.1.1
+nuget WindowsAzure.Storage
 nuget FSharp.Charting
 nuget FSharp.Formatting
 nuget FSharp.Core

paket.lock

@@ -692,7 +692,7 @@ NUGET
       System.ValueTuple (>= 4.4)
     Unquote (4.0)
       FSharp.Core (>= 4.2.3)
-    WindowsAzure.Storage (9.1.1)
+    WindowsAzure.Storage (9.3.2)
       NETStandard.Library (>= 1.6.1)
       Newtonsoft.Json (>= 10.0.2)
 GITHUB

src/FSharp.Azure.StorageTypeProvider/AzureTypeProvider.fs

@@ -69,8 +69,8 @@ type public AzureTypeProvider(config : TypeProviderConfig) as this =
         let parsedTableSchema = Table.StaticSchema.createSchema config.ResolutionFolder staticTableSchema
 
         match connectionStringValidation, parsedBlobSchema, parsedTableSchema with
-        | Some (Success _), Success blobSchema, Success tableSchema
-        | None, Success blobSchema, Success tableSchema ->
+        | Some (Ok ()), Ok blobSchema, Ok tableSchema
+        | None, Ok blobSchema, Ok tableSchema ->
             let domainTypes = ProvidedTypeDefinition("Domain", Some typeof<obj>)            
             typeProviderForAccount.AddMember(domainTypes)
 
@@ -86,9 +86,9 @@ type public AzureTypeProvider(config : TypeProviderConfig) as this =
                     try builder(connectionString, domainTypes)
                     with ex -> failwithf "An error occurred during initial type generation for %s: %O" name ex))
             typeProviderForAccount
-        | Some (Failure ex), _, _ -> failwithf "Unable to validate connection string (%O)" ex
-        | _, Failure ex, _ -> failwithf "Unable to parse blob schema file (%O)" ex
-        | _, _, Failure ex -> failwithf "Unable to parse table schema file (%O)" ex
+        | Some (Error ex), _, _ -> failwithf "Unable to validate connection string (%O)" ex
+        | _, Error ex, _ -> failwithf "Unable to parse blob schema file (%O)" ex
+        | _, _, Error ex -> failwithf "Unable to parse table schema file (%O)" ex
 
     let createParam (name, defaultValue:'a, help) =
         let providedParameter = ProvidedStaticParameter(name, typeof<'a>, defaultValue)

src/FSharp.Azure.StorageTypeProvider/Blob/BlobMemberFactory.fs

@@ -5,16 +5,18 @@ open FSharp.Azure.StorageTypeProvider.Blob.BlobRepository
 open ProviderImplementation.ProvidedTypes
 open System
 open Microsoft.WindowsAzure.Storage.Blob
+open FSharp.Control.Tasks
 
 let rec private createBlobItem (domainType : ProvidedTypeDefinition) connectionString containerName fileItem = 
     match fileItem with
     | Folder(path, name, contents) ->
         let folderProp = ProvidedTypeDefinition((sprintf "%s.%s" containerName path), Some typeof<BlobFolder>, hideObjectMethods = true)
         domainType.AddMember(folderProp)
-        folderProp.AddMembersDelayed(fun _ -> 
-            (contents.Value
-             |> Array.choose (createBlobItem domainType connectionString containerName)
-             |> Array.toList))
+        folderProp.AddMembersDelayed(fun _ ->
+            contents
+            |> Async.RunSynchronously
+            |> Array.choose (createBlobItem domainType connectionString containerName)
+            |> Array.toList)
         Some <| ProvidedProperty(name, folderProp, getterCode = fun _ -> <@@ ContainerBuilder.createBlobFolder connectionString containerName path @@>)
     | Blob(path, name, blobType, length) -> 
         match blobType, length with
@@ -32,10 +34,11 @@ let private createContainerType (domainType : ProvidedTypeDefinition) connection
     let individualContainerType = ProvidedTypeDefinition(container.Name + "Container", Some typeof<BlobContainer>, hideObjectMethods = true)
     individualContainerType.AddXmlDoc <| sprintf "Provides access to the '%s' container." container.Name
     individualContainerType.AddMembersDelayed(fun _ -> 
-        (container.Contents.Value
-         |> Seq.choose (createBlobItem domainType connectionString container.Name)
-         |> Seq.toList))
-    domainType.AddMember(individualContainerType)
+        container.Contents
+        |> Async.RunSynchronously
+        |> Array.choose (createBlobItem domainType connectionString container.Name)
+        |> Array.toList)
+    domainType.AddMember individualContainerType
     // this local binding is required for the quotation.
     let containerName = container.Name
     let containerProp = 
@@ -49,8 +52,15 @@ let getBlobStorageMembers staticSchema (connectionString, domainType : ProvidedT
     let createContainerType = createContainerType domainType connectionString
 
     match staticSchema with
-    | [] -> containerListingType.AddMembersDelayed(fun _ -> connectionString |> getBlobStorageAccountManifest |> List.map createContainerType)
-    | staticSchema -> staticSchema |> List.map createContainerType |> containerListingType.AddMembers
+    | [] -> containerListingType.AddMembersDelayed(fun _ ->
+        getBlobStorageAccountManifest connectionString
+        |> Async.RunSynchronously
+        |> Array.map createContainerType
+        |> Array.toList)
+    | staticSchema ->
+        staticSchema
+        |> List.map createContainerType
+        |> containerListingType.AddMembers
 
     domainType.AddMember containerListingType
 

src/FSharp.Azure.StorageTypeProvider/Blob/BlobRepository.fs

@@ -1,18 +1,19 @@
 ///Contains reusable helper functions for accessing blobs
-module internal FSharp.Azure.StorageTypeProvider.Blob.BlobRepository
+module FSharp.Azure.StorageTypeProvider.Blob.BlobRepository
 
+open FSharp.Azure.StorageTypeProvider
 open Microsoft.WindowsAzure.Storage
 open Microsoft.WindowsAzure.Storage.Blob
 open System
 open System.IO
 
 type ContainerItem = 
-    | Folder of path : string * name : string * contents : ContainerItem array Lazy
+    | Folder of path : string * name : string * contents : ContainerItem array Async
     | Blob of path : string * name : string * blobType : BlobType * length : int64 option
 
 type LightweightContainer = 
     { Name : string
-      Contents : ContainerItem seq Lazy }
+      Contents : ContainerItem array Async }
 
 let getBlobClient connection = CloudStorageAccount.Parse(connection).CreateCloudBlobClient()
 let getContainerRef(connection, container) = (getBlobClient connection).GetContainerReference(container)
@@ -25,59 +26,79 @@ let private getItemName (item : string) (parent : CloudBlobDirectory) =
     | null -> item
     | parent -> item.Substring(parent.Prefix.Length)
 
-let rec private getContainerStructure wildcard (container : CloudBlobContainer) =
-    container.ListBlobsSegmentedAsync(prefix = wildcard, currentToken = null).Result
-    |> fun s -> s.Results
-    |> Seq.distinctBy (fun b -> b.Uri.AbsoluteUri)
-    |> Seq.choose (function
-       | :? CloudBlobDirectory as directory -> 
-           let path, name = getItemName directory.Prefix directory.Parent
-           Some(Folder(path, name, lazy(container |> getContainerStructure directory.Prefix)))
-       | :? ICloudBlob as blob ->
-           let path, name = getItemName blob.Name blob.Parent
-           Some(Blob(path, name, blob.BlobType, Some blob.Properties.Length))
-       | _ -> None)
-    |> Seq.toArray
+[<AutoOpen>]
+module private SdkExtensions =
+    type CloudBlobClient with
+        member blobClient.ListContainersAsync() =
+            let listContainers token = async {
+                let! results = blobClient.ListContainersSegmentedAsync token |> Async.AwaitTask
+                return results.ContinuationToken, results.Results }
+            Async.segmentedAzureOperation listContainers
 
-let listBlobs incSubDirs (container:CloudBlobContainer) prefix = 
-    container.ListBlobsSegmentedAsync(prefix = prefix, useFlatBlobListing = incSubDirs, blobListingDetails = BlobListingDetails.All, maxResults = Nullable(), currentToken = null, options = BlobRequestOptions(), operationContext = null).Result
-    |> fun s -> s.Results
-    |> Seq.choose(function
-        | :? ICloudBlob as blob -> 
-            let path, name = getItemName blob.Name blob.Parent
-            Some(Blob(path, name, blob.Properties.BlobType, Some blob.Properties.Length))
-        | _ -> None)    //can safely ignore folder types as we have a flat structure if & only if we want to include items from sub directories
+    type CloudBlobContainer with
+        member container.ListBlobsAsync incSubDirs prefix =
+            let listBlobs token = async {
+                let! results = container.ListBlobsSegmentedAsync(prefix = prefix, useFlatBlobListing = incSubDirs, blobListingDetails = BlobListingDetails.None, maxResults = Nullable(), currentToken = token, options = BlobRequestOptions(), operationContext = null) |> Async.AwaitTask
+                return results.ContinuationToken, results.Results }
+            Async.segmentedAzureOperation listBlobs
 
-let getBlobStorageAccountManifest connection = 
-    (getBlobClient connection).ListContainersSegmentedAsync(null).Result
-    |> fun s -> s.Results
-    |> Seq.toList
-    |> List.map (fun container -> 
-        { Name = container.Name
-          Contents =
-            lazy
-                container
-                |> getContainerStructure null
-                |> Seq.cache })
+let listBlobs incSubDirs (container:CloudBlobContainer) prefix = async {
+    let! results = container.ListBlobsAsync incSubDirs prefix
 
-let downloadFolder (connectionDetails, path) =
+    //can safely ignore folder types as we have a flat structure if & only if we want to include items from sub directories
+    return
+        [| for result in results do
+               match result with
+               | :? ICloudBlob as blob -> 
+                   let path, name = getItemName blob.Name blob.Parent
+                   yield Blob(path, name, blob.Properties.BlobType, Some blob.Properties.Length)
+               | _ -> () |] }
+
+let getBlobStorageAccountManifest (connectionString:string) =
+    let rec getContainerStructureAsync prefix (container:CloudBlobContainer) = async {
+        let! blobs = container.ListBlobsAsync false prefix
+        let blobs = blobs |> Array.distinctBy (fun b -> b.Uri.AbsoluteUri)
+        return
+            [| for blob in blobs do
+                   match blob with
+                   | :? CloudBlobDirectory as directory -> 
+                       let path, name = getItemName directory.Prefix directory.Parent
+                       yield Folder(path, name, container |> getContainerStructureAsync directory.Prefix)
+                   | :? ICloudBlob as blob ->
+                       let path, name = getItemName blob.Name blob.Parent
+                       yield Blob(path, name, blob.Properties.BlobType, Some blob.Properties.Length)
+                   | _ -> () |] }
+
+    async {
+        let client = (CloudStorageAccount.Parse connectionString).CreateCloudBlobClient()
+        let! containers = client.ListContainersAsync() 
+        return!
+            containers
+            |> Array.map (fun container -> async {
+                let structure = container |> getContainerStructureAsync null
+                return { Name = container.Name; Contents = structure } })
+            |> Async.Parallel }
+
+let downloadFolder (connectionDetails, path) = async {
     let downloadFile (blobRef:ICloudBlob) destination =
         let targetDirectory = Path.GetDirectoryName(destination)
         if not (Directory.Exists targetDirectory) then Directory.CreateDirectory targetDirectory |> ignore
         blobRef.DownloadToFileAsync(destination, FileMode.Create) |> Async.AwaitTask
 
     let connection, container, folderPath = connectionDetails
     let containerRef = (getBlobClient connection).GetContainerReference(container)
-    containerRef.ListBlobsSegmentedAsync(prefix = folderPath, useFlatBlobListing = true, blobListingDetails = BlobListingDetails.All, maxResults = Nullable(), currentToken = null, options = BlobRequestOptions(), operationContext = null).Result
-    |> fun s -> s.Results
-    |> Seq.choose (function
-        | :? ICloudBlob as b -> Some b
-        | _ -> None)
-    |> Seq.map (fun blob -> 
-        let targetName = 
-            match folderPath with
-            | folderPath when String.IsNullOrEmpty folderPath -> blob.Name
-            | _ -> blob.Name.Replace(folderPath, String.Empty)
-        downloadFile blob (Path.Combine(path, targetName)))
-    |> Async.Parallel
-    |> Async.Ignore
+    let! blobs = containerRef.ListBlobsAsync true folderPath
+
+    return!
+        blobs
+        |> Array.choose (function
+            | :? ICloudBlob as b -> Some b
+            | _ -> None)
+        |> Array.map (fun blob -> 
+            let targetName = 
+                match folderPath with
+                | folderPath when String.IsNullOrEmpty folderPath -> blob.Name
+                | _ -> blob.Name.Replace(folderPath, String.Empty)
+            downloadFile blob (Path.Combine(path, targetName)))
+        |> Async.Parallel
+        |> Async.Ignore }

src/FSharp.Azure.StorageTypeProvider/Blob/ProvidedBlobTypes.fs

@@ -164,19 +164,22 @@ module BlobBuilder =
                 | _ -> false
                 -> return None }
 
-    let listBlobs defaultConnectionString container file includeSubfolders prefix connectionString =
+    let listBlobs defaultConnectionString container file includeSubfolders prefix connectionString = async {
         let connectionString = connectionString |> defaultArg <| defaultConnectionString
         let includeSubfolders = includeSubfolders |> defaultArg <| false
         let container = getContainerRef (connectionString, container)
         let prefix = file + (prefix |> Option.toObj)
-        listBlobs includeSubfolders container prefix
-        |> Seq.choose (function
-            | Blob(path, _, blobType, _) -> 
-                match blobType with
-                | BlobType.PageBlob -> (createPageBlobFile connectionString container.Name path) :> BlobFile 
-                | _ -> (createBlockBlobFile connectionString container.Name path) :> BlobFile 
-                |> Some
-            | _ -> None)
+        let! blobs = listBlobs includeSubfolders container prefix
+
+        return
+            blobs
+            |> Array.choose (function
+                | Blob(path, _, blobType, _) -> 
+                    match blobType with
+                    | BlobType.PageBlob -> (createPageBlobFile connectionString container.Name path) :> BlobFile 
+                    | _ -> (createBlockBlobFile connectionString container.Name path) :> BlobFile 
+                    |> Some
+                | _ -> None) }
 
 /// Represents a pseudo-folder in blob storage.
 type BlobFolder internal (defaultConnectionString, container, file) = 

src/FSharp.Azure.StorageTypeProvider/Blob/StaticSchema.fs

@@ -23,33 +23,34 @@ let rec buildBlobItem prevPath (elementName, contents) =
         Blob (prevPath + elementName, elementName, blobType, None)
     | Folder, Json.ObjectOrNull children ->
         let path = prevPath + elementName
-        Folder (path, elementName, lazy (children |> Array.map (buildBlobItem path)))
+        Folder (path, elementName, async { return children |> Array.map (buildBlobItem path) })
     | _ -> failInvalidJson()
 
 let buildBlobSchema (json:Json.Json) =
     json.AsObject
     |> Array.map (fun (containerName, containerElements) ->
         { Name = containerName
           Contents =
-              lazy
-                  match containerElements with
-                  | Json.ObjectOrNull elements -> elements |> Seq.map (buildBlobItem "")
-                  | _ -> failInvalidJson() })
+            async {
+                return
+                    match containerElements with
+                    | Json.ObjectOrNull elements -> elements |> Array.map (buildBlobItem "")
+                    | _ -> failInvalidJson() } } )
     |> Array.toList
 
 let createSchema resolutionFolder path =
     path
     |> Option.map(fun path ->
         let paths = [ path; Path.Combine(resolutionFolder, path) ]
         match paths |> List.tryFind File.Exists with
-        | None -> Failure (exn (sprintf "Could not locate schema file. Searched: %A " paths))
+        | None -> Error (exn (sprintf "Could not locate schema file. Searched: %A " paths))
         | Some file ->
             try
             file
             |> File.ReadAllText
             |> JToken.Parse
             |> Json.ofJToken
             |> buildBlobSchema
-            |> Success
-            with ex -> Failure ex)
-    |> defaultArg <| Success []
+            |> Ok
+            with ex -> Error ex)
+    |> defaultArg <| Ok []

src/FSharp.Azure.StorageTypeProvider/Configuration.fs

@@ -5,8 +5,6 @@ open System.Configuration
 open System.IO
 open Microsoft.WindowsAzure.Storage
 
-type Result<'T> = Success of result:'T | Failure of exn
-
 let private doesFileExist folder file =
     let fullPath = Path.Combine(folder, file) 
     if fullPath |> File.Exists then Some fullPath else None
@@ -56,8 +54,9 @@ module ConnectionValidation =
                 .CreateCloudTableClient()
                 .GetTableReference("a")
                 .ExistsAsync()
-                .Result //throws an exception if attempted with an invalid connection string
+                |> Async.AwaitTask
+                |> Async.RunSynchronously //throws an exception if attempted with an invalid connection string
                 |> ignore
-            Success()
-        with | ex -> Failure ex
+            Ok()
+        with | ex -> Error ex
     let validateConnectionString = memoize checkConnectionString