New lazy API for sessions

Banyan/Project.toml

@@ -4,21 +4,21 @@ authors = ["Banyan <support@banyancomputing.com>"]
 version = "0.4.1"
 
 [deps]
+AWS = "fbe9abb3-538b-5e4e-ba9e-bc94f4f92ebc"
 AWSCore = "4f1ea46c-232b-54a6-9b17-cc2d0f3e6598"
 AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95"
-AWSSQS = "6e80b5ca-5733-51f9-999e-c18680912812"
+Arrow = "69666777-d1a9-59fb-9406-91d4454c9d45"
 Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
 FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
 FilePathsBase = "48062228-2e41-5def-b9a4-89aafe57970f"
 HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
-IniFile = "83e8ac13-25f8-5344-8a64-a9f2b223428f"
 JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433"
-MethodAnalysis = "85b6ec6f-f7df-4429-9514-a64bcd9ee824"
 MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195"
+MethodAnalysis = "85b6ec6f-f7df-4429-9514-a64bcd9ee824"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
@@ -29,15 +29,14 @@ TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
 TimeZones = "f269a46b-ccf7-5d73-abea-4c690281aa53"
 
 [compat]
-AWSCore = "0.6"
-AWSS3 = "0.7"
-AWSSQS = "0.6"
+AWS = "1"
+AWSS3 = "0.8"
+Arrow = "2"
 DataStructures = "0.18"
-Downloads = "^1.4"
+Downloads = "1.4"
 FileIO = "1.9.1"
 FilePathsBase = "^0.9.15"
 HTTP = "^0.9.17"
-IniFile = "0.5"
 JSON = "0.21"
 MPI = "^0.19.0"
 MethodAnalysis = "0.4"

Banyan/some_file

@@ -0,0 +1 @@
+hello world

Banyan/src/Banyan.jl

@@ -21,9 +21,7 @@ global NOT_USING_MODULES = String["ProfileView", "SnoopCompileCore"]
 using FilePathsBase: joinpath, isempty
 using Base: notnothing, env_project_file
 
-using AWSCore,
-    AWSS3,
-    AWSSQS,
+using Arrow,
     Base64,
     DataStructures,
     Dates,
@@ -32,18 +30,27 @@ using AWSCore,
     FilePathsBase,
     HTTP,
     JSON,
-    IniFile,
     LibGit2,
     MPI,
     ProgressMeter,
     Random,
     Serialization,
     TOML
 
+using AWS
+AWS.DEFAULT_BACKEND[] = AWS.DownloadsBackend()
+s3 = set_features(AWS.AWSServices.s3; use_response_type=true)
+using AWS.AWSExceptions
+using AWS: @service
+# TODO: Remove @service S3 since we just use AWSS3 and s3
+@service S3 use_response_type = true
+@service SQS use_response_type = true
+using AWSS3
+
 global BANYAN_API_ENDPOINT
 
 # Account management
-export configure
+export configure, get_organization_id
 
 # Cluster management
 export Cluster,
@@ -82,25 +89,23 @@ export AbstractFuture, Future, partitioned_computation, compute_inplace, compute
 
 # Samples
 export Sample, ExactSample, sample, sample_for_grouping, SampleForGrouping, setsample!
-export sample_memory_usage, total_memory_usage, sample_axes, sample_keys, sample_by_key
+export sample_memory_usage, sample_memory_usage, sample_axes, sample_keys, sample_by_key
 export NOTHING_SAMPLE
+export SamplingConfig
 
 # Locations
 export Location, LocationSource, LocationDestination, located, sourced, destined
-export Value, Size, Client, Disk, None
-export invalidate_all_locations, invalidate_metadata, invalidate_sample
-export NOTHING_LOCATION, INVALID_LOCATION
+export Value, Size, Client, Disk, None, RemoteSource
+export invalidate_all_locations, invalidate_location, invalidate_metadata, invalidate_samples, invalidate
+export NOTHING_LOCATION, INVALID_LOCATION, NO_LOCATION_PATH
 export has_separate_metadata, get_sample, get_metadata, get_sample_and_metadata
-export get_remotepath_id,
-    get_meta_path,
-    get_location_path,
-    get_cached_location,
-    cache_location,
-    get_max_exact_sample_length,
-    set_max_exact_sample_length
+export LocationPath, SamplingConfig
+export has_metadata, has_sample, get_sample_rate, configure_sampling, get_sampling_config, get_sampling_configs, set_sampling_configs
+export type_to_str, str_to_type
+export banyan_metadata_bucket_name, banyan_samples_bucket_name, get_metadata_path, get_sample_path_prefix, get_sample_path
 
 # Serialization
-export from_jl_value_contents, to_jl_value_contents
+export from_jl_string, to_jl_string
 
 # Queues
 export receive_from_client, send_to_client, get_sqs_dict_from_url
@@ -165,8 +170,8 @@ export is_debug_on,
     get_partition_idx_from_divisions,
     isoverlapping,
     to_jl_value,
-    to_jl_value_contents,
-    from_jl_value_contents,
+    to_jl_string,
+    from_jl_string,
     get_divisions,
     getpath,
     buftovbuf,
@@ -189,7 +194,7 @@ export is_debug_on,
 export Empty, EMPTY, nonemptytype, disallowempty, empty_handler
 
 # Utilities for location constructors
-export get_cached_location, cache_location, get_sample_from_data, sample_from_range
+export get_sample_from_data, sample_from_range
 
 # Partitioning functions for usage in sessions that run on the cluster; dispatched
 # based on `res/pf_dispatch_table.json`.

Banyan/src/annotation.jl

@@ -611,8 +611,8 @@ function apply_mutation(old::Future, new::Future)
     new.mutated,
     old.stale,
     new.stale,
-    old.total_memory_usage,
-    new.total_memory_usage,
+    old.sample_memory_usage,
+    new.sample_memory_usage,
     session_locations[old.value_id],
     session_locations[new.value_id] =
     new.value,
@@ -623,8 +623,8 @@ function apply_mutation(old::Future, new::Future)
     old.mutated,
     new.stale,
     old.stale,
-    new.total_memory_usage,
-    old.total_memory_usage,
+    new.sample_memory_usage,
+    old.sample_memory_usage,
     session_locations[new.value_id],
     session_locations[old.value_id]
 end
@@ -675,11 +675,11 @@ function finish_partitioned_code_region(splatted_futures::Vector{Future})
 
     # Get the initial memory usage
     for fut in splatted_futures
-        fut_initial_memory_usage::Int64 = if is_total_memory_usage_known(fut)
-            fut.total_memory_usage
+        fut_initial_memory_usage::Int64 = if is_sample_memory_usage_known(fut)
+            fut.sample_memory_usage
         else
             tmu::Int64 = try
-                get_location(fut).total_memory_usage
+                get_location(fut).sample_memory_usage
             catch e
                 if e isa MethodError
                     error("Future with value ID $(fut.value_id) has no initial memory usage even in location with source name $(get_location(fut).src_name)")
@@ -877,7 +877,7 @@ function finish_partitioned_code_region(splatted_futures::Vector{Future})
 
     # Destroy value IDs that are no longer needed because of mutation
     for fut in splatted_futures
-        fut.total_memory_usage = task.memory_usage[fut.value_id]["final"]
+        fut.sample_memory_usage = task.memory_usage[fut.value_id]["final"]
 
         # Issue destroy request for mutated futures that are no longer
         # going to be used