From b127259ea618a413bbdf380203f121035cb616d6 Mon Sep 17 00:00:00 2001 From: vsoch Date: Wed, 29 Nov 2023 18:14:53 -0700 Subject: [PATCH] add early experiment planning for spot The spot_instances.py is refactored to include spot prices, and we have an idea of the overall design. I next need to write a test setup that will implement the features that I want, namely using the metrics operator to run lammps, hwloc, and then pushing to a remote oras cache (needs to be developed in the oras-operator) and also using the aws locality / topology API to get metadata for each group. Primarily I am interested in testing the different sizes scoped in the README against problem sizes to better estimate the total time and thus cost. Signed-off-by: vsoch --- aws/spot-instances/README.md | 6 + aws/spot-instances/run1/.gitignore | 1 + aws/spot-instances/run1/README.md | 237 ++++++++++++++ aws/spot-instances/run1/requirements.txt | 2 + aws/spot-instances/run1/spot_instances.py | 359 ++++++++++++++++++++++ 5 files changed, 605 insertions(+) create mode 100644 aws/spot-instances/README.md create mode 100644 aws/spot-instances/run1/.gitignore create mode 100644 aws/spot-instances/run1/README.md create mode 100644 aws/spot-instances/run1/requirements.txt create mode 100644 aws/spot-instances/run1/spot_instances.py diff --git a/aws/spot-instances/README.md b/aws/spot-instances/README.md new file mode 100644 index 0000000..a06d5a2 --- /dev/null +++ b/aws/spot-instances/README.md @@ -0,0 +1,6 @@ +# Spot Instances + +These are experiments with spot instances. + + - [run0](run0): Based on [these experiments](https://github.com/converged-computing/cloud-select/tree/main/examples/spot-instances/experiments/request-success) I want to very naively try running LAMMPS (with the metrics operator) on a Potpurri of nodes. + - [run1](run1): A set of three experiments that test deploying spot instances using the Flux Operator. diff --git a/aws/spot-instances/run1/.gitignore b/aws/spot-instances/run1/.gitignore new file mode 100644 index 0000000..bdaab25 --- /dev/null +++ b/aws/spot-instances/run1/.gitignore @@ -0,0 +1 @@ +env/ diff --git a/aws/spot-instances/run1/README.md b/aws/spot-instances/run1/README.md new file mode 100644 index 0000000..8b1fb80 --- /dev/null +++ b/aws/spot-instances/run1/README.md @@ -0,0 +1,237 @@ +# Spot Instances Experiments + +We want to test LAMMPS performance (runtime, and MPItrace metrics) when we run on a managed node group +of spot instances. This experiment directory will include three approaches: + + - **test**: preparing for experiments (just small test runs to time different sizes primarily) + - **region**: select from a region (but no placement group) + - **placement group**: TBA + - **fleet**: testing out AWS fleet (also TBA) + +## Experiment Designs + +Importantly, we want to ultimately test the ability of different machines types from spot to run LAMMPS, and not the result of the selection process itself. This decision drives the design below. + +- For each of 20 batches: + - Filter down the initial set to some number above a certain cost threshold + - Randomly select 4 from that set AND give to the AWS API to create 8 nodes (this is flattened into one operation) + - Then we have an instance group, 8 nodes for some unique set of instances from the set of 4 + - Run LAMMPS 20x, collect MPI trace too, lstopo and the AWS topology API + +With the above we can calculate cost as: + +```console +total cost = 20 batches x 1 selection of nodes x 20 runs x [TIME TO RUN EXPERIMENT] seconds +``` + +We will likely need to do some tests to estimate time to run for different sizes to properly prepare for this. +From Rajib we know that hpc7g (128 vCPU) were between 110-120 seconds, and hpc6a (192 vCPU) were 82-86 seconds. But I tested hpc7g earlier and it was much slower, so I think we probably need to do some new test runs. + +### Environment + +For the steps below (and experiments) you should install requirements.txt. + +```bash +pip install -r requirements.txt +``` + +Ideally from a virtual environment or similar. + +### Instance Selection + +See [thinking process here](https://gist.github.com/vsoch/ad19f4270a0500a49c47008e4a853f62). + +We want to (maybe) mimic the following instance types: + +|Instance |Physical Cores | Memory (GiB) | EFA Network Bandwidth (Gbps) | Network Bandwidth (Gbps)* | +| hpc6a.48xlarge | 96 (192 vCPU) | 384 | 100 | 25 | +| hpc7g.16xlarge | 64 (128 vCPU) | 128 | 200 | 25 | + +Note that the website says "physical cores" so that means we need to search for 96x2 == 192 vCPU. +Our starting problem size is `64 x 16 x 16`. + +## Estimating Cost + +**Important** this relies on the [pull request branch here](https://github.com/converged-computing/cloud-select/pull/35). You can clone that and pip install. + +The spot_instances.py can be used (and shared) between experiments to generate cost tables. To generate (don't run this if you already have an instances-aws.csv and it's recent. + +```bash +python spot_instances.py gen +``` + +### 128 vCPU + +We need to find a cost that (divided by 3) is approximately 1k, which is our spending limit for these experiments, and assuming we do them for each of the cases described above. I first tried a range of vCPU we wanted to emulate: + +```bash +$ python spot_instances.py select --min-vcpu 128 --max-vcpu 128 --number 20 +``` + +Note that defaults to bare metal false. We aren't going to mix those. + +
+ +Price estimation for 128 vCPU + +```bash +$ python spot_instances.py select --min-vcpu 128 --max-vcpu 128 --number 20 +``` +```console +Selected subset table: + instance bare_metal arch vcpu threads_per_core memory_mb gpu spot_price price +465 c6a.32xlarge False x86_64 128 2 262144 False 2.201160 4.89600 +211 c7a.32xlarge False x86_64 128 1 262144 False 2.323425 6.56896 +691 c6i.32xlarge False x86_64 128 2 262144 False 2.478600 5.44000 +248 m6a.32xlarge False x86_64 128 2 524288 False 2.628120 5.52960 +209 m6i.32xlarge False x86_64 128 2 524288 False 2.676000 6.14400 +343 c6id.32xlarge False x86_64 128 2 262144 False 2.687360 6.45120 +298 r6a.32xlarge False x86_64 128 2 1048576 False 2.930780 7.25760 +1 m7a.32xlarge False x86_64 128 1 524288 False 3.117800 7.41888 +273 c6in.32xlarge False x86_64 128 2 262144 False 3.239400 7.25760 +193 m6id.32xlarge False x86_64 128 2 524288 False 3.303780 7.59360 +359 r6i.32xlarge False x86_64 128 2 1048576 False 3.371040 8.06400 +203 r6id.32xlarge False x86_64 128 2 1048576 False 3.631980 9.67680 +718 i4i.32xlarge False x86_64 128 2 1048576 False 3.704200 10.98240 +335 r7a.32xlarge False x86_64 128 1 1048576 False 3.727750 9.73760 +685 m6idn.32xlarge False x86_64 128 2 524288 False 3.984450 10.18368 +431 m6in.32xlarge False x86_64 128 2 524288 False 4.009550 8.91072 +316 x1.32xlarge False x86_64 128 2 1998848 False 4.393550 13.33800 +213 r7iz.32xlarge False x86_64 128 2 1048576 False 4.446850 11.90400 +39 x2idn.32xlarge False x86_64 128 2 2097152 False 4.623940 13.33800 +4 r6in.32xlarge False x86_64 128 2 1048576 False 5.122650 11.15712 + +😸️ Final selection of spot: +c6a.32xlarge +c7a.32xlarge +c6i.32xlarge +m6a.32xlarge +m6i.32xlarge +c6id.32xlarge +r6a.32xlarge +m7a.32xlarge +c6in.32xlarge +m6id.32xlarge +r6i.32xlarge +r6id.32xlarge +i4i.32xlarge +r7a.32xlarge +m6idn.32xlarge +m6in.32xlarge +x1.32xlarge +r7iz.32xlarge +x2idn.32xlarge +r6in.32xlarge + +🤓️ Mean (std) of price +$3.43 ($0.82) +``` + +
+ +### 192 vCPU + +I think likely we can't do this size because there aren't a ton of instances to choose from. + +
+ +Price estimation for 128 vCPU + +```bash +$ python spot_instances.py select --min-vcpu 192 --max-vcpu 192 --number 20 +``` +```console +Selected subset table: + instance bare_metal arch vcpu threads_per_core memory_mb gpu spot_price price +581 c6a.48xlarge False x86_64 192 2 393216 False 3.207520 7.34400 +381 c7a.48xlarge False x86_64 192 1 393216 False 3.671550 9.85344 +152 m6a.48xlarge False x86_64 192 2 786432 False 3.735820 8.29440 +689 c7i.48xlarge False x86_64 192 2 393216 False 3.948450 8.56800 +698 m7i.48xlarge False x86_64 192 2 786432 False 4.011800 9.67680 +150 r6a.48xlarge False x86_64 192 2 1572864 False 4.505600 10.88640 +566 r7i.48xlarge False x86_64 192 2 1572864 False 4.588400 12.70080 +449 m7a.48xlarge False x86_64 192 1 786432 False 4.720575 11.12832 +238 inf2.48xlarge False x86_64 192 2 786432 False 4.758775 12.98127 +712 r7a.48xlarge False x86_64 192 1 1572864 False 6.843625 14.60640 + +😸️ Final selection of spot: +c6a.48xlarge +c7a.48xlarge +m6a.48xlarge +c7i.48xlarge +m7i.48xlarge +r6a.48xlarge +r7i.48xlarge +m7a.48xlarge +inf2.48xlarge +r7a.48xlarge + +🤓️ Mean (std) of price +$4.4 ($1.0) +``` + +### 64 vCPU + +What if we try closer to what we did on Google Cloud, closer to 50 vCPU. It looks like the closest we can get is 64 vCPU. A size 64 vCPU is fairly good, because we might have 32 actual CPU per node. + +```bash +$ python spot_instances.py select --min-vcpu 64 --max-vcpu 64 --number 20 +``` + +
+ +Price estimation for 128 vCPU + +```console +Selected subset table: + instance bare_metal arch vcpu threads_per_core memory_mb gpu spot_price price +679 c6a.16xlarge False x86_64 64 2 131072 False 1.151780 2.44800 +212 c5ad.16xlarge False x86_64 64 2 131072 False 1.163000 2.75200 +729 c5a.16xlarge False x86_64 64 2 131072 False 1.210240 2.46400 +10 m5a.16xlarge False x86_64 64 2 262144 False 1.314160 2.75200 +474 c6i.16xlarge False x86_64 64 2 131072 False 1.325060 2.72000 +25 m6a.16xlarge False x86_64 64 2 262144 False 1.364100 2.76480 +515 m5.16xlarge False x86_64 64 2 262144 False 1.369840 3.07200 +234 c6id.16xlarge False x86_64 64 2 131072 False 1.380580 3.22560 +671 c7i.16xlarge False x86_64 64 2 131072 False 1.394750 2.85600 +402 m4.16xlarge False x86_64 64 2 262144 False 1.398133 3.20000 +354 m7a.16xlarge False x86_64 64 1 262144 False 1.403450 3.70944 +291 c7a.16xlarge False x86_64 64 1 131072 False 1.411625 3.28448 +221 r5a.16xlarge False x86_64 64 2 524288 False 1.511780 3.61600 +635 m6i.16xlarge False x86_64 64 2 262144 False 1.515600 3.07200 +153 r6i.16xlarge False x86_64 64 2 524288 False 1.515940 4.03200 +525 r6a.16xlarge False x86_64 64 2 524288 False 1.544920 3.62880 +347 m7i.16xlarge False x86_64 64 2 262144 False 1.587725 3.22560 +27 m5d.16xlarge False x86_64 64 2 262144 False 1.603920 3.61600 +720 m5ad.16xlarge False x86_64 64 2 262144 False 1.615520 3.29600 +721 m6id.16xlarge False x86_64 64 2 262144 False 1.615700 3.79680 + +😸️ Final selection of spot: +c6a.16xlarge +c5ad.16xlarge +c5a.16xlarge +m5a.16xlarge +c6i.16xlarge +m6a.16xlarge +m5.16xlarge +c6id.16xlarge +c7i.16xlarge +m4.16xlarge +m7a.16xlarge +c7a.16xlarge +r5a.16xlarge +m6i.16xlarge +r6i.16xlarge +r6a.16xlarge +m7i.16xlarge +m5d.16xlarge +m5ad.16xlarge +m6id.16xlarge + +🤓️ Mean (std) of price +$1.42 ($0.14) +``` + +
+ +That also gives us many choices under $2, so I am leaning toward this as our choice (but need to test timing and problem sizes). diff --git a/aws/spot-instances/run1/requirements.txt b/aws/spot-instances/run1/requirements.txt new file mode 100644 index 0000000..4765030 --- /dev/null +++ b/aws/spot-instances/run1/requirements.txt @@ -0,0 +1,2 @@ +kubescaler +pandas diff --git a/aws/spot-instances/run1/spot_instances.py b/aws/spot-instances/run1/spot_instances.py new file mode 100644 index 0000000..85e8108 --- /dev/null +++ b/aws/spot-instances/run1/spot_instances.py @@ -0,0 +1,359 @@ +#!/usr/bin/env python + +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# HPCIC DevTools Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (MIT) +# +# Save a cache of spot instance pricing. +# Note that you can get the spot placement score via this call +# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ec2/client/get_spot_placement_scores.html#EC2.Client.get_spot_placement_scores +# However it needs a certain instance type and number + +import argparse +import os +import sys +import boto3 +import statistics + +import pandas + +from cloudselect.logger import setup_logger +from cloudselect.main import Client +import cloudselect.utils as utils + +here = os.path.dirname(os.path.abspath(__file__)) + +# Default architecture (also can do arm) +default_arch = "x86_64" + + +def get_parser(): + parser = argparse.ArgumentParser( + description="Cloud Select Spot Instance Grouper", + formatter_class=argparse.RawTextHelpFormatter, + ) + parser.add_argument( + "--data", + help="exported data.csv (to generate or use)", + default=os.path.join(here, "instances-aws.csv"), + ) + subparsers = parser.add_subparsers( + help="cloudselect actions", + title="actions", + description="actions", + dest="command", + ) + gen = subparsers.add_parser( + "gen", + description="generate instance data frame", + formatter_class=argparse.RawTextHelpFormatter, + ) + gen.add_argument( + "--no-cache", help="do not use cache", action="store_true", default=False + ) + + # On the fly updates to config params + gen.add_argument( + "-c", + dest="config_params", + help=""""customize a config value on the fly to ADD/SET/REMOVE for a command +cloud-select -c set:key:value +cloud-select -c add:registry:/tmp/registry +cloud-select -c rm:registry:/tmp/registry""", + action="append", + ) + + # Select step + select = subparsers.add_parser( + "select", + description="select from instance data frame", + formatter_class=argparse.RawTextHelpFormatter, + ) + select.add_argument( + "--arch", + default=default_arch, + help="architecture", + ) + select.add_argument( + "--gpu", help="select instances with GPU", action="store_true", default=False + ) + select.add_argument( + "--bare-metal", + help="select instances with bare metal", + action="store_true", + default=False, + ) + select.add_argument( + "--randomize", + help="randomize list (do not sort by price)", + action="store_true", + default=False, + ) + select.add_argument("--min-vcpu", help="minimum vCPU", type=int, default=32) + select.add_argument("--max-vcpu", help="maximum vCPU", type=int, default=64) + select.add_argument( + "--max-threads-per-core", + dest="threads_per_core", + help="threads per core", + type=int, + default=2, + ) + select.add_argument("--min-mem", help="minimum memory MB", type=int) + select.add_argument("--max-mem", help="maximum memory MB", type=int) + select.add_argument("--max-price", help="maximum price to set (USD/hour)", type=int) + select.add_argument( + "-n", "--number", help="number to select", type=int, dest="number" + ) + return parser + + +def run(): + parser = get_parser() + + # If an error occurs while parsing the arguments, the interpreter will exit with value 2 + args, extra = parser.parse_known_args() + + # retrieve subparser (with help) from parser + subparsers_actions = [ + action + for action in parser._actions + if isinstance(action, argparse._SubParsersAction) + ] + for subparsers_action in subparsers_actions: + for choice, subparser in subparsers_action.choices.items(): + if choice == args.command: + break + + # Be more verbose + setup_logger(quiet=False, debug=True) + + if args.command == "gen": + generate_data(args) + elif args.command == "select": + select_instances( + args.data, + arch=args.arch, + number=args.number, + has_gpu=args.gpu, + min_mem=args.min_mem, + max_mem=args.max_mem, + min_vcpu=args.min_vcpu, + max_vcpu=args.max_vcpu, + max_price=args.max_price, + randomize=args.randomize, + max_threads_per_core=args.threads_per_core, + bare_metal=args.bare_metal, + ) + + +def select_instances( + datafile, + min_vcpu, + max_vcpu=None, + arch=default_arch, + max_threads_per_core=2, + min_mem=None, + max_mem=None, + number=None, + has_gpu=False, + max_price=None, + randomize=False, + bare_metal=False, +): + """ + Given a csv of data, filter down / sort and show final set. + """ + if not os.path.exists(datafile): + sys.exit(f"Input data table {datafile} does not exist.") + + # Read in data frame + df = pandas.read_csv(datafile, index_col=0) + + # Filter to arch and cpu (required) + subset = df[df.arch == arch] + subset = subset[subset.vcpu >= min_vcpu] + subset = subset[subset.vcpu <= max_vcpu] + subset = subset[subset.threads_per_core <= max_threads_per_core] + + # GPU + subset = subset[subset.gpu == has_gpu] + + # Optional memory + if min_mem: + subset = subset[min_mem <= subset.memory_mb] + if max_mem: + subset = subset[subset.memory_mb <= max_mem] + if max_price: + subset = subset[subset.price <= max_price] + + if bare_metal: + subset = subset[subset.bare_metal == True] + else: + subset = subset[subset.bare_metal == False] + + # If not randomize, sort by vcpu and then price + if not randomize: + sorted_df = subset.sort_values(["vcpu", "spot_price"]) + else: + sorted_df = subset.sample(frac=1) + + instance_names = list(sorted_df.instance.values) + + # Honor a user threshold, if set + if number and len(instance_names) > number: + instance_names = instance_names[:number] + + sorted_df = sorted_df[sorted_df.instance.isin(instance_names)] + print("Selected subset table:") + print(sorted_df) + + print("\n😸️ Final selection of spot:") + for instance_name in instance_names: + print(instance_name) + + # Give mean cost and std + print("\n🤓️ Mean (std) of price") + mean = round(sorted_df.spot_price.mean(), 2) + std = round(sorted_df.spot_price.std(), 2) + print(f"${mean} (${std})") + return sorted_df + + +def get_price_lookup(prices, spot_prices): + """ + Given a cloud, generate lookup of instance type by float price + + Note that I'm seeing instance types with USD 0.0 which doesn't make sense... + """ + # Put together lookup by name. We will take mean across availability zones + lookup = {} + means = {} + for instance_type, zones in spot_prices.items(): + sps = [float(x["SpotPrice"]) for _, x in zones.items()] + means[instance_type] = statistics.mean(sps) + + for price in prices.data: + if "instanceType" not in price["product"]["attributes"]: + continue + if price["product"]["attributes"]["operatingSystem"] != "Linux": + continue + # WARNING: this is hard coded here to match the prices data default + if price["product"]["attributes"]["regionCode"] != "us-east-1": + continue + name = price["product"]["attributes"]["instanceType"] + + # Skip instance that don't have spot + if name not in means: + continue + + # We need OnDemand too + if "OnDemand" not in price["terms"]: + continue + + # This seems to be the best way to find the on demand linux for the instance type + # There are a LOT + usd = None + for _, meta in price["terms"].items(): + for _, pds in meta.items(): + for _, pd in pds["priceDimensions"].items(): + if pd["unit"] != "Hrs": + continue + if f"On Demand Linux {name}" in pd["description"]: + usd = pd["pricePerUnit"]["USD"] + + # This isn't the right instance type + if not usd: + continue + if usd and name in lookup: + print(f"Warning: found previous price for {name}: {lookup[name]}") + lookup[name] = {"price": float(usd), "spot_price": means[name]} + return lookup + + +def generate_data(args): + """ + Generate and save data frame + """ + # The client has a cache on it, cli.cache. + # Set use_cache to False so we don't rely on it + # This is currently just for aws + cli = Client(use_cache=not args.no_cache, clouds=["aws"]) + + # Update config settings on the fly + cli.settings.update_params(args.config_params) + + # Get new prices data for each cloud + for cloud in cli.get_clouds(): + # Spot instances are aws for now + if cloud.name != "aws": + continue + instances = cli.instances()["aws"] + prices = cli.prices()["aws"] + + # Get spot prices + spot_prices = cloud.spot_prices(instances) + lookup = get_price_lookup(prices, spot_prices) + + # Filter down to those that support spot (only a small number don't) + instances = [x for x in instances.data if "spot" in x["SupportedUsageClasses"]] + + print(f"Found {len(instances)} instances on {cloud.name}") + + # There is more data here, this is what we want for now + df = instances_to_table(instances, lookup) + print(df) + print(f"Saving {df.shape[0]} instances in table to {args.data}") + df.to_csv(args.data) + + +def instances_to_table(instances, lookup): + """ + Given listing of instances and price lookup, convert to table. + """ + df = pandas.DataFrame( + columns=[ + "instance", + "bare_metal", + "arch", + "vcpu", + "threads_per_core", + "memory_mb", + "gpu", + "spot_price", + "price", + ] + ) + + # Put into pandas data frame so we can sort / filter by: + # architecture + # memory + # cpu + # to start we will just care about arch and cpu + idx = 0 + for i in instances: + for arch in i["ProcessorInfo"]["SupportedArchitectures"]: + if "window" in arch.lower(): + continue + # What is the difference between DefaultCores and DefaultVCpu? + cpus = i["VCpuInfo"] + gpu = "GpuInfo" in i + price = lookup.get(i["InstanceType"])["price"] + spot_price = lookup.get(i["InstanceType"])["spot_price"] + df.loc[idx, :] = [ + i["InstanceType"], + i["BareMetal"], + arch, + cpus["DefaultVCpus"], + cpus["DefaultThreadsPerCore"], + i["MemoryInfo"]["SizeInMiB"], + gpu, + spot_price, + price, + ] + idx += 1 + return df + + +if __name__ == "__main__": + run()