diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 151e117..07b3f25 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,17 +1,12 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.0.1 + rev: v4.1.0 hooks: - id: check-yaml - args: ['--unsafe'] + args: ["--unsafe"] - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/psf/black - rev: 21.7b0 + rev: 22.1.0 hooks: - id: black - - repo: https://github.com/commitizen-tools/commitizen - rev: v2.18.0 - hooks: - - id: commitizen - stages: [commit-msg] diff --git a/wraeblast/cmd.py b/wraeblast/cmd.py index 9f3191a..5b087da 100644 --- a/wraeblast/cmd.py +++ b/wraeblast/cmd.py @@ -1,9 +1,13 @@ import asyncio import datetime import json +import os import pathlib +import tempfile +import boto3 import cleo +import pandas as pd import pkg_resources import structlog @@ -60,6 +64,7 @@ class RenderFilterCommand(BaseCommand): {--l|league=TEMP : Current league name} {--N|no-sync : Prevents automatic insights syncing} {--no-insights : Disables all economy data fetching} + {--s|store-path= : Fetch HDF from the given path} {--p|preset=default : Preset name} """ @@ -86,6 +91,20 @@ def handle(self) -> None: options_filename = str(self.option("options-file")).strip() tmpl_filename = str(self.argument("file")).strip() output_path.parent.mkdir(parents=True, exist_ok=True) + store_path = str(self.option("store-path")) + store_is_s3 = store_path.startswith("s3://") + + if store_is_s3: + bucket, key = store_path[5:].split("/", 1) + key_basename = pathlib.Path(key).name + self.line(f"Downloading data from S3: {bucket}/{key}") + tempdir = tempfile.TemporaryDirectory() + key_dest = str(pathlib.Path(tempdir.name) / key_basename) + s3 = boto3.resource("s3") + s3.Bucket(bucket).download_file(key, key_dest) + store = insights._create_hdfstore(key_dest) + else: + store = insights._create_hdfstore(store_path) loop = asyncio.get_event_loop() filter_context = None @@ -94,6 +113,7 @@ def handle(self) -> None: insights.initialize_filter_context( league=league, no_sync=bool(self.option("no-sync")), + store=store, ), ) else: @@ -154,6 +174,7 @@ class SyncInsightsCommand(BaseCommand): """Fetch Path of Exile economy insights sync_insights + {--s|store-path= : Store HDF at the given path} {--l|league=TEMP : Current league name} """ @@ -163,6 +184,24 @@ def handle(self) -> None: self.line("Syncing economy data from poe.ninja") league = check_league_option(str(self.option("league"))) loop = asyncio.get_event_loop() - loop.run_until_complete( - insights.initialize_insights_cache(league=league), + store_path = str(self.option("store-path")) + store_is_s3 = store_path.startswith("s3://") + if store_is_s3: + tempdir = tempfile.TemporaryDirectory() + store = insights._create_hdfstore( + str(pathlib.Path(tempdir.name) / f"wraeblast-{league}.h5") + ) + else: + store = insights._create_hdfstore(store_path) + store = loop.run_until_complete( + insights.initialize_insights_cache( + league=league, + store=store, + ), ) + if store_is_s3: + print(store_path) + bucket, key = store_path[5:].split("/", 1) + self.line(f"Syncing to S3 bucket: {bucket}/{key}") + s3 = boto3.resource("s3") + s3.Bucket(bucket).upload_file(store._path, key)