From 52f05f0fe7620e8b7ba147ce53f0b157928aecab Mon Sep 17 00:00:00 2001 From: Carlos garcia Date: Wed, 27 Mar 2024 17:07:26 -0700 Subject: [PATCH] Initial commit --- .gitignore copy | 160 +++++++++++++++++++++++++++++++++++++++++++++++ LICENSE.md | 21 +++++++ README.md | 76 ++++++++++++++++++++++ converter.py | 125 ++++++++++++++++++++++++++++++++++++ main.py | 28 +++++++++ requirements.txt | 3 + 6 files changed, 413 insertions(+) create mode 100644 .gitignore copy create mode 100644 LICENSE.md create mode 100644 README.md create mode 100644 converter.py create mode 100755 main.py create mode 100644 requirements.txt diff --git a/.gitignore copy b/.gitignore copy new file mode 100644 index 0000000..68bc17f --- /dev/null +++ b/.gitignore copy @@ -0,0 +1,160 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..36946e7 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Carlos Garcia - carlosgarciadev.com + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..fdbc1f0 --- /dev/null +++ b/README.md @@ -0,0 +1,76 @@ +# Transactify + +Transactify is a brother tool of Money Mind app -- Visit [www.carlosgarciadev.com](http://www.carlosgarciadev.com) for more information + +This tool that allows you to transform QFX files and Excel files from Questrade into JSON files, which the Money Mind app will use to ingest the information. + +## Features + +- Convert QFX files to JSON format +- Convert Questrade Excel files to JSON format + +## Installation + +1. You can either clone the repository or download the ZIP file from the releases page: + + - To download the ZIP file, go to the releases page and download the latest release on zip file. + + - To clone the repository, use the following command: + + ```shell + git clone https://github.com/your-username/Transactify.git + ``` + +2. Create a virtual environment and activate it: + + ```shell + python -m venv .venv + source .venv/bin/activate + ``` + +3. Install the required dependencies: + + ```shell + pip install -r requirements.txt + ``` + +Now you have successfully installed the required dependencies for Transactify. + +## Usage + +To use Transactify, follow these steps: + +1. Navigate to the directory where `main.py` is located: + + ```shell + cd path/to/Transactify + ``` + +2. Run the `main.py` script with the following command: + + ```shell + ./main.py + ``` + + By default, the script will look for files to convert in your `Downloads` folder. If your files are located in a different folder, you can specify the path using the `--files-path` option: + + ```shell + ./main.py --files-path /path/to/your/files + ``` + +3. The converted JSON files will be created in the same directory as the original files. + +### Example + +Suppose you have a QFX file and a Questrade Excel file in your `Downloads` folder. After running the script, you will find the corresponding JSON files in the same folder: + +- `your_file.qfx` will be converted to `your_file.json` +- `your_questrade_file.xlsx` will be converted to `your_questrade_file.json` + +## Contributing + +If you would like to contribute to the development of Transactify, please feel free to submit a pull request or open an issue on the GitHub repository. + +## License + +Transactify is released under the MIT License. See the LICENSE file for more details. diff --git a/converter.py b/converter.py new file mode 100644 index 0000000..4c0a3be --- /dev/null +++ b/converter.py @@ -0,0 +1,125 @@ +# MIT License 2024, Carlos Garcia +import json +import pathlib +from hashlib import md5 +from typing import Union + +import ofxparse +import pandas as pd + + +class Converter: + def __init__(self, files_path: Union[pathlib.Path, None]) -> None: + self.ofx_parser = ofxparse.OfxParser() + if not files_path: + self.files_path = pathlib.Path.home() / "Downloads" + else: + self.files_path = files_path + + def consistent_hash(self, string): + result = abs(int(md5(string.encode()).hexdigest(), 16)) + return int(str(result)[:14]) + + def create_transactions_files_excel(self) -> None: + excel_trans_path = self.files_path / "transactions_excel" + excel_files = list(self.files_path.glob("*.xlsx")) + + if not excel_files: + print("No Excel files found.") + return + + if not excel_trans_path.exists(): + excel_trans_path.mkdir() + + all_transactions = [] + + for excel_file in excel_files: + print(f"Processing {excel_file}...") + df = pd.read_excel(excel_file) + + df["Settlement Date"] = pd.to_datetime(df["Settlement Date"], format="%Y-%m-%d %I:%M:%S %p").dt.strftime( + "%Y-%m-%d" + ) + df["Transaction Date"] = pd.to_datetime(df["Transaction Date"], format="%Y-%m-%d %I:%M:%S %p").dt.strftime( + "%Y-%m-%d" + ) + + df["id"] = df.apply( + lambda row: self.consistent_hash(str(row["Net Amount"]) + str(row["Price"]) + row["Transaction Date"]), + axis=1, + ) + df["date"] = df["Settlement Date"] + df["amount"] = df["Net Amount"] + df["accountId"] = df["Account #"] + df["name"] = df.apply( + lambda row: ( + row["Activity Type"] + if pd.isna(row["Symbol"]) or row["Symbol"] == "" + else row["Symbol"] + " " + row["Activity Type"] + ), + axis=1, + ) + df["balance"] = 0 + + # Selecting only the specified columns for the JSON file + json_df = df[["id", "date", "amount", "name", "accountId", "balance"]] + + # Convert the adjusted dataframe to JSON format + formatted_json_data = json_df.to_json(orient="records", date_format="iso") + + # Convert the JSON data to a list of dictionaries + transactions = json.loads(formatted_json_data) + all_transactions.extend(transactions) + + # Create a JSON file with transactions + json_file_path = f"{excel_trans_path}/all_transactions.json" + with open(json_file_path, "w") as file: + json.dump(all_transactions, file, indent=4) + + def create_transaction_files_qfx(self) -> None: + all_transactions = [] + qfx_transaction_files = list(self.files_path.rglob("*.QFX")) + list(self.files_path.rglob("*.qfx")) + print(f"qfx_transaction_files: {qfx_transaction_files}") + if not qfx_transaction_files: + print("No QFX files found.") + return + + qfx_trans_path = self.files_path / "qfx_transactions" + if not qfx_trans_path.exists(): + qfx_trans_path.mkdir() + + for qfx_file in qfx_transaction_files: + print(f"Processing {qfx_file}...") + with open(qfx_file) as file: + ofx_data = self.ofx_parser.parse(file) + + # Check if the file contains multiple accounts + if hasattr(ofx_data, "accounts"): + accounts = ofx_data.accounts + else: + accounts = [ofx_data.account] + + for account in accounts: + print(f"Processing account {account.account_id}...") + transactions = [] + for transaction in account.statement.transactions: + name = transaction.payee.strip() + while " " in name: + name = name.replace(" ", " ") + + row = { + "id": self.consistent_hash(account.account_id + str(transaction.id)), + "date": transaction.date.strftime("%Y-%m-%d"), + "amount": float(transaction.amount), + "name": name, + "accountId": int(account.account_id), + "balance": float(account.statement.balance), + } + transactions.append(row) + + all_transactions.extend(transactions) + + # Create a JSON file with transactions + json_file_path = f"{qfx_trans_path}/all_transactions.json" + with open(json_file_path, "w") as file: + json.dump(all_transactions, file, indent=4) diff --git a/main.py b/main.py new file mode 100755 index 0000000..e748d5f --- /dev/null +++ b/main.py @@ -0,0 +1,28 @@ +#!.venv/bin/python3 +import pathlib +from typing import Union + +import typer + +from converter import Converter + + +def main( + files_path=typer.Option( + pathlib.Path.home() / "Downloads", + help="Path to the files to convert, usually the Downloads folder.", + ), +): + if isinstance(files_path, str): + files_path = pathlib.Path(files_path) + convert_transactions(files_path) + + +def convert_transactions(files_path: Union[pathlib.Path, None]) -> None: + converter = Converter(files_path) + converter.create_transactions_files_excel() + converter.create_transaction_files_qfx() + + +if __name__ == "__main__": + typer.run(main) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e3bb8b7 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +typer==0.9.0 +pandas==2.1.3 +ofxparse==0.21 \ No newline at end of file