Skip to content

Commit

Permalink
Merge pull request #63 from dickwolff/feat/Interact-With-Ghostfolio
Browse files Browse the repository at this point in the history
Automatic import & validation with Ghostfolio
  • Loading branch information
dickwolff committed May 4, 2024
2 parents 9b65a5f + 5fc7579 commit edb7777
Show file tree
Hide file tree
Showing 6 changed files with 211 additions and 6 deletions.
15 changes: 15 additions & 0 deletions .env.sample
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,18 @@ DEBUG_LOGGING = false

# Default account currency that will be used when the currency is not present in the input file.
#XTB_ACCOUNT_CURRENCY = "EUR"

## Settings for automatic validation and upload.

# Set to true to automatically validate the generated export.
#GHOSTFOLIO_VALIDATE=false

# Set to true to automatically import a valid export.
#GHOSTFOLIO_IMPORT=false

# Your local Ghostfolio instance URL
# PLEASE DO NOT USE THE ONLINE GHOSTFOLIO SERVICE!!
#GHOSTFOLIO_URL="http://192.168.1.x:3333"

# Your Ghostfolio secret (with which you log in)
#GHOSTFOLIO_SECRET = ""
4 changes: 4 additions & 0 deletions .github/pull_request_template.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,3 +11,7 @@
- [ ] Added relevant changes to README (if applicable)
- [ ] Added relevant test(s)
- [ ] Updated GitVersion file

## Related issue (if applicable)

Fixes #..
2 changes: 1 addition & 1 deletion GitVersion.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
next-version: 0.11.0
next-version: 0.12.0
assembly-informational-format: "{NuGetVersion}"
mode: ContinuousDeployment
branches:
Expand Down
11 changes: 9 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -120,12 +120,15 @@ The following parameters can be given to the Docker run command.
| `--env DEBUG_LOGGING=true` | Y | When set to true, the container will show logs in more detail, useful for error tracing. |
| `--env FORCE_DEGIRO_V2=true` | Y | When set to true, the converter will use the DEGIRO V2 converter (currently in beta) when a DEGIRO file was found. |
| `--env PURGE_CACHE=true` | Y | When set to true, the file cache will be purged on start. |
| `--env GHOSTFOLIO_VALIDATE=true` | Y | When set to true, the tool with automatically validate the generated file against Ghostfolio. |
| `--env GHOSTFOLIO_IMPORT=true` | Y | When set to true, the tool will try to automatically import the generated file into Ghostfolio. |
| `--env GHOSTFOLIO_URL=http://xxxxxxx` | Y | The endpoint of your **local** Ghostfolio instance. E.g. `http://192.168.1.15:3333`. **Use ONLY with a local Ghostfolio instance!** |
| `--env GHOSTFOLIO_SECRET=xxxxxxx` | Y | The credentials of your Ghostfolio user. Used to authenticate with the `import` API endpoint. **Use ONLY with a local Ghostfolio instance!** |

[^1]: You can retrieve your Ghostfolio account ID by going to Accounts > Edit for your account and copying the Account ID field

![image](assets/account_settings.png)


### How to use by generating your own image

Use this option if you wish to run using an isolated docker environment where you have full control over the image and thus can trust it to contain only what is expected.
Expand Down Expand Up @@ -174,11 +177,15 @@ You can now run `npm run start [exporttype]`. See the table with run commands be
| Exporter | Run command |
| ----------- | ----------------------------------- |
| DEGIRO | `run start degiro` |
| eToro | `run start etoro` |
| Finpension | `run start finpension` (or `fp`) |
| Freetrade | `run start freetrade` (or `ft`) |
| Swissquote | `run start swissquote` (or `sq`) |
| IBKR | `run start ibkr` |
| Rabobank | `run start rabobank` |
| Schwab | `run start schwab` |
| Swissquote | `run start swissquote` (or `sq`) |
| Trading 212 | `run start trading212` (or `t212`) |
| XTB | `run start xtb` |

### Caching

Expand Down
41 changes: 38 additions & 3 deletions src/converter.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import path from "path";
import * as fs from "fs";
import dayjs from "dayjs";
import { SecurityService } from "./securityService";
import GhostfolioService from "./ghostfolioService";
import { AbstractConverter } from "./converters/abstractconverter";
import { DeGiroConverter } from "./converters/degiroConverter";
import { DeGiroConverterV2 } from "./converters/degiroConverterV2";
Expand All @@ -14,7 +16,6 @@ import { SchwabConverter } from "./converters/schwabConverter";
import { SwissquoteConverter } from "./converters/swissquoteConverter";
import { Trading212Converter } from "./converters/trading212Converter";
import { XtbConverter } from "./converters/xtbConverter";
import { SecurityService } from "./securityService";

async function createAndRunConverter(converterType: string, inputFilePath: string, outputFilePath: string, completionCallback: CallableFunction, errorCallback: CallableFunction) {

Expand All @@ -24,15 +25,15 @@ async function createAndRunConverter(converterType: string, inputFilePath: strin
}

// If DEBUG_LOGGING is enabled, set spaces to 2 else null for easier to read JSON output.
const spaces = (Boolean(process.env.DEBUG_LOGGING) == true) ? 2 : null;
const spaces = process.env.DEBUG_LOGGING === "true" ? 2 : null;

const converterTypeLc = converterType.toLocaleLowerCase();

// Determine convertor type.
const converter = await createConverter(converterTypeLc);

// Map the file to a Ghostfolio import.
converter.readAndProcessFile(inputFilePath, (result: GhostfolioExport) => {
converter.readAndProcessFile(inputFilePath, async (result: GhostfolioExport) => {

console.log("[i] Processing complete, writing to file..")

Expand All @@ -43,6 +44,8 @@ async function createAndRunConverter(converterType: string, inputFilePath: strin

console.log(`[i] Wrote data to '${outputFileName}'!`);

await tryAutomaticValidationAndImport(outputFileName);

completionCallback();

}, (error) => errorCallback(error));
Expand Down Expand Up @@ -120,6 +123,38 @@ async function createConverter(converterType: string): Promise<AbstractConverter
return converter;
}

async function tryAutomaticValidationAndImport(outputFileName: string) {

try {
const ghostfolioService = new GhostfolioService();

// When automatic validation is enabled, do this.
if (process.env.GHOSTFOLIO_VALIDATE === "true") {
console.log('[i] Automatic validation is allowed. Start validating..');
const validationResult = await ghostfolioService.validate(outputFileName);
console.log(`[i] Finished validation. ${validationResult ? 'Export was valid!' : 'Export was not valid!'}`);
}
else {
console.log('[i] You can now automatically validate the generated file against Ghostfolio. Set GHOSTFOLIO_VALIDATE=true in your environment variables to enable this feature.');
}

// When automatic import is enabled, do this.
if (process.env.GHOSTFOLIO_IMPORT === "true") {
console.log('[i] Automatic import is allowed. Start importing..');
console.log('[i] THIS IS AN EXPERIMENTAL FEATURE!! Use this at your own risk!');
const importResult = await ghostfolioService.import(outputFileName);
console.log(`[i] Finished importing. ${importResult > 0 ? `Succesfully imported ${importResult} activities!` : 'Import failed!'}`);
}
else {
console.log('[i] You can now automatically import the generated file into Ghostfolio. Set GHOSTFOLIO_IMPORT=true in your environment variables to enable this feature');
console.log('[i] THIS IS AN EXPERIMENTAL FEATURE!! Use this at your own risk!');
}
}
catch (e) {
console.log(`[e] Did not complete automatic import & validation due to errors: ${e}`);
}
}

export {
createAndRunConverter,
createConverter
Expand Down
144 changes: 144 additions & 0 deletions src/ghostfolioService.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
/* istanbul ignore file */

import * as fs from "fs";

export default class GhostfolioService {

private cachedBearerToken: string;

constructor() {

if (!process.env.GHOSTFOLIO_URL) {
throw new Error("Ghostfolio URL not provided!");
}

if (!process.env.GHOSTFOLIO_SECRET) {
throw new Error("Ghostfolio secret not provided!");
}
}

/**
* Validate an export file to Ghostfolio
*
* @param path The path to the Ghostfolio export file.
* @returns Wether the export file is valid and can be processed by Ghostfolio.
*/
public async validate(path: string, retryCount: number = 0): Promise<boolean> {

// Check wether validation is allowed.
if (!process.env.GHOSTFOLIO_VALIDATE) {
throw new Error("Validate is not allowed by config!");
}

// Stop after retrycount 3, if it doesn't work now it probably never will...
if (retryCount === 3) {
throw new Error("Failed to validate export file because of authentication error..")
}

// Read file and prepare request body.
const fileToValidate = fs.readFileSync(path, { encoding: "utf-8" });
const requestBody = {
activities: JSON.parse(fileToValidate).activities
}

// Try validation.
const validationResult = await fetch(`${process.env.GHOSTFOLIO_URL}/api/v1/import?dryRun=true`, {
method: "POST",
headers: [["Authorization", `Bearer ${this.cachedBearerToken}`], ["Content-Type", "application/json"]],
body: JSON.stringify(requestBody)
});

// Check if response was unauthorized. If so, refresh token and try again.
if (validationResult.status === 401) {

await this.authenticate(true);
return await this.validate(path, retryCount++);
}

// If status is 400, then import failed.
// Look in response for reasons and log those.
if (validationResult.status === 400) {

console.log(`[e] Validation failed!`);

var response = await validationResult.json();
response.message.forEach(message => {
console.log(`[e]\t${message}`);
});

return false;
}

return validationResult.status === 201;
}

/**
* Import an export file into Ghostfolio
*
* @param path The path to the Ghostfolio export file.
* @returns The amount of records imported.
*/
public async import(path: string, retryCount: number = 0): Promise<number> {

// Check wether validation is allowed.
if (!process.env.GHOSTFOLIO_IMPORT) {
throw new Error("Auto import is not allowed by config!");
}

// Stop after retrycount 3, if it doesn't work now it probably never will...
if (retryCount === 3) {
throw new Error("Failed to automatically import export file because of authentication error..")
}

// Read file and prepare request body.
const fileToValidate = fs.readFileSync(path, { encoding: "utf-8" });
const requestBody = {
activities: JSON.parse(fileToValidate).activities
}

// Try import.
const importResult = await fetch(`${process.env.GHOSTFOLIO_URL}/api/v1/import?dryRun=false`, {
method: "POST",
headers: [["Authorization", `Bearer ${this.cachedBearerToken}`], ["Content-Type", "application/json"]],
body: JSON.stringify(requestBody)
});

// Check if response was unauthorized. If so, refresh token and try again.
if (importResult.status === 401) {

await this.authenticate(true);
return await this.import(path, retryCount++);
}

var response = await importResult.json();

// If status is 400, then import failed.
// Look in response for reasons and log those.
if (importResult.status === 400) {

console.log(`[e] Import failed!`);

response.message.forEach(message => {
console.log(`[e]\t${message}`);
});

// It failed, so throw erro and stop.
throw new Error("Automatic import failed! See the logs for more details.");
}

return response.activities.length;
}

private async authenticate(refresh: boolean = false): Promise<void> {

// Only get bearer when it isn't set or has to be refreshed.
if (!this.cachedBearerToken || refresh) {

// Retrieve bearer token for authentication.
const bearerResponse = await fetch(`${process.env.GHOSTFOLIO_URL}/api/v1/auth/anonymous/${process.env.GHOSTFOLIO_SECRET}`);
const bearer = await bearerResponse.json();
this.cachedBearerToken = bearer.authToken;
return;
}
}
}

0 comments on commit edb7777

Please sign in to comment.