Merge pull request #60 from dickwolff/feat/Mock-YahooFinance-During-T…

…esting

Mock yahoo finance service during testing

GitVersion.yml

@@ -1,4 +1,4 @@
-next-version: 0.10.0
+next-version: 0.11.0
 assembly-informational-format: "{NuGetVersion}"
 mode: ContinuousDeployment
 branches:

README.md

@@ -18,7 +18,7 @@ This tool allows you to convert a multiple transaction exports (CSV) to an impor
 - [Trading 212](https://trading212.com)
 - [XTB](https://www.xtb.com/int)
 
-Is your broker not in the list? Feel free to create an [issue](https://github.com/dickwolff/Export-To-Ghostfolio/issues/new) or, even better, build it yourself and create a [pull request](https://github.com/dickwolff/Export-To-Ghostfolio/compare)!
+Is your broker not in the list? Feel free to create an [issue](https://github.com/dickwolff/Export-To-Ghostfolio/issues/new) or, even better, build it yourself and create a [pull request](https://github.com/dickwolff/Export-To-Ghostfolio/compare)! For more info, see [contributing](#contributing).
 
 ## Download transaction export
 
@@ -190,4 +190,15 @@ The tool uses `cacache` to store data retrieved from Yahoo Finance on disk. This
 
 The export file can now be imported in Ghostfolio by going to Portfolio > Activities and pressing the 3 dots at the top right of the table. Since Ghostfolio 1.221.0, you can now preview the import and validate the data has been converted correctly. When it is to your satisfaction, press import to add the activities to your portfolio.
 
-![image](https://user-images.githubusercontent.com/5620002/203356387-1f42ca31-7cff-44a5-8f6c-84045cf7101e.png)
+![image](https://user-images.githubusercontent.com/5620002/203356387-1f42ca31-7cff-44a5-8f6c-84045cf7101e.png)
+
+---
+
+## Contributing
+
+We welcome any contribution to the repository. Feel free to create an [issue](https://github.com/dickwolff/Export-To-Ghostfolio/issues/new) or, even better, build it yourself and create a [pull request](https://github.com/dickwolff/Export-To-Ghostfolio/compare)!
+
+The tool can be run two ways, manually and via Docker. Both entrypoints of the tool can be found in the ‘src/‘ folder. 
+The tool uses a mock in the tests, which allow the tests to be run in a consistent and repeatable manner. This way there is no necessity for a live Yahoo Finance service. The mock was added because of inconsistencies in between test runs and rate-limiting issues with Yahoo Finance (with multiple consequetive runs, especially when running locally).
+
+Whenever you add a new converter or create a fix for an existing one, please refer to the [Wiki](https://github.com/dickwolff/Export-To-Ghostfolio/wiki/Add-new-testdata-to-Yahoo-Finance-mock) for instructions on how to extend the mock with testdata.

src/converter.ts

@@ -14,7 +14,7 @@ import { SchwabConverter } from "./converters/schwabConverter";
 import { SwissquoteConverter } from "./converters/swissquoteConverter";
 import { Trading212Converter } from "./converters/trading212Converter";
 import { XtbConverter } from "./converters/xtbConverter";
-import { YahooFinanceService } from "./yahooFinanceService";
+import { SecurityService } from "./securityService";
 
 async function createAndRunConverter(converterType: string, inputFilePath: string, outputFilePath: string, completionCallback: CallableFunction, errorCallback: CallableFunction) {
 
@@ -50,9 +50,9 @@ async function createAndRunConverter(converterType: string, inputFilePath: strin
 
 async function createConverter(converterType: string): Promise<AbstractConverter> {
 
-    const yahooFinanceService = new YahooFinanceService();
+    const securityService = new SecurityService();
 
-    const cacheSize = await yahooFinanceService.loadCache();
+    const cacheSize = await securityService.loadCache();
     console.log(`[i] Restored ${cacheSize[0]} ISIN-symbol pairs and ${cacheSize[1]} symbols from cache..`);
 
     let converter: AbstractConverter;
@@ -65,53 +65,53 @@ async function createConverter(converterType: string): Promise<AbstractConverter
             console.log("[i] The new converter has multiple record parsing improvements and also supports platform fees.");
             console.log("[i] The new converter is currently in beta and we're looking for your feedback!");
             console.log("[i] You can run the beta converter with the command 'npm run start degiro-v2'.");
-            converter = new DeGiroConverter(yahooFinanceService);
+            converter = new DeGiroConverter(securityService);
             break;
         case "degiro-v2":
             console.log("[i] Processing file using DeGiro converter (V2 Beta)");
             console.log("[i] NOTE: You are running a converter that is currently in beta.");
             console.log("[i] If you have any issues, please report them on GitHub. Many thanks!");
-            converter = new DeGiroConverterV2(yahooFinanceService);
+            converter = new DeGiroConverterV2(securityService);
             break;
         case "etoro":
             console.log("[i] Processing file using Etoro converter");
-            converter = new EtoroConverter(yahooFinanceService);
+            converter = new EtoroConverter(securityService);
             break;
         case "fp":
         case "finpension":
             console.log("[i] Processing file using Finpension converter");
-            converter = new FinpensionConverter(yahooFinanceService);
+            converter = new FinpensionConverter(securityService);
             break;
         case "ft":
         case "freetrade":
             console.log("[i] Processing file using Freetrade converter");
-            converter = new FreetradeConverter(yahooFinanceService);
+            converter = new FreetradeConverter(securityService);
             break;
         case "ibkr":
             console.log("[i] Processing file using IBKR converter");
-            converter = new IbkrConverter(yahooFinanceService);
+            converter = new IbkrConverter(securityService);
             break;
         case "rabobank":
             console.log("[i] Processing file using Rabobank converter");
-            converter = new RabobankConverter(yahooFinanceService);
+            converter = new RabobankConverter(securityService);
             break;
         case "schwab":
             console.log("[i] Processing file using Schwab converter");
-            converter = new SchwabConverter(yahooFinanceService);
+            converter = new SchwabConverter(securityService);
             break;
         case "sq":
         case "swissquote":
             console.log("[i] Processing file using Swissquote converter");
-            converter = new SwissquoteConverter(yahooFinanceService);
+            converter = new SwissquoteConverter(securityService);
             break;
         case "t212":
         case "trading212":
             console.log("[i] Processing file using Trading212 converter");
-            converter = new Trading212Converter(yahooFinanceService);
+            converter = new Trading212Converter(securityService);
             break;
         case "xtb":
             console.log("[i] Processing file using XTB converter");
-            converter = new XtbConverter(yahooFinanceService);
+            converter = new XtbConverter(securityService);
             break;
         default:
             throw new Error(`Unknown converter '${converterType}' provided`);

src/converters/abstractconverter.ts

@@ -1,16 +1,16 @@
 import * as fs from "fs";
 import * as cliProgress from "cli-progress";
-import { YahooFinanceService } from "../yahooFinanceService";
+import { SecurityService } from "../securityService";
 
 export abstract class AbstractConverter {
 
-    protected yahooFinanceService: YahooFinanceService;
+    protected securityService: SecurityService;
 
     protected progress: cliProgress.MultiBar;
 
-    constructor(yahooFinanceService: YahooFinanceService) {
+    constructor(securityService: SecurityService) {
 
-        this.yahooFinanceService = yahooFinanceService;
+        this.securityService = securityService;
 
         this.progress = new cliProgress.MultiBar(
             {

src/converters/degiroConverter.ts

@@ -4,16 +4,16 @@ import dayjs from "dayjs";
 import { parse } from "csv-parse";
 import { DeGiroRecord } from "../models/degiroRecord";
 import { AbstractConverter } from "./abstractconverter";
-import { YahooFinanceService } from "../yahooFinanceService";
+import { SecurityService } from "../securityService";
 import { GhostfolioExport } from "../models/ghostfolioExport";
 import YahooFinanceRecord from "../models/yahooFinanceRecord";
 import customParseFormat from "dayjs/plugin/customParseFormat";
 import { GhostfolioOrderType } from "../models/ghostfolioOrderType";
 
 export class DeGiroConverter extends AbstractConverter {
 
-  constructor(yahooFinanceService: YahooFinanceService) {
-    super(yahooFinanceService);
+  constructor(securityService: SecurityService) {
+    super(securityService);
 
     dayjs.extend(customParseFormat);
   }
@@ -80,7 +80,7 @@ export class DeGiroConverter extends AbstractConverter {
 
         let security: YahooFinanceRecord;
         try {
-          security = await this.yahooFinanceService.getSecurity(
+          security = await this.securityService.getSecurity(
             record.isin,
             null,
             record.product,

src/converters/degiroConverterV2.test.ts

@@ -1,11 +1,12 @@
 import { DeGiroConverterV2 } from "./degiroConverterV2";
-import { YahooFinanceService } from "../yahooFinanceService";
+import { SecurityService } from "../securityService";
 import { GhostfolioExport } from "../models/ghostfolioExport";
+import YahooFinanceServiceMock from "../testing/yahooFinanceServiceMock";
 
 describe("degiroConverterV2", () => {
 
   beforeEach(() => {
-   // jest.spyOn(console, "log").mockImplementation(jest.fn());
+    jest.spyOn(console, "log").mockImplementation(jest.fn());
   });
 
   afterEach(() => {
@@ -15,7 +16,7 @@ describe("degiroConverterV2", () => {
   it("should construct", () => {
 
     // Act
-    const sut = new DeGiroConverterV2(new YahooFinanceService());
+    const sut = new DeGiroConverterV2(new SecurityService(new YahooFinanceServiceMock()));
 
     // Assert
     expect(sut).toBeTruthy();
@@ -24,7 +25,7 @@ describe("degiroConverterV2", () => {
   it("should process sample CSV file", (done) => {
 
     // Arange
-    const sut = new DeGiroConverterV2(new YahooFinanceService());
+    const sut = new DeGiroConverterV2(new SecurityService(new YahooFinanceServiceMock()));
     const inputFile = "samples/degiro-export.csv";
 
     // Act
@@ -43,7 +44,7 @@ describe("degiroConverterV2", () => {
     it("the input file does not exist", (done) => {
 
       // Arrange
-      const sut = new DeGiroConverterV2(new YahooFinanceService());
+      const sut = new DeGiroConverterV2(new SecurityService(new YahooFinanceServiceMock()));
 
       let tempFileName = "tmp/testinput/degiro-filedoesnotexist.csv";
 
@@ -60,7 +61,7 @@ describe("degiroConverterV2", () => {
     it("the input file is empty", (done) => {
 
       // Arrange
-      const sut = new DeGiroConverterV2(new YahooFinanceService());
+      const sut = new DeGiroConverterV2(new SecurityService());
 
       let tempFileContent = "";
       tempFileContent += "Datum,Tijd,Valutadatum,Product,ISIN,Omschrijving,FX,Mutatie,,Saldo,,Order Id\n";
@@ -85,9 +86,9 @@ describe("degiroConverterV2", () => {
       tempFileContent += `15-12-2022,16:55,15-12-2022,VICI PROPERTIES INC. C,US9256521090,"Koop 1 @ 33,9 USD",,USD,"-33,90",USD,"-33,90",5925d76b-eb36-46e3-b017-a61a6d03c3e7`;
 
       // Mock Yahoo Finance service to throw error.
-      const yahooFinanceService = new YahooFinanceService();
-      jest.spyOn(yahooFinanceService, "getSecurity").mockImplementation(() => { throw new Error("Unit test error"); });
-      const sut = new DeGiroConverterV2(yahooFinanceService);
+      const yahooFinanceServiceMock = new YahooFinanceServiceMock();
+      jest.spyOn(yahooFinanceServiceMock, "search").mockImplementation(() => { throw new Error("Unit test error"); });
+      const sut = new DeGiroConverterV2(new SecurityService(yahooFinanceServiceMock));
 
       // Act
       sut.processFileContents(tempFileContent, () => { done.fail("Should not succeed!"); }, (err: Error) => {
@@ -109,10 +110,10 @@ describe("degiroConverterV2", () => {
     tempFileContent += `15-12-2022,16:55,15-12-2022,VICI PROPERTIES INC. C,US9256521090,DEGIRO Transactiekosten en/of kosten van derden,,EUR,"-1,00",EUR,"31,98",5925d76b-eb36-46e3-b017-a61a6d03c3e7\n`;
     tempFileContent += `15-12-2022,16:55,15-12-2022,VICI PROPERTIES INC. C,US9256521090,"Koop 1 @ 33,9 USD",,USD,"-33,90",USD,"-33,90",5925d76b-eb36-46e3-b017-a61a6d03c3e7`;
 
-    // Mock Yahoo Finance service to return null.
-    const yahooFinanceService = new YahooFinanceService();
-    jest.spyOn(yahooFinanceService, "getSecurity").mockImplementation(() => { return null });
-    const sut = new DeGiroConverterV2(yahooFinanceService);
+    // Mock Yahoo Finance service to return no quotes.
+    const yahooFinanceServiceMock = new YahooFinanceServiceMock();
+    jest.spyOn(yahooFinanceServiceMock, "search").mockImplementation(() => { return Promise.resolve({ quotes: [] }) });
+    const sut = new DeGiroConverterV2(new SecurityService(yahooFinanceServiceMock));
 
     // Bit hacky, but it works.
     const consoleSpy = jest.spyOn((sut as any).progress, "log");

src/converters/degiroConverterV2.ts

@@ -2,7 +2,7 @@ import dayjs from "dayjs";
 import { parse } from "csv-parse";
 import { DeGiroRecord } from "../models/degiroRecord";
 import { AbstractConverter } from "./abstractconverter";
-import { YahooFinanceService } from "../yahooFinanceService";
+import { SecurityService } from "../securityService";
 import { GhostfolioExport } from "../models/ghostfolioExport";
 import YahooFinanceRecord from "../models/yahooFinanceRecord";
 import customParseFormat from "dayjs/plugin/customParseFormat";
@@ -11,8 +11,8 @@ import { GhostfolioOrderType } from "../models/ghostfolioOrderType";
 
 export class DeGiroConverterV2 extends AbstractConverter {
 
-  constructor(yahooFinanceService: YahooFinanceService) {
-    super(yahooFinanceService);
+  constructor(securityService: SecurityService) {
+    super(securityService);
 
     dayjs.extend(customParseFormat);
   }
@@ -90,7 +90,7 @@ export class DeGiroConverterV2 extends AbstractConverter {
         // Look for the security for the current record.
         let security: YahooFinanceRecord;
         try {
-          security = await this.yahooFinanceService.getSecurity(
+          security = await this.securityService.getSecurity(
             record.isin,
             null,
             record.product,

src/converters/etoroConverter.test.ts

@@ -1,6 +1,7 @@
 import { EtoroConverter } from "./etoroConverter";
-import { YahooFinanceService } from "../yahooFinanceService";
+import { SecurityService } from "../securityService";
 import { GhostfolioExport } from "../models/ghostfolioExport";
+import YahooFinanceServiceMock from "../testing/yahooFinanceServiceMock";
 
 describe("etoroConverter", () => {
 
@@ -15,7 +16,7 @@ describe("etoroConverter", () => {
   it("should construct", () => {
 
     // Act
-    const sut = new EtoroConverter(new YahooFinanceService());
+    const sut = new EtoroConverter(new SecurityService(new YahooFinanceServiceMock()));
 
     // Assert
     expect(sut).toBeTruthy();
@@ -24,7 +25,7 @@ describe("etoroConverter", () => {
   it("should process sample CSV file", (done) => {
 
     // Arange
-    const sut = new EtoroConverter(new YahooFinanceService());
+    const sut = new EtoroConverter(new SecurityService(new YahooFinanceServiceMock()));
     const inputFile = "samples/etoro-export.csv";
 
     // Act
@@ -43,7 +44,7 @@ describe("etoroConverter", () => {
     it("the input file does not exist", (done) => {
 
       // Arrange
-      const sut = new EtoroConverter(new YahooFinanceService());
+      const sut = new EtoroConverter(new SecurityService(new YahooFinanceServiceMock()));
 
       let tempFileName = "tmp/testinput/etoro-filedoesnotexist.csv";
 
@@ -60,7 +61,7 @@ describe("etoroConverter", () => {
     it("the input file is empty", (done) => {
 
       // Arrange
-      const sut = new EtoroConverter(new YahooFinanceService());
+      const sut = new EtoroConverter(new SecurityService(new YahooFinanceServiceMock()));
 
       let tempFileContent = "";
       tempFileContent += "Date,Type,Details,Amount,Units,Realized Equity Change,Realized Equity,Balance,Position ID,Asset type,NWA\n";
@@ -84,9 +85,9 @@ describe("etoroConverter", () => {
       tempFileContent += `02/01/2024 00:10:33,Dividend,NKE/USD,0.17,-,0.17,"4,581.91",99.60,2272508626,Stocks,0.00`;
 
       // Mock Yahoo Finance service to throw error.
-      const yahooFinanceService = new YahooFinanceService();
-      jest.spyOn(yahooFinanceService, "getSecurity").mockImplementation(() => { throw new Error("Unit test error"); });
-      const sut = new EtoroConverter(yahooFinanceService);
+      const yahooFinanceServiceMock = new YahooFinanceServiceMock();
+      jest.spyOn(yahooFinanceServiceMock, "search").mockImplementation(() => { throw new Error("Unit test error"); });
+      const sut = new EtoroConverter(new SecurityService(yahooFinanceServiceMock));
 
       // Act
       sut.processFileContents(tempFileContent, () => { done.fail("Should not succeed!"); }, (err: Error) => {
@@ -107,10 +108,10 @@ describe("etoroConverter", () => {
     tempFileContent += "Date,Type,Details,Amount,Units,Realized Equity Change,Realized Equity,Balance,Position ID,Asset type,NWA\n";
     tempFileContent += `02/01/2024 00:10:33,Dividend,NKE/USD,0.17,-,0.17,"4,581.91",99.60,2272508626,Stocks,0.00`;
 
-    // Mock Yahoo Finance service to return null.
-    const yahooFinanceService = new YahooFinanceService();
-    jest.spyOn(yahooFinanceService, "getSecurity").mockImplementation(() => { return null });
-    const sut = new EtoroConverter(yahooFinanceService);
+    // Mock Yahoo Finance service to return no quotes.
+    const yahooFinanceServiceMock = new YahooFinanceServiceMock();
+    jest.spyOn(yahooFinanceServiceMock, "search").mockImplementation(() => { return Promise.resolve({ quotes: [] }) });
+    const sut = new EtoroConverter(new SecurityService(yahooFinanceServiceMock));
 
     // Bit hacky, but it works.
     const consoleSpy = jest.spyOn((sut as any).progress, "log");

src/converters/etoroConverter.ts

@@ -2,16 +2,16 @@ import dayjs from "dayjs";
 import { parse } from "csv-parse";
 import { EtoroRecord } from "../models/etoroRecord";
 import { AbstractConverter } from "./abstractconverter";
-import { YahooFinanceService } from "../yahooFinanceService";
+import { SecurityService } from "../securityService";
 import { GhostfolioExport } from "../models/ghostfolioExport";
 import YahooFinanceRecord from "../models/yahooFinanceRecord";
 import customParseFormat from "dayjs/plugin/customParseFormat";
 import { GhostfolioOrderType } from "../models/ghostfolioOrderType";
 
 export class EtoroConverter extends AbstractConverter {
 
-    constructor(yahooFinanceService: YahooFinanceService) {
-        super(yahooFinanceService);
+    constructor(securityService: SecurityService) {
+        super(securityService);
 
         dayjs.extend(customParseFormat);
     }
@@ -120,7 +120,7 @@ export class EtoroConverter extends AbstractConverter {
 
                 let security: YahooFinanceRecord;
                 try {
-                    security = await this.yahooFinanceService.getSecurity(
+                    security = await this.securityService.getSecurity(
                         null,
                         symbol,
                         null,