Merge pull request #52 from Menighin/v3

V3

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cei-crawler",
-  "version": "2.5.0",
+  "version": "3.0.0",
   "description": "Crawler para pegar dados do Canal Eletronico do Investidor",
   "main": "src/app.js",
   "repository": {
@@ -24,11 +24,8 @@
   "license": "MIT",
   "dependencies": {
     "@babel/runtime": "^7.9.6",
-    "abort-controller": "^3.0.0",
-    "cheerio": "^1.0.0-rc.3",
-    "node-fetch": "^2.6.1",
-    "normalize-html-whitespace": "^1.0.0",
-    "tough-cookie": "^4.0.0"
+    "axios": "^0.21.1",
+    "puppeteer-core": "^10.1.0"
   },
   "devDependencies": {
     "@babel/core": "^7.9.0",

src/lib/AccountStatementCrawler.js

@@ -0,0 +1,42 @@
+const typedefs = require("./typedefs");
+const CeiUtils = require('./CeiUtils');
+const AxiosWrapper = require('./AxiosWrapper');
+
+const URLS = {
+    LIST_DATA: 'https://investidor.b3.com.br/api/extrato/v1/movimentacao/:page',
+};
+
+class AccountStatementCrawler {
+
+    /**
+     * Crawls the tab "Movimentacao" at CEI
+     * @param {Date} startDate - The start date of the range
+     * @param {Date} endDate - The end date of the range
+     * @param {Number} page - The page of the data
+     * @param {typedefs.CeiCrawlerOptions} [options] - Options for the crawler
+     * @returns {Promise<typedefs.CeiListData<typedefs.AccountStatement>} - The account statement
+     */
+    static async getAccountStatement(startDate, endDate, page, options = {}) {
+        const lastExecution = options.lastExecutionInfo.generalDate;
+        const startDateStr = CeiUtils.getDateForQueryParam(startDate || CeiUtils.subtractMonth(lastExecution));
+        const endDateStr = CeiUtils.getDateForQueryParam(endDate || lastExecution);
+
+        if (options.debug)
+            console.log(`[AccountStatementCrawler] Crawling statement for period ${startDateStr} - ${endDateStr}`);
+
+        const response = await AxiosWrapper.request(URLS.LIST_DATA, {
+            queryParams: {
+                dataInicio: startDateStr,
+                dataFim: endDateStr,
+            },
+            pathParams: {
+                page: page
+            }
+        });
+
+        return response;
+    }
+
+}
+
+module.exports = AccountStatementCrawler;

src/lib/AxiosWrapper.js

@@ -0,0 +1,53 @@
+const axios = require('axios').default;
+const https = require('https');
+const { CeiCrawlerError, CeiErrorTypes } = require('./CeiCrawlerError');
+
+class AxiosWrapper {
+
+    static setup(options) {
+        const httpsAgent = new https.Agent({
+            rejectUnauthorized: false,
+        });
+
+        axios.interceptors.request.use(config => {
+            config.headers = {
+                'Authorization': `Bearer ${options.auth.token}`
+            };
+            config.params['cache-guid'] = options.auth['cache-guid'];
+            config.httpsAgent = httpsAgent;
+
+            if (options.debug)
+                console.log(`[AxiosWrapper] ${config.method.toUpperCase()} ${config.url} ${JSON.stringify(config.params)}`);
+
+            return config;
+        });
+    }
+
+    static async request(url, opts = {}) {
+        const pathParams = opts.pathParams || {};
+        const queryParams = opts.queryParams || {};
+        try {
+            const urlWithParams = Object.keys(pathParams)
+                .reduce((p, v) => {
+                    return p.replace(`:${v}`, pathParams[v]);
+                }, url);
+
+            const response = await axios.get(urlWithParams, {
+                params: {
+                    ...queryParams
+                }
+            });
+            return response.data;
+        } catch (e) {
+            const msgStr = e.response.data != null ? (e.response.data.message || e.response.data.trim()) : e.message;
+            const msg = msgStr === '' ? e.message : msgStr;
+
+            if (e.response.status === 401)
+                throw new CeiCrawlerError(CeiErrorTypes.UNAUTHORIZED, msg, e.response.status);
+
+            throw new CeiCrawlerError(CeiErrorTypes.BAD_REQUEST, msg, e.response.status);
+        }
+    }
+}
+
+module.exports = AxiosWrapper;