diff --git a/README.md b/README.md
index 45d18eb..79b99c6 100644
--- a/README.md
+++ b/README.md
@@ -95,6 +95,7 @@ ___
requests: [
// Every request is passed to an `axios.create` instance
{
+ skip: false, // skip a request
endpoint: 'https://awesome-api.com/give-me-my-blazing-fast-data', // default = `axios.url`
method: 'get', // default = `axios.method || 'get'`
// The params of the request, you can pass a graph-ql query too, check it in the example folder
@@ -102,11 +103,21 @@ ___
// Use this to map the response in a custom `key`
field: 'categories', // default = `the actual index in this array of requests`
// Usually, your data is always nested in one or more objects
- pathToData: 'data.categories.listCategories',
+ pathToData: 'data.categories.listCategories.items', // Check `dot-object` to know more
// In case of no-response, what value do you prefer for your empty data?
emptyValue: [],
// Like headers, authentication or everything is required by this request
config: {},
+ // New, available after with version >= 1.2
+ id: -1, // useful for debugging purpose, default = `the actual index in this array of requests + 1`
+ // For recursive api calls with lists or nested data
+ pagination: {
+ pathBodyToPaginationParamValue: 'variables.nextToken', // [REQUIRED]
+ pathResponseToTheNextPaginationValue: 'data.categories.listCategories.nextToken', // useful with Graphql, default = null
+ step: 1, // It always start with the `pathBodyToPaginationParamValue` param if specified, so this is used to increase this numeric value
+ maxIterations: 15, // Max deep of iterations
+ lastPaginationValue: null // stop the recursion if 'pathResponseToTheNextPaginationValue' or 'pathBodyToPaginationParamValue' is this value, and is useful to stop the Iteration if the next value is matched
+ },
},
],
},
@@ -223,6 +234,7 @@ ___
- Nuxt [staticDir](https://nuxtjs.org/api/configuration-dir/);
- Nuxt [buildModules](https://nuxtjs.org/guide/modules/#build-only-modules);
- [fs-extra.outputJson](https://github.com/jprichardson/node-fs-extra/blob/master/docs/outputJson.md);
+- `pathTo` data handled with [dot-notation](https://github.com/rhalff/dot-object);
- [Axios.create](https://github.com/axios/axios#creating-an-instance);
- [@nuxtjs/axios](https://axios.nuxtjs.org/).
diff --git a/example/nuxt.config.js b/example/nuxt.config.js
index f40256f..ec35449 100644
--- a/example/nuxt.config.js
+++ b/example/nuxt.config.js
@@ -3,7 +3,10 @@ import { resolve } from 'path';
import * as PACKAGE from '../package.json';
// GraphQL RAW Queries
-import { GRAPHQL, LOCATIONS } from './graphql';
+import {
+ GRAPHQL,
+ LOCATIONS,
+} from './graphql';
// Configuration
const apisToFile = {
@@ -11,13 +14,56 @@ const apisToFile = {
baseURL: 'https://jsonplaceholder.typicode.com',
},
requests: [
+ // Rest API
{
endpoint: '/posts',
field: 'posts',
+ body: {
+ params: {
+ '_page': 1,
+ '_limit': 10,
+ },
+ },
+ },
+ {
+ endpoint: '/posts',
+ field: 'postsPaginated',
+ body: {
+ params: {
+ '_page': 1,
+ '_limit': 10,
+ },
+ },
+ // New settings
+ pagination: {
+ pathBodyToPaginationParamValue: 'params._page',
+ maxIterations: 3,
+ },
},
{
endpoint: '/comments',
field: 'comments',
+ body: {
+ params: {
+ '_limit': 10,
+ },
+ },
+ },
+ {
+ endpoint: '/comments',
+ field: 'commentsPaginated',
+ body: {
+ params: {
+ '_page': 1,
+ '_limit': 15,
+ },
+ },
+ // New settings
+ pagination: {
+ pathBodyToPaginationParamValue: 'params._page',
+ step: 2,
+ lastPaginationValue: 7,
+ },
},
// GraphQL
{
@@ -31,7 +77,7 @@ const apisToFile = {
{
endpoint: 'https://kdonz3bavvbbhmocoletnw4w2q.appsync-api.eu-west-1.amazonaws.com/graphql',
method: 'post',
- field: 'locations',
+ field: 'graphqlLocations',
pathToData: 'data.listLocations',
config: {
headers: {
@@ -42,7 +88,25 @@ const apisToFile = {
},
emptyValue: {},
body: LOCATIONS,
+ },
+ {
+ endpoint: 'https://kdonz3bavvbbhmocoletnw4w2q.appsync-api.eu-west-1.amazonaws.com/graphql',
+ method: 'post',
+ field: 'graphqlLocationsPaginated',
+ pathToData: 'data.listLocations.items',
+ config: {
+ headers: {
+ 'Content-Type': 'application/json',
+ 'x-api-key': 'da2-jy2nym3ybbgubehdqhf5rjgbxq',
+ 'x-region': 'eu-west-1',
+ },
+ },
+ body: LOCATIONS,
// New settings
+ pagination: {
+ pathResponseToTheNextPaginationValue: 'data.listLocations.nextToken',
+ pathBodyToPaginationParamValue: 'variables.nextToken',
+ },
},
],
};
@@ -51,7 +115,7 @@ const apisToFile = {
export default {
// Plugin options
apisToFile,
- // Other options
+ // Options
modern: true,
srcDir: __dirname,
rootDir: resolve(
@@ -68,11 +132,34 @@ export default {
'../lib/module'
),
],
+ watch: [
+ resolve(
+ __dirname,
+ '../lib/module'
+ ),
+ ],
+ // Meta
head: {
htmlAttrs: {
lang: 'en',
},
title: PACKAGE.name,
+ link: [
+ {
+ once: true,
+ hid: 'favicon',
+ rel: 'shortcut icon',
+ type: 'image/x-icon',
+ href: '/favicon.ico',
+ },
+ {
+ once: true,
+ hid: 'humans',
+ rel: 'author',
+ type: 'text/plain',
+ href: '/humans.txt',
+ },
+ ],
meta: [
{
once: true,
@@ -101,6 +188,67 @@ export default {
'../docs'
),
},
+ /*
+ * Build
+ */
+ build: {
+ loaders: {
+ vue: {
+ compilerOptions: {
+ preserveWhitespace: false,
+ whitespace: 'condense',
+ },
+ },
+ },
+ /*
+ ** Minifier
+ */
+ html: {
+ minify: {
+ collapseBooleanAttributes: true,
+ decodeEntities: true,
+ minifyCSS: true,
+ minifyJS: true,
+ processConditionalComments: true,
+ collapseInlineTagWhitespace: true,
+ removeOptionalTags: true,
+ removeAttributeQuotes: true,
+ removeEmptyAttributes: true,
+ removeRedundantAttributes: true,
+ trimCustomFragments: true,
+ useShortDoctype: true,
+ collapseWhitespace: true,
+ removeScriptTypeAttributes: true,
+ removeStyleLinkTypeAttributes: true,
+ removeComments: true,
+ continueOnParseError: true,
+ },
+ },
+ /*
+ ** Run lint on save
+ */
+ extend(
+ config,
+ {
+ isDev,
+ isClient,
+ },
+ ) {
+
+ /*
+ ** ESLint loaded
+ */
+ isDev && isClient && config.module.rules.push(
+ {
+ enforce: 'pre',
+ test: /\.(js|vue)$/,
+ loader: 'eslint-loader',
+ exclude: /(node_modules)/,
+ },
+ );
+
+ },
+ },
/*
* Server
*/
diff --git a/example/pages/index.css b/example/pages/index.css
new file mode 100644
index 0000000..6465800
--- /dev/null
+++ b/example/pages/index.css
@@ -0,0 +1,30 @@
+.page h3,
+.page p {
+
+ display: block;
+ width: 100%;
+ max-width: 500px;
+ margin: 0;
+ text-overflow: ellipsis;
+ white-space: nowrap;
+ overflow: hidden;
+
+}
+
+.page h3 {
+
+ margin-top: 16px;
+
+}
+
+.page em {
+
+ display: block;
+
+}
+
+.page button {
+
+ margin: 32px;
+
+}
diff --git a/example/pages/index.vue b/example/pages/index.vue
index c96fbff..f1b204a 100644
--- a/example/pages/index.vue
+++ b/example/pages/index.vue
@@ -1,8 +1,31 @@
-
+
NUXT Apis to file
Nuxt module to merge and transform API calls into a single file, during the build phase.. Like a payload extractor
+
+
+
+ Total number of posts preloaded:
+
+
+
+
+
+
+
+ Total number of posts (paginated) preloaded:
+
+
+
+
+
-
+
+
- Total number graphql data preloaded:
+ GraphQL Object loaded:
+
+
+ Total number of locations (paginated) preloaded:
+
+
+
+
+
@@ -80,3 +116,8 @@
},
};
+
+
diff --git a/example/static/.htaccess b/example/static/.htaccess
new file mode 100644
index 0000000..16008cc
--- /dev/null
+++ b/example/static/.htaccess
@@ -0,0 +1,121 @@
+# SP BEGIN php handler
+
+ AddHandler fcgid-script .php .php5 .php7 .phtml
+ FcgidWrapper /usr/local/cpanel/cgi-sys/sp-ea-php74 .php
+ FcgidWrapper /usr/local/cpanel/cgi-sys/sp-ea-php74 .php5
+ FcgidWrapper /usr/local/cpanel/cgi-sys/sp-ea-php74 .php7
+ FcgidWrapper /usr/local/cpanel/cgi-sys/sp-ea-php74 .phtml
+
+# SP END php handler
+
+# Validation + force https
+
+
+ RewriteEngine On
+
+ # Force HTTPS
+ RewriteCond %{HTTPS} !on
+ RewriteRule (.*) https://%{HTTP_HOST}%{REQUEST_URI}
+
+ # Rules to correctly serve gzip compressed CSS and JS files.
+ # Requires both mod_rewrite and mod_headers to be enabled.
+
+
+ # Serve brotli compressed CSS files if they exist and the client accepts gzip.
+ RewriteCond %{HTTP:Accept-encoding} br
+ RewriteCond %{REQUEST_FILENAME}\.br -s
+ RewriteRule ^(.*)\.css $1\.css\.br [QSA]
+
+ # Serve gzip compressed CSS files if they exist and the client accepts gzip.
+ RewriteCond %{HTTP:Accept-encoding} gzip
+ RewriteCond %{REQUEST_FILENAME}\.gz -s
+ RewriteRule ^(.*)\.css $1\.css\.gz [QSA]
+
+ # Serve brotli compressed JS files if they exist and the client accepts gzip.
+ RewriteCond %{HTTP:Accept-encoding} br
+ RewriteCond %{REQUEST_FILENAME}\.br -s
+ RewriteRule ^(.*)\.js$ $1\.js\.br [QSA]
+
+ # Serve gzip compressed JS files if they exist and the client accepts gzip.
+ RewriteCond %{HTTP:Accept-encoding} gzip
+ RewriteCond %{REQUEST_FILENAME}\.gz -s
+ RewriteRule ^(.*)\.js$ $1\.js\.gz [QSA]
+
+ # Serve correct content types, and prevent mod_deflate double gzip.
+ RewriteRule \.css\.gz$ - [T=text/css,E=no-gzip:1]
+ RewriteRule \.js\.gz$ - [T=text/javascript,E=no-gzip:1]
+ RewriteRule \.css\.br$ - [T=text/css,E=no-gzip:1]
+ RewriteRule \.js\.br$ - [T=text/javascript,E=no-gzip:1]
+
+
+ # Serve correct encoding type.
+ Header set Content-Encoding gzip
+ # Force proxies to cache gzipped & non-gzipped css/js files separately.
+ Header append Vary Accept-Encoding
+
+
+
+ # Serve correct encoding type.
+ Header set Content-Encoding br
+ # Force proxies to cache gzipped & non-gzipped css/js files separately.
+ Header append Vary Accept-Encoding
+
+
+
+
+
+
+# Compression BEGIN
+
+
+ AddOutputFilterByType DEFLATE application/json
+ AddOutputFilterByType DEFLATE application/javascript
+ AddOutputFilterByType DEFLATE application/rss+xml
+ AddOutputFilterByType DEFLATE application/vnd.ms-fontobject
+ AddOutputFilterByType DEFLATE application/x-font
+ AddOutputFilterByType DEFLATE application/x-font-opentype
+ AddOutputFilterByType DEFLATE application/x-font-otf
+ AddOutputFilterByType DEFLATE application/x-font-truetype
+ AddOutputFilterByType DEFLATE application/x-font-ttf
+ AddOutputFilterByType DEFLATE application/x-javascript
+ AddOutputFilterByType DEFLATE application/xhtml+xml
+ AddOutputFilterByType DEFLATE application/xml
+ AddOutputFilterByType DEFLATE font/opentype
+ AddOutputFilterByType DEFLATE font/otf
+ AddOutputFilterByType DEFLATE font/ttf
+ AddOutputFilterByType DEFLATE image/svg+xml
+ AddOutputFilterByType DEFLATE image/x-icon
+ AddOutputFilterByType DEFLATE text/css
+ AddOutputFilterByType DEFLATE text/html
+ AddOutputFilterByType DEFLATE text/javascript
+ AddOutputFilterByType DEFLATE text/plain
+ AddOutputFilterByType DEFLATE text/xml
+
+
+# Compression END
+
+# Service worker BEGIN
+
+
+ # YEAR
+
+ Header set Cache-Control "max-age=29030400"
+
+
+ # WEEK
+
+ Header set Cache-Control "max-age=604800"
+
+
+ # 60 MIN
+
+ Header set Cache-Control "max-age=3000"
+
+
+ # Service worker
+
+ Header set Cache-Control "max-age=0, no-cache"
+
+
+
+# Service worker END
diff --git a/example/static/favicon.ico b/example/static/favicon.ico
new file mode 100644
index 0000000..9e2bb99
Binary files /dev/null and b/example/static/favicon.ico differ
diff --git a/example/static/humans.txt b/example/static/humans.txt
new file mode 100644
index 0000000..8da229d
--- /dev/null
+++ b/example/static/humans.txt
@@ -0,0 +1,10 @@
+/* TEAM */
+Full stack developer: Luca Iaconelli
+Twitter: @luxdamore
+Instagram: @luxdamore
+From: Faenza, Italia
+
+/* SITE */
+Language: Italian
+Doctype: HTML5
+IDE: Visual Studio Code
diff --git a/example/static/icon.png b/example/static/icon.png
new file mode 100644
index 0000000..2b5660d
Binary files /dev/null and b/example/static/icon.png differ
diff --git a/example/static/robots.txt b/example/static/robots.txt
new file mode 100644
index 0000000..c2a49f4
--- /dev/null
+++ b/example/static/robots.txt
@@ -0,0 +1,2 @@
+User-agent: *
+Allow: /
diff --git a/example/store/build-data.js b/example/store/build-data.js
index 57084c1..fc79ca9 100644
--- a/example/store/build-data.js
+++ b/example/store/build-data.js
@@ -11,29 +11,42 @@ export const actions = {
{ commit }
) {
- let preloadedComments = []
- , preloadedPosts = []
+ let preloadedPosts = []
+ , preloadedCommentsPaginated = []
+ , preloadedComments = []
+ , preloadedPostsPaginated = []
, preloadedGraphQl = {}
, preloadedLocations = {}
+ , preloadedLocationsPaginated = []
;
try {
const {
- comments,
posts,
+ postsPaginated,
+ comments,
+ commentsPaginated,
graphql,
- locations,
+ graphqlLocations,
+ graphqlLocationsPaginated,
} = await getFile();
- if( comments )
- preloadedComments = comments;
if( posts )
preloadedPosts = posts;
+ if( postsPaginated )
+ preloadedPostsPaginated = postsPaginated;
+ if( comments )
+ preloadedComments = comments;
+ if( commentsPaginated )
+ preloadedCommentsPaginated = commentsPaginated;
+
if( graphql )
preloadedGraphQl = graphql;
- if( preloadedLocations )
- preloadedLocations = locations;
+ if( graphqlLocations )
+ preloadedLocations = graphqlLocations;
+ if( graphqlLocationsPaginated )
+ preloadedLocationsPaginated = graphqlLocationsPaginated;
} catch( e ) {
@@ -45,6 +58,23 @@ export const actions = {
}
+
+ commit(
+ 'posts/SET_ITEMS',
+ preloadedPosts,
+ {
+ root: true,
+ }
+ );
+
+ commit(
+ 'posts/SET_ITEMS_PAGINATED',
+ preloadedPostsPaginated,
+ {
+ root: true,
+ }
+ );
+
commit(
'comments/SET_ITEMS',
preloadedComments,
@@ -54,8 +84,16 @@ export const actions = {
);
commit(
- 'posts/SET_ITEMS',
- preloadedPosts,
+ 'comments/SET_ITEMS_PAGINATED',
+ preloadedCommentsPaginated,
+ {
+ root: true,
+ }
+ );
+
+ commit(
+ 'graphql/SET_ITEMS',
+ preloadedGraphQl,
{
root: true,
}
@@ -70,8 +108,8 @@ export const actions = {
);
commit(
- 'graphql/SET_ITEMS',
- preloadedGraphQl,
+ 'locations/SET_ITEMS_PAGINATED',
+ preloadedLocationsPaginated,
{
root: true,
}
diff --git a/example/store/comments.js b/example/store/comments.js
index 8815ed5..524e498 100644
--- a/example/store/comments.js
+++ b/example/store/comments.js
@@ -1,6 +1,7 @@
export const state = () => (
{
items: [],
+ paginated: [],
}
);
@@ -13,4 +14,12 @@ export const mutations = {
state.items = items;
},
+ SET_ITEMS_PAGINATED(
+ state,
+ items,
+ ) {
+
+ state.paginated = items;
+
+ },
};
diff --git a/example/store/locations.js b/example/store/locations.js
index 375b52e..2759c75 100644
--- a/example/store/locations.js
+++ b/example/store/locations.js
@@ -1,6 +1,7 @@
export const state = () => (
{
data: {},
+ paginated: [],
}
);
@@ -13,4 +14,12 @@ export const mutations = {
state.data = value;
},
+ SET_ITEMS_PAGINATED(
+ state,
+ items,
+ ) {
+
+ state.paginated = items;
+
+ },
};
diff --git a/example/store/posts.js b/example/store/posts.js
index 8815ed5..524e498 100644
--- a/example/store/posts.js
+++ b/example/store/posts.js
@@ -1,6 +1,7 @@
export const state = () => (
{
items: [],
+ paginated: [],
}
);
@@ -13,4 +14,12 @@ export const mutations = {
state.items = items;
},
+ SET_ITEMS_PAGINATED(
+ state,
+ items,
+ ) {
+
+ state.paginated = items;
+
+ },
};
diff --git a/lib/module.js b/lib/module.js
index 141f323..f420354 100644
--- a/lib/module.js
+++ b/lib/module.js
@@ -3,6 +3,7 @@ import { resolve } from 'path';
// External
import { create, all } from 'axios';
+import { str, pick } from 'dot-object';
import { outputJson } from 'fs-extra';
// Log
@@ -27,24 +28,58 @@ const moduleName = 'apis-to-file'
dir: null,
requests: [],
axios: {},
+ // New
+ genericErrorMessage: 'Generic error',
+ pagination: null,
}
- , checkIfUndefinedAndReturnDifferentData = (
- check,
- value,
- ) => {
+;
+
+// Find the data
+function findValue(
+ pathway,
+ data = {},
+ emptyValue = [],
+) {
+
+ if( ! pathway ) {
- return typeof check === 'undefined'
- ? value
- : check
- ;
+ if( typeof data !== 'undefined' && data !== null )
+ return data;
+
+ return emptyValue;
}
-;
+
+ try {
+
+ const value = pick(
+ pathway,
+ data
+ );
+
+ if( typeof value !== 'undefined' && value !== null )
+ return value;
+
+ } catch( e ) {
+
+ logger.error(
+ '\x1B[31m%s\x1B[0m',
+ moduleName,
+ '[dot-object] error',
+ e
+ );
+
+ }
+
+ return emptyValue;
+
+}
export default async function(
moduleOptions,
) {
+ // Default options
const options = {
... defaultConfig,
hideGenericMessagesInConsole: ! this.options.dev,
@@ -54,59 +89,101 @@ export default async function(
... this.options.apisToFile || {},
};
+ // Check for requests
if( ! options.requests || ! options.requests.length ) {
- ! options.hideErrorsInConsole && logger.info(
+ logger.info(
'\x1B[32m%s\x1B[0m',
moduleName,
- 'skipping, no requests found in the configuration',
+ 'skipping, no requests found in the configuration object',
);
return;
}
- // Promise generator
+ // Promises generator
const axiosWithConfig = create(
options.axios,
)
- , requests = options.requests.filter(
+ // Requests filtering
+ , validRequests = options.requests.filter(
obj => ! obj.skip,
)
- , promises = []
+ , {
+ requestsInParallel,
+ requestsWithPagination,
+ } = validRequests.reduce(
+ (
+ accumulator,
+ item,
+ index
+ ) => {
+
+ if( ! item.id )
+ item.id = index + 1;
+
+ if( item.pagination && item.pagination.pathBodyToPaginationParamValue ) {
+
+ accumulator.requestsWithPagination.push(
+ item
+ );
+
+ } else {
+
+ accumulator.requestsInParallel.push(
+ item
+ );
+
+ }
+
+ return accumulator;
+
+ },
+ {
+ requestsInParallel: [],
+ requestsWithPagination: [],
+ }
+ )
+ , errors = []
;
- for( let index = 0; index < requests.length; index ++ ) {
-
- const request = requests[ index ]
- , {
- method = options.axios.method || 'get',
- endpoint = options.axios.url,
- body = {},
- config = {},
- } = request
- ;
-
- promises.push(
- axiosWithConfig[ method ](
- endpoint,
- body,
- config,
- ),
- );
+ let fileContent = {};
- }
+ // Parallel requests
+ if( requestsInParallel.length ) {
- let fileContent = null;
+ // Handle results
+ const promises = [];
- try {
+ // Parallel promises generation
+ for( let index = 0; index < requestsInParallel.length; index ++ ) {
+ const request = requestsInParallel[ index ]
+ , {
+ method = options.axios.method || 'get',
+ endpoint = options.axios.url,
+ body = {},
+ config = {},
+ } = request
+ ;
+
+ promises.push(
+ axiosWithConfig[ method ](
+ endpoint,
+ body,
+ config,
+ ),
+ );
+
+ }
+
+ // API
const responses = await all(
- promises,
- )
- , errors = []
- ;
+ promises,
+ );
+ // Start the file creation
fileContent = responses.reduce(
(
accumulator,
@@ -114,48 +191,31 @@ export default async function(
index,
) => {
+ const {
+ id,
+ field: key = index,
+ emptyValue,
+ pathToData,
+ } = requestsInParallel[ index ]
+ , value = findValue(
+ pathToData,
+ data,
+ emptyValue
+ )
+ ;
+
+ // Keep errors for later
data.error && errors.push(
- data.error
+ `[${ id }] ${ data.error }`
);
data.errors && data.errors.length && errors.push(
... data.errors.map(
- error => error.message || 'Generic error',
+ error => `[${ id }] ${ error.message }`
),
);
- const {
- field: key = index,
- emptyValue = [],
- pathToData,
- } = requests[ index ]
- , value = (
- pathToData
- ? pathToData
- .split(
- '.',
- )
- .reduce(
- (
- acc,
- key,
- ) => (
- typeof acc !== 'undefined' && acc !== null
- ? checkIfUndefinedAndReturnDifferentData(
- acc[ key ],
- emptyValue
- )
- : acc || {}
- ),
- data || {},
- )
- : checkIfUndefinedAndReturnDifferentData(
- data,
- emptyValue
- )
- )
- ;
-
+ // Return data
return {
... accumulator,
[ key ]: value,
@@ -165,38 +225,201 @@ export default async function(
{},
);
- // Console
- errors.length && ! options.hideErrorsInConsole && logger.error(
- '\x1B[31m%s\x1B[0m',
+ }
+
+ // Recursive requests
+ if( requestsWithPagination.length ) {
+
+ // Pagination
+ for( let index = 0; index < requestsWithPagination.length; index ++ ) {
+
+ const request = requestsWithPagination[ index ]
+ , {
+ method = options.axios.method || 'get',
+ endpoint = options.axios.url,
+ body = {},
+ config = {},
+ // Paths
+ id,
+ field: key = index,
+ emptyValue = [],
+ pathToData,
+ // Pagination
+ pagination: {
+ pathBodyToPaginationParamValue,
+ pathResponseToTheNextPaginationValue = null,
+ step = 1,
+ maxIterations = 15,
+ lastPaginationValue = null,
+ },
+ } = request
+ // Recursive data
+ , recursion = async() => {
+
+ // Value and limit control
+ let value = emptyValue
+ , count = 1
+ // Params
+ , params = body
+ ;
+
+ async function next(
+ token
+ ) {
+
+ if( token ) {
+
+ params = str(
+ pathBodyToPaginationParamValue,
+ token,
+ params
+ );
+
+ }
+
+ const { data = {} } = await axiosWithConfig[ method ](
+ endpoint,
+ params,
+ config,
+ )
+ , newValue = findValue(
+ pathToData,
+ data,
+ emptyValue,
+ )
+ ;
+
+ if( value && newValue ) {
+
+ if(
+ Array.isArray(
+ value
+ )
+ ) {
+
+ value.push(
+ ... newValue
+ );
+
+ } else {
+
+ value = {
+ ... value,
+ ... newValue,
+ };
+
+ }
+
+ }
+
+ // Keep errors for later
+ data.error && errors.push(
+ `[${ id }] ${ data.error }`
+ );
+
+ data.errors && data.errors.length && errors.push(
+ ... data.errors.map(
+ error => `[${ id }] ${ error.message }`
+ ),
+ );
+
+ const nextToken = (
+ pathResponseToTheNextPaginationValue
+ ? findValue(
+ pathResponseToTheNextPaginationValue,
+ data,
+ null
+ )
+ : (
+ parseInt(
+ findValue(
+ pathBodyToPaginationParamValue,
+ params,
+ step
+ ),
+ 10
+ ) + step
+ )
+ )
+ ;
+
+ if( ! nextToken || nextToken === lastPaginationValue || maxIterations <= count )
+ return value;
+
+ count ++;
+
+ await next(
+ nextToken
+ );
+
+ }
+
+ await next();
+
+ return value;
+
+ }
+ // Do it
+ , data = await recursion()
+ ;
+
+ fileContent[ key ] = data;
+
+ }
+
+ }
+
+ // Console
+ if( ! options.hideGenericMessagesInConsole ) {
+
+ logger.info(
+ '\x1B[32m%s\x1B[0m',
moduleName,
- 'error during request',
- errors
+ 'total number of Parallel APIs requests',
+ requestsInParallel.length,
);
- ! options.hideGenericMessagesInConsole && logger.info(
+ logger.info(
'\x1B[32m%s\x1B[0m',
moduleName,
- 'total number of APIs calls',
- responses.length,
+ 'total number of Parallel APIs requests',
+ requestsWithPagination.length,
);
- } catch( e ) {
+ logger.info(
+ '\x1B[32m%s\x1B[0m',
+ moduleName,
+ 'total number of APIs requests',
+ validRequests.length,
+ );
- ! options.hideErrorsInConsole && logger.error(
+ }
+
+ if( ! options.hideErrorsInConsole && errors.length ) {
+
+ logger.error(
'\x1B[31m%s\x1B[0m',
moduleName,
- 'there was a problem calling some APIs',
- e,
+ 'total number of errors',
+ errors.length
+ );
+
+ logger.error(
+ '\x1B[31m%s\x1B[0m',
+ moduleName,
+ 'recorded errors',
+ errors
);
}
- // File generator
+ // File data
const completeFilePath = resolve(
__dirname,
`${ options.file.startFromStaticDir ? this.options.dir.static : this.options.srcDir }/${ options.file.path || '' }/${ options.dir || '' }/${ options.file.name || moduleName }.${ options.file.ext || 'json' }`,
);
+ // File generation
try {
await outputJson(
@@ -225,6 +448,7 @@ export default async function(
}
+// Exports
const meta = PACKAGE;
export { meta };
diff --git a/lib/plugin.js b/lib/plugin.js
deleted file mode 100644
index 2d1ec23..0000000
--- a/lib/plugin.js
+++ /dev/null
@@ -1 +0,0 @@
-export default () => {};
diff --git a/test/module.test.js b/test/module.test.js
index f226193..ea756d1 100644
--- a/test/module.test.js
+++ b/test/module.test.js
@@ -62,51 +62,66 @@ describe(
}
);
+ // Utils
+ const getElement = async(
+ selector,
+ value
+ ) => {
+
+ const html = await get(
+ BASE_URL
+ )
+ , { window } = new JSDOM(
+ html
+ )
+ , element = window.document.querySelector(
+ selector
+ )
+ , number = element.querySelector(
+ '.number'
+ )
+ , numberValue = number.textContent
+ ;
+
+ expect(
+ element
+ ).toBeDefined();
+
+ expect(
+ number
+ ).toBeDefined();
+
+ expect(
+ numberValue
+ ).toBeDefined();
+
+ expect(
+ parseInt(
+ numberValue
+ )
+ ).toEqual(
+ value
+ );
+
+ };
+
+ // Rest API
describe(
- 'data',
+ 'rest-api',
() => {
- const getElement = async(
- selector,
- value
- ) => {
-
- const html = await get(
- BASE_URL
- )
- , { window } = new JSDOM(
- html
- )
- , element = window.document.querySelector(
- selector
- )
- , number = element.querySelector(
- '.number'
- )
- , numberValue = number.textContent
- ;
-
- expect(
- element
- ).toBeDefined();
-
- expect(
- number
- ).toBeDefined();
-
- expect(
- numberValue
- ).toBeDefined();
+ // Posts and comments
+ test(
+ 'posts',
+ async() => {
- expect(
- parseInt(
- numberValue
- )
- ).toEqual(
- value
- );
+ await getElement(
+ '.posts',
+ 10,
+ );
- };
+ }
+ );
test(
'comments',
@@ -114,26 +129,54 @@ describe(
await getElement(
'.comments',
- 500,
+ 10,
);
}
);
- test(
- 'posts',
- async() => {
+ // paginated
+ describe(
+ 'paginated',
+ () => {
- await getElement(
- '.posts',
- 100,
+ test(
+ 'posts',
+ async() => {
+
+ await getElement(
+ '.posts-paginated',
+ 30,
+ );
+
+ }
+ );
+
+ test(
+ 'comments',
+ async() => {
+
+ await getElement(
+ '.comments-paginated',
+ 45,
+ );
+
+ }
);
}
);
+ }
+ );
+
+ // GraphQL
+ describe(
+ 'graphql',
+ () => {
+
test(
- 'graphql',
+ 'render',
async() => {
const html = await get(
@@ -149,6 +192,37 @@ describe(
}
);
+ test(
+ 'locations',
+ async() => {
+
+ await getElement(
+ '.locations',
+ 11,
+ );
+
+ }
+ );
+
+ describe(
+ 'paginated',
+ () => {
+
+ test(
+ 'locations',
+ async() => {
+
+ await getElement(
+ '.locations-paginated',
+ 20,
+ );
+
+ }
+ );
+
+ }
+ );
+
}
);
Total number of comments preloaded: @@ -14,20 +37,21 @@ />