Skip to content
Permalink
Browse files

feat: news indexing script (#36512)

* feat: Add script to poll Ghost for all published posts and construct index for Algolia.

* feat: Add script to poll Ghost for all published posts and construct index for Algolia.
  • Loading branch information...
scissorsneedfoodtoo authored and raisedadead committed Jul 29, 2019
1 parent 04e9878 commit 9f372d014d4d334a4bf7e829eda064716d0e6260
Showing with 104 additions and 0 deletions.
  1. +2 −0 sample.env
  2. +62 −0 search-indexing/data-sources/news/getAllPosts.js
  3. +39 −0 search-indexing/package-lock.json
  4. +1 −0 search-indexing/package.json
@@ -34,3 +34,5 @@ LOCALE=english

TEST_CHALLENGES_FOR_LANGS=english
DOCKER_HOST_LOCATION=localhost

GHOST_CLIENT_KEY=123abc
@@ -0,0 +1,62 @@
const path = require('path');

const envPath = path.resolve(__dirname, '../../../.env');
require('dotenv').config({ path: envPath });

const { GHOST_CLIENT_KEY } = process.env;

const axios = require('axios');
const fs = require('fs');

const getJson = async url => {
return axios
.get(url)
.then(res => res.data)
.catch(err => console.log(err));
};

const constructIndex = async () => {
let currPage = 1;
let lastPage = 5;
const delay = m => new Promise(resolve => setTimeout(resolve, m));
const posts = [];

while (currPage && currPage <= lastPage) {
const data = await getJson(
`https://www.freecodecamp.org/news/ghost/api/v2/content/posts/?key=${GHOST_CLIENT_KEY}&include=tags,authors&page=${currPage}`
);

data.posts.forEach(post => {
const thisPost = {
title: post.title,
author: {
name: post.primary_author.name,
url: post.primary_author.url,
profileImage: post.primary_author.profile_image
},
tags: post.tags.map(obj => {
return {
name: obj.name,
url: obj.url
};
}),
url: post.url,
featureImage: post.feature_image,
ghostId: post.id,
publishedAt: post.published_at
};

posts.push(thisPost);
});

currPage = data.meta.pagination.next;
lastPage = data.meta.pagination.pages;

console.log(posts);

fs.writeFileSync('posts.json', JSON.stringify(posts, null, 2));
await delay(1000);
}
};

constructIndex();

Some generated files are not rendered by default. Learn more.

@@ -28,6 +28,7 @@
"homepage": "https://github.com/freeCodeCamp/freeCodeCamp#readme",
"dependencies": {
"algoliasearch": "^3.25.1",
"axios": "^0.19.0",
"chalk": "^2.3.2",
"debug": "^4.1.1",
"dotenv": "^6.2.0",

0 comments on commit 9f372d0

Please sign in to comment.
You can’t perform that action at this time.