From acee5925bd2c58649397e8cbcc8c290fdef9f560 Mon Sep 17 00:00:00 2001 From: Furkan Yavuz Date: Thu, 25 Apr 2019 17:50:34 +0300 Subject: [PATCH] Part-2: Implementing Cron Scheduler to daily update GitHub repositories --- config/env.config.js | 4 +- controller/cron.controller.js | 12 +++ controller/repository.controller.js | 115 ++++++++++++++++++++++++++++ index.js | 2 + package.json | 2 + 5 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 controller/cron.controller.js diff --git a/config/env.config.js b/config/env.config.js index acbaebf..fd66cda 100644 --- a/config/env.config.js +++ b/config/env.config.js @@ -1,5 +1,7 @@ module.exports = { "port": process.env.PORT || 3000, "environment": "dev", - "mongoDbUri": process.env.MONGODB_URI || "mongodb://localhost/github-consumer" + "mongoDbUri": process.env.MONGODB_URI || "mongodb://localhost/github-consumer", + "githubEndpoint": "https://api.github.com/graphql", + "githubAccessToken": process.env.GITHUB_ACCESS_TOKEN }; diff --git a/controller/cron.controller.js b/controller/cron.controller.js new file mode 100644 index 0000000..21ba229 --- /dev/null +++ b/controller/cron.controller.js @@ -0,0 +1,12 @@ +const RepositoryController = require('./repository.controller'); +const CronJob = require('cron').CronJob; + +function updateDaily() { + RepositoryController.updateRepositories(); +} + +exports.startCronJobs = function () { + new CronJob('0 0 * * *', function () { + updateDaily() + }, null, true, 'UTC'); +}; diff --git a/controller/repository.controller.js b/controller/repository.controller.js index 34ba897..820f908 100644 --- a/controller/repository.controller.js +++ b/controller/repository.controller.js @@ -1,4 +1,9 @@ const RepositoryModel = require('../model/repository.model'); +const Axios = require('axios'); +const Config = require('../config/env.config'); + +const GITHUB_API_URL = Config.githubEndpoint; +const GITHUB_ACCESS_TOKEN = Config.githubAccessToken; exports.insert = (req, res) => { RepositoryModel.create(req.body) @@ -37,3 +42,113 @@ exports.deleteById = (req, res) => { res.status(204).send({}); }); }; + +// Cron job Update methods + +async function updateDatabase(responseData, owner, name) { + + let createdAt = ''; + let resourcePath = ''; + let tagName = ''; + let releaseDescription = ''; + let homepageUrl = ''; + let repositoryDescription = ''; + let avatarUrl = ''; + + if (responseData.repository.releases) { + + createdAt = responseData.repository.releases.nodes[0].createdAt; + resourcePath = responseData.repository.releases.nodes[0].resourcePath; + tagName = responseData.repository.releases.nodes[0].tagName; + releaseDescription = responseData.repository.releases.nodes[0].description; + homepageUrl = responseData.repository.homepageUrl; + repositoryDescription = responseData.repository.description; + + if (responseData.organization && responseData.organization.avatarUrl) { + avatarUrl = responseData.organization.avatarUrl; + } else if (responseData.user && responseData.user.avatarUrl) { + avatarUrl = responseData.user.avatarUrl; + } + + const repositoryData = { + owner: owner, + name: name, + createdAt: createdAt, + resourcePath: resourcePath, + tagName: tagName, + releaseDescription: releaseDescription, + homepageUrl: homepageUrl, + repositoryDescription: repositoryDescription, + avatarUrl: avatarUrl + }; + + await RepositoryModel.findByOwnerAndName(owner, name) + .then((oldGitHubRelease) => { + if (!oldGitHubRelease[0]) { + RepositoryModel.create(repositoryData); + } else { + RepositoryModel.patchById(oldGitHubRelease[0].id, repositoryData); + } + console.log(`Updated latest release: http://github.com${repositoryData.resourcePath}`); + }); + } +} + +async function getLatestRelease(repository) { + + const owner = repository.owner; + const name = repository.name; + + console.log(`Getting latest release for: http://github.com/${owner}/${name}`); + + const query = ` + query { + organization(login: "${owner}") { + avatarUrl + } + user(login: "${owner}") { + avatarUrl + } + repository(owner: "${owner}", name: "${name}") { + homepageUrl + description + releases(first: 1, orderBy: {field: CREATED_AT, direction: DESC}) { + nodes { + createdAt + resourcePath + tagName + description + } + } + } + }`; + + const jsonQuery = JSON.stringify({query}); + + const headers = { + 'User-Agent': 'Release Tracker', + 'Authorization': `Bearer ${GITHUB_ACCESS_TOKEN}` + }; + + await Axios.post(GITHUB_API_URL, jsonQuery, {headers: headers} + ).then((response) => { + return updateDatabase(response.data.data, owner, name); + }); +} + +async function asyncUpdate() { + + await RepositoryModel.list().then((array) => { + const promises = array.map(getLatestRelease); + + return Promise.all(promises); + }); +} + +exports.updateRepositories = async function update() { + console.log('GitHub Repositories Update Started'); + + await asyncUpdate().then(() => { + console.log('GitHub Repositories Update Finished'); + }); +}; diff --git a/index.js b/index.js index 641699c..3f54154 100644 --- a/index.js +++ b/index.js @@ -3,6 +3,7 @@ const express = require('express'); const bodyParser = require('body-parser'); const Router = require('./config/routes.config'); +const CronController = require('./controller/cron.controller'); const app = express(); @@ -21,6 +22,7 @@ app.use(function (req, res, next) { app.use(bodyParser.json()); Router.routesConfig(app); +CronController.startCronJobs(); app.listen(config.port, function () { console.log('app listening at port %s', config.port); diff --git a/package.json b/package.json index bcfee26..6a064f8 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,9 @@ }, "homepage": "https://github.com/furknyavuz/sample-github-api-consumer-nodejs#readme", "dependencies": { + "axios": "^0.18.0", "body-parser": "1.7.0", + "cron": "^1.7.0", "express": "^4.8.7", "moment": "^2.17.1", "moment-timezone": "^0.5.13",