task: Update oss-contributions and adjust github lambda

README.md

@@ -139,6 +139,17 @@ Lever webhooks are extremely limited so we have to write our own lambda to check
 
 The zap is within the zapier account registered to `apis@yld.io`
 
+#### Contributions
+
+We have 4 lambdas to update contribution data on contentful, considering each contribution type (_pullRequest_, _pullRequestReview_, _commit_, _issue_):
+
+- `contributions-pull-requests`
+- `contributions-pull-request-reviews`
+- `contributions-commits`
+- `contributions-issues`
+
+Each time one of these lambdas runs, it'll fetch the contributions using the github graphQL API and _Members Data_ from Contenful, considering each members working periods, and update the contribution list for that type on Contentful (_OS contributions_).
+
 #### Github
 
 Local development requires:
@@ -151,7 +162,7 @@ Local development requires:
 
 `./src/functions/github.js`
 
-The aim of this lammbda is to have up to date metrics of yld's open source contribution on the site. We do this by aggregating data on a lambda and publishing it to Contentful. It is split into two sections, `repos` and `meta`.
+The aim of this lammbda is to have up to date metrics of yld's open source contribution on the site. We do this by aggregating the contributions from different types, that should be up to date on Contenful via the contribution types lambdas, processing the data and publishing it to Contentful. It is split into two sections, `repos` and `meta`.
 
 - Repos
   We want to store data regarding specific repos that members of yld have contributed to e.g. node, react, enyzyme etc. By creating a `githubRepo` content type on Contentful with only a URL value edtiable, we are able to create references to these repos that can be used throughout the site but have metrics that are _only_ available to update via the API medium. This ensures data is always valid and accurate.
@@ -161,6 +172,21 @@ The aim of this lammbda is to have up to date metrics of yld's open source contr
 
 Main [`@yld.io/oss-stats`](https://www.npmjs.com/package/@yldio/oss-stats) to aggregate all open source contributions for `yldio` organization members.
 
+#### Members
+
+This lambda takes a log with members join/leave dates from contentful(_Member log_), as:
+
+```
+Sérgio Ramos,Joined YLD,Mar 2015,sergioramos
+Filipe Pinheiro,Joined YLD,Sep 2015,fampinheiro
+Fábio Moreira,Joined YLD,Apr 2018,fabiommmoreira
+Fábio Antunes (2),Joined YLD,Feb 2017,FabioAntunes
+Fábio Antunes (2),Left YLD,May 2018,FabioAntunes
+Fábio Antunes (1),Joined YLD,Mar 2019,FabioAntunes
+```
+
+calls [`@yld.io/oss-stats`](https://www.npmjs.com/package/@yldio/oss-stats) to fetch additional data from github (id and url) and return an object with the member data formatted in the way it's intended to be used within the contributions aggregator and stores it on Contenful - _Members Data_.
+
 ## Content Model notes
 
 This section serves as a information repository for some of our content models, stating what they represent and explaining some of their fields, if needed.

gatsby-config.js

@@ -28,6 +28,7 @@ module.exports = {
   },
   plugins: [
     `gatsby-plugin-react-helmet`,
+    `gatsby-plugin-sharp`,
     `gatsby-transformer-sharp`,
     `gatsby-plugin-styled-components`,
     `gatsby-plugin-sitemap`,

package.json

@@ -33,7 +33,7 @@
     "@mdx-js/mdx": "^1.5.1",
     "@mdx-js/react": "^1.5.0",
     "@sindresorhus/slugify": "^0.9.1",
-    "@yldio/oss-stats": "^2.0.1",
+    "@yldio/oss-stats": "^3.0.0",
     "airbnb-prop-types": "^2.13.2",
     "apr-intercept": "^3.0.3",
     "apr-reduce": "^3.0.3",

src/functions/contributions-commits.js

@@ -0,0 +1,32 @@
+/* eslint-disable no-console */
+/**
+ *
+ * This lambda is to keep commit contributions up to date on Contentful
+ *
+ * lambda triggered --> get member data and current contributions data from contenfult -> fetch new commit contributions data from github -> compare with the current contributions data, if it's different update it on contentful
+ */
+
+const Auth = require('./utils/auth');
+const updateContributionType = require('./oss/updateContributionType');
+
+exports.handler = async evt =>
+  Auth(evt, async () => {
+    const { CONTENTFUL_SPACE, CMS_CRUD, GITHUB_TOKEN } = process.env;
+
+    if ((!CONTENTFUL_SPACE, !CMS_CRUD, !GITHUB_TOKEN)) {
+      throw new Error(`Missing env variables, check set up`);
+    }
+
+    const {
+      contributionsByRepo,
+      totalContributions,
+    } = await updateContributionType('commit');
+
+    return {
+      statusCode: 200,
+      body: JSON.stringify({
+        contributionsByRepo,
+        totalContributions,
+      }),
+    };
+  });

src/functions/contributions-issues.js

@@ -0,0 +1,32 @@
+/* eslint-disable no-console */
+/**
+ *
+ * This lambda is to keep issue contributions up to date on Contentful
+ *
+ * lambda triggered --> get member data and current contributions data from contenfult -> fetch new issue contributions data from github -> compare with the current contributions data, if it's different update it on contentful
+ */
+
+const Auth = require('./utils/auth');
+const updateContributionType = require('./oss/updateContributionType');
+
+exports.handler = async evt =>
+  Auth(evt, async () => {
+    const { CONTENTFUL_SPACE, CMS_CRUD, GITHUB_TOKEN } = process.env;
+
+    if ((!CONTENTFUL_SPACE, !CMS_CRUD, !GITHUB_TOKEN)) {
+      throw new Error(`Missing env variables, check set up`);
+    }
+
+    const {
+      contributionsByRepo,
+      totalContributions,
+    } = await updateContributionType('issue');
+
+    return {
+      statusCode: 200,
+      body: JSON.stringify({
+        contributionsByRepo,
+        totalContributions,
+      }),
+    };
+  });

src/functions/contributions-pull-request-reviews.js

@@ -0,0 +1,32 @@
+/* eslint-disable no-console */
+/**
+ *
+ * This lambda is to keep pull request review contributions up to date on Contentful
+ *
+ * lambda triggered --> get member data and current contributions data from contenfult -> fetch new pull request review contributions data from github -> compare with the current contributions data, if it's different update it on contentful
+ */
+
+const Auth = require('./utils/auth');
+const updateContributionType = require('./oss/updateContributionType');
+
+exports.handler = async evt =>
+  Auth(evt, async () => {
+    const { CONTENTFUL_SPACE, CMS_CRUD, GITHUB_TOKEN } = process.env;
+
+    if ((!CONTENTFUL_SPACE, !CMS_CRUD, !GITHUB_TOKEN)) {
+      throw new Error(`Missing env variables, check set up`);
+    }
+
+    const {
+      contributionsByRepo,
+      totalContributions,
+    } = await updateContributionType('pullRequestReview');
+
+    return {
+      statusCode: 200,
+      body: JSON.stringify({
+        contributionsByRepo,
+        totalContributions,
+      }),
+    };
+  });

src/functions/contributions-pull-requests.js

@@ -0,0 +1,32 @@
+/* eslint-disable no-console */
+/**
+ *
+ * This lambda is to keep pull request contributions up to date on Contentful
+ *
+ * lambda triggered --> get member data and current contributions data from contenfult -> fetch new pull-request contributions data from github -> compare with the current contributions data, if it's different update it on contentful
+ */
+
+const Auth = require('./utils/auth');
+const updateContributionType = require('./oss/updateContributionType');
+
+exports.handler = async evt =>
+  Auth(evt, async () => {
+    const { CONTENTFUL_SPACE, CMS_CRUD, GITHUB_TOKEN } = process.env;
+
+    if ((!CONTENTFUL_SPACE, !CMS_CRUD, !GITHUB_TOKEN)) {
+      throw new Error(`Missing env variables, check set up`);
+    }
+
+    const {
+      contributionsByRepo,
+      totalContributions,
+    } = await updateContributionType('pullRequest');
+
+    return {
+      statusCode: 200,
+      body: JSON.stringify({
+        contributionsByRepo,
+        totalContributions,
+      }),
+    };
+  });

src/functions/github.js

@@ -5,31 +5,21 @@
  *
  * lambda triggered --> get repo urls stored in contentful --> get current contentful data -->  get current github data --> if any differences, write new data to contentful
  */
-
-const OssStats = require('@yldio/oss-stats');
 const { createClient } = require('contentful-management');
 
 const Auth = require('./utils/auth');
 const Meta = require('./oss/meta');
 const Repos = require('./oss/repos');
-
-const org = 'yldio';
+const { LOCALE } = require('./utils/constants');
 
 exports.handler = async evt =>
   Auth(evt, async () => {
-    const { getContributionStats } = OssStats.contributions;
     const { CONTENTFUL_SPACE, CMS_CRUD, GITHUB_TOKEN } = process.env;
 
     if ((!CONTENTFUL_SPACE, !CMS_CRUD, !GITHUB_TOKEN)) {
       throw new Error(`Missing env variables, check set up`);
     }
 
-    // Get github data
-    const {
-      contributionsByRepository,
-      totalContributions,
-    } = await getContributionStats(org, GITHUB_TOKEN);
-
     // Get contentful data
     const client = createClient({
       accessToken: CMS_CRUD,
@@ -38,6 +28,47 @@ exports.handler = async evt =>
     const space = await client.getSpace(CONTENTFUL_SPACE);
     const environment = await space.getEnvironment('master');
 
+    const { items: osContributions } = await environment.getEntries({
+      content_type: 'osContributions',
+    });
+
+    const {
+      contributionsByRepository,
+      totalContributions,
+    } = osContributions.reduce(
+      (acc, curr) => {
+        const {
+          fields: { contributions },
+        } = curr;
+        // prettier-ignore
+        const { contributionsByRepo, totalContributions } = contributions[LOCALE];
+
+        contributionsByRepo.forEach(repositoryContribution => {
+          const repositoryEntry = acc.contributionsByRepository.find(
+            ({ repository }) =>
+              repository && repository.id === repositoryContribution.id,
+          );
+
+          if (repositoryEntry) {
+            repositoryEntry.contributions.totalCount +=
+              repositoryContribution.totalCount;
+          } else {
+            acc.contributionsByRepository.push({
+              repository: repositoryContribution,
+              contributions: {
+                totalCount: repositoryContribution.totalCount,
+              },
+            });
+          }
+        });
+
+        acc.totalContributions += totalContributions;
+
+        return acc;
+      },
+      { contributionsByRepository: [], totalContributions: 0 },
+    );
+
     const [meta, { updatedRepos, missingRepos }] = await Promise.all([
       Meta(environment, {
         contributionsCount: totalContributions,

src/functions/members.js

@@ -0,0 +1,99 @@
+/* eslint-disable no-console */
+/**
+ *
+ * This lambda is to keep yld members data up to date on Contentful
+ *
+ * lambda triggered --> get member log stored in contentful --> get current member data --> generate new member data from log --> if any differences, write new data to contentful
+ */
+
+const OssStats = require('@yldio/oss-stats');
+const { createClient } = require('contentful-management');
+const { head, isEqual } = require('lodash');
+
+const Auth = require('./utils/auth');
+const { LOCALE } = require('./utils/constants');
+
+exports.handler = async evt =>
+  Auth(evt, async () => {
+    const { getOrgMembers } = OssStats.org;
+    const {
+      CONTENTFUL_SPACE,
+      CMS_CRUD,
+      GITHUB_TOKEN,
+      LAMBDA_ENV = 'development',
+    } = process.env;
+    const isProd = LAMBDA_ENV === 'production';
+
+    if ((!CONTENTFUL_SPACE, !CMS_CRUD, !GITHUB_TOKEN)) {
+      throw new Error(`Missing env variables, check set up`);
+    }
+
+    // Get contentful data
+    const client = createClient({
+      accessToken: CMS_CRUD,
+    });
+
+    const space = await client.getSpace(CONTENTFUL_SPACE);
+    const environment = await space.getEnvironment('master');
+
+    const {
+      items: [
+        {
+          fields: { log },
+        },
+      ],
+    } = await environment.getEntries({
+      content_type: 'memberLog',
+    });
+
+    const membersLog = log[LOCALE];
+
+    const members = await getOrgMembers(membersLog);
+
+    const { items: membersData } = await environment.getEntries({
+      content_type: 'membersData',
+    });
+
+    const contentfulEntry = head(membersData);
+
+    const {
+      fields: { membersDetails },
+    } = contentfulEntry;
+
+    const fieldsAreEqual = isEqual(members, membersDetails.members);
+
+    if (isProd && !fieldsAreEqual) {
+      contentfulEntry.fields = {
+        ...contentfulEntry.fields,
+        membersDetails: { [LOCALE]: members },
+      };
+      const id = await contentfulEntry.update();
+      const updatedEntry = await environment.getEntry(id.sys.id);
+
+      console.log(
+        `Publishing updated entry for memberDetails`,
+        JSON.stringify(contentfulEntry.fields, null, 2),
+      );
+      await updatedEntry.publish();
+    } else {
+      console.log(
+        fieldsAreEqual
+          ? `Values for github meta data have not changed. Not updating!`
+          : `Not prod so not updating contentful github meta data `,
+        JSON.stringify(
+          {
+            generatedMembersDetails: members,
+          },
+          null,
+          2,
+        ),
+      );
+    }
+
+    return {
+      statusCode: 200,
+      body: JSON.stringify({
+        members,
+      }),
+    };
+  });