GitHub - Add GET request to certain Instant Webhook-Based Sources

components/github/github.app.mjs

@@ -83,6 +83,20 @@ export default {
         }));
       },
     },
+    branch: {
+      label: "Branch",
+      description: "Branch to monitor for new commits",
+      type: "string",
+      async options({ repoFullname }) {
+        const branches = await this.getBranches({
+          repoFullname,
+        });
+        return branches.map((branch) => ({
+          label: branch.name,
+          value: branch.commit.sha,
+        }));
+      },
+    },
     pullNumber: {
       type: "integer",
       label: "PR Number",
@@ -136,7 +150,7 @@ export default {
     async removeWebhook({
       repoFullname, webhookId,
     }) {
-      return this._client().request(`DELETE /webhooks/${repoFullname}/hooks/${webhookId}`, {});
+      return this._client().request(`DELETE /repos/${repoFullname}/hooks/${webhookId}`, {});
     },
     async getOrganizations() {
       const response = await this._client().request("GET /user/orgs", {});
@@ -273,5 +287,29 @@ export default {
 
       return response.data;
     },
+    async getCommits({
+      repoFullname, ...data
+    }) {
+      const { data: commits } = await this._client().request(`GET /repos/${repoFullname}/commits`, {
+        ...data,
+      });
+      return commits;
+    },
+    async getBranches({
+      repoFullname, ...data
+    }) {
+      const { data: branches } = await this._client().request(`GET /repos/${repoFullname}/branches`, {
+        ...data,
+      });
+      return branches;
+    },
+    async getProjectCards({
+      columnId, ...data
+    }) {
+      const { data: cards } = await this._client().request(`GET /projects/columns/${columnId}/cards`, {
+        ...data,
+      });
+      return cards;
+    },
   },
 };

components/github/readme.md

@@ -249,3 +249,9 @@ If an issue _doesn't_ yet exist and you need to create one, please [use the issu
 You can read about our platform security and privacy [here](https://pipedream.com/docs/privacy-and-security/).
 
 If you'd like to report a suspected vulnerability or security issue, or have any questions about the security of the product, please contact our security team at **security@pipedream.com**.
+
+## Troubleshooting
+
+Note: Event Source [New Card in Column](https://github.com/PipedreamHQ/pipedream/blob/master/components/github/sources/new-card-in-column/new-card-in-column.mjs) only supports legacy (classic) projects.
+
+Please [reach out](https://pipedream.com/support/) to the Pipedream team with any technical issues or questions about the Github integration. We're happy to help!

components/github/sources/common/common-webhook.mjs

@@ -25,8 +25,17 @@ export default {
     getWebhookEvents() {
       throw new Error("getWebhookEvents is not implemented");
     },
+    generateMeta() {
+      throw new Error("generateMeta is not implemented");
+    },
+    loadHistoricalEvents() {
+      return true;
+    },
   },
   hooks: {
+    async deploy() {
+      await this.loadHistoricalEvents();
+    },
     async activate() {
       const response = await this.github.createWebhook({
         repoFullname: this.repoFullname,
@@ -39,7 +48,6 @@ export default {
           events: this.getWebhookEvents(),
         },
       });
-
       this._setWebhookId(response.id);
     },
     async deactivate() {

components/github/sources/new-card-in-column/new-card-in-column.mjs

@@ -5,7 +5,7 @@ export default {
   key: "github-new-card-in-column",
   name: "New Card in Column (Instant)",
   description: "Emit new event when a project card is created or moved to a specific column",
-  version: "0.0.1",
+  version: "0.1.0",
   type: "source",
   props: {
     ...common.props,
@@ -59,24 +59,39 @@ export default {
         ts: Date.parse(card.updated_at),
       };
     },
+    async loadHistoricalEvents() {
+      const cards = await this.github.getProjectCards({
+        columnId: this.getThisColumnValue(),
+        per_page: 25,
+      });
+      for (const card of cards) {
+        await this.processCard(card);
+      }
+    },
+    async processCard(card) {
+      const meta = this.generateMeta(card);
+      const issue = await this.github.getIssueFromProjectCard({
+        repoFullname: this.repoFullname,
+        cardId: card.id,
+      });
+      this.$emit({
+        card,
+        issue,
+      }, meta);
+    },
   },
   async run(event) {
     const card = event.body.project_card;
+    if (!card) {
+      console.log("No card in event. Skipping event.");
+      return;
+    }
 
     if (!this.isCardInThisColumn(card)) {
       console.log(`Card not in ${this.getThisColumnLabel()}. Skipping...`);
       return;
     }
 
-    const meta = this.generateMeta(card);
-    const issue = await this.github.getIssueFromProjectCard({
-      repoFullName: this.repoFullname,
-      cardId: card.id,
-    });
-
-    this.$emit({
-      card,
-      issue,
-    }, meta);
+    this.processCard(card);
   },
 };

components/github/sources/new-commit/new-commit.mjs

@@ -0,0 +1,81 @@
+import common from "../common/common-webhook.mjs";
+
+export default {
+  ...common,
+  key: "github-new-commit",
+  name: "New Commit (Instant)",
+  description: "Emit new events on new commits to a repo or branch",
+  version: "0.1.0",
+  type: "source",
+  dedupe: "unique",
+  props: {
+    ...common.props,
+    branch: {
+      propDefinition: [
+        common.props.github,
+        "branch",
+        (c) => ({
+          repoFullname: c.repoFullname,
+        }),
+      ],
+      description: "Branch to monitor for new commits. Defaults to master",
+      optional: true,
+      withLabel: true,
+    },
+  },
+  methods: {
+    ...common.methods,
+    getWebhookEvents() {
+      return [
+        "push",
+      ];
+    },
+    isEventForThisBranch(branch) {
+      return !this.branch || branch === this.branch.label;
+    },
+    generateMeta(data) {
+      return {
+        id: data.id,
+        summary: data.message,
+        ts: Date.parse(data.timestamp),
+      };
+    },
+    async loadHistoricalEvents() {
+      const commitInfo = await this.github.getCommits({
+        repoFullname: this.repoFullname,
+        sha: this.branch
+          ? this.branch.value
+          : undefined,
+        per_page: 25,
+      });
+      const commits = commitInfo.map((info) => ({
+        id: info.commit.url.split("/").pop(),
+        timestamp: info.commit.committer.date,
+        ...info.commit,
+      }));
+      this.processCommits(commits);
+    },
+    processCommits(commits) {
+      for (const commit of commits) {
+        const meta = this.generateMeta(commit);
+        this.$emit(commit, meta);
+      }
+    },
+  },
+  async run(event) {
+    const { body } = event;
+
+    // skip initial response from Github
+    if (body?.zen) {
+      console.log(body.zen);
+      return;
+    }
+
+    const branch = body.ref.split("refs/heads/").pop();
+    if (!this.isEventForThisBranch(branch)) {
+      return;
+    }
+
+    this.processCommits(body.commits);
+  },
+};

components/github/sources/webhook-event/webhook-event.mjs

@@ -3,16 +3,16 @@ import constants from "../common/constants.mjs";
 
 export default {
   ...common,
-  key: "github-weebhook-vents",
+  key: "github-webhook-events",
   name: "New Webhook Event (Instant)",
-  description: "Emit new event for each selected event types",
+  description: "Emit new event for each selected event type",
   type: "source",
-  version: "0.0.1",
+  version: "0.0.2",
   props: {
     ...common.props,
     events: {
       label: "Webhook Events",
-      description: "The event will be emited",
+      description: "The event types to be emited",
       type: "string[]",
       options: constants.REPOSITORY_WEBHOOK_EVENTS,
     },
@@ -30,6 +30,12 @@ export default {
       body,
     } = event;
 
+    // skip initial response from Github
+    if (body?.zen) {
+      console.log(body.zen);
+      return;
+    }
+
     this.$emit(body, {
       id: headers["x-github-delivery"],
       summary: `New event ${headers["x-github-hook-installation-target-id"]} of type ${headers["x-github-hook-installation-target-type"]}}`,