From 8e1fcf1628296eb7685179b799dc15f3c13ce807 Mon Sep 17 00:00:00 2001 From: Jorge Cortes Date: Wed, 18 Oct 2023 18:05:19 -0500 Subject: [PATCH 1/3] Added first version --- .../actions/create-issue/create-issue.mjs | 64 +++++++++++ .../actions/delete-user/delete-user.mjs | 25 +++++ .../update-project /update-project .mjs | 77 +++++++++++++ components/redmine/package.json | 4 +- components/redmine/redmine.app.mjs | 103 +++++++++++++++++- .../sources/issue-created/issue-created.mjs | 63 +++++++++++ .../project-updated/project-updated.mjs | 55 ++++++++++ 7 files changed, 384 insertions(+), 7 deletions(-) create mode 100644 components/redmine/actions/create-issue/create-issue.mjs create mode 100644 components/redmine/actions/delete-user/delete-user.mjs create mode 100644 components/redmine/actions/update-project /update-project .mjs create mode 100644 components/redmine/sources/issue-created/issue-created.mjs create mode 100644 components/redmine/sources/project-updated/project-updated.mjs diff --git a/components/redmine/actions/create-issue/create-issue.mjs b/components/redmine/actions/create-issue/create-issue.mjs new file mode 100644 index 0000000000000..f6fdca18e0486 --- /dev/null +++ b/components/redmine/actions/create-issue/create-issue.mjs @@ -0,0 +1,64 @@ +import redmine from "../../redmine.app.mjs"; + +export default { + key: "redmine-create-issue", + name: "Create Issue", + description: "Creates a new issue in Redmine. [See the documentation](https://www.redmine.org/projects/redmine/wiki/rest_issues#creating-an-issue)", + version: "0.0.1", + type: "action", + props: { + redmine, + projectId: { + propDefinition: [ + redmine, + "projectId", + ], + }, + subject: { + propDefinition: [ + redmine, + "subject", + ], + }, + description: { + propDefinition: [ + redmine, + "description", + ], + }, + statusId: { + propDefinition: [ + redmine, + "statusId", + ], + }, + priorityId: { + propDefinition: [ + redmine, + "priorityId", + ], + }, + trackerId: { + propDefinition: [ + redmine, + "trackerId", + ], + }, + }, + async run({ $ }) { + const response = await this.redmine.createIssue({ + data: { + issue: { + project_id: this.projectId, + subject: this.subject, + description: this.description, + status_id: this.statusId, + priority_id: this.priorityId, + tracker_id: this.trackerId, + }, + }, + }); + $.export("$summary", `Successfully created issue with ID: ${response.issue.id}`); + return response; + }, +}; diff --git a/components/redmine/actions/delete-user/delete-user.mjs b/components/redmine/actions/delete-user/delete-user.mjs new file mode 100644 index 0000000000000..0a76d4de85122 --- /dev/null +++ b/components/redmine/actions/delete-user/delete-user.mjs @@ -0,0 +1,25 @@ +import redmine from "../../redmine.app.mjs"; + +export default { + key: "redmine-delete-user", + name: "Delete User", + description: "Deletes a user from the Redmine platform. [See the documentation](https://www.redmine.org/projects/redmine/wiki/rest_users#delete)", + version: "0.0.1", + type: "action", + props: { + redmine, + userId: { + propDefinition: [ + redmine, + "userId", + ], + }, + }, + async run({ $ }) { + const response = await this.redmine.deleteUser({ + userId: this.userId, + }); + $.export("$summary", `Successfully deleted user with ID: ${this.userId}`); + return response; + }, +}; diff --git a/components/redmine/actions/update-project /update-project .mjs b/components/redmine/actions/update-project /update-project .mjs new file mode 100644 index 0000000000000..bb9b7778097e3 --- /dev/null +++ b/components/redmine/actions/update-project /update-project .mjs @@ -0,0 +1,77 @@ +import redmine from "../../redmine.app.mjs"; + +export default { + key: "redmine-update-project", + name: "Update Project", + description: "Updates an existing project in Redmine. [See the documentation](https://www.redmine.org/projects/redmine/wiki/rest_projects#updating-a-project)", + version: "0.0.1", + type: "action", + props: { + redmine, + projectId: { + propDefinition: [ + redmine, + "projectId", + ], + }, + name: { + type: "string", + label: "Name", + description: "The name of the project", + optional: true, + }, + identifier: { + type: "string", + label: "Identifier", + description: "The identifier of the project", + optional: true, + }, + description: { + type: "string", + label: "Description", + description: "The description of the project", + optional: true, + }, + homepage: { + type: "string", + label: "Homepage", + description: "The homepage of the project", + optional: true, + }, + isPublic: { + type: "boolean", + label: "Is Public", + description: "Whether the project is public", + optional: true, + }, + parentId: { + propDefinition: [ + redmine, + "projectId", + ], + optional: true, + }, + inheritMembers: { + type: "boolean", + label: "Inherit Members", + description: "Whether the project should inherit members from its parent", + optional: true, + }, + }, + async run({ $ }) { + const response = await this.redmine.updateProject({ + projectId: this.projectId, + project: { + name: this.name, + identifier: this.identifier, + description: this.description, + homepage: this.homepage, + is_public: this.isPublic, + parent_id: this.parentId, + inherit_members: this.inheritMembers, + }, + }); + $.export("$summary", `Successfully updated project with ID: ${this.projectId}`); + return response; + }, +}; diff --git a/components/redmine/package.json b/components/redmine/package.json index 56a5906166039..17144e2e3db94 100644 --- a/components/redmine/package.json +++ b/components/redmine/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/redmine", - "version": "0.0.1", + "version": "0.1.0", "description": "Pipedream Redmine Components", "main": "redmine.app.mjs", "keywords": [ @@ -12,4 +12,4 @@ "publishConfig": { "access": "public" } -} \ No newline at end of file +} diff --git a/components/redmine/redmine.app.mjs b/components/redmine/redmine.app.mjs index 830f6ecc68ca3..b12ed1a33f6b3 100644 --- a/components/redmine/redmine.app.mjs +++ b/components/redmine/redmine.app.mjs @@ -1,11 +1,104 @@ +import { axios } from "@pipedream/platform"; + export default { type: "app", app: "redmine", - propDefinitions: {}, + propDefinitions: { + projectId: { + type: "integer", + label: "Project ID", + description: "The ID of the project", + }, + userId: { + type: "integer", + label: "User ID", + description: "The ID of the user", + }, + issueId: { + type: "integer", + label: "Issue ID", + description: "The ID of the issue", + }, + subject: { + type: "string", + label: "Subject", + description: "The subject of the issue", + }, + description: { + type: "string", + label: "Description", + description: "The description of the issue", + }, + statusId: { + type: "integer", + label: "Status ID", + description: "The ID of the status", + }, + priorityId: { + type: "integer", + label: "Priority ID", + description: "The ID of the priority", + }, + trackerId: { + type: "integer", + label: "Tracker ID", + description: "The ID of the tracker", + }, + }, methods: { - // this.$auth contains connected account data - authKeys() { - console.log(Object.keys(this.$auth)); + _baseUrl() { + return "https://redmine.org"; + }, + async _makeRequest(opts = {}) { + const { + $ = this, + method = "GET", + path, + headers, + ...otherOpts + } = opts; + return axios($, { + ...otherOpts, + method, + url: this._baseUrl() + path, + headers: { + ...headers, + "X-Redmine-API-Key": `${this.$auth.api_key}`, + }, + }); + }, + async createIssue(opts = {}) { + return this._makeRequest({ + ...opts, + method: "POST", + path: "/issues.json", + }); + }, + async updateProject(opts = {}) { + return this._makeRequest({ + ...opts, + method: "PUT", + path: `/projects/${opts.projectId}.json`, + }); + }, + async deleteUser(opts = {}) { + return this._makeRequest({ + ...opts, + method: "DELETE", + path: `/users/${opts.userId}.json`, + }); + }, + async getIssueCreated(opts = {}) { + return this._makeRequest({ + ...opts, + path: `/issues/${opts.issueId}.json`, + }); + }, + async getProjectUpdated(opts = {}) { + return this._makeRequest({ + ...opts, + path: `/projects/${opts.projectId}.json`, + }); }, }, -}; \ No newline at end of file +}; diff --git a/components/redmine/sources/issue-created/issue-created.mjs b/components/redmine/sources/issue-created/issue-created.mjs new file mode 100644 index 0000000000000..faa4e5f8f57d7 --- /dev/null +++ b/components/redmine/sources/issue-created/issue-created.mjs @@ -0,0 +1,63 @@ +import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform"; +import redmine from "../../redmine.app.mjs"; + +export default { + key: "redmine-issue-created", + name: "New Issue Created", + description: "Emit new event when a new issue is created in Redmine", + version: "0.0.1", + type: "source", + dedupe: "unique", + props: { + redmine, + db: "$.service.db", + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + projectId: { + propDefinition: [ + redmine, + "projectId", + ], + }, + }, + methods: { + _getIssueId() { + return this.db.get("issueId") || 0; + }, + _setIssueId(id) { + this.db.set("issueId", id); + }, + }, + async run() { + const params = { + projectId: this.projectId, + }; + + while (true) { + const { issues } = await this.redmine.getIssueCreated(params); + if (issues.length === 0) { + console.log("No new issues found."); + break; + } + + let maxIssueId = this._getIssueId(); + for (const issue of issues) { + if (issue.id > maxIssueId) { + maxIssueId = issue.id; + this.$emit(issue, { + id: issue.id, + summary: `New issue: ${issue.subject}`, + ts: Date.parse(issue.created_on), + }); + } + } + + this._setIssueId(maxIssueId); + params.offset = (params.offset || 0) + issues.length; + } + }, +}; diff --git a/components/redmine/sources/project-updated/project-updated.mjs b/components/redmine/sources/project-updated/project-updated.mjs new file mode 100644 index 0000000000000..3eea8ffe7b45f --- /dev/null +++ b/components/redmine/sources/project-updated/project-updated.mjs @@ -0,0 +1,55 @@ +import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform"; +import redmine from "../../redmine.app.mjs"; + +export default { + key: "redmine-project-updated", + name: "Project Updated", + description: "Emits an event whenever a project is updated in Redmine", + version: "0.0.1", + type: "source", + dedupe: "unique", + props: { + redmine, + db: "$.service.db", + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + projectId: { + propDefinition: [ + redmine, + "projectId", + ], + }, + }, + methods: { + _getLastUpdatedTime() { + return this.db.get("lastUpdatedTime") ?? 0; + }, + _setLastUpdatedTime(lastUpdatedTime) { + this.db.set("lastUpdatedTime", lastUpdatedTime); + }, + }, + async run() { + const lastUpdatedTime = this._getLastUpdatedTime(); + let maxUpdatedTime = 0; + + const project = await this.redmine.getProjectUpdated({ + projectId: this.projectId, + }); + + const updatedOn = Date.parse(project.updated_on); + if (updatedOn > lastUpdatedTime) { + this.$emit(project, { + id: project.id, + summary: `Project ${project.name} updated`, + ts: updatedOn, + }); + maxUpdatedTime = Math.max(maxUpdatedTime, updatedOn); + } + + this._setLastUpdatedTime(maxUpdatedTime); + }, +}; From 3c377e71ef7674932d0606cf2fb2ffab158edd59 Mon Sep 17 00:00:00 2001 From: Jorge Cortes Date: Fri, 20 Oct 2023 13:32:55 -0500 Subject: [PATCH 2/3] Added tested actions and sources --- .../actions/create-issue/create-issue.mjs | 63 ++--- .../actions/delete-user/delete-user.mjs | 31 ++- .../update-project.mjs} | 56 ++-- components/redmine/common/constants.mjs | 15 ++ components/redmine/common/utils.mjs | 43 ++++ components/redmine/redmine.app.mjs | 243 +++++++++++++----- components/redmine/sources/common/base.mjs | 14 + components/redmine/sources/common/polling.mjs | 58 +++++ .../sources/issue-created/issue-created.mjs | 69 ++--- .../project-updated/project-updated.mjs | 62 ++--- 10 files changed, 440 insertions(+), 214 deletions(-) rename components/redmine/actions/{update-project /update-project .mjs => update-project/update-project.mjs} (58%) create mode 100644 components/redmine/common/constants.mjs create mode 100644 components/redmine/common/utils.mjs create mode 100644 components/redmine/sources/common/base.mjs create mode 100644 components/redmine/sources/common/polling.mjs diff --git a/components/redmine/actions/create-issue/create-issue.mjs b/components/redmine/actions/create-issue/create-issue.mjs index f6fdca18e0486..91ed70602d2a5 100644 --- a/components/redmine/actions/create-issue/create-issue.mjs +++ b/components/redmine/actions/create-issue/create-issue.mjs @@ -1,4 +1,5 @@ -import redmine from "../../redmine.app.mjs"; +import app from "../../redmine.app.mjs"; +import utils from "../../common/utils.mjs"; export default { key: "redmine-create-issue", @@ -7,58 +8,62 @@ export default { version: "0.0.1", type: "action", props: { - redmine, + app, projectId: { propDefinition: [ - redmine, + app, "projectId", ], }, - subject: { + trackerId: { propDefinition: [ - redmine, - "subject", + app, + "trackerId", ], }, + subject: { + type: "string", + label: "Subject", + description: "The subject of the issue", + }, description: { - propDefinition: [ - redmine, - "description", - ], + type: "string", + label: "Description", + description: "The description of the issue", }, statusId: { propDefinition: [ - redmine, + app, "statusId", ], }, priorityId: { propDefinition: [ - redmine, + app, "priorityId", ], }, - trackerId: { - propDefinition: [ - redmine, - "trackerId", - ], + }, + methods: { + createIssue(args = {}) { + return this.app.post({ + path: "/issues.json", + ...args, + }); }, }, - async run({ $ }) { - const response = await this.redmine.createIssue({ + run({ $: step }) { + const { + createIssue, + ...issue + } = this; + + return createIssue({ + step, data: { - issue: { - project_id: this.projectId, - subject: this.subject, - description: this.description, - status_id: this.statusId, - priority_id: this.priorityId, - tracker_id: this.trackerId, - }, + issue: utils.transformProps(issue), }, + summary: (response) => `Successfully created issue with ID: \`${response.issue?.id}\``, }); - $.export("$summary", `Successfully created issue with ID: ${response.issue.id}`); - return response; }, }; diff --git a/components/redmine/actions/delete-user/delete-user.mjs b/components/redmine/actions/delete-user/delete-user.mjs index 0a76d4de85122..177f0853833ff 100644 --- a/components/redmine/actions/delete-user/delete-user.mjs +++ b/components/redmine/actions/delete-user/delete-user.mjs @@ -1,4 +1,4 @@ -import redmine from "../../redmine.app.mjs"; +import app from "../../redmine.app.mjs"; export default { key: "redmine-delete-user", @@ -7,19 +7,34 @@ export default { version: "0.0.1", type: "action", props: { - redmine, + app, userId: { propDefinition: [ - redmine, + app, "userId", ], }, }, - async run({ $ }) { - const response = await this.redmine.deleteUser({ - userId: this.userId, + methods: { + deleteUser({ + userId, ...args + } = {}) { + return this.app.delete({ + path: `/users/${userId}.json`, + ...args, + }); + }, + }, + run({ $: step }) { + const { + deleteUser, + userId, + } = this; + + return deleteUser({ + step, + userId, + summary: () => "Successfully deleted user", }); - $.export("$summary", `Successfully deleted user with ID: ${this.userId}`); - return response; }, }; diff --git a/components/redmine/actions/update-project /update-project .mjs b/components/redmine/actions/update-project/update-project.mjs similarity index 58% rename from components/redmine/actions/update-project /update-project .mjs rename to components/redmine/actions/update-project/update-project.mjs index bb9b7778097e3..73eb6c95ba9c5 100644 --- a/components/redmine/actions/update-project /update-project .mjs +++ b/components/redmine/actions/update-project/update-project.mjs @@ -1,4 +1,5 @@ -import redmine from "../../redmine.app.mjs"; +import app from "../../redmine.app.mjs"; +import utils from "../../common/utils.mjs"; export default { key: "redmine-update-project", @@ -7,10 +8,10 @@ export default { version: "0.0.1", type: "action", props: { - redmine, + app, projectId: { propDefinition: [ - redmine, + app, "projectId", ], }, @@ -20,12 +21,6 @@ export default { description: "The name of the project", optional: true, }, - identifier: { - type: "string", - label: "Identifier", - description: "The identifier of the project", - optional: true, - }, description: { type: "string", label: "Description", @@ -44,13 +39,6 @@ export default { description: "Whether the project is public", optional: true, }, - parentId: { - propDefinition: [ - redmine, - "projectId", - ], - optional: true, - }, inheritMembers: { type: "boolean", label: "Inherit Members", @@ -58,20 +46,30 @@ export default { optional: true, }, }, - async run({ $ }) { - const response = await this.redmine.updateProject({ - projectId: this.projectId, - project: { - name: this.name, - identifier: this.identifier, - description: this.description, - homepage: this.homepage, - is_public: this.isPublic, - parent_id: this.parentId, - inherit_members: this.inheritMembers, + methods: { + updateProject({ + projectId, ...args + } = {}) { + return this.app.put({ + path: `/projects/${projectId}.json`, + ...args, + }); + }, + }, + run({ $: step }) { + const { + updateProject, + projectId, + ...project + } = this; + + return updateProject({ + step, + projectId, + data: { + project: utils.transformProps(project), }, + summary: () => "Successfully updated project", }); - $.export("$summary", `Successfully updated project with ID: ${this.projectId}`); - return response; }, }; diff --git a/components/redmine/common/constants.mjs b/components/redmine/common/constants.mjs new file mode 100644 index 0000000000000..4f26a9a20e03f --- /dev/null +++ b/components/redmine/common/constants.mjs @@ -0,0 +1,15 @@ +const SUMMARY_LABEL = "$summary"; +const DOMAIN_PLACEHOLDER = "{domain}"; +const BASE_URL = `https://${DOMAIN_PLACEHOLDER}`; +const LAST_CREATED_AT = "lastCreatedAt"; +const DEFAULT_MAX = 600; +const DEFAULT_LIMIT = 60; + +export default { + SUMMARY_LABEL, + DOMAIN_PLACEHOLDER, + BASE_URL, + DEFAULT_MAX, + DEFAULT_LIMIT, + LAST_CREATED_AT, +}; diff --git a/components/redmine/common/utils.mjs b/components/redmine/common/utils.mjs new file mode 100644 index 0000000000000..2bcc3ad0a1e86 --- /dev/null +++ b/components/redmine/common/utils.mjs @@ -0,0 +1,43 @@ +function toSnakeCase(str) { + return str?.replace(/([A-Z])/g, "_$1").toLowerCase(); +} + +function keysToSnakeCase(data = {}) { + return Object.entries(data) + .reduce((acc, [ + key, + value, + ]) => ({ + ...acc, + [toSnakeCase(key)]: value, + }), {}); +} + +function transformProps(props) { + if (!props) { + return; + } + + return keysToSnakeCase( + Object.fromEntries( + Object.entries(props) + .filter(([ + key, + value, + ]) => typeof(value) !== "function" && key !== "app"), + ), + ); +} + +async function iterate(iterations) { + const items = []; + for await (const item of iterations) { + items.push(item); + } + return items; +} + +export default { + transformProps, + iterate, +}; diff --git a/components/redmine/redmine.app.mjs b/components/redmine/redmine.app.mjs index b12ed1a33f6b3..53c93eb797952 100644 --- a/components/redmine/redmine.app.mjs +++ b/components/redmine/redmine.app.mjs @@ -1,4 +1,8 @@ -import { axios } from "@pipedream/platform"; +import { + axios, ConfigurationError, +} from "@pipedream/platform"; +import constants from "./common/constants.mjs"; +import utils from "./common/utils.mjs"; export default { type: "app", @@ -8,97 +12,220 @@ export default { type: "integer", label: "Project ID", description: "The ID of the project", + async options() { + const response = await this.listProjects(); + console.log(response); + const { projects } = response; + return projects.map(({ + id: value, name: label, + }) => ({ + label, + value, + })); + }, }, - userId: { - type: "integer", - label: "User ID", - description: "The ID of the user", - }, - issueId: { + trackerId: { type: "integer", - label: "Issue ID", - description: "The ID of the issue", - }, - subject: { - type: "string", - label: "Subject", - description: "The subject of the issue", - }, - description: { - type: "string", - label: "Description", - description: "The description of the issue", + label: "Tracker ID", + description: "The ID of the tracker", + async options() { + const { trackers } = await this.listTrackers(); + return trackers.map(({ + id: value, name: label, + }) => ({ + label, + value, + })); + }, }, statusId: { type: "integer", label: "Status ID", description: "The ID of the status", + async options() { + const { issue_statuses: statuses } = await this.listIssueStatuses(); + return statuses.map(({ + id: value, name: label, + }) => ({ + label, + value, + })); + }, }, priorityId: { type: "integer", label: "Priority ID", description: "The ID of the priority", + async options() { + const { issue_priorities: priorities } = await this.listIssuePriorities(); + return priorities.map(({ + id: value, name: label, + }) => ({ + label, + value, + })); + }, }, - trackerId: { + userId: { type: "integer", - label: "Tracker ID", - description: "The ID of the tracker", + label: "User ID", + description: "The ID of the user", + async options() { + const { users } = await this.listUsers(); + return users.map(({ + id: value, login: label, + }) => ({ + label, + value, + })); + }, + }, + issueId: { + type: "integer", + label: "Issue ID", + description: "The ID of the issue", }, }, methods: { - _baseUrl() { - return "https://redmine.org"; + exportSummary(step) { + if (!step?.export) { + throw new ConfigurationError("The summary method should be bind to the step object aka `$`"); + } + return (msg = "") => step.export(constants.SUMMARY_LABEL, msg); + }, + getUrl(path) { + const baseUrl = constants.BASE_URL + .replace(constants.DOMAIN_PLACEHOLDER, this.$auth.hostname); + return `${baseUrl}${path}`; }, - async _makeRequest(opts = {}) { + async makeRequest({ + step = this, path, headers, summary, ...args + } = {}) { const { - $ = this, - method = "GET", - path, - headers, - ...otherOpts - } = opts; - return axios($, { - ...otherOpts, - method, - url: this._baseUrl() + path, + getUrl, + exportSummary, + $auth: { api_key: apiKey }, + } = this; + + const config = { + ...args, + url: getUrl(path), headers: { ...headers, - "X-Redmine-API-Key": `${this.$auth.api_key}`, + "X-Redmine-API-Key": apiKey, }, + }; + + const response = await axios(step, config); + + if (typeof(summary) === "function") { + exportSummary(step)(summary(response)); + } + + return response || { + success: true, + }; + }, + post(args = {}) { + return this.makeRequest({ + method: "post", + ...args, + }); + }, + put(args = {}) { + return this.makeRequest({ + method: "put", + ...args, + }); + }, + delete(args = {}) { + return this.makeRequest({ + method: "delete", + ...args, }); }, - async createIssue(opts = {}) { - return this._makeRequest({ - ...opts, - method: "POST", + listIssues(args = {}) { + return this.makeRequest({ path: "/issues.json", + ...args, }); }, - async updateProject(opts = {}) { - return this._makeRequest({ - ...opts, - method: "PUT", - path: `/projects/${opts.projectId}.json`, + listProjects(args = {}) { + return this.makeRequest({ + path: "/projects.json", + ...args, }); }, - async deleteUser(opts = {}) { - return this._makeRequest({ - ...opts, - method: "DELETE", - path: `/users/${opts.userId}.json`, + listTrackers(args = {}) { + return this.makeRequest({ + path: "/trackers.json", + ...args, }); }, - async getIssueCreated(opts = {}) { - return this._makeRequest({ - ...opts, - path: `/issues/${opts.issueId}.json`, + listIssueStatuses(args = {}) { + return this.makeRequest({ + path: "/issue_statuses.json", + ...args, }); }, - async getProjectUpdated(opts = {}) { - return this._makeRequest({ - ...opts, - path: `/projects/${opts.projectId}.json`, + listIssuePriorities(args = {}) { + return this.makeRequest({ + path: "/enumerations/issue_priorities.json", + ...args, }); }, + listUsers(args = {}) { + return this.makeRequest({ + path: "/users.json", + ...args, + }); + }, + async *getIterations({ + resourceFn, + resourceFnArgs, + resourceName, + max = constants.DEFAULT_MAX, + }) { + let offset = 0; + let resourcesCount = 0; + + while (true) { + const response = + await resourceFn({ + ...resourceFnArgs, + params: { + offset, + limit: constants.DEFAULT_LIMIT, + ...resourceFnArgs?.params, + }, + }); + + const nextResources = resourceName && response[resourceName] || response; + + if (!nextResources?.length) { + console.log("No more resources found"); + return; + } + + for (const resource of nextResources) { + yield resource; + resourcesCount += 1; + + if (resourcesCount >= max) { + return; + } + } + + if (nextResources.length < constants.DEFAULT_LIMIT) { + console.log("Less resources than the limit found, no more resources to fetch"); + return; + } + + offset += constants.DEFAULT_LIMIT; + } + }, + paginate(args = {}) { + return utils.iterate(this.getIterations(args)); + }, }, }; diff --git a/components/redmine/sources/common/base.mjs b/components/redmine/sources/common/base.mjs new file mode 100644 index 0000000000000..59b0f195959dd --- /dev/null +++ b/components/redmine/sources/common/base.mjs @@ -0,0 +1,14 @@ +import { ConfigurationError } from "@pipedream/platform"; +import app from "../../redmine.app.mjs"; + +export default { + props: { + app, + db: "$.service.db", + }, + methods: { + generateMeta() { + throw new ConfigurationError("generateMeta is not implemented"); + }, + }, +}; diff --git a/components/redmine/sources/common/polling.mjs b/components/redmine/sources/common/polling.mjs new file mode 100644 index 0000000000000..418eb4b8534ab --- /dev/null +++ b/components/redmine/sources/common/polling.mjs @@ -0,0 +1,58 @@ +import { + ConfigurationError, + DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, +} from "@pipedream/platform"; +import common from "./base.mjs"; + +export default { + ...common, + props: { + ...common.props, + timer: { + type: "$.interface.timer", + label: "Polling schedule", + description: "How often to poll the API", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + }, + methods: { + ...common.methods, + getResourceName() { + throw new ConfigurationError("getResourceName is not implemented"); + }, + getResourceFn() { + throw new ConfigurationError("getResourceFn is not implemented"); + }, + getResourceFnArgs() { + throw new ConfigurationError("getResourceFnArgs is not implemented"); + }, + processEvent(resource) { + const meta = this.generateMeta(resource); + this.$emit(resource, meta); + }, + async processResources(resources) { + Array.from(resources) + .reverse() + .forEach(this.processEvent); + }, + }, + async run() { + const { + app, + getResourceFn, + getResourceFnArgs, + getResourceName, + processResources, + } = this; + + const resources = await app.paginate({ + resourceFn: getResourceFn(), + resourceFnArgs: getResourceFnArgs(), + resourceName: getResourceName(), + }); + + processResources(resources); + }, +}; diff --git a/components/redmine/sources/issue-created/issue-created.mjs b/components/redmine/sources/issue-created/issue-created.mjs index faa4e5f8f57d7..70ce8122c3b07 100644 --- a/components/redmine/sources/issue-created/issue-created.mjs +++ b/components/redmine/sources/issue-created/issue-created.mjs @@ -1,63 +1,34 @@ -import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform"; -import redmine from "../../redmine.app.mjs"; +import common from "../common/polling.mjs"; export default { + ...common, key: "redmine-issue-created", name: "New Issue Created", description: "Emit new event when a new issue is created in Redmine", version: "0.0.1", type: "source", dedupe: "unique", - props: { - redmine, - db: "$.service.db", - timer: { - type: "$.interface.timer", - default: { - intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, - }, + methods: { + ...common.methods, + getResourceName() { + return "issues"; }, - projectId: { - propDefinition: [ - redmine, - "projectId", - ], + getResourceFn() { + return this.app.listIssues; }, - }, - methods: { - _getIssueId() { - return this.db.get("issueId") || 0; + getResourceFnArgs() { + return { + params: { + sort: "created_on:desc", + }, + }; }, - _setIssueId(id) { - this.db.set("issueId", id); + generateMeta(resource) { + return { + id: resource.id, + summary: `New Issue: ${resource.subject}`, + ts: Date.parse(resource.created_on), + }; }, }, - async run() { - const params = { - projectId: this.projectId, - }; - - while (true) { - const { issues } = await this.redmine.getIssueCreated(params); - if (issues.length === 0) { - console.log("No new issues found."); - break; - } - - let maxIssueId = this._getIssueId(); - for (const issue of issues) { - if (issue.id > maxIssueId) { - maxIssueId = issue.id; - this.$emit(issue, { - id: issue.id, - summary: `New issue: ${issue.subject}`, - ts: Date.parse(issue.created_on), - }); - } - } - - this._setIssueId(maxIssueId); - params.offset = (params.offset || 0) + issues.length; - } - }, }; diff --git a/components/redmine/sources/project-updated/project-updated.mjs b/components/redmine/sources/project-updated/project-updated.mjs index 3eea8ffe7b45f..89762b3b5f663 100644 --- a/components/redmine/sources/project-updated/project-updated.mjs +++ b/components/redmine/sources/project-updated/project-updated.mjs @@ -1,55 +1,35 @@ -import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform"; -import redmine from "../../redmine.app.mjs"; +import common from "../common/polling.mjs"; export default { + ...common, key: "redmine-project-updated", name: "Project Updated", description: "Emits an event whenever a project is updated in Redmine", version: "0.0.1", type: "source", dedupe: "unique", - props: { - redmine, - db: "$.service.db", - timer: { - type: "$.interface.timer", - default: { - intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, - }, + methods: { + ...common.methods, + getResourceName() { + return "projects"; }, - projectId: { - propDefinition: [ - redmine, - "projectId", - ], + getResourceFn() { + return this.app.listProjects; }, - }, - methods: { - _getLastUpdatedTime() { - return this.db.get("lastUpdatedTime") ?? 0; + getResourceFnArgs() { + return { + params: { + sort: "updated_on:desc", + }, + }; }, - _setLastUpdatedTime(lastUpdatedTime) { - this.db.set("lastUpdatedTime", lastUpdatedTime); + generateMeta(resource) { + const ts = Date.parse(resource.updated_on); + return { + id: `${resource.id}-${ts}`, + summary: `Project Updated: ${resource.name}`, + ts, + }; }, }, - async run() { - const lastUpdatedTime = this._getLastUpdatedTime(); - let maxUpdatedTime = 0; - - const project = await this.redmine.getProjectUpdated({ - projectId: this.projectId, - }); - - const updatedOn = Date.parse(project.updated_on); - if (updatedOn > lastUpdatedTime) { - this.$emit(project, { - id: project.id, - summary: `Project ${project.name} updated`, - ts: updatedOn, - }); - maxUpdatedTime = Math.max(maxUpdatedTime, updatedOn); - } - - this._setLastUpdatedTime(maxUpdatedTime); - }, }; From 4597bf28d2390ca6a80427b6dfa8c12b78629719 Mon Sep 17 00:00:00 2001 From: Jorge Cortes Date: Fri, 20 Oct 2023 15:14:35 -0500 Subject: [PATCH 3/3] Removed issuedId prop --- components/redmine/redmine.app.mjs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/components/redmine/redmine.app.mjs b/components/redmine/redmine.app.mjs index 53c93eb797952..0bc99f353ddbe 100644 --- a/components/redmine/redmine.app.mjs +++ b/components/redmine/redmine.app.mjs @@ -80,11 +80,6 @@ export default { })); }, }, - issueId: { - type: "integer", - label: "Issue ID", - description: "The ID of the issue", - }, }, methods: { exportSummary(step) {