From 900d1c8e753ef45eaaf7719b747d2bc8fed002a6 Mon Sep 17 00:00:00 2001 From: Mikael Wahlberg Date: Thu, 12 May 2022 15:36:31 +0200 Subject: [PATCH 1/8] feat(importers): support for redmine --- packages/import/package.json | 2 +- packages/import/src/cli.ts | 8 + packages/import/src/importIssues.ts | 12 +- .../redmineCsv/RedmineCsvImporter.ts | 141 ++++++++++++++++++ .../import/src/importers/redmineCsv/index.ts | 24 +++ packages/import/src/types.ts | 10 ++ 6 files changed, 195 insertions(+), 2 deletions(-) create mode 100644 packages/import/src/importers/redmineCsv/RedmineCsvImporter.ts create mode 100644 packages/import/src/importers/redmineCsv/index.ts diff --git a/packages/import/package.json b/packages/import/package.json index 390ed85f..e6886992 100644 --- a/packages/import/package.json +++ b/packages/import/package.json @@ -22,7 +22,7 @@ "cli": "ts-node src/cli", "dev:import": "NODE_ENV=development yarn build:import", "build:import": "run-s build:clean build:rollup", - "build:rollup": "npx rollup -c", + "build:rollup": "npx rollup -c -w", "build:clean": "npx rimraf dist" }, "dependencies": { diff --git a/packages/import/src/cli.ts b/packages/import/src/cli.ts index 15abbaa2..9ebaff30 100644 --- a/packages/import/src/cli.ts +++ b/packages/import/src/cli.ts @@ -8,6 +8,7 @@ import { githubImport } from "./importers/github"; import { jiraCsvImport } from "./importers/jiraCsv"; import { linearCsvImporter } from "./importers/linearCsv"; import { pivotalCsvImport } from "./importers/pivotalCsv"; +import { redmineCsvImport } from "./importers/redmineCsv"; import { trelloJsonImport } from "./importers/trelloJson"; import { importIssues } from "./importIssues"; import { ImportAnswers } from "./types"; @@ -43,6 +44,10 @@ inquirer.registerPrompt("filePath", require("inquirer-file-path")); name: "Pivotal (CSV export)", value: "pivotalCsv", }, + { + name: "Redmine (CSV export)", + value: "redmineCsv", + }, { name: "Clubhouse (CSV export)", value: "clubhouseCsv", @@ -74,6 +79,9 @@ inquirer.registerPrompt("filePath", require("inquirer-file-path")); case "pivotalCsv": importer = await pivotalCsvImport(); break; + case "redmineCsv": + importer = await redmineCsvImport(); + break; case "clubhouseCsv": importer = await clubhouseCsvImport(); break; diff --git a/packages/import/src/importIssues.ts b/packages/import/src/importIssues.ts index 618e2fdb..b566b1c4 100644 --- a/packages/import/src/importIssues.ts +++ b/packages/import/src/importIssues.ts @@ -270,7 +270,7 @@ export const importIssues = async (apiKey: string, importer: Importer): Promise< const formattedDueDate = issue.dueDate ? format(issue.dueDate, "yyyy-MM-dd") : undefined; - await client.issueCreate({ + const newIssue = await client.issueCreate({ teamId, projectId: projectId as unknown as string, title: issue.title, @@ -281,6 +281,16 @@ export const importIssues = async (apiKey: string, importer: Importer): Promise< assigneeId, dueDate: formattedDueDate, }); + const newId = newIssue._issue.id; + //console.error(JSON.stringify(await client.issue(newId), null, 4)); + if (!!issue.url) { + client.attachmentLinkURL(newId, issue.url, { title: "Original Redmine issue" }); + } + if (!!issue.extraUrls) { + for (const url of issue.extraUrls) { + client.attachmentLinkURL(newId, url.url, !!url.title ? { title: url.title } : {}); + } + } } console.error( diff --git a/packages/import/src/importers/redmineCsv/RedmineCsvImporter.ts b/packages/import/src/importers/redmineCsv/RedmineCsvImporter.ts new file mode 100644 index 00000000..28c2b9eb --- /dev/null +++ b/packages/import/src/importers/redmineCsv/RedmineCsvImporter.ts @@ -0,0 +1,141 @@ +import csv from "csvtojson"; +//import { roundToNearestMinutes } from "date-fns"; +import { Importer, ImportResult } from "../../types"; +// eslint-disable-next-line @typescript-eslint/no-var-requires +const j2m = require("jira2md"); + +type RedmineStoryType = "Internal Bug" | "Internal Feature"; + +interface RedmineIssueType { + "#": string; + Subject: string; + Tags: string; + Iteration: string; + "Iteration Start": string; + "Iteration End": string; + Tracker: RedmineStoryType; + Estimate: string; + "Current State": string; + "Created at": Date; + "Accepted at": Date; + "Category-Iconik": string; + "Requested By": string; + Description: string; + URL: string; + "Owned By": string; + Blocker: string; + "Blocker Status": string; + Comment: string; +} + +/** + * Import issues from an Redmine Tracker CSV export. + * + * @param filePath path to csv file + * @param orgSlug base Redmine project url + */ +export class RedmineCsvImporter implements Importer { + public constructor(filePath: string) { + this.filePath = filePath; + } + + public get name(): string { + return "Redmine (CSV)"; + } + + public get defaultTeamName(): string { + return "Redmine"; + } + + public import = async (): Promise => { + const data = (await csv().fromFile(this.filePath)) as RedmineIssueType[]; + + const importData: ImportResult = { + issues: [], + labels: {}, + users: {}, + statuses: {}, + }; + + const assignees = Array.from(new Set(data.map(row => row["Assignee"]))); + + for (const user of assignees) { + importData.users[user] = { + name: user, + }; + } + + for (const row of data) { + const title = row.Subject; + if (!title) { + continue; + } + + //const url = row.URL; + const url = "https://redmine.iconik.biz/issues/" + row["#"]; + const mdDesc = j2m.to_markdown(row.Description); + const description = mdDesc; + + // const priority = parseInt(row['Estimate']) || undefined; + + const tags = row.Tags.split(","); + const categories = row["Category-Iconik"].split(","); + const refs = row["Internal Reference"].split("\n"); + const extraUrls = []; + if (!!refs) { + for (const r of refs) { + //console.error(r.trim()); + if (r.startsWith("http")) { + if (r.includes("support.iconik.io")) { + extraUrls.push({ url: r.trim(), title: "Zoho desk issue" }); + } else { + extraUrls.push({ url: r.trim() }); + } + //console.error(`Adding URL: ${r.trim()}`); + } + } + } + + const assigneeId = row["Assignee"] && row["Assignee"].length > 0 ? row["Assignee"] : undefined; + + const status = row["Status"] && (row["Status"] === "Review" || row["Status"] === "Codereview") ? "Done" : "Todo"; + + let labels = tags.filter(tag => !!tag); + if (row.Tracker === "Internal Bug") { + labels.push("bug"); + } + if (row.Tracker === "Internal Feature") { + labels.push("feature"); + } + if (!!categories) { + labels = labels.concat(categories.filter(tag => !!tag)); + } + const createdAt = row["Created"]; + + importData.issues.push({ + title, + description, + status, + url, + extraUrls, + assigneeId, + labels, + createdAt, + }); + + for (const lab of labels) { + if (!importData.labels[lab]) { + importData.labels[lab] = { + name: lab, + }; + } + } + } + + return importData; + }; + + // -- Private interface + + private filePath: string; +} diff --git a/packages/import/src/importers/redmineCsv/index.ts b/packages/import/src/importers/redmineCsv/index.ts new file mode 100644 index 00000000..5b2f0e43 --- /dev/null +++ b/packages/import/src/importers/redmineCsv/index.ts @@ -0,0 +1,24 @@ +import * as inquirer from "inquirer"; +import { Importer } from "../../types"; +import { RedmineCsvImporter } from "./RedmineCsvImporter"; + +const BASE_PATH = process.cwd(); + +export const redmineCsvImport = async (): Promise => { + const answers = await inquirer.prompt(questions); + const redmineImporter = new RedmineCsvImporter(answers.redmineFilePath); + return redmineImporter; +}; + +interface RedmineImportAnswers { + redmineFilePath: string; +} + +const questions = [ + { + basePath: BASE_PATH, + type: "filePath", + name: "redmineFilePath", + message: "Select your exported CSV file of Redmine issues", + }, +]; diff --git a/packages/import/src/types.ts b/packages/import/src/types.ts index 9b62b743..69811614 100644 --- a/packages/import/src/types.ts +++ b/packages/import/src/types.ts @@ -16,6 +16,8 @@ export interface Issue { labels?: string[]; /** Link to original issue. */ url?: string; + /** Link to original issue. */ + extraUrls?: Url[]; /** When the issue was created. */ createdAt?: Date; /** When the issue is due. This is a date string of the format yyyy-MM-dd. */ @@ -36,6 +38,14 @@ export interface Comment { createdAt?: Date; } +/** Issue comment */ +export interface Url { + /** Comment's body in markdown */ + title?: string; + /** User who posted the comments */ + url: string; +} + export type IssueStatus = "backlog" | "unstarted" | "started" | "completed" | "canceled"; /** Import response. */ From 1b643ac8539685755ba993d99e49323b16c58712 Mon Sep 17 00:00:00 2001 From: Mikael Wahlberg Date: Thu, 12 May 2022 19:33:33 +0200 Subject: [PATCH 2/8] Some fixes --- packages/import/package.json | 3 +- .../redmineCsv/RedmineCsvImporter.ts | 31 ++++++++++++++++--- yarn.lock | 5 +++ 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/packages/import/package.json b/packages/import/package.json index e6886992..b971548c 100644 --- a/packages/import/package.json +++ b/packages/import/package.json @@ -34,7 +34,8 @@ "inquirer-file-path": "1.0.1", "jira2md": "2.0.4", "lodash": "4.17.20", - "node-fetch": "2.6.1" + "node-fetch": "2.6.1", + "node-pandoc": "0.3.0" }, "devDependencies": { "@rollup/plugin-commonjs": "^17.1.0", diff --git a/packages/import/src/importers/redmineCsv/RedmineCsvImporter.ts b/packages/import/src/importers/redmineCsv/RedmineCsvImporter.ts index 28c2b9eb..096097ab 100644 --- a/packages/import/src/importers/redmineCsv/RedmineCsvImporter.ts +++ b/packages/import/src/importers/redmineCsv/RedmineCsvImporter.ts @@ -2,7 +2,6 @@ import csv from "csvtojson"; //import { roundToNearestMinutes } from "date-fns"; import { Importer, ImportResult } from "../../types"; // eslint-disable-next-line @typescript-eslint/no-var-requires -const j2m = require("jira2md"); type RedmineStoryType = "Internal Bug" | "Internal Feature"; @@ -14,7 +13,7 @@ interface RedmineIssueType { "Iteration Start": string; "Iteration End": string; Tracker: RedmineStoryType; - Estimate: string; + Priority: string; "Current State": string; "Created at": Date; "Accepted at": Date; @@ -73,9 +72,19 @@ export class RedmineCsvImporter implements Importer { //const url = row.URL; const url = "https://redmine.iconik.biz/issues/" + row["#"]; - const mdDesc = j2m.to_markdown(row.Description); - const description = mdDesc; - + let pandoc = require('node-pandoc'), + src = row.Description, + args = '-f textile -t markdown'; + // Set your callback function + const description :string = await new Promise((resolve, reject) => { + pandoc(src, args, function(err: string, result: string): void { + if (err) { + reject(err) + return; + } + resolve(result) + }) + }); // const priority = parseInt(row['Estimate']) || undefined; const tags = row.Tags.split(","); @@ -96,6 +105,17 @@ export class RedmineCsvImporter implements Importer { } } + var priority = parseInt(row.Priority.substring(0,1)) + if (priority > 7) { + priority = 1; + } else if (priority > 5) { + priority = 2; + } else if (priority > 3) { + priority = 3; + } else { + priority = 4; + } + const assigneeId = row["Assignee"] && row["Assignee"].length > 0 ? row["Assignee"] : undefined; const status = row["Status"] && (row["Status"] === "Review" || row["Status"] === "Codereview") ? "Done" : "Todo"; @@ -121,6 +141,7 @@ export class RedmineCsvImporter implements Importer { assigneeId, labels, createdAt, + priority }); for (const lab of labels) { diff --git a/yarn.lock b/yarn.lock index e303c18c..8a216875 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9847,6 +9847,11 @@ node-notifier@^8.0.0: uuid "^8.3.0" which "^2.0.2" +node-pandoc@0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/node-pandoc/-/node-pandoc-0.3.0.tgz#d46577cd0a72af415353fcb8a0117adb6242f9c8" + integrity sha1-1GV3zQpyr0FTU/y4oBF622JC+cg= + node-releases@^1.1.73: version "1.1.73" resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-1.1.73.tgz#dd4e81ddd5277ff846b80b52bb40c49edf7a7b20" From d8959a102446d94b94b3bb0c4a5dfcd18615dcbc Mon Sep 17 00:00:00 2001 From: Mikael Wahlberg Date: Mon, 16 May 2022 18:45:22 +0200 Subject: [PATCH 3/8] Some fixes --- packages/import/src/importIssues.ts | 13 +++++++++++ .../redmineCsv/RedmineCsvImporter.ts | 22 +++++++++++++++---- packages/import/src/types.ts | 4 ++++ 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/packages/import/src/importIssues.ts b/packages/import/src/importIssues.ts index b566b1c4..4692141b 100644 --- a/packages/import/src/importIssues.ts +++ b/packages/import/src/importIssues.ts @@ -220,6 +220,7 @@ export const importIssues = async (apiKey: string, importer: Importer): Promise< } } + const originalIdmap = {} as {[name: string]: string}; // Create issues for (const issue of importData.issues) { const issueDescription = issue.description @@ -282,6 +283,18 @@ export const importIssues = async (apiKey: string, importer: Importer): Promise< dueDate: formattedDueDate, }); const newId = newIssue._issue.id; + if (!!issue.originalId) { + originalIdmap[issue.originalId] = newId; + console.error(`Adding ${issue.originalId} to ${newId} `) + } + if (!!issue.relatedOriginalIds) { + for (const relatedId of issue.relatedOriginalIds) { + console.error(`Checking ${relatedId}`) + if(!!originalIdmap[relatedId]) { + client.issueRelationCreate({issueId: newId, relatedIssueId: originalIdmap[relatedId], type: "related"}) + } + } + } //console.error(JSON.stringify(await client.issue(newId), null, 4)); if (!!issue.url) { client.attachmentLinkURL(newId, issue.url, { title: "Original Redmine issue" }); diff --git a/packages/import/src/importers/redmineCsv/RedmineCsvImporter.ts b/packages/import/src/importers/redmineCsv/RedmineCsvImporter.ts index 096097ab..ebd114be 100644 --- a/packages/import/src/importers/redmineCsv/RedmineCsvImporter.ts +++ b/packages/import/src/importers/redmineCsv/RedmineCsvImporter.ts @@ -15,7 +15,7 @@ interface RedmineIssueType { Tracker: RedmineStoryType; Priority: string; "Current State": string; - "Created at": Date; + Created: Date; "Accepted at": Date; "Category-Iconik": string; "Requested By": string; @@ -71,9 +71,14 @@ export class RedmineCsvImporter implements Importer { } //const url = row.URL; - const url = "https://redmine.iconik.biz/issues/" + row["#"]; + const originalId = row["#"]; + var pandocSource = row.Description + const url = "https://redmine.iconik.biz/issues/" + originalId; + if (row.Description.startsWith("http")) { + pandocSource = ".\n" + row.Description; + } let pandoc = require('node-pandoc'), - src = row.Description, + src = pandocSource, args = '-f textile -t markdown'; // Set your callback function const description :string = await new Promise((resolve, reject) => { @@ -104,6 +109,13 @@ export class RedmineCsvImporter implements Importer { } } } + const relatedOriginalIds = []; + const relatedIssues=row["Related issues"].split(","); + if(!!relatedIssues){ + for (const i of relatedIssues) { + relatedOriginalIds.push(i.slice(1+i.indexOf("#"))) + } + } var priority = parseInt(row.Priority.substring(0,1)) if (priority > 7) { @@ -141,7 +153,9 @@ export class RedmineCsvImporter implements Importer { assigneeId, labels, createdAt, - priority + priority, + originalId, + relatedOriginalIds, }); for (const lab of labels) { diff --git a/packages/import/src/types.ts b/packages/import/src/types.ts index 69811614..d4348f4f 100644 --- a/packages/import/src/types.ts +++ b/packages/import/src/types.ts @@ -26,6 +26,10 @@ export interface Issue { completedAt?: Date; /** When the issue was started. */ startedAt?: Date; + /** Original ID. */ + originalId?: string; + /** List of related original Ids. */ + relatedOriginalIds?: string[]; } /** Issue comment */ From fc604e03a9c03f9a85b178727dabcf5b73625d22 Mon Sep 17 00:00:00 2001 From: Mikael Wahlberg Date: Mon, 23 May 2022 14:09:53 +0200 Subject: [PATCH 4/8] More stuff for Redmine --- packages/import/package.json | 4 +- packages/import/src/importIssues.ts | 66 +++++++++++++++++-- .../redmineCsv/RedmineCsvImporter.ts | 49 +++++++++++++- yarn.lock | 29 +++++++- 4 files changed, 140 insertions(+), 8 deletions(-) diff --git a/packages/import/package.json b/packages/import/package.json index b971548c..747b4c83 100644 --- a/packages/import/package.json +++ b/packages/import/package.json @@ -35,7 +35,9 @@ "jira2md": "2.0.4", "lodash": "4.17.20", "node-fetch": "2.6.1", - "node-pandoc": "0.3.0" + "node-pandoc": "0.3.0", + "axios-redmine": "0.1.17", + "axios": "0.27.2" }, "devDependencies": { "@rollup/plugin-commonjs": "^17.1.0", diff --git a/packages/import/src/importIssues.ts b/packages/import/src/importIssues.ts index 4692141b..64ff199a 100644 --- a/packages/import/src/importIssues.ts +++ b/packages/import/src/importIssues.ts @@ -6,6 +6,8 @@ import * as inquirer from "inquirer"; import _ from "lodash"; import { Comment, Importer, ImportResult } from "./types"; import { replaceImagesInMarkdown } from "./utils/replaceImages"; +var axios = require('axios'); +var fs = require('fs'); interface ImportAnswers { newTeam: boolean; @@ -223,9 +225,8 @@ export const importIssues = async (apiKey: string, importer: Importer): Promise< const originalIdmap = {} as {[name: string]: string}; // Create issues for (const issue of importData.issues) { - const issueDescription = issue.description - ? await replaceImagesInMarkdown(client, issue.description, importData.resourceURLSuffix) - : undefined; + const issueDescription = issue.description; + const description = importAnswers.includeComments && issue.comments @@ -283,6 +284,8 @@ export const importIssues = async (apiKey: string, importer: Importer): Promise< dueDate: formattedDueDate, }); const newId = newIssue._issue.id; + var descriptionData = newIssue._issue; + console.log(JSON.stringify(newIssue)) if (!!issue.originalId) { originalIdmap[issue.originalId] = newId; console.error(`Adding ${issue.originalId} to ${newId} `) @@ -297,11 +300,64 @@ export const importIssues = async (apiKey: string, importer: Importer): Promise< } //console.error(JSON.stringify(await client.issue(newId), null, 4)); if (!!issue.url) { - client.attachmentLinkURL(newId, issue.url, { title: "Original Redmine issue" }); + await client.attachmentLinkURL(newId, issue.url, { title: "Original Redmine issue" }); } if (!!issue.extraUrls) { for (const url of issue.extraUrls) { - client.attachmentLinkURL(newId, url.url, !!url.title ? { title: url.title } : {}); + await client.attachmentLinkURL(newId, url.url, !!url.title ? { title: url.title } : {}); + } + } + var files = []; + const dir = `/tmp/redmineimporter/${issue.originalId}`; + if(!!issue.originalId) { + if (fs.existsSync(dir)){ + fs.readdirSync(dir).forEach(file => { + console.log(file); + files.push(file) + }); + } + } + if(!!files) { + var desc = description + var attachmentHeader = "# Attachments:\n\n" + for (const file of files) { + var contentType = "application/octet-stream" + var isImage = "" + if (file.toLowerCase().includes(".jpg")) { + contentType = "image/jpg"; + isImage="!"; + } else if (file.toLowerCase().includes(".png")) { + contentType = "image/png"; + isImage="!"; + } + var stats = fs.statSync(dir+"/"+file) + var fileSizeInBytes = stats.size; + + const uploadData = await client.fileUpload(contentType, file, fileSizeInBytes); + console.log(`UPLOAD: ${JSON.stringify(uploadData)}`) + const stream = fs.createReadStream(dir+"/"+file); + var headers = {}; + for (const h of uploadData.uploadFile.headers) { + headers[h.key] = h.value; + } + headers["content-type"] = uploadData.uploadFile?.contentType; + const upload = await axios({ + method: "put", + url: uploadData.uploadFile?.uploadUrl, + data: stream, + headers: headers, + }); + console.log(`RESULT: ${upload.status}`) + const issue = await client.issue(newId); + console.log(JSON.stringify(issue)) + const imageString = `![](${file})`; + if(desc?.includes(imageString)) { + desc = desc.replace(imageString, `${isImage}[${file}](${uploadData.uploadFile?.assetUrl})`) + } else { + desc = desc + `\n${attachmentHeader}${isImage}[${file}](${uploadData.uploadFile?.assetUrl})\n` + attachmentHeader = "" + } + await client.issueUpdate(newId, {description: desc }) } } } diff --git a/packages/import/src/importers/redmineCsv/RedmineCsvImporter.ts b/packages/import/src/importers/redmineCsv/RedmineCsvImporter.ts index ebd114be..74cb79ed 100644 --- a/packages/import/src/importers/redmineCsv/RedmineCsvImporter.ts +++ b/packages/import/src/importers/redmineCsv/RedmineCsvImporter.ts @@ -2,6 +2,8 @@ import csv from "csvtojson"; //import { roundToNearestMinutes } from "date-fns"; import { Importer, ImportResult } from "../../types"; // eslint-disable-next-line @typescript-eslint/no-var-requires +var https = require('https'); +var fs = require('fs'); type RedmineStoryType = "Internal Bug" | "Internal Feature"; @@ -63,6 +65,11 @@ export class RedmineCsvImporter implements Importer { name: user, }; } + const hostname = 'https://redmine.iconik.biz' + const config = { + apiKey: '7db9d4d08352f61f80457e4d31a530de5dd2f1df', + rejectUnauthorized: process.env.REJECT_UNAUTHORIZED + } for (const row of data) { const title = row.Subject; @@ -91,6 +98,46 @@ export class RedmineCsvImporter implements Importer { }) }); // const priority = parseInt(row['Estimate']) || undefined; + const Redmine = require('axios-redmine') + + // protocol required in Hostname, supports both HTTP and HTTPS + var attachments: any[] = []; + const redmine = new Redmine(hostname, config) + const dumpIssue = function (issue: any) { + console.log('Dumping issue:') + for (const item in issue) { + console.log(' ' + item + ': ' + JSON.stringify(issue[item])) + } + } + const params = { include: 'attachments,journals,watchers' } + await redmine + .get_issue_by_id(parseInt(originalId), params) + .then(response => { + attachments = response.data.issue.attachments; + }) + .catch(err => { + console.log(err) + }) + + if (!!attachments && (attachments.length > 0)) { + console.log(`Attachments2: ${JSON.stringify(attachments)}`) + const dir = `/tmp/redmineimporter/${parseInt(originalId)}`; + if (!fs.existsSync(dir)){ + fs.mkdirSync(dir, { recursive: true }); + } + for (const attachment of attachments) { + const file = fs.createWriteStream(`${dir}/${attachment.filename}`); + const request = https.get(attachment.content_url, {headers: {"X-Redmine-API-Key": config.apiKey}}, function(response) { + response.pipe(file); + + // after download completed close filestream + file.on("finish", () => { + file.close(); + console.log("Download Completed"); + }); + }); + } + } const tags = row.Tags.split(","); const categories = row["Category-Iconik"].split(","); @@ -143,7 +190,7 @@ export class RedmineCsvImporter implements Importer { labels = labels.concat(categories.filter(tag => !!tag)); } const createdAt = row["Created"]; - + console.log(description) importData.issues.push({ title, description, diff --git a/yarn.lock b/yarn.lock index 8a216875..030f947f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3656,6 +3656,28 @@ aws4@^1.8.0: resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.11.0.tgz#d61f46d83b2519250e2784daf5b09479a8b41c59" integrity sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA== +axios-redmine@0.1.17: + version "0.1.17" + resolved "https://registry.yarnpkg.com/axios-redmine/-/axios-redmine-0.1.17.tgz#e904782c546244860370e351ac4693fe7a3a996f" + integrity sha512-xKIQMj8R1ncOfwyaBAks7umMikzq3PlgSG7zkNPUWmMBBw9ibE/YV8MhYNScU23U4qoOqPbEYlM6i+EWHoRy3A== + dependencies: + axios "^0.21.1" + +axios@0.27.2: + version "0.27.2" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.27.2.tgz#207658cc8621606e586c85db4b41a750e756d972" + integrity sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ== + dependencies: + follow-redirects "^1.14.9" + form-data "^4.0.0" + +axios@^0.21.1: + version "0.21.4" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575" + integrity sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg== + dependencies: + follow-redirects "^1.14.0" + babel-jest@^26.6.3: version "26.6.3" resolved "https://registry.yarnpkg.com/babel-jest/-/babel-jest-26.6.3.tgz#d87d25cb0037577a0c89f82e5755c5d293c01056" @@ -6395,6 +6417,11 @@ flush-write-stream@^1.0.0: inherits "^2.0.3" readable-stream "^2.3.6" +follow-redirects@^1.14.0, follow-redirects@^1.14.9: + version "1.15.0" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.0.tgz#06441868281c86d0dda4ad8bdaead2d02dca89d4" + integrity sha512-aExlJShTV4qOUOL7yF1U5tvLCB0xQuudbf6toyYA0E/acBNw71mvjFTnLaRp50aQaYocMR0a/RMMBIHeZnGyjQ== + for-in@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/for-in/-/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80" @@ -6405,7 +6432,7 @@ forever-agent@~0.6.1: resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" integrity sha1-+8cfDEGt6zf5bFd60e1C2P2sypE= -form-data@4.0.0: +form-data@4.0.0, form-data@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452" integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww== From 174c03deb6ceb9a4b07362f928bdfb73a463c6ac Mon Sep 17 00:00:00 2001 From: Mikael Gransell Date: Wed, 21 Sep 2022 09:59:28 +0200 Subject: [PATCH 5/8] Add original comments and upload large files --- packages/import/src/importIssues.ts | 70 ++++---- .../redmineCsv/RedmineCsvImporter.ts | 152 +++++++++++------- 2 files changed, 125 insertions(+), 97 deletions(-) diff --git a/packages/import/src/importIssues.ts b/packages/import/src/importIssues.ts index 64ff199a..37cff9f7 100644 --- a/packages/import/src/importIssues.ts +++ b/packages/import/src/importIssues.ts @@ -1,13 +1,13 @@ /* eslint-disable no-console */ import { LinearClient } from "@linear/sdk"; -import { format } from "date-fns"; import chalk from "chalk"; +import { format } from "date-fns"; import * as inquirer from "inquirer"; import _ from "lodash"; import { Comment, Importer, ImportResult } from "./types"; import { replaceImagesInMarkdown } from "./utils/replaceImages"; -var axios = require('axios'); -var fs = require('fs'); +const axios = require("axios"); +const fs = require("fs"); interface ImportAnswers { newTeam: boolean; @@ -222,11 +222,10 @@ export const importIssues = async (apiKey: string, importer: Importer): Promise< } } - const originalIdmap = {} as {[name: string]: string}; + const originalIdmap = {} as { [name: string]: string }; // Create issues for (const issue of importData.issues) { const issueDescription = issue.description; - const description = importAnswers.includeComments && issue.comments @@ -284,17 +283,17 @@ export const importIssues = async (apiKey: string, importer: Importer): Promise< dueDate: formattedDueDate, }); const newId = newIssue._issue.id; - var descriptionData = newIssue._issue; - console.log(JSON.stringify(newIssue)) + const descriptionData = newIssue._issue; + console.log(JSON.stringify(newIssue)); if (!!issue.originalId) { originalIdmap[issue.originalId] = newId; - console.error(`Adding ${issue.originalId} to ${newId} `) + console.error(`Adding ${issue.originalId} to ${newId} `); } if (!!issue.relatedOriginalIds) { for (const relatedId of issue.relatedOriginalIds) { - console.error(`Checking ${relatedId}`) - if(!!originalIdmap[relatedId]) { - client.issueRelationCreate({issueId: newId, relatedIssueId: originalIdmap[relatedId], type: "related"}) + console.error(`Checking ${relatedId}`); + if (!!originalIdmap[relatedId]) { + client.issueRelationCreate({ issueId: newId, relatedIssueId: originalIdmap[relatedId], type: "related" }); } } } @@ -309,36 +308,36 @@ export const importIssues = async (apiKey: string, importer: Importer): Promise< } var files = []; const dir = `/tmp/redmineimporter/${issue.originalId}`; - if(!!issue.originalId) { - if (fs.existsSync(dir)){ + if (!!issue.originalId) { + if (fs.existsSync(dir)) { fs.readdirSync(dir).forEach(file => { console.log(file); - files.push(file) + files.push(file); }); - } + } } - if(!!files) { - var desc = description - var attachmentHeader = "# Attachments:\n\n" + if (!!files) { + let desc = description; + let attachmentHeader = "# Attachments:\n\n"; for (const file of files) { - var contentType = "application/octet-stream" - var isImage = "" + let contentType = "application/octet-stream"; + let isImage = ""; if (file.toLowerCase().includes(".jpg")) { contentType = "image/jpg"; - isImage="!"; + isImage = "!"; } else if (file.toLowerCase().includes(".png")) { contentType = "image/png"; - isImage="!"; + isImage = "!"; } - var stats = fs.statSync(dir+"/"+file) - var fileSizeInBytes = stats.size; + const stats = fs.statSync(dir + "/" + file); + const fileSizeInBytes = stats.size; const uploadData = await client.fileUpload(contentType, file, fileSizeInBytes); - console.log(`UPLOAD: ${JSON.stringify(uploadData)}`) - const stream = fs.createReadStream(dir+"/"+file); - var headers = {}; + console.log(`UPLOAD: ${JSON.stringify(uploadData)}`); + const stream = fs.createReadStream(dir + "/" + file); + const headers = {}; for (const h of uploadData.uploadFile.headers) { - headers[h.key] = h.value; + headers[h.key] = h.value; } headers["content-type"] = uploadData.uploadFile?.contentType; const upload = await axios({ @@ -346,18 +345,19 @@ export const importIssues = async (apiKey: string, importer: Importer): Promise< url: uploadData.uploadFile?.uploadUrl, data: stream, headers: headers, + maxBodyLength: 100_000_000, }); - console.log(`RESULT: ${upload.status}`) + console.log(`RESULT: ${upload.status}`); const issue = await client.issue(newId); - console.log(JSON.stringify(issue)) + console.log(JSON.stringify(issue)); const imageString = `![](${file})`; - if(desc?.includes(imageString)) { - desc = desc.replace(imageString, `${isImage}[${file}](${uploadData.uploadFile?.assetUrl})`) + if (desc?.includes(imageString)) { + desc = desc.replace(imageString, `${isImage}[${file}](${uploadData.uploadFile?.assetUrl})`); } else { - desc = desc + `\n${attachmentHeader}${isImage}[${file}](${uploadData.uploadFile?.assetUrl})\n` - attachmentHeader = "" + desc = desc + `\n${attachmentHeader}${isImage}[${file}](${uploadData.uploadFile?.assetUrl})\n`; + attachmentHeader = ""; } - await client.issueUpdate(newId, {description: desc }) + await client.issueUpdate(newId, { description: desc }); } } } diff --git a/packages/import/src/importers/redmineCsv/RedmineCsvImporter.ts b/packages/import/src/importers/redmineCsv/RedmineCsvImporter.ts index 74cb79ed..d3fa88a2 100644 --- a/packages/import/src/importers/redmineCsv/RedmineCsvImporter.ts +++ b/packages/import/src/importers/redmineCsv/RedmineCsvImporter.ts @@ -1,13 +1,17 @@ +/* eslint-disable no-console */ +/* eslint-disable @typescript-eslint/no-explicit-any */ import csv from "csvtojson"; //import { roundToNearestMinutes } from "date-fns"; -import { Importer, ImportResult } from "../../types"; +import { Comment, Importer, ImportResult } from "../../types"; // eslint-disable-next-line @typescript-eslint/no-var-requires -var https = require('https'); -var fs = require('fs'); +const https = require("https"); +const fs = require("fs"); type RedmineStoryType = "Internal Bug" | "Internal Feature"; interface RedmineIssueType { + Assignee: string; + Status: string; "#": string; Subject: string; Tags: string; @@ -58,18 +62,14 @@ export class RedmineCsvImporter implements Importer { statuses: {}, }; - const assignees = Array.from(new Set(data.map(row => row["Assignee"]))); + const assignees = Array.from(new Set(data.map(row => row.Assignee))); + const users = new Set(assignees); - for (const user of assignees) { - importData.users[user] = { - name: user, - }; - } - const hostname = 'https://redmine.iconik.biz' + const hostname = "https://redmine.iconik.biz"; const config = { - apiKey: '7db9d4d08352f61f80457e4d31a530de5dd2f1df', - rejectUnauthorized: process.env.REJECT_UNAUTHORIZED - } + apiKey: "7db9d4d08352f61f80457e4d31a530de5dd2f1df", + rejectUnauthorized: process.env.REJECT_UNAUTHORIZED, + }; for (const row of data) { const title = row.Subject; @@ -79,74 +79,94 @@ export class RedmineCsvImporter implements Importer { //const url = row.URL; const originalId = row["#"]; - var pandocSource = row.Description + let pandocSource = row.Description; const url = "https://redmine.iconik.biz/issues/" + originalId; - if (row.Description.startsWith("http")) { - pandocSource = ".\n" + row.Description; - } - let pandoc = require('node-pandoc'), + if (row.Description?.startsWith("http")) { + pandocSource = ".\n" + row.Description; + } + pandocSource += "\n\n"; + const pandoc = require("node-pandoc"), src = pandocSource, - args = '-f textile -t markdown'; + args = "-f textile -t markdown"; // Set your callback function - const description :string = await new Promise((resolve, reject) => { - pandoc(src, args, function(err: string, result: string): void { + let description: string = await new Promise((resolve, reject) => { + pandoc(src, args, function (err: string, result: string): void { if (err) { - reject(err) + reject(err); return; } - resolve(result) - }) + resolve(result); + }); }); // const priority = parseInt(row['Estimate']) || undefined; - const Redmine = require('axios-redmine') + const Redmine = require("axios-redmine"); // protocol required in Hostname, supports both HTTP and HTTPS - var attachments: any[] = []; - const redmine = new Redmine(hostname, config) + const redmine = new Redmine(hostname, config); const dumpIssue = function (issue: any) { - console.log('Dumping issue:') + console.log("Dumping issue:"); for (const item in issue) { - console.log(' ' + item + ': ' + JSON.stringify(issue[item])) + console.log(" " + item + ": " + JSON.stringify(issue[item])); } + }; + const params = { include: "attachments,journals,watchers" }; + const response = await redmine.get_issue_by_id(parseInt(originalId), params).catch((err: unknown) => { + console.log(err); + return undefined; + }); + if (response === undefined) { + console.log("Skipping redmine issue:", originalId); + continue; } - const params = { include: 'attachments,journals,watchers' } - await redmine - .get_issue_by_id(parseInt(originalId), params) - .then(response => { - attachments = response.data.issue.attachments; - }) - .catch(err => { - console.log(err) - }) - - if (!!attachments && (attachments.length > 0)) { - console.log(`Attachments2: ${JSON.stringify(attachments)}`) + + const attachments: any[] = response.data.issue.attachments.filter( + (attachment: any) => attachment.filesize < 50000000 + ); + + if (!!attachments && attachments.length > 0) { + console.log(`Attachments2: ${JSON.stringify(attachments)}`); const dir = `/tmp/redmineimporter/${parseInt(originalId)}`; - if (!fs.existsSync(dir)){ + if (!fs.existsSync(dir)) { fs.mkdirSync(dir, { recursive: true }); - } + } for (const attachment of attachments) { const file = fs.createWriteStream(`${dir}/${attachment.filename}`); - const request = https.get(attachment.content_url, {headers: {"X-Redmine-API-Key": config.apiKey}}, function(response) { - response.pipe(file); - - // after download completed close filestream - file.on("finish", () => { + const request = https.get( + attachment.content_url, + { headers: { "X-Redmine-API-Key": config.apiKey } }, + function (response: any) { + response.pipe(file); + + // after download completed close filestream + file.on("finish", () => { file.close(); - console.log("Download Completed"); - }); - }); + // console.log("Download Completed"); + }); + } + ); } } + const journals: any[] = response.data.issue.journals; + const comments = journals + .filter(journal => journal.notes !== undefined && journal.notes.length !== 0) + .map( + (journal: any): Comment => ({ + body: journal.notes, + userId: journal.user.name, + createdAt: new Date(journal.created_on), + }) + ); + comments.forEach(({ userId }) => users.add(userId)); + const tags = row.Tags.split(","); const categories = row["Category-Iconik"].split(","); - const refs = row["Internal Reference"].split("\n"); + const refs = row["Internal Reference"].split(/[\n\s]/); const extraUrls = []; if (!!refs) { for (const r of refs) { //console.error(r.trim()); - if (r.startsWith("http")) { + if (r?.startsWith("http")) { if (r.includes("support.iconik.io")) { extraUrls.push({ url: r.trim(), title: "Zoho desk issue" }); } else { @@ -157,17 +177,17 @@ export class RedmineCsvImporter implements Importer { } } const relatedOriginalIds = []; - const relatedIssues=row["Related issues"].split(","); - if(!!relatedIssues){ + const relatedIssues = row["Related issues"].split(","); + if (!!relatedIssues) { for (const i of relatedIssues) { - relatedOriginalIds.push(i.slice(1+i.indexOf("#"))) + relatedOriginalIds.push(i.slice(1 + i.indexOf("#"))); } } - var priority = parseInt(row.Priority.substring(0,1)) + let priority = parseInt(row.Priority.substring(0, 1)); if (priority > 7) { priority = 1; - } else if (priority > 5) { + } else if (priority > 5) { priority = 2; } else if (priority > 3) { priority = 3; @@ -175,9 +195,9 @@ export class RedmineCsvImporter implements Importer { priority = 4; } - const assigneeId = row["Assignee"] && row["Assignee"].length > 0 ? row["Assignee"] : undefined; + const assigneeId = row.Assignee && row.Assignee.length > 0 ? row.Assignee : undefined; - const status = row["Status"] && (row["Status"] === "Review" || row["Status"] === "Codereview") ? "Done" : "Todo"; + const status = row.Status && (row.Status === "Review" || row.Status === "Codereview") ? "Done" : "Todo"; let labels = tags.filter(tag => !!tag); if (row.Tracker === "Internal Bug") { @@ -189,8 +209,11 @@ export class RedmineCsvImporter implements Importer { if (!!categories) { labels = labels.concat(categories.filter(tag => !!tag)); } - const createdAt = row["Created"]; - console.log(description) + const createdAt = row.Created; + if (comments.length > 0) { + description = description.concat(" \n \n## Original Comments:\n"); + } + // console.log(description); importData.issues.push({ title, description, @@ -203,6 +226,7 @@ export class RedmineCsvImporter implements Importer { priority, originalId, relatedOriginalIds, + comments, }); for (const lab of labels) { @@ -214,6 +238,10 @@ export class RedmineCsvImporter implements Importer { } } + for (const user of users) { + importData.users[user] = { name: user }; + } + return importData; }; From f0a23b326ad22f90ca5c6ff5cc06f927108fc487 Mon Sep 17 00:00:00 2001 From: Mikael Gransell Date: Thu, 17 Nov 2022 15:25:10 +0100 Subject: [PATCH 6/8] Set "Exported to Linear" flag on Redmine issues --- .../import/src/importers/redmineCsv/RedmineCsvImporter.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/import/src/importers/redmineCsv/RedmineCsvImporter.ts b/packages/import/src/importers/redmineCsv/RedmineCsvImporter.ts index d3fa88a2..7a2a569a 100644 --- a/packages/import/src/importers/redmineCsv/RedmineCsvImporter.ts +++ b/packages/import/src/importers/redmineCsv/RedmineCsvImporter.ts @@ -119,6 +119,14 @@ export class RedmineCsvImporter implements Importer { continue; } + await redmine + .update_issue(parseInt(originalId), { + issue: { custom_fields: [{ id: 56, name: "Exported to Linear", value: 1 }] }, + }) + .catch((err: unknown) => { + console.log("Failed updating Exported to Linear field", err); + }); + const attachments: any[] = response.data.issue.attachments.filter( (attachment: any) => attachment.filesize < 50000000 ); From efdc7f4c0d6c78b2d3ae9cd3ade897dbb4f4aa26 Mon Sep 17 00:00:00 2001 From: Mikael Gransell Date: Fri, 18 Nov 2022 10:29:21 +0100 Subject: [PATCH 7/8] Add labels to workspace instead of specific project --- packages/import/src/importIssues.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/import/src/importIssues.ts b/packages/import/src/importIssues.ts index 37cff9f7..306a3f36 100644 --- a/packages/import/src/importIssues.ts +++ b/packages/import/src/importIssues.ts @@ -168,12 +168,15 @@ export const importIssues = async (apiKey: string, importer: Importer): Promise< } const teamInfo = await client.team(teamId); + const organization = await client.organization; const issueLabels = await teamInfo?.labels(); + const organizationLabels = await organization.labels(); const workflowStates = await teamInfo?.states(); const existingLabelMap = {} as { [name: string]: string }; - for (const label of issueLabels?.nodes ?? []) { + const allLabels = (issueLabels.nodes ?? []).concat(organizationLabels.nodes); + for (const label of allLabels) { const labelName = label.name?.toLowerCase(); if (labelName && label.id && !existingLabelMap[labelName]) { existingLabelMap[labelName] = label.id; @@ -194,7 +197,6 @@ export const importIssues = async (apiKey: string, importer: Importer): Promise< name: labelName, description: label.description, color: label.color, - teamId, }); const issueLabel = await labelResponse?.issueLabel; From 4c8e02065fbfac4ff59dc60f4b4a5422d16efb0b Mon Sep 17 00:00:00 2001 From: Mikael Gransell Date: Thu, 8 Dec 2022 16:42:45 +0100 Subject: [PATCH 8/8] Don't fail issue import if a label can't be created --- packages/import/src/importIssues.ts | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/packages/import/src/importIssues.ts b/packages/import/src/importIssues.ts index 306a3f36..1b09be44 100644 --- a/packages/import/src/importIssues.ts +++ b/packages/import/src/importIssues.ts @@ -3,7 +3,6 @@ import { LinearClient } from "@linear/sdk"; import chalk from "chalk"; import { format } from "date-fns"; import * as inquirer from "inquirer"; -import _ from "lodash"; import { Comment, Importer, ImportResult } from "./types"; import { replaceImagesInMarkdown } from "./utils/replaceImages"; const axios = require("axios"); @@ -189,15 +188,21 @@ export const importIssues = async (apiKey: string, importer: Importer): Promise< const labelMapping = {} as { [id: string]: string }; for (const labelId of Object.keys(importData.labels)) { const label = importData.labels[labelId]; - const labelName = _.truncate(label.name.trim(), { length: 20 }); + const labelName = label.name; let actualLabelId = existingLabelMap[labelName.toLowerCase()]; if (!actualLabelId) { - const labelResponse = await client.issueLabelCreate({ - name: labelName, - description: label.description, - color: label.color, - }); + console.log("Label", labelName, "not found. Creating"); + const labelResponse = await client + .issueLabelCreate({ + name: labelName, + description: label.description, + color: label.color, + }) + .catch(() => { + console.log("Unable to create label", labelName); + return undefined; + }); const issueLabel = await labelResponse?.issueLabel; if (issueLabel?.id) { @@ -234,7 +239,9 @@ export const importIssues = async (apiKey: string, importer: Importer): Promise< ? await buildComments(client, issueDescription || "", issue.comments, importData) : issueDescription; - const labelIds = issue.labels ? issue.labels.map(labelId => labelMapping[labelId]) : undefined; + const labelIds = issue.labels + ? issue.labels.map(labelId => labelMapping[labelId]).filter(id => Boolean(id)) + : undefined; let stateId = !!issue.status ? existingStateMap[issue.status.toLowerCase()] : undefined; // Create a new state since one doesn't already exist with this name