diff --git a/packages/import/package.json b/packages/import/package.json index 0a4f380f..d5acd7ee 100644 --- a/packages/import/package.json +++ b/packages/import/package.json @@ -34,7 +34,10 @@ "inquirer-file-path": "1.0.1", "jira2md": "2.0.4", "lodash": "4.17.20", - "node-fetch": "2.6.1" + "node-fetch": "2.6.1", + "node-pandoc": "0.3.0", + "axios-redmine": "0.1.17", + "axios": "0.27.2" }, "devDependencies": { "@rollup/plugin-commonjs": "^17.1.0", diff --git a/packages/import/src/cli.ts b/packages/import/src/cli.ts index 3fa055d6..94b943c2 100644 --- a/packages/import/src/cli.ts +++ b/packages/import/src/cli.ts @@ -7,6 +7,7 @@ import { githubImport } from "./importers/github"; import { jiraCsvImport } from "./importers/jiraCsv"; import { linearCsvImporter } from "./importers/linearCsv"; import { pivotalCsvImport } from "./importers/pivotalCsv"; +import { redmineCsvImport } from "./importers/redmineCsv"; import { shortcutCsvImport } from "./importers/shortcutCsv"; import { trelloJsonImport } from "./importers/trelloJson"; import { importIssues } from "./importIssues"; @@ -43,6 +44,14 @@ inquirer.registerPrompt("filePath", require("inquirer-file-path")); name: "Pivotal (CSV export)", value: "pivotalCsv", }, + { + name: "Redmine (CSV export)", + value: "redmineCsv", + }, + { + name: "Redmine (CSV export)", + value: "redmineCsv", + }, { name: "Shortcut (CSV export)", value: "shortcutCsv", @@ -74,6 +83,9 @@ inquirer.registerPrompt("filePath", require("inquirer-file-path")); case "pivotalCsv": importer = await pivotalCsvImport(); break; + case "redmineCsv": + importer = await redmineCsvImport(); + break; case "shortcutCsv": importer = await shortcutCsvImport(); break; diff --git a/packages/import/src/importIssues.ts b/packages/import/src/importIssues.ts index b1633747..42b66908 100644 --- a/packages/import/src/importIssues.ts +++ b/packages/import/src/importIssues.ts @@ -1,11 +1,12 @@ /* eslint-disable no-console */ import { LinearClient } from "@linear/sdk"; -import { format } from "date-fns"; import chalk from "chalk"; +import { format } from "date-fns"; +import fs from "fs"; import * as inquirer from "inquirer"; -import _ from "lodash"; import { Comment, Importer, ImportResult } from "./types"; import { replaceImagesInMarkdown } from "./utils/replaceImages"; +const axios = require("axios"); interface ImportAnswers { newTeam: boolean; @@ -166,12 +167,15 @@ export const importIssues = async (apiKey: string, importer: Importer): Promise< } const teamInfo = await client.team(teamId); + const organization = await client.organization; const issueLabels = await teamInfo?.labels(); + const organizationLabels = await organization.labels(); const workflowStates = await teamInfo?.states(); const existingLabelMap = {} as { [name: string]: string }; - for (const label of issueLabels?.nodes ?? []) { + const allLabels = (issueLabels.nodes ?? []).concat(organizationLabels.nodes); + for (const label of allLabels) { const labelName = label.name?.toLowerCase(); if (labelName && label.id && !existingLabelMap[labelName]) { existingLabelMap[labelName] = label.id; @@ -184,16 +188,21 @@ export const importIssues = async (apiKey: string, importer: Importer): Promise< const labelMapping = {} as { [id: string]: string }; for (const labelId of Object.keys(importData.labels)) { const label = importData.labels[labelId]; - const labelName = _.truncate(label.name.trim(), { length: 20 }); + const labelName = label.name; let actualLabelId = existingLabelMap[labelName.toLowerCase()]; if (!actualLabelId) { - const labelResponse = await client.createIssueLabel({ - name: labelName, - description: label.description, - color: label.color, - teamId, - }); + console.log("Label", labelName, "not found. Creating"); + const labelResponse = await client + .createIssueLabel({ + name: labelName, + description: label.description, + color: label.color, + }) + .catch(() => { + console.log("Unable to create label", labelName); + return undefined; + }); const issueLabel = await labelResponse?.issueLabel; if (issueLabel?.id) { @@ -220,18 +229,19 @@ export const importIssues = async (apiKey: string, importer: Importer): Promise< } } + const originalIdmap = {} as { [name: string]: string }; // Create issues for (const issue of importData.issues) { - const issueDescription = issue.description - ? await replaceImagesInMarkdown(client, issue.description, importData.resourceURLSuffix) - : undefined; + const issueDescription = issue.description; const description = importAnswers.includeComments && issue.comments ? await buildComments(client, issueDescription || "", issue.comments, importData) : issueDescription; - const labelIds = issue.labels ? issue.labels.map(labelId => labelMapping[labelId]) : undefined; + const labelIds = issue.labels + ? issue.labels.map(labelId => labelMapping[labelId]).filter(id => Boolean(id)) + : undefined; let stateId = !!issue.status ? existingStateMap[issue.status.toLowerCase()] : undefined; // Create a new state since one doesn't already exist with this name @@ -270,7 +280,7 @@ export const importIssues = async (apiKey: string, importer: Importer): Promise< const formattedDueDate = issue.dueDate ? format(issue.dueDate, "yyyy-MM-dd") : undefined; - await client.createIssue({ + const newIssue = await client.createIssue({ teamId, projectId: projectId as unknown as string, title: issue.title, @@ -281,6 +291,91 @@ export const importIssues = async (apiKey: string, importer: Importer): Promise< assigneeId, dueDate: formattedDueDate, }); + const newId = (await newIssue.issue)?.id; + console.log(JSON.stringify(newIssue)); + if (!!newId) { + if (!!issue.originalId) { + originalIdmap[issue.originalId] = newId; + console.error(`Adding ${issue.originalId} to ${newId} `); + } + // if (!!issue.relatedOriginalIds) { + // for (const relatedId of issue.relatedOriginalIds) { + // console.error(`Checking ${relatedId}`); + // if (!!originalIdmap[relatedId]) { + // client.createIssueRelation({ + // issueId: newId, + // relatedIssueId: originalIdmap[relatedId], + // type: IssueRelationType.Related, + // }); + // } + // } + // } + //console.error(JSON.stringify(await client.issue(newId), null, 4)); + if (!!issue.url) { + await client.attachmentLinkURL(newId, issue.url, { title: "Original Redmine issue" }); + } + if (!!issue.extraUrls) { + for (const url of issue.extraUrls) { + await client.attachmentLinkURL(newId, url.url, !!url.title ? { title: url.title } : {}); + } + } + const files: string[] = []; + const dir = `/tmp/redmineimporter/${issue.originalId}`; + if (!!issue.originalId) { + if (fs.existsSync(dir)) { + fs.readdirSync(dir).forEach(file => { + console.log(file); + files.push(file); + }); + } + } + if (files.length > 0) { + let desc = description; + let attachmentHeader = "# Attachments:\n\n"; + for (const file of files) { + let contentType = "application/octet-stream"; + let isImage = ""; + if (file.toLowerCase().includes(".jpg")) { + contentType = "image/jpg"; + isImage = "!"; + } else if (file.toLowerCase().includes(".png")) { + contentType = "image/png"; + isImage = "!"; + } + const stats = fs.statSync(dir + "/" + file); + const fileSizeInBytes = stats.size; + + const uploadData = await client.fileUpload(contentType, file, fileSizeInBytes); + console.log(`UPLOAD: ${JSON.stringify(uploadData)}`); + const stream = fs.createReadStream(dir + "/" + file); + const headers = {}; + for (const h of uploadData.uploadFile?.headers || []) { + headers[h.key] = h.value; + } + headers["content-type"] = uploadData.uploadFile?.contentType; + const upload = await axios({ + method: "put", + url: uploadData.uploadFile?.uploadUrl, + data: stream, + headers: headers, + maxBodyLength: 100_000_000, + }); + console.log(`RESULT: ${upload.status}`); + const issue = await client.issue(newId); + console.log(JSON.stringify(issue)); + const imageString = `![](${file})`; + if (desc?.includes(imageString)) { + desc = desc.replace(imageString, `${isImage}[${file}](${uploadData.uploadFile?.assetUrl})`); + } else { + desc = desc + `\n${attachmentHeader}${isImage}[${file}](${uploadData.uploadFile?.assetUrl})\n`; + attachmentHeader = ""; + } + await client.updateIssue(newId, { description: desc }); + } + } + } else { + console.error("No id on newly created issue"); + } } console.info(chalk.green(`${importer.name} issues imported to your team: https://linear.app/team/${teamKey}/all`)); diff --git a/packages/import/src/importers/redmineCsv/RedmineCsvImporter.ts b/packages/import/src/importers/redmineCsv/RedmineCsvImporter.ts new file mode 100644 index 00000000..7a2a569a --- /dev/null +++ b/packages/import/src/importers/redmineCsv/RedmineCsvImporter.ts @@ -0,0 +1,259 @@ +/* eslint-disable no-console */ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import csv from "csvtojson"; +//import { roundToNearestMinutes } from "date-fns"; +import { Comment, Importer, ImportResult } from "../../types"; +// eslint-disable-next-line @typescript-eslint/no-var-requires +const https = require("https"); +const fs = require("fs"); + +type RedmineStoryType = "Internal Bug" | "Internal Feature"; + +interface RedmineIssueType { + Assignee: string; + Status: string; + "#": string; + Subject: string; + Tags: string; + Iteration: string; + "Iteration Start": string; + "Iteration End": string; + Tracker: RedmineStoryType; + Priority: string; + "Current State": string; + Created: Date; + "Accepted at": Date; + "Category-Iconik": string; + "Requested By": string; + Description: string; + URL: string; + "Owned By": string; + Blocker: string; + "Blocker Status": string; + Comment: string; +} + +/** + * Import issues from an Redmine Tracker CSV export. + * + * @param filePath path to csv file + * @param orgSlug base Redmine project url + */ +export class RedmineCsvImporter implements Importer { + public constructor(filePath: string) { + this.filePath = filePath; + } + + public get name(): string { + return "Redmine (CSV)"; + } + + public get defaultTeamName(): string { + return "Redmine"; + } + + public import = async (): Promise => { + const data = (await csv().fromFile(this.filePath)) as RedmineIssueType[]; + + const importData: ImportResult = { + issues: [], + labels: {}, + users: {}, + statuses: {}, + }; + + const assignees = Array.from(new Set(data.map(row => row.Assignee))); + const users = new Set(assignees); + + const hostname = "https://redmine.iconik.biz"; + const config = { + apiKey: "7db9d4d08352f61f80457e4d31a530de5dd2f1df", + rejectUnauthorized: process.env.REJECT_UNAUTHORIZED, + }; + + for (const row of data) { + const title = row.Subject; + if (!title) { + continue; + } + + //const url = row.URL; + const originalId = row["#"]; + let pandocSource = row.Description; + const url = "https://redmine.iconik.biz/issues/" + originalId; + if (row.Description?.startsWith("http")) { + pandocSource = ".\n" + row.Description; + } + pandocSource += "\n\n"; + const pandoc = require("node-pandoc"), + src = pandocSource, + args = "-f textile -t markdown"; + // Set your callback function + let description: string = await new Promise((resolve, reject) => { + pandoc(src, args, function (err: string, result: string): void { + if (err) { + reject(err); + return; + } + resolve(result); + }); + }); + // const priority = parseInt(row['Estimate']) || undefined; + const Redmine = require("axios-redmine"); + + // protocol required in Hostname, supports both HTTP and HTTPS + const redmine = new Redmine(hostname, config); + const dumpIssue = function (issue: any) { + console.log("Dumping issue:"); + for (const item in issue) { + console.log(" " + item + ": " + JSON.stringify(issue[item])); + } + }; + const params = { include: "attachments,journals,watchers" }; + const response = await redmine.get_issue_by_id(parseInt(originalId), params).catch((err: unknown) => { + console.log(err); + return undefined; + }); + if (response === undefined) { + console.log("Skipping redmine issue:", originalId); + continue; + } + + await redmine + .update_issue(parseInt(originalId), { + issue: { custom_fields: [{ id: 56, name: "Exported to Linear", value: 1 }] }, + }) + .catch((err: unknown) => { + console.log("Failed updating Exported to Linear field", err); + }); + + const attachments: any[] = response.data.issue.attachments.filter( + (attachment: any) => attachment.filesize < 50000000 + ); + + if (!!attachments && attachments.length > 0) { + console.log(`Attachments2: ${JSON.stringify(attachments)}`); + const dir = `/tmp/redmineimporter/${parseInt(originalId)}`; + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + for (const attachment of attachments) { + const file = fs.createWriteStream(`${dir}/${attachment.filename}`); + const request = https.get( + attachment.content_url, + { headers: { "X-Redmine-API-Key": config.apiKey } }, + function (response: any) { + response.pipe(file); + + // after download completed close filestream + file.on("finish", () => { + file.close(); + // console.log("Download Completed"); + }); + } + ); + } + } + + const journals: any[] = response.data.issue.journals; + const comments = journals + .filter(journal => journal.notes !== undefined && journal.notes.length !== 0) + .map( + (journal: any): Comment => ({ + body: journal.notes, + userId: journal.user.name, + createdAt: new Date(journal.created_on), + }) + ); + comments.forEach(({ userId }) => users.add(userId)); + + const tags = row.Tags.split(","); + const categories = row["Category-Iconik"].split(","); + const refs = row["Internal Reference"].split(/[\n\s]/); + const extraUrls = []; + if (!!refs) { + for (const r of refs) { + //console.error(r.trim()); + if (r?.startsWith("http")) { + if (r.includes("support.iconik.io")) { + extraUrls.push({ url: r.trim(), title: "Zoho desk issue" }); + } else { + extraUrls.push({ url: r.trim() }); + } + //console.error(`Adding URL: ${r.trim()}`); + } + } + } + const relatedOriginalIds = []; + const relatedIssues = row["Related issues"].split(","); + if (!!relatedIssues) { + for (const i of relatedIssues) { + relatedOriginalIds.push(i.slice(1 + i.indexOf("#"))); + } + } + + let priority = parseInt(row.Priority.substring(0, 1)); + if (priority > 7) { + priority = 1; + } else if (priority > 5) { + priority = 2; + } else if (priority > 3) { + priority = 3; + } else { + priority = 4; + } + + const assigneeId = row.Assignee && row.Assignee.length > 0 ? row.Assignee : undefined; + + const status = row.Status && (row.Status === "Review" || row.Status === "Codereview") ? "Done" : "Todo"; + + let labels = tags.filter(tag => !!tag); + if (row.Tracker === "Internal Bug") { + labels.push("bug"); + } + if (row.Tracker === "Internal Feature") { + labels.push("feature"); + } + if (!!categories) { + labels = labels.concat(categories.filter(tag => !!tag)); + } + const createdAt = row.Created; + if (comments.length > 0) { + description = description.concat(" \n \n## Original Comments:\n"); + } + // console.log(description); + importData.issues.push({ + title, + description, + status, + url, + extraUrls, + assigneeId, + labels, + createdAt, + priority, + originalId, + relatedOriginalIds, + comments, + }); + + for (const lab of labels) { + if (!importData.labels[lab]) { + importData.labels[lab] = { + name: lab, + }; + } + } + } + + for (const user of users) { + importData.users[user] = { name: user }; + } + + return importData; + }; + + // -- Private interface + + private filePath: string; +} diff --git a/packages/import/src/importers/redmineCsv/index.ts b/packages/import/src/importers/redmineCsv/index.ts new file mode 100644 index 00000000..5b2f0e43 --- /dev/null +++ b/packages/import/src/importers/redmineCsv/index.ts @@ -0,0 +1,24 @@ +import * as inquirer from "inquirer"; +import { Importer } from "../../types"; +import { RedmineCsvImporter } from "./RedmineCsvImporter"; + +const BASE_PATH = process.cwd(); + +export const redmineCsvImport = async (): Promise => { + const answers = await inquirer.prompt(questions); + const redmineImporter = new RedmineCsvImporter(answers.redmineFilePath); + return redmineImporter; +}; + +interface RedmineImportAnswers { + redmineFilePath: string; +} + +const questions = [ + { + basePath: BASE_PATH, + type: "filePath", + name: "redmineFilePath", + message: "Select your exported CSV file of Redmine issues", + }, +]; diff --git a/packages/import/src/types.ts b/packages/import/src/types.ts index 9b62b743..d4348f4f 100644 --- a/packages/import/src/types.ts +++ b/packages/import/src/types.ts @@ -16,6 +16,8 @@ export interface Issue { labels?: string[]; /** Link to original issue. */ url?: string; + /** Link to original issue. */ + extraUrls?: Url[]; /** When the issue was created. */ createdAt?: Date; /** When the issue is due. This is a date string of the format yyyy-MM-dd. */ @@ -24,6 +26,10 @@ export interface Issue { completedAt?: Date; /** When the issue was started. */ startedAt?: Date; + /** Original ID. */ + originalId?: string; + /** List of related original Ids. */ + relatedOriginalIds?: string[]; } /** Issue comment */ @@ -36,6 +42,14 @@ export interface Comment { createdAt?: Date; } +/** Issue comment */ +export interface Url { + /** Comment's body in markdown */ + title?: string; + /** User who posted the comments */ + url: string; +} + export type IssueStatus = "backlog" | "unstarted" | "started" | "completed" | "canceled"; /** Import response. */ diff --git a/yarn.lock b/yarn.lock index c5101a86..d0a8c27f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3688,6 +3688,28 @@ aws4@^1.8.0: resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.12.0.tgz#ce1c9d143389679e253b314241ea9aa5cec980d3" integrity sha512-NmWvPnx0F1SfrQbYwOi7OeaNGokp9XhzNioJ/CSBs8Qa4vxug81mhJEAVZwxXuBmYB5KDRfMq/F3RR0BIU7sWg== +axios-redmine@0.1.17: + version "0.1.17" + resolved "https://registry.yarnpkg.com/axios-redmine/-/axios-redmine-0.1.17.tgz#e904782c546244860370e351ac4693fe7a3a996f" + integrity sha512-xKIQMj8R1ncOfwyaBAks7umMikzq3PlgSG7zkNPUWmMBBw9ibE/YV8MhYNScU23U4qoOqPbEYlM6i+EWHoRy3A== + dependencies: + axios "^0.21.1" + +axios@0.27.2: + version "0.27.2" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.27.2.tgz#207658cc8621606e586c85db4b41a750e756d972" + integrity sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ== + dependencies: + follow-redirects "^1.14.9" + form-data "^4.0.0" + +axios@^0.21.1: + version "0.21.4" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575" + integrity sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg== + dependencies: + follow-redirects "^1.14.0" + babel-jest@^26.6.3: version "26.6.3" resolved "https://registry.yarnpkg.com/babel-jest/-/babel-jest-26.6.3.tgz#d87d25cb0037577a0c89f82e5755c5d293c01056" @@ -6250,6 +6272,11 @@ flush-write-stream@^1.0.0: inherits "^2.0.3" readable-stream "^2.3.6" +follow-redirects@^1.14.0, follow-redirects@^1.14.9: + version "1.15.2" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.2.tgz#b460864144ba63f2681096f274c4e57026da2c13" + integrity sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA== + for-each@^0.3.3: version "0.3.3" resolved "https://registry.yarnpkg.com/for-each/-/for-each-0.3.3.tgz#69b447e88a0a5d32c3e7084f3f1710034b21376e" @@ -6267,7 +6294,7 @@ forever-agent@~0.6.1: resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" integrity sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw== -form-data@4.0.0: +form-data@4.0.0, form-data@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452" integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww== @@ -9695,6 +9722,11 @@ node-notifier@^8.0.0: uuid "^8.3.0" which "^2.0.2" +node-pandoc@0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/node-pandoc/-/node-pandoc-0.3.0.tgz#d46577cd0a72af415353fcb8a0117adb6242f9c8" + integrity sha512-e+kANHQQFBlN7J8wEFf7sNc1sSCLltyPtPqNA/IEapiNsA1LjZ9mtV/B0lU6b2cAdvn7rrAVNBONw+Fo1YsY+A== + node-releases@^2.0.8: version "2.0.10" resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.10.tgz#c311ebae3b6a148c89b1813fd7c4d3c024ef537f"