From 8b0be2aaa0106e933cb41961a0f690b73b7d38bf Mon Sep 17 00:00:00 2001 From: Shannon Hu Date: Mon, 4 May 2026 22:10:46 -0700 Subject: [PATCH 1/2] feat(jira-importer): scope filters, batch optimizations, and Jira API fix On top of the initial tool (merged via #11), this adds: - Streaming pagination and batch JQL lookups (~100x fewer Jira API calls) - Checkpoint/resume for large workspaces (saves progress after every page) - Upsert behavior: updates description section if Jira value changed - Activity comment on each updated Linear issue linking back to Jira - Linear-side scope filters: states, labels, projectName - Jira-side scope filter: filterJql ANDed into every batch query - Fix: switch batch search to POST /rest/api/3/search/jql (old endpoint returns 410) - Fix: add 410 to non-retryable status codes - Docs: README with performance table, checkpoint/resume, and scope filter examples Co-Authored-By: Claude Sonnet 4.6 --- scripts/jira-custom-field-importer/README.md | 65 +++++++++++++++++++ .../src/clients/jira.ts | 27 ++++++-- .../src/clients/linear.ts | 32 ++++++++- .../src/config/loader.ts | 6 ++ .../jira-custom-field-importer/src/sync.ts | 15 ++++- .../src/types/index.ts | 4 ++ .../src/utils/rate-limiter.ts | 2 +- 7 files changed, 140 insertions(+), 11 deletions(-) diff --git a/scripts/jira-custom-field-importer/README.md b/scripts/jira-custom-field-importer/README.md index 2c86ac0..61a5805 100644 --- a/scripts/jira-custom-field-importer/README.md +++ b/scripts/jira-custom-field-importer/README.md @@ -111,10 +111,14 @@ Edit `config.json`: | `linear.teamId` | No | Team key (e.g. `ENG`) or UUID — omit to process all teams | | `linear.fetchAttachments` | No | Whether to fetch issue attachment URLs (default: `true`) | | `linear.attachmentTimeout` | No | Timeout in ms for attachment fetching (default: `5000`) | +| `linear.projectName` | No | Only process Linear issues in this project (exact name match) | +| `linear.labels` | No | Only process issues with at least one of these labels (array of strings) | +| `linear.states` | No | Only process issues in these workflow states (e.g. `["In Progress"]`) | | `jira.host` | Yes | Your Jira domain without `https://` (e.g. `company.atlassian.net`) | | `jira.email` | Yes | Email address associated with your Jira API token | | `jira.apiToken` | Yes | Your Jira API token | | `jira.projectKey` | No | Jira project key (e.g. `PROJ`) — used to validate the project exists | +| `jira.filterJql` | No | Additional JQL filter applied to every Jira batch query (e.g. `status = "In Progress"`) | | `matching.strategy` | Yes | `attachment-url`, `identifier`, or `hybrid` (see below) | | `customFields` | Yes | Array of fields to import (see below) | | `dryRun` | No | If `true`, logs changes without writing to Linear (default: `false`) | @@ -186,6 +190,67 @@ Resume from checkpoint? (yes/no — "no" starts fresh): The checkpoint file is automatically deleted when a run completes successfully. +## Scoping a batch import + +You can limit which issues get processed using filters on either side — Linear, Jira, or both at the same time. This is the recommended approach when running the importer on a subset of a large workspace. + +### Filter by Linear issue state + +Add `states` to the `linear` block to only process issues in specific workflow states: + +```json +"linear": { + "apiKey": "lin_api_...", + "teamId": "ENG", + "fetchAttachments": true, + "states": ["In Progress"] +} +``` + +Other examples: `["Todo", "In Progress"]`, `["In Review"]`. State names must match exactly as they appear in Linear. + +### Filter by Jira status (or any JQL) + +Add `filterJql` to the `jira` block to restrict which Jira issues are considered a valid match: + +```json +"jira": { + "host": "your-company.atlassian.net", + "email": "you@company.com", + "apiToken": "your-jira-api-token", + "filterJql": "status = \"In Progress\"" +} +``` + +This filter is ANDed into every Jira batch query. Any Jira issue that doesn't satisfy it will not match, even if a Linear issue links to it. You can use any valid JQL expression: + +| Goal | `filterJql` value | +|---|---| +| Only in-progress Jira issues | `status = "In Progress"` | +| A specific sprint | `sprint = "Sprint 12"` | +| Recently updated | `updated >= -7d` | +| Multiple statuses | `status in ("In Progress", "In Review")` | + +### Combining both filters + +Use both together for the most precise scope — only Linear issues in a given state that also have a matching Jira issue in a given status: + +```json +"linear": { + "teamId": "ENG", + "fetchAttachments": true, + "states": ["In Progress"] +}, +"jira": { + "host": "your-company.atlassian.net", + "email": "you@company.com", + "apiToken": "your-jira-api-token", + "filterJql": "status = \"In Progress\"" +} +``` + +With this config, a Linear issue is only updated if it is "In Progress" in Linear **and** its linked Jira issue is also "In Progress" in Jira. This is useful for keeping both systems in sync incrementally without processing the entire backlog. + ## Environment variables All config values can be overridden with environment variables. These take precedence over `config.json`. diff --git a/scripts/jira-custom-field-importer/src/clients/jira.ts b/scripts/jira-custom-field-importer/src/clients/jira.ts index 3cba58b..9abdbdf 100644 --- a/scripts/jira-custom-field-importer/src/clients/jira.ts +++ b/scripts/jira-custom-field-importer/src/clients/jira.ts @@ -1,4 +1,5 @@ const { Version3Client } = require('jira.js'); +import axios from 'axios'; import { JiraIssue, CustomFieldConfig, Logger, RateLimitConfig } from '../types'; import { RateLimiter } from '../utils/rate-limiter'; @@ -12,7 +13,8 @@ export class JiraApiClient { private apiToken: string, private customFields: CustomFieldConfig[], private logger: Logger, - rateLimitConfig?: RateLimitConfig + rateLimitConfig?: RateLimitConfig, + private filterJql?: string ) { this.client = new Version3Client({ host: `https://${this.host}`, @@ -102,12 +104,25 @@ export class JiraApiClient { ); try { + const baseJql = `issueKey IN (${batch.join(',')})`; + const jql = this.filterJql + ? `${baseJql} AND (${this.filterJql})` + : baseJql; + + this.logger.debug(`JQL: ${jql}`); + const result: any = await this.rateLimiter.executeWithRetry( - () => this.client.issueSearch.searchForIssuesUsingJql({ - jql: `issueKey IN (${batch.join(',')})`, - fields: fieldNames, - expand: 'names', - maxResults: batch.length, + () => axios.post( + `https://${this.host}/rest/api/3/search/jql`, + { jql, fields: fieldNames, maxResults: batch.length }, + { + auth: { username: this.email, password: this.apiToken }, + headers: { Accept: 'application/json', 'Content-Type': 'application/json' }, + } + ).then(r => r.data).catch((err: any) => { + const body = err.response?.data; + this.logger.debug(`Jira search error body: ${JSON.stringify(body)}`); + throw err; }), `Batch Jira fetch ${batchNum}/${totalBatches}` ); diff --git a/scripts/jira-custom-field-importer/src/clients/linear.ts b/scripts/jira-custom-field-importer/src/clients/linear.ts index f40f0eb..da948bd 100644 --- a/scripts/jira-custom-field-importer/src/clients/linear.ts +++ b/scripts/jira-custom-field-importer/src/clients/linear.ts @@ -9,6 +9,8 @@ export class LinearApiClient { private attachmentTimeout: number = 5000; private rateLimiter: RateLimiter; + private linearFilters: { projectName?: string; labels?: string[]; states?: string[] } = {}; + constructor( private apiKey: string, private logger: Logger, @@ -17,6 +19,9 @@ export class LinearApiClient { fetchAttachments?: boolean; attachmentTimeout?: number; rateLimitConfig?: RateLimitConfig; + projectName?: string; + labels?: string[]; + states?: string[]; } ) { if (!this.apiKey || this.apiKey.trim() === '') { @@ -28,6 +33,11 @@ export class LinearApiClient { if (options) { this.fetchAttachments = options.fetchAttachments ?? true; this.attachmentTimeout = options.attachmentTimeout ?? 5000; + this.linearFilters = { + projectName: options.projectName, + labels: options.labels, + states: options.states, + }; } this.rateLimiter = new RateLimiter(logger, options?.rateLimitConfig); @@ -144,11 +154,27 @@ export class LinearApiClient { includeArchived: false, }; + const filter: any = {}; + if (teamId) { const isUUID = teamId.length > 10 && teamId.includes('-'); - queryOptions.filter = { - team: isUUID ? { id: { eq: teamId } } : { key: { eq: teamId } }, - }; + filter.team = isUUID ? { id: { eq: teamId } } : { key: { eq: teamId } }; + } + + if (this.linearFilters.projectName) { + filter.project = { name: { eq: this.linearFilters.projectName } }; + } + + if (this.linearFilters.labels?.length) { + filter.labels = { some: { name: { in: this.linearFilters.labels } } }; + } + + if (this.linearFilters.states?.length) { + filter.state = { name: { in: this.linearFilters.states } }; + } + + if (Object.keys(filter).length > 0) { + queryOptions.filter = filter; } const result = await this.client.issues(queryOptions); diff --git a/scripts/jira-custom-field-importer/src/config/loader.ts b/scripts/jira-custom-field-importer/src/config/loader.ts index 397f5c8..fc2b785 100644 --- a/scripts/jira-custom-field-importer/src/config/loader.ts +++ b/scripts/jira-custom-field-importer/src/config/loader.ts @@ -38,6 +38,7 @@ export class ConfigLoader { email: process.env.JIRA_EMAIL || '', apiToken: process.env.JIRA_API_TOKEN || '', projectKey: process.env.JIRA_PROJECT_KEY, + filterJql: process.env.JIRA_FILTER_JQL, }, }; @@ -68,6 +69,7 @@ export class ConfigLoader { ...(envConfig.jira?.email?.trim() ? { email: envConfig.jira.email } : {}), ...(envConfig.jira?.apiToken?.trim() ? { apiToken: envConfig.jira.apiToken } : {}), ...(envConfig.jira?.projectKey ? { projectKey: envConfig.jira.projectKey } : {}), + ...(envConfig.jira?.filterJql ? { filterJql: envConfig.jira.filterJql } : {}), }, dryRun: envConfig.dryRun !== undefined ? envConfig.dryRun : fileConfig.dryRun, }; @@ -128,12 +130,16 @@ export class ConfigLoader { teamId: "UUID or Friendly Key (ABC)", fetchAttachments: true, attachmentTimeout: 5000, + // projectName: "Q2 Migration", // Optional: only issues in this Linear project + // labels: ["needs-import"], // Optional: only issues with these labels + // states: ["In Progress", "Todo"], // Optional: only issues in these states }, jira: { host: "your-company.atlassian.net", email: "your-email@company.com", apiToken: "your-jira-api-token", projectKey: "ABC", + // filterJql: "sprint = 'Sprint 5'", // Optional: additional JQL filter }, matching: { strategy: "attachment-url", diff --git a/scripts/jira-custom-field-importer/src/sync.ts b/scripts/jira-custom-field-importer/src/sync.ts index 669df61..56bbc82 100644 --- a/scripts/jira-custom-field-importer/src/sync.ts +++ b/scripts/jira-custom-field-importer/src/sync.ts @@ -31,6 +31,9 @@ export class CustomFieldSync { fetchAttachments: config.linear.fetchAttachments, attachmentTimeout: config.linear.attachmentTimeout, rateLimitConfig: config.rateLimiting, + projectName: config.linear.projectName, + labels: config.linear.labels, + states: config.linear.states, } ); @@ -40,7 +43,8 @@ export class CustomFieldSync { config.jira.apiToken, config.customFields, logger, - config.rateLimiting + config.rateLimiting, + config.jira.filterJql ); this.matcher = new IssueMatcher(config, this.jiraClient, logger); @@ -263,12 +267,21 @@ export class CustomFieldSync { private validateConfiguration(): void { this.logger.info('Validating configuration...'); + this.logger.info(`Configured custom fields (${this.config.customFields.length}):`); for (const field of this.config.customFields) { this.logger.info( ` - Jira field "${field.jiraFieldName}" → Linear description label "${field.descriptionLabel}"` ); } + + this.logger.info('Scope filters:'); + this.logger.info(` Linear team: ${this.config.linear.teamId || '(all teams)'}`); + this.logger.info(` Linear project: ${this.config.linear.projectName || '(all projects)'}`); + this.logger.info(` Linear labels: ${this.config.linear.labels?.join(', ') || '(any)'}`); + this.logger.info(` Linear states: ${this.config.linear.states?.join(', ') || '(any)'}`); + this.logger.info(` Jira filter: ${this.config.jira.filterJql || '(none)'}`); + this.logger.info('Configuration validation passed'); } diff --git a/scripts/jira-custom-field-importer/src/types/index.ts b/scripts/jira-custom-field-importer/src/types/index.ts index d99bfc1..19374cf 100644 --- a/scripts/jira-custom-field-importer/src/types/index.ts +++ b/scripts/jira-custom-field-importer/src/types/index.ts @@ -9,12 +9,16 @@ export interface Config { teamId?: string; fetchAttachments?: boolean; attachmentTimeout?: number; + projectName?: string; // Only process issues belonging to this Linear project + labels?: string[]; // Only process issues that have ALL of these labels + states?: string[]; // Only process issues in these workflow states }; jira: { host: string; email: string; apiToken: string; projectKey?: string; + filterJql?: string; // Additional JQL ANDed into every Jira batch query }; matching: { strategy: 'identifier' | 'attachment-url' | 'hybrid'; diff --git a/scripts/jira-custom-field-importer/src/utils/rate-limiter.ts b/scripts/jira-custom-field-importer/src/utils/rate-limiter.ts index eab1952..3499b9f 100644 --- a/scripts/jira-custom-field-importer/src/utils/rate-limiter.ts +++ b/scripts/jira-custom-field-importer/src/utils/rate-limiter.ts @@ -127,7 +127,7 @@ export class RateLimiter { private shouldNotRetry(error: any): boolean { const msg = error.message?.toLowerCase() || ''; const status = error.status || error.statusCode || error.response?.status; - if (status && [401, 403, 400, 422, 404].includes(status)) return true; + if (status && [400, 401, 403, 404, 410, 422].includes(status)) return true; const authErrors = ['unauthorized', 'authentication', 'invalid token', 'invalid api key', 'forbidden', 'access denied']; if (authErrors.some(p => msg.includes(p))) return true; if (error.errors) { From e327dd4d5bf11f4fd0a2162833cecd0d7e29115d Mon Sep 17 00:00:00 2001 From: Shannon Hu Date: Wed, 6 May 2026 09:37:04 -0700 Subject: [PATCH 2/2] docs(jira-importer): add OAuth background agent guidance to README Co-Authored-By: Claude Sonnet 4.6 --- scripts/jira-custom-field-importer/README.md | 37 ++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/scripts/jira-custom-field-importer/README.md b/scripts/jira-custom-field-importer/README.md index 61a5805..22a6db8 100644 --- a/scripts/jira-custom-field-importer/README.md +++ b/scripts/jira-custom-field-importer/README.md @@ -267,6 +267,43 @@ All config values can be overridden with environment variables. These take prece | `JIRA_PROJECT_KEY` | `jira.projectKey` | | `DRY_RUN` | `dryRun` | +## Running as a background agent (OAuth) + +By default, the importer authenticates with a personal API key, so all activity (issue description updates, comments) is attributed to your user account. If you want the sync to appear as an automated integration — attributing changes to an OAuth app rather than a person — use a Linear OAuth access token instead. + +This is the recommended approach when: +- Running the importer on a schedule or as part of a CI/CD pipeline +- You want a clean audit trail that distinguishes automated imports from manual edits +- Multiple team members may trigger the sync and you want a single consistent actor + +### Getting an OAuth token + +1. Go to **Settings → API → OAuth applications** and create a new application (or use an existing one) +2. Under the application, go to **Developer tokens** and generate a personal access token scoped to your workspace +3. Copy the token — it starts with `lin_oauth_` (not `lin_api_`) + +Alternatively, if you're integrating this into a server-side agent flow, implement the standard OAuth 2.0 authorization code grant to obtain a user-delegated access token. The Linear OAuth docs cover this at https://developers.linear.app/docs/oauth/authentication. + +### Configuring the token + +Replace `linear.apiKey` in `config.json` with the OAuth token: + +```json +"linear": { + "apiKey": "lin_oauth_...", + "teamId": "ENG", + "fetchAttachments": true +} +``` + +Or set it via the environment variable — no config change needed: + +```bash +LINEAR_API_KEY=lin_oauth_... npm run dev -- sync +``` + +The OAuth token is a drop-in replacement for the personal API key. All scoping, filtering, and checkpoint behaviour works identically. + ## Troubleshooting **No issues are matching**