👩‍💻 Dataset deposit

.gitignore

@@ -34,3 +34,6 @@ yarn-error.log*
 _build
 
 src/version.ts
+
+.yalc
+yalc.lock

src/cli/deposit.ts

@@ -3,6 +3,7 @@ import fs from 'node:fs';
 import { Command, Option } from 'commander';
 import inquirer from 'inquirer';
 import { v4 as uuid } from 'uuid';
+import { load as yamlLoad } from 'js-yaml';
 import {
   Session,
   filterPages,
@@ -17,7 +18,7 @@ import {
 import type { ISession } from 'myst-cli';
 import { clirun } from 'myst-cli-utils';
 import type { GenericParent } from 'myst-common';
-import { extractPart, plural } from 'myst-common';
+import { extractPart, plural, toText } from 'myst-common';
 import { JatsSerializer } from 'myst-to-jats';
 import { VFile } from 'vfile';
 import { u } from 'unist-builder';
@@ -28,12 +29,13 @@ import { preprintFromMyst } from '../preprint.js';
 import { addDoiToConfig, element2JatsUnist, transformXrefToLink } from './utils.js';
 import type { ProjectFrontmatter } from 'myst-frontmatter';
 import { selectNewDois } from './generate.js';
-import type { ConferenceOptions, JournalIssue } from '../types.js';
+import type { ConferenceOptions, DatabaseOptions, JournalIssue } from '../types.js';
 import { curvenoteDoiData } from '../utils.js';
 import { conferencePaperFromMyst, conferenceXml } from '../conference.js';
 import { contributorsXmlFromMystEditors } from '../contributors.js';
+import { databaseXml, datasetFromMyst } from '../dataset.js';
 
-type DepositType = 'conference' | 'journal' | 'preprint';
+type DepositType = 'conference' | 'journal' | 'preprint' | 'dataset';
 
 type DepositOptions = {
   type?: DepositType;
@@ -166,11 +168,19 @@ async function getDepositSources(
     depositFile = resp.depositFile;
     return [{ projectPath, depositFile }];
   }
-  // If there is no project on the current path, load all projects in child folders
+  // If there is no project on the current path, load all projects in child folders (up to two levels deep)
   const subdirs = fs
     .readdirSync('.')
     .map((item) => path.resolve(item))
-    .filter((item) => fs.lstatSync(item).isDirectory());
+    .filter((item) => fs.lstatSync(item).isDirectory())
+    .map((dir) => {
+      const files = fs.readdirSync(dir);
+      if (files.includes('myst.yml') || files.includes('curvenote.yml')) return dir;
+      return files
+        .map((item) => path.join(dir, item))
+        .filter((item) => fs.lstatSync(item).isDirectory());
+    })
+    .flat();
   const depositSources = (
     await Promise.all(
       subdirs.map(async (dir) => {
@@ -371,6 +381,7 @@ export async function deposit(session: ISession, opts: DepositOptions) {
           { name: 'Posted Content / Preprint', value: 'preprint' },
           { name: 'Journal', value: 'journal' },
           { name: 'Conference Proceeding', value: 'conference' },
+          { name: 'Dataset', value: 'dataset' },
         ],
       },
     ]);
@@ -553,6 +564,33 @@ export async function deposit(session: ISession, opts: DepositOptions) {
         return conferencePaperFromMyst(frontmatter, dois, abstract);
       }),
     });
+  } else if (depositType === 'dataset') {
+    if (!journalTitle) {
+      throw new Error(`venue title is required for database / datasets`);
+    }
+    console.log('Deposit summary:');
+    console.log('  Database:');
+    console.log(`    Title: ${journalTitle}${journalAbbr ? ` (${journalAbbr})` : ''}`);
+    const database: DatabaseOptions = {
+      contributors: proceedingsEditors,
+      title: journalTitle,
+    };
+    console.log('  Datasets:');
+    depositArticles.forEach(({ frontmatter }) => {
+      console.log(
+        `    ${frontmatter.doi} - ${frontmatter.title?.slice(0, 30)}${(frontmatter.title?.length ?? 0) > 30 ? '...' : ''}`,
+      );
+    });
+    body = databaseXml({
+      ...database,
+      datasets: depositArticles.map(({ frontmatter, dois, abstract, configFile }) => {
+        // TODO: simplify the abstract to b/u/sup/sub etc.
+        return datasetFromMyst(frontmatter, dois, toText(abstract as any), ({ doi }) => ({
+          doi: `${doi}`,
+          resource: `https://zenodo.org/records/${getZenodoId(configFile)}`,
+        }));
+      }),
+    });
   } else {
     if (depositArticles.length > 1) {
       throw new Error('preprint deposit may only use a single article');
@@ -577,7 +615,7 @@ function makeDepositCLI(program: Command) {
     .addOption(new Option('--file <value>', 'File to deposit'))
     .addOption(
       new Option('--type <value>', 'Deposit type')
-        .choices(['conference', 'journal', 'preprint'])
+        .choices(['conference', 'journal', 'preprint', 'dataset'])
         .default('preprint'),
     )
     .addOption(new Option('--id <value>', 'Deposit batch id'))
@@ -593,3 +631,14 @@ function makeDepositCLI(program: Command) {
 export function addDepositCLI(program: Command) {
   program.addCommand(makeDepositCLI(program));
 }
+
+function getZenodoId(configFile: string | undefined): number | undefined {
+  // This shouldn't be needed in the future
+  if (!configFile) return undefined;
+  const data = yamlLoad(fs.readFileSync(configFile).toString()) as {
+    project: { zenodo?: string };
+  };
+  const url = data?.project?.zenodo;
+  if (!url) return undefined;
+  return Number.parseInt(String(url).split('/').slice(-1)[0], 10);
+}

src/dataset.ts

@@ -0,0 +1,152 @@
+import type { Element } from 'xast';
+import { curvenoteDoiData, e } from './utils.js';
+import type { DatabaseOptions, DatasetMetadata, Titles } from './types.js';
+import type { PageFrontmatter } from 'myst-frontmatter';
+import { normalize } from 'doi-utils';
+import { contributorsXmlFromMystAuthors } from './contributors.js';
+import { dateXml } from './dates.js';
+
+function titleXml(title: Titles) {
+  const titles = typeof title === 'string' ? [e('title', title)] : [e('title', title.title)];
+  if (typeof title !== 'string') {
+    if (title.subtitle) titles.push(e('subtitle', title.subtitle));
+    if (title.original_language_title)
+      titles.push(e('original_language_title', title.original_language_title));
+    if (title.original_language_subtitle)
+      titles.push(e('original_language_subtitle', title.original_language_subtitle));
+  }
+  return e('titles', titles);
+}
+
+/**
+ * Create conference paper xml
+ *
+ * Required fields: titles, doi_data
+ *
+ * Optional fields: contributors, publication_date, acceptance_date, pages,
+ * institution, publisher_item, fr:program, ai:program, ct:program, rel:program,
+ * archive_locations, citation_list, jats:abstract, scn_policies, component_list
+ */
+export function datasetXml({
+  type = 'other',
+  contributors,
+  date,
+  title,
+  description,
+  doi_data,
+  relations,
+  citations,
+}: DatasetMetadata) {
+  if (!title) throw new Error('Missing required frontmatter field: title');
+  if (!doi_data?.doi) throw new Error('Missing required frontmatter field: doi');
+  const children: Element[] = [];
+  if (contributors) children.push(contributors);
+  children.push(titleXml(title));
+  if (date) {
+    const dateChildren: Element[] = [];
+    if (date?.created) dateChildren.push(dateXml('creation_date', date.created) as Element);
+    if (date?.published) dateChildren.push(dateXml('publication_date', date.published) as Element);
+    if (date?.updated) dateChildren.push(dateXml('update_date', date.updated) as Element);
+    children.push(e('database_date', dateChildren));
+  }
+  if (description) children.push(e('description', description));
+  if (relations && relations.length > 0) {
+    children.push(
+      e(
+        'program',
+        { name: 'relations', xmlns: 'http://www.crossref.org/relations.xsd' },
+        relations.map((relation) =>
+          e('related_item', [
+            e(
+              relation.kind === 'inter' ? 'inter_work_relation' : 'intra_work_relation',
+              { 'relationship-type': relation.relationship, 'identifier-type': relation.idType },
+              relation.id,
+            ),
+          ]),
+        ),
+      ),
+    );
+  }
+  const doiChildren = [e('doi', doi_data.doi)];
+  if (doi_data.resource) {
+    doiChildren.push(e('resource', { content_version: 'vor' }, doi_data.resource));
+  }
+  if (doi_data.xml || doi_data.pdf || doi_data.zip) {
+    const collectionChildren = [];
+    if (doi_data.xml) {
+      collectionChildren.push(
+        e('item', [e('resource', { mime_type: 'text/xml', content_version: 'vor' }, doi_data.xml)]),
+      );
+    }
+    if (doi_data.pdf) {
+      collectionChildren.push(
+        e('item', [
+          e('resource', { mime_type: 'application/pdf', content_version: 'vor' }, doi_data.pdf),
+        ]),
+      );
+    }
+    if (doi_data.zip) {
+      collectionChildren.push(
+        e('item', [
+          e('resource', { mime_type: 'application/zip', content_version: 'vor' }, doi_data.zip),
+        ]),
+      );
+    }
+    doiChildren.push(e('collection', { property: 'text-mining' }, collectionChildren));
+  }
+  children.push(e('doi_data', doiChildren));
+  if (citations) {
+    children.push(
+      e(
+        'citation_list',
+        Object.entries(citations).map(([key, value]) => {
+          return e('citation', { key }, [e('doi', normalize(value))]);
+        }),
+      ),
+    );
+  }
+  return e('dataset', { dataset_type: type }, children);
+}
+
+export function datasetFromMyst(
+  myst: PageFrontmatter,
+  citations?: Record<string, string>,
+  abstract?: string | Element[],
+  doiResolution: (myst: PageFrontmatter) => { doi: string; resource: string } | undefined = ({
+    doi,
+  }) => curvenoteDoiData(doi as string),
+) {
+  const { title, subtitle, date, venue } = myst;
+  if (!title) throw new Error('Must have a title');
+  const contributors = contributorsXmlFromMystAuthors(myst);
+  const datasetOpts: DatasetMetadata = {
+    contributors,
+    title: { title, subtitle },
+    date: date ? { created: new Date(date) } : undefined,
+    description: abstract,
+  };
+  datasetOpts.doi_data = doiResolution(myst);
+  if (venue?.doi) {
+    datasetOpts.relations = [
+      { kind: 'inter', relationship: 'isPartOf', id: venue.doi, idType: 'doi' },
+    ];
+  }
+  if (citations && Object.keys(citations).length) {
+    datasetOpts.citations = citations;
+  }
+  return datasetXml(datasetOpts);
+}
+
+export function databaseXml({ contributors, title, description, datasets = [] }: DatabaseOptions) {
+  const children: Element[] = [];
+  if (!title) {
+    throw new Error('Missing conference event name');
+  }
+  const databaseMetadataChildren: Element[] = [];
+  if (contributors) databaseMetadataChildren.push(contributors);
+  databaseMetadataChildren.push(titleXml(title));
+  if (description) children.push(e('description', description));
+  children.push(e('database_metadata', { language: 'en' }, databaseMetadataChildren));
+  children.push(...datasets);
+  return e('database', children);
+}