diff --git a/app/actions/definitions/documents.tsx b/app/actions/definitions/documents.tsx
index 46975e985c15..c30ab56058b8 100644
--- a/app/actions/definitions/documents.tsx
+++ b/app/actions/definitions/documents.tsx
@@ -31,7 +31,6 @@ import {
import * as React from "react";
import { toast } from "sonner";
import { ExportContentType, TeamPreference } from "@shared/types";
-import MarkdownHelper from "@shared/utils/MarkdownHelper";
import { getEventFiles } from "@shared/utils/files";
import DocumentDelete from "~/scenes/DocumentDelete";
import DocumentMove from "~/scenes/DocumentMove";
@@ -454,7 +453,7 @@ export const copyDocumentAsMarkdown = createAction({
? stores.documents.get(activeDocumentId)
: undefined;
if (document) {
- copy(MarkdownHelper.toMarkdown(document));
+ copy(document.toMarkdown());
toast.success(t("Markdown copied to clipboard"));
}
},
diff --git a/app/components/Facepile.tsx b/app/components/Facepile.tsx
index bd9097354bf2..0073d194dd8b 100644
--- a/app/components/Facepile.tsx
+++ b/app/components/Facepile.tsx
@@ -5,6 +5,7 @@ import { s } from "@shared/styles";
import User from "~/models/User";
import Avatar from "~/components/Avatar";
import Flex from "~/components/Flex";
+import { AvatarSize } from "./Avatar/Avatar";
type Props = {
users: User[];
@@ -17,7 +18,7 @@ type Props = {
function Facepile({
users,
overflow = 0,
- size = 32,
+ size = AvatarSize.Large,
limit = 8,
renderAvatar = DefaultAvatar,
...rest
@@ -43,7 +44,7 @@ function Facepile({
}
function DefaultAvatar(user: User) {
- return ;
+ return ;
}
const AvatarWrapper = styled.div`
@@ -62,11 +63,11 @@ const More = styled.div<{ size: number }>`
min-width: ${(props) => props.size}px;
height: ${(props) => props.size}px;
border-radius: 100%;
- background: ${(props) => props.theme.slate};
- color: ${s("text")};
+ background: ${(props) => props.theme.textTertiary};
+ color: ${s("white")};
border: 2px solid ${s("background")};
text-align: center;
- font-size: 11px;
+ font-size: 12px;
font-weight: 600;
`;
diff --git a/app/editor/index.tsx b/app/editor/index.tsx
index 15ae8d8457ca..9dc83132e990 100644
--- a/app/editor/index.tsx
+++ b/app/editor/index.tsx
@@ -39,8 +39,8 @@ import { basicExtensions as extensions } from "@shared/editor/nodes";
import Node from "@shared/editor/nodes/Node";
import ReactNode from "@shared/editor/nodes/ReactNode";
import { EventType } from "@shared/editor/types";
-import { UserPreferences } from "@shared/types";
-import ProsemirrorHelper from "@shared/utils/ProsemirrorHelper";
+import { ProsemirrorData, UserPreferences } from "@shared/types";
+import { ProsemirrorHelper } from "@shared/utils/ProsemirrorHelper";
import EventEmitter from "@shared/utils/events";
import Flex from "~/components/Flex";
import { PortalContext } from "~/components/Portal";
@@ -59,7 +59,7 @@ export type Props = {
/** The user id of the current user */
userId?: string;
/** The editor content, should only be changed if you wish to reset the content */
- value?: string;
+ value?: string | ProsemirrorData;
/** The initial editor content as a markdown string or JSON object */
defaultValue: string | object;
/** Placeholder displayed when the editor is empty */
diff --git a/app/models/Document.ts b/app/models/Document.ts
index 048a50a27161..637b0f989a4b 100644
--- a/app/models/Document.ts
+++ b/app/models/Document.ts
@@ -3,12 +3,19 @@ import i18n, { t } from "i18next";
import capitalize from "lodash/capitalize";
import floor from "lodash/floor";
import { action, autorun, computed, observable, set } from "mobx";
+import { Node, Schema } from "prosemirror-model";
+import ExtensionManager from "@shared/editor/lib/ExtensionManager";
+import { richExtensions } from "@shared/editor/nodes";
+import type {
+ JSONObject,
+ NavigationNode,
+ ProsemirrorData,
+} from "@shared/types";
import {
ExportContentType,
FileOperationFormat,
NotificationEventType,
} from "@shared/types";
-import type { JSONObject, NavigationNode } from "@shared/types";
import Storage from "@shared/utils/Storage";
import { isRTL } from "@shared/utils/rtl";
import slugify from "@shared/utils/slugify";
@@ -61,6 +68,9 @@ export default class Document extends ParanoidModel {
@observable
id: string;
+ @observable.shallow
+ data: ProsemirrorData;
+
/**
* The original data source of the document, if imported.
*/
@@ -111,12 +121,6 @@ export default class Document extends ParanoidModel {
@Relation(() => Collection, { onDelete: "cascade" })
collection?: Collection;
- /**
- * The text content of the document as Markdown.
- */
- @observable
- text: string;
-
/**
* The title of the document.
*/
@@ -515,6 +519,17 @@ export default class Document extends ParanoidModel {
recursive?: boolean;
}) => this.store.duplicate(this, options);
+ /**
+ * Returns the first blocks of the document, useful for displaying a preview.
+ *
+ * @param blocks The number of blocks to return, defaults to 4.
+ * @returns A new ProseMirror document.
+ */
+ getSummary = (blocks = 4) => ({
+ ...this.data,
+ content: this.data.content.slice(0, blocks),
+ });
+
@computed
get pinned(): boolean {
return !!this.store.rootStore.pins.orderedData.find(
@@ -535,19 +550,40 @@ export default class Document extends ParanoidModel {
return !this.isDeleted && !this.isTemplate && !this.isArchived;
}
+ @computed
+ get childDocuments() {
+ return this.store.orderedData.filter(
+ (doc) => doc.parentDocumentId === this.id
+ );
+ }
+
@computed
get asNavigationNode(): NavigationNode {
return {
id: this.id,
title: this.title,
- children: this.store.orderedData
- .filter((doc) => doc.parentDocumentId === this.id)
- .map((doc) => doc.asNavigationNode),
+ children: this.childDocuments.map((doc) => doc.asNavigationNode),
url: this.url,
isDraft: this.isDraft,
};
}
+ /**
+ * Returns the markdown representation of the document derived from the ProseMirror data.
+ *
+ * @returns The markdown representation of the document as a string.
+ */
+ toMarkdown = () => {
+ const extensionManager = new ExtensionManager(richExtensions);
+ const serializer = extensionManager.serializer();
+ const schema = new Schema({
+ nodes: extensionManager.nodes,
+ marks: extensionManager.marks,
+ });
+ const markdown = serializer.serialize(Node.fromJSON(schema, this.data));
+ return markdown;
+ };
+
download = (contentType: ExportContentType) =>
client.post(
`/documents.export`,
diff --git a/app/models/Revision.ts b/app/models/Revision.ts
index a2b1954bcce6..1a73a61453df 100644
--- a/app/models/Revision.ts
+++ b/app/models/Revision.ts
@@ -1,4 +1,5 @@
import { computed } from "mobx";
+import { ProsemirrorData } from "@shared/types";
import { isRTL } from "@shared/utils/rtl";
import Document from "./Document";
import User from "./User";
@@ -18,8 +19,8 @@ class Revision extends Model {
/** The document title when the revision was created */
title: string;
- /** Markdown string of the content when revision was created */
- text: string;
+ /** Prosemirror data of the content when revision was created */
+ data: ProsemirrorData;
/** The emoji of the document when the revision was created */
emoji: string | null;
diff --git a/app/scenes/Collection/components/MembershipPreview.tsx b/app/scenes/Collection/components/MembershipPreview.tsx
index 2941f2e12989..e357e9713e7a 100644
--- a/app/scenes/Collection/components/MembershipPreview.tsx
+++ b/app/scenes/Collection/components/MembershipPreview.tsx
@@ -5,6 +5,7 @@ import { useTranslation } from "react-i18next";
import { PAGINATION_SYMBOL } from "~/stores/base/Store";
import Collection from "~/models/Collection";
import Avatar from "~/components/Avatar";
+import { AvatarSize } from "~/components/Avatar/Avatar";
import Facepile from "~/components/Facepile";
import Fade from "~/components/Fade";
import NudeButton from "~/components/NudeButton";
@@ -66,7 +67,7 @@ const MembershipPreview = ({ collection, limit = 8 }: Props) => {
return null;
}
- const overflow = usersCount - groupsCount - collectionUsers.length;
+ const overflow = usersCount + groupsCount - collectionUsers.length;
return (
{
users={sortBy(collectionUsers, "lastActiveAt")}
overflow={overflow}
limit={limit}
- renderAvatar={(user) => }
+ renderAvatar={(item) => (
+
+ )}
/>
diff --git a/app/scenes/Document/Shared.tsx b/app/scenes/Document/Shared.tsx
index 747bc46aa067..f3f431a92124 100644
--- a/app/scenes/Document/Shared.tsx
+++ b/app/scenes/Document/Shared.tsx
@@ -123,10 +123,10 @@ function SharedDocumentScene(props: Props) {
React.useEffect(() => {
async function fetchData() {
try {
- const response = await documents.fetchWithSharedTree(documentSlug, {
+ const res = await documents.fetchWithSharedTree(documentSlug, {
shareId,
});
- setResponse(response);
+ setResponse(res);
} catch (err) {
setError(err);
}
diff --git a/app/scenes/Document/components/CommentThreadItem.tsx b/app/scenes/Document/components/CommentThreadItem.tsx
index 22962a60a12f..ada71ffbc1e3 100644
--- a/app/scenes/Document/components/CommentThreadItem.tsx
+++ b/app/scenes/Document/components/CommentThreadItem.tsx
@@ -8,7 +8,7 @@ import { toast } from "sonner";
import styled, { css } from "styled-components";
import breakpoint from "styled-components-breakpoint";
import { s } from "@shared/styles";
-import { JSONObject } from "@shared/types";
+import { ProsemirrorData } from "@shared/types";
import { dateToRelative } from "@shared/utils/date";
import { Minute } from "@shared/utils/time";
import Comment from "~/models/Comment";
@@ -100,7 +100,7 @@ function CommentThreadItem({
const [isEditing, setEditing, setReadOnly] = useBoolean();
const formRef = React.useRef(null);
- const handleChange = (value: (asString: boolean) => JSONObject) => {
+ const handleChange = (value: (asString: boolean) => ProsemirrorData) => {
setData(value(false));
};
diff --git a/app/scenes/Document/components/DataLoader.tsx b/app/scenes/Document/components/DataLoader.tsx
index ace2b46b0c44..2dd92c917161 100644
--- a/app/scenes/Document/components/DataLoader.tsx
+++ b/app/scenes/Document/components/DataLoader.tsx
@@ -2,6 +2,7 @@ import { observer } from "mobx-react";
import * as React from "react";
import { useLocation, RouteComponentProps, StaticContext } from "react-router";
import { NavigationNode, TeamPreference } from "@shared/types";
+import { ProsemirrorHelper } from "@shared/utils/ProsemirrorHelper";
import { RevisionHelper } from "@shared/utils/RevisionHelper";
import Document from "~/models/Document";
import Revision from "~/models/Revision";
@@ -92,7 +93,7 @@ function DataLoader({ match, children }: Props) {
}
}
void fetchDocument();
- }, [ui, documents, document, shareId, documentSlug]);
+ }, [ui, documents, shareId, documentSlug]);
React.useEffect(() => {
async function fetchRevision() {
@@ -161,7 +162,7 @@ function DataLoader({ match, children }: Props) {
collectionId: document.collectionId,
parentDocumentId: nested ? document.id : document.parentDocumentId,
title,
- text: "",
+ data: ProsemirrorHelper.getEmptyDocument(),
});
return newDocument.url;
diff --git a/app/scenes/Document/components/Document.tsx b/app/scenes/Document/components/Document.tsx
index ce436ec10c0d..abd93bd49311 100644
--- a/app/scenes/Document/components/Document.tsx
+++ b/app/scenes/Document/components/Document.tsx
@@ -1,6 +1,9 @@
+import cloneDeep from "lodash/cloneDeep";
import debounce from "lodash/debounce";
+import isEqual from "lodash/isEqual";
import { action, observable } from "mobx";
import { observer } from "mobx-react";
+import { Node } from "prosemirror-model";
import { AllSelection } from "prosemirror-state";
import * as React from "react";
import { WithTranslation, withTranslation } from "react-i18next";
@@ -16,9 +19,8 @@ import styled from "styled-components";
import breakpoint from "styled-components-breakpoint";
import { s } from "@shared/styles";
import { NavigationNode } from "@shared/types";
-import { Heading } from "@shared/utils/ProsemirrorHelper";
+import { ProsemirrorHelper, Heading } from "@shared/utils/ProsemirrorHelper";
import { parseDomain } from "@shared/utils/domains";
-import getTasks from "@shared/utils/getTasks";
import RootStore from "~/stores/RootStore";
import Document from "~/models/Document";
import Revision from "~/models/Revision";
@@ -34,6 +36,7 @@ import PlaceholderDocument from "~/components/PlaceholderDocument";
import RegisterKeyDown from "~/components/RegisterKeyDown";
import withStores from "~/components/withStores";
import type { Editor as TEditor } from "~/editor";
+import { SearchResult } from "~/editor/components/LinkEditor";
import { client } from "~/utils/ApiClient";
import { replaceTitleVariables } from "~/utils/date";
import { emojiToUrl } from "~/utils/emoji";
@@ -73,13 +76,13 @@ type Props = WithTranslation &
RootStore &
RouteComponentProps & {
sharedTree?: NavigationNode;
- abilities: Record;
+ abilities: Record;
document: Document;
revision?: Revision;
readOnly: boolean;
shareId?: string;
onCreateLink?: (title: string, nested?: boolean) => Promise;
- onSearchLink?: (term: string) => any;
+ onSearchLink?: (term: string) => Promise;
};
@observer
@@ -108,8 +111,6 @@ class DocumentScene extends React.Component {
@observable
headings: Heading[] = [];
- getEditorText: () => string = () => this.props.document.text;
-
componentDidMount() {
this.updateIsDirty();
}
@@ -140,8 +141,8 @@ class DocumentScene extends React.Component {
return;
}
- const { view, parser } = editorRef;
- const doc = parser.parse(template.text);
+ const { view, schema } = editorRef;
+ const doc = Node.fromJSON(schema, template.data);
if (doc) {
view.dispatch(
@@ -168,10 +169,8 @@ class DocumentScene extends React.Component {
if (template.emoji) {
this.props.document.emoji = template.emoji;
}
- if (template.text) {
- this.props.document.text = template.text;
- }
+ this.props.document.data = cloneDeep(template.data);
this.updateIsDirty();
return this.onSave({
@@ -292,15 +291,18 @@ class DocumentScene extends React.Component {
}
// get the latest version of the editor text value
- const text = this.getEditorText ? this.getEditorText() : document.text;
+ const doc = this.editor.current?.view.state.doc;
+ if (!doc) {
+ return;
+ }
// prevent save before anything has been written (single hash is empty doc)
- if (text.trim() === "" && document.title.trim() === "") {
+ if (ProsemirrorHelper.isEmpty(doc) && document.title.trim() === "") {
return;
}
- document.text = text;
- document.tasks = getTasks(document.text);
+ document.data = doc.toJSON();
+ document.tasks = ProsemirrorHelper.getTasksSummary(doc);
// prevent autosave if nothing has changed
if (options.autosave && !this.isEditorDirty && !document.isDirty()) {
@@ -340,12 +342,11 @@ class DocumentScene extends React.Component {
updateIsDirty = () => {
const { document } = this.props;
- const editorText = this.getEditorText().trim();
- this.isEditorDirty = editorText !== document.text.trim();
+ const doc = this.editor.current?.view.state.doc;
+ this.isEditorDirty = !isEqual(doc?.toJSON(), document.data);
// a single hash is a doc with just an empty title
- this.isEmpty =
- (!editorText || editorText === "#" || editorText === "\\") && !this.title;
+ this.isEmpty = (!doc || ProsemirrorHelper.isEmpty(doc)) && !this.title;
};
updateIsDirtyDebounced = debounce(this.updateIsDirty, 500);
@@ -358,9 +359,8 @@ class DocumentScene extends React.Component {
this.isUploading = false;
};
- handleChange = (getEditorText: () => string) => {
+ handleChange = () => {
const { document } = this.props;
- this.getEditorText = getEditorText;
// Keep derived task list in sync
const tasks = this.editor.current?.getTasks();
@@ -503,8 +503,8 @@ class DocumentScene extends React.Component {
isDraft={document.isDraft}
template={document.isTemplate}
document={document}
- value={readOnly ? document.text : undefined}
- defaultValue={document.text}
+ value={readOnly ? document.data : undefined}
+ defaultValue={document.data}
embedsDisabled={embedsDisabled}
onSynced={this.onSynced}
onFileUploadStart={this.onFileUploadStart}
diff --git a/app/scenes/DocumentNew.tsx b/app/scenes/DocumentNew.tsx
index 7fe618cff20d..49bcc1b8e8b3 100644
--- a/app/scenes/DocumentNew.tsx
+++ b/app/scenes/DocumentNew.tsx
@@ -5,6 +5,7 @@ import { useTranslation } from "react-i18next";
import { useHistory, useLocation, useRouteMatch } from "react-router-dom";
import { toast } from "sonner";
import { UserPreference } from "@shared/types";
+import { ProsemirrorHelper } from "@shared/utils/ProsemirrorHelper";
import CenteredContent from "~/components/CenteredContent";
import Flex from "~/components/Flex";
import PlaceholderDocument from "~/components/PlaceholderDocument";
@@ -49,7 +50,7 @@ function DocumentNew({ template }: Props) {
templateId: query.get("templateId") ?? undefined,
template,
title: "",
- text: "",
+ data: ProsemirrorHelper.getEmptyDocument(),
});
history.replace(
template || !user.separateEditMode
diff --git a/app/stores/DocumentsStore.ts b/app/stores/DocumentsStore.ts
index a236fee1e21d..2f83d424bd64 100644
--- a/app/stores/DocumentsStore.ts
+++ b/app/stores/DocumentsStore.ts
@@ -22,6 +22,7 @@ import env from "~/env";
import type {
FetchOptions,
PaginationParams,
+ PartialWithId,
Properties,
SearchResult,
} from "~/types";
@@ -473,6 +474,14 @@ export default class DocumentsStore extends Store {
return this.data.get(res.data.id);
};
+ override fetch = (id: string, options: FetchOptions = {}) =>
+ super.fetch(
+ id,
+ options,
+ (res: { data: { document: PartialWithId } }) =>
+ res.data.document
+ );
+
@action
fetchWithSharedTree = async (
id: string,
@@ -507,7 +516,6 @@ export default class DocumentsStore extends Store {
const res = await client.post("/documents.info", {
id,
shareId: options.shareId,
- apiVersion: 2,
});
invariant(res?.data, "Document not available");
diff --git a/app/stores/RevisionsStore.ts b/app/stores/RevisionsStore.ts
index b859bbe0484b..9e6f07eb211b 100644
--- a/app/stores/RevisionsStore.ts
+++ b/app/stores/RevisionsStore.ts
@@ -35,7 +35,6 @@ export default class RevisionsStore extends Store {
id: "latest",
documentId: document.id,
title: document.title,
- text: document.text,
createdAt: document.updatedAt,
createdBy: document.createdBy,
},
diff --git a/app/stores/base/Store.ts b/app/stores/base/Store.ts
index dcd142e12772..2df6f4759de3 100644
--- a/app/stores/base/Store.ts
+++ b/app/stores/base/Store.ts
@@ -229,7 +229,11 @@ export default abstract class Store {
}
@action
- async fetch(id: string, options: JSONObject = {}): Promise {
+ async fetch(
+ id: string,
+ options: JSONObject = {},
+ accessor = (res: unknown) => (res as { data: PartialWithId }).data
+ ): Promise {
if (!this.actions.includes(RPCAction.Info)) {
throw new Error(`Cannot fetch ${this.modelName}`);
}
@@ -248,7 +252,7 @@ export default abstract class Store {
return runInAction(`info#${this.modelName}`, () => {
invariant(res?.data, "Data should be available");
this.addPolicies(res.policies);
- return this.add(res.data);
+ return this.add(accessor(res));
});
} catch (err) {
if (err instanceof AuthorizationError || err instanceof NotFoundError) {
diff --git a/app/utils/ApiClient.ts b/app/utils/ApiClient.ts
index b8ec84e49510..54640ab950f4 100644
--- a/app/utils/ApiClient.ts
+++ b/app/utils/ApiClient.ts
@@ -84,6 +84,7 @@ class ApiClient {
Accept: "application/json",
"cache-control": "no-cache",
"x-editor-version": EDITOR_VERSION,
+ "x-api-version": "3",
pragma: "no-cache",
...options?.headers,
};
diff --git a/plugins/webhooks/server/tasks/DeliverWebhookTask.ts b/plugins/webhooks/server/tasks/DeliverWebhookTask.ts
index 835f0ccb11ba..c1cad4884941 100644
--- a/plugins/webhooks/server/tasks/DeliverWebhookTask.ts
+++ b/plugins/webhooks/server/tasks/DeliverWebhookTask.ts
@@ -518,7 +518,7 @@ export default class DeliverWebhookTask extends BaseTask {
subscription,
payload: {
id: event.documentId,
- model: model && (await presentDocument(model)),
+ model: model && (await presentDocument(undefined, model)),
},
});
}
@@ -544,7 +544,7 @@ export default class DeliverWebhookTask extends BaseTask {
payload: {
id: event.modelId,
model: model && presentMembership(model),
- document: model && (await presentDocument(model.document!)),
+ document: model && (await presentDocument(undefined, model.document!)),
user: model && presentUser(model.user),
},
});
diff --git a/server/collaboration/PersistenceExtension.ts b/server/collaboration/PersistenceExtension.ts
index 6b847d15a347..64abed67e07e 100644
--- a/server/collaboration/PersistenceExtension.ts
+++ b/server/collaboration/PersistenceExtension.ts
@@ -8,7 +8,7 @@ import * as Y from "yjs";
import Logger from "@server/logging/Logger";
import { trace } from "@server/logging/tracing";
import Document from "@server/models/Document";
-import ProsemirrorHelper from "@server/models/helpers/ProsemirrorHelper";
+import { ProsemirrorHelper } from "@server/models/helpers/ProsemirrorHelper";
import { sequelize } from "@server/storage/database";
import documentCollaborativeUpdater from "../commands/documentCollaborativeUpdater";
import { withContext } from "./types";
@@ -51,11 +51,20 @@ export default class PersistenceExtension implements Extension {
return ydoc;
}
- Logger.info(
- "database",
- `Document ${documentId} is not in state, creating from markdown`
- );
- const ydoc = ProsemirrorHelper.toYDoc(document.text, fieldName);
+ let ydoc;
+ if (document.content) {
+ Logger.info(
+ "database",
+ `Document ${documentId} is not in state, creating from content`
+ );
+ ydoc = ProsemirrorHelper.toYDoc(document.content, fieldName);
+ } else {
+ Logger.info(
+ "database",
+ `Document ${documentId} is not in state, creating from text`
+ );
+ ydoc = ProsemirrorHelper.toYDoc(document.text, fieldName);
+ }
const state = ProsemirrorHelper.toState(ydoc);
await document.update(
{
diff --git a/server/commands/accountProvisioner.ts b/server/commands/accountProvisioner.ts
index e5bb8ef9ea31..06cce25546d4 100644
--- a/server/commands/accountProvisioner.ts
+++ b/server/commands/accountProvisioner.ts
@@ -1,12 +1,23 @@
+import path from "path";
+import { readFile } from "fs-extra";
import invariant from "invariant";
-import { UserRole } from "@shared/types";
+import { CollectionPermission, UserRole } from "@shared/types";
import WelcomeEmail from "@server/emails/templates/WelcomeEmail";
+import env from "@server/env";
import {
InvalidAuthenticationError,
AuthenticationProviderDisabledError,
} from "@server/errors";
import { traceFunction } from "@server/logging/tracing";
-import { AuthenticationProvider, Collection, Team, User } from "@server/models";
+import {
+ AuthenticationProvider,
+ Collection,
+ Document,
+ Team,
+ User,
+} from "@server/models";
+import { DocumentHelper } from "@server/models/helpers/DocumentHelper";
+import { sequelize } from "@server/storage/database";
import teamProvisioner from "./teamProvisioner";
import userProvisioner from "./userProvisioner";
@@ -174,7 +185,7 @@ async function accountProvisioner({
}
if (provision) {
- await team.provisionFirstCollection(user.id);
+ await provisionFirstCollection(team, user);
}
}
@@ -186,6 +197,60 @@ async function accountProvisioner({
};
}
+async function provisionFirstCollection(team: Team, user: User) {
+ await sequelize.transaction(async (transaction) => {
+ const collection = await Collection.create(
+ {
+ name: "Welcome",
+ description: `This collection is a quick guide to what ${env.APP_NAME} is all about. Feel free to delete this collection once your team is up to speed with the basics!`,
+ teamId: team.id,
+ createdById: user.id,
+ sort: Collection.DEFAULT_SORT,
+ permission: CollectionPermission.ReadWrite,
+ },
+ {
+ transaction,
+ }
+ );
+
+ // For the first collection we go ahead and create some intitial documents to get
+ // the team started. You can edit these in /server/onboarding/x.md
+ const onboardingDocs = [
+ "Integrations & API",
+ "Our Editor",
+ "Getting Started",
+ "What is Outline",
+ ];
+
+ for (const title of onboardingDocs) {
+ const text = await readFile(
+ path.join(process.cwd(), "server", "onboarding", `${title}.md`),
+ "utf8"
+ );
+ const document = await Document.create(
+ {
+ version: 2,
+ isWelcome: true,
+ parentDocumentId: null,
+ collectionId: collection.id,
+ teamId: collection.teamId,
+ lastModifiedById: collection.createdById,
+ createdById: collection.createdById,
+ title,
+ text,
+ },
+ { transaction }
+ );
+
+ document.content = await DocumentHelper.toJSON(document);
+
+ await document.publish(collection.createdById, collection.id, {
+ transaction,
+ });
+ }
+ });
+}
+
export default traceFunction({
spanName: "accountProvisioner",
})(accountProvisioner);
diff --git a/server/commands/commentCreator.test.ts b/server/commands/commentCreator.test.ts
index c56d44852830..a4438f166708 100644
--- a/server/commands/commentCreator.test.ts
+++ b/server/commands/commentCreator.test.ts
@@ -21,6 +21,7 @@ describe("commentCreator", () => {
type: "paragraph",
content: [
{
+ content: [],
type: "text",
text: "test",
},
diff --git a/server/commands/commentCreator.ts b/server/commands/commentCreator.ts
index 0837445c7a32..bd810602384b 100644
--- a/server/commands/commentCreator.ts
+++ b/server/commands/commentCreator.ts
@@ -1,4 +1,5 @@
import { Transaction } from "sequelize";
+import { ProsemirrorData } from "@shared/types";
import { Comment, User, Event } from "@server/models";
type Props = {
@@ -6,7 +7,7 @@ type Props = {
/** The user creating the comment */
user: User;
/** The comment as data in Prosemirror schema format */
- data: Record;
+ data: ProsemirrorData;
/** The document to comment within */
documentId: string;
/** The parent comment we're replying to, if any */
diff --git a/server/commands/commentUpdater.ts b/server/commands/commentUpdater.ts
index 8d1a42ae3ccc..cc31e9262379 100644
--- a/server/commands/commentUpdater.ts
+++ b/server/commands/commentUpdater.ts
@@ -1,6 +1,7 @@
import { Transaction } from "sequelize";
+import { ProsemirrorData } from "@shared/types";
import { Event, Comment, User } from "@server/models";
-import ProsemirrorHelper from "@server/models/helpers/ProsemirrorHelper";
+import { ProsemirrorHelper } from "@server/models/helpers/ProsemirrorHelper";
type Props = {
/** The user updating the comment */
@@ -9,8 +10,8 @@ type Props = {
resolvedBy?: User;
/** The existing comment */
comment: Comment;
- /** The index to comment the document at */
- data: Record;
+ /** The comment data */
+ data: ProsemirrorData;
/** The IP address of the user creating the comment */
ip: string;
transaction: Transaction;
diff --git a/server/commands/documentCollaborativeUpdater.ts b/server/commands/documentCollaborativeUpdater.ts
index 31e9b59be281..8f9340fbf8d9 100644
--- a/server/commands/documentCollaborativeUpdater.ts
+++ b/server/commands/documentCollaborativeUpdater.ts
@@ -2,6 +2,7 @@ import { yDocToProsemirrorJSON } from "@getoutline/y-prosemirror";
import uniq from "lodash/uniq";
import { Node } from "prosemirror-model";
import * as Y from "yjs";
+import { ProsemirrorData } from "@shared/types";
import { schema, serializer } from "@server/editor";
import Logger from "@server/logging/Logger";
import { Document, Event } from "@server/models";
@@ -41,7 +42,7 @@ export default async function documentCollaborativeUpdater({
});
const state = Y.encodeStateAsUpdate(ydoc);
- const content = yDocToProsemirrorJSON(ydoc, "default");
+ const content = yDocToProsemirrorJSON(ydoc, "default") as ProsemirrorData;
const node = Node.fromJSON(schema, content);
const text = serializer.serialize(node, undefined);
const isUnchanged = document.text === text;
diff --git a/server/commands/documentCreator.ts b/server/commands/documentCreator.ts
index fd9e0eeb003e..eb286ec167d2 100644
--- a/server/commands/documentCreator.ts
+++ b/server/commands/documentCreator.ts
@@ -1,7 +1,7 @@
import { Transaction } from "sequelize";
import { Optional } from "utility-types";
import { Document, Event, User } from "@server/models";
-import TextHelper from "@server/models/helpers/TextHelper";
+import { TextHelper } from "@server/models/helpers/TextHelper";
type Props = Optional<
Pick<
@@ -10,6 +10,7 @@ type Props = Optional<
| "urlId"
| "title"
| "text"
+ | "content"
| "emoji"
| "collectionId"
| "parentDocumentId"
diff --git a/server/commands/documentDuplicator.ts b/server/commands/documentDuplicator.ts
index 50a6582a60df..eb094bfd7874 100644
--- a/server/commands/documentDuplicator.ts
+++ b/server/commands/documentDuplicator.ts
@@ -48,6 +48,7 @@ export default async function documentDuplicator({
emoji: document.emoji,
template: document.template,
title: title ?? document.title,
+ content: document.content,
text: document.text,
...sharedProperties,
});
diff --git a/server/commands/documentImporter.ts b/server/commands/documentImporter.ts
index 9b5db20dfce9..a17842c62002 100644
--- a/server/commands/documentImporter.ts
+++ b/server/commands/documentImporter.ts
@@ -6,8 +6,8 @@ import parseTitle from "@shared/utils/parseTitle";
import { DocumentValidation } from "@shared/validations";
import { traceFunction } from "@server/logging/tracing";
import { User } from "@server/models";
-import ProsemirrorHelper from "@server/models/helpers/ProsemirrorHelper";
-import TextHelper from "@server/models/helpers/TextHelper";
+import { ProsemirrorHelper } from "@server/models/helpers/ProsemirrorHelper";
+import { TextHelper } from "@server/models/helpers/TextHelper";
import { DocumentConverter } from "@server/utils/DocumentConverter";
import { InvalidRequestError } from "../errors";
diff --git a/server/commands/documentPermanentDeleter.ts b/server/commands/documentPermanentDeleter.ts
index ec02a9d22662..ac583a3f316b 100644
--- a/server/commands/documentPermanentDeleter.ts
+++ b/server/commands/documentPermanentDeleter.ts
@@ -2,9 +2,10 @@ import uniq from "lodash/uniq";
import { Op, QueryTypes } from "sequelize";
import Logger from "@server/logging/Logger";
import { Document, Attachment } from "@server/models";
+import { DocumentHelper } from "@server/models/helpers/DocumentHelper";
+import { ProsemirrorHelper } from "@server/models/helpers/ProsemirrorHelper";
import DeleteAttachmentTask from "@server/queues/tasks/DeleteAttachmentTask";
import { sequelize } from "@server/storage/database";
-import parseAttachmentIds from "@server/utils/parseAttachmentIds";
export default async function documentPermanentDeleter(documents: Document[]) {
const activeDocument = documents.find((doc) => !doc.deletedAt);
@@ -25,7 +26,9 @@ export default async function documentPermanentDeleter(documents: Document[]) {
for (const document of documents) {
// Find any attachments that are referenced in the text content
- const attachmentIdsInText = parseAttachmentIds(document.text);
+ const attachmentIdsInText = ProsemirrorHelper.parseAttachmentIds(
+ DocumentHelper.toProsemirror(document)
+ );
// Find any attachments that were originally uploaded to this document
const attachmentIdsForDocument = (
diff --git a/server/commands/documentUpdater.ts b/server/commands/documentUpdater.ts
index 6baea5c7a5a4..fdfeb9c0cc67 100644
--- a/server/commands/documentUpdater.ts
+++ b/server/commands/documentUpdater.ts
@@ -1,6 +1,6 @@
import { Transaction } from "sequelize";
import { Event, Document, User } from "@server/models";
-import DocumentHelper from "@server/models/helpers/DocumentHelper";
+import { DocumentHelper } from "@server/models/helpers/DocumentHelper";
type Props = {
/** The user updating the document */
diff --git a/server/emails/templates/CommentCreatedEmail.tsx b/server/emails/templates/CommentCreatedEmail.tsx
index c8f432ef1323..a421b84d8677 100644
--- a/server/emails/templates/CommentCreatedEmail.tsx
+++ b/server/emails/templates/CommentCreatedEmail.tsx
@@ -4,8 +4,8 @@ import { Day } from "@shared/utils/time";
import { Collection, Comment, Document } from "@server/models";
import HTMLHelper from "@server/models/helpers/HTMLHelper";
import NotificationSettingsHelper from "@server/models/helpers/NotificationSettingsHelper";
-import ProsemirrorHelper from "@server/models/helpers/ProsemirrorHelper";
-import TextHelper from "@server/models/helpers/TextHelper";
+import { ProsemirrorHelper } from "@server/models/helpers/ProsemirrorHelper";
+import { TextHelper } from "@server/models/helpers/TextHelper";
import BaseEmail, { EmailProps } from "./BaseEmail";
import Body from "./components/Body";
import Button from "./components/Button";
diff --git a/server/emails/templates/CommentMentionedEmail.tsx b/server/emails/templates/CommentMentionedEmail.tsx
index 77329c75b264..ca5d379d1f29 100644
--- a/server/emails/templates/CommentMentionedEmail.tsx
+++ b/server/emails/templates/CommentMentionedEmail.tsx
@@ -4,8 +4,8 @@ import { Day } from "@shared/utils/time";
import { Collection, Comment, Document } from "@server/models";
import HTMLHelper from "@server/models/helpers/HTMLHelper";
import NotificationSettingsHelper from "@server/models/helpers/NotificationSettingsHelper";
-import ProsemirrorHelper from "@server/models/helpers/ProsemirrorHelper";
-import TextHelper from "@server/models/helpers/TextHelper";
+import { ProsemirrorHelper } from "@server/models/helpers/ProsemirrorHelper";
+import { TextHelper } from "@server/models/helpers/TextHelper";
import BaseEmail, { EmailProps } from "./BaseEmail";
import Body from "./components/Body";
import Button from "./components/Button";
diff --git a/server/emails/templates/DocumentPublishedOrUpdatedEmail.tsx b/server/emails/templates/DocumentPublishedOrUpdatedEmail.tsx
index 654897b06a69..042cf157aabf 100644
--- a/server/emails/templates/DocumentPublishedOrUpdatedEmail.tsx
+++ b/server/emails/templates/DocumentPublishedOrUpdatedEmail.tsx
@@ -2,7 +2,7 @@ import * as React from "react";
import { NotificationEventType } from "@shared/types";
import { Day } from "@shared/utils/time";
import { Document, Collection, Revision } from "@server/models";
-import DocumentHelper from "@server/models/helpers/DocumentHelper";
+import { DocumentHelper } from "@server/models/helpers/DocumentHelper";
import HTMLHelper from "@server/models/helpers/HTMLHelper";
import NotificationSettingsHelper from "@server/models/helpers/NotificationSettingsHelper";
import SubscriptionHelper from "@server/models/helpers/SubscriptionHelper";
diff --git a/server/models/Comment.ts b/server/models/Comment.ts
index 4dd5e85c9dd8..7857d1eb64e0 100644
--- a/server/models/Comment.ts
+++ b/server/models/Comment.ts
@@ -10,7 +10,7 @@ import {
DefaultScope,
} from "sequelize-typescript";
import type { ProsemirrorData } from "@shared/types";
-import ProsemirrorHelper from "@shared/utils/ProsemirrorHelper";
+import { ProsemirrorHelper } from "@shared/utils/ProsemirrorHelper";
import { CommentValidation } from "@shared/validations";
import { schema } from "@server/editor";
import Document from "./Document";
diff --git a/server/models/Document.test.ts b/server/models/Document.test.ts
index b6842accec28..8e43d07048db 100644
--- a/server/models/Document.test.ts
+++ b/server/models/Document.test.ts
@@ -1,5 +1,6 @@
import { EmptyResultError } from "sequelize";
import slugify from "@shared/utils/slugify";
+import { parser } from "@server/editor";
import Document from "@server/models/Document";
import {
buildDocument,
@@ -208,20 +209,7 @@ describe("#findByPk", () => {
});
describe("tasks", () => {
- test("should consider all the possible checkTtems", async () => {
- const document = await buildDocument({
- text: `- [x] test
- - [X] test
- - [ ] test
- - [-] test
- - [_] test`,
- });
- const tasks = document.tasks;
- expect(tasks.completed).toBe(4);
- expect(tasks.total).toBe(5);
- });
-
- test("should return tasks keys set to 0 if checkItems isn't present", async () => {
+ test("should return tasks keys set to 0 if check items isn't present", async () => {
const document = await buildDocument({
text: `text`,
});
@@ -230,11 +218,12 @@ describe("tasks", () => {
expect(tasks.total).toBe(0);
});
- test("should return tasks keys set to 0 if the text contains broken checkItems", async () => {
+ test("should return tasks keys set to 0 if the text contains broken check items", async () => {
const document = await buildDocument({
- text: `- [x ] test
- - [ x ] test
- - [ ] test`,
+ text: `
+- [x ] test
+- [ x ] test
+- [ ] test`,
});
const tasks = document.tasks;
expect(tasks.completed).toBe(0);
@@ -243,8 +232,9 @@ describe("tasks", () => {
test("should return tasks", async () => {
const document = await buildDocument({
- text: `- [x] list item
- - [ ] list item`,
+ text: `
+- [x] list item
+- [ ] list item`,
});
const tasks = document.tasks;
expect(tasks.completed).toBe(1);
@@ -253,15 +243,21 @@ describe("tasks", () => {
test("should update tasks on save", async () => {
const document = await buildDocument({
- text: `- [x] list item
- - [ ] list item`,
+ text: `
+- [x] list item
+- [ ] list item`,
});
const tasks = document.tasks;
expect(tasks.completed).toBe(1);
expect(tasks.total).toBe(2);
- document.text = `- [x] list item
- - [ ] list item
- - [ ] list item`;
+ document.content = parser
+ .parse(
+ `
+- [x] list item
+- [ ] list item
+- [ ] list item`
+ )
+ ?.toJSON();
await document.save();
const newTasks = document.tasks;
expect(newTasks.completed).toBe(1);
diff --git a/server/models/Document.ts b/server/models/Document.ts
index 931e600c4a81..2fe19fcde84a 100644
--- a/server/models/Document.ts
+++ b/server/models/Document.ts
@@ -47,8 +47,8 @@ import type {
ProsemirrorData,
SourceMetadata,
} from "@shared/types";
+import { ProsemirrorHelper } from "@shared/utils/ProsemirrorHelper";
import { UrlHelper } from "@shared/utils/UrlHelper";
-import getTasks from "@shared/utils/getTasks";
import slugify from "@shared/utils/slugify";
import { DocumentValidation } from "@shared/validations";
import { ValidationError } from "@server/errors";
@@ -63,7 +63,7 @@ import UserMembership from "./UserMembership";
import View from "./View";
import ParanoidModel from "./base/ParanoidModel";
import Fix from "./decorators/Fix";
-import DocumentHelper from "./helpers/DocumentHelper";
+import { DocumentHelper } from "./helpers/DocumentHelper";
import Length from "./validators/Length";
export const DOCUMENT_VERSION = 2;
@@ -75,9 +75,6 @@ type AdditionalFindOptions = {
};
@DefaultScope(() => ({
- attributes: {
- exclude: ["state"],
- },
include: [
{
model: User,
@@ -337,7 +334,9 @@ class Document extends ParanoidModel<
}
get tasks() {
- return getTasks(this.text || "");
+ return ProsemirrorHelper.getTasksSummary(
+ DocumentHelper.toProsemirror(this)
+ );
}
// hooks
@@ -411,7 +410,7 @@ class Document extends ParanoidModel<
}
@BeforeUpdate
- static processUpdate(model: Document) {
+ static async processUpdate(model: Document) {
// ensure documents have a title
model.title = model.title || "";
@@ -431,7 +430,7 @@ class Document extends ParanoidModel<
// backfill content if it's missing
if (!model.content) {
- model.content = DocumentHelper.toJSON(model);
+ model.content = await DocumentHelper.toJSON(model);
}
// ensure the last modifying user is a collaborator
@@ -608,7 +607,6 @@ class Document extends ParanoidModel<
// allow default preloading of collection membership if `userId` is passed in find options
// almost every endpoint needs the collection membership to determine policy permissions.
const scope = this.scope([
- ...(includeState ? [] : ["withoutState"]),
"withDrafts",
{
method: ["withCollectionPermissions", userId, rest.paranoid],
diff --git a/server/models/Team.ts b/server/models/Team.ts
index 04659f6466ce..841b53a4e378 100644
--- a/server/models/Team.ts
+++ b/server/models/Team.ts
@@ -1,8 +1,5 @@
import crypto from "crypto";
-import fs from "fs";
-import path from "path";
import { URL } from "url";
-import util from "util";
import { subMinutes } from "date-fns";
import {
InferAttributes,
@@ -30,12 +27,7 @@ import {
BeforeCreate,
} from "sequelize-typescript";
import { TeamPreferenceDefaults } from "@shared/constants";
-import {
- CollectionPermission,
- TeamPreference,
- TeamPreferences,
- UserRole,
-} from "@shared/types";
+import { TeamPreference, TeamPreferences, UserRole } from "@shared/types";
import { getBaseDomain, RESERVED_SUBDOMAINS } from "@shared/utils/domains";
import env from "@server/env";
import { ValidationError } from "@server/errors";
@@ -55,8 +47,6 @@ import IsUrlOrRelativePath from "./validators/IsUrlOrRelativePath";
import Length from "./validators/Length";
import NotContainsUrl from "./validators/NotContainsUrl";
-const readFile = util.promisify(fs.readFile);
-
@Scopes(() => ({
withDomains: {
include: [{ model: TeamDomain }],
@@ -279,57 +269,6 @@ class Team extends ParanoidModel<
});
};
- provisionFirstCollection = async (userId: string) => {
- await this.sequelize!.transaction(async (transaction) => {
- const collection = await Collection.create(
- {
- name: "Welcome",
- description: `This collection is a quick guide to what ${env.APP_NAME} is all about. Feel free to delete this collection once your team is up to speed with the basics!`,
- teamId: this.id,
- createdById: userId,
- sort: Collection.DEFAULT_SORT,
- permission: CollectionPermission.ReadWrite,
- },
- {
- transaction,
- }
- );
-
- // For the first collection we go ahead and create some intitial documents to get
- // the team started. You can edit these in /server/onboarding/x.md
- const onboardingDocs = [
- "Integrations & API",
- "Our Editor",
- "Getting Started",
- "What is Outline",
- ];
-
- for (const title of onboardingDocs) {
- const text = await readFile(
- path.join(process.cwd(), "server", "onboarding", `${title}.md`),
- "utf8"
- );
- const document = await Document.create(
- {
- version: 2,
- isWelcome: true,
- parentDocumentId: null,
- collectionId: collection.id,
- teamId: collection.teamId,
- lastModifiedById: collection.createdById,
- createdById: collection.createdById,
- title,
- text,
- },
- { transaction }
- );
- await document.publish(collection.createdById, collection.id, {
- transaction,
- });
- }
- });
- };
-
public collectionIds = async function (paranoid = true) {
const models = await Collection.findAll({
attributes: ["id"],
diff --git a/server/models/helpers/DocumentHelper.test.ts b/server/models/helpers/DocumentHelper.test.ts
index 3a4dc64cdd16..bf2a5471532a 100644
--- a/server/models/helpers/DocumentHelper.test.ts
+++ b/server/models/helpers/DocumentHelper.test.ts
@@ -1,6 +1,6 @@
import Revision from "@server/models/Revision";
import { buildDocument } from "@server/test/factories";
-import DocumentHelper from "./DocumentHelper";
+import { DocumentHelper } from "./DocumentHelper";
describe("DocumentHelper", () => {
beforeAll(() => {
diff --git a/server/models/helpers/DocumentHelper.tsx b/server/models/helpers/DocumentHelper.tsx
index 21153f862c22..212ad46f27c3 100644
--- a/server/models/helpers/DocumentHelper.tsx
+++ b/server/models/helpers/DocumentHelper.tsx
@@ -7,14 +7,14 @@ import { JSDOM } from "jsdom";
import { Node } from "prosemirror-model";
import * as Y from "yjs";
import textBetween from "@shared/editor/lib/textBetween";
-import MarkdownHelper from "@shared/utils/MarkdownHelper";
-import { parser, schema } from "@server/editor";
+import { ProsemirrorData } from "@shared/types";
+import { parser, serializer, schema } from "@server/editor";
import { addTags } from "@server/logging/tracer";
import { trace } from "@server/logging/tracing";
import { Document, Revision } from "@server/models";
import diff from "@server/utils/diff";
-import ProsemirrorHelper from "./ProsemirrorHelper";
-import TextHelper from "./TextHelper";
+import { ProsemirrorHelper } from "./ProsemirrorHelper";
+import { TextHelper } from "./TextHelper";
type HTMLOptions = {
/** Whether to include the document title in the generated HTML (defaults to true) */
@@ -35,38 +35,73 @@ type HTMLOptions = {
};
@trace()
-export default class DocumentHelper {
+export class DocumentHelper {
/**
- * Returns the document as JSON content. This method uses the collaborative state if available,
- * otherwise it falls back to Markdown.
+ * Returns the document as a Prosemirror Node. This method uses the derived content if available
+ * then the collaborative state, otherwise it falls back to Markdown.
*
* @param document The document or revision to convert
- * @returns The document content as JSON
+ * @returns The document content as a Prosemirror Node
*/
- static toJSON(document: Document | Revision) {
+ static toProsemirror(document: Document | Revision) {
+ if ("content" in document && document.content) {
+ return Node.fromJSON(schema, document.content);
+ }
if ("state" in document && document.state) {
const ydoc = new Y.Doc();
Y.applyUpdate(ydoc, document.state);
- return yDocToProsemirrorJSON(ydoc, "default");
+ return Node.fromJSON(schema, yDocToProsemirrorJSON(ydoc, "default"));
}
- const node = parser.parse(document.text) || Node.fromJSON(schema, {});
- return node.toJSON();
+ return parser.parse(document.text) || Node.fromJSON(schema, {});
}
/**
- * Returns the document as a Prosemirror Node. This method uses the collaborative state if
- * available, otherwise it falls back to Markdown.
+ * Returns the document as a plain JSON object. This method uses the derived content if available
+ * then the collaborative state, otherwise it falls back to Markdown.
*
* @param document The document or revision to convert
- * @returns The document content as a Prosemirror Node
+ * @param options Options for the conversion
+ * @returns The document content as a plain JSON object
*/
- static toProsemirror(document: Document | Revision) {
- if ("state" in document && document.state) {
+ static async toJSON(
+ document: Document | Revision,
+ options?: {
+ /** The team context */
+ teamId: string;
+ /** Whether to sign attachment urls, and if so for how many seconds is the signature valid */
+ signedUrls: number;
+ /** Marks to remove from the document */
+ removeMarks?: string[];
+ }
+ ): Promise {
+ let doc: Node | null;
+ let json;
+
+ if ("content" in document && document.content) {
+ doc = Node.fromJSON(schema, document.content);
+ } else if ("state" in document && document.state) {
const ydoc = new Y.Doc();
Y.applyUpdate(ydoc, document.state);
- return Node.fromJSON(schema, yDocToProsemirrorJSON(ydoc, "default"));
+ doc = Node.fromJSON(schema, yDocToProsemirrorJSON(ydoc, "default"));
+ } else {
+ doc = parser.parse(document.text);
}
- return parser.parse(document.text) || Node.fromJSON(schema, {});
+
+ if (doc && options?.signedUrls) {
+ json = await ProsemirrorHelper.signAttachmentUrls(
+ doc,
+ options.teamId,
+ options.signedUrls
+ );
+ } else {
+ json = doc?.toJSON() ?? {};
+ }
+
+ if (options?.removeMarks) {
+ json = ProsemirrorHelper.removeMarks(json, options.removeMarks);
+ }
+
+ return json;
}
/**
@@ -88,19 +123,30 @@ export default class DocumentHelper {
}
/**
- * Returns the document as Markdown. This is a lossy conversion and should
- * only be used for export.
+ * Returns the document as Markdown. This is a lossy conversion and should nly be used for export.
*
* @param document The document or revision to convert
* @returns The document title and content as a Markdown string
*/
static toMarkdown(document: Document | Revision) {
- return MarkdownHelper.toMarkdown(document);
+ const text = serializer
+ .serialize(DocumentHelper.toProsemirror(document))
+ .replace(/\n\\(\n|$)/g, "\n\n")
+ .replace(/“/g, '"')
+ .replace(/”/g, '"')
+ .replace(/‘/g, "'")
+ .replace(/’/g, "'")
+ .trim();
+
+ const title = `${document.emoji ? document.emoji + " " : ""}${
+ document.title
+ }`;
+
+ return `# ${title}\n\n${text}`;
}
/**
- * Returns the document as plain HTML. This is a lossy conversion and should
- * only be used for export.
+ * Returns the document as plain HTML. This is a lossy conversion and should only be used for export.
*
* @param document The document or revision to convert
* @param options Options for the HTML output
diff --git a/server/models/helpers/ProsemirrorHelper.tsx b/server/models/helpers/ProsemirrorHelper.tsx
index e1f3895a7664..2a8dd24ae8f1 100644
--- a/server/models/helpers/ProsemirrorHelper.tsx
+++ b/server/models/helpers/ProsemirrorHelper.tsx
@@ -1,5 +1,8 @@
import { prosemirrorToYDoc } from "@getoutline/y-prosemirror";
import { JSDOM } from "jsdom";
+import compact from "lodash/compact";
+import flatten from "lodash/flatten";
+import uniq from "lodash/uniq";
import { Node, DOMSerializer, Fragment, Mark } from "prosemirror-model";
import * as React from "react";
import { renderToString } from "react-dom/server";
@@ -9,10 +12,14 @@ import EditorContainer from "@shared/editor/components/Styles";
import embeds from "@shared/editor/embeds";
import GlobalStyles from "@shared/styles/globals";
import light from "@shared/styles/theme";
+import { ProsemirrorData } from "@shared/types";
+import { attachmentRedirectRegex } from "@shared/utils/ProsemirrorHelper";
import { isRTL } from "@shared/utils/rtl";
import { schema, parser } from "@server/editor";
import Logger from "@server/logging/Logger";
import { trace } from "@server/logging/tracing";
+import Attachment from "@server/models/Attachment";
+import FileStorage from "@server/storage/files";
export type HTMLOptions = {
/** A title, if it should be included */
@@ -36,15 +43,22 @@ type MentionAttrs = {
};
@trace()
-export default class ProsemirrorHelper {
+export class ProsemirrorHelper {
/**
* Returns the input text as a Y.Doc.
*
* @param markdown The text to parse
* @returns The content as a Y.Doc.
*/
- static toYDoc(markdown: string, fieldName = "default"): Y.Doc {
- let node = parser.parse(markdown);
+ static toYDoc(input: string | ProsemirrorData, fieldName = "default"): Y.Doc {
+ if (typeof input === "object") {
+ return prosemirrorToYDoc(
+ ProsemirrorHelper.toProsemirror(input),
+ fieldName
+ );
+ }
+
+ let node = parser.parse(input);
// in the editor embeds are created at runtime by converting links into
// embeds where they match.Because we're converting to a CRDT structure on
@@ -106,7 +120,7 @@ export default class ProsemirrorHelper {
* @param data The object to parse
* @returns The content as a Prosemirror Node
*/
- static toProsemirror(data: Record) {
+ static toProsemirror(data: ProsemirrorData) {
return Node.fromJSON(schema, data);
}
@@ -116,10 +130,10 @@ export default class ProsemirrorHelper {
* @param node The node to parse mentions from
* @returns An array of mention attributes
*/
- static parseMentions(node: Node) {
+ static parseMentions(doc: Node) {
const mentions: MentionAttrs[] = [];
- node.descendants((node: Node) => {
+ doc.descendants((node: Node) => {
if (
node.type.name === "mention" &&
!mentions.some((m) => m.id === node.attrs.id)
@@ -138,6 +152,117 @@ export default class ProsemirrorHelper {
return mentions;
}
+ /**
+ * Removes all marks from the node that match the given types.
+ *
+ * @param data The ProsemirrorData object to remove marks from
+ * @param marks The mark types to remove
+ * @returns The content with marks removed
+ */
+ static removeMarks(data: ProsemirrorData, marks: string[]) {
+ function removeMarksInner(node: ProsemirrorData) {
+ if (node.marks) {
+ node.marks = node.marks.filter((mark) => !marks.includes(mark.type));
+ }
+ if (node.content) {
+ node.content.forEach(removeMarksInner);
+ }
+ return node;
+ }
+ return removeMarksInner(data);
+ }
+
+ /**
+ * Returns the document as a plain JSON object with attachment URLs signed.
+ *
+ * @param node The node to convert to JSON
+ * @param teamId The team ID to use for signing
+ * @param expiresIn The number of seconds until the signed URL expires
+ * @returns The content as a JSON object
+ */
+ static async signAttachmentUrls(doc: Node, teamId: string, expiresIn = 60) {
+ const attachmentIds = ProsemirrorHelper.parseAttachmentIds(doc);
+ const attachments = await Attachment.findAll({
+ where: {
+ id: attachmentIds,
+ teamId,
+ },
+ });
+
+ const mapping: Record = {};
+
+ await Promise.all(
+ attachments.map(async (attachment) => {
+ const signedUrl = await FileStorage.getSignedUrl(
+ attachment.key,
+ expiresIn
+ );
+ mapping[attachment.redirectUrl] = signedUrl;
+ })
+ );
+
+ const json = doc.toJSON() as ProsemirrorData;
+
+ function replaceAttachmentUrls(node: ProsemirrorData) {
+ if (node.attrs?.src) {
+ node.attrs.src = mapping[node.attrs.src as string] || node.attrs.src;
+ } else if (node.attrs?.href) {
+ node.attrs.href = mapping[node.attrs.href as string] || node.attrs.href;
+ } else if (node.marks) {
+ node.marks.forEach((mark) => {
+ if (mark.attrs?.href) {
+ mark.attrs.href =
+ mapping[mark.attrs.href as string] || mark.attrs.href;
+ }
+ });
+ }
+
+ if (node.content) {
+ node.content.forEach(replaceAttachmentUrls);
+ }
+
+ return node;
+ }
+
+ return replaceAttachmentUrls(json);
+ }
+
+ /**
+ * Returns an array of attachment IDs in the node.
+ *
+ * @param node The node to parse attachments from
+ * @returns An array of attachment IDs
+ */
+ static parseAttachmentIds(doc: Node) {
+ const urls: string[] = [];
+
+ doc.descendants((node) => {
+ node.marks.forEach((mark) => {
+ if (mark.type.name === "link") {
+ urls.push(mark.attrs.href);
+ }
+ });
+ if (["image", "video"].includes(node.type.name)) {
+ urls.push(node.attrs.src);
+ }
+ if (node.type.name === "attachment") {
+ urls.push(node.attrs.href);
+ }
+ });
+
+ return uniq(
+ compact(
+ flatten(
+ urls.map((url) =>
+ [...url.matchAll(attachmentRedirectRegex)].map(
+ (match) => match.groups?.id
+ )
+ )
+ )
+ )
+ );
+ }
+
/**
* Returns the node as HTML. This is a lossy conversion and should only be used
* for export.
diff --git a/server/models/helpers/SearchHelper.ts b/server/models/helpers/SearchHelper.ts
index ce347350c5b8..796c756107b4 100644
--- a/server/models/helpers/SearchHelper.ts
+++ b/server/models/helpers/SearchHelper.ts
@@ -12,7 +12,7 @@ import Share from "@server/models/Share";
import Team from "@server/models/Team";
import User from "@server/models/User";
import { sequelize } from "@server/storage/database";
-import DocumentHelper from "./DocumentHelper";
+import { DocumentHelper } from "./DocumentHelper";
type SearchResponse = {
results: {
diff --git a/server/models/helpers/TextHelper.test.ts b/server/models/helpers/TextHelper.test.ts
index f36acd887eb3..601b93bab386 100644
--- a/server/models/helpers/TextHelper.test.ts
+++ b/server/models/helpers/TextHelper.test.ts
@@ -1,5 +1,5 @@
import { buildUser } from "@server/test/factories";
-import TextHelper from "./TextHelper";
+import { TextHelper } from "./TextHelper";
describe("TextHelper", () => {
beforeAll(() => {
diff --git a/server/models/helpers/TextHelper.ts b/server/models/helpers/TextHelper.ts
index 5fa0b9a8c598..463c23625d1e 100644
--- a/server/models/helpers/TextHelper.ts
+++ b/server/models/helpers/TextHelper.ts
@@ -16,7 +16,7 @@ import parseAttachmentIds from "@server/utils/parseAttachmentIds";
import parseImages from "@server/utils/parseImages";
@trace()
-export default class TextHelper {
+export class TextHelper {
/**
* Replaces template variables in the given text with the current date and time.
*
diff --git a/server/models/validators/TextLength.ts b/server/models/validators/TextLength.ts
index 6dcc6eaacd00..444e327ce60d 100644
--- a/server/models/validators/TextLength.ts
+++ b/server/models/validators/TextLength.ts
@@ -2,7 +2,7 @@ import size from "lodash/size";
import { Node } from "prosemirror-model";
import { addAttributeOptions } from "sequelize-typescript";
import { ProsemirrorData } from "@shared/types";
-import ProsemirrorHelper from "@shared/utils/ProsemirrorHelper";
+import { ProsemirrorHelper } from "@shared/utils/ProsemirrorHelper";
import { schema } from "@server/editor";
/**
diff --git a/server/presenters/document.ts b/server/presenters/document.ts
index 8547071df214..b103fc453930 100644
--- a/server/presenters/document.ts
+++ b/server/presenters/document.ts
@@ -1,6 +1,8 @@
import { traceFunction } from "@server/logging/tracing";
import { Document } from "@server/models";
-import TextHelper from "@server/models/helpers/TextHelper";
+import { DocumentHelper } from "@server/models/helpers/DocumentHelper";
+import { TextHelper } from "@server/models/helpers/TextHelper";
+import { APIContext } from "@server/types";
import presentUser from "./user";
type Options = {
@@ -8,6 +10,7 @@ type Options = {
};
async function presentDocument(
+ ctx: APIContext | undefined,
document: Document,
options: Options | null | undefined = {}
) {
@@ -15,17 +18,32 @@ async function presentDocument(
isPublic: false,
...options,
};
- const text = options.isPublic
- ? await TextHelper.attachmentsToSignedUrls(document.text, document.teamId)
- : document.text;
+
+ const asData = !ctx || Number(ctx?.headers["x-api-version"] ?? 0) >= 3;
+ const text =
+ options.isPublic && !asData
+ ? await TextHelper.attachmentsToSignedUrls(document.text, document.teamId)
+ : document.text;
const data: Record = {
id: document.id,
url: document.url,
urlId: document.urlId,
title: document.title,
+ data: asData
+ ? await DocumentHelper.toJSON(
+ document,
+ options.isPublic
+ ? {
+ signedUrls: 60,
+ teamId: document.teamId,
+ removeMarks: ["comment"],
+ }
+ : undefined
+ )
+ : undefined,
+ text: asData ? undefined : text,
emoji: document.emoji,
- text,
tasks: document.tasks,
createdAt: document.createdAt,
createdBy: undefined,
@@ -41,7 +59,7 @@ async function presentDocument(
collectionId: undefined,
parentDocumentId: undefined,
lastViewedAt: undefined,
- isCollectionDeleted: await document.isCollectionDeleted(),
+ isCollectionDeleted: undefined,
};
if (!!document.views && document.views.length > 0) {
@@ -51,6 +69,7 @@ async function presentDocument(
if (!options.isPublic) {
const source = await document.$get("import");
+ data.isCollectionDeleted = await document.isCollectionDeleted();
data.collectionId = document.collectionId;
data.parentDocumentId = document.parentDocumentId;
data.createdBy = presentUser(document.createdBy);
diff --git a/server/presenters/notification.ts b/server/presenters/notification.ts
index 401f4344a1b9..12372832ecca 100644
--- a/server/presenters/notification.ts
+++ b/server/presenters/notification.ts
@@ -1,8 +1,12 @@
import { Notification } from "@server/models";
+import { APIContext } from "@server/types";
import presentUser from "./user";
import { presentComment, presentDocument } from ".";
-export default async function presentNotification(notification: Notification) {
+export default async function presentNotification(
+ ctx: APIContext | undefined,
+ notification: Notification
+) {
return {
id: notification.id,
viewedAt: notification.viewedAt,
@@ -18,7 +22,7 @@ export default async function presentNotification(notification: Notification) {
: undefined,
documentId: notification.documentId,
document: notification.document
- ? await presentDocument(notification.document)
+ ? await presentDocument(ctx, notification.document)
: undefined,
revisionId: notification.revisionId,
collectionId: notification.collectionId,
diff --git a/server/presenters/revision.ts b/server/presenters/revision.ts
index 24cdc4ad8e47..df6c90875428 100644
--- a/server/presenters/revision.ts
+++ b/server/presenters/revision.ts
@@ -1,7 +1,7 @@
import parseTitle from "@shared/utils/parseTitle";
import { traceFunction } from "@server/logging/tracing";
import { Revision } from "@server/models";
-import DocumentHelper from "@server/models/helpers/DocumentHelper";
+import { DocumentHelper } from "@server/models/helpers/DocumentHelper";
import presentUser from "./user";
async function presentRevision(revision: Revision, diff?: string) {
@@ -12,7 +12,7 @@ async function presentRevision(revision: Revision, diff?: string) {
id: revision.id,
documentId: revision.documentId,
title: strippedTitle,
- text: DocumentHelper.toMarkdown(revision),
+ data: await DocumentHelper.toJSON(revision),
emoji: revision.emoji ?? emoji,
html: diff,
createdAt: revision.createdAt,
diff --git a/server/queues/processors/WebsocketsProcessor.ts b/server/queues/processors/WebsocketsProcessor.ts
index dbe66804712d..2f65dd188d88 100644
--- a/server/queues/processors/WebsocketsProcessor.ts
+++ b/server/queues/processors/WebsocketsProcessor.ts
@@ -81,7 +81,7 @@ export default class WebsocketsProcessor {
if (!document) {
return;
}
- const data = await presentDocument(document);
+ const data = await presentDocument(undefined, document);
const channels = await this.getDocumentEventChannels(event, document);
return socketio.to(channels).emit(event.name, data);
}
@@ -452,7 +452,7 @@ export default class WebsocketsProcessor {
return;
}
- const data = await presentNotification(notification);
+ const data = await presentNotification(undefined, notification);
return socketio.to(`user-${event.userId}`).emit(event.name, data);
}
diff --git a/server/queues/tasks/CleanupDeletedDocumentsTask.ts b/server/queues/tasks/CleanupDeletedDocumentsTask.ts
index 7cae409bd16c..4c04095381c2 100644
--- a/server/queues/tasks/CleanupDeletedDocumentsTask.ts
+++ b/server/queues/tasks/CleanupDeletedDocumentsTask.ts
@@ -18,7 +18,7 @@ export default class CleanupDeletedDocumentsTask extends BaseTask {
`Permanently destroying upto ${limit} documents older than 30 days…`
);
const documents = await Document.scope("withDrafts").findAll({
- attributes: ["id", "teamId", "text", "deletedAt"],
+ attributes: ["id", "teamId", "content", "text", "deletedAt"],
where: {
deletedAt: {
[Op.lt]: subDays(new Date(), 30),
diff --git a/server/queues/tasks/CommentCreatedNotificationsTask.ts b/server/queues/tasks/CommentCreatedNotificationsTask.ts
index d2903794c466..98bc7c4ceeb5 100644
--- a/server/queues/tasks/CommentCreatedNotificationsTask.ts
+++ b/server/queues/tasks/CommentCreatedNotificationsTask.ts
@@ -2,7 +2,7 @@ import { NotificationEventType } from "@shared/types";
import subscriptionCreator from "@server/commands/subscriptionCreator";
import { Comment, Document, Notification, User } from "@server/models";
import NotificationHelper from "@server/models/helpers/NotificationHelper";
-import ProsemirrorHelper from "@server/models/helpers/ProsemirrorHelper";
+import { ProsemirrorHelper } from "@server/models/helpers/ProsemirrorHelper";
import { sequelize } from "@server/storage/database";
import { CommentEvent } from "@server/types";
import BaseTask, { TaskPriority } from "./BaseTask";
diff --git a/server/queues/tasks/CommentUpdatedNotificationsTask.ts b/server/queues/tasks/CommentUpdatedNotificationsTask.ts
index 5cc230b81529..68fb184c6216 100644
--- a/server/queues/tasks/CommentUpdatedNotificationsTask.ts
+++ b/server/queues/tasks/CommentUpdatedNotificationsTask.ts
@@ -1,6 +1,6 @@
import { NotificationEventType } from "@shared/types";
import { Comment, Document, Notification, User } from "@server/models";
-import ProsemirrorHelper from "@server/models/helpers/ProsemirrorHelper";
+import { ProsemirrorHelper } from "@server/models/helpers/ProsemirrorHelper";
import { CommentEvent, CommentUpdateEvent } from "@server/types";
import BaseTask, { TaskPriority } from "./BaseTask";
diff --git a/server/queues/tasks/DocumentPublishedNotificationsTask.ts b/server/queues/tasks/DocumentPublishedNotificationsTask.ts
index 2cfed43d441f..50f160ed287d 100644
--- a/server/queues/tasks/DocumentPublishedNotificationsTask.ts
+++ b/server/queues/tasks/DocumentPublishedNotificationsTask.ts
@@ -1,7 +1,7 @@
import { NotificationEventType } from "@shared/types";
import { createSubscriptionsForDocument } from "@server/commands/subscriptionCreator";
import { Document, Notification, User } from "@server/models";
-import DocumentHelper from "@server/models/helpers/DocumentHelper";
+import { DocumentHelper } from "@server/models/helpers/DocumentHelper";
import NotificationHelper from "@server/models/helpers/NotificationHelper";
import { DocumentEvent } from "@server/types";
import BaseTask, { TaskPriority } from "./BaseTask";
diff --git a/server/queues/tasks/ExportDocumentTreeTask.ts b/server/queues/tasks/ExportDocumentTreeTask.ts
index 083246e05ff8..fdf8f8d74a66 100644
--- a/server/queues/tasks/ExportDocumentTreeTask.ts
+++ b/server/queues/tasks/ExportDocumentTreeTask.ts
@@ -6,7 +6,7 @@ import Logger from "@server/logging/Logger";
import { Collection } from "@server/models";
import Attachment from "@server/models/Attachment";
import Document from "@server/models/Document";
-import DocumentHelper from "@server/models/helpers/DocumentHelper";
+import { DocumentHelper } from "@server/models/helpers/DocumentHelper";
import ZipHelper from "@server/utils/ZipHelper";
import { serializeFilename } from "@server/utils/fs";
import parseAttachmentIds from "@server/utils/parseAttachmentIds";
diff --git a/server/queues/tasks/ExportJSONTask.ts b/server/queues/tasks/ExportJSONTask.ts
index 91112dbc27c2..eedda6b9f08f 100644
--- a/server/queues/tasks/ExportJSONTask.ts
+++ b/server/queues/tasks/ExportJSONTask.ts
@@ -10,12 +10,12 @@ import {
Document,
FileOperation,
} from "@server/models";
-import DocumentHelper from "@server/models/helpers/DocumentHelper";
+import { DocumentHelper } from "@server/models/helpers/DocumentHelper";
+import { ProsemirrorHelper } from "@server/models/helpers/ProsemirrorHelper";
import { presentAttachment, presentCollection } from "@server/presenters";
import { CollectionJSONExport, JSONExportMetadata } from "@server/types";
import ZipHelper from "@server/utils/ZipHelper";
import { serializeFilename } from "@server/utils/fs";
-import parseAttachmentIds from "@server/utils/parseAttachmentIds";
import packageJson from "../../../package.json";
import ExportTask from "./ExportTask";
@@ -87,7 +87,9 @@ export default class ExportJSONTask extends ExportTask {
? await Attachment.findAll({
where: {
teamId: document.teamId,
- id: parseAttachmentIds(document.text),
+ id: ProsemirrorHelper.parseAttachmentIds(
+ DocumentHelper.toProsemirror(document)
+ ),
},
})
: [];
diff --git a/server/queues/tasks/RevisionCreatedNotificationsTask.ts b/server/queues/tasks/RevisionCreatedNotificationsTask.ts
index 95c28e4ac688..46ea22140dea 100644
--- a/server/queues/tasks/RevisionCreatedNotificationsTask.ts
+++ b/server/queues/tasks/RevisionCreatedNotificationsTask.ts
@@ -6,7 +6,7 @@ import { createSubscriptionsForDocument } from "@server/commands/subscriptionCre
import env from "@server/env";
import Logger from "@server/logging/Logger";
import { Document, Revision, Notification, User, View } from "@server/models";
-import DocumentHelper from "@server/models/helpers/DocumentHelper";
+import { DocumentHelper } from "@server/models/helpers/DocumentHelper";
import NotificationHelper from "@server/models/helpers/NotificationHelper";
import { RevisionEvent } from "@server/types";
import BaseTask, { TaskPriority } from "./BaseTask";
diff --git a/server/routes/api/documents/documents.test.ts b/server/routes/api/documents/documents.test.ts
index fa751ce678f6..4e47e8ee908e 100644
--- a/server/routes/api/documents/documents.test.ts
+++ b/server/routes/api/documents/documents.test.ts
@@ -17,7 +17,7 @@ import {
User,
GroupPermission,
} from "@server/models";
-import DocumentHelper from "@server/models/helpers/DocumentHelper";
+import { DocumentHelper } from "@server/models/helpers/DocumentHelper";
import {
buildShare,
buildCollection,
@@ -3467,7 +3467,6 @@ describe("#documents.update", () => {
token: user.getJwtToken(),
id: document.id,
title: document.title,
- text: document.text,
},
});
expect(res.status).toEqual(200);
diff --git a/server/routes/api/documents/documents.ts b/server/routes/api/documents/documents.ts
index a87c5c3b0b4f..5d42330bc874 100644
--- a/server/routes/api/documents/documents.ts
+++ b/server/routes/api/documents/documents.ts
@@ -45,7 +45,8 @@ import {
UserMembership,
} from "@server/models";
import AttachmentHelper from "@server/models/helpers/AttachmentHelper";
-import DocumentHelper from "@server/models/helpers/DocumentHelper";
+import { DocumentHelper } from "@server/models/helpers/DocumentHelper";
+import { ProsemirrorHelper } from "@server/models/helpers/ProsemirrorHelper";
import SearchHelper from "@server/models/helpers/SearchHelper";
import { authorize, cannot } from "@server/policies";
import {
@@ -63,7 +64,6 @@ import FileStorage from "@server/storage/files";
import { APIContext } from "@server/types";
import { RateLimiterStrategy } from "@server/utils/RateLimiter";
import ZipHelper from "@server/utils/ZipHelper";
-import parseAttachmentIds from "@server/utils/parseAttachmentIds";
import { getTeamFromContext } from "@server/utils/passport";
import { assertPresent } from "@server/validation";
import pagination from "../middlewares/pagination";
@@ -191,7 +191,7 @@ router.post(
}
const data = await Promise.all(
- documents.map((document) => presentDocument(document))
+ documents.map((document) => presentDocument(ctx, document))
);
const policies = presentPolicies(user, documents);
ctx.body = {
@@ -224,7 +224,7 @@ router.post(
limit: ctx.state.pagination.limit,
});
const data = await Promise.all(
- documents.map((document) => presentDocument(document))
+ documents.map((document) => presentDocument(ctx, document))
);
const policies = presentPolicies(user, documents);
@@ -287,7 +287,7 @@ router.post(
limit: ctx.state.pagination.limit,
});
const data = await Promise.all(
- documents.map((document) => presentDocument(document))
+ documents.map((document) => presentDocument(ctx, document))
);
const policies = presentPolicies(user, documents);
@@ -343,7 +343,7 @@ router.post(
return document;
});
const data = await Promise.all(
- documents.map((document) => presentDocument(document))
+ documents.map((document) => presentDocument(ctx, document))
);
const policies = presentPolicies(user, documents);
@@ -399,7 +399,7 @@ router.post(
limit: ctx.state.pagination.limit,
});
const data = await Promise.all(
- documents.map((document) => presentDocument(document))
+ documents.map((document) => presentDocument(ctx, document))
);
const policies = presentPolicies(user, documents);
@@ -416,8 +416,9 @@ router.post(
auth({ optional: true }),
validate(T.DocumentsInfoSchema),
async (ctx: APIContext) => {
- const { id, shareId, apiVersion } = ctx.input.body;
+ const { id, shareId } = ctx.input.body;
const { user } = ctx.state.auth;
+ const apiVersion = getAPIVersion(ctx);
const teamFromCtx = await getTeamFromContext(ctx);
const { document, share, collection } = await documentLoader({
id,
@@ -426,7 +427,7 @@ router.post(
teamId: teamFromCtx?.id,
});
const isPublic = cannot(user, "read", document);
- const serializedDocument = await presentDocument(document, {
+ const serializedDocument = await presentDocument(ctx, document, {
isPublic,
});
@@ -435,7 +436,7 @@ router.post(
// Passing apiVersion=2 has a single effect, to change the response payload to
// include top level keys for document, sharedTree, and team.
const data =
- apiVersion === 2
+ apiVersion >= 2
? {
document: serializedDocument,
team: team?.getPreference(TeamPreference.PublicBranding)
@@ -572,7 +573,9 @@ router.post(
contentType === "text/markdown" ? "md" : mime.extension(contentType);
const fileName = slugify(document.titleWithDefault);
- const attachmentIds = parseAttachmentIds(document.text);
+ const attachmentIds = ProsemirrorHelper.parseAttachmentIds(
+ DocumentHelper.toProsemirror(document)
+ );
const attachments = attachmentIds.length
? await Attachment.findAll({
where: {
@@ -729,7 +732,7 @@ router.post(
}
ctx.body = {
- data: await presentDocument(document),
+ data: await presentDocument(ctx, document),
policies: presentPolicies(user, [document]),
};
}
@@ -769,7 +772,7 @@ router.post(
});
const policies = presentPolicies(user, documents);
const data = await Promise.all(
- documents.map((document) => presentDocument(document))
+ documents.map((document) => presentDocument(ctx, document))
);
ctx.body = {
@@ -895,7 +898,7 @@ router.post(
const data = await Promise.all(
results.map(async (result) => {
- const document = await presentDocument(result.document);
+ const document = await presentDocument(ctx, result.document);
return { ...result, document };
})
);
@@ -982,7 +985,7 @@ router.post(
invariant(reloaded, "document not found");
ctx.body = {
- data: await presentDocument(reloaded),
+ data: await presentDocument(ctx, reloaded),
policies: presentPolicies(user, [reloaded]),
};
}
@@ -995,9 +998,10 @@ router.post(
transaction(),
async (ctx: APIContext) => {
const { transaction } = ctx.state;
- const { id, apiVersion, insightsEnabled, publish, collectionId, ...input } =
+ const { id, insightsEnabled, publish, collectionId, ...input } =
ctx.input.body;
const editorVersion = ctx.headers["x-editor-version"] as string | undefined;
+
const { user } = ctx.state.auth;
let collection: Collection | null | undefined;
@@ -1052,15 +1056,7 @@ router.post(
document.collection = collection;
ctx.body = {
- data:
- apiVersion === 2
- ? {
- document: await presentDocument(document),
- collection: collection
- ? presentCollection(collection)
- : undefined,
- }
- : await presentDocument(document),
+ data: await presentDocument(ctx, document),
policies: presentPolicies(user, [document, collection]),
};
}
@@ -1120,7 +1116,7 @@ router.post(
ctx.body = {
data: {
documents: await Promise.all(
- response.map((document) => presentDocument(document))
+ response.map((document) => presentDocument(ctx, document))
),
},
policies: presentPolicies(user, response),
@@ -1173,7 +1169,7 @@ router.post(
ctx.body = {
data: {
documents: await Promise.all(
- documents.map((document) => presentDocument(document))
+ documents.map((document) => presentDocument(ctx, document))
),
collections: await Promise.all(
collections.map((collection) => presentCollection(collection))
@@ -1211,7 +1207,7 @@ router.post(
});
ctx.body = {
- data: await presentDocument(document),
+ data: await presentDocument(ctx, document),
policies: presentPolicies(user, [document]),
};
}
@@ -1276,7 +1272,7 @@ router.post(
auth(),
validate(T.DocumentsUnpublishSchema),
async (ctx: APIContext) => {
- const { id, apiVersion } = ctx.input.body;
+ const { id } = ctx.input.body;
const { user } = ctx.state.auth;
const document = await Document.findByPk(id, {
@@ -1309,15 +1305,7 @@ router.post(
});
ctx.body = {
- data:
- apiVersion === 2
- ? {
- document: await presentDocument(document),
- collection: document.collection
- ? presentCollection(document.collection)
- : undefined,
- }
- : await presentDocument(document),
+ data: await presentDocument(ctx, document),
policies: presentPolicies(user, [document]),
};
}
@@ -1395,7 +1383,7 @@ router.post(
});
ctx.body = {
- data: await presentDocument(document),
+ data: await presentDocument(ctx, document),
policies: presentPolicies(user, [document]),
};
}
@@ -1481,7 +1469,7 @@ router.post(
document.collection = collection;
ctx.body = {
- data: await presentDocument(document),
+ data: await presentDocument(ctx, document),
policies: presentPolicies(user, [document]),
};
}
@@ -1760,4 +1748,16 @@ router.post(
}
);
+// Remove this helper once apiVersion is removed (#6175)
+function getAPIVersion(ctx: APIContext) {
+ return Number(
+ ctx.headers["x-api-version"] ??
+ (typeof ctx.input.body === "object" &&
+ ctx.input.body &&
+ "apiVersion" in ctx.input.body &&
+ ctx.input.body.apiVersion) ??
+ 0
+ );
+}
+
export default router;
diff --git a/server/routes/api/documents/schema.ts b/server/routes/api/documents/schema.ts
index 97f1749d247e..b122d09419d7 100644
--- a/server/routes/api/documents/schema.ts
+++ b/server/routes/api/documents/schema.ts
@@ -118,7 +118,7 @@ export const DocumentsInfoSchema = BaseSchema.extend({
.refine((val) => isUUID(val) || UrlHelper.SHARE_URL_SLUG_REGEX.test(val))
.optional(),
- /** Version of the API to be used */
+ /** @deprecated Version of the API to be used, remove in a few releases */
apiVersion: z.number().optional(),
}),
}).refine((req) => !(isEmpty(req.body.id) && isEmpty(req.body.shareId)), {
@@ -241,7 +241,7 @@ export const DocumentsUpdateSchema = BaseSchema.extend({
/** Boolean to denote if text should be appended */
append: z.boolean().optional(),
- /** Version of the API to be used */
+ /** @deprecated Version of the API to be used, remove in a few releases */
apiVersion: z.number().optional(),
/** Whether the editing session is complete */
@@ -287,7 +287,7 @@ export type DocumentsDeleteReq = z.infer;
export const DocumentsUnpublishSchema = BaseSchema.extend({
body: BaseIdSchema.extend({
- /** Version of the API to be used */
+ /** @deprecated Version of the API to be used, remove in a few releases */
apiVersion: z.number().optional(),
}),
});
diff --git a/server/routes/api/notifications/notifications.ts b/server/routes/api/notifications/notifications.ts
index 2bab82e4ad66..3a003b7361f6 100644
--- a/server/routes/api/notifications/notifications.ts
+++ b/server/routes/api/notifications/notifications.ts
@@ -117,7 +117,9 @@ router.post(
pagination: { ...ctx.state.pagination, total },
data: {
notifications: await Promise.all(
- notifications.map(presentNotification)
+ notifications.map((notification) =>
+ presentNotification(ctx, notification)
+ )
),
unseen,
},
@@ -172,7 +174,7 @@ router.post(
});
ctx.body = {
- data: await presentNotification(notification),
+ data: await presentNotification(ctx, notification),
policies: presentPolicies(user, [notification]),
};
}
diff --git a/server/routes/api/pins/pins.ts b/server/routes/api/pins/pins.ts
index 3f34af262b6c..c359866ff2c7 100644
--- a/server/routes/api/pins/pins.ts
+++ b/server/routes/api/pins/pins.ts
@@ -96,7 +96,7 @@ router.post(
data: {
pins: pins.map(presentPin),
documents: await Promise.all(
- documents.map((document: Document) => presentDocument(document))
+ documents.map((document: Document) => presentDocument(ctx, document))
),
},
policies,
diff --git a/server/routes/api/revisions/revisions.test.ts b/server/routes/api/revisions/revisions.test.ts
index c34e8939a593..7db8e718749f 100644
--- a/server/routes/api/revisions/revisions.test.ts
+++ b/server/routes/api/revisions/revisions.test.ts
@@ -105,7 +105,23 @@ describe("#revisions.diff", () => {
});
await Revision.createFromDocument(document);
- await document.update({ text: "New text" });
+ await document.update({
+ content: {
+ type: "doc",
+ content: [
+ {
+ type: "paragraph",
+ content: [
+ {
+ content: [],
+ type: "text",
+ text: "New text",
+ },
+ ],
+ },
+ ],
+ },
+ });
const revision1 = await Revision.createFromDocument(document);
const res = await server.post("/api/revisions.diff", {
diff --git a/server/routes/api/revisions/revisions.ts b/server/routes/api/revisions/revisions.ts
index 45d39dd517f2..dd6e213cbc9e 100644
--- a/server/routes/api/revisions/revisions.ts
+++ b/server/routes/api/revisions/revisions.ts
@@ -6,7 +6,7 @@ import { ValidationError } from "@server/errors";
import auth from "@server/middlewares/authentication";
import validate from "@server/middlewares/validate";
import { Document, Revision } from "@server/models";
-import DocumentHelper from "@server/models/helpers/DocumentHelper";
+import { DocumentHelper } from "@server/models/helpers/DocumentHelper";
import { authorize } from "@server/policies";
import { presentRevision } from "@server/presenters";
import { APIContext } from "@server/types";
diff --git a/server/routes/api/stars/stars.ts b/server/routes/api/stars/stars.ts
index 742dc4693d31..2b3be99fdbad 100644
--- a/server/routes/api/stars/stars.ts
+++ b/server/routes/api/stars/stars.ts
@@ -112,7 +112,7 @@ router.post(
data: {
stars: stars.map(presentStar),
documents: await Promise.all(
- documents.map((document: Document) => presentDocument(document))
+ documents.map((document: Document) => presentDocument(ctx, document))
),
},
policies,
diff --git a/server/routes/api/userMemberships/userMemberships.ts b/server/routes/api/userMemberships/userMemberships.ts
index be925a98b011..d9b8e3d296d8 100644
--- a/server/routes/api/userMemberships/userMemberships.ts
+++ b/server/routes/api/userMemberships/userMemberships.ts
@@ -63,7 +63,7 @@ router.post(
data: {
memberships: memberships.map(presentMembership),
documents: await Promise.all(
- documents.map((document: Document) => presentDocument(document))
+ documents.map((document: Document) => presentDocument(ctx, document))
),
},
policies,
diff --git a/server/scripts/20231119000000-backfill-document-content.ts b/server/scripts/20231119000000-backfill-document-content.ts
index 9dd6d160a06d..821f22ea9fc9 100644
--- a/server/scripts/20231119000000-backfill-document-content.ts
+++ b/server/scripts/20231119000000-backfill-document-content.ts
@@ -2,6 +2,7 @@ import "./bootstrap";
import { yDocToProsemirrorJSON } from "@getoutline/y-prosemirror";
import { Node } from "prosemirror-model";
import * as Y from "yjs";
+import { ProsemirrorData } from "@shared/types";
import { parser, schema } from "@server/editor";
import { Document } from "@server/models";
@@ -31,7 +32,10 @@ export default async function main(exit = false) {
if ("state" in document && document.state) {
const ydoc = new Y.Doc();
Y.applyUpdate(ydoc, document.state);
- document.content = yDocToProsemirrorJSON(ydoc, "default");
+ document.content = yDocToProsemirrorJSON(
+ ydoc,
+ "default"
+ ) as ProsemirrorData;
} else {
const node = parser.parse(document.text) || Node.fromJSON(schema, {});
document.content = node.toJSON();
diff --git a/server/test/factories.ts b/server/test/factories.ts
index e6ca51ca7bb6..929cb0672070 100644
--- a/server/test/factories.ts
+++ b/server/test/factories.ts
@@ -13,6 +13,7 @@ import {
NotificationEventType,
UserRole,
} from "@shared/types";
+import { parser } from "@server/editor";
import {
Share,
Team,
@@ -371,10 +372,12 @@ export async function buildDocument(
overrides.collectionId = collection.id;
}
+ const text = overrides.text ?? "This is the text in an example document";
const document = await Document.create(
{
title: faker.lorem.words(4),
- text: "This is the text in an example document",
+ content: overrides.content ?? parser.parse(text)?.toJSON(),
+ text,
publishedAt: isNull(overrides.collectionId) ? null : new Date(),
lastModifiedById: overrides.userId,
createdById: overrides.userId,
@@ -410,6 +413,7 @@ export async function buildComment(overrides: {
type: "paragraph",
content: [
{
+ content: [],
type: "text",
text: "test",
},
diff --git a/server/utils/ProsemirrorHelper.test.ts b/server/utils/ProsemirrorHelper.test.ts
index 0068d04de4a7..02bd92e5d7e6 100644
--- a/server/utils/ProsemirrorHelper.test.ts
+++ b/server/utils/ProsemirrorHelper.test.ts
@@ -1,5 +1,5 @@
import { Node } from "prosemirror-model";
-import ProsemirrorHelper from "@shared/utils/ProsemirrorHelper";
+import { ProsemirrorHelper } from "@shared/utils/ProsemirrorHelper";
import { schema } from "@server/editor";
// Note: The test is here rather than shared to access the schema
diff --git a/server/utils/parseAttachmentIds.ts b/server/utils/parseAttachmentIds.ts
index 832cdfb54cbc..2cd008fd1fb5 100644
--- a/server/utils/parseAttachmentIds.ts
+++ b/server/utils/parseAttachmentIds.ts
@@ -1,10 +1,9 @@
import compact from "lodash/compact";
import uniq from "lodash/uniq";
-
-const attachmentRedirectRegex =
- /\/api\/attachments\.redirect\?id=(?[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/gi;
-const attachmentPublicRegex =
- /public\/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\/(?[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/gi;
+import {
+ attachmentPublicRegex,
+ attachmentRedirectRegex,
+} from "@shared/utils/ProsemirrorHelper";
export default function parseAttachmentIds(
text: string,
diff --git a/shared/editor/version.ts b/shared/editor/version.ts
index b087359ec374..0cbb6c801192 100644
--- a/shared/editor/version.ts
+++ b/shared/editor/version.ts
@@ -1,3 +1,3 @@
-const EDITOR_VERSION = "12.0.0";
+const EDITOR_VERSION = "13.0.0";
export default EDITOR_VERSION;
diff --git a/shared/types.ts b/shared/types.ts
index 5d41f6970530..edf9e5002f85 100644
--- a/shared/types.ts
+++ b/shared/types.ts
@@ -374,4 +374,18 @@ export type JSONValue =
export type JSONObject = { [x: string]: JSONValue };
-export type ProsemirrorData = JSONObject;
+export type ProsemirrorData = {
+ type: string;
+ content: ProsemirrorData[];
+ text?: string;
+ attrs?: JSONObject;
+ marks?: {
+ type: string;
+ attrs: JSONObject;
+ }[];
+};
+
+export type ProsemirrorDoc = {
+ type: "doc";
+ content: ProsemirrorData[];
+};
diff --git a/shared/utils/MarkdownHelper.test.ts b/shared/utils/MarkdownHelper.test.ts
deleted file mode 100644
index 24713a4c7450..000000000000
--- a/shared/utils/MarkdownHelper.test.ts
+++ /dev/null
@@ -1,18 +0,0 @@
-import MarkdownHelper from "./MarkdownHelper";
-
-describe("#MarkdownHelper", () => {
- it("should serialize title and text", () => {
- expect(MarkdownHelper.toMarkdown({ title: "Title", text: "Test" })).toEqual(
- "# Title\n\nTest"
- );
- });
-
- it("should trim backslashes", () => {
- expect(
- MarkdownHelper.toMarkdown({
- title: "Title",
- text: "One\n\\\nTest\n\\",
- })
- ).toEqual("# Title\n\nOne\n\nTest");
- });
-});
diff --git a/shared/utils/MarkdownHelper.ts b/shared/utils/MarkdownHelper.ts
deleted file mode 100644
index d7167487ba95..000000000000
--- a/shared/utils/MarkdownHelper.ts
+++ /dev/null
@@ -1,29 +0,0 @@
-interface DocumentInterface {
- emoji?: string | null;
- title: string;
- text: string;
-}
-
-export default class MarkdownHelper {
- /**
- * Returns the document as cleaned Markdown for export.
- *
- * @param document The document or revision to convert
- * @returns The document title and content as a Markdown string
- */
- static toMarkdown(document: DocumentInterface) {
- const text = document.text
- .replace(/\n\\(\n|$)/g, "\n\n")
- .replace(/“/g, '"')
- .replace(/”/g, '"')
- .replace(/‘/g, "'")
- .replace(/’/g, "'")
- .trim();
-
- const title = `${document.emoji ? document.emoji + " " : ""}${
- document.title
- }`;
-
- return `# ${title}\n\n${text}`;
- }
-}
diff --git a/shared/utils/ProsemirrorHelper.ts b/shared/utils/ProsemirrorHelper.ts
index 938c20a8e01e..a28177ee7bae 100644
--- a/shared/utils/ProsemirrorHelper.ts
+++ b/shared/utils/ProsemirrorHelper.ts
@@ -1,6 +1,7 @@
import { Node, Schema } from "prosemirror-model";
import headingToSlug from "../editor/lib/headingToSlug";
import textBetween from "../editor/lib/textBetween";
+import { ProsemirrorData } from "../types";
export type Heading = {
/* The heading in plain text */
@@ -27,7 +28,30 @@ export type Task = {
completed: boolean;
};
-export default class ProsemirrorHelper {
+export const attachmentRedirectRegex =
+ /\/api\/attachments\.redirect\?id=(?[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/gi;
+
+export const attachmentPublicRegex =
+ /public\/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\/(?[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/gi;
+
+export class ProsemirrorHelper {
+ /**
+ * Get a new empty document.
+ *
+ * @returns A new empty document as JSON.
+ */
+ static getEmptyDocument(): ProsemirrorData {
+ return {
+ type: "doc",
+ content: [
+ {
+ content: [],
+ type: "paragraph",
+ },
+ ],
+ };
+ }
+
/**
* Returns the node as plain text.
*
@@ -160,6 +184,21 @@ export default class ProsemirrorHelper {
return tasks;
}
+ /**
+ * Returns a summary of total and completed tasks in the node.
+ *
+ * @param doc Prosemirror document node
+ * @returns Object with completed and total keys
+ */
+ static getTasksSummary(doc: Node): { completed: number; total: number } {
+ const tasks = ProsemirrorHelper.getTasks(doc);
+
+ return {
+ completed: tasks.filter((t) => t.completed).length,
+ total: tasks.length,
+ };
+ }
+
/**
* Iterates through the document to find all of the headings and their level.
*
diff --git a/shared/utils/getTasks.ts b/shared/utils/getTasks.ts
deleted file mode 100644
index ef5e99166d92..000000000000
--- a/shared/utils/getTasks.ts
+++ /dev/null
@@ -1,23 +0,0 @@
-const CHECKBOX_REGEX = /\[(X|\s|_|-)\]\s(.*)?/gi;
-
-export default function getTasks(text: string) {
- const matches = [...text.matchAll(CHECKBOX_REGEX)];
- const total = matches.length;
-
- if (!total) {
- return {
- completed: 0,
- total: 0,
- };
- } else {
- const notCompleted = matches.reduce(
- (accumulator, match) =>
- match[1] === " " ? accumulator + 1 : accumulator,
- 0
- );
- return {
- completed: total - notCompleted,
- total,
- };
- }
-}