diff --git a/backend/package.json b/backend/package.json index 031a43df05..954f40c700 100644 --- a/backend/package.json +++ b/backend/package.json @@ -87,6 +87,7 @@ "metascraper-url": "^5.6.5", "metascraper-video": "^5.6.5", "metascraper-youtube": "^5.6.3", + "minimatch": "^3.0.4", "neo4j-driver": "~1.7.5", "neo4j-graphql-js": "^2.6.3", "neode": "^0.3.0", diff --git a/backend/src/middleware/handleHtmlContent/handleContentData.spec.js b/backend/src/middleware/handleHtmlContent/handleContentData.spec.js index f2e3c23035..6fe5d3891c 100644 --- a/backend/src/middleware/handleHtmlContent/handleContentData.spec.js +++ b/backend/src/middleware/handleHtmlContent/handleContentData.spec.js @@ -64,7 +64,7 @@ describe('currentUser { notifications }', () => { let post const title = 'Mentioning Al Capone' const content = - 'Hey @al-capone how do you do?' + 'Hey @al-capone how do you do?' beforeEach(async () => { const createPostMutation = gql` @@ -88,7 +88,7 @@ describe('currentUser { notifications }', () => { it('sends you a notification', async () => { const expectedContent = - 'Hey @al-capone how do you do?' + 'Hey @al-capone how do you do?' const expected = { currentUser: { notifications: [ @@ -108,14 +108,22 @@ describe('currentUser { notifications }', () => { ).resolves.toEqual(expected) }) - describe('who mentions me again', () => { + describe('who mentions me many times', () => { beforeEach(async () => { - const updatedContent = `${post.content} One more mention to @al-capone` - // The response `post.content` contains a link but the XSSmiddleware - // should have the `mention` CSS class removed. I discovered this - // during development and thought: A feature not a bug! This way we - // can encode a re-mentioning of users when you edit your post or - // comment. + const updatedContent = ` + One more mention to + + @al-capone + + and again: + + @al-capone + + and again + + @al-capone + + ` const updatePostMutation = gql` mutation($id: ID!, $title: String!, $content: String!) { UpdatePost(id: $id, content: $content, title: $title) { @@ -136,7 +144,7 @@ describe('currentUser { notifications }', () => { it('creates exactly one more notification', async () => { const expectedContent = - 'Hey @al-capone how do you do? One more mention to @al-capone' + '
One more mention to

@al-capone

and again:

@al-capone

and again

@al-capone

' const expected = { currentUser: { notifications: [ diff --git a/backend/src/middleware/handleHtmlContent/notifications/extractMentionedUsers.js b/backend/src/middleware/handleHtmlContent/notifications/extractMentionedUsers.js index c2fcf169c8..d08309f0ba 100644 --- a/backend/src/middleware/handleHtmlContent/notifications/extractMentionedUsers.js +++ b/backend/src/middleware/handleHtmlContent/notifications/extractMentionedUsers.js @@ -1,20 +1,13 @@ import cheerio from 'cheerio' -const ID_REGEX = /\/profile\/([\w\-.!~*'"(),]+)/g export default function(content) { if (!content) return [] const $ = cheerio.load(content) - const urls = $('.mention') + let userIds = $('a.mention[data-mention-id]') .map((_, el) => { - return $(el).attr('href') + return $(el).attr('data-mention-id') }) .get() - const ids = [] - urls.forEach(url => { - let match - while ((match = ID_REGEX.exec(url)) != null) { - ids.push(match[1]) - } - }) - return ids + userIds = userIds.map(id => id.trim()).filter(id => !!id) + return userIds } diff --git a/backend/src/middleware/handleHtmlContent/notifications/extractMentionedUsers.spec.js b/backend/src/middleware/handleHtmlContent/notifications/extractMentionedUsers.spec.js index f39fbc859d..a631b64a32 100644 --- a/backend/src/middleware/handleHtmlContent/notifications/extractMentionedUsers.spec.js +++ b/backend/src/middleware/handleHtmlContent/notifications/extractMentionedUsers.spec.js @@ -1,5 +1,12 @@ import extractMentionedUsers from './extractMentionedUsers' +const contentWithMentions = + '

Something inspirational about @bob-der-baumeister and @jenny-rostock.

' +const contentEmptyMentions = + '

Something inspirational about @bob-der-baumeister and @jenny-rostock.

' +const contentWithPlainLinks = + '

Something inspirational about @bob-der-baumeister and @jenny-rostock.

' + describe('extractMentionedUsers', () => { describe('content undefined', () => { it('returns empty array', () => { @@ -7,53 +14,17 @@ describe('extractMentionedUsers', () => { }) }) - describe('searches through links', () => { - it('ignores links without .mention class', () => { - const content = - '

Something inspirational about @bob-der-baumeister and @jenny-rostock.

' - expect(extractMentionedUsers(content)).toEqual([]) - }) - - describe('given a link with .mention class', () => { - it('extracts ids', () => { - const content = - '

Something inspirational about @bob-der-baumeister and @jenny-rostock.

' - expect(extractMentionedUsers(content)).toEqual(['u2', 'u3']) - }) - - describe('handles links', () => { - it('with slug and id', () => { - const content = - '

Something inspirational about @bob-der-baumeister and @jenny-rostock.

' - expect(extractMentionedUsers(content)).toEqual(['u2', 'u3']) - }) - - it('with domains', () => { - const content = - '

Something inspirational about @bob-der-baumeister and @jenny-rostock.

' - expect(extractMentionedUsers(content)).toEqual(['u2', 'u3']) - }) - - it('special characters', () => { - const content = - '

Something inspirational about @bob-der-baumeister and @jenny-rostock.

' - expect(extractMentionedUsers(content)).toEqual(['u!*(),2', 'u.~-3']) - }) - }) + it('ignores links without .mention class', () => { + expect(extractMentionedUsers(contentWithPlainLinks)).toEqual([]) + }) - describe('does not crash if', () => { - it('`href` contains no user id', () => { - const content = - '

Something inspirational about @bob-der-baumeister and @jenny-rostock.

' - expect(extractMentionedUsers(content)).toEqual([]) - }) + describe('given a link with .mention class and `data-mention-id` attribute ', () => { + it('extracts ids', () => { + expect(extractMentionedUsers(contentWithMentions)).toEqual(['u3']) + }) - it('`href` is empty or invalid', () => { - const content = - '

Something inspirational about @bob-der-baumeister and @jenny-rostock.

' - expect(extractMentionedUsers(content)).toEqual([]) - }) - }) + it('ignores empty `data-mention-id` attributes', () => { + expect(extractMentionedUsers(contentEmptyMentions)).toEqual([]) }) }) }) diff --git a/backend/src/middleware/xssMiddleware.js b/backend/src/middleware/xssMiddleware.js index 6894e86015..f98ab9d619 100644 --- a/backend/src/middleware/xssMiddleware.js +++ b/backend/src/middleware/xssMiddleware.js @@ -2,30 +2,16 @@ import walkRecursive from '../helpers/walkRecursive' // import { getByDot, setByDot, getItems, replaceItems } from 'feathers-hooks-common' import sanitizeHtml from 'sanitize-html' // import { isEmpty, intersection } from 'lodash' -import cheerio from 'cheerio' import linkifyHtml from 'linkifyjs/html' -const embedToAnchor = content => { - const $ = cheerio.load(content) - $('div[data-url-embed]').each((i, el) => { - const url = el.attribs['data-url-embed'] - const aTag = $(`${url}`) - $(el).replaceWith(aTag) - }) - return $('body').html() -} - function clean(dirty) { if (!dirty) { return dirty } - // Convert embeds to a-tags - dirty = embedToAnchor(dirty) dirty = linkifyHtml(dirty) dirty = sanitizeHtml(dirty, { allowedTags: [ - 'iframe', 'img', 'p', 'h3', @@ -50,35 +36,24 @@ function clean(dirty) { a: ['href', 'class', 'target', 'data-*', 'contenteditable'], span: ['contenteditable', 'class', 'data-*'], img: ['src'], - iframe: ['src', 'class', 'frameborder', 'allowfullscreen'], }, allowedIframeHostnames: ['www.youtube.com', 'player.vimeo.com'], parser: { lowerCaseTags: true, }, transformTags: { - iframe: function(tagName, attribs) { - return { - tagName: 'a', - text: attribs.src, - attribs: { - href: attribs.src, - target: '_blank', - 'data-url-embed': '', - }, - } - }, h1: 'h3', h2: 'h3', h3: 'h3', h4: 'h4', h5: 'strong', i: 'em', - a: function(tagName, attribs) { + a: (tagName, attribs) => { return { tagName: 'a', attribs: { - href: attribs.href, + ...attribs, + href: attribs.href || '', target: '_blank', rel: 'noopener noreferrer nofollow', }, @@ -86,33 +61,6 @@ function clean(dirty) { }, b: 'strong', s: 'strike', - img: function(tagName, attribs) { - const src = attribs.src - - if (!src) { - // remove broken images - return {} - } - - // if (isEmpty(hook.result)) { - // const config = hook.app.get('thumbor') - // if (config && src.indexOf(config < 0)) { - // // download image - // // const ThumborUrlHelper = require('../helper/thumbor-helper') - // // const Thumbor = new ThumborUrlHelper(config.key || null, config.url || null) - // // src = Thumbor - // // .setImagePath(src) - // // .buildUrl('740x0') - // } - // } - return { - tagName: 'img', - attribs: { - // TODO: use environment variables - src: `http://localhost:3050/images?url=${src}`, - }, - } - }, }, }) @@ -120,8 +68,6 @@ function clean(dirty) { dirty = dirty // remove all tags with "space only" .replace(/<[a-z-]+>[\s]+<\/[a-z-]+>/gim, '') - // remove all iframes - .replace(/(]*)(>)[^>]*\/*>/gim, '') .replace(/[\n]{3,}/gim, '\n\n') .replace(/(\r\n|\n\r|\r|\n)/g, '
$1') @@ -144,8 +90,7 @@ const fields = ['content', 'contentExcerpt'] export default { Mutation: async (resolve, root, args, context, info) => { args = walkRecursive(args, fields, clean) - const result = await resolve(root, args, context, info) - return result + return resolve(root, args, context, info) }, Query: async (resolve, root, args, context, info) => { const result = await resolve(root, args, context, info) diff --git a/backend/src/schema/resolvers/embeds/findProvider.js b/backend/src/schema/resolvers/embeds/findProvider.js new file mode 100644 index 0000000000..491cbb9e84 --- /dev/null +++ b/backend/src/schema/resolvers/embeds/findProvider.js @@ -0,0 +1,26 @@ +import fs from 'fs' +import path from 'path' +import minimatch from 'minimatch' + +let oEmbedProvidersFile = fs.readFileSync(path.join(__dirname, './providers.json'), 'utf8') +// some providers allow a format parameter +// we need JSON +oEmbedProvidersFile = oEmbedProvidersFile.replace(/\{format\}/g, 'json') +const oEmbedProviders = JSON.parse(oEmbedProvidersFile) + +export default function(embedUrl) { + for (const provider of oEmbedProviders) { + for (const endpoint of provider.endpoints) { + const { schemes = [], url } = endpoint + if (schemes.some(scheme => minimatch(embedUrl, scheme))) return url + } + const { hostname } = new URL(embedUrl) + if (provider.provider_url.includes(hostname)) { + const { + endpoints: [{ url }], + } = provider + return url + } + } + return null +} diff --git a/backend/src/schema/resolvers/embeds/findProvider.spec.js b/backend/src/schema/resolvers/embeds/findProvider.spec.js new file mode 100644 index 0000000000..cc8cdcb707 --- /dev/null +++ b/backend/src/schema/resolvers/embeds/findProvider.spec.js @@ -0,0 +1,35 @@ +import findProvider from './findProvider' + +describe('Vimeo', () => { + it('matches `https://vimeo.com/showcase/2098620/video/4082288`', () => { + expect(findProvider('https://vimeo.com/showcase/2098620/video/4082288')).toEqual( + 'https://vimeo.com/api/oembed.json', + ) + }) +}) + +describe('RiffReporter', () => { + it('matches `https://www.riffreporter.de/flugbegleiter-koralle/`', () => { + expect(findProvider('https://www.riffreporter.de/flugbegleiter-koralle/')).toEqual( + 'https://www.riffreporter.de/service/oembed', + ) + }) +}) + +describe('Youtube', () => { + it('matches `https://www.youtube.com/watch?v=qkdXAtO40Fo`', () => { + expect(findProvider('https://www.youtube.com/watch?v=qkdXAtO40Fo')).toEqual( + 'https://www.youtube.com/oembed', + ) + }) + + it('matches `https://youtu.be/qkdXAtO40Fo`', () => { + expect(findProvider(`https://youtu.be/qkdXAtO40Fo`)).toEqual('https://www.youtube.com/oembed') + }) + + it('matches `https://youtu.be/qkdXAtO40Fo?t=41`', () => { + expect(findProvider(`https://youtu.be/qkdXAtO40Fo?t=41`)).toEqual( + 'https://www.youtube.com/oembed', + ) + }) +}) diff --git a/backend/src/schema/resolvers/embeds/scraper.js b/backend/src/schema/resolvers/embeds/scraper.js index 607f7aeb92..bbf4fc999b 100644 --- a/backend/src/schema/resolvers/embeds/scraper.js +++ b/backend/src/schema/resolvers/embeds/scraper.js @@ -1,12 +1,11 @@ import Metascraper from 'metascraper' import fetch from 'node-fetch' -import fs from 'fs' -import path from 'path' import { ApolloError } from 'apollo-server' import isEmpty from 'lodash/isEmpty' import isArray from 'lodash/isArray' import mergeWith from 'lodash/mergeWith' +import findProvider from './findProvider' const error = require('debug')('embed:error') @@ -30,24 +29,11 @@ const metascraper = Metascraper([ // require('./rules/metascraper-embed')() ]) -let oEmbedProvidersFile = fs.readFileSync(path.join(__dirname, './providers.json'), 'utf8') - -// some providers allow a format parameter -// we need JSON -oEmbedProvidersFile = oEmbedProvidersFile.replace('{format}', 'json') - -const oEmbedProviders = JSON.parse(oEmbedProvidersFile) - const fetchEmbed = async url => { - const provider = oEmbedProviders.find(provider => { - return provider.provider_url.includes(url.hostname) - }) - if (!provider) return {} - const { - endpoints: [endpoint], - } = provider - const endpointUrl = new URL(endpoint.url) - endpointUrl.searchParams.append('url', url.href) + let endpointUrl = findProvider(url) + if (!endpointUrl) return {} + endpointUrl = new URL(endpointUrl) + endpointUrl.searchParams.append('url', url) endpointUrl.searchParams.append('format', 'json') let json try { @@ -70,7 +56,7 @@ const fetchEmbed = async url => { const fetchResource = async url => { const response = await fetch(url) const html = await response.text() - const resource = await metascraper({ html, url: url.href }) + const resource = await metascraper({ html, url }) return { sources: ['resource'], ...resource, @@ -78,12 +64,6 @@ const fetchResource = async url => { } export default async function scrape(url) { - url = new URL(url) - if (url.hostname === 'youtu.be') { - // replace youtu.be to get proper results - url.hostname = 'youtube.com' - } - const [meta, embed] = await Promise.all([fetchResource(url), fetchEmbed(url)]) const output = mergeWith(meta, embed, (objValue, srcValue) => { if (isArray(objValue)) { diff --git a/backend/src/schema/types/type/Tag.gql b/backend/src/schema/types/type/Tag.gql index ecbd0b46a0..47021bf823 100644 --- a/backend/src/schema/types/type/Tag.gql +++ b/backend/src/schema/types/type/Tag.gql @@ -7,4 +7,4 @@ type Tag { taggedCountUnique: Int! @cypher(statement: "MATCH (this)<-[:TAGGED]-(p)<-[:WROTE]-(u:User) RETURN COUNT(DISTINCT u)") deleted: Boolean disabled: Boolean -} \ No newline at end of file +} diff --git a/cypress/integration/common/steps.js b/cypress/integration/common/steps.js index f996db9922..387273effa 100644 --- a/cypress/integration/common/steps.js +++ b/cypress/integration/common/steps.js @@ -230,7 +230,7 @@ When("I choose {string} as the title of the post", title => { When("I type in the following text:", text => { lastPost.content = text.replace("\n", " "); - cy.get(".ProseMirror").type(lastPost.content); + cy.get(".editor .ProseMirror").type(lastPost.content); }); Then("the post shows up on the landing page at position {int}", index => { diff --git a/webapp/components/Comment.vue b/webapp/components/Comment.vue index 7684c3f758..9096fb6e82 100644 --- a/webapp/components/Comment.vue +++ b/webapp/components/Comment.vue @@ -45,7 +45,11 @@ {{ $t('comment.show.more') }} -
+
{{ $t('comment.show.less') }} @@ -62,6 +66,7 @@ import gql from 'graphql-tag' import { mapGetters, mapMutations } from 'vuex' import HcUser from '~/components/User' import ContentMenu from '~/components/ContentMenu' +import ContentViewer from '~/components/Editor/ContentViewer' import HcEditCommentForm from '~/components/comments/EditCommentForm/EditCommentForm' export default { @@ -74,6 +79,7 @@ export default { components: { HcUser, ContentMenu, + ContentViewer, HcEditCommentForm, }, props: { diff --git a/webapp/components/Editor/ContentViewer.vue b/webapp/components/Editor/ContentViewer.vue new file mode 100644 index 0000000000..993cf32b96 --- /dev/null +++ b/webapp/components/Editor/ContentViewer.vue @@ -0,0 +1,32 @@ + + + diff --git a/webapp/components/Editor/Editor.story.js b/webapp/components/Editor/Editor.story.js new file mode 100644 index 0000000000..aaaa0eb585 --- /dev/null +++ b/webapp/components/Editor/Editor.story.js @@ -0,0 +1,132 @@ +import { storiesOf } from '@storybook/vue' +import { withA11y } from '@storybook/addon-a11y' +import HcEditor from '~/components/Editor/Editor.vue' +import helpers from '~/storybook/helpers' +import Vue from 'vue' + +const embed = { + html: + '', +} + +const plugins = [ + (app = {}) => { + app.$apollo = { + mutate: () => {}, + query: () => { + return { data: { embed } } + }, + } + Vue.prototype.$apollo = app.$apollo + return app + }, +] +helpers.init({ plugins }) + +const users = [{ id: 1, slug: 'peter' }, { id: 2, slug: 'sandra' }, { id: 3, slug: 'jane' }] + +storiesOf('Editor', module) + .addDecorator(withA11y) + .addDecorator(storyFn => { + const ctx = storyFn() + return { + components: { ctx }, + template: ` + + + + `, + } + }) + .addDecorator(helpers.layout) + .add('Empty', () => ({ + components: { HcEditor }, + store: helpers.store, + data: () => ({ + users, + }), + template: ``, + })) + .add('Basic formatting', () => ({ + components: { HcEditor }, + store: helpers.store, + data: () => ({ + users, + content: ` +

Basic formatting

+

+ Here is some italic, bold and underline text. +
+ Also do we have some
inline links here. +

+

Heading 3

+

At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua.

+

Heading 4

+

At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua.

+
Heading 5
+

At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua.

+ +

Unordered List

+
    +
  • Also some list

  • +
  • with

  • +
  • several

  • +
  • points

  • +
+ +

Ordered List

+
    +
  1. justo

  2. +
  3. dolores

  4. +
  5. et ea rebum

  6. +
  7. kasd gubergren

  8. +
+ `, + }), + template: ``, + })) + .add('@Mentions', () => ({ + components: { HcEditor }, + store: helpers.store, + data: () => ({ + users, + content: ` +

+ Here you can mention people like + @sandra and others. + Try it out! +

+ `, + }), + template: ``, + })) + .add('#Hashtags', () => ({ + components: { HcEditor }, + store: helpers.store, + data: () => ({ + users, + content: ` +

+ This text contains #hashtags for projects like #human-connection + Try to add more by typing #. +

+ `, + }), + template: ``, + })) + .add('Embeds', () => ({ + components: { HcEditor }, + store: helpers.store, + data: () => ({ + users, + content: ` +

+ The following link should render a youtube video in addition to the link. +

+ + https://www.youtube.com/watch?v=qkdXAtO40Fo + + `, + }), + template: ``, + })) diff --git a/webapp/components/Editor/Editor.vue b/webapp/components/Editor/Editor.vue index 4413bfa0d5..4a4880eeb8 100644 --- a/webapp/components/Editor/Editor.vue +++ b/webapp/components/Editor/Editor.vue @@ -183,30 +183,16 @@