diff --git a/bin/analyze-page b/bin/analyze-page deleted file mode 100755 index 1fbacc6d12..0000000000 --- a/bin/analyze-page +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env python -""" -This script helps you analyze some page on the internet using Page Shot scraping - -You have to have Page Shot installed and running locally in server mode. - -Usage: - - $ analyze-page URL > results.md - -This creates a report about the page, including information on all the CSS rules -that aren't used in the page, and a list of resources (e.g., images) that ARE -used on a page. - -You can do this on a couple pages (ideally all unique pages from your site) and -see if a rule isn't used anywhere. You also will have a list of resources, and -can look for files that aren't on that list. -""" -import urllib -import sys -import json -import os -import re -import urlparse - -if not sys.argv[1:] or '-h' in sys.argv: - print("Usage:") - print(" analyze-page URL > results.md") - print(" To set host:") - print(" PAGESHOT_HOST=http://localhost:10082 analyze-page URL") - sys.exit() - -port = os.environ.get("PAGESHOT_HOST", 10082) - -url = sys.argv[1] -parsed = urlparse.urlparse(url) -base_url = "%s://%s/" % (parsed.scheme, parsed.netloc) - -pageshot = "http://localhost:%s/data/?url=%s&inlineCss=true&debugInlineCss=true" % (port, urllib.quote(url)) - -response = urllib.urlopen(pageshot).read() -try: - page = json.loads(response) -except: - print("Bad response:") - print(response) - raise - -def display_url(u): - if u.startswith(base_url): - u = "/" + u[len(base_url):] - u = re.sub(r'\?.*', '', u) - return u - -omitted = re.compile(r'Omitted: (.*?) \(from ([^)]*)\) [*]/') -styles = {} -for match in omitted.finditer(page["head"]): - rule = match.group(1) - href = match.group(2) - if href not in styles: - styles[href] = [] - styles[href].append(rule) -if styles: - print("# Unused CSS rules:") - for href in sorted(styles): - short = display_url(href) - if short != href: - print(" - [%s](%s)" % (short, href)) - else: - print(" - %s" % href) - for rule in styles[href]: - print(" - `%s`" % rule) - print("") -else: - print("# No unused CSS rules") - -print("") -print("# Used resources:") -for resource in page["resources"].values(): - resource_url = resource["url"] - short = display_url(resource_url) - if short != resource_url: - print(" - [%s](%s)" % (short, resource_url)) - else: - print(" - %s" % resource_url) diff --git a/bin/load_test_exercise.py b/bin/load_test_exercise.py index 8c0eedcc28..5895349e12 100755 --- a/bin/load_test_exercise.py +++ b/bin/load_test_exercise.py @@ -103,8 +103,6 @@ def make_example_shot(): createdDate=int(time.time()*1000), favicon=None, siteName="test site", - isPublic=True, - showPage=False, clips={ make_uuid(): dict( createdDate=int(time.time()*1000), diff --git a/docs/testing-the-api.md b/docs/testing-the-api.md index dd49e3f55b..c94e2efcd9 100644 --- a/docs/testing-the-api.md +++ b/docs/testing-the-api.md @@ -76,24 +76,15 @@ Authenticated. Creates or updates a shot. Takes a JSON body. Looks like: "y": 383 }, "title": null, - "alt": "my_shots", - "isReadable": false + "alt": "my_shots" } ], - "head": "...", - "body": "<div>...", - "htmlAttrs": [["attrname", "attrvalue"]], - "bodyAttrs": [["attrname", "attrvalue"]], - "headAttrs": [], "siteName": "GitHub", "documentSize": { "width": 1127, "height": 2086 }, "fullScreenThumbnail": "data:...", - "isPublic": true, - "resources": {}, - "showPage": false, "clips": { "cplocgk9m1nc": { "createdDate": 1468983771992, @@ -148,16 +139,12 @@ Then you get a response back that converts the `data:...` URLs to other URLs, li `GET /data/{random_id}/{domain}` -Not authenticated. Gets the JSON back, but with `head` and `body` removed. +Not authenticated. Gets the JSON back. `POST /api/delete-shot` Authenticated. Takes only one parameter, `id` (the `"{random_id}/{domain}"`) -`POST /api/add-saved-shot-data/{random_id}/{domain}` - -Authenticated. Takes a JSON body, with keys `head`, `body`, `headAttrs`, `bodyAttrs`, and `htmlAttrs` - `POST /api/set-expiration` Authenticated. Takes URL encoded parameters, `id` and `expiration` (milliseconds) diff --git a/package.json b/package.json index 5ecf061748..d4e128017c 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,6 @@ "convict": "2.0.0", "cookies": "0.6.2", "core-js": "2.4.1", - "domino": "1.0.28", "envc": "2.4.1", "escape-html": "1.0.3", "express": "4.14.1", diff --git a/server/db-patches/patch-13-14.sql b/server/db-patches/patch-13-14.sql new file mode 100644 index 0000000000..9863599551 --- /dev/null +++ b/server/db-patches/patch-13-14.sql @@ -0,0 +1,2 @@ +ALTER TABLE data DROP COLUMN head; +ALTER TABLE data DROP COLUMN body; diff --git a/server/db-patches/patch-14-13.sql b/server/db-patches/patch-14-13.sql new file mode 100644 index 0000000000..1061dc4f28 --- /dev/null +++ b/server/db-patches/patch-14-13.sql @@ -0,0 +1,2 @@ +ALTER TABLE data ADD COLUMN head TEXT; +ALTER TABLE data ADD COLUMN body TEXT; diff --git a/server/schema.sql b/server/schema.sql index 6c15f2ee65..1b8c00f23f 100644 --- a/server/schema.sql +++ b/server/schema.sql @@ -7,8 +7,6 @@ CREATE TABLE data ( deviceid character varying(200), created timestamp without time zone DEFAULT now(), value text NOT NULL, - head text, - body text, url text NOT NULL, expire_time timestamp without time zone DEFAULT (now() + '14 days'::interval), deleted boolean DEFAULT false NOT NULL, @@ -25,7 +23,8 @@ CREATE TABLE devices ( last_login timestamp without time zone, created timestamp without time zone DEFAULT now(), session_count integer DEFAULT 0, - secret_hashed text + secret_hashed text, + ab_tests text ); CREATE TABLE images ( id character varying(200) NOT NULL, @@ -73,4 +72,4 @@ ALTER TABLE ONLY images ADD CONSTRAINT images_shotid_fkey FOREIGN KEY (shotid) REFERENCES data(id) ON DELETE CASCADE; ALTER TABLE ONLY states ADD CONSTRAINT states_deviceid_fkey FOREIGN KEY (deviceid) REFERENCES devices(id) ON DELETE CASCADE; --- pg-patch version: 12 +-- pg-patch version: 14 diff --git a/server/src/contentcheck.js b/server/src/contentcheck.js deleted file mode 100644 index 2fc8c94ba9..0000000000 --- a/server/src/contentcheck.js +++ /dev/null @@ -1,81 +0,0 @@ -const domino = require("domino"); - -/** These elements are never sent: */ -const skipElementsBadTags = { - SCRIPT: true, - NOSCRIPT: true -}; - -const checkAttrsForLinks = { - src: true, - href: true, - content: true -}; - -exports.checkContent = function (htmlString) { - let window = domino.createWindow(htmlString); - let errors = []; - let els = window.document.getElementsByTagName("*"); - let elLength = els.length; - for (let i=0; i<elLength; i++) { - let el = els[i]; - if (skipElementsBadTags[el.tagName]) { - errors.push(`Bad element: ${describeTag(el)}`); - } - if (el.attributes) { - let attrList = []; - for (let name in el.attributes) { - if (name == "element") { - // Internal/private variable - continue; - } - let value = el.attributes[name].data; - attrList.push([name, value]); - } - errors = errors.concat(exports.checkAttributes(attrList, el)); - } - } - // FIXME: should confirm <base href> is correct - return errors; -}; - -exports.checkAttributes = function (attrList, el) { - if (! (attrList && attrList.length)) { - return []; - } - let errors = []; - for (let i=0; i<attrList.length; i++) { - let name = attrList[i][0].toLowerCase(); - let value = attrList[i][1]; - if (name.startsWith("on")) { - errors.push(`Bad attribute ${safeString(name)} on ${describeTag(el)}`); - } - if (checkAttrsForLinks[name]) { - if (value.search(/^javascript:/i) !== -1) { - errors.push(`Bad attribute ${safeString(name)} with javascript link on ${describeTag(el)}`); - } - } - } - return errors; -}; - -function describeTag(el) { - if (typeof el == "string") { - return `<${safeString(el)}>`; - } - let s = el.tagName; - if (el.className) { - s += safeString("." + el.className.split().join(".")); - } - if (el.id) { - s += safeString("#" + el.id); - } - return s; -} - -/** Return a string that contains only identifier-ish characters. Because we - ** are creating potentially loggable error messages with user-created content, we - ** want to resist any XSS attacks against our logs themselves */ -function safeString(s) { - return s.replace(/[^a-zA-Z0-9_\-.,#]/, ""); -} diff --git a/server/src/dbschema.js b/server/src/dbschema.js index e56e9fb24a..378f9971b6 100644 --- a/server/src/dbschema.js +++ b/server/src/dbschema.js @@ -4,7 +4,7 @@ const pgpatcher = require("pg-patcher"); const path = require("path"); const mozlog = require("mozlog")("dbschema"); -const MAX_DB_LEVEL = exports.MAX_DB_LEVEL = 13; +const MAX_DB_LEVEL = exports.MAX_DB_LEVEL = 14; exports.forceDbVersion = function (version) { mozlog.info("forcing-db-version", {db: db.constr, version}); diff --git a/server/src/pages/shot/controller.js b/server/src/pages/shot/controller.js index ba60f2875e..371d22b366 100644 --- a/server/src/pages/shot/controller.js +++ b/server/src/pages/shot/controller.js @@ -179,51 +179,6 @@ function sendShowElement(clipId) { } } -/* -function requestHasSavedShot(id) { - let event = document.createEvent("CustomEvent"); - event.initCustomEvent("has-saved-shot", true, true, id); - document.dispatchEvent(event); -} - -exports.requestSavedShot = function () { - let event = document.createEvent("CustomEvent"); - event.initCustomEvent("request-saved-shot", true, true, model.shot.id); - document.dispatchEvent(event); -}; - -function addSavedShotData(data) { - if (! data) { - model.hasSavedShot = false; - exports.render(); - return; - } - for (let attr in data) { - if (! data[attr]) { - delete data[attr]; - } - } - data.showPage = true; - model.shot.update(data); - model.hasSavedShot = false; - let url = model.backend + "/api/add-saved-shot-data/" + model.shot.id; - let req = new XMLHttpRequest(); - req.onload = function () { - if (req.status >= 300) { - window.alert("Error saving expiration: " + req.status + " " + req.statusText); - return; - } - let event = document.createEvent("CustomEvent"); - event.initCustomEvent("remove-saved-shot", true, true, model.shot.id); - document.dispatchEvent(event); - exports.render(); - }; - req.open("POST", url); - req.setRequestHeader("content-type", "application/json"); - req.send(JSON.stringify(data)); -} -*/ - exports.setTitle = function (title) { title = title || null; let url = model.backend + "/api/set-title/" + model.shot.id; diff --git a/server/src/pages/shot/view.js b/server/src/pages/shot/view.js index aa132109ea..ba09010277 100644 --- a/server/src/pages/shot/view.js +++ b/server/src/pages/shot/view.js @@ -93,7 +93,7 @@ class Head extends React.Component { if (! this.props.shot) { return null; } - let title = this.props.shot.ogTitle || + let title = (this.props.shot.openGraph && this.props.shot.openGraph.title) || (this.props.shot.twitterCard && this.props.shot.twitterCard.title) || this.props.shot.title; let og = [ diff --git a/server/src/server.js b/server/src/server.js index 0e6d3977a0..a9a74ac31e 100644 --- a/server/src/server.js +++ b/server/src/server.js @@ -64,7 +64,6 @@ const morgan = require("morgan"); const linker = require("./linker"); const { randomBytes } = require("./helpers"); const errors = require("../shared/errors"); -const { checkContent, checkAttributes } = require("./contentcheck"); const buildTime = require("./build-time").string; const ua = require("universal-analytics"); const urlParse = require("url").parse; @@ -649,51 +648,12 @@ app.put("/data/:id/:domain", function (req, res) { throw new Error(`Got unexpected req.body type: ${typeof bodyObj}`); } let shotId = `${req.params.id}/${req.params.domain}`; - - if (! bodyObj.deviceId) { - console.warn("No deviceId in request body", req.url); - let keys = "No keys"; - try { - keys = Object.keys(bodyObj); - } catch (e) { - // ignore - } - sendRavenMessage( - req, "Attempt PUT without deviceId in request body", - {extra: - { - "typeof bodyObj": typeof bodyObj, - keys - } - } - ); - simpleResponse(res, "No deviceId in body", 400); - return; - } if (! req.deviceId) { console.warn("Attempted to PUT without logging in", req.url); sendRavenMessage(req, "Attempt PUT without authentication"); simpleResponse(res, "Not logged in", 401); return; } - if (req.deviceId != bodyObj.deviceId) { - // FIXME: this doesn't make sense for comments or other stuff, see https://github.com/mozilla-services/pageshot/issues/245 - console.warn("Attempted to PUT a page with a different deviceId than the login deviceId"); - sendRavenMessage(req, "Attempted to save page for another user"); - simpleResponse(res, "Cannot save a page on behalf of another user", 403); - return; - } - let errors = checkContent(bodyObj.head) - .concat(checkContent(bodyObj.body)) - .concat(checkAttributes(bodyObj.headAttrs, "head")) - .concat(checkAttributes(bodyObj.bodyAttrs, "body")) - .concat(checkAttributes(bodyObj.htmlAttrs, "html")); - if (errors.length) { - console.warn("Attempted to submit page with invalid HTML:", errors.join("; ").substr(0, 60)); - sendRavenMessage(req, "Errors in submission", {extra: {errors: errors}}); - simpleResponse(res, "Errors in submission", 400); - return; - } let shot = new Shot(req.deviceId, req.backend, shotId, bodyObj); let responseDelay = Promise.resolve() if (slowResponse) { @@ -724,7 +684,6 @@ app.get("/data/:id/:domain", function (req, res) { } else { let value = data.value; value = JSON.parse(value); - delete value.deviceId; value = JSON.stringify(value); if ('format' in req.query) { value = JSON.stringify(JSON.parse(value), null, ' '); @@ -767,9 +726,9 @@ app.post("/api/set-title/:id/:domain", function (req, res) { simpleResponse(res, "Not logged in", 401); return; } - Shot.get(req.backend, shotId).then((shot) => { - if (shot.deviceId !== req.deviceId) { - simpleResponse(res, "Not the owner", 403); + Shot.get(req.backend, shotId, req.deviceId).then((shot) => { + if (! shot) { + simpleResponse(res, "No such shot", 404); return; } shot.userTitle = userTitle; @@ -781,46 +740,6 @@ app.post("/api/set-title/:id/:domain", function (req, res) { }); }); -/* -app.post("/api/add-saved-shot-data/:id/:domain", function (req, res) { - let shotId = `${req.params.id}/${req.params.domain}`; - let bodyObj = req.body; - Shot.get(req.backend, shotId).then((shot) => { - if (! shot) { - sendRavenMessage(req, "Attempt to add saved shot data when no shot exists"); - simpleResponse(res, "No such shot", 404); - return; - } - let errors = checkContent(bodyObj.head) - .concat(checkContent(bodyObj.body)) - .concat(checkAttributes(bodyObj.headAttrs, "head")) - .concat(checkAttributes(bodyObj.bodyAttrs, "body")) - .concat(checkAttributes(bodyObj.htmlAttrs, "html")); - if (errors.length) { - console.warn("Attempted to submit page with invalid HTML:", errors.join("; ").substr(0, 60)); - sendRavenMessage(req, "Errors in submission when adding saved shot", {extra: {errors: errors}}); - simpleResponse(res, "Errors in submission", 400); - return; - } - for (let attr in bodyObj) { - if (! ["body", "head", "headAttrs", "bodyAttrs", "htmlAttrs", "showPage", "readable", "resources"].includes(attr)) { - console.warn("Unexpected attribute in update:", attr); - sendRavenMessage(req, "Unexpected attribute in submission", {extra: {attr}}); - simpleResponse(res, "Unexpected attribute in submission", 400); - return; - } - shot[attr] = bodyObj[attr]; - } - return shot.update().then(() => { - simpleResponse(res, "ok", 200); - }); - }).catch((err) => { - errorResponse(res, "Error serving data:", err); - }); - -}); -*/ - app.post("/api/set-expiration", function (req, res) { if (! req.deviceId) { sendRavenMessage(req, "Attempt to set expiration without login"); diff --git a/server/src/servershot.js b/server/src/servershot.js index 17286cb656..ef33c629b1 100644 --- a/server/src/servershot.js +++ b/server/src/servershot.js @@ -164,19 +164,15 @@ class Shot extends AbstractShot { clipRewrites.rewriteShotUrls(); let oks = clipRewrites.commands(); let json = this.asJson(); - let head = json.head; - let body = json.body; let title = this.title; - json.head = null; - json.body = null; oks.push({setHead: null}); oks.push({setBody: null}); - let searchable = this._makeSearchableText(9); + let searchable = this._makeSearchableText(7); return db.queryWithClient( client, - `INSERT INTO data (id, deviceid, value, head, body, url, title, searchable_version, searchable_text) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, ${searchable.query})`, - [this.id, this.ownerId, JSON.stringify(json), head, body, json.url, title, searchable.version].concat(searchable.args) + `INSERT INTO data (id, deviceid, value, url, title, searchable_version, searchable_text) + VALUES ($1, $2, $3, $4, $5, $6, ${searchable.query})`, + [this.id, this.ownerId, JSON.stringify(json), json.url, title, searchable.version].concat(searchable.args) ).then((rowCount) => { return clipRewrites.commit(client); }).then(() => { @@ -199,34 +195,14 @@ class Shot extends AbstractShot { let oks = clipRewrites.commands(); let json = this.asJson(); return db.transaction((client) => { - let head = json.head; - let body = json.body; - json.head = null; - json.body = null; - if (head !== null) { - oks.push({setHead: null}); - } - if (body !== null) { - oks.push({setBody: null}); - } let promise; - if (head === null && body === null) { - let searchable = this._makeSearchableText(7); - promise = db.queryWithClient( - client, - `UPDATE data SET value = $1, url = $2, title=$3, searchable_version = $4, searchable_text = ${searchable.query} - WHERE id = $5 AND deviceid = $6`, - [JSON.stringify(json), this.url, this.title, searchable.version, this.id, this.ownerId].concat(searchable.args) - ); - } else { - let searchable = this._makeSearchableText(9); - promise = db.queryWithClient( - client, - `UPDATE data SET value = $1, url=$2, title=$3, head = $4, body = $5, searchable_version = $6, searchable_text = ${searchable.query} - WHERE id = $7 AND deviceid = $8`, - [JSON.stringify(json), this.url, this.title, head, body, searchable.version, this.id, this.ownerId].concat(searchable.args) - ); - } + let searchable = this._makeSearchableText(7); + promise = db.queryWithClient( + client, + `UPDATE data SET value = $1, url = $2, title=$3, searchable_version = $4, searchable_text = ${searchable.query} + WHERE id = $5 AND deviceid = $6`, + [JSON.stringify(json), this.url, this.title, searchable.version, this.id, this.ownerId].concat(searchable.args) + ); return promise.then((rowCount) => { if (! rowCount) { throw new Error("No row updated"); @@ -291,14 +267,6 @@ class Shot extends AbstractShot { addWeight(clip.image && clip.image.text, 'A', 'clip text'); } } - let readableBody = this.readable ? this.readable.content.replace(/<[^>]*>/g, " ") : null; - let wholeBody = this.body ? this.body.replace(/<[^>]*>/g, " ") : null; - if (readableBody) { - addWeight(readableBody, 'C', 'readable'); - addWeight(wholeBody, 'D', 'body'); - } else { - addWeight(wholeBody, 'C', 'body'); - } return { query: queryParts.join(' || '), args: texts, @@ -356,8 +324,8 @@ class ServerClip extends AbstractShot.prototype.Clip { Shot.prototype.Clip = ServerClip; -Shot.get = function (backend, id) { - return Shot.getRawValue(id).then((rawValue) => { +Shot.get = function (backend, id, deviceId) { + return Shot.getRawValue(id, deviceId).then((rawValue) => { if (! rawValue) { return null; } @@ -365,7 +333,7 @@ Shot.get = function (backend, id) { if (! json.url && rawValue.url) { json.url = rawValue.url; } - let jsonTitle = json.userTitle || json.ogTitle || (json.openGraph && json.openGraph.title) || json.docTitle; + let jsonTitle = json.userTitle || (json.openGraph && json.openGraph.title) || json.docTitle; if (! jsonTitle) { json.docTitle = rawValue.title; } @@ -383,7 +351,7 @@ Shot.getFullShot = function (backend, id) { throw new Error("Empty id: " + id); } return db.select( - `SELECT value, deviceid, head, body FROM data + `SELECT value, deviceid FROM data WHERE data.id = $1`, [id] ).then((rows) => { @@ -393,19 +361,23 @@ Shot.getFullShot = function (backend, id) { let row = rows[0]; let json = JSON.parse(row.value); let shot = new Shot(row.userid, backend, id, json); - shot.head = row.head; - shot.body = row.body; return shot; }); }; -Shot.getRawValue = function (id) { +Shot.getRawValue = function (id, deviceId) { if (! id) { throw new Error("Empty id: " + id); } + let query = `SELECT value, deviceid, url, title, expire_time, deleted FROM data WHERE id = $1`; + let params = [id]; + if (deviceId) { + query += ` AND deviceid = $2`; + params.push(deviceId); + } return db.select( - `SELECT value, deviceid, url, title, expire_time, deleted FROM data WHERE id = $1`, - [id] + query, + params ).then((rows) => { if (! rows.length) { return null; @@ -713,7 +685,7 @@ Shot.cleanDeletedShots = function () { client, ` UPDATE data - SET value = '{}', head = NULL, body = NULL, deleted = TRUE + SET value = '{}', deleted = TRUE WHERE expire_time + ($1 || ' SECONDS')::INTERVAL < CURRENT_TIMESTAMP AND NOT deleted `, diff --git a/shared/shot.js b/shared/shot.js index 8c03f8c330..6bab040f59 100644 --- a/shared/shot.js +++ b/shared/shot.js @@ -36,20 +36,6 @@ function isUrl(url) { return (/^https?:\/\/[a-z0-9\.\-]+[a-z0-9](:[0-9]+)?\/?/i).test(url); } -/** Tests if a value is a set of attribute pairs, like [["name", "value"], ...] */ -function isAttributePairs(val) { - if (! Array.isArray(val)) { - return false; - } - let good = true; - val.forEach((pair) => { - if (typeof pair[0] != "string" || (! pair[0]) || typeof pair[1] != "string") { - good = false; - } - }); - return good; -} - /** Check if the given object has all of the required attributes, and no extra attributes exception those in optional */ function checkObject(obj, required, optional) { @@ -175,73 +161,29 @@ function makeUuid() { return id; } -function formatAttributes(attrs) { - if (! attrs) { - return ""; - } - let result = []; - for (let item of attrs) { - let name = item[0]; - let value = item[1]; - result.push(` ${name}="${escapeAttribute(value)}"`); - } - return result.join(""); -} - -function escapeAttribute(value) { - if (! value) { - return ""; - } - value = value.replace(/&/g, "&"); - value = value.replace(/"/g, """); - return value; -} - class AbstractShot { constructor(backend, id, attrs) { - this.clearDirty(); attrs = attrs || {}; assert((/^[a-zA-Z0-9]+\/[a-z0-9\.-]+$/).test(id), "Bad ID (should be alphanumeric):", JSON.stringify(id)); this._backend = backend; this._id = id; this.url = attrs.url; this.docTitle = attrs.docTitle || null; - this.ogTitle = attrs.ogTitle || null; this.userTitle = attrs.userTitle || null; this.createdDate = attrs.createdDate || Date.now(); - this.createdDevice = attrs.createdDevice || null; this.favicon = attrs.favicon || null; - this.body = attrs.body || null; - this.head = attrs.head || null; - this.htmlAttrs = attrs.htmlAttrs || null; - this.bodyAttrs = attrs.bodyAttrs || null; - this.headAttrs = attrs.headAttrs || null; - this._comments = []; - if (attrs.comments) { - this._comments = attrs.comments.map( - (json) => new this.Comment(json)); - } - this.hashtags = attrs.hashtags || null; this.siteName = attrs.siteName || null; this.images = []; if (attrs.images) { this.images = attrs.images.map( (json) => new this.Image(json)); } - this.readable = null; - if (attrs.readable) { - this.readable = new this.Readable(attrs.readable); - } - this.deviceId = attrs.deviceId || null; this.openGraph = attrs.openGraph || null; this.twitterCard = attrs.twitterCard || null; this.documentSize = attrs.documentSize || null; this.fullScreenThumbnail = attrs.fullScreenThumbnail || null; - this.isPublic = attrs.isPublic === undefined || attrs.isPublic === null ? null : !! attrs.isPublic; - this.showPage = attrs.showPage || false; this.abTests = attrs.abTests || null; - this.resources = attrs.resources || {}; this._clips = {}; if (attrs.clips) { for (let clipId in attrs.clips) { @@ -259,8 +201,6 @@ class AbstractShot { assert(attrs.id === this.id); } } - // Reset all the dirty items that were unnecessarily set: - this.clearDirty(); } /** Update any and all attributes in the json object, with deep updating @@ -299,20 +239,6 @@ class AbstractShot { } - _dirty(name) { - this._dirtyItems[name] = true; - } - - _dirtyClip(clipId) { - this._dirtyClips[clipId] = true; - } - - /** Clears the dirty attribute checking (e.g., to use after save) */ - clearDirty() { - this._dirtyItems = {}; - this._dirtyClips = {}; - } - /** Returns a JSON version of this shot */ asJson() { let result = {}; @@ -346,79 +272,6 @@ class AbstractShot { return result; } - /** Returns a JSON version of any dirty attributes of this object */ - dirtyJson() { - var result = {}; - for (let attr in this._dirtyItems) { - let val = this[attr]; - if (val && val.asJson) { - val = val.asJson(); - } - result[attr] = val; - } - for (let clipId of this._dirtyClips) { - if (! result.clips) { - result.clips = {}; - } - var clip = this.getClip(clipId); - if (clip) { - result.clips[clipId] = clip.asJson(); - } else { - result.clips[clipId] = null; - } - } - return result; - } - - staticHtml(options) { - options = options || ""; - let head = this.head; - let body = this.body; - let rewriter = (html) => { - if (! html) { - return html; - } - let keys = Object.keys(this.resources); - if (! keys.length) { - return html; - } - for (let key of keys) { - if (key.search(/^[a-zA-Z0-9.\-]+$/) === -1) { - console.warn("Bad resource name:", key); - return; - } - } - let re = new RegExp(keys.join("|"), "g"); - let newHtml = html.replace(re, (match) => { - return options.rewriteLinks(match, this.resources[match]); - }); - newHtml = newHtml.replace(/"data:text\/html;base64,([^"]*)"/g, (match, group) => { - let html = this.atob(group); - html = rewriter(html); - let link = this.btoa(html); - return `"data:text/html;base64,${link}"`; - }); - return newHtml; - }; - if (options.rewriteLinks && this.resources) { - head = rewriter(head); - body = rewriter(body); - } - return `<!DOCTYPE html> -<html${formatAttributes(this.htmlAttrs)}> -<head${formatAttributes(this.headAttrs)}> -<meta charset="UTF-8"> -${options.addHead || ""} -<base href="${escapeAttribute(this.url)}"> -${head} -</head> -<body${formatAttributes(this.bodyAttrs)}> -${body} -${options.addBody || ""} -</body> -</html>`; - } - get backend() { return this._backend; } @@ -432,7 +285,6 @@ ${options.addBody || ""} } set url(val) { assert(val && isUrl(val), "Bad URL:", val); - this._dirty("url"); this._url = val; } @@ -487,57 +339,19 @@ ${options.addBody || ""} return this.backend + "/oembed?url=" + encodeURIComponent(this.viewUrl); } - get createdDevice() { - return this._createdDevice; - } - set createdDevice(val) { - assert(val === null || (typeof val == "string" && val), "Bad createdDevice:", val); - this._dirty("createdDevice"); - this._createdDevice = val; - } - get docTitle() { return this._title; } set docTitle(val) { assert(val === null || typeof val == "string", "Bad docTitle:", val); - this._dirty("docTitle"); this._title = val; } - get comments() { - // Kind of a simulation of a read-only array: - // (because writes are ignored) - return this._comments.slice(); - } - addComment(json) { - let comment = new this.Comment(json); - this._comments.push(comment); - this._dirty("comments"); - } - updateComment(index, json) { - let comment = new this._shot.Comment(json); - this._comments[index] = comment; - this._dirty("comments"); - } - // FIXME: no delete, nor comment editing - - // FIXME: deprecate - get ogTitle() { - return this._ogTitle; - } - set ogTitle(val) { - assert(val === null || typeof val == "string", "Bad ogTitle:", val); - this._dirty("ogTitle"); - this._ogTitle = val; - } - get openGraph() { return this._openGraph || null; } set openGraph(val) { assert(val === null || typeof val == "object", "Bad openGraph:", val); - this._dirty("openGraph"); if (val) { assert(checkObject(val, [], this._OPENGRAPH_PROPERTIES), "Bad attr to openGraph:", Object.keys(val)); this._openGraph = val; @@ -551,7 +365,6 @@ ${options.addBody || ""} } set twitterCard(val) { assert(val === null || typeof val == "object", "Bad twitterCard:", val); - this._dirty("twitterCard"); if (val) { assert(checkObject(val, [], this._TWITTERCARD_PROPERTIES), "Bad attr to twitterCard:", Object.keys(val)); this._twitterCard = val; @@ -565,14 +378,14 @@ ${options.addBody || ""} } set userTitle(val) { assert(val === null || typeof val == "string", "Bad userTitle:", val); - this._dirty("userTitle"); this._userTitle = val; } get title() { // FIXME: we shouldn't support both openGraph.title and ogTitle let ogTitle = this.openGraph && this.openGraph.title; - let title = this.userTitle || this.ogTitle || ogTitle || this.docTitle || this.url; + let twitterTitle = this.twitterCard && this.twitterCard.title; + let title = this.userTitle || ogTitle || twitterTitle || this.docTitle || this.url; if (Array.isArray(title)) { title = title[0]; } @@ -584,7 +397,6 @@ ${options.addBody || ""} } set createdDate(val) { assert(val === null || typeof val == "number", "Bad createdDate:", val); - this._dirty("createdDate"); this._createdDate = val; } @@ -596,36 +408,9 @@ ${options.addBody || ""} if (val) { val = resolveUrl(this.url, val); } - this._dirty("favicon"); this._favicon = val; } - get hashtags() { - return this._hashtags || []; - } - set hashtags(val) { - assert (val === null || Array.isArray(val), ".hashtags must be an array:", val, typeof val); - if (val) { - val.forEach( - (v) => assert(typeof v == "string", "hashtags array may only contain strings:", v)); - } - this._dirty("hashtags"); - this._hashtags = val; - } - - get readable() { - return this._readable; - } - set readable(val) { - assert(typeof val == "object" || ! val, "Bad Shot readable:", val); - if (! val) { - this._readable = val; - } else { - this._readable = new this.Readable(val); - } - this._dirty("readable"); - } - clipNames() { let names = Object.getOwnPropertyNames(this._clips); names.sort(function (a, b) { @@ -643,14 +428,12 @@ ${options.addBody || ""} } setClip(name, val) { let clip = new this.Clip(this, name, val); - this._dirtyClip(name); this._clips[name] = clip; } delClip(name) { if (! this._clips[name]) { throw new Error("No existing clip with id: " + name); } - this._dirtyClip(name); delete this._clips[name]; } biggestClipSortOrder() { @@ -674,67 +457,9 @@ ${options.addBody || ""} } set siteName(val) { assert(typeof val == "string" || ! val); - this._dirty("siteName"); this._siteName = val; } - get head() { - return this._head; - } - set head(val) { - assert(typeof val == "string" || ! val, "Bad head:", val); - this._dirty("head"); - this._head = val; - } - - get body() { - return this._body; - } - set body(val) { - assert(typeof val == "string" || ! val, "Bad body:", val); - this._dirty("body"); - this._body = val; - } - - get bodyAttrs() { - return this._bodyAttrs; - } - set bodyAttrs(val) { - if (! val) { - this._bodyAttrs = null; - } else { - assert(isAttributePairs(val), "Bad bodyAttrs:", val); - this._bodyAttrs = val; - } - this._dirty("bodyAttrs"); - } - - get htmlAttrs() { - return this._htmlAttrs; - } - set htmlAttrs(val) { - if (! val) { - this._htmlAttrs = null; - } else { - assert(isAttributePairs(val), "Bad htmlAttrs:", val); - this._htmlAttrs = val; - } - this._dirty("htmlAttrs"); - } - - get headAttrs() { - return this._headAttrs; - } - set headAttrs(val) { - if (! val) { - this._headAttrs = null; - } else { - assert(isAttributePairs(val), "Bad headAttrs:", val); - this._headAttrs = val; - } - this._dirty("headAttrs"); - } - get deviceId() { return this._deviceId; } @@ -742,7 +467,6 @@ ${options.addBody || ""} assert(typeof val == "string" || ! val); val = val || null; this._deviceId = val; - this._dirty("deviceId"); } get documentSize() { @@ -758,7 +482,6 @@ ${options.addBody || ""} } else { this._documentSize = null; } - this._dirty("documentSize"); } get fullScreenThumbnail() { @@ -772,44 +495,6 @@ ${options.addBody || ""} } else { this._fullScreenThumbnail = null; } - this._dirty("fullScreenThumbnail"); - } - - get isPublic() { - return this._isPublic; - } - set isPublic(val) { - assert(val === null || val === false || val === true, "isPublic should be true/false/null, not:", typeof val, val, JSON.stringify(val)); - this._isPublic = val; - this._dirty("isPublic"); - } - - get showPage() { - return this._showPage; - } - set showPage(val) { - assert(val === true || val === false, "showPage should true or false"); - this._showPage = val; - this._dirty("showPage"); - } - - get resources() { - return this._resources; - } - set resources(val) { - if (val === null || val === undefined) { - this._resources = null; - this._dirty("resources"); - return; - } - assert(typeof val == "object", "resources should be an object, not:", typeof val); - for (let key in val) { - assert(key.search(/^[a-zA-Z0-9\.\-]+$/) === 0, "Bad resource name: " + key); - let obj = val[key]; - assert(checkObject(obj, ['url', 'tag'], ['elId', 'attr', 'hash', 'rel']), "Invalid resource " + key + ": " + JSON.stringify(obj)); - } - this._resources = val; - this._dirty("resources"); } get abTests() { @@ -818,7 +503,6 @@ ${options.addBody || ""} set abTests(val) { if (val === null || val === undefined) { this._abTests = null; - this._dirty("abTests"); return; } assert(typeof val == "object", "abTests should be an object, not:", typeof val); @@ -832,19 +516,19 @@ ${options.addBody || ""} } AbstractShot.prototype.REGULAR_ATTRS = (` -deviceId url docTitle ogTitle userTitle createdDate createdDevice favicon -comments hashtags images readable head body htmlAttrs bodyAttrs -headAttrs siteName openGraph twitterCard documentSize -fullScreenThumbnail isPublic resources showPage abTests +deviceId url docTitle userTitle createdDate favicon images +siteName openGraph twitterCard documentSize +fullScreenThumbnail abTests `).split(/\s+/g); // Attributes that will be accepted in the constructor, but ignored/dropped AbstractShot.prototype.DEPRECATED_ATTRS = (` -microdata history +microdata history ogTitle createdDevice head body htmlAttrs bodyAttrs headAttrs +readable hashtags comments showPage isPublic resources `).split(/\s+/g); AbstractShot.prototype.RECALL_ATTRS = (` -deviceId url docTitle ogTitle userTitle createdDate createdDevice favicon +deviceId url docTitle userTitle createdDate favicon openGraph twitterCard images fullScreenThumbnail `).split(/\s+/g); @@ -863,36 +547,13 @@ card site title description image player player:width player:height player:stream player:stream:content_type `).split(/\s+/g); -/** Represents one comment, on a clip or shot */ -class _Comment { - // FIXME: either we have to notify the shot of updates, or make - // this read-only (as a result this is read-only *but not enforced*) - constructor(json) { - assert(checkObject(json, ["user", "createdDate", "text"], ["hidden", "flagged"]), "Bad attrs for Comment:", Object.keys(json)); - assert(typeof json.user == "string" && json.user, "Bad Comment user:", json.user); - this.user = json.user; - assert(typeof json.createdDate == "number", "Bad Comment createdDate:", json.createdDate); - this.createdDate = json.createdDate; - assert(typeof json.text == "string", "Bad Comment text:", json.text); - this.text = json.text; - this.hidden = !! json.hidden; - this.flagged = !! json.flagged; - } - - asJson() { - return jsonify(this, ["user", "createdDate", "text"], ["hidden", "flagged"]); - } -} - -AbstractShot.prototype.Comment = _Comment; - /** Represents one found image in the document (not a clip) */ class _Image { // FIXME: either we have to notify the shot of updates, or make // this read-only constructor(json) { assert(typeof json === "object", "Clip Image given a non-object", json); - assert(checkObject(json, ["url"], ["dimensions", "isReadable", "title", "alt"]), "Bad attrs for Image:", Object.keys(json)); + assert(checkObject(json, ["url"], ["dimensions", "title", "alt"]), "Bad attrs for Image:", Object.keys(json)); assert(isUrl(json.url), "Bad Image url:", json.url); this.url = json.url; assert((! json.dimensions) || @@ -903,48 +564,20 @@ class _Image { this.title = json.title; assert(typeof json.alt == "string" || ! json.alt, "Bad Image alt:", json.alt); this.alt = json.alt; - this.isReadable = !! json.isReadable; } asJson() { - return jsonify(this, ["url"], ["dimensions", "isReadable"]); + return jsonify(this, ["url"], ["dimensions"]); } } AbstractShot.prototype.Image = _Image; -/** Represents the readable representation of the page that the Readability library returns */ -class _Readable { - // FIXME: either we have to notify the shot of updates, or make - // this read-only - constructor(json) { - assert(checkObject(json, ["content"], ["title", "byline", "dir", "length", "excerpt", "readableIds"]), "Bad attrs for Readable:", Object.keys(json)); - assert(typeof json.title == "string" || ! json.title, "Bad Readable title:", json.title); - this.title = json.title; - assert(typeof json.byline == "string" || ! json.byline, "Bad Readable byline:", json.byline); - this.byline = json.byline; - this.dir = json.dir; - assert(typeof json.content == "string" && json.content, "Bad Readable content:", json.content); - this.content = json.content; - assert(typeof json.length == "number" || ! json.length, "Bad Readable length:", json.length); - this.length = json.length; - assert(typeof json.excerpt == "string" || ! json.excerpt, "Bad Readable excerpt:", json.excerpt); - this.excerpt = json.excerpt; - } - - asJson() { - return jsonify(this, ["content"], ["title", "byline", "dir", "length", "excerpt"]); - } -} - -AbstractShot.prototype.Readable = _Readable; - /** Represents a clip, either a text or image clip */ class _Clip { constructor(shot, id, json) { this._shot = shot; - this._initialized = false; - assert(checkObject(json, ["createdDate"], ["sortOrder", "image", "text", "comments"]), "Bad attrs for Clip:", Object.keys(json)); + assert(checkObject(json, ["createdDate", "image"], ["sortOrder"]), "Bad attrs for Clip:", Object.keys(json)); assert(typeof id == "string" && id, "Bad Clip id:", id); this._id = id; this.createdDate = json.createdDate; @@ -955,73 +588,26 @@ class _Clip { let biggestOrder = shot.biggestClipSortOrder(); this.sortOrder = biggestOrder + 100; } - assert(! (json.image && json.text), "Clip cannot have both .image and .text", Object.keys(json)); - if (json.image) { - this.image = json.image; - } else if (json.text) { - this.text = json.text; - } else { - assert(false, "No .image or .text"); - } - if (json.comments) { - this._comments = json.comments.map( - (commentJson) => new shot.Comment(commentJson)); - } else { - this._comments = []; - } - // From here after we track dirty attributes: - this._initialized = true; + this.image = json.image; } toString() { - let s = `[Shot Clip id=${this.id} sortOrder=${this.sortOrder}`; - if (this.image) { - s += ` image ${this.image.dimensions.x}x${this.image.dimensions.y}]`; - } else { - s += ` text length ${this.text.text.length}]`; - } - return s; - } - - _dirty(property) { - if (this._initialized) { - this._shot._dirtyClip(this.id); - } + return `[Shot Clip id=${this.id} sortOrder=${this.sortOrder} image ${this.image.dimensions.x}x${this.image.dimensions.y}]`; } asJson() { - var result = jsonify(this, ["createdDate"], ["sortOrder", "image", "text"]); - if (this.comments.length) { - result.comments = this.comments.map( - (comment) => comment.asJson()); - } - return result; + return jsonify(this, ["createdDate"], ["sortOrder", "image"]); } get id() { return this._id; } - get comments() { - return this._comments.slice(); - } - addComment(json) { - let comment = new this._shot.Comment(json); - this._comments.push(comment); - this._dirty("comments"); - } - updateComment(index, json) { - let comment = new this._shot.Comment(json); - this._comments[index] = comment; - this._dirty("comments"); - } - get createdDate() { return this._createdDate; } set createdDate(val) { assert(typeof val == "number" || ! val, "Bad Clip createdDate:", val); - this._dirty("createdDate"); this._createdDate = val; } @@ -1031,7 +617,6 @@ class _Clip { set image(image) { if (! image) { this._image = undefined; - this._dirty("image"); return; } assert(checkObject(image, ["url"], ["dimensions", "text", "location", "captureType"]), "Bad attrs for Clip Image:", Object.keys(image)); @@ -1059,7 +644,6 @@ class _Clip { "Bad Clip image element location:", image.location); } assert(! this._text, "Clip with .image cannot have .text", JSON.stringify(this._text)); - this._dirty("image"); this._image = image; } @@ -1069,39 +653,11 @@ class _Clip { } } - get text() { - return this._text; - } - set text(text) { - if (! text) { - this._text = undefined; - this._dirty("text"); - return; - } - assert(checkObject(text, ["html"], ["text", "location"]), "Bad attrs in Clip text:", Object.keys(text)); - assert(typeof text.html == "string" && text.html, "Bad Clip text html:", text.html); - assert(typeof text.text == "string" || ! text.text, "Bad Clip text text:", text.text); - if (text.location) { - assert( - typeof text.location.contextStart == "string" && - typeof text.location.contextEnd == "string" && - typeof text.location.selectionStart == "string" && - typeof text.location.selectionEnd == "string" && - typeof text.location.startOffset == "number" && - typeof text.location.endOffset == "number", - "Bad Clip text location:", JSON.stringify(text.location)); - } - assert(! this._image, "Clip with .text cannot have .image", JSON.stringify(this._image)); - this._dirty("text"); - this._text = text; - } - get sortOrder() { return this._sortOrder || null; } set sortOrder(val) { assert(typeof val == "number" || ! val, "Bad Clip sortOrder:", val); - this._dirty("sortOrder"); this._sortOrder = val; }