Extract hashtags out of project descriptions and store them in an ind…

…exed JSONB array

env.sample

@@ -21,3 +21,5 @@ export NEW_RELIC_HOME='./node_modules/newrelic'
 export NEW_RELIC_LICENSE_KEY=newrelic license key
 export NEW_RELIC_APP_NAME=api.webmaker.org
 export NEW_RELIC_LOG_LEVEL=info
+
+export FEATURED_TAGS=""

migrations/03_add_hashtags.sql

@@ -0,0 +1,8 @@
+BEGIN;
+
+ALTER TABLE projects
+ADD COLUMN metadata jsonb DEFAULT '{tags:[]}'::jsonb
+
+CREATE INDEX ON projects USING gin ((metadata -> 'tags'));
+
+COMMIT;

package.json

@@ -6,7 +6,7 @@
   "scripts": {
     "start": "node server.js",
     "migrate": "node scripts/migrate",
-    "test": "npm run test:api && npm run lint",
+    "test": "npm run lint && npm run test:api",
     "test:api": "npm run test:setup && lab -t 100 -c --verbose --colors --assert code --timeout 5000",
     "test:setup": "npm run test:droptables && npm run test:createtables && npm run test:insertdata",
     "test:droptables": "node scripts/drop-tables",

scripts/create-tables.sql

@@ -26,6 +26,7 @@ CREATE TABLE IF NOT EXISTS "projects"
   deleted_at timestamp DEFAULT NULL,
   thumbnail jsonb NOT NULL DEFAULT '{}'::JSONB,
   description text NOT NULL DEFAULT '',
+  metadata jsonb DEFAULT '{"tags":[]}'::jsonb,
   CONSTRAINT projects_id_pk PRIMARY KEY (id)
 );
 
@@ -65,6 +66,7 @@ CREATE INDEX deleted_at_remixed_from_idx on projects (deleted_at, remixed_from);
 CREATE INDEX deleted_at_featured_idx on projects (deleted_at, featured);
 CREATE INDEX project_id_deleted_at_idx ON pages (project_id, deleted_at);
 CREATE INDEX deleted_at_page_id_idx ON elements (deleted_at, page_id);
+CREATE INDEX ON projects USING gin ((metadata -> 'tags'));
 
 /* Triggers */
 CREATE OR REPLACE FUNCTION update_updated_at()

scripts/test-data.sql

@@ -46,26 +46,26 @@ BEGIN
   bobby_id := get_user_id('bobby_testing');
 
   -- create some test project data
-  INSERT INTO projects (user_id, remixed_From, version, title, thumbnail, featured) VALUES
-    (chris_id, NULL, 'test', 'test_project_1', empty_json, FALSE),
-    (chris_id, NULL, 'test', 'test_project_2', thumb, TRUE),
-    (jon_id, NULL, 'test', 'test_project_3', empty_json, FALSE),
-    (jon_id, NULL, 'test', 'test_project_4', thumb, TRUE),
-    (andrew_id, NULL, 'test', 'test_project_5', empty_json, FALSE),
-    (andrew_id, NULL, 'test', 'test_project_6', thumb, TRUE),
-    (chris_id, NULL, 'test', 'test_project_7', empty_json, TRUE),
-    (andrew_id, NULL, 'test', 'test_project_8', empty_json, TRUE),
-    (jon_id, NULL, 'test', 'test_project_9', empty_json, TRUE),
-    (andrew_id, NULL, 'test', 'test_project_10', empty_json, TRUE),
-    (atique_id, NULL, 'test', 'test_project_11', empty_json, TRUE),
-    (atique_id, NULL, 'test', 'test_project_12', empty_json, TRUE),
-    (ina_id, NULL, 'test', 'test_project_13', empty_json, TRUE),
-    (ina_id, NULL, 'test', 'test_project_14', empty_json, TRUE),
-    (ina_id, NULL, 'test', 'test_project_15', empty_json, TRUE),
-    (bobby_id, NULL, 'test', 'test_project_16', empty_json, TRUE),
-    (bobby_id, NULL, 'test', 'test_project_17', empty_json, TRUE),
-    (bobby_id, NULL, 'test', 'test_project_18', empty_json, TRUE),
-    (bobby_id, NULL, 'test', 'test_project_19', empty_json, TRUE);
+  INSERT INTO projects (user_id, remixed_From, version, title, thumbnail, featured, description, metadata) VALUES
+    (chris_id, NULL, 'test', 'test_project_1', empty_json, FALSE, '', '{}'),
+    (chris_id, NULL, 'test', 'test_project_2', thumb, TRUE, '', '{}'),
+    (jon_id, NULL, 'test', 'test_project_3', empty_json, FALSE, '', '{}'),
+    (jon_id, NULL, 'test', 'test_project_4', thumb, TRUE, '', '{}'),
+    (andrew_id, NULL, 'test', 'test_project_5', empty_json, FALSE, '', '{}'),
+    (andrew_id, NULL, 'test', 'test_project_6', thumb, TRUE, '', '{}'),
+    (chris_id, NULL, 'test', 'test_project_7', empty_json, TRUE, '', '{}'),
+    (andrew_id, NULL, 'test', 'test_project_8', empty_json, TRUE, '', '{}'),
+    (jon_id, NULL, 'test', 'test_project_9', empty_json, TRUE, '', '{}'),
+    (andrew_id, NULL, 'test', 'test_project_10', empty_json, TRUE, '', '{}'),
+    (atique_id, NULL, 'test', 'test_project_11', empty_json, TRUE, '', '{}'),
+    (atique_id, NULL, 'test', 'test_project_12', empty_json, TRUE, '', '{}'),
+    (ina_id, NULL, 'test', 'test_project_13', empty_json, TRUE, '#mozilla', '{"tags":["mozilla"]}'),
+    (ina_id, NULL, 'test', 'test_project_14', empty_json, TRUE, '', '{}'),
+    (ina_id, NULL, 'test', 'test_project_15', empty_json, TRUE, '', '{}'),
+    (bobby_id, NULL, 'test', 'test_project_16', empty_json, TRUE, '', '{}'),
+    (bobby_id, NULL, 'test', 'test_project_17', empty_json, TRUE, '', '{}'),
+    (bobby_id, NULL, 'test', 'test_project_18', empty_json, TRUE, '', '{}'),
+    (bobby_id, NULL, 'test', 'test_project_19', empty_json, TRUE, '', '{}');
 
   -- create some remixes
   INSERT INTO projects (user_id, remixed_From, version, title, thumbnail) VALUES

services/api/handlers/projects.js

@@ -9,7 +9,8 @@ exports.post = {
         request.server.methods.utils.version(),
         request.payload.title,
         JSON.stringify(request.payload.thumbnail),
-        request.payload.description
+        request.payload.description,
+        request.pre.tags
       ],
       function(err, result) {
         if ( err ) {
@@ -57,6 +58,31 @@ exports.post = {
 };
 
 exports.get = {
+  withTag: function(request, reply) {
+    request.server.methods.projects.findWithTags(
+      [ JSON.stringify([request.params.tag]),
+        request.query.count,
+        request.pre.offset
+      ],
+      function(err, result) {
+        if ( err ) {
+          return reply(err);
+        }
+
+        reply({
+          status: 'success',
+          projects: result.rows.map(function(project) {
+            return request.server.methods.utils.formatProject(project);
+          })
+        });
+      }
+    );
+  },
+  featuredTags: function(request, reply) {
+    reply({
+      featured: request.server.featuredTags()
+    });
+  },
   oneShallow: function(request, reply) {
     request.server.methods.projects.findOneShallow(
       [ request.params.project ],
@@ -214,6 +240,7 @@ exports.patch = {
       [
         request.pre.title,
         request.pre.description,
+        request.pre.tags,
         request.params.project
       ],
       function(err, result) {

services/api/lib/bulk.js

@@ -10,13 +10,19 @@ exports.register = function(server, options, done) {
           server.methods.utils.version(),
           data.title,
           JSON.stringify(data.thumbnail),
-          data.description
+          data.description,
+          {
+            tags: server.methods.utils.extractTags(data.description)
+          }
         ];
       },
       update: function(userId, data) {
         return [
           data.title,
           data.description,
+          {
+            tags: server.methods.utils.extractTags(data.description)
+          },
           data.id
         ];
       },

services/api/lib/postgre.js

@@ -150,6 +150,10 @@ module.exports = function (pg) {
         executeQuery(queries.projects.findOneShallow, values, done);
       }, {});
 
+      server.method('projects.findWithTags', function(values, done) {
+        executeQuery(queries.projects.findWithTag, values, done);
+      }, {});
+
       server.method('projects.create', function(values, done) {
         var project;
         var page;
@@ -208,7 +212,8 @@ module.exports = function (pg) {
               server.methods.utils.version(),
               dataToRemix.title,
               dataToRemix.thumbnail,
-              dataToRemix.description
+              dataToRemix.description,
+              dataToRemix.metadata
             ]
           );
         }).then(function(result) {

services/api/lib/prerequisites.js

@@ -246,6 +246,21 @@ exports.setDescription = {
       return reply(request.payload.description);
     }
 
-    reply(request.pre.project.description);
+    if (request.pre.project) {
+      return reply(request.pre.project.description);
+    }
+
+    reply('');
+  }
+};
+
+exports.extractTags = {
+  assign: 'tags',
+  method: function(request, reply) {
+    var metadata = {
+      tags: request.server.methods.utils.extractTags(request.pre.description)
+    };
+
+    reply(JSON.stringify(metadata));
   }
 };

services/api/lib/queries.js

@@ -8,7 +8,8 @@ var projectCols = [
   "projects.updated_at",
   "projects.thumbnail",
   "projects.user_id",
-  "projects.description"
+  "projects.description",
+  "projects.metadata->'tags' as tags"
 ].join(", ");
 
 var projectUserCols = [
@@ -55,6 +56,7 @@ var remixCols = [
   "projects.title AS project_title",
   "projects.thumbnail AS project_thumbnail",
   "projects.description AS project_description",
+  "projects.metadata AS project_metadata",
   "pages.id AS page_id",
   "pages.project_id AS project_id",
   "pages.x AS page_x",
@@ -91,9 +93,10 @@ module.exports = {
   projects: {
     // Create project
     // Params:user_id bigint, remixed_from bigint, version varchar, title varchar, thumbnail jsonb
-    create: "INSERT INTO projects (user_id, remixed_from, version, title, thumbnail, description)" +
-      " VALUES ($1, $2, $3, $4, $5, $6) RETURNING id, user_id, remixed_from, version, title, featured," +
-      " created_at, updated_at, thumbnail, description;",
+
+    create: "INSERT INTO projects (user_id, remixed_from, version, title, thumbnail, description, metadata)" +
+      " VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING id, user_id, remixed_from, version, title, featured," +
+      " created_at, updated_at, thumbnail, description, metadata->'tags' AS tags;",
 
     // Find all projects, sorted by created_at DESC
     // Params: limit integer, offset integer
@@ -121,6 +124,10 @@ module.exports = {
       " FROM projects INNER JOIN users ON users.id = projects.user_id " +
       " WHERE projects.deleted_at IS NULL AND projects.id = $1;",
 
+    findWithTag: "SELECT " + projectUserCols + " FROM projects INNER JOIN users ON users.id = projects.user_id" +
+      " WHERE projects.deleted_at is NULL AND projects.deleted_at IS NULL AND projects.metadata->'tags' @> $1::jsonb" +
+      " ORDER BY created_at DESC LIMIT $2 OFFSET $3",
+
     // Retrieve data in a project for remixing (joins pages and elements)
     // params: project_id bigint
     findDataForRemix: "SELECT " + remixCols + " FROM projects LEFT OUTER JOIN pages ON projects.id = " +
@@ -129,9 +136,9 @@ module.exports = {
 
     // Update project
     // Params title varchar, project_id bigint
-    update: "UPDATE projects SET (title, description) = ($1, $2) WHERE deleted_at IS NULL" +
-      " AND id = $3 RETURNING id, user_id, remixed_from, version, title, featured," +
-      " created_at, updated_at, thumbnail, description;",
+    update: "UPDATE projects SET (title, description, metadata) = ($1, $2, $3) WHERE deleted_at IS NULL" +
+      " AND id = $4 RETURNING id, user_id, remixed_from, version, title, featured, description, metadata" +
+      " created_at, updated_at, thumbnail;",
 
     // Update project thumbnail
     // Params thumbnail jsonb, project_id bigint
@@ -256,5 +263,19 @@ module.exports = {
       "INNER JOIN pages " +
       "ON pages.id = elements.page_id " +
       "WHERE projects.id = pages.project_id)"
+  },
+  tags: {
+    autocomplete: `
+      SELECT tags FROM (
+        SELECT DISTINCT tags, COUNT(tags)
+        FROM (
+          SELECT jsonb_array_elements_text(metadata->'tags') AS tags
+          FROM projects jsonb_to_recordset(x)
+        ) metadata
+        GROUP BY tags
+      ) distinct_tags
+      WHERE tags LIKE $1
+      ORDER BY count DESC;
+    `
   }
 };

services/api/lib/utils.js

@@ -29,6 +29,7 @@ function formatProject(project) {
   formatted.remixed_from = project.remixed_from;
   formatted.featured = project.featured;
   formatted.description = project.description;
+  formatted.tags = project.tags;
 
   // Updates/creates don't include user data
   if ( project.username ) {
@@ -163,24 +164,50 @@ function formatRemixData(rows) {
     title: rows[0].project_title,
     thumbnail: rows[0].project_thumbnail,
     description: rows[0].project_description,
+    metadata: rows[0].project_metadata,
     pages: pages
   };
 }
 
+function extractTags(description) {
+  var tagRegex = /#([A-Za-z\d]+)/g;
+  var tags = [];
+  var tag;
+
+  if (description && description.length) {
+    while ((tag = tagRegex.exec(description)) !== null) {
+      if (tags.indexOf(tag[1]) === -1) {
+        tags.push(tag[1]);
+      }
+    }
+  }
+
+  return tags;
+}
+
 var API_VERSION = process.env.API_VERSION;
 
 function version() {
   return API_VERSION;
 }
 
+var FEATURED_TAGS = process.env.FEATURED_TAGS;
+FEATURED_TAGS = FEATURED_TAGS.split(' ').map((tag) => tag.trim());
+
+function featuredTags() {
+  return FEATURED_TAGS;
+}
+
 exports.register = function(server, options, done) {
   server.method('utils.formatUser', formatUser, { callback: false });
   server.method('utils.formatProject', formatProject, { callback: false });
   server.method('utils.formatPage', formatPage, { callback: false });
   server.method('utils.formatPages', formatPages, { callback: false });
   server.method('utils.formatElement', formatElement, { callback: false });
   server.method('utils.formatRemixData', formatRemixData, { callback: false });
+  server.method('utils.extractTags', extractTags, { callback: false });
   server.method('utils.version', version, { callback: false });
+  server.decorate('server', 'featuredTags', featuredTags);
   done();
 };
 

services/api/routes/authenticated.js

@@ -146,13 +146,15 @@ var routes = [
           thumbnail: Joi.object().keys({
             320: Joi.string().optional()
           }).default({}),
-          description: Joi.string().max(100).default('')
+          description: Joi.string().max(140).default('')
         }
       },
       pre: [
         prerequisites.getUser,
         prerequisites.getTokenUser,
-        prerequisites.canCreate
+        prerequisites.canCreate,
+        prerequisites.setDescription,
+        prerequisites.extractTags
       ],
       cors: {
         methods: ['OPTIONS', 'POST', 'GET']
@@ -284,7 +286,7 @@ var routes = [
         payload: {
           title: Joi.string().max(250).optional(),
           thumbnail: Joi.object().optional(),
-          description: Joi.string().max(100).optional()
+          description: Joi.string().max(140).optional()
         }
       },
       pre: [
@@ -293,7 +295,8 @@ var routes = [
         prerequisites.getProject,
         prerequisites.canWrite,
         prerequisites.setTitle,
-        prerequisites.setDescription
+        prerequisites.setDescription,
+        prerequisites.extractTags
       ],
       cors: {
         methods: ['OPTIONS', 'GET', 'PATCH', 'DELETE']