WIP: elasticsearch

.env.example

@@ -12,6 +12,9 @@ DATABASE_URL=postgres://localhost:5432/republik
 # redis url, leave blank for default: 127.0.0.1:6379
 #REDIS_URL=
 
+# elasticsearch url, leave blank for default: localhost:9200
+#ELASTIC_URL=
+
 # url to republik-frontend
 # used by
 #  - auth to construct token links
@@ -121,6 +124,9 @@ AWS_SECRET_ACCESS_KEY=
 # log ratelimit every 15min to stdout
 #GITHUB_LOG_RATELIMIT=true
 
+# List of all potential GitHub organizations data my stem from (optional)
+#GITHUB_ORGS=organization-one,organization-two
+
 # checkout the README on how to get these
 GITHUB_LOGIN=
 GITHUB_APP_ID=

.gitignore

@@ -55,6 +55,9 @@ typings/
 # Yarn Integrity file
 .yarn-integrity
 
+# Mapped Docker Volumes
+docker-data/
+
 # dotenv environment variables file
 .env
 .test.env

.nvmrc

@@ -0,0 +1 @@
+10

.travis.yml

@@ -1,6 +1,4 @@
 language: node_js
-node_js:
-  - '10.2.1'
 cache:
   directories:
     - node_modules
@@ -10,6 +8,7 @@ notifications:
 addons:
   postgresql: "9.6"
 services:
+  - elasticsearch
   - postgresql
   - redis-server
 branches:
@@ -47,5 +46,11 @@ env:
 before_install:
   - curl -o- -L https://yarnpkg.com/install.sh | bash -s -- --version 1.7.0
   - export PATH=$HOME/.yarn/bin:$PATH
+  - curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.2.3.deb && sudo dpkg -i --force-confnew elasticsearch-6.2.3.deb && sudo service elasticsearch restart
+before_script:
+  # ElasticSearch takes few seconds to start, to make sure it is available
+  # when the build script runs add a small delay to your build script
+  # see https://docs.travis-ci.com/user/database-setup/#ElasticSearch
+  - sleep 10
 script:
   - cd $TEST_DIR && yarn run test:travis

docker-compose.republik-backend.yml

@@ -0,0 +1,34 @@
+version: "3.4"
+services:
+  republik-backend:
+    build:
+      context: ./servers/republik
+    image: republik-backend:latest
+    links:
+      - redis
+      - postgres
+    environment:
+      - DATABASE_URL=postgres://postgres@postgres:5432/republik
+      - REDIS_URL=redis:6379
+      - PUBLIC_URL=http://localhost:8080
+      - PUBLIC_WS_URL_BASE=ws://localhost:8080
+      - FRONTEND_BASE_URL
+      - PUBLIC_WS_URL_PATH
+      - SESSION_SECRET
+      - CORS_WHITELIST_URL
+      - SEND_MAILS
+      - MANDRILL_API_KEY
+      - DEFAULT_MAIL_FROM_NAME
+      - DEFAULT_MAIL_FROM_ADDRESS
+      - AUTH_MAIL_FROM_ADDRESS
+      - DISPLAY_AUTHOR_SECRET
+      - EXO_KEY
+      - EXO_SECRET
+      - S3BUCKET
+      - ASSETS_BASE_URL
+      - KEYCDN_API_KEY
+      - KEYCDN_ZONE_ID
+      - KEYCDN_ZONE_URL
+      - PHANTOMJSCLOUD_API_KEY
+    ports:
+        - 8080:3020

docker-compose.yml

@@ -1,50 +1,46 @@
 version: "3.4"
 services:
-  republik-backend:
-    build:
-      context: ./servers/republik
-    image: republik-backend:latest
-    links:
-      - redis
-      - postgres
+  elastic:
+    # in order to get elastic to work, see https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html#docker-cli-run-prod-mode
+    image: docker.elastic.co/elasticsearch/elasticsearch:6.2.3
+    container_name: elasticsearch
     environment:
-      - DATABASE_URL=postgres://postgres@postgres:5432/republik
-      - REDIS_URL=redis:6379
-      - PUBLIC_URL=http://localhost:8080
-      - PUBLIC_WS_URL_BASE=ws://localhost:8080
-      - FRONTEND_BASE_URL
-      - PUBLIC_WS_URL_PATH
-      - SESSION_SECRET
-      - CORS_WHITELIST_URL
-      - SEND_MAILS
-      - MANDRILL_API_KEY
-      - DEFAULT_MAIL_FROM_NAME
-      - DEFAULT_MAIL_FROM_ADDRESS
-      - AUTH_MAIL_FROM_ADDRESS
-      - DISPLAY_AUTHOR_SECRET
-      - EXO_KEY
-      - EXO_SECRET
-      - S3BUCKET
-      - ASSETS_BASE_URL
-      - KEYCDN_API_KEY
-      - KEYCDN_ZONE_ID
-      - KEYCDN_ZONE_URL
-      - PHANTOMJSCLOUD_API_KEY
+      - "ES_JAVA_OPTS=-Xms1024m -Xmx1024m"
+      - xpack.graph.enabled=false
+      - xpack.ml.enabled=false
+      - xpack.monitoring.enabled=false
+      - xpack.security.enabled=false
+      - xpack.watcher.enabled=false
+      - ELASTIC_PASSWORD=elastic
+      - "discovery.type=single-node"
+      - TAKE_FILE_OWNERSHIP=true
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    volumes:
+      - ./docker-data/elastic:/usr/share/elasticsearch/data
+    ports:
+      - 9200:9200
+
+  kibana:
+    image: docker.elastic.co/kibana/kibana:6.2.3
+    container_name: kibana
     ports:
-        - 8080:3020
+      - 5601:5601
+    depends_on:
+      -  elastic
 
   redis:
-    image: redis:4-alpine
+    image: redis:alpine
     ports:
-        - 6379:6379
+      - 6379:6379
+    volumes:
+      - ./docker-data/redis:/data
 
   postgres:
-    image: postgres:10-alpine
+    image: postgres:10
     volumes:
-      - postgres-data:/var/lib/postgresql
+      - ./docker-data/postgres:/var/lib/postgresql/data
     ports:
-        - 5432:5432
-
-
-volumes:
-  postgres-data:
+      - 5432:5432

package.json

@@ -55,6 +55,7 @@
     "commitmsg": "validate-commit-msg",
     "roleUser": "node packages/auth/script/roleUser",
     "roleUsers": "node packages/auth/script/roleUsers",
-    "pull:redis": "node servers/publikator/script/pullRedis.js"
+    "pull:redis": "node servers/publikator/script/pullRedis.js",
+    "pull:elasticsearch": "node packages/search/script/pullElasticsearch.js"
   }
 }

packages/assets/lib/Repo.js

@@ -5,7 +5,10 @@ const {
   upload: uploadS3,
   getHead
 } = require('./s3')
-const debug = require('debug')('assets')
+const Promise = require('bluebird')
+const debug = require('debug')('assets:lib:Repo')
+
+const CONCURRENCY = 2
 
 const {
   ASSETS_SERVER_BASE_URL,
@@ -22,46 +25,48 @@ const uploadImages = async (repoId, paths) => {
   const { githubRest } = await createGithubClients()
   const [owner, repo] = repoId.split('/')
 
-  await Promise.all(
-    paths.map( async (path) => {
-      // the filename is the blobSha
-      const blobSha = path
-        .split('/')
-        .pop()
-        .split('.')[0]
+  await Promise.map(paths, async path => {
+    // the filename is the blobSha
+    const blobSha = path
+      .split('/')
+      .pop()
+      .split('.')[0]
+
+    const s3Path = getS3PathForGithubPath(repoId, path)
 
-      const s3Path = getS3PathForGithubPath(repoId, path)
+    if (!await getHead({ bucket: AWS_S3_BUCKET, path: s3Path })) {
+      let result
+      let error
 
-      if(!await getHead({
-        bucket: AWS_S3_BUCKET,
-        path: s3Path
-      })) {
-        let result
-        let error
-        try {
-          result = await githubRest.gitdata.getBlob({
-            owner,
-            repo,
-            sha: blobSha
-          })
-        } catch(e) {
-          error = e
-          result = null
-        }
-        if (error || !result || !result.data || !result.data.content) {
-          console.error('getBlob failed for ', {owner, repo, blobSha, path, error})
-          throw new Error('getBlob failed for: '+ s3Path)
-        }
-        return uploadS3({
-          stream: Buffer.from(result.data.content, 'base64'),
-          path: s3Path,
-          bucket: AWS_S3_BUCKET
+      try {
+        result = await githubRest.gitdata.getBlob({
+          owner,
+          repo,
+          file_sha: blobSha
         })
-      } else {
-        debug('uploadImages exists %s', s3Path)
+      } catch (e) {
+        error = e
+        result = null
       }
-    })
-  )
+
+      if (error || !result || !result.data || !result.data.content) {
+        console.error(
+          'getBlob failed for ',
+          { owner, repo, blobSha, path, error }
+        )
+        throw new Error(`getBlob failed for: ${s3Path}`)
+      }
+
+      return uploadS3({
+        stream: Buffer.from(result.data.content, 'base64'),
+        path: s3Path,
+        bucket: AWS_S3_BUCKET
+      })
+    } else {
+      debug('uploadImages exists %s', s3Path)
+    }
+  },
+  { concurrency: CONCURRENCY })
 }
 
 module.exports = {

packages/assets/package.json

@@ -23,6 +23,7 @@
     "isomorphic-unfetch": "^2.0.0"
   },
   "dependencies": {
+    "bluebird": "^3.5.1",
     "aws-sdk": "^2.246.1",
     "check-env": "^1.3.0",
     "debug": "^3.1.0",

packages/base/express/graphql.js

@@ -8,6 +8,7 @@ const cookieParser = require('cookie-parser')
 const checkEnv = require('check-env')
 const { transformUser } = require('@orbiting/backend-modules-auth')
 const redis = require('../lib/redis')
+const elasticsearch = require('../lib/elastic')
 
 checkEnv([
   'PUBLIC_WS_URL_BASE',
@@ -33,7 +34,8 @@ module.exports = (
     pgdb,
     user,
     pubsub,
-    redis
+    redis,
+    elastic: elasticsearch.client()
   })
 
   const subscriptionServer = SubscriptionServer.create(

packages/base/lib/elastic.js

@@ -0,0 +1,11 @@
+const elasticsearch = require('elasticsearch')
+
+const {
+  ELASTIC_URL
+} = process.env
+
+module.exports.client = () => {
+  return new elasticsearch.Client({
+    host: ELASTIC_URL || 'localhost:9200'
+  })
+}

packages/base/package.json

@@ -36,6 +36,7 @@
     "cookie-parser": "^1.4.3",
     "cors": "^2.8.4",
     "d3-time-format": "^2.1.1",
+    "elasticsearch": "^14.2.2",
     "export-files": "^2.1.1",
     "express": "^4.16.3",
     "graphql-redis-subscriptions": "^1.4.0",

packages/documents/graphql/resolvers/Document.js

@@ -16,7 +16,7 @@ const { lib: { webp: {
 
 const shouldDeliverWebP = (argument = 'auto', req) => {
   if (argument === 'auto') {
-    return req.get('Accept').indexOf('image/webp') > -1
+    return req && req.get('Accept').indexOf('image/webp') > -1
   }
   return !!argument
 }

packages/documents/graphql/resolvers/_queries/document.js

@@ -2,10 +2,11 @@ const getDocuments = require('./documents')
 
 module.exports = async (_, args, context) => {
   const {
+    id,
     path,
     repoId
   } = args
 
-  return getDocuments(_, { path, repoId }, context)
+  return getDocuments(_, { id, path, repoId }, context)
     .then(docCon => docCon.nodes[0])
 }