diff --git a/.build_scripts/deploy.conf b/.build_scripts/deploy.conf deleted file mode 100644 index ebb7549..0000000 --- a/.build_scripts/deploy.conf +++ /dev/null @@ -1,8 +0,0 @@ -[stage] -host 52.11.15.48 -user ubuntu -repo https://github.com/hotosm/oam-uploader-api.git -path /home/ubuntu/oam-uploader-api -ref origin/develop -needs_tty yes -post-deploy .build_scripts/post-deploy.sh diff --git a/.build_scripts/docker/run.sh b/.build_scripts/docker/run.sh deleted file mode 100755 index 49d850f..0000000 --- a/.build_scripts/docker/run.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash - -# This script runs the docker container for this application, by: -# - passing in the environment variables defined in local.env, if it exists -# - overriding the above with any relevant variables in the current environment -# (see config.js) -# - exposing the $PORT (default: 3000), on which the app listens -# - setting the container to use the host OS's networking stack (better -# performance, worse isolation) -# - mounting the current directory--which should be the project root--as -# `/local`. -# -# The intent of the last item is that the app can be setup (npm install) and -# run directly from its location on the host OS, but using the container as its -# environment to make dependency management trivial. - -if [ -f local.env ] ; then - ENVFILE="--env-file local.env" -else - ENVFILE="" -fi - -if [[ -t 0 || -p /dev/stdin ]] ; then - echo "Running in tty mode." - MODE=-it -else - MODE= -fi - -PORT=${PORT:-3000} - -ARGS_LENGTH=$(($#-1)) -ARGS=${@:1:$ARGS_LENGTH} -COMMAND=${@:$((ARGS_LENGTH+1)):1} - -exec docker run $MODE --rm \ - -p $PORT \ - $ENVFILE \ - -e OAM_TEST \ - -e PORT \ - -e HOST \ - -e OIN_BUCKET \ - -e DBURI \ - -e DBURI_TEST \ - -e MAX_WORKERS \ - -e ADMIN_PASSWORD \ - -e ADMIN_USERNAME \ - -e AWS_SECRET_KEY_ID \ - -e AWS_SECRET_ACCESS_KEY \ - -e AWS_REGION \ - --net=\"host\" \ - -v $(pwd):/app \ - $ARGS \ - oam-uploader-api $COMMAND diff --git a/.build_scripts/docker/start.sh b/.build_scripts/docker/start.sh deleted file mode 100755 index 10776c6..0000000 --- a/.build_scripts/docker/start.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -# Run the server inside docker. - -npm start diff --git a/.build_scripts/docker/test.sh b/.build_scripts/docker/test.sh deleted file mode 100755 index b5d1789..0000000 --- a/.build_scripts/docker/test.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -# Run the tests inside docker. - -npm test diff --git a/.build_scripts/post-deploy.sh b/.build_scripts/post-deploy.sh deleted file mode 100755 index 0a6ee17..0000000 --- a/.build_scripts/post-deploy.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -.build_scripts/docker/run.sh /install.sh -sudo stop oam-uploader-api -sudo start oam-uploader-api diff --git a/.build_scripts/reset-db.js b/.build_scripts/reset-db.js deleted file mode 100755 index aadfa3e..0000000 --- a/.build_scripts/reset-db.js +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env node -'use strict'; -var readline = require('readline'); -var MongoClient = require('mongodb').MongoClient; -var dbUri = require('../config').dbUri; - -var rl = readline.createInterface({ - input: process.stdin, - output: process.stdout -}); - -rl.question("Type 'yes' if you really want to clear the database: ", function (answer) { - if (answer === 'yes') { - console.log('Okay, doing it!'); - MongoClient.connect(dbUri, function (err, connection) { - if (err) throw err; - console.log('Connected to db.'); - console.log('Dropping workers'); - connection.dropCollection('workers') - .then(function () { - console.log('Dropping uploads'); - return connection.dropCollection('uploads'); - }) - .then(function () { - console.log('Dropping tokens'); - return connection.dropCollection('tokens'); - }) - .then(function () { - console.log('Dropping images'); - return connection.dropCollection('images'); - }) - .then(function () { - console.log('Done.'); - connection.close(); - }) - .catch(function (err) { - console.error(err); - connection.close(); - }); - }); - } else { - console.log('Abort! Whew!'); - } - rl.close(); -}); diff --git a/.build_scripts/upstart.conf b/.build_scripts/upstart.conf deleted file mode 100644 index 1b54294..0000000 --- a/.build_scripts/upstart.conf +++ /dev/null @@ -1,30 +0,0 @@ -#!upstart -description "oam-uploader-api server" -author "anand" - -start on filesystem and started docker -stop on runlevel [!2345] - -env USER=ubuntu -env OAM_UPLOADER_DIR=/home/ubuntu/oam-uploader-api/current - -script - echo $$ > /var/run/oam-uploader-api.pid - cd $OAM_UPLOADER_DIR - exec sudo -u $USER $OAM_UPLOADER_DIR/.build_scripts/docker/run.sh --name=oam-uploader-api /start.sh >> /var/log/oam-uploader-api.sys.log 2>&1 -end script - -pre-start script - echo "[`date -u +%Y-%m-%dT%T.%3NZ`] (sys) Starting" >> /var/log/oam-uploader-api.sys.log -end script - -pre-stop script - rm /var/run/oam-uploader-api.pid - echo "[`date -u +%Y-%m-%dT%T.%3NZ`] (sys) Stopping" >> /var/log/oam-uploader-api.sys.log - docker stop oam-uploader-api -end script - -post-stop script - rm /var/run/oam-uploader-api.pid - echo "[`date -u +%Y-%m-%dT%T.%3NZ`] (sys) Stopped" >> /var/log/oam-uploader-api.sys.log -end script diff --git a/.dockerignore b/.dockerignore index 3b2a527..717eb47 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,4 @@ +Dockerfile node_modules newrelic_agent.log npm-debug.log diff --git a/.env.sample b/.env.sample index 68aad59..6d739ee 100644 --- a/.env.sample +++ b/.env.sample @@ -1,2 +1,38 @@ -LOCALOAM_VOLUME=~/hotosm/data -HOST_PREFIX=http://localhost:4999 +# Environment variables. +# +# * This is a canonical refrence. *All* ENV vars needed in *all* +# environments should be placed here. +# * A `.env` is a requirement of *all* environments, including tests. +# * Copy `.env.sample` to `.env` for it to be automatically sourced. +# * See /config.js for individual explanations of each variable. +# * Dont use "" quotes + +# API +PORT=4000 +HOST=0.0.0.0 +COOKIE_PASSWORD=123abc +DB_URI=mongodb://localhost:27017/oam-api + +# OAM-specific +# OAM_DEBUG=true|false should be used on the command line +OIN_REGISTER_URL=http://localhost:8080/catalog/fixtures/oin-buckets-testing.json +OIN_BUCKET=oin-hotosm-staging +UPLOAD_BUCKET=oam-uploader-staging-temp +# Run quickly like this only for non-production +CRON_TIME=* * * * * +GDRIVE_KEY=abc123 +MAX_WORKERS=1 +TILER_BASE_URL=http://tiles.openaerialmap.org + +# Third party +NEW_RELIC_LICENSE_KEY=123abc +SENDGRID_API_KEY=123 +SENDGRID_FROM=info@hotosm.org +AWS_ACCESS_KEY_ID=123 +AWS_SECRET_ACCESS_KEY=abc +AWS_REGION=us-east-1 + +# To be deprecated +ADMIN_USERNAME=admin +ADMIN_PASSWORD=admin +SECRET_TOKEN=insecuretoken diff --git a/.eslintrc b/.eslintrc index 79298b0..aa0ffcd 100644 --- a/.eslintrc +++ b/.eslintrc @@ -1,7 +1,8 @@ { "extends": ["standard", "standard-react"], "env": { - "es6": true + "es6": true, + "mocha": true }, "rules": { "semi": [2, "always"], diff --git a/Dockerfile b/Dockerfile index cd07a11..5f33f92 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,34 +1,11 @@ FROM quay.io/hotosm/oam-dynamic-tiler-tools -ENV NPM_CONFIG_LOGLEVEL warn - -# Install app dependencies -ADD package.json /tmp/package.json -RUN cd /tmp && npm install - -# Create app directory -RUN mkdir -p /usr/src/app && cp -a /tmp/node_modules /usr/src/app -WORKDIR /usr/src/app - -# Bundle app source -ADD . /usr/src/app - -# Go ahead and install nodemon for convenience while developing -RUN \ - npm install -g nodemon \ - && rm -rf /root/.npm - -# Set TMPDIR environment variable -ENV TMPDIR /tmp +RUN mkdir -p /app COPY . /app - WORKDIR /app -RUN \ - npm install \ - && rm -rf /root/.npm +RUN npm install EXPOSE 4000 -CMD [ "node", "index.js" ] - +CMD ["./node_modules/.bin/nf", "start"] diff --git a/Procfile b/Procfile index 1da0cd6..dafd8d8 100644 --- a/Procfile +++ b/Procfile @@ -1 +1,2 @@ web: node index.js +worker: node catalog-worker.js diff --git a/README.md b/README.md index b6ce4c3..27aee99 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ The steps below will walk you through setting up your own instance of the oam-ca ### Install Project Dependencies - [MongoDB](https://www.mongodb.org/) -- [Node.js](https://nodejs.org/) (v4.5.x) +- [Node.js](https://nodejs.org/) ### Install Application Dependencies @@ -42,183 +42,21 @@ npm install ### Usage -#### Starting the database: - -``` -mongod -``` - -The database is responsible for storing metadata about the imagery and analytics. - #### Starting the API: ``` node index.js ``` -The API exposes endpoints used to access information form the system via a RESTful interface. - #### Starting the background worker: ``` node worker.js ``` -The worker process runs on a schedule (every 5 minutes by default) and checks for new data, update database when it finds anything to add. - -### Environment Variables - -- `OAM_DEBUG` - Debug mode `true` or `false` (default) -- `AWS_SECRET_KEY_ID` - AWS secret key id for reading OIN buckets -- `AWS_SECRET_ACCESS_KEY` - AWS secret access key for reading OIN buckets -- `OIN_REGISTER_URL` - URL to register file containing location of imagery buckets -- `DBURI` - MongoDB connection url -- `CRON_TIME` - A valid cron string (default is every 5 minutes) for the worker schedule -- `SECRET_TOKEN` - The token used for post requests to `/tms` endpoint -- `NEW_RELIC_LICENSE_KEY` - Your New Relic API monitoring license key - -- `NODE_ENV` - Node environment. When in production should be set to `production`, otherwise can be ignored. -- `PORT` - The port to listen on (Default to 4000). -- `HOST` - The hostname or ip address (Default to 0.0.0.0). -- `COOKIE_PASSWORD` - Password used for cookie encoding. Should be at least 32 characters long. IMPORTANT to change the default one for production. -- `AWS_ACCESS_KEY_ID` - AWS secret key id for reading `OIN_BUCKET`. -- `AWS_SECRET_ACCESS_KEY` - AWS secret access key for reading `OIN_BUCKET`. -- `AWS_REGION` - AWS region of `OIN_BUCKET` (Default to us-west-2). -- `SENDGRID_API_KEY` - Sendgrid API key, for sending notification emails. -- `SENDGRID_FROM` - Email address from which to send notification emails (Default to info@hotosm.org). -- `OIN_BUCKET` - The OIN bucket that will receive the uploads (Default to oam-uploader). -- `DBURI` - MongoDB connection url (Default to mongodb://localhost/oam-uploader) -- `DBURI_TEST` - MongoDB connection url for tests. Will be loaded when `NODE_ENV` is `test`. It's not needed for production. (Default to mongodb://localhost/oam-uploader-test) -- `ADMIN_USERNAME` - Username to access the [Token Management](https://github.com/hotosm/oam-uploader-admin) panel. -- `ADMIN_PASSWORD` - Password to access the [Token Management](https://github.com/hotosm/oam-uploader-admin) panel. -- `GDRIVE_KEY` - Google Api key. Needed to use the upload from google drive functionality. -- `GDAL_TRANSLATE_BIN` - Full path to the gdal bin (Default to /usr/bin/gdal_translate) -- `MAX_WORKERS` - Max number of workers used to process the uploads (Default to 1) -- `NEW_RELIC_LICENSE_KEY` - New relic license key. -- `TILER_BASE_URL` - Base URL for dynamic TMS/WMTS endpoints. Defaults to `http://tiles.openaerialmap.org`. - -If you are running a local OAM bucket, here are additional environment variables you can configure (more information in the Docker > Local Indexing section) -- `HOST_PREFIX`: - Used by the localoam service to construct URIs that point to the images (default: `http://localoam`) -- `LOCAL_OAM_BUCKET`: - Used by the localoam service as the location of the HTTP server (default is a docker context service name: `http://localoam:4999`) - -For development purposes, `NEW_RELIC_LICENSE_KEY` can be omitted. Although the system will work some functionality will not be available and errors may be triggered. - - -## Endpoints and Parameters - -More API documentation can be found at: [tbd]. - -### Available Endpoints - -- `/meta` -XGET -- `/meta/add` -XPOST -- `/tms` -XGET -- `/tms` -XPOST -- `/analytics` -XGET - -### POST parameters for `/tms`: - -To add/update `/tms` endpoint, the following JSON format should be used: - -```json -{ - "uri": "http://example.com/tms_uri", - "images": [ - { - "uuid": "http://example.com/image_uri.tif" - } - ] -} -``` -*Note that the `/tms` endpoint requires authenticated access.* - -### Search parameters for `/meta`: - -#### bbox: - -- format: `?bbox=[lon_min],[lat_min],[lon_max],[lat_max]` -- example: `/meta?bbox=-66.15966796875,46.45678142812658,-65.63232421875,46.126556302418514` - -#### title: - -- format: `?title=string` -- example: `/meta?title=sometitle` - - -#### provider: - -- format: `?provider=string` -- example: `/meta?provider=someprovider` - -#### GSD (Resolution): - -- format: `?gsd_from=value&gsd_to=value` -- example: `/meta?gsd_from=0.005&gsd_to=20` - -*Note that gsd_from and gsd_to can be used on their own. Values should be provided in meters.* - -#### has tiled service?: - -- format: `?has_tiled` -- example: `/meta?has_tiled` - -#### page: - -- format: `?page=number` -- example: `/meta?page=2` - -#### date: -- format: `?acquisition_from=date&acquisition_to=date` -- example: `/meta?acquisition_from=2015-04-10&acquisition_to=2015-05-01` - -*Note that acquisition_from and acquisition_to can be used on their own.* - -#### limit: - -default is `100`. - -- format: `?limit=number` -- example: `/meta?limit=1000` - -#### sorting and ordering: - -- format: `?order_by=property&sort=asc|desc` -- example: `/meta?order_by=acquisition_start&sort=asc` - -*Note that `sort` and `order_by` are required together and one alone will not be recognized. Default is to show higher resolution and newer imagery first.* - - -### Documentation - -The documentation for the different endpoints can be generated by running: -``` -npm run docs -``` -This will compile the documentation and place it inside `docs`. The docs site can then be run by any web server. - -The documentation is also automatically built and deployed by [Travis CI](https://travis-ci.org/) whenever a Pull Request is merged to the production branch (in this case `master`). The deployment is done to `gh-pages`. (http://hotosm.github.io/oam-catalog/) - -### Deployment - -Changes to the `master` branch are automatically deployed via Travis to https://oam-catalog.herokuapp.com. - -## Docker -To package the app as a container: - -- Ensure that docker and docker-compose are installed -- Copy `local.sample.env` to `local.env` and fill the values according to the instructions above. -- Run `docker-compose up`. - -The app will be available at `http://localhost:4000` - - -### Local indexing -For local indexing, an alternative `docker-compose` configuration file can be used. This will launch an additional service, a file server for a folder with JSON metadata files that fit the OAM spec. The metadata files need a `HOST_PREFIX` (such as the default `http://localhost:4999`) that points to the location of the `localoam` service running on the network. +The worker currently has 2 main functions: polling buckets for new imagery and processing uploads. -- Copy `.env.sample` to `.env` and add the path to the volume you want to index as well as the `HOST_PREFIX` -- Copy `local.sample.env` to `local.env` and fill the values according to the instructions above. -- Run `docker-compose -f docker-compose-local.yml up` +### API Documentation -## License -Oam Catalog is licensed under **BSD 3-Clause License**, see the [LICENSE](LICENSE) file for more details. +See: https://hotosm.github.io/oam-catalog/ diff --git a/about.md b/about.md deleted file mode 100644 index b311094..0000000 --- a/about.md +++ /dev/null @@ -1,9 +0,0 @@ -## Getting Started - -OpenAerialMap Catalog is an application programming interface (API) that serves metadata about available images and map layers. The API provides access to all imagery metadata information. The API provides endpoints for lists of images available, map layer services available, as well as simple analytics. - -More information on [installation and running a stand-alone instance](https://github.com/hotosm/oam-catalog) of the API can be found on Github. - -See [status.openaerialmap.org](https://oam-status.herokuapp.com/) for more information on system status. - -
diff --git a/app.json b/app.json deleted file mode 100644 index 260cd0c..0000000 --- a/app.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "name": "oam-catalog", - "description": "A catalog for OpenAerialMap imagery", - "repository": "https://github.com/hotosm/oam-catalog", - "keywords": [ - "node", - "REST", - "satellite", - "imagery" - ], - "addons": [ - "mongolab", - "newrelic" - ], - "env": { - "OAM_DEBUG": { - "description": "Debug mode true or false (default)", - "required": false - }, - "AWS_SECRET_KEY_ID": { - "description": "AWS secret key id for reading OIN buckets", - "required": true - }, - "AWS_SECRET_ACCESS_KEY": { - "description": "AWS secret access key for reading OIN buckets", - "required": true - }, - "DBURI": { - "description": "MongoDB connection url", - "required": true - }, - "SECRET_TOKEN": { - "description": "The token used for post requests to /tms endpoint", - "required": true, - "generator": "secret" - } - } -} diff --git a/worker.js b/catalog-worker.js similarity index 68% rename from worker.js rename to catalog-worker.js index 078fc18..e8e15a3 100644 --- a/worker.js +++ b/catalog-worker.js @@ -1,11 +1,13 @@ 'use strict'; -require('envloader').load(); +console.log('Starting catalog worker...'); + +require('dotenv').config(); var _ = require('lodash'); var S3 = require('./services/s3.js'); -var LocalOAM = require('./services/localoam.js'); var async = require('async'); +var config = require('./config'); var analytics = require('./controllers/analytics.js'); var Meta = require('./models/meta.js'); // Replace mongoose's deprecated promise library (mpromise) with bluebird @@ -14,9 +16,6 @@ mongoose.Promise = require('bluebird'); var request = require('request'); var cron = require('node-cron'); -var registerURL = process.env.OIN_REGISTER_URL || 'https://raw.githubusercontent.com/openimagerynetwork/oin-register/master/master.json'; -var localRegisterURL = process.env.LOCAL_REGISTER_URL || ''; - var db = mongoose.connection; var consoleLog = function (err, msg) { @@ -35,7 +34,7 @@ var consoleLog = function (err, msg) { var getBucketList = function (cb) { request.get({ json: true, - uri: registerURL + uri: config.oinRegisterURL }, function (err, res, remoteData) { if (err) { return cb(err); @@ -49,31 +48,7 @@ var getBucketList = function (cb) { return node.locations; }); buckets = _.flatten(buckets); - - if (localRegisterURL.length > 0) { - request.get({ - json: true, - uri: localRegisterURL - }, function (err, res, localData) { - if (err || (res.statusCode !== 200)) { - // There was an error retrieving local buckets - // Return remote buckets - cb(null, buckets); - return console.error('Unable to get local register list.'); - } else { - // We got local buckets, merge them with the buckets object - var localBuckets = _.map(localData.nodes, function (node) { - return node.locations; - }); - - localBuckets = _.flatten(localBuckets); - buckets = _.union(localBuckets, buckets); - cb(null, buckets); - } - }); - } else { - cb(null, buckets); - } + cb(null, buckets); }); }; @@ -131,7 +106,7 @@ var readBuckets = function (tasks) { * The main function to get the registered buckets, read them and update metadata */ var getListAndReadBuckets = function () { - // Start of by getting the last time the system was updated. + // Start off by getting the last time the system was updated. analytics.getLastUpdateTime(function (err, lastSystemUpdate) { if (err) { return console.error(err); @@ -145,17 +120,14 @@ var getListAndReadBuckets = function () { // Generate array of tasks to run in parallel var tasks = _.map(buckets, function (bucket) { - if (bucket.type === 's3') { - return function (done) { - var s3 = new S3(null, null, bucket.bucket_name); + return function (done) { + if (bucket.type === 's3') { + var s3 = new S3(bucket.bucket_name); s3.readBucket(lastSystemUpdate, consoleLog, done); - }; - } else if (bucket.type === 'localoam') { - return function (done) { - var localOAM = new LocalOAM(bucket.url); - localOAM.readBucket(lastSystemUpdate, done); - }; - } + } else { + console.error('Unknown bucket type: ' + bucket.type); + } + }; }); // Read the buckets and store metadata @@ -165,8 +137,12 @@ var getListAndReadBuckets = function () { }; // Kick it all off -cron.schedule(process.env.CRON_TIME || '*/5 * * * *', // Run every 5 minutes +cron.schedule( + config.cronTime, function () { + console.log( + 'Running a catalog worker (cron time: ' + config.cronTime + ')' + ); getListAndReadBuckets(); } ); diff --git a/config.js b/config.js index 7e2522e..dde9579 100644 --- a/config.js +++ b/config.js @@ -1,27 +1,62 @@ -var xtend = require('xtend'); +// Canonical configuration settings. +// All configuration and 'magic numbers' should be defined here. +// Strive to set all ENV values through `.env`. -// Default configuration options used by the app. Most of these can be -// overriden by environment variables (see below). -var defaults = { - host: '0.0.0.0', // cosmetic - port: 4000, // port to listen on - dbUri: process.env.NODE_ENV === 'test' ? 'mongodb://localhost/oam-uploader-test' : 'mongodb://localhost/oam-uploader', // the mongodb database uri (mongodb://user:pass@host:port/db) - adminPassword: 'admin', // the administrator username - adminUsername: 'admin', // the administrator password - oinBucket: 'oam-uploader', // name of the OpenImageryNetwork bucket to which imagery should be uploaded - uploadBucket: 'oam-uploader-temp', // name of the bucket for temporary storage for direct uploads - thumbnailSize: 300, // (very) approximate thumbnail size, in kilobytes - maxWorkers: 1, // the maximum number of workers - sendgridApiKey: null, // sendgrid API key, for sending notification emails - sendgridFrom: 'info@hotosm.org', // the email address from which to send notification emails - gdriveKey: null, +// Amendments for test env +if (process.env.NODE_ENV === 'test') { + process.env.DB_URI = 'mongodb://localhost:27017/oam-api-test'; + process.env.OAM_DEBUG = process.env.OAM_DEBUG || 'false'; +} + +const config = { + debug: process.env.OAM_DEBUG, + + // Server setup + host: process.env.HOST, + port: process.env.PORT, + + // DB connection + dbUri: process.env.DB_URI, + + // OIN bucket in which imagery ultimately lives + oinBucket: process.env.OIN_BUCKET, + // OIN bucket for temporary storage of direct uploads + uploadBucket: process.env.UPLOAD_BUCKET, + // How often to poll OIN buckets for new imagery + cronTime: process.env.CRON_TIME, + // Location of master record of OIN buckets to poll + oinRegisterUrl: process.env.OIN_REGISTER_URL, + // Approximate thumbnail size, in kilobytes + thumbnailSize: 300, + + // Base URL for accessing the slippy map TMS endpoint for imagery. This is + // the entrypoint for using the Dynamic Tiler to serve imagery. + tilerBaseUrl: process.env.TILER_BASE_URL, + + // Maximum number of workers. + maxWorkers: process.env.MAX_WORKERS, + + // AWS credendtials + awsKey: process.env.AWS_ACCESS_KEY_ID, + awsSecret: process.env.AWS_SECRET_ACCESS_KEY, + awsRegion: process.env.AWS_REGION, + + // Google drive is an available method for uploading imagery + gdriveKey: process.env.GDRIVE_KEY, + + // Sendgrid sends emails + sendgridApiKey: process.env.SENDGRID_API_KEY, + sendgridFrom: process.env.SENDGRID_FROM, emailNotification: { subject: '[ OAM Uploader ] Imagery upload submitted', text: 'Your upload has been successfully submitted and is now being ' + 'processed. You can check on the status of the upload at ' + 'http://upload.openaerialmap.org/#/status/{UPLOAD_ID}.' }, - cookiePassword: '3b296ce42ec560abeabaef', + + // For encrypting/decrypting cookie data + cookiePassword: process.env.COOKIE_PASSWORD, + logOptions: { opsInterval: 3000, reporters: [{ @@ -35,34 +70,16 @@ var defaults = { } }] }, - tilerBaseUrl: 'http://tiles.openaerialmap.org' -}; -// Environment variable overrides -var environment = { - port: process.env.PORT, - host: process.env.HOST, - oinBucket: process.env.OIN_BUCKET, - uploadBucket: process.env.UPLOAD_BUCKET, - dbUri: process.env.NODE_ENV === 'test' ? process.env.DBURI_TEST : process.env.DBURI, - maxWorkers: process.env.MAX_WORKERS, + // TODO: Deprecate the following once user accounts have been implemented. + // Credentials for Uploader Admin adminPassword: process.env.ADMIN_PASSWORD, adminUsername: process.env.ADMIN_USERNAME, - sendgridApiKey: process.env.SENDGRID_API_KEY, - sendgridFrom: process.env.SENDGRID_FROM, - gdriveKey: process.env.GDRIVE_KEY, - tilerBaseUrl: process.env.TILER_BASE_URL, - cookiePassword: process.env.COOKIE_PASSWORD + // Token to access POST requests to /tms and /meta + tokenForPostRequests: process.env.SECRET_TOKEN }; -var config = xtend(defaults); -for (var k in environment) { - if (typeof environment[k] !== 'undefined') { - config[k] = environment[k]; - } -} - -// override json.stringify behavior so we don't accidentally log secret keys +// Override json.stringify behavior so we don't accidentally log secret keys config.toJSON = function () { return '[ hidden ]'; }; diff --git a/controllers/analytics.js b/controllers/analytics.js index af0d460..5c8de0a 100644 --- a/controllers/analytics.js +++ b/controllers/analytics.js @@ -69,10 +69,3 @@ module.exports.getLastUpdateTime = function (cb) { }); }; -/** - * The date callback that returns the error and date. - * - * @callback responseCallback - * @param {error} err - The error message - * @param {int} date - The last updated date - */ diff --git a/docker-compose-local.yml b/docker-compose-local.yml deleted file mode 100644 index 6576651..0000000 --- a/docker-compose-local.yml +++ /dev/null @@ -1,32 +0,0 @@ -version: '2' -services: - localoam: - build: localoam - ports: - - "4999:4999" - volumes: - - ${LOCALOAM_VOLUME}:/usr/src/app/data - environment: - - HOST=localoam - - HOST_PREFIX=${HOST_PREFIX} - mongo: - image: mongo - worker: - build: . - environment: - - DBURI=mongodb://mongo/oam-uploader - - LOCAL_REGISTER_URL=http://localoam:4999/master.json - env_file: - - local.env - command: ['node', 'worker.js'] - web: - build: . - environment: - - DBURI=mongodb://mongo/oam-uploader - env_file: - - local.env - ports: - - "4000:4000" - depends_on: - - localoam - - mongo diff --git a/docker-compose.yml b/docker-compose.yml index e2362f9..c08832e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,22 +1,8 @@ version: '2' services: - mongo: - image: mongo - worker: + app: build: . - environment: - - DBURI=mongodb://mongo/oam-uploader - env_file: - - local.env - command: ['node', 'worker.js'] - web: - build: . - environment: - - DBURI=mongodb://mongo/oam-uploader - env_file: - - local.env - ports: - - "4000:4000" - depends_on: - - mongo + image: oam-api + restart: always + diff --git a/index.js b/index.js index beb92a7..199abcb 100644 --- a/index.js +++ b/index.js @@ -1,12 +1,15 @@ 'use strict'; -require('envloader').load(); +require('dotenv').config(); require('newrelic'); + +var config = require('./config'); + var Conn = require('./services/db.js'); var Server = require('./services/server.js'); var db = new Conn(); db.start(); -var server = new Server(process.env.PORT || 4000); +var server = new Server(config.port); server.start(); diff --git a/local.sample.env b/local.sample.env deleted file mode 100644 index 9940fa3..0000000 --- a/local.sample.env +++ /dev/null @@ -1,30 +0,0 @@ -export AWS_SECRET_KEY_ID= -export AWS_SECRET_ACCESS_KEY= -export AWS_REGION=us-west-2 -export OIN_REGISTER_URL= -export SECRET_TOKEN= -export NEW_RELIC_LICENSE_KEY= - -export PORT=4000 -export HOST=0.0.0.0 -export COOKIE_PASSWORD=3b296ce42ec560abeabaef - -export SENDGRID_API_KEY= -export SENDGRID_FROM= - -export OIN_BUCKET=oam-uploader -export UPLOAD_BUCKET=oam-uploader-temp - -export DBURI=mongodb://localhost/oam-uploader -export DBURI_TEST=mongodb://localhost/oam-uploader-test - -export ADMIN_USERNAME=admin -export ADMIN_PASSWORD=admin - -export GDRIVE_KEY= - -export GDAL_TRANSLATE_BIN=/usr/bin/gdal_translate - -export MAX_WORKERS=1 - -export NEW_RELIC_LICENSE_KEY= diff --git a/localoam/.dockerignore b/localoam/.dockerignore deleted file mode 100644 index ab7b1b6..0000000 --- a/localoam/.dockerignore +++ /dev/null @@ -1,5 +0,0 @@ -node_modules -newrelic_agent.log -npm-debug.log -.travis.yml -.build_scripts \ No newline at end of file diff --git a/localoam/.gitignore b/localoam/.gitignore deleted file mode 100644 index 1269488..0000000 --- a/localoam/.gitignore +++ /dev/null @@ -1 +0,0 @@ -data diff --git a/localoam/Dockerfile b/localoam/Dockerfile deleted file mode 100644 index 4781de7..0000000 --- a/localoam/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM node:4 -ENV NPM_CONFIG_LOGLEVEL warn - -# Install app dependencies -ADD package.json /tmp/package.json -RUN cd /tmp && npm install - -# Create app directory -RUN mkdir -p /usr/src/app && cp -a /tmp/node_modules /usr/src/app -WORKDIR /usr/src/app - -# Bundle app source -ADD . /usr/src/app - -CMD [ "node", "index.js" ] diff --git a/localoam/index.js b/localoam/index.js deleted file mode 100644 index 8092d50..0000000 --- a/localoam/index.js +++ /dev/null @@ -1,93 +0,0 @@ -var http = require('http'); -var filed = require('filed'); -var glob = require('glob'); -var es = require('event-stream'); -var responseStream = require('response-stream'); -var oppressor = require('oppressor'); -var path = require('path'); -var fs = require('fs'); - -var HOST = process.env.HOST_PREFIX || 'http://localoam'; -var LOCAL_BUCKET_URL = process.env.LOCAL_BUCKET_URL || 'http://localoam:4999'; - -var master = { - 'nodes': [ - { - 'locations': [ - { - 'type': 'localoam', - 'url': LOCAL_BUCKET_URL - } - ] - } - ] -}; - -/** - * Helper method, stringifies JSON and sets the - * content header type to JSON - */ -function sendJSON (json, res) { - res.setHeader('Content-Type', 'application/json'); - if (typeof json !== 'string') { - json = JSON.stringify(json); - } - res.end(json); -} - -/** - * Given a JSON metadata filename - * Create a tif filename based on the metadata filename and - * a HOST prefix - */ -function insertMetadata (url) { - return responseStream(es.mapSync(function (s) { - if (s === 'Not Found') { - return s; - } - var json = JSON.parse(String(s)); - var uuid = HOST + url.slice(0, url.lastIndexOf('.')) + '.tif'; - json.uuid = uuid; - json.meta_uri = HOST + url; - - return JSON.stringify(json); - })); -} - -var server = http.createServer(function (req, res) { - var url = req.url; - var dataRoot = process.env.DIRECTORY || process.argv[2] || path.join(__dirname, 'data'); - if (url === '/list') { - // Return all the JSON metadata files with their last modified date - glob(dataRoot + '/**/*.json', {ignore: [dataRoot + '/node_modules/**', dataRoot + '/package.json']}, function (err, files) { - if (err) { - console.error(err); - res.writeHead('500', {'Content-Type': 'text/plain'}); - return res.end('Server Error'); - } else { - var records = files.map(function (file) { - // Get last modified time for each file - var stats = fs.statSync(file); - return { - file: '/' + path.relative(dataRoot, file), - lastModified: stats.mtime - }; - }); - } - return sendJSON(records, res); - }); - } else if (url === '/master.json') { - return sendJSON(master, res); - } else if (url.slice(url.lastIndexOf('.') + 1) === 'json') { - // We're asking for a metadata file - return filed(path.join(dataRoot, url)) - .pipe(insertMetadata(url)) - .pipe(oppressor(req)) - .pipe(res); - } else { - // else return the file - return filed(path.join(dataRoot, url)).pipe(res); - } -}); - -server.listen(process.env.PORT || 4999); diff --git a/localoam/package.json b/localoam/package.json deleted file mode 100644 index 2308d96..0000000 --- a/localoam/package.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "name": "localoam", - "version": "0.0.1", - "description": "", - "main": "index.js", - "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" - }, - "keywords": [], - "author": "Development Seed", - "license": "BSD-3-Clause", - "dependencies": { - "event-stream": "^3.3.4", - "filed": "^0.1.0", - "glob": "^7.1.1", - "oppressor": "0.0.1", - "response-stream": "0.0.0" - } -} diff --git a/models/upload.js b/models/upload.js index c127362..9424f84 100644 --- a/models/upload.js +++ b/models/upload.js @@ -1,3 +1,5 @@ +// Not a mongoose model :/ + var Joi = require('joi'); var infoSchema = Joi.object().keys({ diff --git a/package.json b/package.json index db7a5ff..a19735d 100644 --- a/package.json +++ b/package.json @@ -4,9 +4,9 @@ "description": "A catalog for Open Aerial Map Imagery", "main": "index.js", "scripts": { - "test": "npm run lint && npm run catalog_tests && npm run uploader_tests", - "lint": "eslint . --ext .js", "docs": "apidoc -i routes/ -o docs/", + "lint": "eslint . --ext .js", + "test": "npm run lint && npm run catalog_tests && npm run uploader_tests", "uploader_tests": "lab -T test/uploader/babel.js test/uploader/test__*.js", "catalog_tests": "mocha test/catalog" }, @@ -23,15 +23,15 @@ "dependencies": { "amazon-s3-url-signer": "^0.1.0", "async": "2.1.4", - "aws-sdk": "^2.1.44", "babel": "^5.8.21", "bluebird": "^3.4.1", "boom": "^2.8.0", "cron": "^1.2.1", - "envloader": "0.0.2", + "dotenv": "^4.0.0", "es6-promise": "^2.3.0", "es6-promisify": "^3.0.0", "exit-hook": "^1.1.1", + "foreman": "^2.0.0", "fs-extra": "^0.18.2", "good": "^6.3.0", "good-console": "^5.0.2", @@ -66,6 +66,7 @@ "eslint-plugin-promise": "^3.4.0", "eslint-plugin-react": "^6.7.1", "eslint-plugin-standard": "^2.0.1", + "http-server": "^0.10.0", "lab": "^5.15.0", "mocha": "^2.2.4", "omit-deep": "^0.1.2", diff --git a/plugins/authentication.js b/plugins/authentication.js index 821e75b..0ebb56d 100644 --- a/plugins/authentication.js +++ b/plugins/authentication.js @@ -36,8 +36,6 @@ var Authentication = { password: config.cookiePassword, cookie: 'oam-uploader-api', redirectTo: false, - // Change for production. - isSecure: process.env.NODE_ENV === 'production', validateFunc: validateUserCookie() }); diff --git a/plugins/workers.js b/plugins/workers.js index 1749989..35ea95e 100644 --- a/plugins/workers.js +++ b/plugins/workers.js @@ -16,6 +16,7 @@ module.exports = function register (server, options, next) { next(); function spawn () { + console.log('Running an upload worker'); var available = Object.keys(myWorkers); var workers = db.collection('workers'); server.log(['worker', 'debug'], 'Maybe spawn... available: ' + @@ -56,14 +57,14 @@ module.exports = function register (server, options, next) { function spawnWorker () { server.log(['worker', 'info'], 'Spawning worker.'); - var cp = fork(path.join(__dirname, '../worker')) - .on('message', function (msg) { - myWorkers[msg.workerId] = cp.pid; - myProcesses[cp.pid] = msg.workerId; - var id = msg.workerId.slice(0, 6); - var message = msg.message.length === 1 ? msg.message[0] : msg.message; - server.log(['worker', 'worker ' + id].concat(msg.tags), message); - }) + var cp = fork(path.join(__dirname, '../worker/index.js')) + .on('message', function (msg) { + myWorkers[msg.workerId] = cp.pid; + myProcesses[cp.pid] = msg.workerId; + var id = msg.workerId.slice(0, 6); + var message = msg.message.length === 1 ? msg.message[0] : msg.message; + server.log(['worker', 'worker ' + id].concat(msg.tags), message); + }) .on('exit', function (info) { server.log(['worker', 'debug'], 'Worker exited: ' + JSON.stringify(info)); delete myWorkers[myProcesses[cp.pid]]; diff --git a/routes/uploads.js b/routes/uploads.js index fafd101..c7bca9f 100644 --- a/routes/uploads.js +++ b/routes/uploads.js @@ -13,9 +13,9 @@ var config = require('../config'); var sendgrid = require('sendgrid')(config.sendgridApiKey); AWS.config = { - region: 'us-east-1', - accessKeyId: config.awsSecretKeyId, - secretAccessKey: config.awsSecretAccessKey + region: config.awsRegion, + accessKeyId: config.awsKey, + secretAccessKey: config.awsSecret }; function insertImages (db, scene, callback) { diff --git a/services/auth.js b/services/auth.js index 2d25726..261f75b 100644 --- a/services/auth.js +++ b/services/auth.js @@ -1,10 +1,13 @@ 'use strict'; +var config = require('../config'); + // This is a place holder with a basic token authentication -// More advanced authentiction will be added later +// More advanced authentiction will be added later. +// TODO: Deprecate once user accounts have been implemented. module.exports = function (token, callback) { - var secretToken = process.env.SECRET_TOKEN || 'insecuretoken'; + var secretToken = config.tokenForPostRequests; if (token === secretToken) { callback(null, true, {token: token}); diff --git a/services/db.js b/services/db.js index 7df395d..614b8b3 100644 --- a/services/db.js +++ b/services/db.js @@ -11,7 +11,7 @@ var Connection = function () { Connection.prototype.start = function (cb) { this.db.on('error', console.error.bind(console, 'connection error:')); this.db.once('open', function () { - console.log('Successfully connected to database.'); + console.log('Successfully connected to ' + config.dbUri); if (cb) { cb(); } diff --git a/services/localoam.js b/services/localoam.js deleted file mode 100644 index 6deda11..0000000 --- a/services/localoam.js +++ /dev/null @@ -1,66 +0,0 @@ -'use strict'; - -var meta = require('../controllers/meta.js'); -var path = require('path'); -var request = require('request'); - -/** -* LocalOAM Constructor that handles intractions with local OAM bucket server -* -* @constructor -* @param {String} bucket (optional) - Local OAM Bucket URL. Can be set by LOCAL_OAM_BUCKET env var. -*/ -var LocalOAM = function (bucket) { - this.params = { - bucket: bucket || process.env.LOCAL_OAM_BUCKET - }; -}; - -/** -* Read bucket method for LocalOAM. It reads the LocalOAM bucket and adds all the `.json` metadata to Meta model -* -* @param {finishedCallback} done - The callback that handles when reading is done -*/ - -LocalOAM.prototype.readBucket = function (lastSystemUpdate, done) { - console.info('--- Reading from bucket: ' + this.params.bucket + ' ---'); - - var self = this; - self.tasks = []; - - // List keys in local bucket - request({ - json: true, - uri: self.params.bucket + '/list' - }, function (err, response, payload) { - if (err) { return done(err); } - - // We have a successful response - if (response.statusCode === 200 && payload != null) { - payload.forEach(function (item) { - var format = path.extname(item.file); - - if (format === '.json') { - // Get the last time the metadata file was modified so we can determine - // if we need to update it. - var lastModified = item.LastModified; - var url = self.params.bucket + item.file; - var task = function (done) { - meta.addRemoteMeta(url, lastModified, lastSystemUpdate, done); - }; - self.tasks.push(task); - } - }); - done(null, self.tasks); - } - }); -}; - - /** - * The finished callback calls back to the worker with the list of tasks - * - * @callback finishedCallback - * @param {Error} err - The error - * @param {Function[]} tasks - The tasks - */ -module.exports = LocalOAM; diff --git a/services/s3.js b/services/s3.js index 9b49a69..aaa9e81 100644 --- a/services/s3.js +++ b/services/s3.js @@ -1,17 +1,15 @@ 'use strict'; +var config = require('../config'); var s3 = require('s3'); var meta = require('../controllers/meta.js'); /** * S3 Constructor that handles intractions with S3 * -* @constructor -* @param {String} secretId (optional) - AWS secret key id. Can be set by AWS_SECRET_KEY_ID env var. -* @param {String} secretKey (optional) - AWS secret access key. Can be set by AWS_SECRET_ACCESS_KEY env var. -* @param {String} bucket (optional) - S3 Bucket name. Can be set by S3_BUCKET_NAME env var. +* @param {String} bucket - S3 Bucket name. */ -var S3 = function (secretId, secretKey, bucket) { +var S3 = function (bucket) { this.client = s3.createClient({ maxAsyncS3: 20, // this is the default s3RetryCount: 3, // this is the default @@ -19,14 +17,14 @@ var S3 = function (secretId, secretKey, bucket) { multipartUploadThreshold: 20971520, // this is the default (20 MB) multipartUploadSize: 15728640, // this is the default (15 MB) s3Options: { - accessKeyId: secretId || process.env.AWS_SECRET_KEY_ID, - secretAccessKey: secretKey || process.env.AWS_SECRET_ACCESS_KEY + accessKeyId: config.awsKey, + secretAccessKey: config.awsSecret } }); this.params = { s3Params: { - Bucket: bucket || process.env.S3_BUCKET_NAME /* required */ + Bucket: bucket }, recursive: true }; @@ -75,17 +73,3 @@ S3.prototype.readBucket = function (lastSystemUpdate, cb, done) { module.exports = S3; -/** - * The response callback returns the error and success message. - * - * @callback responseCallback - * @param {error} err - The error message - * @param {string} msg - The success message - */ - - /** - * The finished callback just calls back to the worker to let it know there is - * no more data coming. - * - * @callback finishedCallback - */ diff --git a/services/server.js b/services/server.js index f97c1d7..3a97f38 100644 --- a/services/server.js +++ b/services/server.js @@ -1,6 +1,7 @@ 'use strict'; var Hapi = require('hapi'); +var config = require('../config'); var Server = function (port) { this.port = port; @@ -13,8 +14,8 @@ var Server = function (port) { stripTrailingSlash: true } }, - debug: process.env.OAM_DEBUG ? { - log: [ 'error' ], + debug: config.debug === 'true' ? { + log: [ 'error', 'debug', 'info', 'worker' ], request: [ 'error', 'received', 'response' ] } : false }); @@ -51,10 +52,12 @@ Server.prototype.start = function (cb) { if (err) throw err; }); - self.hapi.plugins.workers.spawn(); - self.hapi.start(function () { - console.log('Server running at:', self.hapi.info.uri); + console.log( + 'Server (' + process.env.NODE_ENV + ') running at:', + self.hapi.info.uri + ); + self.hapi.plugins.workers.spawn(); if (cb) { cb(); } diff --git a/test/.setup.js b/test/.setup.js new file mode 100644 index 0000000..80ecd96 --- /dev/null +++ b/test/.setup.js @@ -0,0 +1,6 @@ +process.env.NODE_ENV = 'test' +require('dotenv').config(); + +var Conn = require('../services/db.js'); +var dbWrapper = new Conn(); +dbWrapper.start(); diff --git a/test/catalog/fixtures/oin-buckets.json b/test/catalog/fixtures/oin-buckets.json new file mode 100644 index 0000000..db835da --- /dev/null +++ b/test/catalog/fixtures/oin-buckets.json @@ -0,0 +1,12 @@ +{ + "nodes": [{ + "name": "Testing/Development Bucket", + "contact": "info@hotosm.org", + "locations": [ + { + "type": "s3", + "bucket_name": "oin-hotosm-staging" + } + ] + }] +} diff --git a/test/catalog/test_meta.js b/test/catalog/test_meta.js index 6fc8a69..7ed43f7 100644 --- a/test/catalog/test_meta.js +++ b/test/catalog/test_meta.js @@ -1,8 +1,6 @@ -/* global before, after, describe, it */ 'use strict'; var expect = require('chai').expect; -// var should = require('chai').should(); var request = require('request'); var Conn = require('../../services/db.js'); var Server = require('../../services/server.js'); diff --git a/test/catalog/test_tms.js b/test/catalog/test_tms.js index abe2fac..b6b338b 100644 --- a/test/catalog/test_tms.js +++ b/test/catalog/test_tms.js @@ -1,14 +1,11 @@ -/* global before, after, describe, it */ 'use strict'; var expect = require('chai').expect; -// var should = require('chai').should(); var request = require('request'); var async = require('async'); var Conn = require('../../services/db.js'); var Server = require('../../services/server.js'); var meta = require('./sample_meta.json'); -var testDb = 'oam_catalog_test'; describe('TMS endpoint', function () { this.timeout(15000); @@ -48,7 +45,7 @@ describe('TMS endpoint', function () { }; before(function (done) { - self.db = new Conn(testDb); + self.db = new Conn(); self.db.start(function (err) { if (err) { console.log(err); diff --git a/test/docker-compose.yml b/test/docker-compose.yml new file mode 100644 index 0000000..926c887 --- /dev/null +++ b/test/docker-compose.yml @@ -0,0 +1,29 @@ +# Run from project root with: +# `docker-compose -f test/docker-compose.yml up` + +version: '2' +services: + test-app: + extends: + service: app + file: ../docker-compose.yml + + # So you can override from the CLI + environment: + NODE_ENV: $NODE_ENV + OAM_DEBUG: $OAM_DEBUG + + # Make container's localhost the same as host's localhost. + # So the DB should either be on the host or remote, not in a + # docker container. + network_mode: "host" + + # Mount your code live into the container so you don't need + # to rebuild the container everytime. + volumes: + - ../:/app + + # Run both foreman and a static file server for testing purposes + command: /bin/sh -c + "./node_modules/.bin/http-server ./test & + ./node_modules/.bin/nf start" diff --git a/test/helper.js b/test/helper.js new file mode 100644 index 0000000..d4bd5ae --- /dev/null +++ b/test/helper.js @@ -0,0 +1,5 @@ +var connection = require('mongoose').connection; + +beforeEach(function (done) { + connection.db.dropDatabase(done); +}); diff --git a/test/integration/test_main.js b/test/integration/test_main.js new file mode 100644 index 0000000..6e686b7 --- /dev/null +++ b/test/integration/test_main.js @@ -0,0 +1,52 @@ +'use strict'; + +var expect = require('chai').expect; +require('../helper'); +var request = require('request'); +var db = require('mongoose').connection; + +var apiBase = 'http://localhost:4000'; +var auth = { 'Authorization': 'Bearer letmein' }; + +function waitForImage (id, callback) { + var getOptions = { + url: apiBase + '/uploads/' + id, + headers: auth, + json: true + }; + + request.get(getOptions, function (_err, httpResponse, body) { + var status = body.results.scenes[0].images[0].status; + if (status === 'finished') { + callback(); + } + }); + setTimeout(waitForImage, 100, id, callback); +} + +describe('Uploading imagery', function () { + beforeEach(function (done) { + db.collection('tokens').insert({ + token: 'letmein', + status: 'active', + expiration: new Date(Date.now() + 1000000) + }, done); + }); + + it('should upload and process an image', function (done) { + // Needs time to process the image + this.timeout(3 * 60 * 1000); + + var postOptions = { + url: apiBase + '/uploads', + json: require('../uploader/fixture/NE1_50M_SR.input.json'), + headers: auth + }; + + request.post(postOptions, function (_err, httpResponse, body) { + expect(httpResponse.statusCode).to.eq(200); + var imageId = body.results.upload; + waitForImage(imageId, done); + }); + }); +}); diff --git a/test/mocha.opts b/test/mocha.opts new file mode 100644 index 0000000..5c3597a --- /dev/null +++ b/test/mocha.opts @@ -0,0 +1 @@ +--require test/.setup.js diff --git a/test/uploader/babel.js b/test/uploader/babel.js index 9d91f05..38f19c4 100644 --- a/test/uploader/babel.js +++ b/test/uploader/babel.js @@ -1,3 +1,7 @@ +// Shouldn't really be here, but tests should be merged to +// mocha soon. +require('dotenv').config(); + var Babel = require('babel'); module.exports = [ diff --git a/test/uploader/fixture/NE1_50M_SR.input.json b/test/uploader/fixture/NE1_50M_SR.input.json index 2ba69ad..d0cc98c 100644 --- a/test/uploader/fixture/NE1_50M_SR.input.json +++ b/test/uploader/fixture/NE1_50M_SR.input.json @@ -18,7 +18,7 @@ "acquisition_end": "2015-04-30T00:00:00.000", "tms": null, "urls": [ - "http://localhost:8080/NE1_50M_SR.tif" + "http://localhost:8080/uploader/fixture/NE1_50M_SR.tif" ] } ] diff --git a/test/uploader/fixture/upload-status.json b/test/uploader/fixture/upload-status.json index ff3c6a0..82191a7 100644 --- a/test/uploader/fixture/upload-status.json +++ b/test/uploader/fixture/upload-status.json @@ -20,7 +20,7 @@ "images": [ { "_id": "55e0c86a24c379c000544d23", - "url": "http://localhost:8080/NE1_50M_SR.tif", + "url": "http://localhost:8080/uploader/fixture/NE1_50M_SR.tif", "status": "initial", "messages": [], "startedAt": "2015-08-28T20:45:31.062Z", diff --git a/worker/process-image.js b/worker/process-image.js index 7e17e99..cd27ac7 100644 --- a/worker/process-image.js +++ b/worker/process-image.js @@ -2,7 +2,6 @@ var cp = require('child_process'); -var AWS = require('aws-sdk'); var promisify = require('es6-promisify'); var request = require('request'); @@ -54,10 +53,9 @@ function _processImage (scene, sourceUrl, targetPrefix, callback) { args.push(sourceUrl, output); var child = cp.spawn('process.sh', args, { - AWS_ACCESS_KEY_ID: AWS.config.credentials.accessKeyId, - AWS_DEFAULT_REGION: AWS.config.region, - AWS_SECRET_ACCESS_KEY: AWS.config.credentials.secretAccessKey, - AWS_SESSION_TOKEN: AWS.config.credentials.sessionToken, + AWS_ACCESS_KEY_ID: config.awsKey, + AWS_SECRET_ACCESS_KEY: config.awsSecret, + AWS_DEFAULT_REGION: config.awsRegion, THUMBNAIL_SIZE: config.thumbnailSize, TILER_BASE_URL: config.tilerBaseUrl }); diff --git a/worker/queue.js b/worker/queue.js index 682af52..90a53db 100644 --- a/worker/queue.js +++ b/worker/queue.js @@ -21,20 +21,21 @@ JobQueue.prototype._initialize = function init () { if (this._initialized) { return Promise.resolve(true); } this._initialized = true; return promisify(MongoClient.connect.bind(MongoClient))(config.dbUri) - .then((connection) => { - this.db = connection; - this.workers = this.db.collection('workers'); - this.images = this.db.collection('images'); + .then((connection) => { + console.log('Upload worker connected to: ' + config.dbUri); + this.db = connection; + this.workers = this.db.collection('workers'); + this.images = this.db.collection('images'); - return this.workers.insertOne({ state: 'working' }) - .then((result) => { - this.workerId = result.ops[0]._id; - log.workerId = this.workerId; - this._setupQueries(); - log('Initialized'); - }) - .catch(this.cleanup.bind(this)); - }); + return this.workers.insertOne({ state: 'working' }) + .then((result) => { + this.workerId = result.ops[0]._id; + log.workerId = this.workerId; + this._setupQueries(); + log('Initialized'); + }) + .catch(this.cleanup.bind(this)); + }); }; JobQueue.prototype._setupQueries = function _setupQueries () { @@ -81,7 +82,8 @@ JobQueue.prototype._mainloop = function mainloop () { .findOneAndUpdate({ status: 'initial' }, this.update.jobClaimed, { returnOriginal: false }) - .then((result) => { + .then((result) => { + log(['info'], result); if (!result.value) { // no jobs left; try to shut down. // avoid race condition by making sure our state wasn't changed from