Find file History
Fetching latest commit…
Cannot retrieve the latest commit at this time.
Permalink
Type Name Latest commit message Commit time
..
Failed to load latest commit information.
config
README.md
index.js
package-lock.json
package.json

README.md

redshift_batch

Batches kafka messages into redshift.

Install

npm install

Copy config/default.json to config/local.json then edit the production.json

{
  "queueSize": 50, // number of msgs to queue up before inserting
  "timeout": 3000, // max time to queue before inserting
  "database": "postgres://fritzy@localhost:5432/fritzy", // pg connection string
  "kafka": {  
    "topic": "ecommerce-logs", // kafka topic
    "group": "redshift-batch", // consumer group id
    "config": { // no-kafka configuration object
      "kafkaHost": "kafka://localhost:9092",
      "ssl": {
        "key": "",
        "cert: ""
      }
    }
  }
}
CREATE TABLE ecommercelogs(id INT IDENTITY(1, 1), time TIMESTAMP WITH TIME ZONE, session VARCHAR(255), action VARCHAR(255), product VARCHAR(255), category VARCHAR(255), campaign VARCHAR(255))

Running

node index.js