diff --git a/docs/api.md b/docs/api.md index 2aef363..3937d3d 100644 --- a/docs/api.md +++ b/docs/api.md @@ -135,4 +135,73 @@ a HTTP DELETE request to the url `/v1/flush`. Like this: https://autocompeter.com/v1/flush This will reset the counts all related to your domain. The only thing that -isn't removed is your auth key. +isn't removed is your auth key. + +## Bulk upload + +Instead of submitting one "document" at a time you can instead send in a +whole big JSON blob. The struct needs to be like this example: + + { + "documents": [ + { + "url": "http://example.com/page1", + "title": "Page One" + }, + { + "url": "http://example.com/page2", + "title": "Other page", + "popularity": 123 + }, + { + "url": "http://example.com/page3", + "title": "Last page", + "group": "admins" + }, + ] + } + +Note that the `popularity` and the `group` keys are optional. Each +dictionary in the array called `documents` needs to have a `url` and `title`. + +The endpoint to use `https://autocompeter.com/v1/bulk` and you need to do a +HTTP POST or a HTTP PUT. + +Here's an example using curl: + + url -X POST -H "Auth-Key: 3b14d7c280bf525b779d0a01c601fe44" \ + -d '{"documents": [{"url":"/url", "title":"My Title", "popularity":1001}]}' \ + https://autocompeter.com/v1/bulk + +And here's an example using +Python [requests](http://requests.readthedocs.org/en/latest/): + + +```python +import json +import requests + +documents = [ + { + 'url': '/some/page', + 'title': 'Some title', + 'popularity': 10 + }, + { + 'url': '/other/page', + 'title': 'Other title', + }, + { + 'url': '/private/page', + 'title': 'Other private page', + 'group': 'private' + }, +] +print requests.post( + 'https://autocompeter.com/v1/bulk', + data=json.dumps({'documents': documents}), + headers={ + 'Auth-Key': '3b14d7c280bf525b779d0a01c601fe44', + } +) +``` diff --git a/sampleloader/populate.py b/sampleloader/populate.py index 0a27edb..c0e5b14 100755 --- a/sampleloader/populate.py +++ b/sampleloader/populate.py @@ -3,6 +3,7 @@ import os import json import hashlib +import time import click import requests @@ -28,7 +29,7 @@ def get_events(): yield (item['title'], item['url'], item['popularity'], group) -def populate(database, destination, domain, flush=False): +def populate(database, destination, domain, flush=False, bulk=False): c = redis.StrictRedis(host='localhost', port=6379, db=database) if flush: c.flushdb() @@ -38,6 +39,16 @@ def populate(database, destination, domain, flush=False): #items = get_blogposts() items = get_events() + t0 = time.time() + if bulk: + _in_bulk(destination, items) + else: + _one_at_a_time(destination, items) + t1 = time.time() + print "TOOK", t1 - t0 + + +def _one_at_a_time(destination, items): for title, url, popularity, group in items: _url = destination + '/v1' data = { @@ -56,14 +67,42 @@ def populate(database, destination, domain, flush=False): assert r.status_code == 201, r.status_code +def _in_bulk(destination, items): + data = { + 'documents': [ + dict( + title=t, + url=u, + popularity=p, + group=g + ) + for t, u, p, g in items + ] + } + _url = destination + '/v1/bulk' + r = requests.post( + _url, + data=json.dumps(data), + headers={'Auth-Key': key} + ) + assert r.status_code == 201, r.status_code + + @click.command() @click.option('--database', '-d', default=8) @click.option('--destination', default='http://autocompeter.com') @click.option('--domain', default='autocompeter.com') @click.option('--flush', default=False, is_flag=True) -def run(database, destination, domain, flush=False): +@click.option('--bulk', default=False, is_flag=True) +def run(database, destination, domain, flush=False, bulk=False): #print (database, domain, flush) - populate(database, destination, domain, flush=flush) + populate( + database, + destination, + domain, + flush=flush, + bulk=bulk, + ) if __name__ == '__main__': run() diff --git a/server.go b/server.go index d3bb23e..a0d74bd 100644 --- a/server.go +++ b/server.go @@ -3,6 +3,7 @@ package main import ( "crypto/md5" "encoding/base64" + "encoding/json" "fmt" "github.com/codegangsta/negroni" "github.com/fiam/gounidecode/unidecode" @@ -132,7 +133,7 @@ func updateHandler(w http.ResponseWriter, req *http.Request) { } form.Title = strings.Trim(form.Title, " ") form.URL = strings.Trim(form.URL, " ") - group := form.Group + // group := form.Group c, err := redisPool.Get() errHndlr(err) @@ -145,30 +146,44 @@ func updateHandler(w http.ResponseWriter, req *http.Request) { return } + insertDocument( + domain, + form.Title, + form.URL, + form.Group, + form.Popularity, + c, + ) + + output := map[string]string{"message": "OK"} + renderer.JSON(w, http.StatusCreated, output) +} + +func insertDocument(domain, title, url, group string, popularity float64, c *redis.Client) { encoded := encodeString(domain) - encodedURL := encodeString(form.URL) + encodedURL := encodeString(url) - title, _ := c.Cmd("HGET", encoded+"$titles", encodedURL).Str() - if title == "" { - err = c.Cmd("HINCRBY", "$domaindocuments", domain, 1).Err + existingTitle, _ := c.Cmd("HGET", encoded+"$titles", encodedURL).Str() + if existingTitle == "" { + err := c.Cmd("HINCRBY", "$domaindocuments", domain, 1).Err errHndlr(err) } pipedCommands := 0 - for _, prefix := range getPrefixes(form.Title) { + for _, prefix := range getPrefixes(title) { if group != "" { encodedGroup := encodeString(group) - c.Append("ZADD", encoded+encodedGroup+prefix, form.Popularity, encodedURL) + c.Append("ZADD", encoded+encodedGroup+prefix, popularity, encodedURL) c.Append("HSET", encoded+"$groups", encodedURL, encodedGroup) pipedCommands++ } else { - c.Append("ZADD", encoded+prefix, form.Popularity, encodedURL) + c.Append("ZADD", encoded+prefix, popularity, encodedURL) } pipedCommands++ } - c.Append("HSET", encoded+"$titles", encodedURL, form.Title) + c.Append("HSET", encoded+"$titles", encodedURL, title) pipedCommands++ - c.Append("HSET", encoded+"$urls", encodedURL, form.URL) + c.Append("HSET", encoded+"$urls", encodedURL, url) pipedCommands++ for i := 1; i <= pipedCommands; i++ { if err := c.GetReply().Err; err != nil { @@ -176,6 +191,54 @@ func updateHandler(w http.ResponseWriter, req *http.Request) { } } +} + +type bulkDocuments struct { + Documents []bulkDocument `json:"documents"` +} + +type bulkDocument struct { + URL string `json:"url"` + Title string `json:"title"` + Popularity float64 `json:"popularity"` + Group string `json:"group"` +} + +func bulkHandler(w http.ResponseWriter, req *http.Request) { + key := req.Header.Get("AUTH-KEY") + if key == "" { + output := map[string]string{"error": "Auth-Key header not set"} + renderer.JSON(w, http.StatusForbidden, output) + return + } + c, err := redisPool.Get() + errHndlr(err) + defer redisPool.Put(c) + + domain, err := GetDomain(key, c) + if domain == "" { + output := map[string]string{"error": "Auth-Key not recognized"} + renderer.JSON(w, http.StatusForbidden, output) + return + } + + // encoded := encodeString(domain) + + decoder := json.NewDecoder(req.Body) + var bs bulkDocuments + err = decoder.Decode(&bs) + errHndlr(err) + for _, b := range bs.Documents { + // fmt.Println(b.URL, b.Title, b.Popularity, b.Group) + insertDocument( + domain, + b.Title, + b.URL, + b.Group, + b.Popularity, + c, + ) + } output := map[string]string{"message": "OK"} renderer.JSON(w, http.StatusCreated, output) } @@ -674,6 +737,7 @@ func main() { mux.HandleFunc("/v1", deleteHandler).Methods("DELETE") mux.HandleFunc("/v1/stats", privateStatsHandler).Methods("GET") mux.HandleFunc("/v1/flush", flushHandler).Methods("DELETE") + mux.HandleFunc("/v1/bulk", bulkHandler).Methods("POST", "PUT") n := negroni.Classic() diff --git a/templates/index.tmpl b/templates/index.tmpl index 9df9577..4170f1e 100644 --- a/templates/index.tmpl +++ b/templates/index.tmpl @@ -196,6 +196,7 @@ Autocompeter(