Permalink
Browse files

Tasks for fetching repos from GitHub and updating design documents

  • Loading branch information...
0 parents commit b0b9b6e7cf76dfbb129e95a9640f81bcf8446faf @lackac committed May 15, 2011
Showing with 10,192 additions and 0 deletions.
  1. +109 −0 Jakefile.coffee
  2. +33 −0 design.coffee
  3. +9 −0 node_modules/async/.gitmodules
  4. +19 −0 node_modules/async/LICENSE
  5. +21 −0 node_modules/async/Makefile
  6. +970 −0 node_modules/async/README.md
  7. BIN node_modules/async/async.min.js.gzip
  8. +70 −0 node_modules/async/deps/nodeunit.css
  9. +1,966 −0 node_modules/async/deps/nodeunit.js
  10. +1 −0 node_modules/async/dist/async.min.js
  11. +3 −0 node_modules/async/index.js
  12. +632 −0 node_modules/async/lib/async.js
  13. +4 −0 node_modules/async/nodelint.cfg
  14. +16 −0 node_modules/async/package.json
  15. +1,367 −0 node_modules/async/test/test-async.js
  16. +24 −0 node_modules/async/test/test.html
  17. +20 −0 node_modules/cradle/LICENSE
  18. +10 −0 node_modules/cradle/Makefile
  19. +260 −0 node_modules/cradle/README.md
  20. +628 −0 node_modules/cradle/lib/cradle.js
  21. +102 −0 node_modules/cradle/lib/cradle/cache.js
  22. +123 −0 node_modules/cradle/lib/cradle/response.js
  23. +1 −0 node_modules/cradle/node_modules/.bin/vows
  24. +20 −0 node_modules/cradle/node_modules/vargs/LICENSE
  25. +34 −0 node_modules/cradle/node_modules/vargs/README.md
  26. +70 −0 node_modules/cradle/node_modules/vargs/lib/vargs.js
  27. +12 −0 node_modules/cradle/node_modules/vargs/package.json
  28. +20 −0 node_modules/cradle/node_modules/vows/LICENSE
  29. +7 −0 node_modules/cradle/node_modules/vows/Makefile
  30. +39 −0 node_modules/cradle/node_modules/vows/README.md
  31. +421 −0 node_modules/cradle/node_modules/vows/bin/vows
  32. +27 −0 node_modules/cradle/node_modules/vows/lib/assert/error.js
  33. +180 −0 node_modules/cradle/node_modules/vows/lib/assert/macros.js
  34. +195 −0 node_modules/cradle/node_modules/vows/lib/vows.js
  35. +93 −0 node_modules/cradle/node_modules/vows/lib/vows/console.js
  36. +55 −0 node_modules/cradle/node_modules/vows/lib/vows/context.js
  37. +28 −0 node_modules/cradle/node_modules/vows/lib/vows/extras.js
  38. +69 −0 node_modules/cradle/node_modules/vows/lib/vows/reporters/dot-matrix.js
  39. +10 −0 node_modules/cradle/node_modules/vows/lib/vows/reporters/json.js
  40. +8 −0 node_modules/cradle/node_modules/vows/lib/vows/reporters/silent.js
  41. +74 −0 node_modules/cradle/node_modules/vows/lib/vows/reporters/spec.js
  42. +38 −0 node_modules/cradle/node_modules/vows/lib/vows/reporters/watch.js
  43. +90 −0 node_modules/cradle/node_modules/vows/lib/vows/reporters/xunit.js
  44. +321 −0 node_modules/cradle/node_modules/vows/lib/vows/suite.js
  45. +20 −0 node_modules/cradle/node_modules/vows/node_modules/eyes/LICENSE
  46. +4 −0 node_modules/cradle/node_modules/vows/node_modules/eyes/Makefile
  47. +72 −0 node_modules/cradle/node_modules/vows/node_modules/eyes/README.md
  48. +233 −0 node_modules/cradle/node_modules/vows/node_modules/eyes/lib/eyes.js
  49. +14 −0 node_modules/cradle/node_modules/vows/node_modules/eyes/package.json
  50. +55 −0 node_modules/cradle/node_modules/vows/node_modules/eyes/test/eyes-test.js
  51. +14 −0 node_modules/cradle/node_modules/vows/package.json
  52. +101 −0 node_modules/cradle/node_modules/vows/test/assert-test.js
  53. +373 −0 node_modules/cradle/node_modules/vows/test/vows-test.js
  54. +13 −0 node_modules/cradle/package.json
  55. +104 −0 node_modules/cradle/test/cache-test.js
  56. +612 −0 node_modules/cradle/test/cradle-test.js
  57. +76 −0 node_modules/cradle/test/response-test.js
  58. +49 −0 node_modules/cradle/test/scripts/prepare-db.js
  59. +3 −0 node_modules/progress/.gitignore
  60. +4 −0 node_modules/progress/.npmignore
  61. +16 −0 node_modules/progress/History.md
  62. +5 −0 node_modules/progress/Makefile
  63. +104 −0 node_modules/progress/Readme.md
  64. +2 −0 node_modules/progress/index.js
  65. +109 −0 node_modules/progress/lib/node-progress.js
  66. +10 −0 node_modules/progress/package.json
@@ -0,0 +1,109 @@
+{print} = require 'sys'
+http = require 'http'
+{whilst} = require 'async'
+ProgressBar = require 'progress'
+
+desc "Fetch GitHub repos that have more than 1 watcher"
+task 'fetch', (n) ->
+ {db} = getDB()
+ n = parseInt(n ? 500)
+ progress = new ProgressBar "[:bar] :percent (:elapseds)", total: n, width: 60
+ console.log "Fetching repo information..."
+ page = 1
+ whilst (-> page <= n),
+ (next) ->
+ getSearchPage page++, (err, repos) ->
+ return next(err) if err
+ updateRepos db, repos, (err) ->
+ progress.tick()
+ next(err)
+ (err) ->
+ throw err if err
+ console.log "done."
+ complete()
+, true
+
+desc "Push design doc to the database"
+task 'design', ->
+ {couch, db} = getDB()
+ ddoc = require './design.coffee'
+ for name, view of ddoc.views
+ for key, fun of view
+ view[key] = fun.toString().replace(/\n +/g, '')
+ for key, fun of ddoc.lists
+ ddoc.lists[key] = fun.toString().replace(/\n +/g, '')
+
+ console.log "Pushing design doc #{ddoc._id}..."
+ db.get ddoc._id, (err, doc) ->
+ if err and err.error != "not_found"
+ console.log(err)
+ complete()
+ else
+ ddoc._rev = doc._rev if doc
+ db.save ddoc._id, ddoc, (err, doc) ->
+ console.log(err) if err
+ complete()
+, true
+
+desc "Trigger view indexing"
+task 'trigger', ->
+ {couch, db} = getDB()
+ console.log "Triggering view indexing..."
+ ddoc = require './design.coffee'
+ completed = false
+ db.view 'repos/by_watchers', limit: 1, (err, res) ->
+ completed = true
+ console.log(err) if err
+ complete()
+ taskRe = new RegExp("#{db.name}.* #{ddoc._id}$")
+ logStatus = ->
+ couch.activeTasks (err, tasks) ->
+ if tasks
+ for {task, status} in tasks when taskRe.test(task)
+ console.log(status)
+ setTimeout(logStatus, 2000) unless completed
+ logStatus()
+, true
+
+
+updateRepos = (db, repos, callback) ->
+ ids = repos.map (repo) -> "#{repo.owner}:#{repo.name}"
+ db.get ids, (err, rows) ->
+ callback(err) if err
+ bulk_docs = []
+ for row, i in rows
+ repos[i]._id = ids[i]
+ if row.error == "not_found" or not row.doc
+ bulk_docs.push(repos[i])
+ else if needsUpdate(row.doc, repos[i])
+ repos[i]._rev = row.doc._rev
+ bulk_docs.push(repos[i])
+ db.save bulk_docs, (err, docs) -> callback(err, docs)
+
+needsUpdate = (doc, repo) ->
+ doc.description != repo.description or doc.watchers != repo.watchers or doc.forks != repo.forks
+
+getSearchPage = (page, callback) ->
+ req = http.get
+ host: 'github.com'
+ path: "/api/v2/json/repos/search/+?start_page=#{page}"
+ (res) ->
+ resp_body = ""
+ res.on "data", (chunk) -> resp_body += chunk
+ res.on "end", ->
+ repos = JSON.parse(resp_body).repositories
+ callback(null, repos)
+ res.on "error", (err) ->
+ callback(err)
+
+getDB = ->
+ uri = require("url").parse(process.env.DB || "http://localhost:5984/gh-repos")
+ db_options = { cache: true, raw: false, cacheSize: 1000, bulkSize: 500 }
+ db_options.host = "#{uri.protocol}//#{uri.hostname}"
+ db_options.port = uri.port if uri.port
+ if uri.auth
+ [user, pass] = uri.auth.split(":")
+ db_options.auth = { username: user, password: pass }
+ db_name = uri.pathname.substr(1)
+ connection = new (require('cradle').Connection)(db_options)
+ { couch: connection, db: connection.database(db_name) }
@@ -0,0 +1,33 @@
+module.exports =
+ _id: '_design/repos'
+ languange: 'javascript'
+
+ views:
+ by_watchers:
+ map: (doc) -> emit doc.watchers, doc.description if doc.watchers
+ by_language:
+ map: (doc) -> emit doc.languange, doc.description if doc.languange
+ by_prefix:
+ map: (doc) ->
+ return unless doc.type is "repo"
+
+ # emit all prefixes of name
+ for c, i in doc.name
+ emit [doc.name.substr(0, i), doc.watchers], doc.description
+
+ # emit all prefixes of owner
+ for c, i in doc.owner
+ prefix = doc.owner.substr(0, i)
+ emit [prefix, doc.watchers], doc.description
+
+ # emit all combined prefixes
+ for d, j in doc.name
+ emit [prefix + '/' + doc.name.substr(0,j), doc.watchers], doc.description
+
+ lists:
+ complete: (head, req) ->
+ start
+ headers: { 'Content-Type': 'text/plain' }
+ while row = getRow()
+ send "#{row.id.replace(':', '/')} (#{row.key[1]}) #{row.value}\n"
+ return

Some generated files are not rendered by default. Learn more.

Oops, something went wrong.

Some generated files are not rendered by default. Learn more.

Oops, something went wrong.

Some generated files are not rendered by default. Learn more.

Oops, something went wrong.
Oops, something went wrong.

0 comments on commit b0b9b6e

Please sign in to comment.