Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Atomic publish

instead of the problematic 3-stage PUT process, just do one PUT with the
tarball data inline.
  • Loading branch information...
commit e9fbeb8b67f249394f735c74ef11fe4720d46ca0 1 parent 137e098
@isaacs isaacs authored
View
153 lib/publish.js
@@ -3,9 +3,11 @@ module.exports = publish
var path = require("path")
, url = require("url")
+ , semver = require("semver")
+ , crypto = require("crypto")
+ , fs = require("fs")
function publish (data, tarball, cb) {
-
var email = this.conf.get('email')
var auth = this.conf.get('_auth')
var username = this.conf.get('username')
@@ -16,14 +18,30 @@ function publish (data, tarball, cb) {
return cb(er)
}
- // add the dist-url to the data, pointing at the tarball.
- // if the {name} isn't there, then create it.
- // if the {version} is already there, then fail.
- // then:
- // PUT the data to {config.registry}/{data.name}/{data.version}
- var registry = this.conf.get('registry')
+ if (data.name !== encodeURIComponent(data.name).toLowerCase())
+ return cb(new Error('invalid name: must be lowercase and url-safe'))
+
+ var ver = semver.clean(data.version)
+ if (!ver)
+ return cb(new Error('invalid semver: ' + data.version))
+ data.version = ver
+
+ var self = this
+ fs.stat(tarball, function(er, s) {
+ if (er) return cb(er)
+ fs.readFile(tarball, 'base64', function(er, tardata) {
+ if (er) return cb(er)
+ putFirst.call(self, data, tardata, s, username, email, cb)
+ })
+ })
+}
- var fullData =
+function putFirst (data, tardata, stat, username, email, cb) {
+ // optimistically try to PUT all in one single atomic thing.
+ // If 409, then GET and merge, try again.
+ // If other error, then fail.
+
+ var root =
{ _id : data.name
, name : data.name
, description : data.description
@@ -37,76 +55,93 @@ function publish (data, tarball, cb) {
]
}
+ root.versions[ data.version ] = data
+ var tag = data.tag || this.conf.get('tag') || "latest"
+ root["dist-tags"][tag] = data.version
+
+ var registry = this.conf.get('registry')
var tbName = data.name + "-" + data.version + ".tgz"
, tbURI = data.name + "/-/" + tbName
data._id = data.name+"@"+data.version
data.dist = data.dist || {}
+ data.dist.shasum = crypto.createHash("sha1").update(tardata).digest("hex")
data.dist.tarball = url.resolve(registry, tbURI)
.replace(/^https:\/\//, "http://")
-
- // first try to just PUT the whole fullData, and this will fail if it's
- // already there, because it'll be lacking a _rev, so couch'll bounce it.
- this.request("PUT", encodeURIComponent(data.name), fullData,
- function (er, parsed, json, response) {
- // get the rev and then upload the attachment
- // a 409 is expected here, if this is a new version of an existing package.
- if (er
- && !(response && response.statusCode === 409)
- && !( parsed
- && parsed.reason ===
- "must supply latest _rev to update existing package" )) {
- this.log.error("publish", "Failed PUT response "
- +(response && response.statusCode))
+ root._attachments = {}
+ root._attachments[ tbName ] = {
+ content_type: 'application/octet-stream',
+ data: tardata,
+ length: stat.size
+ };
+
+ this.request("PUT", data.name, root, function (er, parsed, json, res) {
+ var r409 = "must supply latest _rev to update existing package"
+ var r409b = "Document update conflict."
+ var conflict = res && res.statusCode === 409
+ if (parsed && (parsed.reason === r409 || parsed.reason === r409b))
+ conflict = true
+
+ // a 409 is typical here. GET the data and merge in.
+ if (er && !conflict) {
+ this.log.error("publish", "Failed PUT "
+ +(res && res.statusCode))
return cb(er)
}
- var dataURI = encodeURIComponent(data.name)
- + "/" + encodeURIComponent(data.version)
- var tag = data.tag || this.conf.get('tag') || "latest"
- dataURI += "/-tag/" + tag
+ if (!er && !conflict)
+ return cb(er, parsed, json, res)
// let's see what versions are already published.
- // could be that we just need to update the bin dist values.
- this.request("GET", data.name, function (er, fullData) {
- if (er) return cb(er)
-
- function handle(er) {
- if (er.message.indexOf("conflict Document update conflict.") === 0) {
- return cb(conflictError.call(this, data._id));
- }
- this.log.error("publish", "Error uploading package");
+ this.request("GET", data.name, function (er, current) {
+ if (er)
return cb(er)
- }
-
- var exists = fullData.versions && fullData.versions[data.version]
- if (exists) return cb(conflictError.call(this, data._id))
-
- var rev = fullData._rev;
- attach.call(this, data.name, tarball, tbName, rev, function (er) {
- if (er) return handle.call(this, er)
- this.log.verbose("publish", "attached", [data.name, tarball, tbName])
- this.request("PUT", dataURI, data, function (er) {
- if (er) return handle.call(this, er)
- return cb(er)
- }.bind(this))
- }.bind(this))
+ putNext.call(this, data.version, root, current, cb)
}.bind(this))
- }.bind(this)) // pining for fat arrows.
+ }.bind(this))
+}
+
+function putNext(newVersion, root, current, cb) {
+ // already have the tardata on the root object
+ // just merge in existing stuff
+ // if the version already exists, and not a --force, then raise error
+ var force = this.conf.get('force')
+ var curVers = Object.keys(current.versions || {}).map(function (v) {
+ return semver.clean(v, true)
+ })
+
+ if (!force && curVers.indexOf(newVersion) !== -1) {
+ return cb(conflictError(root.name))
+ }
+
+ current.versions[newVersion] = root.versions[newVersion]
+ for (var i in root) {
+ switch (i) {
+ // objects that copy over the new stuffs
+ case 'dist-tags':
+ case 'versions':
+ case '_attachments':
+ for (var j in root[i])
+ current[i][j] = root[i][j]
+ break
+
+ // ignore these
+ case 'maintainers':
+ break;
+
+ // copy
+ default:
+ current[i] = root[i]
+ }
+ }
+
+ this.request("PUT", root.name, current, cb)
}
function conflictError (pkgid) {
- var e = new Error("publish fail")
+ var e = new Error("cannot modify existing version")
e.code = "EPUBLISHCONFLICT"
e.pkgid = pkgid
return e
}
-
-function attach (doc, file, filename, rev, cb) {
- doc = encodeURIComponent(doc)
- var revu = "-rev/"+rev
- , attURI = doc + "/-/" + encodeURIComponent(filename) + "/" + revu
- this.log.verbose("uploading", [attURI, file])
- this.upload(attURI, file, cb)
-}
View
4 test/fixtures/server.js
@@ -17,10 +17,10 @@ function handler (req, res) {
, mu = req.method + ' ' + req.url
var k = server._expect[mu] ? mu : server._expect[u] ? u : null
- if (!k) throw Error('unexpected request', req.method, req.url)
+ if (!k) throw Error('unexpected request: ' + req.method + ' ' + req.url)
var fn = server._expect[k].shift()
- if (!fn) throw Error('unexpected request', req.method, req.url)
+ if (!fn) throw Error('unexpected request' + req.method + ' ' + req.url)
var remain = (Object.keys(server._expect).reduce(function (s, k) {
View
81 test/publish-again.js
@@ -0,0 +1,81 @@
+var tap = require('tap')
+var server = require('./fixtures/server.js')
+var RC = require('../')
+var client = new RC(
+ { cache: __dirname + '/fixtures/cache'
+ , registry: 'http://localhost:' + server.port
+ , username: "username"
+ , password: "password"
+ , email: "i@izs.me"
+ , _auth: new Buffer("username:password").toString('base64')
+ , "always-auth": true
+ })
+
+var fs = require("fs")
+
+tap.test("publish again", function (t) {
+ var lastTime = null
+ server.expect("/npm-registry-client", function (req, res) {
+ t.equal(req.method, "PUT")
+ var b = ""
+ req.setEncoding('utf8')
+ req.on("data", function (d) {
+ b += d
+ })
+
+ req.on("end", function () {
+ var o = lastTime = JSON.parse(b)
+ t.equal(o._id, "npm-registry-client")
+ t.equal(o["dist-tags"].latest, pkg.version)
+ t.has(o.versions[pkg.version], pkg)
+ t.same(o.maintainers, [ { name: 'username', email: 'i@izs.me' } ])
+ var att = o._attachments[ pkg.name + '-' + pkg.version + '.tgz' ]
+ t.same(att.data, pd)
+ res.statusCode = 409
+ res.json({reason: "must supply latest _rev to update existing package"})
+ })
+ })
+
+ server.expect("/npm-registry-client", function (req, res) {
+ t.equal(req.method, "GET")
+ t.ok(lastTime)
+ for (var i in lastTime.versions) {
+ var v = lastTime.versions[i]
+ delete lastTime.versions[i]
+ lastTime.versions["0.0.2"] = v
+ lastTime["dist-tags"] = { latest: "0.0.2" }
+ }
+ lastTime._rev = "asdf"
+ res.json(lastTime)
+ })
+
+ server.expect("/npm-registry-client", function (req, res) {
+ t.equal(req.method, "PUT")
+ t.ok(lastTime)
+
+ var b = ""
+ req.setEncoding('utf8')
+ req.on("data", function (d) {
+ b += d
+ })
+
+ req.on("end", function() {
+ var o = JSON.parse(b)
+ t.equal(o._rev, "asdf")
+ t.deepEqual(o.versions["0.0.2"], o.versions[pkg.version])
+ res.statusCode = 201
+ res.json({created: true})
+ })
+ })
+
+
+ // not really a tarball, but doesn't matter
+ var tarball = require.resolve('../package.json')
+ var pd = fs.readFileSync(tarball, 'base64')
+ var pkg = require('../package.json')
+ client.publish(pkg, tarball, function (er, data, raw, res) {
+ if (er) throw er
+ t.deepEqual(data, { created: true })
+ t.end()
+ })
+})
View
48 test/publish.js
@@ -0,0 +1,48 @@
+var tap = require('tap')
+var server = require('./fixtures/server.js')
+var RC = require('../')
+var client = new RC(
+ { cache: __dirname + '/fixtures/cache'
+ , registry: 'http://localhost:' + server.port
+ , username: "username"
+ , password: "password"
+ , email: "i@izs.me"
+ , _auth: new Buffer("username:password").toString('base64')
+ , "always-auth": true
+ })
+
+var fs = require("fs")
+
+tap.test("publish", function (t) {
+ server.expect("/npm-registry-client", function (req, res) {
+ t.equal(req.method, "PUT")
+ var b = ""
+ req.setEncoding('utf8')
+ req.on("data", function (d) {
+ b += d
+ })
+
+ req.on("end", function () {
+ var o = JSON.parse(b)
+ console.error('PUT req', o)
+ t.equal(o._id, "npm-registry-client")
+ t.equal(o["dist-tags"].latest, pkg.version)
+ t.has(o.versions[pkg.version], pkg)
+ t.same(o.maintainers, [ { name: 'username', email: 'i@izs.me' } ])
+ var att = o._attachments[ pkg.name + '-' + pkg.version + '.tgz' ]
+ t.same(att.data, pd)
+ res.statusCode = 201
+ res.json({created:true})
+ })
+ })
+
+ // not really a tarball, but doesn't matter
+ var tarball = require.resolve('../package.json')
+ var pd = fs.readFileSync(tarball, 'base64')
+ var pkg = require('../package.json')
+ client.publish(pkg, tarball, function (er, data, raw, res) {
+ if (er) throw er
+ t.deepEqual(data, { created: true })
+ t.end()
+ })
+})
Please sign in to comment.
Something went wrong with that request. Please try again.