Allow global configuration of gcloud

[ryanseys]

From discussion at #168 (comment)

Support supplying credentials earlier in the gcloud setup process.

var gcloud = require('gcloud');
gcloud.options({ keyFilename: !ON_COMPUTE && '/path/to/the/key.json' });

[stephenplusplus]

#46

If we're continuing the discussion here, I'll just add a couple notes.

// Would be nice if it worked on gce
var gcloud = require('gcloud');
var dataset = new gcloud.datastore.Dataset({ projectId: 'hi' });

// Should work on gae & elsewhere
var gcloud = require('gcloud')({ credentials: {/*... */} });
var dataset = new gcloud.datastore.Dataset({ projectId: 'hi' });

So, a couple things are different from your example:

1. No need for a user on gce to require any connection configuration. That can be handled for them by us like it is currently (right?)
2. Remove the .options method, and instead pass the credentials/keyfilename to the gcloud required object.

Here's how I would envision that working...

// lib/index.js
var connection = require('./common/connection.js');
function gcloud(connectionOptions) {
  var conn = new connection(connectionOptions);
  gcloud.datastore.connection = conn;
  gcloud.pubsub.connection = conn;
  gcloud.storage.connection = conn;
}
gcloud.datastore = require('./datastore/index.js');
gcloud.pubsub = require('./pubsub/index.js');
gcloud.storage = require('./storage/index.js');

module.exports = gcloud;

// lib/storage/index.js
var connection = require('./common/connection.js');
var conn = module.exports.connection;
if (!module.exports.connection) {
  // no connection was configured, so we're on gce
  conn = new connection();
}
function Bucket() {}
// ...
module.exports.Bucket = Bucket;

If that example is awful, feel free to re-think it completely. Trying to wrangle a baby over here, so my thought 🚋 could be way off :)

[jgeewax]

I'd like to take this to an extra level and include projectId in the connection parameters (it's uncommon for a single chunk of code to operate on more than one project simultaneously).

That is:

var gcloud = require('gcloud')({
  projectId: 'my-project',
  keyFile: '/path/to/keyfile.json'
});

// This is a weird one because there's currently one
// dataset per project, but this may change over time.
var dataset = new gcloud.datastore.Dataset() 

// This one is pretty standard...
var bucket = new gcloud.storage.Bucket({ bucketName: 'my-bucket' });

If you do want to do more than one project, the old config should still be able to be overridden:

// Assuming we have gcloud set as above...
var otherDataset = new gcloud.datastore.Dataset({
  projectId: 'my-other-project',
  keyFile: '/path/to/other/keyfile.json'
});

Or even:

var gcloud = require('gcloud');

var projectOne = gcloud.configure({
  projectId: 'my-project',
  keyFile: '/path/to/keyfile.json'
});

var projectTwo = gcloud.configure({
  projectId: 'my-other-project',
  keyFile: '/path/to/other/keyfile.json'
});

var bucketOne = new projectOne.storage.Bucket( ... );
var bucketTwo = new projectTwo.storage.Bucket( ... );

Thoughts?

CC: @proppy @ryanseys @stephenplusplus @silvolu

[jgeewax]

(I should add that if we allow it to be done this way, the Authentication section of the README becomes much easier to write, as the demo isn't service-specific... See #193 )

[stephenplusplus]

I'd like to take this to an extra level and include projectId in the connection parameters (it's uncommon for a single chunk of code to operate on more than one project simultaneously).

Makes sense 👍

How about still avoiding a new method:

var gcloud = require('gcloud');

// one project.
var myProject = gcloud({
  projectId: 'my-project',
  keyFile: '/path/to/keyfile.json'
});

// another project.
var anotherProject = gcloud({
  projectId: 'my-other-project',
  keyFile: '/path/to/other/keyfile.json'
});

[jgeewax]

SGTM. Who wants to take this one?

[stephenplusplus]

I'm happy to give it a go, unless @ryanseys wants the honors 👑

[silvolu]

SGTM as long as we allow overwriting for a single service instance, e.g. as @jgeewax mentioned

var otherDataset = new gcloud.datastore.Dataset({
  projectId: 'my-other-project',
  keyFile: '/path/to/other/keyfile.json'
});

[proppy]

Note that on GCE you can query the projectId from the metadata server, and since you don't need the .json credentials either: var myProject = gcloud(); should just work.

[rakyll]

var dataset = new gcloud.datastore.Dataset()

Your service account may have read/right permissions for other projects. So, this is not a weird one.

[rakyll]

I should add that if we allow it to be done this way, the Authentication section of the README becomes much easier to write, as the demo isn't service-specific..

There is no service specific auth. All services work with a credentials object that looks more or less the same -- we can make the object exactly the same for each service.

The credentials object is the global configuration. We don't need need interfaces.

var conf = {
  projectId: 'my-other-project',
  keyFile: '/path/to/other/keyfile.json'
};

var ds = gcloud.datastore.dataset(conf);
// or on GCE
var ds = gcloud.datastore.dataset();
var bucket = gcloud.storage.bucket(conf);

This proposal is not looking good to me.

[stephenplusplus]

That covered the credentials, but not any further config options:

var conf = {
  projectId: 'my-other-project',
  keyFile: '/path/to/other/keyfile.json'
};
// ...
var bucket = gcloud.storage.bucket(conf); // bucketName?

It's not an exhaustive amount of effort we would force on the developer to figure that problem out-- they would probably just inline the entire configuration object again, as opposed to extending the object. However, if we know the developer is going to do something the same way each time they invoke a sub-module, why not plan for this and allow up-front specification?

var gcloud = require('gcloud')({
  projectId: 'my-other-project',
  keyFile: '/path/to/other/keyfile.json'
});
var ds = gcloud.datastore.dataset();
var bucket = gcloud.storage.bucket({ bucketName: 'hi' });

And from config-aware envs:

var gcloud = require('gcloud');
var ds = gcloud.datastore.dataset();
var bucket = gcloud.storage.bucket({ bucketName: 'hi' });

And if you need multiple connection details:

var gcloud = require('gcloud');
var myProject = gcloud({
  projectId: 'my-project',
  keyFile: '/path/to/keyfile.json'
});
var myOtherProject = gcloud({
  projectId: 'my-other-project',
  keyFile: '/path/to/other/keyfile.json'
});
var ds = myProject.datastore.dataset();
var bucket = myOtherProject.storage.bucket({ bucketName: 'hi' });

[stephenplusplus]

I would say, however, providing anything but connection info to the gcloud module can introduce some confusion. I would vote to keep it limited to just credentials or keyFilename. But, maybe there are use-cases where being allowed to specify additional defaults would be beneficial?

[rakyll]

But, maybe there are use-cases where being allowed to specify additional defaults would be beneficial?

There could be global options. But if there are not default, the configuration object on the service layer (called conf above) should require them as well.

Non-optional options can be set optionally via gcloud.configure(opts), so we still can eliminate gcloud initialisation duality.

var bucket = gcloud.storage.bucket(conf); // bucketName?

We actually never need a projectId field for datastore and storage clients. The configuration is already fragmented. The only option you can pass is the keyFilename or credentials.

Overriding creates more confusion.

// Confusion #1: Should this authorise via metadata server or not?
var gcloud = require('gcloud');
var myProject = gcloud({
  projectId: 'my-project',
  keyFile: '/path/to/keyfile.json'
});
var ds = myProject.datastore.dataset();


// Confusion #2: Will this override keyFilename or not?
var gcloud = require('gcloud');
var myProject = gcloud({
  projectId: 'my-project',
  keyFile: '/path/to/keyfile.json'
});
var ds = myProject.datastore.dataset({ credentials: blah });

// Confusion #3: Will this discover my GCE project Id or not?
var gcloud = require('gcloud');
var myProject = gcloud({
  keyFilename: '/path/to/keyfile.json'
});
var ds = myProject.datastore.dataset();

// Confusion #4: Will this work on GAE devappserver?
var gcloud = require('gcloud');
var ds = myProject.datastore.dataset(); // this will work
var bucket = myProject.storage.bucket(); // this won't work
// Whereas on prod, both will work. So, if this is a GAE user, global config is not usable.

So, configuration object differs per service. Auth behaves differently according to the environment. Some services support devappserver, some don't.

Why do we need to mess with all the logic here (and explaining it) instead of asking user to repeat himself twice by passing a credentials object or a keyFilename string?

Our current way of doing auth is already extremely implicit -- GAE and GCE behaviours are magical. I don't want to add more confusion.

[stephenplusplus]

Our current way of doing auth is already extremely implicit -- GAE and GCE behaviours are magical. I don't want to add more confusion.

I can agree with that. Something similar to this would be nice to have:

var gcloud = require('gcloud');
gcloud.setEnvironment('gae');

[jgeewax]

We do need the project ID for storage. When creating a bucket, we need to know which project to put it under. If running outside of GCE, we need to explicitly specify the project ID, right?

With Confusion 4: gcloud({keyFilename: ...}).storage.bucket() in GCE... does that actually work? I think we really need a bucket name... right?

[rakyll]

We do need the project ID for storage. When creating a bucket, we need to know which project to put it under. If running outside of GCE, we need to explicitly specify the project ID, right?

We don't need a project ID. We need a bucket name. A project may contain more than a single bucket.

Datastore is not any different. A dataset can be managed by any service account as long as it has permissions and no project ID is required -- just the dataset ID.

gcloud({keyFilename: ...}).storage.bucket()

It currently requires an explicit bucket name, so it won't work (until we support GAE devappserver's mock Storage server). But the confusion is still there. A dev feels the necessity to read the code to predict the behaviour and we create more complexity by introducing overriding.

User will just need to pass the same credentials object, and will do it at the construction time for once. Why are we trying to improve a two-line repetition?

[proppy]

So maybe gcloud could have a notion of global and local credentials.

You could do something like this:

var gcloud = require('gcloud');
gcloud.auth({key: '/path/to/json'}); // auth the default connection w/ this credentials
var dataset = gcloud.datastore.dataset('datasetId'); // use the default connection
var bucket = gcloud.storage.bucket('bucketId'); // use the default connection

You can even even go further and similarly to #191 (comment) have gcloud.auth() return a AuthorizedConnection and have .datastore and .bucket be child of that connection.

var conn1 = require('gcloud').auth({key: ...}); // auth and return a new connection
var conn2 = require('gcloud').auth({key: ...}); // auth and return a new connection

var dataset = conn1.datastore.dataset('datasetId'); // access datasetId with conn1 credentials
var bucket = conn2.storage.bucket('bucketId'); // access bucketId with conn2 credentials

I agree with @rakyll that projectId is not useful here, what really matters is:

• the identity you want to use (.json file)
• the resource you want to access (bucket, datasetId, ...)

[silvolu]

+1 to @proppy's proposal.

[rakyll]

Or

var gcloud = require('gcloud')(auth);
gcloud.datastore.dataset('datasetId');
gcloud.storage.bucket('bucketId');

Global credentials, local identifier, no overriding.

+1 to @proppy.

[stephenplusplus]

@rakyll This looks like what #196 implemented, minus the overriding. Can you answer your earlier points of confusion?

// Confusion #1: Should this authorise via metadata server or not?
var gcloud = require('gcloud');
var myProject = gcloud({
  projectId: 'my-project',
  keyFile: '/path/to/keyfile.json'
});
var ds = myProject.datastore.dataset();
// would then be...
var gcloud = require('gcloud');
var myProject = gcloud({
  keyFile: '/path/to/keyfile.json'
});
var ds = myProject.datastore.dataset('my-project');


// Confusion #2: Will this override keyFilename or not?
-- resolved --

// Confusion #3: Will this discover my GCE project Id or not?
var gcloud = require('gcloud');
var myProject = gcloud({
  keyFilename: '/path/to/keyfile.json'
});
var ds = myProject.datastore.dataset();

// Confusion #4: Will this work on GAE devappserver?
var gcloud = require('gcloud');
var ds = myProject.datastore.dataset(); // this will work
var bucket = myProject.storage.bucket(); // this won't work
// Whereas on prod, both will work. So, if this is a GAE user, global config is not usable.

Will we still support this for gae/gce?:

var gcloud = require('gcloud');
var ds = gcloud.datastore.dataset('projectId'); // connection auto-configured

[rakyll]

Isn't this resolved? There is a single way to set auth -- globally. And since, there are specified credentials, myProject.datastore.dataset('my-project') will use them instead of GCE metadata server.

// Confusion #1: Should this authorise via metadata server or not?
// would then be...
var gcloud = require('gcloud');
var myProject = gcloud({
  keyFile: '/path/to/keyfile.json'
});
var ds = myProject.datastore.dataset('my-project');

This is still not resolved, we should throw if global auth exists but no local identifier is provided. Such validation should be done in lib/datastore/index.js

var gcloud = require('gcloud');
var myProject = gcloud({
  keyFilename: '/path/to/keyfile.json'
});
var ds = myProject.datastore.dataset();

This is not resolved. GAE users will not be able to use the same gcloud instance.

var gcloud = require('gcloud');
var ds = myProject.datastore.dataset(); // this will work
var bucket = myProject.storage.bucket(); // this won't work
// Whereas on prod, both will work. So, if this is a GAE user, global config is not usable.

They need to create one with credentials to talk to Storage, another one with no credentials for Datastore :'(

var gcloud = require('gcloud');
var ds = gcloud.datastore.dataset(); // this will work via devappserver mock or metadata server

var gcloud2 = require('gcloud')({ keyFilename: '...' });
var bucket = gcloud2.storage.bucket('name');

Will we still support this for gae/gce?:

Yes, we should keep supporting GAE/GCE.

Pass the credentials object twice, just saying :)

[stephenplusplus]

Yeah, at this point, I vote no global config of anything.

[jgeewax]

This thread has gotten pretty long, and I think I'm at least a little bit confused... (It's also tricky to reply in-line like an e-mail, or comment on a doc...)

To summarize... I'd love to be able to write...

// Pull in gcloud and deal with auth.
var gcloud = require('gcloud')({
  // ... credential details here ...
  // (Also, allow a default project ID, as most people have only one project...)
  // nulls here means... "default to the 'magic' auth via GAE/GCE SAs if possible".
});

// These don't think about auth.
var bucket = gcloud.storage.bucket('bucket');
var datastore = gcloud.datastore.dataset('dataset');

If I'm required to set credentials in every call to .bucket(), I need to pull in both gcloud and the credentials piece everywhere I want to make that call, which seems kind of obnoxious. Ideally I'd like to pull in my authenticated instance of gcloud so that the code where I'm using it (ie, gcloud.storage.bucket()) doesn't even think about auth.

Is this going to be possible? or are we deciding not to do that?

---

Regarding confusion...

I don't see the confusion outlined above, making me think I'm missing something big...

1. project ID is necessary when creating a bucket.
2. dataset ID is currently the same as a project ID
3. specifying a project id means use that project id, even if it fails
4. not specifying a project id means try to find the project id with magic from the environment
5. specifying a keyfile means use that keyfile, even if it fails
6. not specifying a keyfile means try to authenticate with magic from the environment

---

Regarding the different environments...

Can we expose something that lets you check the environment, or set different credentials depending on the environment? The common one is... "If it's not in a 'special' environment, here are the credentials...", ie:

var gcloud = require('gcloud')({projectId: 'my-project'});

// How do we use stuff if we're running in EC2 or on a non-special Linux box?
if (!gcloud.isSpecialMagicalEnvironment()) {
  gcloud = gcloud({
    // ... here are my non-special credentials...
  });
}
var bucket = gcloud.storage.bucket('bucket name');

---

Huge apology if this is derailing the whole conversation...

[stephenplusplus]

Will we still support this for gae/gce?:

Yes, we should keep supporting GAE/GCE.

To clarify this, I meant not requiring invocation of the returned gcloud value:

// g[ca]e
var gcloud = require('gcloud');
var ds = gcloud.datastore.dataset({});
// vs.
var gcloud = require('gcloud')({ creds });
vs ds = gcloud.datastore.dataset({});

That means we have to support and document using the gcloud module in these two ways (I believe you had a concern about this earlier, "initialisation duality"?)

[rakyll]

// nulls here means... "default to the 'magic' auth via GAE/GCE SAs if possible". 

This only works if you are on GCE prod and datastore GAE dev. To be able to work with Storage and Pubsub you'll need a new gcloud instance inited with different credentials if you are a GAE/GCE user.

// GAE local user to use datastore
var gcloud = require('gcloud')();
gcloud.datastore.dataset();

// GAE prod user to use datastore
var gcloud = require('gcloud')();
gcloud.datastore.dataset();

// GAE local user to use storage
var gcloud = require('gcloud')({ credentials: ... });
gcloud.storage.bucket('name');

// GAE prod user to use storage
var gcloud = require('gcloud')();
gcloud.storage.bucket('name');

if (!gcloud.isSpecialMagicalEnvironment())

Yes, this has become a must.

We need to support gcloud.isGAEDev(), gcloud.isGAEProd() and gcloud.isGCE() to help the user to correctly initialize gcloud :(

[rakyll]

That means we have to support and document using the gcloud module in these two ways (I believe you had a concern about this earlier, "initialisation duality"?)

require('gcloud')() looks uglier than the duality IMHO :)