Add url signing as an alternative to forbid direct access

docs/source/catalog.rst

@@ -572,6 +572,10 @@ Whether a particular catalog entry supports direct or proxied access is determin
 - ``force``: Force all clients to access the data directly.  If they do not have the required driver, an exception will
   be raised.
 
+- ``signed``: Clients access the data directly, but are provided with a temporary URLs. This enables authentication
+  systems to still be effective without requiring the server to load all the data itself. Similar to ``force`` this
+  will raise an error if the client doesn't have the needed driver.
+
 Note that when the client is loading a data source via direct access, the catalog server will need to send the driver
 arguments to the client.  Do not include sensitive credentials in a data source that allows direct access.
 

intake/cli/server/server.py

@@ -13,6 +13,7 @@
 import tornado.gen
 import tornado.ioloop
 import tornado.web
+from fsspec import get_fs_token_paths
 
 from intake.config import conf
 from intake.container import serializer
@@ -239,7 +240,7 @@ def post(self):
         action = request['action']
         head = self.request.headers
         logger.debug('Source POST: %s' % request)
-        
+
         if action == 'search':
             if 'source-id' in head:
                 cat = self._cache.get(head['source-id'])
@@ -319,7 +320,27 @@ def post(self):
                                      metadata=source.metadata))
                 self.write(msgpack.packb(response, **pack_kwargs))
                 self.finish()
-            elif direct_access == 'force' and not client_has_plugin:
+            elif direct_access == 'signed' and client_has_plugin:
+                response = open_desc
+                user_parameters['plugin'] = plugin_name
+                response['args'] = entry._entry._create_open_args(user_parameters)[1]
+                response_args = response['args']
+                fs, *_ = get_fs_token_paths(response_args['urlpath'],
+                                            storage_options=response_args.get('storage_options', {}))
+                try:
+                    response_args['urlpath'] = fs.sign(response_args['urlpath'],
+                                                       expiration=response_args.get('expiration', 100))
+                except NotImplementedError:
+                    msg = 'signed urls not supported for the filesystem associated with "%s"' % entry_name
+                    raise tornado.web.HTTPError(status_code=400, log_message=msg,
+                                                reason=msg)
+                except Exception as e:
+                    msg = 'server could not create signed url for "%s"' % entry_name
+                    raise tornado.web.HTTPError(status_code=400, log_message=msg,
+                                                reason=msg)
+                self.write(msgpack.packb(response, **pack_kwargs))
+                self.finish()
+            elif (direct_access in {'signed', 'force'}) and not client_has_plugin:
                 msg = 'client must have plugin "%s" to access source "%s"' \
                       '' % (plugin_name, entry_name)
                 raise tornado.web.HTTPError(status_code=400, log_message=msg,