Add multiple repository support and config file support

Fixes #11
Fixes #12

config.json

@@ -0,0 +1,6 @@
+{
+  "bin-dir" : "$SUBUSERDIR/bin",
+  "installed-programs.json" : "$SUBUSERDIR/installed-programs.json",
+  "user-set-permissions-dir" : "$SUBUSERDIR/permissions/",
+  "program-home-dirs-dir" : "$SUBUSERDIR/homes/"
+}

docs/config-dot-json-file-format.md

@@ -0,0 +1,38 @@
+The config.json file format
+--------------------------------
+
+In the context of subuser, a `config.json` file is a file which subuser settings.
+
+The `config.json` files are to be arranged into a fallback-hierarchy.  Subuser will first look up properties in the file `~/.subuser/config.json` falling back to:
+
+ * `/etc/subuser/config.json`
+
+ * `$SUBUSERDIR/config.json`
+
+Each config file may be partial.  That is, if the user wants to specify some options, but leave others as their default values, they can simply ommit those options that they do not wish to change.
+
+`$SUBUSERDIR` is the directory where the subuser source resides.  It corresponds to the root of this git repository.
+
+Each config.json file is to be a valid [json](http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf) file containing a single json object.
+
+Properties:
+-----------
+The defaults settings for these properties may be found in `$SUBUSERDIR/config.json`.
+
+**Note on paths**: Each path is to be absolute.  Any environment variables of the form `$VARIABLE_NAME` will be expanded.
+
+ * `bin-dir`: This is the directory where subuser "executables" are to be installed.  This directory should be in the user's path.
+
+  `type`: path to directory
+
+ * `installed-programs.json` : This is the path to the installed-programs registry file.
+
+ `type`: path to file
+
+ * `user-set-permissions-dir`: NOT YET IMPLEMENTED
+
+ `type`: path to directory
+
+ * `program-home-dirs-dir`: The directory where subuser is to store the home directories of each subuser program.
+
+ `type`: path to directory

docs/repositories-dot-json-file-format.md

@@ -0,0 +1,39 @@
+The repositories.json file format
+--------------------------------
+
+In the context of subuser, a `repositories.json` file is a file which describes where subuser programs may be installed from.
+
+The `repositories.json` files are to be arranged into a fallback-hierarchy.  Subuser will first look up properties in the file `~/.subuser/repositories.json` falling back to:
+
+ * `/etc/subuser/repositories.json`
+
+ * `$SUBUSERDIR/repositories.json`
+
+`$SUBUSERDIR` is the directory where the subuser source resides.  It corresponds to the root of this git repository.
+
+
+Each repositories.json file is to be a valid [json](http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf) file containing a single json object.
+
+This object is a set of key value pairs where each key is the name of a subuser repository.  The value is a json object with the following properties:
+
+Properties:
+-----------
+**Note on paths**: Each path is to be absolute.  Any environment variables of the form `$VARIABLE_NAME` will be expanded.
+
+ * `path`: The path to the repository.
+
+ Ex:
+
+ ````
+ "path" : "$SUBUSERDIR/programsThatCanBeInstalled/"
+ ````
+
+Example repositories.json file:
+
+````
+{
+ "default" : {"path" : "$SUBUSERDIR/programsThatCanBeInstalled/"}
+}
+````
+
+This file states that there is one repository named `default` which can be found in the `$SUBUSERDIR/programsThatCanBeInstalled` directory.

logic/subuserCommands/install

@@ -8,10 +8,6 @@ import subuserlib.installCommon
 import subuserlib.describe
 import subuserlib.commandLineArguments
 
-######################################################
-home = os.path.expanduser("~")
-######################################################
-
 def printHelp():
   """
     Display a help message for the install mode of the subuser command.

logic/subuserCommands/subuserlib/availablePrograms.py

@@ -6,12 +6,15 @@
 
 def available(programName):
   """ Returns True if the program is available for instalation. """
-  return os.path.exists(paths.getProgramSrcDir(programName))
+  return not paths.getProgramSrcDir(programName) == None
 
 def getAvailablePrograms():
   """ Returns a list of program's available for instalation. """
-  availableProgramsPath = paths.getAvailableProgramsPath()
-  return os.listdir(availableProgramsPath)
+  repoPaths = paths.getRepoPaths()
+  availablePrograms = []
+  for path in repoPaths:
+    availablePrograms += os.listdir(path)
+  return availablePrograms
 
 def getAvailableProgramsText(addNewLine=False, indentSpaces=0):
   """ Returns a string representing a sorted list of available program names.

logic/subuserCommands/subuserlib/config.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+# This file should be compatible with both Python 2 and 3.
+# If it is not, please file a bug report.
+import os
+import inspect
+import json
+
+home = os.path.expanduser("~")
+
+def _getSubuserDir():
+  """ Get the toplevel directory for subuser. """
+  return os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))))) # BLEGH!
+
+def _getSubuserConfigPaths():
+  """ Returns a list of paths to config.json files in order that they should be looked in. """
+  _configsPaths = []
+  _configsPaths.append(os.path.join(home,".subuser","config.json"))
+  _configsPaths.append("/etc/subuser/config.json") # TODO how does this work on windows?
+  _configsPaths.append(os.path.join(_getSubuserDir(),"config.json"))
+  configsPaths = []
+  for path in _configsPaths:
+   if os.path.exists(path):
+    configsPaths.append(path)
+  return configsPaths
+
+def _addIfUnrepresented(identifier,path,paths):
+  """ Add the tuple to the dictionary if it's key is not yet in the dictionary. """
+  if not identifier in paths.keys():
+    paths[identifier] = path
+
+def expandPathInConfig(path,config):
+  """ Expand the path of a given setting. """
+  config[path] = os.path.expandvars(config[path])
+
+def expandPathsInConfig(paths,config):
+  for path in paths:
+    expandPathInConfig(path,config)
+
+def expandVarsInPaths(config):
+  """ Go through a freshly loaded config file and expand any environment variables in the paths. """
+  os.environ["SUBUSERDIR"] = _getSubuserDir()
+  expandPathsInConfig(["bin-dir","installed-programs.json","user-set-permissions-dir","program-home-dirs-dir"],config)
+
+def getConfig():
+  """ Returns a dictionary of settings used by subuser. """
+  configPaths = _getSubuserConfigPaths()
+  config = {}
+  for _configFile in configPaths:
+    with open(_configFile, 'r') as configFile:
+      _config = json.load(configFile)
+      for identifier,setting in _config.iteritems():
+        _addIfUnrepresented(identifier,setting,config)
+  expandVarsInPaths(config)
+  return config

logic/subuserCommands/subuserlib/installCommon.py

@@ -20,8 +20,8 @@ def installExecutable(programName):
     st = os.stat(executablePath)
     os.chmod(executablePath, stat.S_IMODE(st.st_mode) | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
 
-def installFromBaseImage(programName):
-  buildImageScriptPath = paths.getBuildImageScriptPath(programName)
+def installFromBaseImage(programName,programSrcDir):
+  buildImageScriptPath = paths.getBuildImageScriptPath(programSrcDir)
   #Report to user
   while True:
     sys.stdout.write("""\nATTENTION!!!
@@ -33,7 +33,7 @@ def installFromBaseImage(programName):
   - Do you want to view the full contents of this shell script [v]?
   - Do you want to continue? (Type "run" to run the shell script)
 
-  [v/run/q]: """.format(programName, paths.getBuildImageScriptPath(programName)))
+  [v/run/q]: """.format(programName, buildImageScriptPath))
     sys.stdout.flush()
     try:
       userInput = sys.stdin.readline().strip()
@@ -73,26 +73,28 @@ def installFromBaseImage(programName):
     os.chmod(buildImageScriptPath, stat.S_IMODE(st.st_mode) | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
     subprocessExtras.subprocessCheckedCall([buildImageScriptPath])
 
+def installFromDockerfile(programName, useCache):
+  if useCache:
+    cacheArg = "--no-cache=false"
+  else:
+    cacheArg = "--no-cache=true"
+  docker.runDockerAndExitIfItFails(["build","-rm",cacheArg,"--tag=subuser-"+programName+"",dockerImageDir])
+
 
 def installProgram(programName, useCache):
   """
   Build the docker image associated with a program and create a tiny executable to add that image to your path.
   """
   print("Installing {0} ...".format(programName))
-
-  dockerImageDir = os.path.join(paths.getProgramSrcDir(programName), "docker-image")
-  _DockerfilePath = os.path.join(dockerImageDir, 'Dockerfile')
+  programSrcDir = paths.getProgramSrcDir(programName)
+  _DockerfilePath = paths.getDockerfilePath(programSrcDir)
   # Check if we use a 'Dockerfile' or a 'BuildImage.sh'
-  if os.path.isfile(paths.getBuildImageScriptPath(programName)):
-    installFromBaseImage(programName, cacheArg)
+  if os.path.isfile(paths.getBuildImageScriptPath(programSrcDir)):
+    installFromBaseImage(programName,programSrcDir)
   elif os.path.isfile(_DockerfilePath):
-    if useCache:
-      cacheArg = "--no-cache=false"
-    else:
-      cacheArg = "--no-cache=true"
-    docker.runDockerAndExitIfItFails(["build","-rm",cacheArg,"--tag=subuser-"+programName+"",dockerImageDir])
+    installFromDockerfile(programName,programSrcDir,useCache)
   else:
-    sys.exit("No buildfile found: need one of: 'Dockerfile' or 'BuildImage.sh'. PATH: {0}".format(dockerImageDir))
+    sys.exit("No buildfile found: There needs to be a 'Dockerfile' or a 'BuildImage.sh' in the docker-image directory.")
 
   _permissions = permissions.getPermissions(programName)
 

logic/subuserCommands/subuserlib/paths.py

@@ -2,41 +2,70 @@
 # This file should be compatible with both Python 2 and 3.
 # If it is not, please file a bug report.
 import os
+import sys
 import inspect
 import permissions
+import json
+import config
+import repositories
 
-home = os.path.expanduser("~")
+home = os.path.expanduser("~") 
 
 def getSubuserDir():
   """ Get the toplevel directory for subuser. """
   return os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))))) # BLEGH!
 
-def getProgramSrcDir(progName):
+def getRepoPaths():
+  """
+  Return a list of paths to the subuser repositories.
+  """
+  try:
+    _repositories = repositories.getRepositories()
+    repoPaths = []
+    for repo,info in _repositories.iteritems():
+      repoPaths.append(info["path"])
+    return repoPaths
+  except KeyError:
+    print("Looking up repo-paths failed. Your repositories.json file is invalid.")
+    sys.exit(1)
+
+def getProgramSrcDir(programName):
   """
   Get the directory where the "source" of the application is stored.  That is the permissions list and the docker-image directory.
+
+Returns None if the program cannot be found.
+
   """
-  programSourceDir = os.path.join(getSubuserDir(),"programsThatCanBeInstalled",progName)
-  return programSourceDir
+  for repoPath in getRepoPaths():
+   programSourceDir = os.path.join(repoPath,programName)
+   if os.path.exists(programSourceDir):
+     return programSourceDir
+  return None
 
 def getExecutablePath(progName):
   """
   Get the path to the executable that we will be installing.
   """
-  executablePath = os.path.join(getSubuserDir(),"bin",progName)
-  return executablePath
+  return os.path.join(config.getConfig()["bin"],progName)
 
 def getPermissionsFilePath(programName):
-  """ Return the path to the given programs permissions file. """
-  sourceDir = getProgramSrcDir(programName)
-  return os.path.join(sourceDir,"permissions.json")
+  """ Return the path to the given programs permissions file.
+Returns None if no permission file is found.
+ """
+  userPermissionsPath = os.path.join(config.getConfig()["user-set-permissions-dir"],programName,"permissions.json")
+  if os.path.exists(userPermissionsPath):
+    return userPermissionsPath
+  else:
+    repoPaths = getRepoPaths()
+    for repoPath in repoPaths:
+      permissionsPath = os.path.join(repoPath,programName,"permissions.json")
+      if os.path.exists(permissionsPath):
+        return permissionsPath
+  return None
 
 def getProgramRegistryPath():
   """ Return the path to the list of installed programs json file. """
-  return os.path.join(getSubuserDir(),"installed-programs.json")
-
-def getAvailableProgramsPath():
-  """ Return the path to the directory which contains sources of programs available for instalation. """
-  return os.path.join(getSubuserDir(),"programsThatCanBeInstalled")
+  return config.getConfig()["installed-programs.json"]
 
 def getProgramHomeDirOnHost(programName):
   """ Each program has it's own home directory(or perhaps a shared one).
@@ -47,15 +76,21 @@ def getProgramHomeDirOnHost(programName):
   programPermissions = permissions.getPermissions(programName)
   sharedHome = permissions.getSharedHome(programPermissions)
   if sharedHome:
-    return os.path.join(getSubuserDir(),"homes",sharedHome)
+    return os.path.join(config.getConfig()["program-home-dirs-dir"],sharedHome)
   else:
-    return os.path.join(getSubuserDir(),"homes",programName)
+    return os.path.join(config.getConfig()["program-home-dirs-dir"],programName)
 
 def getDockersideScriptsPath():
   return os.path.join(getSubuserDir(),"logic","dockerside-scripts")
 
-def getBuildImageScriptPath(programName):
+def getBuildImageScriptPath(programSrcDir):
+  """
+  Get path to the BuildImage.sh. From the program's docker-image directory.
+  """
+  return os.path.join(programSrcDir,"docker-image","BuildImage.sh")
+
+def getDockerfilePath(programSrcDir):
   """
-  Get path to the BuildImage.sh. From the program docker-image directory.
+  Get path to the Dockerfile From the program's docker-image directory.
   """
-  return os.path.join(getProgramSrcDir(programName), "docker-image", 'BuildImage.sh')
+  return os.path.join(programSrcDir,"docker-image","Dockerfile")

logic/subuserCommands/subuserlib/repositories.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+# This file should be compatible with both Python 2 and 3.
+# If it is not, please file a bug report.
+
+# TODO, refactor by putting helper functions for both repositories.py and configs.py in one place.
+
+import os
+import inspect
+import json
+import collections
+
+home = os.path.expanduser("~")
+
+def _getSubuserDir():
+  """ Get the toplevel directory for subuser. """
+  return os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))))) # BLEGH!
+
+def _getRepositoryListPaths():
+  """ Returns a list of paths to repositories.json files in order that they should be looked in. """
+  _repositoryListPaths = []
+  _repositoryListPaths.append(os.path.join(home,".subuser","repositories.json"))
+  _repositoryListPaths.append("/etc/subuser/repositories.json") # TODO how does this work on windows?
+  _repositoryListPaths.append(os.path.join(_getSubuserDir(),"repositories.json"))
+  repositoryListPaths = []
+  for path in _repositoryListPaths:
+   if os.path.exists(path):
+    repositoryListPaths.append(path)
+  return repositoryListPaths
+
+def _addIfUnrepresented(identifier,path,paths):
+  """ Add the tuple to the dictionary if it's key is not yet in the dictionary. """
+  if not identifier in paths.keys():
+    paths[identifier] = path
+
+def expandVarsInPaths(repositories):
+  """ Go through a freshly loaded list of repositories and expand any environment variables in the paths. """
+  os.environ["SUBUSERDIR"] = _getSubuserDir()
+  for reponame,info in repositories.iteritems():
+    info["path"] = os.path.expandvars(info["path"])
+
+def getRepositories():
+  """ Returns a dictionary of repositories used by subuser. """
+  repositoryListPaths = _getRepositoryListPaths()
+  repositories = {}
+  for _repositoryListFile in repositoryListPaths:
+    with open(_repositoryListFile, 'r') as repositoryListFile:
+      _repositories = json.load(repositoryListFile, object_pairs_hook=collections.OrderedDict)
+      for identifier,repository in _repositories.iteritems():
+        _addIfUnrepresented(identifier,repository,repositories)
+  expandVarsInPaths(repositories)
+  return repositories

repositories.json

@@ -0,0 +1,3 @@
+{
+ "default" : {"path" : "$SUBUSERDIR/programsThatCanBeInstalled/"}
+}