Skip to content

Commit

Permalink
Start of doc update.
Browse files Browse the repository at this point in the history
  • Loading branch information
@starcraftman
starcraftman committed Nov 9, 2015
1 parent b868040 commit 12dc59c
Show file tree
Hide file tree
Showing 5 changed files with 87 additions and 71 deletions.
88 changes: 49 additions & 39 deletions pakit/conf.py
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
"""
All configuration of pakit is done here.
YamlMixin: A mixin class that can read/write yaml files.
Config: Handles global configuration of pakit.
InstallDB: Handles the database of installed programs.
RecipeURIDB: Store and track recipe URIs.
"""
from __future__ import absolute_import

Expand Down Expand Up @@ -34,10 +36,13 @@
'recipes': os.path.expanduser('~/.pakit'),
'source': '/tmp/pakit/src',
},
'recipe_uris': [
'https://github.com/pakit/base_recipes',
'user_recipes',
],
'recipe': {
'update_interval': 60 * 60 * 24,
'uris': [
'https://github.com/pakit/base_recipes',
'user_recipes',
],
},
},
}

Expand Down Expand Up @@ -104,35 +109,12 @@ def write_to(self, obj):

class Config(YamlMixin):
"""
The main configuration file, users can modify global behaviour here.
For example, after parsing the YAML the config may hold:
config = {
'pakit': {
'command': {
'timeout': 120
},
'defaults': {
'repo': 'stable',
},
'log': {
'enabled': True,
'file': '/tmp/pakit/main.log',
'level': 'debug',
},
'paths': {
'link': '/tmp/pakit/links',
'prefix': '/tmp/pakit/builds',
'recipes': '/home/username/.pakit/recipes',
'source': '/tmp/pakit/src',
},
},
'ag': {
'repo': 'unstable'
}
}
The main configuration class, users can modify global behaviour here.
Details:
The yaml file is read and mapped to a tiered python dictionary.
See pakit.conf.TEMPLATE for the defaults.
Details of config:
pakit.command.timeout
The timeout for commands.
Expand Down Expand Up @@ -166,6 +148,15 @@ class Config(YamlMixin):
pakit.paths.source
The path where source code will be downloaded & built.
pakit.recipe.update_interval
After this amount of seconds, pakit will check configured
recipe uris for updates.
pakit.recipe.uris
A list of uris to retrieve recipes from. Recipes paths are indexed
in order of the list. For possible uri specification format,
see pakit.recipe.RecipeManager.
pakit.defaults
A dictionary of default options made available to all recipes.
Anything in this, will be available inside recipes as self.opts.
Expand Down Expand Up @@ -201,7 +192,6 @@ def __str__(self):
def __contains__(self, key_str):
obj = self.conf
for key in key_str.split('.'):
logging.error(key)
if key not in obj:
return False
obj = obj[key]
Expand Down Expand Up @@ -406,15 +396,21 @@ def write(self):

class RecipeURIDB(InstallDB):
"""
Store information on configured remote uris and the paths
to index.
Store information on configured recipe uris and the paths to index them.
"""
def __init__(self, filename):
super(RecipeURIDB, self).__init__(filename)

def add(self, *args):
"""
Update the database for the URI remote.
Add an entry to the database based on the uri.
Will overwrite the entry if it exists in the database prior.
Args:
uri: A uri that is unique in the database.
path: An absolute path to the recipes.
is_vcs: True if and only if it is a version control repository.
kwargs: Optional, a dict of kwargs for the vcs_factory.
"""
uri, path, is_vcs = args[0], args[1], args[2]
kwargs = args[3] if len(args) == 4 else None
Expand All @@ -428,7 +424,10 @@ def add(self, *args):

def update_time(self, uri):
"""
Update the timestmap on the entry.
Update the timestmap on a uri in the database.
Args:
uri: A valid uri in the database.
"""
timestamp = time.time()
self.conf[uri].update({
Expand All @@ -439,7 +438,13 @@ def update_time(self, uri):

def select_path(self, preferred):
"""
Make best effort to honor, ensure returned path is unique in the db.
Select a path that is unique within the database.
Args:
preferred: The preferred path.
Returns:
The selected path that is unique in the database.
"""
existing = [self.conf[key]['path'] for key in self.conf]
new_path = preferred
Expand All @@ -452,7 +457,12 @@ def select_path(self, preferred):

def need_updates(self, interval):
"""
Return remotes that need updating.
Returns a list of uris that should be updated because they are older
than the supplied interval.
Args:
interval: A number of seconds, uris that haven't been updated
in this interval will be selected.
"""
to_update = []
for uri in self.conf:
Expand Down
39 changes: 20 additions & 19 deletions pakit/recipe.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
Recipe: The base class for all recipes.
RecipeDB: The database that indexes all recipes.
RecipeDecorator: Provides some functionality by wrapping Recipes at runtime.
RecipeManager: Retrieves and manages remote recipe sources.
"""
from __future__ import absolute_import
from abc import ABCMeta, abstractmethod
Expand Down Expand Up @@ -441,21 +442,22 @@ def index(self, path):
path: The folder containing recipes to index.
"""
# TODO: Iterate all classes in file, only index subclassing Recipe
# TODO: Try/finally to cleanup
check_package(path)
sys.path.insert(0, os.path.dirname(path))

new_recs = glob.glob(os.path.join(path, '*.py'))
new_recs = [os.path.basename(fname)[0:-3] for fname in new_recs]
if '__init__' in new_recs:
new_recs.remove('__init__')

mod = os.path.basename(path)
for cls in new_recs:
obj = self.recipe_obj(mod, cls)
self.rdb.update({cls: obj})

sys.path = sys.path[1:]
try:
sys.path.insert(0, os.path.dirname(path))

new_recs = glob.glob(os.path.join(path, '*.py'))
new_recs = [os.path.basename(fname)[0:-3] for fname in new_recs]
if '__init__' in new_recs:
new_recs.remove('__init__')

mod = os.path.basename(path)
for cls in new_recs:
obj = self.recipe_obj(mod, cls)
self.rdb.update({cls: obj})
finally:
if os.path.dirname(path) in sys.path:
sys.path.remove(os.path.dirname(path))

def names(self, desc=False):
"""
Expand Down Expand Up @@ -521,11 +523,12 @@ def __init__(self, config):
Args:
config: The pakit configuration.
"""
self.interval = config.get('pakit.recipe.update_interval')
self.root = config.path_to('recipes')
self.uri_db = RecipeURIDB(os.path.join(self.root, 'uris.yml'))
self.active_kwargs = {}
self.active_uris = []
for uri in config.get('pakit.recipe_uris'):
for uri in config.get('pakit.recipe.uris'):
if isinstance(uri, type({})):
kwargs = uri
uri = kwargs.pop('uri')
Expand All @@ -535,7 +538,7 @@ def __init__(self, config):
@property
def paths(self):
"""
The paths to all active recipe locations on the system.
Returns the paths to all active recipe locations on the system.
"""
return [self.uri_db[uri]['path'] for uri in self.active_uris]

Expand Down Expand Up @@ -563,9 +566,7 @@ def check_for_updates(self):
- it has not been updated since interval
- the kwargs between uri_db and active_kwargs differ
"""
# TODO: Interval in pakit.yml
interval = 60 * 60 * 24
need_updates = self.uri_db.need_updates(interval)
need_updates = self.uri_db.need_updates(self.interval)
vcs_uris = [uri for uri, entry in self.uri_db if entry['is_vcs']]
for uri in set(self.active_uris).intersection(vcs_uris):
db_kwargs = self.uri_db[uri].get('kwargs', {})
Expand Down
4 changes: 3 additions & 1 deletion tests/pakit.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,9 @@ pakit:
prefix: /tmp/test_pakit/builds
recipes: /tmp/test_pakit/recipes
source: /tmp/test_pakit/src
recipe_uris:
recipe:
update_interval: 86400
uris:
- https://github.com/pakit/base_recipes
- https://github.com/pakit/test_recipes
ag:
Expand Down
11 changes: 7 additions & 4 deletions tests/test_conf.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,10 +88,13 @@ def test__str__(self):
' "recipes": "{0}",'.format(user_recs),
' "source": "/tmp/pakit/src"',
' },',
' "recipe_uris": [',
' "https://github.com/pakit/base_recipes",',
' "user_recipes"',
' ]',
' "recipe": {',
' "update_interval": 86400,',
' "uris": [',
' "https://github.com/pakit/base_recipes",',
' "user_recipes"',
' ]',
' }',
' }',
'}',
]
Expand Down
16 changes: 8 additions & 8 deletions tests/test_recipe.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -208,7 +208,7 @@ def test_names(self):
assert prog in pakit.recipe.RDB.names()

def test_index(self):
recipe_base = os.path.basename(tc.CONF.get('pakit.recipe_uris')[1])
recipe_base = os.path.basename(tc.CONF.get('pakit.recipe.uris')[1])
test_recipes = os.path.join(tc.CONF.path_to('recipes'), recipe_base)
rdb = RecipeDB(tc.CONF)
rdb.index(test_recipes)
Expand All @@ -223,7 +223,7 @@ def test_recipe_obj(self):
class TestRecipeManager(object):
def setup(self):
self.config = copy.deepcopy(tc.CONF)
self.config.set('pakit.recipe_uris', [tc.GIT])
self.config.set('pakit.recipe.uris', [tc.GIT])
self.config.set('pakit.paths.recipes',
os.path.join(tc.STAGING, 'test_recipes'))
self.manager = None
Expand All @@ -243,14 +243,14 @@ def test_init_new_uris_vcs(self):
assert os.path.exists(expect)

def test_init_new_uris_vcs_unsupported(self):
self.config.set('pakit.recipe_uris', ['https://www.google.ca'])
self.config.set('pakit.recipe.uris', ['https://www.google.ca'])
self.manager = RecipeManager(self.config)
with pytest.raises(PakitError):
self.manager.init_new_uris()

def test_init_new_uris_vcs_kwargs(self):
with_kwargs = {'tag': '0.31.0', 'uri': tc.GIT}
self.config.set('pakit.recipe_uris', [with_kwargs])
self.config.set('pakit.recipe.uris', [with_kwargs])
self.manager = RecipeManager(self.config)
self.manager.init_new_uris()
expect = os.path.join(self.config.path_to('recipes'),
Expand All @@ -261,7 +261,7 @@ def test_init_new_uris_vcs_kwargs(self):
def test_init_new_uris_local_path(self):
uri = 'user_recipes'
expect = os.path.join(self.config.path_to('recipes'), uri)
self.config.set('pakit.recipe_uris', [uri])
self.config.set('pakit.recipe.uris', [uri])
self.manager = RecipeManager(self.config)
self.manager.init_new_uris()
assert os.path.exists(expect)
Expand All @@ -270,15 +270,15 @@ def test_init_new_uris_local_path_exists(self):
uri = 'user_recipes'
expect = os.path.join(self.config.path_to('recipes'), uri)
os.makedirs(expect)
self.config.set('pakit.recipe_uris', [uri])
self.config.set('pakit.recipe.uris', [uri])
self.manager = RecipeManager(self.config)
self.manager.init_new_uris()
assert os.path.exists(expect)

def test_paths(self):
uri = 'user_recipes'
expect = [os.path.join(self.config.path_to('recipes'), uri)]
self.config.set('pakit.recipe_uris', [uri])
self.config.set('pakit.recipe.uris', [uri])
self.manager = RecipeManager(self.config)
self.manager.init_new_uris()
assert self.manager.paths == expect
Expand Down Expand Up @@ -312,7 +312,7 @@ def test_check_for_updates_kwargs(self):
self.manager = RecipeManager(self.config)
self.manager.init_new_uris()
with_kwargs = {'tag': '0.31.0', 'uri': tc.GIT}
self.config.set('pakit.recipe_uris', [with_kwargs])
self.config.set('pakit.recipe.uris', [with_kwargs])
self.manager = RecipeManager(self.config)

old_time = self.manager.uri_db[uri]['time']
Expand Down

0 comments on commit 12dc59c

Please sign in to comment.