Permalink
Browse files

fix caching bug, preparing for 0.1.3 release

  • Loading branch information...
1 parent 5698bec commit beac5e7f3185d4b1f5a63b38bc8f8ec9aadf8c8e @garbas committed Jul 9, 2011
Showing with 152 additions and 99 deletions.
  1. +6 −0 README.rst
  2. +42 −41 githubcollective/__init__.py
  3. +74 −47 githubcollective/config.py
  4. +7 −2 githubcollective/github.py
  5. +23 −9 githubcollective/sync.py
View
@@ -119,6 +119,12 @@ Credits
Changelog
=========
+0.1.3 - 2011-07-09
+------------------
+
+ - fix caching file bug, cache now working
+ [garbas]
+
0.1.2 - 2011-07-03
------------------
@@ -9,23 +9,23 @@
from githubcollective.config import ConfigGithub
-def select_config(config):
+def config_type(config):
if config.endswith('.cfg'):
- return ConfigCFG(config)
+ return ConfigCFG, config
elif config.endswith('.json'):
- return Config(config)
+ return Config, config
else:
raise NotImplemented
-
-class Mailer(object):
-
- def __init__(self, type_):
- raise NotImplemented
-
- def send(self, to, msg):
- raise NotImplemented
-
+def cache_type(cache):
+ if os.path.exists(cache):
+ f = open(cache)
+ data = f.read()
+ if data:
+ f.seek(0)
+ return f
+ f.close()
+ return open(cache, 'w+')
def run():
parser = argparse.ArgumentParser(
@@ -36,13 +36,12 @@ def run():
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
- parser.add_argument('-c', '--config', type=select_config, required=True,
+ parser.add_argument('-c', '--config', type=config_type, required=True,
help="path to configuration file (could also be remote location). "
"eg. http://collective.github.com/permissions.cfg")
- parser.add_argument('-M', '--mailer', type=lambda x: Mailer(x),
- help="TODO")
- parser.add_argument('-C', '--cache', type=str,
- help="path to file where to cache results from github.")
+ parser.add_argument('-C', '--cache', type=cache_type,
+ help="path to file where to cache results from github.",
+ default=None)
parser.add_argument('-o', '--github-org', type=str, required=True,
help="github organisation.")
parser.add_argument('-u', '--github-username', type=str, required=True,
@@ -55,34 +54,36 @@ def run():
args = parser.parse_args()
github = Github(
- args.github_org,
- args.github_username,
- args.github_password,
- args.verbose,
- args.pretend,
- )
+ args.github_org,
+ args.github_username,
+ args.github_password,
+ args.verbose,
+ args.pretend,
+ )
- new = args.config
- old = None
- if args.cache and \
- os.path.exists(args.cache):
- old = Config(args.cache)
- if old is None or \
- (not old.teams and not old.repos):
- old = ConfigGithub(github)
+ new = args.config[0](
+ args.config[1],
+ args.verbose,
+ args.pretend,
+ )
- sync = Sync(github,
- mailer=args.mailer,
- verbose=args.verbose,
- pretend=args.pretend,
+ old = ConfigGithub(
+ github,
+ args.cache,
+ args.verbose,
+ args.pretend,
)
- sync_ok = sync.run(new, old)
+
+ sync_ok = Sync(
+ github,
+ args.verbose,
+ args.pretend,
+ ).run(new, old)
if sync_ok:
- if args.cache and \
- (not args.pretend or not os.path.exists(args.cache)):
- cache = open(args.cache, 'w+')
- old.dumps(cache)
- cache.close()
+ if args.cache:
+ if not args.pretend:
+ old.dumps(args.cache)
+ args.cache.close()
sys.exit()
sys.exit(1)
View
@@ -19,11 +19,15 @@
class Config(object):
- def __init__(self, filename):
+ def __init__(self, filename, verbose, pretend):
self._teams = {}
self._repos = {}
self._fork_urls = {}
+
self.filename = filename
+ self.verbose = verbose
+ self.pretend = pretend
+
if type(filename) is file:
data = filename.read()
elif type(filename) in [str, unicode] and \
@@ -37,37 +41,38 @@ def __init__(self, filename):
data = f.read()
f.close()
else:
- raise NotImplemented
+ data = filename
if data:
- self.parse(data)
+ self._teams, self._repos, self._fork_urls = self.parse(data)
def parse(self, data):
- data = json.loads(data)
+ teams, repos = {}, {}
+ try:
+ data = json.loads(data)
+ except:
+ import ipdb; ipdb.set_trace()
+
for team in data['teams']:
team = Team(**team)
- self._teams[team.name] = team
+ teams[team.name] = team
+
for repo in data['repos']:
repo = Repo(**repo)
- self._repos[repo.name] = repo
-
- def dumps(self, filename=None):
- if filename is None:
- filename = self.filename
+ repos[repo.name] = repo
- if type(filename) is file:
- f = filename
- elif self.is_url(filename):
- raise Exception('Can save only locally, not remotly. '
- 'Wrong filename: %s!' % filename)
- else:
- f = open(filename, 'w+')
+ return teams, repos, data['fork_urls']
+ def dumps(self, cache):
+ if cache.mode != 'w+':
+ cache = open(cache.name, 'w+')
+ cache.truncate(0)
+ cache.seek(0)
json.dump({
'teams': [self._teams[name].dumps() for name in self.teams],
'repos': [self._repos[name].dumps() for name in self.repos],
- }, f, indent=4)
- f.close()
+ 'fork_urls': self._fork_urls,
+ }, cache, indent=4)
def is_url(self, url):
return url.startswith('http')
@@ -107,23 +112,24 @@ def get_fork_url(self, repo):
class ConfigCFG(Config):
def parse(self, data):
+ teams, repos, fork_urls = {}, {}, {}
config = ConfigParser.SafeConfigParser()
config.readfp(StringIO.StringIO(data))
for section in config.sections():
if section.startswith('repo:'):
# add repo
name = section[len('repo:'):]
- self._repos[name] = Repo(name)
+ repos[name] = Repo(name)
# add fork
- self._fork_urls[name] = None
if config.has_option(section, 'fork'):
- self._fork_urls[name] = config.get(section, 'fork')
+ fork_urls[name] = config.get(section, 'fork')
# add owners team
- team_name = TEAM_PREFIX + name + TEAM_OWNERS_SUFFIX
- team_members = config.get(section, 'owners').split()
- self._teams[team_name] = Team(team_name, 'admin',
- members=team_members, repos=[name])
+ if config.has_option(section, 'owners'):
+ team_name = TEAM_PREFIX + name + TEAM_OWNERS_SUFFIX
+ team_members = config.get(section, 'owners').split()
+ teams[team_name] = Team(team_name, 'admin',
+ members=team_members, repos=[name])
elif section.startswith('team:'):
# add team
name = TEAM_PREFIX + section[len('team:'):]
@@ -133,31 +139,42 @@ def parse(self, data):
members = []
if config.has_option(section, 'members'):
members = config.get(section, 'members').split()
- repos = []
+ team_repos = []
if config.has_option(section, 'repos'):
- repos = config.get(section, 'repos').split()
- self._teams[name] = Team(name, permission,
- members=members, repos=repos)
+ team_repos = config.get(section, 'repos').split()
+ teams[name] = Team(name, permission,
+ members=members, repos=team_repos)
# add repos to teams (defined with repo: section
for section in config.sections():
if section.startswith('repo:'):
if config.has_option(section, 'teams'):
for team in config.get(section, 'teams').split():
- self._teams[TEAM_PREFIX + team].repos.add(
+ teams[TEAM_PREFIX + team].repos.add(
section[len('repo:'):],
)
+ return teams, repos, fork_urls
+
class ConfigGithub(Config):
- def __init__(self, github=None):
- self._cache = {}
+ def __init__(self, github, cache, verbose=False, pretend=False):
self.github = github
-
- @property
- def _teams(self):
- if 'teams' not in self._cache.keys():
- self._cache['teams'] = {}
+ self._github = {'teams': {}, 'repos': {}}
+
+ data = None
+ if cache:
+ data = cache.read()
+ super(ConfigGithub, self).__init__(data, verbose, pretend)
+ if cache and not data:
+ print 'CACHE DOES NOT EXISTS! CACHING...'
+ self.dumps(cache)
+ print 'CACHE WRITTEN TO %s!' % cache.name
+
+ def _get_teams(self):
+ if 'teams' not in self._github.keys() or \
+ not self._github['teams']:
+ self._github['teams'] = {}
for item in self.github._gh_org_teams():
if not item['name'].startswith(TEAM_PREFIX):
continue
@@ -169,15 +186,25 @@ def _teams(self):
if team.repos_count > 0:
team.repos.update([i['name']
for i in self.github._gh_team_repos(item['id'])])
- self._cache['teams'][team.name] = team
- return self._cache['teams']
-
- @property
- def _repos(self):
- if 'repos' not in self._cache.keys():
- self._cache['repos'] = {}
+ self._github['teams'][team.name] = team
+ return self._github['teams']
+ def _set_teams(self, value):
+ self._github['teams'] = value
+ def _del_teams(self):
+ del self._github['teams']
+ _teams = property(_get_teams, _set_teams, _del_teams)
+
+ def _get_repos(self):
+ if 'repos' not in self._github.keys() or \
+ not self._github['repos']:
+ self._github['repos'] = {}
for item in self.github._gh_org_repos():
repo = Repo(**item)
- self._cache['repos'][repo.name] = repo
- return self._cache['repos']
+ self._github['repos'][repo.name] = repo
+ return self._github['repos']
+ def _set_repos(self, value):
+ self._github['repos'] = value
+ def _del_repos(self):
+ del self._github['repos']
+ _repos = property(_get_repos, _set_repos, _del_repos)
@@ -41,7 +41,6 @@ def _request(self, method, url, data=None):
self._request_count += 1
self._request_limit = response.headers['x-ratelimit-limit']
self._request_remaining = response.headers['x-ratelimit-remaining']
- response.raise_for_status()
if self.verbose:
print '%s - %s/%s - %s - %s' % (
self._request_count,
@@ -50,6 +49,10 @@ def _request(self, method, url, data=None):
method.__name__.upper(),
kw['url'],
)
+ try:
+ response.raise_for_status()
+ except:
+ import ipdb; ipdb.set_trace()
return response
def _get_request(self, url):
@@ -97,7 +100,9 @@ def _gh_org_fork_repo(self, fork_url):
return self._post_request('/repos/%s/forks' % fork_url, {'org': self.org})
def _gh_org_create_repo(self, name):
- return self._post_request('/orgs/%s/repos' % self.org, {'name': name})
+ return self._post_request('/orgs/%s/repos' % self.org, json.dumps({
+ 'name': name,
+ }))
def _gh_org_create_team(self, name, permission='pull'):
assert permission in ['pull', 'push', 'admin']
Oops, something went wrong.

0 comments on commit beac5e7

Please sign in to comment.