From 598edc7e75e4e37a4d41940add85528e14739fae Mon Sep 17 00:00:00 2001 From: Andrew Shearer Date: Fri, 17 Dec 2010 12:29:21 -0500 Subject: [PATCH 1/6] Show progress with percent complete during import from Delicious --- delicious_import.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/delicious_import.py b/delicious_import.py index b92ec6e7..f82c2518 100644 --- a/delicious_import.py +++ b/delicious_import.py @@ -25,7 +25,8 @@ def __init__(self, username, password=''): x = minidom.parseString(content) # sample post: - for post in x.getElementsByTagName("post"): + post_list = x.getElementsByTagName('post') + for post_index, post in enumerate(post_list): url = post.getAttribute('href') desc = post.getAttribute('description') tags = ",".join([t for t in post.getAttribute('tag').split()]) @@ -39,6 +40,10 @@ def __init__(self, username, password=''): args = [url] g = gitMark(options, args) + if post_list.length > 1: + print '%d of %d bookmarks imported (%d%%)' % ( + post_index + 1, post_list.length, + (post_index + 1.0) / post_list.length * 100) if __name__ == '__main__': From 0a1a493e24130139fc9f3da8594a4dd5f8876f66 Mon Sep 17 00:00:00 2001 From: Andrew Shearer Date: Fri, 17 Dec 2010 12:33:20 -0500 Subject: [PATCH 2/6] Fix errors due to quotes or other shell-recognized characters in description --- gitmark.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/gitmark.py b/gitmark.py index 9e71e85e..55a655a3 100644 --- a/gitmark.py +++ b/gitmark.py @@ -56,8 +56,7 @@ def gitAdd(self, files): pipe.wait() def gitCommit(self, msg): - pipe = subprocess.Popen("git commit -m '%s'" % msg, shell=True) - pipe.wait() + subprocess.call(['git', 'commit', '-m', msg]) def gitPush(self): pipe = subprocess.Popen("git push origin master", shell=True) @@ -106,4 +105,4 @@ def getContent(self, url): opts = {'push': options.push, 'tags': options.tags, 'msg': options.msg} - g = gitMark(opts, args) \ No newline at end of file + g = gitMark(opts, args) From e8dd60438048ec0d8cfb50614f769d4f3c7124b7 Mon Sep 17 00:00:00 2001 From: Andrew Shearer Date: Fri, 17 Dec 2010 12:34:14 -0500 Subject: [PATCH 3/6] Add each list of files in a single git command --- gitmark.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/gitmark.py b/gitmark.py index 55a655a3..bfbe9f86 100644 --- a/gitmark.py +++ b/gitmark.py @@ -50,11 +50,8 @@ def __init__(self, options, args): self.gitPush() def gitAdd(self, files): - for f in files: - cmd = 'git add %s' % f - pipe = subprocess.Popen(cmd, shell=True) - pipe.wait() - + subprocess.call(['git', 'add'] + files) + def gitCommit(self, msg): subprocess.call(['git', 'commit', '-m', msg]) From 91e0667d9d98117ec968b69097a986e9fd1f4281 Mon Sep 17 00:00:00 2001 From: Andrew Shearer Date: Fri, 17 Dec 2010 12:58:00 -0500 Subject: [PATCH 4/6] Log an error but save the bookmark and continue if downloading content fails --- gitmark.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/gitmark.py b/gitmark.py index bfbe9f86..736e2b36 100644 --- a/gitmark.py +++ b/gitmark.py @@ -86,9 +86,15 @@ def generateHash(self, text): return m.hexdigest() def getContent(self, url): - h = urllib.urlopen(url) - content = h.read() - h.close() + try: + h = urllib.urlopen(url) + content = h.read() + h.close() + except IOError, e: + print >>sys.stderr, ("Error: could not retrieve the content of a" + " URL. The bookmark will be saved, but its content won't be" + " searchable. URL: <%s>. Error: %s" % (url, e)) + content = '' return content From d8f9b8bfc923e202033713a954fc52fe7c8a10dc Mon Sep 17 00:00:00 2001 From: Andrew Shearer Date: Fri, 17 Dec 2010 13:10:20 -0500 Subject: [PATCH 5/6] Restore Windows compatibility, which my shell escaping fix would have broken. Windows needs to go through the shell in order to find git on the PATH, but it's OK to use the shell there because it will automatically escape discrete arguments. Warning: this is theoretical work. Untested on Windows. --- gitmark.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/gitmark.py b/gitmark.py index 736e2b36..023878be 100644 --- a/gitmark.py +++ b/gitmark.py @@ -17,6 +17,12 @@ from settings import * +# Arguments are passed directly to git, not through the shell, to avoid the +# need for shell escaping. On Windows, however, commands need to go through the +# shell for git to be found on the PATH, but escaping is automatic there. So +# send git commands through the shell on Windows, and directly everywhere else. +USE_SHELL = os.name == 'nt' + class gitMark(object): def __init__(self, options, args): @@ -50,10 +56,10 @@ def __init__(self, options, args): self.gitPush() def gitAdd(self, files): - subprocess.call(['git', 'add'] + files) + subprocess.call(['git', 'add'] + files, shell=USE_SHELL) def gitCommit(self, msg): - subprocess.call(['git', 'commit', '-m', msg]) + subprocess.call(['git', 'commit', '-m', msg], shell=USE_SHELL) def gitPush(self): pipe = subprocess.Popen("git push origin master", shell=True) From b3912ba3ea9b8c83099011c87b0912280db13782 Mon Sep 17 00:00:00 2001 From: Andrew Shearer Date: Fri, 17 Dec 2010 13:57:33 -0500 Subject: [PATCH 6/6] Preserve bookmark timestamps during Delicious import. --- delicious_import.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/delicious_import.py b/delicious_import.py index f82c2518..b3763469 100644 --- a/delicious_import.py +++ b/delicious_import.py @@ -37,9 +37,16 @@ def __init__(self, username, password=''): options['push'] = False options['msg'] = desc - args = [url] + # Set the authoring date of the commit based on the imported + # timestamp. git reads the GIT_AUTHOR_DATE environment var. + os.environ['GIT_AUTHOR_DATE'] = timestamp + args = [url] g = gitMark(options, args) + + # Reset authoring timestamp (abundance of caution) + del os.environ['GIT_AUTHOR_DATE'] + if post_list.length > 1: print '%d of %d bookmarks imported (%d%%)' % ( post_index + 1, post_list.length,