@@ -36,8 +36,6 @@ def main(): if __name__ == "__main__": - parser = optparse.OptionParser(add_help_option=None) ; a = parser.add_option - parser.usage = "< <file>" - a("-h", action="callback", callback=io.opts_help, help="Display this message") - (opts, args) = parser.parse_args() + (parser, a) = io.init_opts("< <file>") + (opts, args) = io.parse_args(parser) main() dumpstream.py @@ -3,7 +3,6 @@ import ftplib import httplib import mimetools -import optparse import os import socket import sys @@ -220,7 +219,6 @@ class Fetcher(object): if blocknum == 1: self.started = True - #step = 12 step = 5 if blocknum > 0 and blocknum % step == 0: t = time.time() @@ -327,11 +325,9 @@ class Fetcher(object): if __name__ == "__main__": - parser = optparse.OptionParser(add_help_option=None) ; a = parser.add_option - parser.usage = "<url> [<file>] [options]" + (parser, a) = io.init_opts("<url> [<file>] [options]") a("--spidertest", action="store_true", help="Test spider with url") - a("-h", action="callback", callback=io.opts_help, help="Display this message") - (opts, args) = parser.parse_args() + (opts, args) = io.parse_args(parser) try: urllib._urlopener = urllib.FancyURLopener() url = args[0] fetch.py @@ -2,6 +2,7 @@ import gzip import cPickle as pickle # cPickle is supposed to be faster +import optparse import os import tempfile import sys @@ -90,6 +91,16 @@ def opts_help(option, opt_str, value, parser): write_err(" %s %s\n" % (argument.strip().ljust(25), o.help)) sys.exit(2) +def init_opts(usage): + parser = optparse.OptionParser(add_help_option=None) + parser.usage = usage + return parser, parser.add_option + +def parse_args(parser): + a = parser.add_option + a("-h", action="callback", callback=opts_help, help="Display this message") + return parser.parse_args() + if __name__ == "__main__": io.py @@ -1,7 +1,6 @@ #!/usr/bin/env python import itertools -import optparse import re import urllib @@ -137,12 +136,10 @@ def colorize_shell(str): if __name__ == "__main__": - parser = optparse.OptionParser(add_help_option=None) ; a = parser.add_option - parser.usage = "[ <url> [options] | --test ]" + (parser, a) = io.init_opts("[ <url> [options] | --test ]") a("--dump", action="store_true", help="Dump urls") - a("-h", action="callback", callback=io.opts_help, help="Display this message") a("--test", action="store_true", help="Run spider testsuite") - (opts, args) = parser.parse_args() + (opts, args) = io.parse_args(parser) try: if opts.test: data = testcases spider.py @@ -1,6 +1,5 @@ #!/usr/bin/env python -import optparse import os import re import sys @@ -186,15 +185,13 @@ def main(queue, rules, wb): if __name__ == "__main__": - parser = optparse.OptionParser(add_help_option=None) ; a = parser.add_option - parser.usage = "<url> [<pattern>] [options]" + (parser, a) = io.init_opts("<url> [<pattern>] [options]") a("--recipe", metavar="<recipe>", dest="recipe", help="Use a spidering recipe") a("--fetch", action="store_true", help="Fetch urls, don't dump") a("--dump", action="store_true", help="Dump urls, don't fetch") a("--host", action="store_true", help="Only spider this host") a("--depth", type="int", metavar="<depth>", dest="depth", help="Spider to this depth") - a("-h", action="callback", callback=io.opts_help, help="Display this message") - (opts, args) = parser.parse_args() + (opts, args) = io.parse_args(parser) try: if opts.fetch: os.environ["FETCH_ALL"] = str(True) spiderfetch.py @@ -1,6 +1,5 @@ #!/usr/bin/env python -import optparse import pickle import sys @@ -172,8 +171,7 @@ class Web(object): if __name__ == "__main__": - parser = optparse.OptionParser(add_help_option=None) ; a = parser.add_option - parser.usage = "<web> [options]" + (parser, a) = io.init_opts("<web> [options]") a("--dump", action="store_true", help="Dump all urls in web") a("--in", metavar="<url>", dest="into", help="Find incoming urls to <url>") a("--out", metavar="<url>", help="Find outgoing urls from <url>") @@ -182,9 +180,8 @@ if __name__ == "__main__": a("--trace", metavar="<url>", help="Trace path from root to <url>") a("--deepest", action="store_true", help="Trace url furthest from root") a("--popular", action="store_true", help="Find the most referenced urls") - a("-h", action="callback", callback=io.opts_help, help="Display this message") a("--test", action="store_true", help="Run trace loop test") - (opts, args) = parser.parse_args() + (opts, args) = io.parse_args(parser) try: if opts.test: wb = Web() web.py 78874ff665d7f6c4403b039b0af6bc0a2bffcb19 Martin Matusiak numerodix@gmail.com http://github.com/numerodix/spiderfetch/commit/e93e1634c9f75343401e9ab7a9fe20aa78f24ac3 e93e1634c9f75343401e9ab7a9fe20aa78f24ac3 2008-06-27T13:33:20-07:00 2008-06-27T13:33:20-07:00 rewrite opts logic 46fb041a9ba955bf1f8396698d5057ae94e9f44b Martin Matusiak numerodix@gmail.com