@@ -339,7 +339,7 @@ if __name__ == "__main__": if len(args) > 1: filename = args[1] else: - os.environ["ORIG_FILENAMES"] = os.environ.get("ORIG_FILENAMES") or str(True) + os.environ["ORIG_FILENAMES"] = os.environ.get("ORIG_FILENAMES") or "1" filename = io.safe_filename(urlrewrite.url_to_filename(url)) Fetcher(mode=Fetcher.FETCH, url=url, filename=filename).launch() except filetype.WrongFileTypeError: fetch.py @@ -12,7 +12,7 @@ import shcolor _help_header = "spiderfetch tool suite\n\n" -_tools_help="""\ +_help_tools="""\ == spiderfetch == Spiders recursively for urls, starting from <url>. Driven either by <pattern> @@ -46,6 +46,19 @@ An automation module for use with mplayer to record media streams. Reads urls from a file and records with mplayer. """ +_help_vars="""\ +SOCKET_TIMEOUT Seconds to wait before calling a socket timeout. +TRIES Number of tries on 503 Service Unavailable. + +ORIG_FILENAMES Save files with their original filenames on the host (1) or + use filenames generated from the full url to avoid name collisions (0). +TMPDIR Temp directory for downloads. +LOGDIR Directory to use for logfiles. + +TERM When set and not 'dumb' gives color output. +DEBUG_FETCH Write newlines after every update to see the full output. +""" + #LOGDIR = os.environ.get("LOGDIR") or "logs" LOGDIR = os.environ.get("LOGDIR") or "." @@ -60,7 +73,7 @@ def write_abort(): write_err("\n%s\n" % shcolor.color(shcolor.RED, "User aborted")) def get_tempfile(): - return tempfile.mkstemp(prefix=os.path.basename("." + sys.argv[0]) + ".") + return tempfile.mkstemp(prefix="."+os.path.basename(sys.argv[0])+".") def safe_filename(filename, dir=None): if dir: @@ -133,13 +146,18 @@ def opts_help(option, opt_str, value, parser): sys.exit(2) def help_tools(option, opt_str, value, parser): - write_err(_help_header+_tools_help) + write_err(_help_header+_help_tools) + sys.exit(2) + +def help_vars(option, opt_str, value, parser): + write_err(_help_header+_help_vars) sys.exit(2) def parse_args(parser): a = parser.add_option a("-h", action="callback", callback=opts_help, help="Display this message") a("--tools", action="callback", callback=help_tools, help="Descriptions of the tools") + a("--vars", action="callback", callback=help_vars, help="Environmental variables") return parser.parse_args() io.py @@ -194,11 +194,11 @@ if __name__ == "__main__": (opts, args) = io.parse_args(parser) try: if opts.fetch: - os.environ["FETCH_ALL"] = str(True) + os.environ["FETCH_ALL"] = "1" elif opts.dump: - os.environ["DUMP_ALL"] = str(True) + os.environ["DUMP_ALL"] = "1" if opts.host: - os.environ["HOST_FILTER"] = str(True) + os.environ["HOST_FILTER"] = "1" if opts.depth: os.environ["DEPTH"] = str(opts.depth) spiderfetch.py 8f6c5d066b1eea6d894f99509aa2072b2efeab90 Martin Matusiak numerodix@gmail.com http://github.com/numerodix/spiderfetch/commit/c5130c4195cddfdd52cdf79ef4df7186e443d8c7 c5130c4195cddfdd52cdf79ef4df7186e443d8c7 2008-06-27T14:30:21-07:00 2008-06-27T14:30:21-07:00 add help for env vars a7946633ede895f6319cacfcf6ae728493114fd9 Martin Matusiak numerodix@gmail.com