@@ -38,33 +38,39 @@ def safe_filename(filename, dir=None): filename = os.path.join(path, root + "-" + str(serial) + ext) return os.path.basename(filename) -def create_logdir(): - if not os.path.exists(LOGDIR): - os.makedirs(LOGDIR) +def create_dir(dir): + if not os.path.exists(dir): + os.makedirs(dir) -def logdir(filename): - path = os.path.dirname(filename) - if path: - return filename - return os.path.join(LOGDIR, os.path.basename(filename)) +def file_exists(filename, dir=None): + if dir: + filename = os.path.join(dir, filename) + return os.path.exists(filename) + +def delete(filename, dir=None): + if dir: + filename = os.path.join(dir, filename) + return os.unlink(filename) def savelog(s, filename, mode=None): - create_logdir() + create_dir(LOGDIR) mode = mode or 'w' - open(logdir(filename), mode).write(s) + open(os.path.join(LOGDIR, filename), mode).write(s) -def serialize(o, filename): - create_logdir() +def serialize(o, filename, dir=None): + if dir: + create_dir(dir) + filename = os.path.join(dir, filename) try: getattr(o, "_to_pickle")() except AttributeError: pass #fp = gzip.GzipFile(logdir(filename), 'w', compresslevel=1) - pickle.dump(o, open(logdir(filename), 'w'), pickle.HIGHEST_PROTOCOL) + pickle.dump(o, open(filename, 'w'), pickle.HIGHEST_PROTOCOL) -def deserialize(filename): - if not os.path.exists(filename): - filename = logdir(filename) +def deserialize(filename, dir=None): + if dir: + filename = os.path.join(dir, filename) #fp = gzip.GzipFile(filename, 'r') o = pickle.load(open(filename, 'r')) try: io.py @@ -22,31 +22,31 @@ def save_session(wb, queue=None): filename = urlrewrite.hostname_to_filename(hostname) io.write_err("Saving session to %s ..." % shcolor.color(shcolor.YELLOW, filename+".{web,session}")) - io.serialize(wb, filename + ".web") + io.serialize(wb, filename + ".web", dir=io.LOGDIR) if queue: - io.serialize(queue, filename + ".session") + io.serialize(queue, filename + ".session", dir=io.LOGDIR) # only web being saved, ie. spidering complete, remove old session - elif os.path.exists(io.logdir(filename + ".session")): - os.unlink(io.logdir(filename + ".session")) + elif io.file_exists(filename + ".session", dir=io.LOGDIR): + io.delete(filename + ".session", dir=io.LOGDIR) io.write_err(shcolor.color(shcolor.GREEN, "done\n")) def restore_session(url): hostname = urlrewrite.get_hostname(url) filename = urlrewrite.hostname_to_filename(hostname) - if (os.path.exists(io.logdir(filename+".session")) and - os.path.exists(io.logdir(filename+".web"))): + if (io.file_exists(filename + ".session", dir=io.LOGDIR) and + io.file_exists(filename + ".web", dir=io.LOGDIR)): io.write_err("Restoring session from %s ..." % shcolor.color(shcolor.YELLOW, filename+".{web,session}")) - q = io.deserialize(filename + ".session") + q = io.deserialize(filename + ".session", dir=io.LOGDIR) q = recipe.overrule_records(q) - wb = io.deserialize(filename + ".web") + wb = io.deserialize(filename + ".web", dir=io.LOGDIR) io.write_err(shcolor.color(shcolor.GREEN, "done\n")) return q, wb return None, None def log_exc(exc, url, wb): exc_filename = io.safe_filename("exc", dir=io.LOGDIR) - io.serialize(exc, exc_filename) + io.serialize(exc, exc_filename, dir=io.LOGDIR) s = traceback.format_exc() s += "\nBad url: |%s|\n" % url node = wb.get(url) spiderfetch.py 49f20e35e9b289d39cb8ea934db6de11f371e95f Martin Matusiak numerodix@gmail.com http://github.com/numerodix/spiderfetch/commit/eda049dfd6c600c99c1de34ec0240139dde742b1 eda049dfd6c600c99c1de34ec0240139dde742b1 2008-06-27T10:10:49-07:00 2008-06-27T10:10:49-07:00 kill faulty logdir() wrapping 19866618abc4957393c53f62007a417808e9f3cf Martin Matusiak numerodix@gmail.com