Permalink
Cannot retrieve contributors at this time
executable file
431 lines (341 sloc)
15.2 KB
| #!/usr/bin/env python3 | |
| # by teknohog | |
| # Automatic assignment handler for manual testing at mersenne.org and | |
| # optionally gpu72.com. | |
| # Written with mfakto in mind, this only handles trial factoring work | |
| # for now. It should work with mfaktc as well. | |
| # This version can run in parallel with the factoring program, as it | |
| # uses lockfiles to avoid conflicts when updating files. | |
| import http.cookiejar | |
| import math | |
| import os | |
| import re | |
| import sys | |
| import urllib | |
| from datetime import datetime | |
| from optparse import OptionParser | |
| from time import sleep | |
| primenet_baseurl = "https://www.mersenne.org/" | |
| gpu72_baseurl = "https://www.gpu72.com/" | |
| def ass_generate(assignment): | |
| output = "" | |
| for key in assignment: | |
| output += key + "=" + assignment[key] + "&" | |
| #return output.rstrip("&") | |
| return output | |
| def debug_print(text): | |
| if options.debug: | |
| print(progname + ": " + text) | |
| def exp_increase(l, max_exp): | |
| output = [] | |
| for line in l: | |
| # Increase the upper limit to max_exp | |
| s = re.search(r",([0-9]+)$", line) | |
| if s: | |
| exp = int(s.groups()[0]) | |
| new_exp = str(max(exp, max_exp)) | |
| output.append(re.sub(r",([0-9]+)$", "," + new_exp, line)) | |
| return output | |
| def greplike(pattern, l): | |
| output = [] | |
| for line in l: | |
| s = re.search(r"(" + pattern + ")$", line) | |
| if s: | |
| output.append(s.groups()[0]) | |
| return output | |
| def num_topup(l, targetsize): | |
| num_existing = len(l) | |
| num_needed = targetsize - num_existing | |
| return max(num_needed, 0) | |
| def ghzd_topup(l, ghdz_target): | |
| ghzd_existing = 0.0 | |
| for line in l: | |
| pieces = line.split(",") | |
| # calculate ghz-d http://mersenneforum.org/showpost.php?p=152280&postcount=204 | |
| exponent = int(pieces[1]) | |
| first_bit = int(pieces[2]) + 1 | |
| for bits in range(first_bit, int(pieces[3]) + 1): | |
| if bits > 65: | |
| timing = 28.50624 # 2.4 * 0.00707 * 1680.0 | |
| elif bits == 64: | |
| timing = 28.66752 # 2.4 * 0.00711 * 1680.0 | |
| elif bits == 63 or bits == 62: | |
| timing = 29.95776 # 2.4 * 0.00743 * 1680.0 | |
| elif bits >= 48: | |
| timing = 18.7488 # 2.4 * 0.00465 * 1680.0 | |
| else: | |
| continue | |
| bit_ghzd = timing * (1 << (bits - 48)) / exponent | |
| # if there is a checkpoint file, subtract the work done | |
| if bits == first_bit: | |
| checkpoint_file = os.path.join(workdir, "M"+str(exponent)+".ckp") | |
| if os.path.isfile(checkpoint_file): | |
| File = open(checkpoint_file, "r") | |
| checkpoint = File.readline() | |
| File.close() | |
| checkpoint_pieces = checkpoint.split(" ") | |
| if checkpoint_pieces[4] == "mfakto": | |
| progress_index = 6 | |
| else: | |
| progress_index = 5 | |
| percent_done = float(checkpoint_pieces[progress_index]) / float(checkpoint_pieces[3]) | |
| bit_ghzd *= 1 - percent_done | |
| debug_print(str(datetime.now()) + " " + "Found checkpoint file for assignment M"+str(exponent)+" indicating "+str(round(percent_done*100,2))+"% done.") | |
| ghzd_existing += bit_ghzd | |
| debug_print(str(datetime.now()) + " " + "Found " + str(ghzd_existing) + " of existing GHz-days of work") | |
| return max(0, math.ceil(ghdz_target - ghzd_existing)) | |
| def readonly_file(filename): | |
| # Used when there is no intention to write the file back, so don't | |
| # check or write lockfiles. Also returns a single string, no list. | |
| if os.path.exists(filename): | |
| File = open(filename, "r") | |
| contents = File.read() | |
| File.close() | |
| else: | |
| contents = "" | |
| return contents | |
| def read_list_file(filename): | |
| # Used when we plan to write the new version, so use locking | |
| lockfile = filename + ".lck" | |
| try: | |
| fd = os.open(lockfile, os.O_CREAT | os.O_EXCL) | |
| os.close(fd) | |
| if os.path.exists(filename): | |
| File = open(filename, "r") | |
| contents = File.readlines() | |
| File.close() | |
| return map(lambda x: x.rstrip(), contents) | |
| else: | |
| return [] | |
| except OSError as e: | |
| if e.errno == 17: | |
| return "locked" | |
| else: | |
| raise | |
| def write_list_file(filename, l, mode="w"): | |
| # Assume we put the lock in upon reading the file, so we can | |
| # safely write the file and remove the lock | |
| lockfile = filename + ".lck" | |
| # A "null append" is meaningful, as we can call this to clear the | |
| # lockfile. In this case the main file need not be touched. | |
| if mode != "a" or len(l) > 0: | |
| content = "\n".join(l) + "\n" | |
| File = open(filename, mode) | |
| File.write(content) | |
| File.close() | |
| os.remove(lockfile) | |
| def primenet_fetch(num_to_get, ghzd_to_get = 0): | |
| if not primenet_login: | |
| return [] | |
| # Manual assignment settings; trial factoring = 2 | |
| assignment = {"cores": "1", | |
| "num_to_get": str(num_to_get), | |
| "ghz_to_get": "", | |
| "pref": "1", | |
| "pref2": "1", | |
| "exp_lo": "10000000", | |
| "exp_hi": "999999999", | |
| } | |
| if ghzd_to_get: | |
| assignment[ghz_to_get] = str(ghzd_to_get) | |
| try: | |
| r = primenet.open(primenet_baseurl + "manual_gpu_assignment/?" + ass_generate(assignment) + "B1=Get+Assignments") | |
| lines = [line.decode(encoding="ISO-8859-1") for line in r.readlines()] | |
| return exp_increase(greplike(workpattern, lines), int(options.max_exp)) | |
| except urllib.error.URLError: | |
| debug_print(str(datetime.now()) + " URL open error at primenet_fetch") | |
| return [] | |
| def gpu72_fetch(num_to_get, ghzd_to_get = 0): | |
| if options.gpu72_type == "dctf": | |
| gpu72_type = "dctf" | |
| else: | |
| gpu72_type = "lltf" | |
| if options.gpu72_option == "lowest_tf_level": | |
| option = "1" | |
| elif options.gpu72_option == "highest_tf_level": | |
| option = "2" | |
| elif options.gpu72_option == "lowest_exponent": | |
| option = "3" | |
| elif options.gpu72_option == "oldest_exponent": | |
| option = "4" | |
| elif gpu72_type == "dctf" and options.gpu72_option == "no_p1_done": | |
| option = "5" | |
| elif gpu72_type == "lltf" and options.gpu72_option == "lhm_bit_first": | |
| option = "6" | |
| elif gpu72_type == "lltf" and options.gpu72_option == "lhm_depth_first": | |
| option = "7" | |
| elif options.gpu72_option == "let_gpu72_decide": | |
| option = "9" | |
| else: | |
| option = "0" | |
| if ghzd_to_get > 0: | |
| num_to_get_str = "0" | |
| ghzd_to_get_str = str(ghzd_to_get) | |
| else: | |
| num_to_get_str = str(num_to_get) | |
| ghzd_to_get_str = "" | |
| assignment = {"Number": num_to_get_str, | |
| "GHzDays": ghzd_to_get_str, | |
| "Low": "0", | |
| "High": "10000000000", | |
| "Pledge": str(max(70, int(options.max_exp))), | |
| "Option": option, | |
| } | |
| try: | |
| # This makes a POST instead of GET | |
| data = urllib.parse.urlencode(assignment).encode() | |
| r = gpu72.open(gpu72_baseurl + "account/getassignments/" + gpu72_type + "/", data) | |
| lines = [line.decode(encoding="ISO-8859-1") for line in r.readlines()] | |
| new_tasks = greplike(workpattern, lines) | |
| # Remove dupes | |
| return list(set(new_tasks)) | |
| except urllib.error.HTTPError as e: | |
| debug_print(str(datetime.now()) + " " + "HTTP error at gpu72_fetch: " + str(e.code) + " " + e.reason) | |
| except urllib.error.URLError: | |
| debug_print(str(datetime.now()) + " " + "URL open error at gpu72_fetch") | |
| return [] | |
| def get_assignment(): | |
| w = read_list_file(workfile) | |
| if w == "locked": | |
| return "locked" | |
| tasks = greplike(workpattern, w) | |
| if options.ghzd_cache != "": | |
| ghzd_to_get = ghzd_topup(tasks, int(options.ghzd_cache)) | |
| num_to_get = 0 | |
| else: | |
| ghzd_to_get = 0 | |
| num_to_get = num_topup(tasks, int(options.num_cache)) | |
| if num_to_get < 1 and ghzd_to_get == 0: | |
| debug_print(str(datetime.now()) + " Cache full, not getting new work") | |
| # Must write something anyway to clear the lockfile | |
| new_tasks = [] | |
| else: | |
| if use_gpu72 and ghzd_to_get > 0: | |
| debug_print(str(datetime.now()) + " Fettching " + str(ghzd_to_get) + " GHz-days of assignments") | |
| new_tasks = gpu72_fetch(num_to_get, ghzd_to_get) | |
| elif use_gpu72 and num_to_get > 0: | |
| debug_print(str(datetime.now()) + " Fetching " + str(num_to_get) + " assignments") | |
| new_tasks = gpu72_fetch(num_to_get) | |
| else: | |
| new_tasks = [] | |
| # Fallback to primenet in case of problems | |
| if options.fallback == "1" and len(new_tasks) == 0: | |
| debug_print(str(datetime.now()) + " Retrieved nothing from gpu72. Fetching from PrimeNet.") | |
| new_tasks = primenet_fetch(num_to_get, ghzd_to_get) | |
| debug_print(str(datetime.now()) + " Fetched " + str(len(new_tasks)) + " new assignment(s).") | |
| write_list_file(workfile, new_tasks, "a") | |
| def mersenne_find(line, complete=True): | |
| work = readonly_file(workfile) | |
| s = re.search(r"M([0-9]*) ", line) | |
| if s: | |
| mersenne = s.groups()[0] | |
| if not "," + mersenne + "," in work: | |
| return complete | |
| else: | |
| return not complete | |
| else: | |
| return False | |
| def submit_work(): | |
| # Only submit completed work, i.e. the exponent must not exist in | |
| # worktodo.txt any more | |
| files = [resultsfile, sentfile] | |
| rs = list(map(read_list_file, files)) | |
| if "locked" in rs: | |
| # Remove the lock in case one of these was unlocked at start | |
| for i in range(len(files)): | |
| if rs[i] != "locked": | |
| write_list_file(files[i], [], "a") | |
| return "locked" | |
| results = rs[0] | |
| # Only for new results, to be appended to results_sent | |
| sent = [] | |
| # Use the textarea form to submit several results at once. | |
| # Useless lines (not including a M#) are now discarded completely. | |
| results_send = list(filter(mersenne_find, results)) | |
| results_keep = list(filter(lambda x: mersenne_find(x, complete=False), results)) | |
| if len(results_send) == 0: | |
| debug_print(str(datetime.now()) + " " + "No complete results found to send.") | |
| # Don't just return here, files are still locked... | |
| else: | |
| while len(results_send) > 0: | |
| sendbatch = [] | |
| while sum(map(len, sendbatch)) < sendlimit and \ | |
| len(results_send) > 0: | |
| sendbatch.append(results_send.pop(0)) | |
| data = "\n".join(sendbatch) | |
| debug_print(str(datetime.now()) + " " + "Submitting\n" + data) | |
| try: | |
| post_data = urllib.parse.urlencode({"data": data}).encode() | |
| r = primenet.open(primenet_baseurl + "manual_result/default.php", post_data) | |
| res = r.read().decode() | |
| if "processing:" in res or "Accepted" in res: | |
| sent += sendbatch | |
| else: | |
| results_keep += sendbatch | |
| debug_print(str(datetime.now()) + " " + "Submission failed.") | |
| except urllib.error.URLError: | |
| results_keep += sendbatch | |
| debug_print(str(datetime.now()) + " " + "URL open error") | |
| write_list_file(resultsfile, results_keep) | |
| write_list_file(sentfile, sent, "a") | |
| parser = OptionParser() | |
| parser.add_option("-d", "--debug", action="store_true", dest="debug", default=False, help="Display debugging info") | |
| parser.add_option("-e", "--exp", dest="max_exp", default="72", help="Upper limit of exponent, default 72") | |
| parser.add_option("-T", "--gpu72type", dest="gpu72_type", default="lltf", help="GPU72 type of work, lltf or dctf, default lltf.") | |
| parser.add_option("-o", "--gpu72option", dest="gpu72_option", default="what_makes_sense", help="GPU72 Option to fetch, default what_makes_sense. Other valid values are lowest_tf_level, highest_tf_level, lowest_exponent, oldest_exponent, no_p1_done (dctf only), lhm_bit_first (lltf only), lhm_depth_first (lltf only), and let_gpu72_decide (let_gpu72_decide may override max_exp).") | |
| parser.add_option("-u", "--username", dest="username", help="Primenet user name") | |
| parser.add_option("-p", "--password", dest="password", help="Primenet password") | |
| parser.add_option("-w", "--workdir", dest="workdir", default=".", help="Working directory with worktodo.txt and results.txt, default current") | |
| parser.add_option("-U", "--gpu72user", dest="guser", help="GPU72 user name", default="") | |
| parser.add_option("-P", "--gpu72pass", dest="gpass", help="GPU72 password") | |
| parser.add_option("-n", "--num_cache", dest="num_cache", default="1", help="Number of assignments to cache, default 1") | |
| parser.add_option("-g", "--ghzd_cache", dest="ghzd_cache", default="", help="GHz-days of assignments to cache, taking into account checkpoint files. Overrides num_cache.") | |
| parser.add_option("-f", "--fallback", dest="fallback", default="1", help="Fall back to mersenne.org when GPU72 fails or has no work, default 1.") | |
| parser.add_option("-t", "--timeout", dest="timeout", default="3600", help="Seconds to wait between network updates, default 3600. Use 0 for a single update without looping.") | |
| (options, args) = parser.parse_args() | |
| use_gpu72 = (len(options.guser) > 0) | |
| progname = os.path.basename(sys.argv[0]) | |
| workdir = os.path.expanduser(options.workdir) | |
| timeout = int(options.timeout) | |
| workfile = os.path.join(workdir, "worktodo.txt") | |
| resultsfile = os.path.join(workdir, "results.txt") | |
| # A cumulative backup | |
| sentfile = os.path.join(workdir, "results_sent.txt") | |
| # Trial factoring | |
| workpattern = r"Factor=[^,]*(,[0-9]+){3}" | |
| # mersenne.org limit is about 4 KB; stay on the safe side | |
| sendlimit = 3000 | |
| # adapted from http://stackoverflow.com/questions/923296/keeping-a-session-in-python-while-making-http-requests | |
| primenet_cj = http.cookiejar.CookieJar() | |
| primenet = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(primenet_cj)) | |
| if use_gpu72: | |
| # Basic http auth | |
| password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm() | |
| password_mgr.add_password(None, gpu72_baseurl + "account/", options.guser, options.gpass) | |
| handler = urllib.request.HTTPBasicAuthHandler(password_mgr) | |
| gpu72 = urllib.request.build_opener(handler) | |
| while True: | |
| # Log in to primenet | |
| try: | |
| login_data = {"user_login": options.username, | |
| "user_password": options.password, | |
| } | |
| # This makes a POST instead of GET | |
| data = urllib.parse.urlencode(login_data).encode() | |
| r = primenet.open(primenet_baseurl + "default.php", data) | |
| if not options.username + "<br>logged in" in r.read().decode(): | |
| primenet_login = False | |
| debug_print(str(datetime.now()) + " " + "Login failed.") | |
| else: | |
| primenet_login = True | |
| while submit_work() == "locked": | |
| debug_print(str(datetime.now()) + " " + "Waiting for results file access...") | |
| sleep(2) | |
| except urllib.error.URLError: | |
| debug_print(str(datetime.now()) + " " + "Primenet URL open error") | |
| while get_assignment() == "locked": | |
| debug_print(str(datetime.now()) + " " + "Waiting for worktodo.txt access...") | |
| sleep(2) | |
| if timeout <= 0: | |
| break | |
| sleep(timeout) |