Module implementing a multi-threaded hdd io queue.
pip install mhddq
import mhddq
import time
# calls are enqueued instead of being called immediately
# 2 alternating io threads handle all io functions
# ideally while one thread is dealing with queue maintenance the
# other thread is using the hdd, and then they trade off when done
# after a function has been completed by the io threads its output
# is enqueued for a third thread to handle the callback as to not
# interfere with the two io threads
mhddq.file_exists(callback = cb_file_exists, filename = './file.txt')
def cb_file_exists(args):
# args = {
# 'action': str, # function name, 'file_exists' in this case
# 'result': bool, # if the function encoutered any errors then this is False
# 'params': dict, # the params you initially passed, { 'callback': function, 'filename': str }
# 'output': dict # the output of the function, in this case { 'exists': bool }
# # if result is False then this would be { 'exception': str }
# }
# print the result, 'file.txt exists: False'
print(args['params']['filename'] + ' exists: ' + str(args['output']['exists']))
# if a callback throws an exception it is caught and printed to stderr
# if multiple operations need to be grouped together you can use an oplist
fileops = mhddq.create_empty_oplist()
# oplists have their own sets of function calls
# in practice it ends up being the same as mhddq.file_exists()
# except the io thread considers the entire oplist as a single
# operation and won't yield to it's sister thread until the list
# has been completed
mhddq.oplist_file_exists(oplist = fileops, callback = cb_file_exists, filename = './file.txt')
# you can add an oplist directly to the io queue like this
mhddq.enqueue_oplist(fileops)
# or you can embed the oplist in another oplist
superlist = mhddq.create_empty_oplist()
mhddq.embed_oplist(superlist, fileops)
# embedded lists are processed as a single operation inside it's
# parent list, while it's parent list is processed as a single operation
# in the io queue
mhddq.enqueue_oplist(superlist)
# when it is time to turn off the app you can do a graceful shutdown
# this will prevent any new items from being enqueued and blocks the
# calling thread until the io and callback threads have cleared their
# queues and exited their processing loops
mhddq.shutdown()
# other threads can wait for a graceful shutdown by using a loop like this
while mhddq.is_shutdown() is False:
time.sleep(0.01)
# once shutdown, mhddq can not be restarted without restarting the app