2.019 not holding _f_lock when not necessary

xeHentai/const.py

@@ -97,3 +97,7 @@
 ERR_RPC_METHOD_NOT_FOUND = -32601
 ERR_RPC_INVALID_PARAMS = -32602
 ERR_RPC_EXEC_ERROR = -32603
+
+
+class DownloadAbortedException(Exception):
+    pass

xeHentai/core.py

@@ -276,9 +276,9 @@ def _do_task(self, task_guid):
                     tid = 'down-%d' % (i + 1)
                     _ = self._get_httpworker(tid, task.img_q,
                         filters.download_file_wrapper(task.config['dir']),
-                        lambda x, tid = tid: (task.save_file(x[1], x[2], x[0]),
-                            self.logger.debug(i18n.XEH_FILE_DOWNLOADED % (task.get_fname(x[1]))),
-                            mon.vote(tid, 0)),
+                        lambda x, tid = tid: (task.save_file(x[1], x[2], x[0]) and \
+                            (self.logger.debug(i18n.XEH_FILE_DOWNLOADED % (task.get_fname(x[1]))),
+                                mon.vote(tid, 0))),
                         lambda x, tid = tid: (
                             task.page_q.put(task.get_reload_url(x[1])),# if x[0] != ERR_QUOTA_EXCEEDED else None,
                             task.reload_map.pop(x[1]) if x[1] in task.reload_map else None, # delete old url in reload_map

xeHentai/filters.py

@@ -111,10 +111,12 @@ def _(r, suc, fail):
             fail((ERR_CONNECTION_ERROR, r._real_url))
         elif r.status_code == 403:
             fail((ERR_KEY_EXPIRED, r._real_url))
-        elif r.status_code == 509 or len(r.content) in [925, 28658, 144, 210, 1009] or '509.gif' in r.url:
+        elif r.status_code == 509 or r.content_length in [925, 28658, 144, 210, 1009] or '509.gif' in r.url:
             fail((ERR_QUOTA_EXCEEDED, r._real_url))
             # will not call the decorated filter
-        elif len(r.content) < 200 and re.findall("exceeded your image viewing limits", r.text):
+        elif r.content_length < 200 and \
+                r.headers.get('content-type') and r.headers.get('content-type').startswith('text') and \
+                re.findall("exceeded your image viewing limits", r.text):
             fail((ERR_QUOTA_EXCEEDED, r._real_url))
             # will not call the decorated filter
         else:
@@ -187,8 +189,8 @@ def download_file(r, suc, fail, dirpath = dirpath):
         # if multiple hash-size-h-w-type is found, use the last one
         # the first is original and the last is scaled
         # _FakeReponse will be filtered in flt_quota_check
-        if not r.headers.get('content-length') or \
-            p and p[-1] and int(p[-1][1]) != int(r.headers.get('content-length')):
+        if not r.content_length or \
+            p and p[-1] and int(p[-1][1]) != r.content_length:
             return fail((ERR_IMAGE_BROKEN, r._real_url, r.url))
         if not hasattr(r, 'iter_content_cb'):
             return fail((ERR_STREAM_NOT_IMPLEMENTED, r._real_url, r.url))
@@ -200,9 +202,9 @@ def _yield(chunk_size=16384, _r=r):
                 length_read += len(_)
                 _r.iter_content_cb(_)
                 yield _
-            if length_read != int(r.headers.get('content-length')):
+            if length_read != r.content_length:
                 fail((ERR_IMAGE_BROKEN, r._real_url, r.url))
-                raise StopIteration()
+                raise DownloadAbortedException()
 
         suc((_yield, r._real_url, r.url))
 

xeHentai/task.py

@@ -199,16 +199,20 @@ def save_file(self, imgurl, redirect_url, binary_iter):
         self._cnt_lock.acquire()
         self.meta['finished'] += 1
         self._cnt_lock.release()
+        # create a femp file first
+        # we don't need _f_lock because this will not be in a sequence
+        # and we can't do that otherwise we are breaking the multi threading
+        fn_tmp = os.path.join(fpath, ".%s" % self.get_fidpad(int(fid)))
+        try:
+            with open(fn_tmp, "wb") as f:
+                for binary in binary_iter():
+                    f.write(binary)
+        except DownloadAbortedException as ex:
+            os.remove(fn_tmp)
+            return
         self._f_lock.acquire()
         try:
-            try:
-                with open(fn, "wb") as f:
-                    for binary in binary_iter():
-                        f.write(binary)
-            except StopIteration as ex:
-                os.remove(fn)
-                self._f_lock.release()
-                return
+            os.rename(fn_tmp, fn)
             if imgurl in self.filehash_map:
                 for fid, _ in self.filehash_map[imgurl]:
                     fn_rep = os.path.join(fpath, self.get_fidpad(fid))
@@ -219,6 +223,7 @@ def save_file(self, imgurl, redirect_url, binary_iter):
             self._f_lock.release()
             raise ex
         self._f_lock.release()
+        return True
 
     def get_fname(self, imgurl):
         pageurl, fname = self.reload_map[imgurl]

xeHentai/worker.py

@@ -56,7 +56,13 @@ def request(self, method, url, _filter, suc, fail, data=None, stream_cb=None):
                 self.logger.warning("%s-%s %s %s: %s" % (i18n.THREAD, self.tname, method, url, ex))
                 time.sleep(random.random() + 0.618)
             else:
-                self.logger.verbose("%s-%s %s %s %d %d" % (i18n.THREAD, self.tname, method, url, r.status_code, len(r.content)))
+                if r.headers.get('content-length'):
+                    r.content_length = int(r.headers.get('content-length'))
+                elif not stream_cb:
+                    r.content_length = len(r.content)
+                else:
+                    r.content_length = 0
+                self.logger.verbose("%s-%s %s %s %d %d" % (i18n.THREAD, self.tname, method, url, r.status_code, r.content_length))
                 # if it's a redirect, 3xx
                 if r.status_code > 300 and r.status_code < 400:
                     _new_url = r.headers.get("location")
@@ -68,9 +74,7 @@ def request(self, method, url, _filter, suc, fail, data=None, stream_cb=None):
                         url = _new_url
                         continue
                 # intercept some error to see if we can change IP
-                if self.proxy and \
-                    ((r.headers.get('content-length') and int(r.headers.get('content-length')) < 1024) or \
-                        (not stream_cb and len(r.content) < 1024)) and \
+                if self.proxy and r.content_length < 1024 and \
                     re.match("Your IP address has been temporarily banned", r.text):
                     _t = util.parse_human_time(r.text)
                     self.logger.warn(i18n.PROXY_DISABLE_BANNED % _t)