Skip to content

Commit

Permalink
Fixed concurrent access to cache file when using tldextract in multip…
Browse files Browse the repository at this point in the history
…le threads
  • Loading branch information
Thomas Perrot authored and john-kurkowski committed Aug 6, 2020
1 parent 5a2dd21 commit a6e7277
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 4 deletions.
2 changes: 1 addition & 1 deletion pylintrc
Original file line number Diff line number Diff line change
Expand Up @@ -358,7 +358,7 @@ max-statements=50
max-parents=7

# Maximum number of attributes for a class (see R0902).
max-attributes=7
max-attributes=8

# Minimum number of public methods for a class (see R0903).
min-public-methods=1
Expand Down
9 changes: 6 additions & 3 deletions tldextract/tldextract.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@
import os
import pkgutil
import re
import threading

import idna

Expand Down Expand Up @@ -193,6 +194,7 @@ def __init__(self, cache_file=CACHE_FILE, suffix_list_urls=PUBLIC_SUFFIX_LIST_UR
self.suffix_list_urls = tuple(url.strip() for url in suffix_list_urls if url.strip())

self.cache_file = os.path.expanduser(cache_file or '')
self.cache_file_lock = threading.Lock()
self.fallback_to_snapshot = fallback_to_snapshot
if not (self.suffix_list_urls or self.cache_file or self.fallback_to_snapshot):
raise ValueError("The arguments you have provided disable all ways for tldextract "
Expand Down Expand Up @@ -264,8 +266,8 @@ def _get_tld_extractor(self):

if self._extractor:
return self._extractor

tlds = self._get_cached_tlds()
with self.cache_file_lock:
tlds = self._get_cached_tlds()
if tlds:
tlds.extend(self.extra_suffixes)
self._extractor = _PublicSuffixListTLDExtractor(tlds)
Expand All @@ -289,7 +291,8 @@ def _get_tld_extractor(self):
raise Exception("tlds is empty, but fallback_to_snapshot is set"
" to false. Cannot proceed without tlds.")

self._cache_tlds(tlds)
with self.cache_file_lock:
self._cache_tlds(tlds)

tlds.extend(self.extra_suffixes)
self._extractor = _PublicSuffixListTLDExtractor(tlds)
Expand Down

0 comments on commit a6e7277

Please sign in to comment.