-
Notifications
You must be signed in to change notification settings - Fork 0
/
roaring_bitmap_search.py
98 lines (66 loc) · 2.83 KB
/
roaring_bitmap_search.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
import sys
import time
from collections import defaultdict
import humanize
from loguru import logger
from pyroaring import BitMap
from shared.command_line_args_support import base_arg_parser
from shared.memory_usage import get_current_memory_usage
from shared.valid_records_generator import get_all_valid_records
DATA_LIMIT = None
expired_passports = defaultdict(BitMap)
def load_expired_passports_into_roaring_bitmap(input_file: str) -> None:
records_added = 0
timestamp_begin = time.monotonic()
for index, record in enumerate(get_all_valid_records(input_file, logger)):
if DATA_LIMIT and (index >= DATA_LIMIT):
break
expired_passports[record[0]].add(record[1])
records_added += 1
if records_added and (not (records_added % 1_000_000)):
time_elapsed = time.monotonic() - timestamp_begin
logger.info(
f"Added records count: {records_added} (time spent: {humanize.precisedelta(time_elapsed)})"
)
logger.info(f"Records added: {records_added}")
def is_passport_blocked(series: int, number: int) -> bool:
if series not in expired_passports:
return False
if number not in expired_passports[series]:
return False
return True
def main():
logger.info("Application started")
parser = base_arg_parser()
args = parser.parse_args()
logger.debug(f"args: {args}")
logger.info("Uploading records into database")
timestamp_begin = time.monotonic()
load_expired_passports_into_roaring_bitmap(args.input_file)
timestamp_end = time.monotonic()
time_elapsed = timestamp_end - timestamp_begin
logger.info(f"Time elapsed for data uploading: {humanize.precisedelta(time_elapsed)}")
while True:
passport_series = input("Passport series: ")
passport_number = input("Passport series: ")
passport_series = passport_series.strip()
passport_number = passport_number.strip()
if (not passport_series) and (not passport_number):
logger.info("Empty request, going exit...")
break
passport_series = int(passport_series)
passport_number = int(passport_number)
timestamp_begin = time.perf_counter_ns()
passport_expired = is_passport_blocked(passport_series, passport_number)
timestamp_end = time.perf_counter_ns()
lookup_time_ns = (timestamp_end - timestamp_begin)
lookup_time_ms = lookup_time_ns / 1_000_000
logger.info(
f"Lookup time {round(lookup_time_ms, 3)} ms ({lookup_time_ns} ns). Is passport expired: {passport_expired}"
)
human_readable_memory_usage = humanize.naturalsize(get_current_memory_usage())
logger.info(f"Used memory: {human_readable_memory_usage}")
logger.info("Application finished")
if __name__ == "__main__":
main()
sys.exit(0)