diff --git a/remove-duplicate/Readme.md b/remove-duplicate/Readme.md new file mode 100644 index 000000000..ffe85d0ac --- /dev/null +++ b/remove-duplicate/Readme.md @@ -0,0 +1,16 @@ +# Python Clean Duplicate String from file + +## Setup and activate virtual environment : +For Unix based systems please execute the following command to create venv and install requirements. +``` +make init +source .venv/bin/activate +``` + +## Running : + +``` +pip install -r requirements.txt +python clean.py +``` + diff --git a/remove-duplicate/clean.py b/remove-duplicate/clean.py new file mode 100644 index 000000000..88ebbfa49 --- /dev/null +++ b/remove-duplicate/clean.py @@ -0,0 +1,20 @@ +import tqdm +import time +listnya = input('input list : ') +output = input('Output Destination : ') +lists = open(listnya) +cok = set(lists.readlines()) +total = len(list(open(listnya))) +print("[+] Found Total {} List".format(str(total))) +print("[+] Start Remove Duplicate ...\n\n") +time.sleep(1) +count = 0 +result = listnya.replace(".txt", "") +file = result + '-clean.txt' +folder = str(output) +for i in tqdm.tqdm(cok): + time.sleep(0) + count += 1 + open(folder + file, 'a').write(i) +final = total - count +print("\nSuccess Remove {} From list".format(str(final))) diff --git a/remove-duplicate/requirements.txt b/remove-duplicate/requirements.txt new file mode 100644 index 000000000..a083eaa95 --- /dev/null +++ b/remove-duplicate/requirements.txt @@ -0,0 +1 @@ +tqdm==4.62.3