Python Remove Duplicate line from file

remove-duplicate/Readme.md

@@ -0,0 +1,16 @@
+# Python Clean Duplicate String from file
+
+## Setup and activate virtual environment :
+For Unix based systems please execute the following command to create venv and install requirements.
+```
+make init
+source .venv/bin/activate
+```
+
+## Running :
+
+```
+pip install -r requirements.txt
+python clean.py
+```
+

remove-duplicate/clean.py

@@ -0,0 +1,20 @@
+import tqdm
+import time
+listnya = input('input list : ')
+output = input('Output Destination : ')
+lists = open(listnya)
+cok = set(lists.readlines())
+total = len(list(open(listnya)))
+print("[+] Found Total {} List".format(str(total)))
+print("[+] Start Remove Duplicate ...\n\n")
+time.sleep(1)
+count = 0
+result = listnya.replace(".txt", "")
+file = result + '-clean.txt'
+folder = str(output)
+for i in tqdm.tqdm(cok):
+    time.sleep(0)
+    count += 1
+    open(folder + file, 'a').write(i)
+final = total - count
+print("\nSuccess Remove {} From list".format(str(final)))

remove-duplicate/requirements.txt

@@ -0,0 +1 @@
+tqdm==4.62.3