-
Notifications
You must be signed in to change notification settings - Fork 29
/
pysort.py
418 lines (394 loc) · 24.8 KB
/
pysort.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
from __future__ import print_function
import os
import sys
import time
import mmap
"""
Author Github: https://github.com/g666gle
Author Twitter: https://twitter.com/g666gle1
Date: 1/29/2019
Description: Takes in one file at a time as command line input. processes each line in the file and places the
information into the correct subdirectory of the data folder.
Usage: python3 pysort.py file.txt
Version: 1.0.0
Python Version: 3.6.7
"""
args = sys.argv
# TODO fix bug where spaces in import files
# TODO make another log file for usage
# TODO make an option for counting the number of files
# TODO Add zstd to compress the files once all of the files are imported
# TODO Add support for SQL vbull CSV Json
# FIXED
# - Bug where password:email files were being imputed as email:password
# - in run.sh fixed and added SimplyEmail as an option for harvesting email addresses
def check_duplicate(full_file_path, line):
"""
This function takes in a path to the file and the specific line we want to check for duplicates with. First the file
is checked to make sure it isn't empty, then the file is opened as a binary so we can store the lines as a mmap obj.
Next if the line is a duplicate then False is returned else True
:param full_file_path: Path to the file
:param line: The line being checked
:return: True if the line should be written to the file; else False
"""
# Check to see if the file is not empty
if not os.stat(full_file_path).st_size == 0:
# Open the file as a binary file and store it in a mmap obj
with open(full_file_path, 'rb', 0) as fp, mmap.mmap(fp.fileno(), 0, access=mmap.ACCESS_READ) as s:
# Check to see if the line already exists in the file
if s.find(str.encode(line.strip())) != -1:
return False # string is in file so do not re-write it
return True # string is not in file so write it to the file
return True # Write to the file
def place_data(line, path):
"""
This function takes in the line of the current file and the root path to the BaseQuery directory. Checks the format
of the file to make sure each line is in the email:password format. Then determines the depth of the correct characters
ex) ex|ample@gmail.com ---> would result in a depth of 2. Then checking each directory to see if it already exists
the username:password combo is correctly placed into a easy to query file. If a invalid character is determined in the
first 4 chars then it will be put in a '0UTLIERS.txt' file.
:param line: email:password
:param path: full path to file
:return: Either a 1 or a 0 depending on if a line has been written or not
"""
if line[0] == ":":
line = line[1:]
emailPaswd = line.split(':', 2)
# Checks to see if the users format is "Password:Username" instead of "Username:Password"
if len(emailPaswd) >= 2 and '@' in emailPaswd[1] and '.' in emailPaswd[1]:
# Switches the position of the username and password
temp = emailPaswd[0]
emailPaswd[0] = emailPaswd[1].lower()
emailPaswd[1] = temp
else:
# Change all of the email usernames to be lowercase; to be uniform
emailPaswd[0] = emailPaswd[0].lower()
try:
# check to see if you have a valid email address and the username is >= 4; also checks if there is a '@' in the username
if '@' in emailPaswd[0].strip() and len(emailPaswd[0].strip().split('@')[0]) >= 4 and len(emailPaswd) >= 2 and emailPaswd[0].strip().count('@') == 1:
first_letter = emailPaswd[0][0]
second_letter = emailPaswd[0][1]
third_letter = emailPaswd[0][2]
fourth_letter = emailPaswd[0][3]
# Check to see if the username has an invalid character and at what spot
if str(first_letter).isalnum():
folder_depth = 1
if str(second_letter).isalnum():
folder_depth = 2
if str(third_letter).isalnum():
folder_depth = 3
if str(fourth_letter).isalnum():
folder_depth = 4
else:
folder_depth = 0
# Check to see if the first letter doesn't have a directory
if not os.path.isdir(path + "/data/" + first_letter):
# Check to see if we start with at least one valid char
if folder_depth >= 1:
# Make the directory
os.makedirs(path + "/data/" + first_letter)
else:
# If the outlier dir doesn't exist; make it and start the file
if not os.path.isdir(path + "/data/0UTLIERS"):
os.makedirs(path + "/data/0UTLIERS")
# Don't need to check for duplicates because its a new file
with open(path + "/data/0UTLIERS/0utliers.txt", 'a') as fp:
length = len(emailPaswd)
# Iterate through each index of the list and write it to the file
for index in range(length):
if index != length - 1:
fp.write(emailPaswd[index] + ":")
else: # Don't add a ':' at the end of the line
fp.write(emailPaswd[index])
fp.write("\n")
return 1
else: # If the outlier dir already exists append the line to the file
if check_duplicate(path + "/data/0UTLIERS/0utliers.txt", line):
# Checks to see if there are duplicates already in the file, returns true if there isn't
with open(path + "/data/0UTLIERS/0utliers.txt", 'a') as fp:
length = len(emailPaswd)
# Iterate through each index of the list and write it to the file
for index in range(length):
if index != length - 1:
fp.write(emailPaswd[index] + ":")
else: # Don't add a ':' at the end of the line
fp.write(emailPaswd[index])
fp.write("\n")
return 1
return 0
else: # The directory already exists
if folder_depth == 0: # There is not at least one consecutive valid char
# If the outlier dir doesn't exist; make it and start the file
if not os.path.isdir(path + "/data/0UTLIERS"):
os.makedirs("mkdir " + path + "/data/0UTLIERS")
with open(path + "/data/0UTLIERS/0utliers.txt", 'a') as fp:
length = len(emailPaswd)
# Iterate through each index of the list and write it to the file
for index in range(length):
if index != length - 1:
fp.write(emailPaswd[index] + ":")
else: # Don't add a ':' at the end of the line
fp.write(emailPaswd[index])
fp.write("\n")
return 1
else: # If the outlier dir already exists append the line to the file
if check_duplicate(path + "/data/0UTLIERS/0utliers.txt", line):
with open(path + "/data/0UTLIERS/0utliers.txt", 'a') as fp:
length = len(emailPaswd)
# Iterate through each index of the list and write it to the file
for index in range(length):
if index != length - 1:
fp.write(emailPaswd[index] + ":")
else: # Don't add a ':' at the end of the line
fp.write(emailPaswd[index])
fp.write("\n")
return 1
return 0
# Check to see if the second letter doesn't have a directory
if not os.path.isdir(path + "/data/" + first_letter + "/" + second_letter):
# Check to see if we start with at least two valid char
if folder_depth >= 2:
# Make the directory
os.makedirs(path + "/data/" + first_letter + "/" + second_letter)
else:
# If the outlier dir doesn't exist; make it and start the file
if not os.path.isdir(path + "/data/" + first_letter + "/0UTLIERS"):
os.makedirs(path + "/data/" + first_letter + "/0UTLIERS")
with open(path + "/data/" + first_letter + "/0UTLIERS/0utliers.txt", 'a') as fp:
length = len(emailPaswd)
# Iterate through each index of the list and write it to the file
for index in range(length):
if index != length - 1:
fp.write(emailPaswd[index] + ":")
else: # Don't add a ':' at the end of the line
fp.write(emailPaswd[index])
fp.write("\n")
return 1
else:
# Check for duplicates
if check_duplicate(path + "/data/" + first_letter + "/0UTLIERS/0utliers.txt", line):
with open(path + "/data/" + first_letter + "/0UTLIERS/0utliers.txt", 'a') as fp:
length = len(emailPaswd)
# Iterate through each index of the list and write it to the file
for index in range(length):
if index != length - 1:
fp.write(emailPaswd[index] + ":")
else: # Don't add a ':' at the end of the line
fp.write(emailPaswd[index])
fp.write("\n")
return 1
return 0
else: # The directory already exists
if folder_depth <= 1: # There is not at least two consecutive valid char
# If the outlier dir doesn't exist; make it and start the file
if not os.path.isdir(path + "/data/" + first_letter + "/0UTLIERS"):
os.makedirs(path + "/data/" + first_letter + "/0UTLIERS")
with open(path + "/data/" + first_letter + "/0UTLIERS/0utliers.txt", 'a') as fp:
length = len(emailPaswd)
# Iterate through each index of the list and write it to the file
for index in range(length):
if index != length - 1:
fp.write(emailPaswd[index] + ":")
else: # Don't add a ':' at the end of the line
fp.write(emailPaswd[index])
fp.write("\n")
return 1
else: # If the outlier dir already exists append the line to the file
if check_duplicate(path + "/data/" + first_letter + "/0UTLIERS/0utliers.txt", line):
with open(path + "/data/" + first_letter + "/0UTLIERS/0utliers.txt", 'a') as fp:
length = len(emailPaswd)
# Iterate through each index of the list and write it to the file
for index in range(length):
if index != length - 1:
fp.write(emailPaswd[index] + ":")
else: # Don't add a ':' at the end of the line
fp.write(emailPaswd[index])
fp.write("\n")
return 1
return 0
# Check to see if the third letter doesn't have a directory
if not os.path.isdir(path + "/data/" + first_letter + "/" + second_letter + "/" + third_letter):
# Check to see if we start with at least three valid char
if folder_depth >= 3:
# Make the directory
os.makedirs(path + "/data/" + first_letter + "/" + second_letter + "/" + third_letter)
else:
if not os.path.isdir(path + "/data/" + first_letter + "/" + second_letter + "/0UTLIERS"):
os.makedirs(path + "/data/" + first_letter + "/" + second_letter + "/0UTLIERS")
with open(path + "/data/" + first_letter + "/" + second_letter + "/0UTLIERS/0utliers.txt", 'a') as fp:
length = len(emailPaswd)
# Iterate through each index of the list and write it to the file
for index in range(length):
if index != length - 1:
fp.write(emailPaswd[index] + ":")
else: # Don't add a ':' at the end of the line
fp.write(emailPaswd[index])
fp.write("\n")
return 1
else: # If the outlier dir already exists append the line to the file
if check_duplicate(path + "/data/" + first_letter + "/" + second_letter + "/0UTLIERS/0utliers.txt", line):
with open(path + "/data/" + first_letter + "/" + second_letter + "/0UTLIERS/0utliers.txt", 'a') as fp:
length = len(emailPaswd)
# Iterate through each index of the list and write it to the file
for index in range(length):
if index != length - 1:
fp.write(emailPaswd[index] + ":")
else: # Don't add a ':' at the end of the line
fp.write(emailPaswd[index])
fp.write("\n")
return 1
return 0
else: # The directory already exists
if folder_depth <= 2: # There is not at least three consecutive valid char
# If the outlier dir doesn't exist; make it and start the file
if not os.path.isdir(path + "/data/" + first_letter + "/" + second_letter + "/0UTLIERS"):
os.makedirs(path + "/data/" + first_letter + "/" + second_letter + "/0UTLIERS")
with open(path + "/data/" + first_letter + "/" + second_letter + "/0UTLIERS/0utliers.txt", 'a') as fp:
length = len(emailPaswd)
# Iterate through each index of the list and write it to the file
for index in range(length):
if index != length - 1:
fp.write(emailPaswd[index] + ":")
else: # Don't add a ':' at the end of the line
fp.write(emailPaswd[index])
fp.write("\n")
return 1
else: # If the outlier dir already exists append the line to the file
if check_duplicate(path + "/data/" + first_letter + "/" + second_letter + "/0UTLIERS/0utliers.txt", line):
with open(path + "/data/" + first_letter + "/" + second_letter + "/0UTLIERS/0utliers.txt", 'a') as fp:
length = len(emailPaswd)
# Iterate through each index of the list and write it to the file
for index in range(length):
if index != length - 1:
fp.write(emailPaswd[index] + ":")
else: # Don't add a ':' at the end of the line
fp.write(emailPaswd[index])
fp.write("\n")
return 1
return 0
# Checks to see if the file in the third directory doesn't exists
if not os.path.isfile(path + "/data/" + first_letter + "/" + second_letter + "/" + third_letter + "/" + fourth_letter + ".txt"):
if folder_depth == 4: # The file doesn't exist in the third dir but there is 4 valid chars
# Make the file
with open(path + "/data/" + first_letter + "/" + second_letter + "/" + third_letter + "/" + fourth_letter + ".txt", 'a') as output_file:
length = len(emailPaswd)
# Iterate through each index of the list and write it to the file
for index in range(length):
if index != length-1:
output_file.write(emailPaswd[index] + ":")
else: # Don't add a ':' at the end of the line
output_file.write(emailPaswd[index])
output_file.write("\n")
return 1
elif folder_depth == 3: # Check to see if the fourth letter is an outlier EX) exa!mple@example.com
if not os.path.isdir(path + "/data/" + first_letter + "/" + second_letter + "/" + third_letter + "/0UTLIERS"):
os.makedirs(path + "/data/" + first_letter + "/" + second_letter + "/" + third_letter + "/0UTLIERS")
# Make the 0UTLIERS file
with open(path + "/data/" + first_letter + "/" + second_letter + "/" + third_letter + "/0UTLIERS/0utliers.txt", 'a') as output_file:
if check_duplicate(path + "/data/" + first_letter + "/" + second_letter + "/" + third_letter + "/0UTLIERS/0utliers.txt", line):
length = len(emailPaswd)
# Iterate through each index of the list and write it to the file
for index in range(length):
if index != length - 1:
output_file.write(emailPaswd[index] + ":")
else: # Dont add a ':' at the end of the line
output_file.write(emailPaswd[index])
output_file.write("\n")
return 1
return 0
else: # The file exists
if folder_depth == 4: # The file does exist in the third dir but there is 4 valid chars
if check_duplicate(path + "/data/" + first_letter + "/" + second_letter + "/" + third_letter + "/" + fourth_letter + ".txt", line):
# Append the file
with open(path + "/data/" + first_letter + "/" + second_letter + "/" + third_letter + "/" + fourth_letter + ".txt", 'a') as output_file:
length = len(emailPaswd)
# Iterate through each index of the list and write it to the file
for index in range(length):
if index != length - 1:
output_file.write(emailPaswd[index] + ":")
else: # Dont add a ':' at the end of the line
output_file.write(emailPaswd[index])
output_file.write("\n")
return 1
return 0
elif folder_depth == 3: # The file does exist in the third dir but there is only 3 valid chars
# Check to see if you need to make the 0UTLIERS dir
if not os.path.isdir(path + "/data/" + first_letter + "/" + second_letter + "/" + third_letter + "/0UTLIERS"):
os.makedirs(path + "/data/" + first_letter + "/" + second_letter + "/" + third_letter + "/0UTLIERS")
# Check for duplicates and then write to the file
if check_duplicate(path + "/data/" + first_letter + "/" + second_letter + "/" + third_letter + "/0UTLIERS/0utliers.txt", line):
# Append the 0UTLIERS file
with open(path + "/data/" + first_letter + "/" + second_letter + "/" + third_letter + "/0UTLIERS/0utliers.txt", 'a') as output_file:
length = len(emailPaswd)
# Iterate through each index of the list and write it to the file
for index in range(length):
if index != length - 1:
output_file.write(emailPaswd[index] + ":")
else: # Dont add a ':' at the end of the line
output_file.write(emailPaswd[index])
output_file.write("\n")
return 1
return 0
# NOT a valid email address or the username is NOT >= 4; or there is more than one '@' in the username
else:
if not os.path.isdir(path + "/data/NOTVALID"):
os.makedirs(path + "/data/NOTVALID/")
with open(path + "/data/NOTVALID/FAILED_TEST.txt", 'a') as fp:
length = len(emailPaswd)
# Iterate through each index of the list and write it to the file
for index in range(length):
if index != length - 1:
fp.write(emailPaswd[index] + ":")
else: # Don't add a ':' at the end of the line
fp.write(emailPaswd[index])
fp.write("\n")
return 1
else: # The directory already exists
if line != "":
if check_duplicate(path + "/data/NOTVALID/FAILED_TEST.txt", line):
# Open the file; check if it's a duplicate and write to the file
with open(path + "/data/NOTVALID/FAILED_TEST.txt", 'a') as fp:
length = len(emailPaswd)
# Iterate through each index of the list and write it to the file
for index in range(length):
if index != length - 1:
fp.write(emailPaswd[index] + ":")
else: # Don't add a ':' at the end of the line
fp.write(emailPaswd[index])
fp.write("\n")
return 1
return 0
except OSError:
raise
return 0
def main():
start_time = time.time()
total_lines = 0 # The amount of lines that are not white-space
written_lines = 0 # The amount of lines written
RED = '\033[0;31m'
GREEN = '\033[0;32m'
YELLOW = '\033[1;33m'
NC = '\033[0m' # No Color
# Check to see if the arguments are correct
if len(args) == 2 and args[1] != "":
path = os.getcwd()
print(GREEN + "[+]" + NC + " Attempting to open file " + GREEN + args[1] + NC)
# Directory guaranteed to exist from previous check in Import.sh
with open(path + "/PutYourDataBasesHere/" + args[1], 'r') as fp:
try:
for line in fp:
if total_lines % 10000 == 0 and total_lines != 0:
print(GREEN + "[+]" + NC + " Processing line number: " + str(total_lines) + "\nLine: " + line)
if line.strip() != "":
written_lines += place_data(line.strip(), path)
total_lines += 1
except Exception as e:
print(RED + "Exception: " + str(e) + NC)
else:
print(YELLOW + "[!]" + NC + " Invalid arguments provided")
stop_time = time.time()
print()
print(GREEN + "[+]" + NC + " Total time: " + str(("%.2f" % (stop_time - start_time)) + " seconds"))
print(GREEN + "[+]" + NC + " Total lines: " + str(("%.2f" % total_lines)))
print(GREEN + "[+]" + NC + " Written lines: " + str(("%.2f" % written_lines)))
main()