-
Notifications
You must be signed in to change notification settings - Fork 0
/
main.py
101 lines (83 loc) · 2.91 KB
/
main.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
from chat_downloader import ChatDownloader, errors
import scrapetube
from datetime import datetime
from os import listdir, mkdir
import json
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("-c", "--Channel", help="Channel ID")
args = parser.parse_args()
if not args.Channel:
channel_id = str(input("Enter Channel ID"))
else:
channel_id = args.Channel
errorss = ""
import sqlite3
conn = sqlite3.connect("database.db")
cursor = conn.cursor()
vids = scrapetube.get_channel(channel_id, content_type="streams")
print(channel_id)
channel_id = channel_id.replace(
"-", "_"
) # YT is weird with channel ids. some have - in them. but sql tables cant have - in them. so we replace it with _
cursor.execute(
f"CREATE TABLE IF NOT EXISTS {channel_id} (stream_id varchar(255), user_id varchar(255), user_name varchar(255), user_avatar varchar(255), message_timestamp varchar(255), message_origin_time varchar(255), message_content varchar(255))"
)
conn.commit()
"""
for vid in video_ids_list:
for x in range(5):
cursor.execute(f"INSERT INTO {channel_id} (stream_id) VALUES ('{vid}')")
conn.commit()"""
cursor.execute(f"SELECT DISTINCT stream_id FROM {channel_id}")
stream_ids = cursor.fetchall()
x = stream_ids
stream_ids = []
stream_ids = [y[0] for y in x]
print(stream_ids)
videos = [vid for vid in vids if vid["videoId"] not in stream_ids and vid["thumbnailOverlays"][0]["thumbnailOverlayTimeStatusRenderer"]["style"] != "LIVE"]
processed_count = 0
def ignore_exc(iterable):
iterator = iter(iterable)
while True:
try:
item = next(iterator)
except StopIteration:
break
except:
continue
yield item
for vid in videos:
processed_count += 1
stream_id = vid["videoId"]
print(f"processing {stream_id}")
print(f"{processed_count}/{len(videos)}")
try:
chat = ChatDownloader().get_chat(stream_id)
except Exception as e:
continue
# print(json.dumps(vid, indent=4))
for message in ignore_exc(chat):
try:
user_id = message["author"]["id"]
user_name = message["author"]["name"]
user_avatar = message["author"]["images"][0]["url"]
message_timestamp = message["time_in_seconds"]
message_content = message["message"]
time_of_message = message["timestamp"] / 1000000
except Exception as e:
continue
cursor.execute(
f"INSERT INTO {channel_id} (stream_id, user_id, user_name, user_avatar, message_timestamp, message_origin_time, message_content) VALUES (?, ?, ?, ?, ?, ?, ?)",
(
stream_id,
user_id,
user_name,
user_avatar,
message_timestamp,
time_of_message,
message_content,
),
)
# print(json.dumps(message, indent=4))
conn.commit()