-
Notifications
You must be signed in to change notification settings - Fork 0
/
tables_8ch_p4c.py
160 lines (143 loc) · 9.03 KB
/
tables_8ch_p4c.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
#-------------------------------------------------------------------------------
# Name: module1
# Purpose:
#
# Author: User
#
# Created: 19-11-2018
# Copyright: (c) User 2018
# Licence: <your licence>
#-------------------------------------------------------------------------------
# StdLib
import logging
import logging.handlers
import datetime
# Remote libraries
import sqlalchemy# For talking to DBs
from sqlalchemy.ext.declarative import declarative_base
# local
def table_factory_boards(Base):# Boards table
""" We're Java now!
Make the 'x_board' table for an 8chan archive.
see https://stackoverflow.com/questions/19163911/dynamically-setting-tablename-for-sharding-in-sqlalchemy
TODO: Sane database design
"""
table_name = u'boards'
logging.debug(u'Naming the board table {0!r}'.format(table_name))
class Boards(Base):
__tablename__ = table_name
# From: https://github.com/bibanon/py8chan/blob/master/py8chan/board.py#L76
# Use 'b_VarName' format to maybe avoid fuckery. (Name confusion, name collission, reserved keywords such as 'id', etc)
b_id = sqlalchemy.Column(sqlalchemy.Integer, nullable=True, unique=True)
b_name = sqlalchemy.Column(sqlalchemy.Unicode, nullable=True)
b_title = sqlalchemy.Column(sqlalchemy.Unicode, nullable=True)
b_is_worksafe = sqlalchemy.Column(sqlalchemy.Boolean, nullable=True)
b_page_count= sqlalchemy.Column(sqlalchemy.Integer, nullable=True)
b_threads_per_page = sqlalchemy.Column(sqlalchemy.Integer, nullable=True)
# Misc recordkeeping (internal use and also for exporting dumps more easily)
row_created = sqlalchemy.Column(sqlalchemy.DateTime, nullable=True, default=datetime.datetime.utcnow)
row_updated = sqlalchemy.Column(sqlalchemy.DateTime, nullable=True, onupdate=datetime.datetime.utcnow)
primary_key = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True)
return Boards
def table_factory_threads(Base, board_name):# Threads table
"""We're Java now!
Make the 'x_threads' table for an 8chan board.
see https://stackoverflow.com/questions/19163911/dynamically-setting-tablename-for-sharding-in-sqlalchemy
TODO: Sane database design"""
logging.debug(u'table_factory_threads() args={0!r}'.format(locals()))# Record arguments.
assert(type(board_name) in [unicode])
table_name = u'{0}_threads'.format(board_name)
logging.debug(u'Naming the thread table {0!r}'.format(table_name))
class Threads(Base):
__tablename__ = table_name
# From: https://github.com/bibanon/py8chan/blob/master/py8chan/thread.py#L10
# Use 't_VarName' format to maybe avoid fuckery. (Name confusion, name collission, reserved keywords such as 'id', etc)
t_thread_id = sqlalchemy.Column(sqlalchemy.Integer, nullable=True, unique=True)
t_board = sqlalchemy.Column(sqlalchemy.Integer, nullable=True)
t_last_reply_id = sqlalchemy.Column(sqlalchemy.Integer, nullable=True, unique=True)
# py8chan.Thread class attributes
t_closed = sqlalchemy.Column(sqlalchemy.Boolean, nullable=True)
t_sticky = sqlalchemy.Column(sqlalchemy.Unicode, nullable=True)
t_topic = sqlalchemy.Column(sqlalchemy.Unicode, nullable=True)
t_url = sqlalchemy.Column(sqlalchemy.Unicode, nullable=True, unique=True)
# Misc recordkeeping: (internal use and also for exporting dumps more easily)
row_created = sqlalchemy.Column(sqlalchemy.DateTime, nullable=True, default=datetime.datetime.utcnow)
row_updated = sqlalchemy.Column(sqlalchemy.DateTime, nullable=True, onupdate=datetime.datetime.utcnow)
primary_key = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True)
return Threads
def table_factory_posts(Base, board_name):# Posts table
"""We're Java now!
Make the 'x_posts' table for an 8chan board.
see https://stackoverflow.com/questions/19163911/dynamically-setting-tablename-for-sharding-in-sqlalchemy
TODO: Sane database design"""
logging.debug(u'table_factory_posts() args={0!r}'.format(locals()))# Record arguments.
assert(type(board_name) in [unicode])
table_name = u'{0}_posts'.format(board_name)
logging.debug(u'Naming the posts table {0!r}'.format(table_name))
class Posts(Base):
__tablename__ = table_name
# From: https://github.com/bibanon/py8chan/blob/master/py8chan/post.py#L13
# Use 'p_VarName' format to maybe avoid fuckery. (Name confusion, name collission, reserved keywords such as 'id', etc)
p_thread_id = sqlalchemy.Column(sqlalchemy.Integer, nullable=True)
# py8chan.Post class attributes
p_post_id = sqlalchemy.Column(sqlalchemy.Integer, nullable=True, unique=True)
p_poster_id = sqlalchemy.Column(sqlalchemy.Integer, nullable=True)
p_name = sqlalchemy.Column(sqlalchemy.Unicode, nullable=True)
p_email = sqlalchemy.Column(sqlalchemy.Unicode, nullable=True)
p_tripcode = sqlalchemy.Column(sqlalchemy.Unicode, nullable=True)
p_subject = sqlalchemy.Column(sqlalchemy.Unicode, nullable=True)
p_comment = sqlalchemy.Column(sqlalchemy.Unicode, nullable=True)
p_html_comment = sqlalchemy.Column(sqlalchemy.Unicode, nullable=True)
p_text_comment = sqlalchemy.Column(sqlalchemy.Unicode, nullable=True)
p_is_op = sqlalchemy.Column(sqlalchemy.Boolean, nullable=True)
p_timestamp = sqlalchemy.Column(sqlalchemy.Integer, nullable=True)
p_datetime = sqlalchemy.Column(sqlalchemy.Integer, nullable=True)
p_has_file = sqlalchemy.Column(sqlalchemy.Boolean, nullable=True)
p_has_extra_files = sqlalchemy.Column(sqlalchemy.Boolean, nullable=True)
p_url = sqlalchemy.Column(sqlalchemy.Unicode, nullable=True, unique=True)
# Misc recordkeeping (internal use and also for exporting dumps more easily)
row_created = sqlalchemy.Column(sqlalchemy.DateTime, nullable=True, default=datetime.datetime.utcnow)
row_updated = sqlalchemy.Column(sqlalchemy.DateTime, nullable=True, onupdate=datetime.datetime.utcnow)
primary_key = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True)
return Posts
def table_factory_files(Base, board_name):# File table
"""We're Java now!
Make the 'x_files' table for an 8chan board.
see https://stackoverflow.com/questions/19163911/dynamically-setting-tablename-for-sharding-in-sqlalchemy
TODO: Sane database design"""
logging.debug(u'table_factory_files() args={0!r}'.format(locals()))# Record arguments.
assert(type(board_name) in [unicode])
table_name = u'{0}_files'.format(board_name)
logging.debug(u'Naming the file table {0!r}'.format(table_name))
class File(Base):
__tablename__ = table_name
# https://github.com/bibanon/py8chan/blob/master/py8chan/file.py#L15
# Use 'm_VarName' format to maybe avoid fuckery. (Name confusion, name collission, reserved keywords such as 'id', etc)
m_post_id = sqlalchemy.Column(sqlalchemy.Integer, nullable=True, unique=True)# TODO: Make this a foreign key linked to posts table
# py8chan.File class attributes
m_file_md5_hex = sqlalchemy.Column(sqlalchemy.Unicode, nullable=True)
m_filename_original = sqlalchemy.Column(sqlalchemy.Unicode, nullable=True)
m_filename = sqlalchemy.Column(sqlalchemy.Unicode, nullable=True)
m_file_url = sqlalchemy.Column(sqlalchemy.Unicode, nullable=True)
m_file_extension = sqlalchemy.Column(sqlalchemy.Unicode, nullable=True)
m_file_size = sqlalchemy.Column(sqlalchemy.Integer, nullable=True)
m_file_width = sqlalchemy.Column(sqlalchemy.Integer, nullable=True)
m_file_height = sqlalchemy.Column(sqlalchemy.Integer, nullable=True)
m_thumbnail_width = sqlalchemy.Column(sqlalchemy.Integer, nullable=True)
m_thumbnail_height = sqlalchemy.Column(sqlalchemy.Integer, nullable=True)
m_thumbnail_fname = sqlalchemy.Column(sqlalchemy.Unicode, nullable=True)
m_thumbnail_url = sqlalchemy.Column(sqlalchemy.Unicode, nullable=True)
# archive-side values
archive_filename = sqlalchemy.Column(sqlalchemy.Unicode, nullable=True)# What the file is called on the archive's disk
file_saved = sqlalchemy.Column(sqlalchemy.Boolean, nullable=True)# Set to False if the file is lost from the server. Will be set to True when file is saved.
thumbnail_saved = sqlalchemy.Column(sqlalchemy.Boolean, nullable=True)# Set to False if the thumbnail is lost from the server. Will be set to True when thumbnail is saved.
forbidden = sqlalchemy.Column(sqlalchemy.Boolean, nullable=True)# If True any file matching this MD5 will not be downloaded.
# Misc recordkeeping: (internal use and also for exporting dumps more easily)
row_created = sqlalchemy.Column(sqlalchemy.DateTime, nullable=True, default=datetime.datetime.utcnow)
row_updated = sqlalchemy.Column(sqlalchemy.DateTime, nullable=True, onupdate=datetime.datetime.utcnow)
primary_key = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True)
return File
def main():
pass
if __name__ == '__main__':
main()