Skip to content
Permalink
Browse files

Merge pull request #772 from sl1-1/Convert_User_Timestamps

Convert user timestamps from WeasylTimeStampColumn to TIMESTAMP

Reviewed-by: Charmander <~@charmander.me>
  • Loading branch information
charmander committed Mar 22, 2020
2 parents c031e5e + c128c7d commit e1301438c35d336c75be3c977b125ece274cafde
@@ -0,0 +1,34 @@
"""Convert profile timestamps to Datetime
Revision ID: 21660aaeb0d7
Revises: 096beed48d55
Create Date: 2020-02-27 18:02:57.260000
"""

# revision identifiers, used by Alembic.
revision = '21660aaeb0d7'
down_revision = 'e0320dc462db'

from alembic import op # lgtm[py/unused-import]
import sqlalchemy as sa # lgtm[py/unused-import]
from sqlalchemy.dialects.postgresql import TIMESTAMP

def upgrade():
op.alter_column('profile', 'unixtime',
existing_type=sa.INTEGER(),
type_=TIMESTAMP(timezone=True),
existing_nullable=False,
server_default=sa.text(u'now()'),
new_column_name="created_at",
postgresql_using="to_timestamp(unixtime + 18000)")


def downgrade():
op.alter_column('profile', 'created_at', server_default=None)
op.alter_column('profile', 'created_at',
existing_type=TIMESTAMP(timezone=True),
type_=sa.INTEGER(),
existing_nullable=False,
postgresql_using="extract(epoch from created_at) - 18000",
new_column_name='unixtime')
@@ -0,0 +1,31 @@
"""Convert last_login on login table to Datetime
Revision ID: 2e25bc9a0896
Revises: 3618355655c2
Create Date: 2020-02-27 15:02:32.668000
"""

# revision identifiers, used by Alembic.
revision = '2e25bc9a0896'
down_revision = '21660aaeb0d7'

from alembic import op # lgtm[py/unused-import]
import sqlalchemy as sa # lgtm[py/unused-import]
from sqlalchemy.dialects.postgresql import TIMESTAMP


def upgrade():
op.alter_column('login', 'last_login',
existing_type=sa.INTEGER(),
type_=TIMESTAMP(timezone=True),
existing_nullable=False,
postgresql_using="to_timestamp(last_login + 18000)")


def downgrade():
op.alter_column('login', 'last_login',
existing_type=TIMESTAMP(timezone=True),
type_=sa.INTEGER(),
existing_nullable=False,
postgresql_using="extract(epoch from last_login) - 18000")
@@ -0,0 +1,35 @@
"""Convert logincreate timestamps to TIMESTAMP(timzone=True)
Revision ID: 3accc3d526ba
Revises: 2e25bc9a0896
Create Date: 2020-02-27 16:06:48.018000
"""

# revision identifiers, used by Alembic.
revision = '3accc3d526ba'
down_revision = '2e25bc9a0896'

from alembic import op # lgtm[py/unused-import]
import sqlalchemy as sa # lgtm[py/unused-import]
from sqlalchemy.dialects.postgresql import TIMESTAMP


def upgrade():
op.alter_column('logincreate', 'unixtime',
existing_type=sa.INTEGER(),
server_default=sa.func.now(),
type_=TIMESTAMP(timezone=True),
existing_nullable=False,
postgresql_using="to_timestamp(unixtime + 18000)",
new_column_name='created_at')


def downgrade():
op.alter_column('logincreate', 'created_at', server_default=None)
op.alter_column('logincreate', 'created_at',
existing_type=TIMESTAMP(timezone=True),
type_=sa.INTEGER(),
existing_nullable=False,
postgresql_using="extract(epoch from created_at) - 18000",
new_column_name='unixtime')
@@ -326,7 +326,7 @@ def default_fkey(*args, **kwargs):
'login', metadata,
Column('userid', Integer(), primary_key=True, nullable=False),
Column('login_name', String(length=40), nullable=False, unique=True),
Column('last_login', WeasylTimestampColumn(), nullable=False),
Column('last_login', TIMESTAMP(timezone=True), nullable=False),
Column('force_password_reset', Boolean(), nullable=False, server_default='f'),
Column('email', String(length=100), nullable=False, server_default=''),
Column('twofa_secret', String(length=420), nullable=True),
@@ -356,7 +356,7 @@ def default_fkey(*args, **kwargs):
Column('hashpass', String(length=100), nullable=False),
Column('email', String(length=100), nullable=False, unique=True),
Column('birthday', WeasylTimestampColumn(), nullable=False),
Column('unixtime', WeasylTimestampColumn(), nullable=False),
Column('created_at', TIMESTAMP(timezone=True), nullable=False, server_default=func.now()),
# Used to determine if a record is invalid for purposes of plausible deniability of email addresses
# AKA, create a logincreate entry if an in-use email address is provided, thus preserving the effect of
# a pending username triggering a username taken error.
@@ -463,7 +463,7 @@ def default_fkey(*args, **kwargs):
Column('full_name', String(length=100), nullable=False),
Column('catchphrase', String(length=200), nullable=False, server_default=''),
Column('artist_type', String(length=100), nullable=False, server_default=''),
Column('unixtime', WeasylTimestampColumn(), nullable=False),
Column('created_at', TIMESTAMP(timezone=True), nullable=False, server_default=func.now()),
Column('latest_submission_time', ArrowColumn(), nullable=False, server_default='epoch'),
Column('profile_text', String(length=100000), nullable=False, server_default=''),
Column('settings', String(length=20), nullable=False, server_default='ccci'),
@@ -37,8 +37,8 @@ def user_with_age(age):
login_name = 'user%d' % next(_user_counter)
return users.Login(
info=users.UserInfo(birthday=birthday),
profile=users.Profile(username=login_name, full_name=login_name, unixtime=arrow.get(0)),
last_login=arrow.get(0),
profile=users.Profile(username=login_name, full_name=login_name, created_at=arrow.get(0).datetime),
last_login=arrow.get(0).datetime,
login_name=login_name)


@@ -134,7 +134,7 @@ def admincontrol_pending_accounts_get_(request):
:return: A Pyramid response with a webpage containing the pending accounts.
"""
query = d.engine.execute("""
SELECT token, username, email, invalid, invalid_email_addr, unixtime
SELECT token, username, email, invalid, invalid_email_addr, created_at
FROM logincreate
ORDER BY username
""").fetchall()
@@ -57,8 +57,8 @@ def run_periodic_tasks():
# Purge stale logincreate records older than two days
db.execute("""
DELETE FROM logincreate
WHERE unixtime < %(time)s
""", time=time_now - (86400 * 2))
WHERE created_at < (NOW() - INTERVAL '2 days')
""")
log.msg('cleared stale account creation records')

db.execute("UPDATE cron_runs SET last_run = %(now)s", now=now.naive)
@@ -582,6 +582,8 @@ def convert_to_localtime(target):
tz = get_current_request().weasyl_session.timezone
if isinstance(target, arrow.Arrow):
return tz.localtime(target.datetime)
elif isinstance(target, datetime.datetime):
return tz.localtime(target)
else:
target = int(get_time() if target is None else target) - _UNIXTIME_OFFSET
return tz.localtime_from_timestamp(target)
@@ -1033,7 +1035,7 @@ def metric(*a, **kw):


def iso8601(unixtime):
if isinstance(unixtime, arrow.Arrow):
if isinstance(unixtime, arrow.Arrow) or isinstance(unixtime, datetime.datetime):
return unixtime.isoformat().partition('.')[0] + 'Z'
else:
return datetime.datetime.utcfromtimestamp(unixtime - _UNIXTIME_OFFSET).isoformat() + 'Z'
@@ -7,6 +7,7 @@
import bcrypt
from publicsuffixlist import PublicSuffixList
from sqlalchemy.sql.expression import select
import sqlalchemy

from libweasyl import security
from libweasyl import staff
@@ -66,7 +67,7 @@ def clean_display_name(text):

def signin(request, userid, ip_address=None, user_agent=None):
# Update the last login record for the user
d.execute("UPDATE login SET last_login = %i WHERE userid = %i", [d.get_time(), userid])
d.execute("UPDATE login SET last_login = NOW() WHERE userid = %i", [userid])

# Log the successful login and increment the login count
d.append_to_log('login.success', userid=userid, ip=d.get_address())
@@ -269,7 +270,6 @@ def create(form):
"hashpass": passhash(password),
"email": email,
"birthday": birthday,
"unixtime": arrow.now(),
})

# Send verification email
@@ -287,7 +287,6 @@ def create(form):
"hashpass": passhash(password),
"email": token,
"birthday": arrow.now(),
"unixtime": arrow.now(),
"invalid": True,
# So we have a way for admins to determine which email address collided in the View Pending Accounts Page
"invalid_email_addr": email,
@@ -315,12 +314,13 @@ def verify(token, ip_address=None):
db = d.connect()
with db.begin():
# Create login record
userid = db.scalar(lo.insert().returning(lo.c.userid), {
"login_name": d.get_sysname(query.username),
"last_login": arrow.now(),
"email": query.email,
"ip_address_at_signup": ip_address,
})
userid = db.scalar(
lo.insert().values(
login_name=d.get_sysname(query.username),
last_login=sqlalchemy.func.now(),
email=query.email,
ip_address_at_signup=ip_address
).returning(lo.c.userid))

# Create profile records
db.execute(d.meta.tables["authbcrypt"].insert(), {
@@ -331,7 +331,6 @@ def verify(token, ip_address=None):
"userid": userid,
"username": query.username,
"full_name": query.username,
"unixtime": arrow.now(),
"config": "kscftj",
})
db.execute(d.meta.tables["userinfo"].insert(), {
@@ -368,11 +368,11 @@ def finduser(userid, form):

# Filter for date-time
if form.dateafter and form.datebefore:
q = q.where(d.sa.between(pr.c.unixtime, arrow.get(form.dateafter), arrow.get(form.datebefore)))
q = q.where(d.sa.between(pr.c.created_at, arrow.get(form.dateafter).datetime, arrow.get(form.datebefore).datetime))
elif form.dateafter:
q = q.where(pr.c.unixtime >= arrow.get(form.dateafter))
q = q.where(pr.c.created_at >= arrow.get(form.dateafter).datetime)
elif form.datebefore:
q = q.where(pr.c.unixtime <= arrow.get(form.datebefore))
q = q.where(pr.c.created_at <= arrow.get(form.datebefore).datetime)

# Apply any row offset
if form.row_offset:
@@ -127,7 +127,7 @@ def resolve_by_login(login):

def select_profile(userid, viewer=None):
query = d.engine.execute("""
SELECT pr.username, pr.full_name, pr.catchphrase, pr.unixtime, pr.profile_text,
SELECT pr.username, pr.full_name, pr.catchphrase, pr.created_at, pr.profile_text,
pr.settings, pr.stream_url, pr.config, pr.stream_text, us.end_time
FROM profile pr
INNER JOIN login lo USING (userid)
@@ -703,7 +703,7 @@ def select_manage(userid):
query = d.engine.execute("""
SELECT
lo.userid, lo.last_login, lo.email, lo.ip_address_at_signup,
pr.unixtime, pr.username, pr.full_name, pr.catchphrase,
pr.created_at, pr.username, pr.full_name, pr.catchphrase,
ui.birthday, ui.gender, ui.country, pr.config
FROM login lo
INNER JOIN profile pr USING (userid)
@@ -117,7 +117,7 @@ def parse(cls, query_string, find_default):
def select_users(q):
terms = q.lower().split()
statement = """
SELECT userid, full_name, unixtime, username FROM profile
SELECT userid, full_name, created_at, username FROM profile
WHERE LOWER(username) SIMILAR TO ('%%(' || %(terms)s || ')%%') ESCAPE ''
OR LOWER(full_name) SIMILAR TO ('%%(' || %(terms)s || ')%%') ESCAPE ''
ORDER BY username
@@ -131,7 +131,7 @@ def select_users(q):
"userid": i.userid,
"title": i.full_name,
"rating": "",
"unixtime": i.unixtime,
"unixtime": i.created_at,
"username": i.username,
} for i in query]
media.populate_with_user_media(ret)
@@ -46,7 +46,7 @@ def create_user(full_name="", birthday=arrow.get(586162800), config=None,

while True:
user = add_entity(users.Login(login_name=get_sysname(username),
last_login=arrow.get(0)))
last_login=arrow.get(0).datetime))

if user.userid not in staff.MODS and user.userid not in staff.DEVELOPERS:
break
@@ -56,7 +56,7 @@ def create_user(full_name="", birthday=arrow.get(586162800), config=None,
db.flush()

add_entity(users.Profile(userid=user.userid, username=username,
full_name=full_name, unixtime=arrow.get(0), config=config))
full_name=full_name, created_at=arrow.get(0).datetime, config=config))
add_entity(users.UserInfo(userid=user.userid, birthday=birthday))
# Verify this user
if verified:
@@ -191,7 +191,6 @@ def test_create_fails_if_pending_account_has_same_email():
"hashpass": login.passhash(raw_password),
"email": email_addr,
"birthday": arrow.Arrow(2000, 1, 1),
"unixtime": arrow.now(),
})
form = Bag(username="test", password='0123456789', passcheck='0123456789',
email=email_addr, emailcheck=email_addr,
@@ -253,7 +252,6 @@ def test_usernames_cannot_match_pending_account_usernames():
"hashpass": login.passhash(raw_password),
"email": "test0003@weasyl.com",
"birthday": arrow.Arrow(2000, 1, 1),
"unixtime": arrow.now(),
})
form = Bag(username=user_name, password='0123456789', passcheck='0123456789',
email=email_addr, emailcheck=email_addr,
@@ -28,7 +28,6 @@ def test_acct_verif_token_returned_if_email_provided_to_function():
"hashpass": login.passhash(raw_password),
"email": form.email,
"birthday": arrow.Arrow(2000, 1, 1),
"unixtime": arrow.now(),
})
acct_verification_token = login.get_account_verification_token(email=form.email, username=None)
assert token == acct_verification_token
@@ -46,7 +45,6 @@ def test_acct_verif_token_returned_if_username_provided_to_function():
"hashpass": login.passhash(raw_password),
"email": form.email,
"birthday": arrow.Arrow(2000, 1, 1),
"unixtime": arrow.now(),
})
acct_verification_token = login.get_account_verification_token(email=None, username=form.username)
assert token == acct_verification_token
@@ -1,5 +1,7 @@
from __future__ import absolute_import

from datetime import datetime
import pytz
import pytest

from pyramid.threadlocal import get_current_request
@@ -18,7 +20,8 @@ def test_verify_login_record_is_updated():
db = d.connect()
db.add(sess)
db.flush()
d.engine.execute("UPDATE login SET last_login = -1 WHERE userid = %(id)s", id=user_id)
time = datetime(2020, 1, 1, 00, 00, 1, tzinfo=pytz.UTC) # Arbitrary date that should be earlier than now.
d.engine.execute("UPDATE login SET last_login = %(timestamp)s WHERE userid = %(id)s", id=user_id, timestamp=time)
login.signin(get_current_request(), user_id)
last_login = d.engine.scalar("SELECT last_login FROM login WHERE userid = %(id)s", id=user_id)
assert last_login > -1
assert last_login > time
@@ -21,7 +21,6 @@ def _create_pending_account(invalid=False):
"hashpass": login.passhash('0123456789'),
"email": email,
"birthday": arrow.Arrow(2000, 1, 1),
"unixtime": arrow.now(),
"invalid": invalid,
})

0 comments on commit e130143

Please sign in to comment.
You can’t perform that action at this time.