Skip to content

Commit

Permalink
De-register functions after migration, refs #217
Browse files Browse the repository at this point in the history
  • Loading branch information
simonw committed Sep 3, 2023
1 parent 06d35c8 commit 87af2dd
Showing 1 changed file with 10 additions and 5 deletions.
15 changes: 10 additions & 5 deletions llm/embeddings_migrations.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,29 +52,34 @@ def m004_store_content_hash(db):
)
)

# Backfill content_hash
@db.register_function
# Register functions manually so we can de-register later
def md5(text):
return hashlib.md5(text.encode("utf8")).digest()

@db.register_function
def random_md5():
return hashlib.md5(str(time.time()).encode("utf8")).digest()

db.conn.create_function("temp_md5", 1, md5)
db.conn.create_function("temp_random_md5", 0, random_md5)

with db.conn:
db.execute(
"""
update embeddings
set content_hash = md5(content)
set content_hash = temp_md5(content)
where content is not null
"""
)
db.execute(
"""
update embeddings
set content_hash = random_md5()
set content_hash = temp_random_md5()
where content is null
"""
)

db["embeddings"].create_index(["content_hash"])

# De-register functions
db.conn.create_function("temp_md5", 1, None)
db.conn.create_function("temp_random_md5", 0, None)

0 comments on commit 87af2dd

Please sign in to comment.