Increase csv field size limit in metadata backfill (#5274)

AudiusProject · May 23, 2023 · 63e6025 · 63e6025
1 parent 594915d
commit 63e6025
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/discovery-provider/src/tasks/backfill_cid_data.py b/discovery-provider/src/tasks/backfill_cid_data.py
@@ -41,6 +41,8 @@ def backfill_cid_data(db: SessionManager):
         with db.scoped_session() as session:
             # Load cid data from csv in chunks...
             with open(tmp.name, "r") as file:
+                # Set 370KB limit for csv fields
+                csv.field_size_limit(370_000)
                 while True:
                     csv_reader = csv.reader(file, delimiter="\t")
                     lines = list(islice(csv_reader, chunk_size))