Fix proerly remove duplicates in vehicle positions ingestor using onl…

…y subset of columns How could I be so stupid? Really?
VMois · Apr 27, 2024 · 9e6400d · 9e6400d
1 parent 4cc17f2
commit 9e6400d
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/gtfs_realtime_ingestor/ingest_vehicle_positions.py b/gtfs_realtime_ingestor/ingest_vehicle_positions.py
@@ -117,7 +117,9 @@ def extract_vehicle_location():
 
         if chunks_left == 0:
             chunks_left = CHUNKS_TO_LOAD
-            df = pl.DataFrame(flattened_data).unique(keep="last")
+            df = pl.DataFrame(flattened_data).unique(
+                subset=["vehicle_id", "timestamp"], keep="last"
+            )
 
             flattened_data.clear()
             object_path = f"realtime/vehicle_{current_hash}.csv"