Refactor tcx extraction

Author: alhankeser

Dockerfile

@@ -32,11 +32,16 @@ RUN --mount=type=cache,target=/root/.cache/pip \
     --mount=type=bind,source=requirements.txt,target=requirements.txt \
     python -m pip install -r requirements.txt
 
+# Copy the source code into the container.
+COPY . .
+
+# Setup zig executable
+RUN python build_zig.py
+
 # Switch to the non-privileged user to run the application.
 USER dockeruser
 
-# Copy the source code into the container.
-COPY . .
+# Give dockeruser a few rights
 ADD --chown=dockeruser --chmod=700 ./activity_files ./activity_files
 ADD --chown=dockeruser --chmod=700 ./.token ./.token
 

build_zig.py

@@ -0,0 +1,4 @@
+# The poor man's solution to getting zig executable for machine in question
+import tcx_extract as tcx
+
+tcx.build_zig()

lib/file_manager/file_manager.py

@@ -5,6 +5,7 @@
 import pandas as pd
 from bs4 import BeautifulSoup
 from dotenv import load_dotenv
+import tcx_extract as tcx
 
 load_dotenv()
 
@@ -30,60 +31,45 @@ def save_activity_raw_file(activity_id, activity_raw_data):
 def to_unix(timestamp):
     unix_time_start = pd.Timestamp("1970-01-01").tz_localize("UTC")
     increment = pd.Timedelta("1s")
-    timestamp = pd.Timestamp(timestamp)
+    timestamp = pd.to_datetime(timestamp, errors='coerce')
     return (timestamp - unix_time_start) // increment
 
 def convert_activity_file(
     activity_id, original_file_path, format="csv"
 ):
-    selectors = {
-            "heart_rate": "HeartRateBpm > Value",
-            "watts": "ns3:Watts",
-            "time": "Time",
-        }
-    with open(f"{original_file_path}", "rb") as file:
-        soup = BeautifulSoup(file, features="lxml-xml")
-        track_points = soup.find_all('Trackpoint')
-        if len(track_points) == 0:
-            return False
-        data = []
-        for track_point in track_points:
-            time_point = track_point.find(selectors['time']).text
-            try:
-                heart_rate_point = track_point.select(selectors['heart_rate'])[0].text
-            except:
-                heart_rate_point = 0
-            try:
-                watts_point = track_point.find(selectors['watts']).text
-            except:
-                watts_point = 0
-            
-            data.append({
-                'time': to_unix(time_point),
-                'heart_rate': int(heart_rate_point),
-                'watts': int(watts_point)
-            })
-        
-        data_df = pd.DataFrame(data)
-        data_dict = data_df.to_dict(orient='list')
+    time_points = tcx.extract(original_file_path, "Time")
+    watts = tcx.extract(original_file_path, "ns3:Watts")
+    heart_rate = tcx.extract(original_file_path, "Value")
+
+    data_df = pd.DataFrame({
+        'time': time_points,
+        'heart_rate': heart_rate,
+        'watts': watts
+    })
+    
+    data_df['time'] = to_unix(data_df['time']).fillna(0).astype('int64')
+    data_df['heart_rate'] = pd.to_numeric(data_df['heart_rate'], errors='coerce').fillna(0).astype('uint16')
+    data_df['watts'] = pd.to_numeric(data_df['watts'], errors='coerce').fillna(0).astype('uint16')
+
+    data_dict = data_df.to_dict(orient='list')
 
-        activity_data = {
-            "activity_id": activity_id,
-            "start_time": data_df.iloc[0]['time'],
-            "data": data_dict
+    activity_data = {
+        "activity_id": activity_id,
+        "start_time": data_df.iloc[0]['time'],
+        "data": data_dict
+    }
+    df = pd.DataFrame([activity_data])
+    df = df.astype(
+        {
+            "activity_id": "int64", 
+            "start_time": "int64", 
+            "data": "string"
         }
-        df = pd.DataFrame([activity_data])
-        df = df.astype(
-            {
-                "activity_id": "int64", 
-                "start_time": "int64", 
-                "data": "string"
-             }
-        )
-        if format == "csv":
-            converted_file_path = f"{CONVERTED_FILES_PATH}/{activity_id}.csv"
-            df.to_csv(converted_file_path, index=False)
-        if format == "parquet":
-            converted_file_path = f"{CONVERTED_FILES_PATH}/{activity_id}.parquet"
-            df.to_parquet(converted_file_path, index=False)
-        return converted_file_path
+    )
+    if format == "csv":
+        converted_file_path = f"{CONVERTED_FILES_PATH}/{activity_id}.csv"
+        df.to_csv(converted_file_path, index=False)
+    if format == "parquet":
+        converted_file_path = f"{CONVERTED_FILES_PATH}/{activity_id}.parquet"
+        df.to_parquet(converted_file_path, index=False)
+    return converted_file_path

main.py

@@ -1,5 +1,6 @@
 import os
 import logging
+import time
 from lib.api.garmin import garmin
 from lib.file_manager import file_manager
 from dotenv import load_dotenv

requirements.txt

@@ -8,4 +8,5 @@ numpy==1.26.3
 pandas==2.2.0
 pyarrow==15.0.0
 python-dotenv==1.0.0
-google-cloud-storage==2.14.0
+google-cloud-storage==2.14.0
+tcx_extract==0.1.0.1