[168] Adding some changes that make it possible to get more insight i…

…nto running requests.

.pre-commit-config.yaml

@@ -21,11 +21,13 @@ repos:
       - id: pyupgrade
         args: ["--py37-plus"]
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.782
+    rev: v1.7.1
     hooks:
       - id: mypy
         args:
           - --pretty
           - --show-error-codes
           - --show-error-context
           - --ignore-missing-imports
+        additional_dependencies:
+          - types-python-dateutil

myfitnesspal/client.py

@@ -4,8 +4,10 @@
 import json
 import logging
 import re
+import uuid
 from collections import OrderedDict
 from http.cookiejar import CookieJar
+from pathlib import Path
 from typing import Any, Dict, List, Optional, Union, cast, overload
 
 import browser_cookie3
@@ -63,7 +65,16 @@ def __init__(
         self,
         cookiejar: CookieJar = None,
         unit_aware: bool = False,
+        log_requests_to: Path | None = None,
     ):
+        self._client_instance_id = uuid.uuid4()
+        self._request_counter = 0
+        if log_requests_to:
+            self._log_requests_to = log_requests_to / Path(
+                str(self._client_instance_id)
+            )
+            self._log_requests_to.mkdir(parents=True, exist_ok=True)
+
         self.unit_aware = unit_aware
 
         self.session = requests.Session()
@@ -199,6 +210,14 @@ def _get_request_for_url(
         headers: Optional[Dict[str, str]] = None,
         **kwargs,
     ) -> requests.Response:
+        request_id = uuid.uuid4()
+        self._request_counter += 1
+        logger.debug(
+            "Sending request %s (#%s for client) to url %s",
+            self._request_counter,
+            request_id,
+            url,
+        )
         if headers is None:
             headers = {}
 
@@ -212,7 +231,38 @@ def _get_request_for_url(
             if self.user_id:
                 headers["mfp-user-id"] = self.user_id
 
-        return self.session.get(url, headers=headers, **kwargs)
+        result = self.session.get(url, headers=headers, **kwargs)
+        if self._log_requests_to:
+            with open(
+                self._log_requests_to
+                / Path(
+                    str(self._request_counter).zfill(3) + "__" + str(request_id)
+                ).with_suffix(".json"),
+                "w",
+                encoding="utf-8",
+            ) as outf:
+                outf.write(
+                    json.dumps(
+                        {
+                            "request": {
+                                "url": url,
+                                "send_token": send_token,
+                                "user_id": self.user_id if send_token else None,
+                                "headers": headers,
+                                "kwargs": kwargs,
+                            },
+                            "response": {
+                                "headers": dict(result.headers),
+                                "status_code": result.status_code,
+                                "content": result.content.decode("utf-8"),
+                            },
+                        },
+                        indent=4,
+                        sort_keys=True,
+                    )
+                )
+
+        return result
 
     def _get_content_for_url(self, *args, **kwargs) -> str:
         return self._get_request_for_url(*args, **kwargs).content.decode("utf8")
@@ -389,7 +439,6 @@ def _get_exercise(self, document):
 
                 # If name is empty string:
                 if columns[0].find("a") is None or not name:
-
                     # check for `td > div > a`
                     if columns[0].find("div").find("a") is None:
                         # then check for just `td > div`
@@ -638,7 +687,6 @@ def _get_measurements(self, document):
 
         # create a dictionary out of the date and value of each entry
         for entry in trs:
-
             # ensure there are measurement entries on the page
             if len(entry) == 1:
                 return measurements
@@ -658,7 +706,6 @@ def _get_measurements(self, document):
         return measurements
 
     def _get_measurement_ids(self, document) -> Dict[str, int]:
-
         # find the option element for all of the measurement choices
         options = document.xpath("//select[@id='type']/option")
 

myfitnesspal/cmdline.py

@@ -1,6 +1,7 @@
 import argparse
 import logging
 import sys
+from pathlib import Path
 
 from rich.console import Console
 from rich.logging import RichHandler
@@ -20,6 +21,7 @@ def main(args=None):
     parser.add_argument("command", type=str, nargs=1, choices=COMMANDS.keys())
     parser.add_argument("--loglevel", type=str, default="INFO")
     parser.add_argument("--traceback-locals", action="store_true")
+    parser.add_argument("--log-requests-to", type=Path, default=None)
     parser.add_argument("--debugger", action="store_true")
     args, extra = parser.parse_known_args()
 

myfitnesspal/commands.py

@@ -53,7 +53,7 @@ def decorator(fn):
 @command(
     "Display MyFitnessPal data for a given date.",
 )
-def day(args, *extra, **kwargs):
+def day(super_args, *extra, **kwargs):
     parser = argparse.ArgumentParser()
     parser.add_argument(
         "date",
@@ -64,7 +64,7 @@ def day(args, *extra, **kwargs):
     )
     args = parser.parse_args(extra)
 
-    client = Client()
+    client = Client(log_requests_to=super_args.log_requests_to)
     day = client.get_date(args.date)
 
     date_str = args.date.strftime("%Y-%m-%d")