Merge pull request #181 from sdementen/fix-yahoo-change

fix user-agent and crumble for yahoo api

CHANGELOG.rst

@@ -4,8 +4,11 @@ What's new
 devel
 ~~~~~
 
+
+- fix changes from yahoo API (fix user-agent to "", remove recovery of crumble)
 - fix tzlocal>=2.3 issue with change of tz type returned by tzlocal.get_localzone() (fix #180)
 
+
 Version 1.1.7 (2021-04-04)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 

piecash/yahoo_client.py

@@ -6,14 +6,12 @@
 import csv
 import datetime
 import logging
-import re
 from collections import namedtuple
 from decimal import Decimal
 from time import sleep
 
 import pytz
 
-
 MAX_ATTEMPT = 5
 
 YAHOO_BASE_URL = "https://query1.finance.yahoo.com/v7/finance"
@@ -27,7 +25,9 @@
 def get_latest_quote(symbol):
     import requests
 
-    resp = requests.get("{}/quote".format(YAHOO_BASE_URL), params={"symbols": symbol})
+    resp = requests.get("{}/quote".format(YAHOO_BASE_URL),
+                        params={"symbols": symbol},
+                        headers={"user-agent": ""})
     resp.raise_for_status()
 
     try:
@@ -54,21 +54,7 @@ def get_latest_quote(symbol):
     )
 
 
-crumble_link = "https://finance.yahoo.com/quote/{0}/history?p={0}"
-crumble_regex = r'CrumbStore":{"crumb":"(.*?)"}'
-quote_link = "https://query1.finance.yahoo.com/v7/finance/download/{}?period1={}&period2={}&interval=1d&events=history&crumb={}"
-
-
-def get_crumble_and_cookie(symbol):
-    import requests
-
-    link = crumble_link.format(symbol)
-    response = requests.get(link)
-    cookie_str = response.headers["set-cookie"]
-
-    match = re.search(crumble_regex, response.text)
-    crumble_str = match.group(1)
-    return crumble_str, cookie_str
+quote_link = "https://query1.finance.yahoo.com/v7/finance/download/{}?period1={}&period2={}&interval=1d&events=history"
 
 
 def download_quote(symbol, date_from, date_to, tz=None):
@@ -98,11 +84,9 @@ def normalize(d):
             "{} attempt to download quotes for symbol {} from yahoo".format(i, symbol)
         )
 
-        crumble_str, cookie_str = get_crumble_and_cookie(symbol)
-
-        link = quote_link.format(symbol, time_stamp_from, time_stamp_to, crumble_str)
+        link = quote_link.format(symbol, time_stamp_from, time_stamp_to)
 
-        resp = requests.get(link, headers={"Cookie": cookie_str})
+        resp = requests.get(link, headers={"user-agent": ""})
         try:
             resp.raise_for_status()
         except Exception as e: