diff --git a/src/pycookiecheat/__init__.py b/src/pycookiecheat/__init__.py index 9d791c8..bf41fcc 100644 --- a/src/pycookiecheat/__init__.py +++ b/src/pycookiecheat/__init__.py @@ -1,6 +1,7 @@ """__init__.py :: Exposes chrome_cookies function.""" from pycookiecheat.pycookiecheat import chrome_cookies +from pycookiecheat.firefox import firefox_cookies __author__ = "Nathan Henrie" __email__ = "nate@n8henrie.com" @@ -8,4 +9,5 @@ __all__ = [ "chrome_cookies", + "firefox_cookies", ] diff --git a/src/pycookiecheat/firefox.py b/src/pycookiecheat/firefox.py new file mode 100644 index 0000000..dd63bc9 --- /dev/null +++ b/src/pycookiecheat/firefox.py @@ -0,0 +1,211 @@ +import configparser +from pathlib import Path +import shutil +import sqlite3 +import sys +import tempfile +from typing import Dict +from typing import List +from typing import Optional +import urllib +import urllib.error +import urllib.parse + +FIREFOX_COOKIE_SELECT_SQL = """ + SELECT name, value, `path`, isSecure, expiry + FROM moz_cookies + WHERE host = ?; +""" + +FIREFOX_OS_PROFILE_DIRS: Dict[str, Dict[str, str]] = { + "linux": { + "Firefox": "~/.mozilla/firefox", + }, + "osx": { + "Firefox": "~/Library/Application Support/Firefox/Profiles", + }, + "windows": { + "Firefox": "~/AppData/Roaming/Mozilla/Firefox/Profiles", + }, +} + + +class FirefoxProfileNotPopulatedError(Exception): + """Raised when the Firefox profile has never been used.""" + + pass + + +def _get_profiles_dir_for_os(os: str, browser: str = "Firefox") -> Path: + """Retrieve the default directory containing the user profiles.""" + browser = browser.title() + try: + os_config = FIREFOX_OS_PROFILE_DIRS[os] + except KeyError: + raise ValueError( + f"OS must be one of {list(FIREFOX_OS_PROFILE_DIRS.keys())}" + ) + try: + return Path(os_config[browser]).expanduser() + except KeyError: + raise ValueError(f"Browser must be one of {list(os_config.keys())}") + + +def _find_firefox_default_profile(firefox_dir: Path) -> str: + """ + Return the name of the default Firefox profile. + + Args: + firefox_dir: Path to the Firefox config directory + Returns: + Name of the default profile + + Firefox' profiles.ini file in the Firefox config directory that lists all + available profiles. + + In Firefox versions 66 and below the default profile is simply marked with + `Default=1` in the profile section. Firefox 67 started to support multiple + installs of Firefox on the same machine and the default profile is now set + in `Install...` sections. The install section contains the name of its + default profile in the `Default` key. + + https://support.mozilla.org/en-US/kb/understanding-depth-profile-installation + """ + profiles_ini = configparser.ConfigParser() + profiles_ini.read(firefox_dir / "profiles.ini") + installs = [s for s in profiles_ini.sections() if s.startswith("Install")] + if installs: # Firefox >= 67 + # Heuristic: Take the first install, that's probably the system install + return profiles_ini[installs[0]]["Default"] + else: # Firefox < 67 + profiles = [ + s for s in profiles_ini.sections() if s.startswith("Profile") + ] + for profile in profiles: + if profiles_ini[profile].get("Default") == "1": + return profiles_ini[profile]["Path"] + if profiles: + return profiles_ini[profiles[0]]["Path"] + raise Exception("no profiles found at {}".format(firefox_dir)) + + +def _copy_if_exists(src: List[Path], dest: Path): + for file in src: + try: + shutil.copy2(file, dest) + except FileNotFoundError: + pass + + +def _load_firefox_cookie_db( + profiles_dir: Path, + tmp_dir: Path, + profile_name: Optional[str] = None, +) -> Path: + """ + Return a file path to the selected browser profile's cookie database. + + Args: + profiles_dir: Browser+OS paths profiles_dir path + tmp_dir: A temporary directory to copy the DB file(s) into + profile_name: Name (or glob pattern) of the Firefox profile to search + for cookies -- if none given it will find the configured + default profile + Returns: + Path to the "deWAL'ed" temporary copy of cookies.sqlite + + Firefox stores its cookies in an SQLite3 database file. While Firefox is + running it has an exclusive lock on this file and other processes can't + read from it. To circumvent this, copy the cookies file to the given + temporary directory and read it from there. + + The SQLite database uses a feature called WAL ("write-ahead logging") that + writes transactions for the database into a second file _prior_ to writing + it to the actual DB. When copying the database this method also copies the + WAL file and then merges any outstanding writes, to make sure the cookies + DB has the most recent data. + """ + if not profile_name: + profile_name = _find_firefox_default_profile(profiles_dir) + for profile_dir in profiles_dir.glob(profile_name): + if (profile_dir / "cookies.sqlite").exists(): + break + else: + raise FirefoxProfileNotPopulatedError(profiles_dir / profile_name) + cookies_db = profile_dir / "cookies.sqlite" + cookies_wal = profile_dir / "cookies.sqlite-wal" + _copy_if_exists([cookies_db, cookies_wal], tmp_dir) + db_file = tmp_dir / "cookies.sqlite" + if not db_file.exists(): + raise FileNotFoundError(f"no Firefox cookies DB in temp dir {tmp_dir}") + with sqlite3.connect(db_file) as con: + con.execute("PRAGMA journal_mode=OFF;") # merge WAL + return db_file + + +def firefox_cookies( + url: str, + profile_name: Optional[str] = None, + browser: str = "Firefox", + curl_cookie_file: Optional[str] = None, +) -> Dict[str, str]: + """Retrieve cookies from Chrome/Chromium on OSX or Linux. + + Args: + url: Domain from which to retrieve cookies, starting with http(s) + profile_name: Name (or glob pattern) of the Firefox profile to search + for cookies -- if none given it will find the configured + default profile + browser: Name of the browser's cookies to read (must be 'Firefox') + curl_cookie_file: Path to save the cookie file to be used with cURL + Returns: + Dictionary of cookie values for URL + """ + parsed_url = urllib.parse.urlparse(url) + if parsed_url.scheme: + domain = parsed_url.netloc + else: + raise urllib.error.URLError("You must include a scheme with your URL.") + + if sys.platform.startswith("linux"): + os = "linux" + elif sys.platform == "darwin": + os = "osx" + elif sys.platform == "win32": + os = "windows" + else: + raise OSError( + "This script only works on " + + ", ".join(FIREFOX_OS_PROFILE_DIRS.keys()) + ) + profiles_dir = _get_profiles_dir_for_os(os, browser) + + cookies = {} + curl_cookies = [] + with tempfile.TemporaryDirectory() as tmp_dir: + db_file = _load_firefox_cookie_db( + profiles_dir, Path(tmp_dir), profile_name + ) + with sqlite3.connect(db_file) as con: + res = con.execute(FIREFOX_COOKIE_SELECT_SQL, (domain,)) + for key, value, path, is_secure, expiry in res.fetchall(): + cookies[key] = value + if curl_cookie_file: + # http://www.cookiecentral.com/faq/#3.5 + curl_cookies.append( + "\t".join( + [ + domain, + "TRUE", + path, + "TRUE" if is_secure else "FALSE", + str(expiry), + key, + value, + ] + ) + ) + if curl_cookie_file: + with open(curl_cookie_file, "w") as text_file: + text_file.write("\n".join(curl_cookies) + "\n") + return cookies diff --git a/tests/test_firefox.py b/tests/test_firefox.py new file mode 100644 index 0000000..4f9cee3 --- /dev/null +++ b/tests/test_firefox.py @@ -0,0 +1,359 @@ +from datetime import datetime +from datetime import timedelta +from http.cookies import SimpleCookie +from http.server import BaseHTTPRequestHandler +from http.server import HTTPServer +from pathlib import Path +import re +from textwrap import dedent +from threading import Thread +from typing import Iterator +from typing import Optional +from unittest.mock import patch +from urllib.error import URLError + +from playwright.sync_api import sync_playwright +import pytest +from pytest import FixtureRequest +from pytest import TempPathFactory + +from pycookiecheat.firefox import _find_firefox_default_profile +from pycookiecheat.firefox import _get_profiles_dir_for_os +from pycookiecheat.firefox import _load_firefox_cookie_db +from pycookiecheat.firefox import firefox_cookies +from pycookiecheat.firefox import FirefoxProfileNotPopulatedError + + +TEST_PROFILE_NAME = "test-profile" +TEST_PROFILE_DIR = f"1234abcd.{TEST_PROFILE_NAME}" + +PROFILES_INI_VERSION1 = dedent( + f""" + [General] + StartWithLastProfile=1 + + [Profile0] + Name={TEST_PROFILE_NAME} + IsRelative=1 + Path={TEST_PROFILE_DIR} + Default=1 + + [Profile1] + Name={TEST_PROFILE_NAME}2 + IsRelative=1 + Path=abcdef01.{TEST_PROFILE_NAME}2 + """ +) + +PROFILES_INI_VERSION2 = dedent( + f""" + [Install8149948BEF895A0D] + Default={TEST_PROFILE_DIR} + Locked=1 + + [General] + StartWithLastProfile=1 + Version=2 + + [Profile0] + Name={TEST_PROFILE_NAME} + IsRelative=1 + Path={TEST_PROFILE_DIR} + Default=1 + """ +) + +PROFILES_INI_EMPTY = dedent( + """ + [General] + StartWithLastProfile=1 + Version=2 + """ +) + +PROFILES_INI_VERSION1_NO_DEFAULT = dedent( + f""" + [General] + StartWithLastProfile=1 + Version=2 + + [Profile0] + Name={TEST_PROFILE_NAME} + IsRelative=1 + Path={TEST_PROFILE_DIR} + """ +) + +PROFILES_INI_VERSION2_NO_DEFAULT = dedent( + f""" + [Install8149948BEF895A0D] + Default={TEST_PROFILE_DIR} + Locked=1 + + [General] + StartWithLastProfile=1 + Version=2 + + [Profile0] + Name={TEST_PROFILE_NAME} + IsRelative=1 + Path={TEST_PROFILE_DIR} + """ +) + + +def _make_test_profiles( + tmp_path: Path, profiles_ini_content: str, populate: bool = True +) -> Iterator[Path]: + """ + All of the fixtures using this function use the pytest builtin `tmp_path` + or `tmp_path_factory` fixtures to create their temporary directories. + """ + profile_dir = tmp_path / TEST_PROFILE_DIR + profile_dir.mkdir() + (tmp_path / "profiles.ini").write_text(profiles_ini_content) + if populate: + with sync_playwright() as p: + p.firefox.launch_persistent_context( + user_data_dir=profile_dir, + headless=True, + ).close() + with patch( + "pycookiecheat.firefox._get_profiles_dir_for_os", + return_value=tmp_path, + ): + yield tmp_path + + +@pytest.fixture(scope="module") +def profiles(tmp_path_factory: TempPathFactory) -> Iterator[Path]: + """Create a populated Firefox data dir with a profile & cookie DB""" + yield from _make_test_profiles( + tmp_path_factory.mktemp("_"), PROFILES_INI_VERSION2 + ) + + +@pytest.fixture( + scope="module", + params=[ + PROFILES_INI_VERSION1, + PROFILES_INI_VERSION2, + PROFILES_INI_VERSION1_NO_DEFAULT, + PROFILES_INI_VERSION2_NO_DEFAULT, + ], +) +def profiles_ini_versions( + tmp_path_factory: TempPathFactory, request: FixtureRequest +) -> Iterator[Path]: + """ + Create a populated Firefox data dir using varius `profiles.ini` file format + versions and contents. + """ + yield from _make_test_profiles(tmp_path_factory.mktemp("_"), request.param) + + +@pytest.fixture(scope="module") +def no_profiles(tmp_path_factory: TempPathFactory) -> Iterator[Path]: + """Create a Firefox data dir with a `profiles.ini` but no profiles.""" + yield from _make_test_profiles( + tmp_path_factory.mktemp("_"), PROFILES_INI_EMPTY + ) + + +# TODO: Making this fixture module-scoped breaks the tests using the `profiles` +# fixture. Find out why. +@pytest.fixture +def profiles_unpopulated(tmp_path: Path) -> Iterator[Path]: + """ + Create a Firefox data dir with a valid `profiles.ini` file, but an + unpopulated (i.e. never-used) profile dir. + """ + yield from _make_test_profiles( + tmp_path, PROFILES_INI_VERSION2, populate=False + ) + + +@pytest.fixture(scope="session") +def cookie_server() -> Iterator[int]: + """ + Start an `HTTPServer` on localhost which replies to GET requests by + setting a cookie. Used as fixture for testing cookie retrieval. + + Returns: + The port of the server on localhost. + """ + + class CookieSetter(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + cookie = SimpleCookie() + cookie["foo"] = "bar" + cookie["foo"]["path"] = "/" + # Needs an expiry time, otherwise it's a session cookie, which are + # never saved to disk. (Well, _technically_ they sometimes are, + # when the browser is set to resume the session on restart, but we + # aren't concerned with that here.) + this_time_tomorrow = datetime.utcnow() + timedelta(days=1) + cookie["foo"]["expires"] = this_time_tomorrow.strftime( + "%a, %d %b %Y %H:%M:%S GMT" + ) + self.send_header("Set-Cookie", cookie["foo"].OutputString()) + self.end_headers() + + def log_message(self, *_): + pass # Suppress logging + + with HTTPServer(("localhost", 0), CookieSetter) as server: + Thread(target=server.serve_forever, daemon=True).start() + yield server.server_port + server.shutdown() + + +@pytest.fixture +def set_cookie(profiles: Path, cookie_server: int) -> Iterator[None]: + """ + Launch Firefox and visit the cookie-setting server. The cookie is set, + saved to the DB and the browser closes. Ideally the browser should still + be running while the cookie tests run, but the synchronous playwright API + doesn't support that. + """ + profile_dir = profiles / TEST_PROFILE_DIR + with sync_playwright() as p, p.firefox.launch_persistent_context( + user_data_dir=profile_dir + ) as context: + context.new_page().goto( + f"http://localhost:{cookie_server}", + # Fail quickly because it's localhost. If it's not there in 100ms + # the problem is the server or the test setup, not the network. + timeout=100, + ) + # This `yield` should be indented twice more, inside the launched + # firefox context manager, but the synchronous playwright API doesn't + # support it. This means the tests don't test getting cookies while + # Firefox is running. + # TODO: Try using the async playwright API instead. + yield + + +### _get_profiles_dir_for_os() + + +@pytest.mark.parametrize( + "os_name,expected_dir", + [ + ("linux", "~/.mozilla/firefox"), + ("osx", "~/Library/Application Support/Firefox/Profiles"), + ("windows", "~/AppData/Roaming/Mozilla/Firefox/Profiles"), + ], +) +def test_get_profiles_dir_for_os_valid(os_name: str, expected_dir: str): + # Test only implicit "Firefox" default, since it's the only type we + # currently support + profiles_dir = _get_profiles_dir_for_os(os_name, "Firefox") + assert profiles_dir == Path(expected_dir).expanduser() + + +def test_get_profiles_dir_for_os_invalid(): + with pytest.raises(ValueError, match="OS must be one of"): + _get_profiles_dir_for_os("invalid") + with pytest.raises(ValueError, match="Browser must be one of"): + _get_profiles_dir_for_os("linux", "invalid") + + +### _find_firefox_default_profile() + + +def test_firefox_get_default_profile_valid(profiles_ini_versions: Path): + profile_dir = profiles_ini_versions / _find_firefox_default_profile( + profiles_ini_versions + ) + assert profile_dir.is_dir() + assert (profile_dir / "cookies.sqlite").is_file() + + +def test_firefox_get_default_profile_invalid(no_profiles: Path): + with pytest.raises(Exception, match="no profiles found"): + _find_firefox_default_profile(no_profiles) + + +### _load_firefox_cookie_db() + + +def test_load_firefox_cookie_db_populated(tmp_path: Path, profiles: Path): + db_path = _load_firefox_cookie_db(profiles, tmp_path) + assert db_path == tmp_path / "cookies.sqlite" + assert db_path.exists() + + +@pytest.mark.parametrize("profile_name", [TEST_PROFILE_DIR, None]) +def test_load_firefox_cookie_db_unpopulated( + tmp_path: Path, + profile_name: Optional[str], + profiles_unpopulated: Path, +): + with pytest.raises(FirefoxProfileNotPopulatedError): + _load_firefox_cookie_db( + profiles_unpopulated, + tmp_path, + profile_name, + ) + + +def test_load_firefox_cookie_db_copy_error(tmp_path: Path, profiles: Path): + # deliberately break copying + with patch("shutil.copy2"), pytest.raises( + FileNotFoundError, match="no Firefox cookies DB in temp dir" + ): + _load_firefox_cookie_db( + profiles, + tmp_path, + TEST_PROFILE_DIR, + ) + + +### firefox_cookies() + + +def test_firefox_cookies(set_cookie: None): + cookies = firefox_cookies("http://localhost", TEST_PROFILE_DIR) + assert len(cookies) > 0 + assert cookies["foo"] == "bar" + + +def test_firefox_no_cookies(profiles: Path): + cookies = firefox_cookies("http://example.org", TEST_PROFILE_DIR) + assert len(cookies) == 0 + + +def test_firefox_cookies_no_scheme(): + with pytest.raises( + URLError, match="You must include a scheme with your URL" + ): + firefox_cookies("localhost") + + +def test_firefox_cookies_curl_cookie_file(tmp_path: Path, set_cookie: None): + cookie_file = tmp_path / "cookies.txt" + firefox_cookies( + "http://localhost", + profile_name=TEST_PROFILE_DIR, + curl_cookie_file=str(cookie_file), + ) + assert cookie_file.exists() + assert re.fullmatch( + r"localhost\tTRUE\t/\tFALSE\t[0-9]+\tfoo\tbar\n", + cookie_file.read_text(), + ) + + +@pytest.mark.parametrize("fake_os", ["linux", "darwin", "win32"]) +def test_firefox_cookies_os(fake_os, profiles: Path): + with patch("sys.platform", fake_os): + cookies = firefox_cookies("http://example.org", TEST_PROFILE_DIR) + assert isinstance(cookies, dict) + + +def test_firefox_cookies_os_invalid(profiles: Path): + with patch("sys.platform", "invalid"): + with pytest.raises(OSError): + firefox_cookies("http://localhost")