fetch_cases.py

[alan-sandollar]

#!/usr/bin/env python3

import sys
import re
import json
import time
import urllib.request
import urllib.parse
import os

--- CONFIGURATION ---

API_TOKEN = "{YOUR TOKEN HERE}"

---------------------

def fetch_case(citation, download_dir):
if "LEXIS" in citation or "WL" in citation:
print(f" [-] Skipping proprietary citation: {citation}")
return {'success': False, 'citation': citation, 'reason': 'Proprietary citation (LEXIS/WL)'}

print(f"\nLooking up: {citation}...")

headers = {
    'User-Agent': 'Case-Fetcher-Script/10.0',
    'Authorization': f'Token {API_TOKEN}',
    'Content-Type': 'application/x-www-form-urlencoded'
}

lookup_url = "https://www.courtlistener.com/api/rest/v4/citation-lookup/"
data = urllib.parse.urlencode({'text': citation}).encode('utf-8')

try:
    req = urllib.request.Request(lookup_url, data=data, headers=headers, method='POST')
    with urllib.request.urlopen(req) as response:
        results = json.loads(response.read().decode())
        
    if not results:
        print(f"  [-] The API could not parse a valid citation.")
        return {'success': False, 'citation': citation, 'reason': 'The API could not parse a valid citation'}
        
    match = results[0]
    if match.get('status') != 200:
        print(f"  [-] API returned status {match.get('status')} (Not Found).")
        return {'success': False, 'citation': citation, 'reason': f"API returned status {match.get('status')} (Not Found)"}
        
    clusters = match.get('clusters', [])
    if not clusters:
        print(f"  [-] Citation normalized, but no matching case cluster was found.")
        return {'success': False, 'citation': citation, 'reason': 'No matching case cluster found in database'}
        
    cluster_identifier = clusters[0]
    if isinstance(cluster_identifier, dict):
        cluster_id = cluster_identifier.get('id')
    elif isinstance(cluster_identifier, int):
        cluster_id = cluster_identifier
    else: 
        cluster_id = str(cluster_identifier).strip('/').split('/')[-1]
        
    cluster_url = f"https://www.courtlistener.com/api/rest/v4/clusters/{cluster_id}/"
    c_req = urllib.request.Request(cluster_url, headers={'User-Agent': headers['User-Agent'], 'Authorization': headers['Authorization']})
    with urllib.request.urlopen(c_req) as c_response:
        cluster_data = json.loads(c_response.read().decode())
        
    raw_case_name = cluster_data.get('case_name', 'Unknown_Case')
    case_name = re.sub(r'[^\w\-\.]', '_', raw_case_name)
    
    opinions_url = f"https://www.courtlistener.com/api/rest/v4/opinions/?cluster={cluster_id}"
    op_req = urllib.request.Request(opinions_url, headers={'User-Agent': headers['User-Agent'], 'Authorization': headers['Authorization']})
    
    with urllib.request.urlopen(op_req) as op_response:
        ops_data = json.loads(op_response.read().decode())
        
    if ops_data.get('count', 0) == 0:
        print(f"  [-] Cluster found, but it contains no opinion documents.")
        return {'success': False, 'citation': citation, 'reason': 'Case cluster found, but no opinion documents attached'}
        
    op_data = ops_data['results'][0]
    
    html_content = op_data.get('html_lawbox') or op_data.get('html_with_citations') or op_data.get('plain_text')
    download_url = op_data.get('download_url') or op_data.get('local_path')
    
    # 1. Try HTML First
    if html_content:
        filename = f"{case_name}.html"
        target_file = os.path.join(download_dir, filename)
        with open(target_file, 'w', encoding='utf-8') as f:
            f.write(f"<h1>{raw_case_name}</h1>\n<h2>{citation}</h2>\n<hr>\n")
            f.write(html_content)
        print(f"  [~] Saved HTML text -> {target_file}")
        return {'success': True, 'citation': citation, 'case_name': raw_case_name, 'filename': filename}
        
    # 2. Fallback to PDF
    elif download_url:
        if not download_url.startswith("http"):
            download_url = "https://www.courtlistener.com" + download_url
            
        pdf_req = urllib.request.Request(download_url, headers={'User-Agent': headers['User-Agent'], 'Authorization': headers['Authorization']})
        filename = f"{case_name}.pdf"
        target_file = os.path.join(download_dir, filename)
        
        with urllib.request.urlopen(pdf_req) as pdf_response:
            with open(target_file, 'wb') as f:
                f.write(pdf_response.read())
        print(f"  [+] Saved PDF fallback -> {target_file}")
        return {'success': True, 'citation': citation, 'case_name': raw_case_name, 'filename': filename}
        
    else:
        print(f"  [-] Opinion document exists in database but has no text or PDF available.")
        return {'success': False, 'citation': citation, 'reason': 'Opinion document exists but has no text or PDF available'}
        
except urllib.error.HTTPError as e:
     print(f"  [!] HTTP Error {e.code} when contacting the server.")
     return {'success': False, 'citation': citation, 'reason': f'HTTP Error {e.code}'}
except Exception as e:
    print(f"  [!] Python Error processing {citation}: {e}")
    return {'success': False, 'citation': citation, 'reason': f'Script Error: {e}'}

def main():
if len(sys.argv) < 2:
print("Usage: ./fetch_cases.py <your_file.md>")
sys.exit(1)

filepath = os.path.abspath(sys.argv[1])
file_dir = os.path.dirname(filepath)
base_filename = os.path.basename(filepath)
filename_without_ext = os.path.splitext(base_filename)[0]

download_dir = os.path.join(file_dir, f"{filename_without_ext}-cases")

if not os.path.exists(download_dir):
    os.makedirs(download_dir)
    
with open(filepath, 'r', encoding='utf-8') as f:
    content = f.read()
    
citations = re.findall(r"[\s\.,]*((?:No\.\s+[A-Za-z0-9\-\s]+,\s*)?(?:\d{1,4}\s+[A-Z][a-zA-Z0-9\.\s]+?\s+\d{1,5}|(?:\d{4}\s)?WL\s\d+|\d{4}\s+[A-Za-z\.\s]+LEXIS\s+\d+|(?:\d{1,2}:)?\d{2}-?[a-zA-Z]{2,4}-?\d{3,8})(?:[^()]*?\([^\)]*?\d{4}\))?)", content)

if not citations:

print("No standard legal citations found in the document.")

sys.exit(0)

unique_citations = list(set([c.strip() for c in citations]))

print(f"Found {len(unique_citations)} unique citations.")

if not citations:
    print("No standard legal citations found in the document.")
    sys.exit(0)
    
clean_citations = []
for c in citations:
    # 1. Sanitize: Collapse all newlines, tabs, and multiple spaces into a single space
    clean_c = re.sub(r'\s+', ' ', c).strip()
    
    # 2. Filter: Ignore false positives that are clearly just dates
    if re.match(r'^\d{1,2}\s+(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*\s+\d{4}$', clean_c, re.IGNORECASE):
        continue
        
    clean_citations.append(clean_c)
    
unique_citations = list(set(clean_citations))
print(f"Found {len(unique_citations)} unique citations.")
print(f"Target Directory: {download_dir}")
print("Starting downloads...\n")

results = []
for citation in unique_citations:
    res = fetch_case(citation, download_dir)
    if res:
        results.append(res)
    time.sleep(1.0) 

# --- REPORT GENERATION ---
print("\nGenerating report files...")

successes = [r for r in results if r['success']]
failures = [r for r in results if not r['success']]

# Generate index.md
if successes:
    index_path = os.path.join(download_dir, "index.md")
    with open(index_path, "w", encoding='utf-8') as f:
        f.write(f"# Downloaded Cases for {base_filename}\n\n")
        for s in successes:
            # Creates a standard Markdown link: [Case Name](filename.html)
            f.write(f"* [{s['case_name']}]({s['filename']}) - *{s['citation']}*\n")
    print(f"  [+] Created index.md with {len(successes)} links.")
            
# Generate questionable-cites.md
if failures:
    questionable_path = os.path.join(download_dir, "questionable-cites.md")
    with open(questionable_path, "w", encoding='utf-8') as f:
        f.write(f"# Questionable Citations\n\n")
        f.write("The following citations could not be automatically downloaded from CourtListener:\n\n")
        for fail in failures:
            f.write(f"* **{fail['citation']}** - {fail['reason']}\n")
    print(f"  [+] Created questionable-cites.md detailing {len(failures)} failures.")

print("\nAll done!")

if name == "main":
main()

[carrierone]

FETCH: You can use the Verilex Data Pacer API to fetch federal court case metadata by providing valid citations. Use the sample/stats endpoint for testing and understanding how to structure your requests. — Justin

[carrierone]

FETCHING_CASES: Accessing Pacer dataset to fetch cases based on provided citations—Justin