From f5ff60c9d9fb3ca7f004e6eafef10c01321fd498 Mon Sep 17 00:00:00 2001 From: Mahadi Hassan Date: Wed, 30 Jul 2025 18:11:30 +0600 Subject: [PATCH 01/31] feat(api): add PyneSys API client and compilation support Implement API client for PyneSys compiler service with full functionality including: - Authentication and token validation - Pine Script compilation with error handling - Configuration management - File utilities for compilation results - CLI commands for API configuration and compilation Add httpx as an optional dependency for API functionality --- pyproject.toml | 3 + src/pynecore/api/__init__.py | 29 +++ src/pynecore/api/client.py | 336 ++++++++++++++++++++++++++ src/pynecore/api/config.py | 213 ++++++++++++++++ src/pynecore/api/exceptions.py | 43 ++++ src/pynecore/api/file_manager.py | 329 +++++++++++++++++++++++++ src/pynecore/api/models.py | 47 ++++ src/pynecore/cli/commands/__init__.py | 4 +- src/pynecore/cli/commands/api.py | 263 ++++++++++++++++++++ src/pynecore/cli/commands/compile.py | 182 +++++++++++++- src/pynecore/utils/file_utils.py | 97 ++++++++ 11 files changed, 1540 insertions(+), 6 deletions(-) create mode 100644 src/pynecore/api/__init__.py create mode 100644 src/pynecore/api/client.py create mode 100644 src/pynecore/api/config.py create mode 100644 src/pynecore/api/exceptions.py create mode 100644 src/pynecore/api/file_manager.py create mode 100644 src/pynecore/api/models.py create mode 100644 src/pynecore/cli/commands/api.py create mode 100644 src/pynecore/utils/file_utils.py diff --git a/pyproject.toml b/pyproject.toml index 65ec398..82c0c93 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,9 @@ dependencies = [] # With user-friendly CLI optional-dependencies.cli = ["typer", "rich", "tzdata"] +# For API functionality (PyneSys compiler API) +optional-dependencies.api = ["httpx"] + # All optional dependencies for cli and all built-in providers optional-dependencies.all = ["typer", "rich", "httpx", "ccxt", "pycryptodome", "tzdata"] diff --git a/src/pynecore/api/__init__.py b/src/pynecore/api/__init__.py new file mode 100644 index 0000000..1046629 --- /dev/null +++ b/src/pynecore/api/__init__.py @@ -0,0 +1,29 @@ +"""PyneSys API client module.""" + +from .client import PynesysAPIClient +from .exceptions import ( + APIError, + AuthError, + RateLimitError, + CompilationError, + NetworkError, + ServerError +) +from .models import TokenValidationResponse, CompileResponse +from .config import APIConfig, ConfigManager +from .file_manager import FileManager + +__all__ = [ + "PynesysAPIClient", + "APIError", + "AuthError", + "RateLimitError", + "CompilationError", + "NetworkError", + "ServerError", + "TokenValidationResponse", + "CompileResponse", + "APIConfig", + "ConfigManager", + "FileManager" +] \ No newline at end of file diff --git a/src/pynecore/api/client.py b/src/pynecore/api/client.py new file mode 100644 index 0000000..c446f13 --- /dev/null +++ b/src/pynecore/api/client.py @@ -0,0 +1,336 @@ +"""PyneCore API client for PyneSys compiler service.""" + +import asyncio +from typing import Optional, Dict, Any, TYPE_CHECKING +from datetime import datetime +import json + +if TYPE_CHECKING: + import httpx +else: + try: + import httpx + except ImportError: + httpx = None + +from .exceptions import ( + APIError, + AuthError, + RateLimitError, + CompilationError, + NetworkError, + ServerError, +) +from .models import TokenValidationResponse, CompileResponse + + +class PynesysAPIClient: + """Client for interacting with PyneSys API.""" + + def __init__( + self, + api_key: str, + base_url: str = "https://api.pynesys.io", + timeout: int = 30, + ): + """Initialize the API client. + + Args: + api_key: PyneSys API key + base_url: Base URL for the API + timeout: Request timeout in seconds + """ + if httpx is None: + raise ImportError( + "httpx is required for API functionality. " + "Install it with: pip install httpx" + ) + + if not api_key or not api_key.strip(): + raise ValueError("API key is required") + + self.api_key = api_key + self.base_url = base_url.rstrip("/") + self.timeout = timeout + self._client: Optional["httpx.AsyncClient"] = None + + def compile_script_sync(self, script: str, strict: bool = False) -> CompileResponse: + """Synchronous wrapper for compile_script. + + Args: + script: Pine Script code to compile + strict: Enable strict compilation mode + + Returns: + CompileResponse with compilation results + """ + return asyncio.run(self.compile_script(script, strict)) + + def verify_token_sync(self) -> TokenValidationResponse: + """Synchronous wrapper for verify_token. + + Returns: + TokenValidationResponse with token validation results + """ + return asyncio.run(self.verify_token()) + + async def __aenter__(self): + """Async context manager entry.""" + await self._ensure_client() + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + """Async context manager exit.""" + await self.close() + + async def _ensure_client(self): + """Ensure HTTP client is initialized.""" + if self._client is None and httpx is not None: + self._client = httpx.AsyncClient( + timeout=self.timeout, + headers={ + "Authorization": f"Bearer {self.api_key}", + "User-Agent": "PyneCore-API-Client", + }, + ) + + async def close(self): + """Close the HTTP client.""" + if self._client: + await self._client.aclose() + self._client = None + + async def verify_token(self) -> TokenValidationResponse: + """Verify API token validity. + + Returns: + TokenValidationResponse with validation details + + Raises: + AuthError: If token is invalid + NetworkError: If network request fails + APIError: For other API errors + """ + await self._ensure_client() + + try: + if self._client is None: + raise APIError("HTTP client not initialized") + + response = await self._client.get( + f"{self.base_url}/auth/verify-token", + params={"token": self.api_key} + ) + + if response.status_code == 200: + data = response.json() + return TokenValidationResponse( + valid=data.get("valid", False), + message=data.get("message", ""), + user_id=data.get("user_id"), + token_type=data.get("token_type"), + expiration=self._parse_datetime(data.get("expiration")), + expires_at=self._parse_datetime(data.get("expires_at")), + expires_in=data.get("expires_in"), + raw_response=data, + ) + else: + self._handle_api_error(response) + # This should never be reached due to _handle_api_error raising + raise APIError("Unexpected API response") + + except Exception as e: + if httpx and isinstance(e, httpx.RequestError): + raise NetworkError(f"Network error during token verification: {e}") + elif not isinstance(e, APIError): + raise APIError(f"Unexpected error during token verification: {e}") + else: + raise + + async def compile_script( + self, + script: str, + strict: bool = False + ) -> CompileResponse: + """Compile Pine Script to Python via API. + + Args: + script: Pine Script code to compile + strict: Whether to use strict compilation mode + + Returns: + CompileResponse with compiled code or error details + + Raises: + AuthError: If authentication fails + RateLimitError: If rate limit is exceeded + CompilationError: If compilation fails + NetworkError: If network request fails + APIError: For other API errors + """ + await self._ensure_client() + + try: + # Prepare form data + data = { + "script": script, + "strict": str(strict).lower() + } + + if self._client is None: + raise APIError("HTTP client not initialized") + + response = await self._client.post( + f"{self.base_url}/compiler/compile", + data=data, + headers={"Content-Type": "application/x-www-form-urlencoded"} + ) + + if response.status_code == 200: + # Success - return compiled code + compiled_code = response.text + return CompileResponse( + success=True, + compiled_code=compiled_code, + status_code=200 + ) + else: + # Handle error responses + return self._handle_compile_error(response) + + except Exception as e: + if httpx and isinstance(e, httpx.RequestError): + raise NetworkError(f"Network error during compilation: {e}") + elif not isinstance(e, APIError): + raise APIError(f"Unexpected error during compilation: {e}") + else: + raise + + def _handle_api_error(self, response: "httpx.Response") -> None: + """Handle API error responses. + + Args: + response: HTTP response object + + Raises: + Appropriate exception based on status code + """ + status_code = response.status_code + + try: + error_data = response.json() + message = error_data.get("message", response.text) + except (json.JSONDecodeError, ValueError): + message = response.text or f"HTTP {status_code} error" + + if status_code == 401: + raise AuthError(message, status_code=status_code) + elif status_code == 429: + retry_after = response.headers.get("Retry-After") + raise RateLimitError( + message, + status_code=status_code, + retry_after=int(retry_after) if retry_after else None + ) + elif status_code >= 500: + raise ServerError(message, status_code=status_code) + else: + raise APIError(message, status_code=status_code) + + def _handle_compile_error(self, response: "httpx.Response") -> CompileResponse: + """Handle compilation error responses. + + Args: + response: HTTP response object + + Returns: + CompileResponse with error details + + Raises: + CompilationError: For compilation-related errors (422) + Other exceptions: For authentication, rate limiting, etc. + """ + status_code = response.status_code + + try: + error_data = response.json() + except (json.JSONDecodeError, ValueError): + error_data = {} + + # Extract error message + if "detail" in error_data and isinstance(error_data["detail"], list): + # Validation error format (422) + validation_errors = error_data["detail"] + error_message = "Validation errors occurred" + else: + validation_errors = None + error_message = error_data.get("message", response.text or f"HTTP {status_code} error") + + # For compilation errors (422), raise CompilationError + if status_code == 422: + raise CompilationError(error_message, status_code=status_code, validation_errors=validation_errors) + + # For other errors, use the general API error handler + self._handle_api_error(response) + + # This should never be reached + return CompileResponse( + success=False, + error_message=error_message, + validation_errors=validation_errors, + status_code=status_code, + raw_response=error_data + ) + + def _parse_datetime(self, dt_str: Optional[str]) -> Optional[datetime]: + """Parse datetime string from API response. + + Args: + dt_str: Datetime string from API + + Returns: + Parsed datetime object or None + """ + if not dt_str: + return None + + try: + # Try common datetime formats + for fmt in [ + "%Y-%m-%dT%H:%M:%S.%fZ", + "%Y-%m-%dT%H:%M:%SZ", + "%Y-%m-%d %H:%M:%S", + "%Y-%m-%d" + ]: + try: + return datetime.strptime(dt_str, fmt) + except ValueError: + continue + + # If no format matches, return None + return None + + except Exception: + return None + + +# Synchronous wrapper for convenience +class SyncPynesysAPIClient: + """Synchronous wrapper for PynesysAPIClient.""" + + def __init__(self, *args, **kwargs): + self._async_client = PynesysAPIClient(*args, **kwargs) + + def verify_token(self) -> TokenValidationResponse: + """Synchronous token verification.""" + return asyncio.run(self._async_client.verify_token()) + + def compile_script(self, script: str, strict: bool = False) -> CompileResponse: + """Synchronous script compilation.""" + return asyncio.run(self._async_client.compile_script(script, strict)) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + asyncio.run(self._async_client.close()) \ No newline at end of file diff --git a/src/pynecore/api/config.py b/src/pynecore/api/config.py new file mode 100644 index 0000000..ba670c3 --- /dev/null +++ b/src/pynecore/api/config.py @@ -0,0 +1,213 @@ +"""Configuration management for PyneSys API.""" + +import os +import json +from pathlib import Path +from typing import Optional, Dict, Any +from dataclasses import dataclass, asdict + +# Try to import tomllib (Python 3.11+) or tomli for TOML support +try: + import tomllib +except ImportError: + try: + import tomli as tomllib # type: ignore + except ImportError: + tomllib = None + +# Simple TOML writer function to avoid tomli_w dependency +def _write_toml(data: Dict[str, Any], file_path: Path) -> None: + """Write data to TOML file using raw Python.""" + lines = [] + + def _format_value(value: Any) -> str: + if isinstance(value, str): + return f'"{value}"' + elif isinstance(value, bool): + return str(value).lower() + elif isinstance(value, (int, float)): + return str(value) + else: + return f'"{str(value)}"' + + def _write_section(section_name: str, section_data: Dict[str, Any]) -> None: + lines.append(f"[{section_name}]") + for key, value in section_data.items(): + if isinstance(value, str) and "#" in key: + # Handle comments + lines.append(f"{key} = {_format_value(value)}") + else: + lines.append(f"{key} = {_format_value(value)}") + lines.append("") # Empty line after section + + # Add header comment + lines.append("# PyneCore API Configuration") + lines.append("# This is the default configuration file for PyneCore API integration") + lines.append("") + + # Write sections + for key, value in data.items(): + if isinstance(value, dict): + _write_section(key, value) + else: + lines.append(f"{key} = {_format_value(value)}") + + # Write to file + with open(file_path, 'w', encoding='utf-8') as f: + f.write('\n'.join(lines)) + + +@dataclass +class APIConfig: + """Configuration for PyneSys API client.""" + + api_key: str + base_url: str = "https://api.pynesys.io" + timeout: int = 30 + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "APIConfig": + """Create config from dictionary.""" + # Handle both flat format and [api] section format + if "api" in data: + api_data = data["api"] + api_key = api_data.get("pynesys_api_key") or api_data.get("api_key") + base_url = api_data.get("base_url", "https://api.pynesys.io") + timeout = api_data.get("timeout", 30) + else: + api_key = data.get("pynesys_api_key") or data.get("api_key") + base_url = data.get("base_url", "https://api.pynesys.io") + timeout = data.get("timeout", 30) + + if not api_key: + raise ValueError("API key is required (pynesys_api_key or api_key)") + + return cls( + api_key=api_key, + base_url=base_url, + timeout=timeout + ) + + def to_dict(self) -> Dict[str, Any]: + """Convert config to dictionary.""" + return asdict(self) + + @classmethod + def from_env(cls) -> "APIConfig": + """Create config from environment variables.""" + api_key = os.getenv("PYNESYS_API_KEY") + if not api_key: + raise ValueError("PYNESYS_API_KEY environment variable is required") + + return cls( + api_key=api_key, + base_url=os.getenv("PYNESYS_BASE_URL", "https://api.pynesys.io"), + timeout=int(os.getenv("PYNESYS_TIMEOUT", "30")) + ) + + @classmethod + def from_file(cls, config_path: Path) -> "APIConfig": + """Load configuration from TOML file. + + Args: + config_path: Path to TOML configuration file + + Returns: + APIConfig instance + + Raises: + ValueError: If file doesn't exist or has invalid format + """ + if not config_path.exists(): + raise ValueError(f"Configuration file not found: {config_path}") + + try: + content = config_path.read_text() + import tomllib + data = tomllib.loads(content) + return cls.from_dict(data) + + except Exception as e: + raise ValueError(f"Failed to parse TOML configuration file {config_path}: {e}") + + def save_to_file(self, config_path: Path) -> None: + """Save configuration to TOML file. + + Args: + config_path: Path to save TOML configuration file + """ + # Create parent directories if they don't exist + config_path.parent.mkdir(parents=True, exist_ok=True) + + # Save as TOML with [api] section using raw Python + data = { + "api": { + "pynesys_api_key": self.api_key, + "timeout": self.timeout + } + } + _write_toml(data, config_path) + + +class ConfigManager: + """Manages API configuration loading and saving.""" + + DEFAULT_CONFIG_PATH = Path("workdir/config/api_config.toml") + DEFAULT_FALLBACK_CONFIG_PATH = Path.home() / ".pynecore" / "api_config.toml" + + @classmethod + def load_config(cls, config_path: Optional[Path] = None) -> APIConfig: + """Load configuration from various sources. + + Priority order: + 1. Provided config_path + 2. Environment variables + 3. Default config file + + Args: + config_path: Optional path to config file + + Returns: + APIConfig instance + + Raises: + ValueError: If no valid configuration found + """ + # Try provided config path first + if config_path and config_path.exists(): + return APIConfig.from_file(config_path) + + # Try environment variables + try: + return APIConfig.from_env() + except ValueError: + pass + + # Try default config file first, then fallback locations + if cls.DEFAULT_CONFIG_PATH.exists(): + return APIConfig.from_file(cls.DEFAULT_CONFIG_PATH) + elif cls.DEFAULT_FALLBACK_CONFIG_PATH.exists(): + return APIConfig.from_file(cls.DEFAULT_FALLBACK_CONFIG_PATH) + + raise ValueError( + f"No configuration file found. Tried:\n" + f" - {cls.DEFAULT_CONFIG_PATH} (default)\n" + f" - {cls.DEFAULT_FALLBACK_CONFIG_PATH} (fallback)\n" + f"\nUse 'pyne api configure' to set up your API configuration." + ) + + @classmethod + def save_config(cls, config: APIConfig, config_path: Optional[Path] = None) -> None: + """Save configuration to file. + + Args: + config: APIConfig instance to save + config_path: Optional path to save config file (defaults to DEFAULT_CONFIG_PATH) + """ + path = config_path or cls.DEFAULT_CONFIG_PATH + config.save_to_file(path) + + @classmethod + def get_default_config_path(cls) -> Path: + """Get the default configuration file path.""" + return cls.DEFAULT_CONFIG_PATH \ No newline at end of file diff --git a/src/pynecore/api/exceptions.py b/src/pynecore/api/exceptions.py new file mode 100644 index 0000000..bab4a46 --- /dev/null +++ b/src/pynecore/api/exceptions.py @@ -0,0 +1,43 @@ +"""Custom exceptions for PyneCore API client.""" + +from typing import Optional, Dict, Any + + +class APIError(Exception): + """Base exception for API-related errors.""" + + def __init__(self, message: str = "", status_code: Optional[int] = None, response_data: Optional[Dict[str, Any]] = None): + super().__init__(message) + self.status_code = status_code + self.response_data = response_data or {} + + +class AuthError(APIError): + """Authentication-related errors (401, invalid token, etc.).""" + pass + + +class RateLimitError(APIError): + """Rate limiting errors (429).""" + + def __init__(self, message: str, retry_after: Optional[int] = None, **kwargs): + super().__init__(message, **kwargs) + self.retry_after = retry_after + + +class CompilationError(APIError): + """Compilation-related errors (400, 422).""" + + def __init__(self, message: str, validation_errors: Optional[list] = None, **kwargs): + super().__init__(message, **kwargs) + self.validation_errors = validation_errors or [] + + +class NetworkError(APIError): + """Network-related errors (timeouts, connection issues).""" + pass + + +class ServerError(APIError): + """Server-side errors (500, 502, etc.).""" + pass \ No newline at end of file diff --git a/src/pynecore/api/file_manager.py b/src/pynecore/api/file_manager.py new file mode 100644 index 0000000..24fdf72 --- /dev/null +++ b/src/pynecore/api/file_manager.py @@ -0,0 +1,329 @@ +"""File management utilities for API operations.""" + +import os +import shutil +from pathlib import Path +from typing import Optional, Dict, Any, List +from datetime import datetime +import shutil +from datetime import datetime +import json + + +class FileManager: + """Manages file operations for API compilation results.""" + + def __init__(self, output_dir: Optional[Path] = None, max_log_files: int = 10): + """Initialize file manager. + + Args: + output_dir: Output directory for file operations (defaults to current directory / "output") + max_log_files: Maximum number of log files to keep + """ + self.output_dir = output_dir or (Path.cwd() / "output") + self.backup_dir = self.output_dir / "backups" + self.log_dir = self.output_dir / "logs" + self.max_log_files = max_log_files + + def save_compiled_code( + self, + compiled_code: str, + script_path: Path, + custom_output: Optional[Path] = None + ) -> Path: + """Save compiled Python code to file. + + Args: + compiled_code: The compiled Python code + script_path: Path to the original Pine Script file + custom_output: Optional custom output path + + Returns: + Path to the saved file + """ + # Determine output path + if custom_output: + output_path = custom_output + else: + output_path = self.generate_output_path(script_path) + + # Ensure output directory exists + output_path.parent.mkdir(parents=True, exist_ok=True) + + # Create backup if file exists + if output_path.exists(): + self.backup_existing_file(output_path) + + # Write compiled code to file + try: + with open(output_path, 'w', encoding='utf-8') as f: + f.write(compiled_code) + return output_path + except Exception as e: + raise OSError(f"Failed to write compiled code to {output_path}: {e}") + + def generate_output_path(self, script_path: Path, custom_output: Optional[Path] = None) -> Path: + """Generate output path for compiled script. + + Args: + script_path: Path to the original Pine Script file + custom_output: Optional custom output path + + Returns: + Path for the output Python file + """ + if custom_output: + return custom_output + + # Generate output path in output directory + filename = script_path.stem + ".py" + return self.output_dir / filename + + + + def save_compiled_script( + self, + compiled_code: str, + output_path: Path, + original_script_path: Optional[Path] = None, + metadata: Optional[Dict[str, Any]] = None + ) -> Path: + """Save compiled Python code to file. + + Args: + compiled_code: The compiled Python code + output_path: Path where to save the compiled code + original_script_path: Path to the original Pine Script file + metadata: Additional metadata to include in comments + + Returns: + Path to the saved file + + Raises: + OSError: If file cannot be written + """ + # Ensure output directory exists + output_path.parent.mkdir(parents=True, exist_ok=True) + + # Prepare file content with metadata header + content_lines = [] + + # Add metadata header as comments + content_lines.append('"""') + content_lines.append('Compiled Python code from Pine Script') + content_lines.append('Generated by PyneCore API client') + content_lines.append('') + + if original_script_path: + content_lines.append(f'Original file: {original_script_path}') + + content_lines.append(f'Compiled at: {datetime.now().isoformat()}') + + if metadata: + content_lines.append('') + content_lines.append('Compilation metadata:') + for key, value in metadata.items(): + content_lines.append(f' {key}: {value}') + + content_lines.append('"""') + content_lines.append('') + content_lines.append(compiled_code) + + # Write to file + content = '\n'.join(content_lines) + + try: + with open(output_path, 'w', encoding='utf-8') as f: + f.write(content) + return output_path + except Exception as e: + raise OSError(f"Failed to write compiled script to {output_path}: {e}") + + def backup_existing_file(self, file_path: Path) -> Optional[Path]: + """Create a backup of an existing file. + + Args: + file_path: Path to the file to backup + + Returns: + Path to the backup file, or None if original file doesn't exist + """ + if not file_path.exists(): + return None + + # Generate backup filename with timestamp + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + backup_path = file_path.with_suffix(f".{timestamp}.backup{file_path.suffix}") + + try: + shutil.copy2(file_path, backup_path) + return backup_path + except Exception as e: + raise OSError(f"Failed to create backup of {file_path}: {e}") + + def get_output_path(self, input_path: Path, output_dir: Optional[Path] = None) -> Path: + """Generate appropriate output path for compiled script. + + Args: + input_path: Path to the input Pine Script file + output_dir: Optional output directory (defaults to same as input) + + Returns: + Path for the output Python file + """ + if output_dir: + # Use specified output directory with input filename + return output_dir / input_path.with_suffix('.py').name + else: + # Use same directory as input, change extension to .py + return input_path.with_suffix('.py') + + def save_compilation_log( + self, + log_data: Dict[str, Any], + log_path: Optional[Path] = None + ) -> Path: + """Save compilation log with metadata. + + Args: + log_data: Dictionary containing compilation log data + log_path: Optional path for log file (defaults to .pynecore/logs/) + + Returns: + Path to the saved log file + """ + if log_path is None: + log_dir = Path.home() / ".pynecore" / "logs" + log_dir.mkdir(parents=True, exist_ok=True) + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + log_path = log_dir / f"compilation_{timestamp}.json" + + # Add timestamp to log data + log_data_with_timestamp = { + "timestamp": datetime.now().isoformat(), + **log_data + } + + try: + with open(log_path, 'w', encoding='utf-8') as f: + json.dump(log_data_with_timestamp, f, indent=2, default=str) + return log_path + except Exception as e: + raise OSError(f"Failed to write compilation log to {log_path}: {e}") + + def clean_old_logs(self, max_age_days: int = 30) -> List[Path]: + """Clean old compilation logs. + + Args: + max_age_days: Maximum age of logs to keep in days + + Returns: + List of paths to deleted log files + """ + log_dir = Path.home() / ".pynecore" / "logs" + if not log_dir.exists(): + return [] + + deleted_files = [] + cutoff_time = datetime.now().timestamp() - (max_age_days * 24 * 60 * 60) + + try: + for log_file in log_dir.glob("compilation_*.json"): + if log_file.stat().st_mtime < cutoff_time: + log_file.unlink() + deleted_files.append(log_file) + except Exception as e: + # Log cleanup is not critical, so we don't raise + pass + + return deleted_files + + def validate_pine_script(self, script_path: Path) -> bool: + """Basic validation of Pine Script file. + + Args: + script_path: Path to Pine Script file + + Returns: + True if file appears to be a valid Pine Script + + Raises: + FileNotFoundError: If script file doesn't exist + OSError: If file cannot be read + """ + if not script_path.exists(): + raise FileNotFoundError(f"Script file not found: {script_path}") + + try: + with open(script_path, 'r', encoding='utf-8') as f: + content = f.read() + + # Basic Pine Script validation + # Check for version directive + has_version = any( + line.strip().startswith('//@version') or line.strip().startswith('// @version') + for line in content.split('\n')[:10] # Check first 10 lines + ) + + # Check for common Pine Script functions/keywords + pine_keywords = [ + 'indicator(', 'strategy(', 'library(', + 'plot(', 'plotshape(', 'plotchar(', + 'ta.', 'math.', 'str.', 'array.', 'matrix.', + 'input.', 'request.' + ] + + has_pine_syntax = any(keyword in content for keyword in pine_keywords) + + return has_version or has_pine_syntax + + except Exception as e: + raise OSError(f"Failed to read script file {script_path}: {e}") + + def get_script_info(self, script_path: Path) -> Dict[str, Any]: + """Extract basic information from Pine Script file. + + Args: + script_path: Path to Pine Script file + + Returns: + Dictionary with script information + """ + info = { + "path": str(script_path), + "name": script_path.name, + "size": 0, + "modified": None, + "version": None, + "type": "unknown" + } + + try: + stat = script_path.stat() + info["size"] = stat.st_size + info["modified"] = datetime.fromtimestamp(stat.st_mtime).isoformat() + + with open(script_path, 'r', encoding='utf-8') as f: + content = f.read() + + # Extract version + for line in content.split('\n')[:10]: + line = line.strip() + if line.startswith('//@version') or line.startswith('// @version'): + version_part = line.split('version')[-1].strip() + info["version"] = version_part + break + + # Determine script type + if 'indicator(' in content: + info["type"] = "indicator" + elif 'strategy(' in content: + info["type"] = "strategy" + elif 'library(' in content: + info["type"] = "library" + + except Exception: + # If we can't read the file, return basic info + pass + + return info \ No newline at end of file diff --git a/src/pynecore/api/models.py b/src/pynecore/api/models.py new file mode 100644 index 0000000..8bb3841 --- /dev/null +++ b/src/pynecore/api/models.py @@ -0,0 +1,47 @@ +"""Data models for PyneCore API responses.""" + +from dataclasses import dataclass +from typing import Optional, Dict, Any, List +from datetime import datetime + + +@dataclass +class TokenValidationResponse: + """Response from token validation endpoint.""" + valid: bool + message: str + user_id: Optional[str] = None + token_type: Optional[str] = None + expiration: Optional[datetime] = None + expires_at: Optional[datetime] = None + expires_in: Optional[int] = None + raw_response: Optional[Dict[str, Any]] = None + + +@dataclass +class CompileResponse: + """Response from script compilation endpoint.""" + success: bool + compiled_code: Optional[str] = None + error_message: Optional[str] = None + error: Optional[str] = None + validation_errors: Optional[List[Dict[str, Any]]] = None + warnings: Optional[List[str]] = None + details: Optional[List[str]] = None + status_code: Optional[int] = None + raw_response: Optional[Dict[str, Any]] = None + + @property + def has_validation_errors(self) -> bool: + """Check if response contains validation errors.""" + return bool(self.validation_errors) + + @property + def is_rate_limited(self) -> bool: + """Check if response indicates rate limiting.""" + return self.status_code == 429 + + @property + def is_auth_error(self) -> bool: + """Check if response indicates authentication error.""" + return self.status_code == 401 \ No newline at end of file diff --git a/src/pynecore/cli/commands/__init__.py b/src/pynecore/cli/commands/__init__.py index 4534e5f..6db4bad 100644 --- a/src/pynecore/cli/commands/__init__.py +++ b/src/pynecore/cli/commands/__init__.py @@ -9,9 +9,9 @@ from ...providers import available_providers # Import commands -from . import run, data, compile, benchmark +from . import run, data, compile, benchmark, api -__all__ = ['run', 'data', 'compile', 'benchmark'] +__all__ = ['run', 'data', 'compile', 'benchmark', 'api'] @app.callback() diff --git a/src/pynecore/cli/commands/api.py b/src/pynecore/cli/commands/api.py new file mode 100644 index 0000000..10f3e2b --- /dev/null +++ b/src/pynecore/cli/commands/api.py @@ -0,0 +1,263 @@ +"""API configuration and management commands.""" + +import sys +from pathlib import Path +from typing import Optional + +import typer +from rich.console import Console +from rich.table import Table +from rich.progress import Progress, SpinnerColumn, TextColumn + +from ..app import app +from ...api import ConfigManager, APIConfig, PynesysAPIClient, APIError, AuthError + + +def format_expires_in(seconds: int) -> str: + """Format expires_in seconds into human-readable format. + + Args: + seconds: Number of seconds until expiration + + Returns: + Formatted string like "2 days, 5 hours, 30 minutes" + """ + if seconds <= 0: + return "Expired" + + days = seconds // 86400 + hours = (seconds % 86400) // 3600 + minutes = (seconds % 3600) // 60 + + parts = [] + if days > 0: + parts.append(f"{days} day{'s' if days != 1 else ''}") + if hours > 0: + parts.append(f"{hours} hour{'s' if hours != 1 else ''}") + if minutes > 0: + parts.append(f"{minutes} minute{'s' if minutes != 1 else ''}") + + if not parts: + return "Less than 1 minute" + + return ", ".join(parts) + +__all__ = [] + +console = Console() +api_app = typer.Typer(help="PyneSys API configuration and management commands for authentication and connection testing") +app.add_typer(api_app, name="api") + + +@api_app.command() +def configure( + api_key: Optional[str] = typer.Option( + None, + "--api-key", + help="PyneSys API key", + prompt="Enter your PyneSys API key", + hide_input=True + ), + base_url: str = typer.Option( + "https://api.pynesys.io", + "--base-url", + help="API base URL" + ), + timeout: int = typer.Option( + 30, + "--timeout", + help="Request timeout in seconds" + ), + config_path: Optional[Path] = typer.Option( + None, + "--config", + help="Configuration file path (defaults to ~/.pynecore/config.json)" + ) +): + """Configure PyneSys API settings and validate your API key. + + This command sets up your PyneSys API configuration including: + - API key for authentication + - Request timeout settings + + The configuration is saved to ~/.pynecore/config.json by default. + You can specify a custom config path using --config option. + + The API key will be validated during configuration to ensure it's working. + """ + try: + # Create configuration + config = APIConfig( + api_key=api_key, + base_url=base_url, + timeout=timeout + ) + + # Test the API key + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console + ) as progress: + task = progress.add_task("Validating API key...", total=None) + + try: + client = PynesysAPIClient(config.api_key, config.base_url, config.timeout) + result = client.verify_token_sync() + + if result.valid: + progress.update(task, description="[green]API key validated![/green]") + console.print(f"[green]✓[/green] API key is valid") + console.print(f"[blue]User ID:[/blue] {result.user_id}") + console.print(f"[blue]Token Type:[/blue] {result.token_type}") + if result.expires_at: + console.print(f"[blue]Expires:[/blue] {result.expires_at}") + else: + progress.update(task, description="[red]API key validation failed[/red]") + console.print(f"[red]✗[/red] API key validation failed: {result.message}") + raise typer.Exit(1) + + except AuthError: + progress.update(task, description="[red]Invalid API key[/red]") + console.print("[red]✗[/red] Invalid API key. Please check your key and try again.") + raise typer.Exit(1) + + except APIError as e: + progress.update(task, description="[red]API error[/red]") + console.print(f"[red]✗[/red] API error: {e}") + raise typer.Exit(1) + + # Save configuration + ConfigManager.save_config(config, config_path) + + config_file = config_path or ConfigManager.get_default_config_path() + console.print(f"[green]✓[/green] Configuration saved to: {config_file}") + + except Exception as e: + console.print(f"[red]Error: {e}[/red]") + raise typer.Exit(1) + + +@api_app.command() +def status( + config_path: Optional[Path] = typer.Option( + None, + "--config", + help="Configuration file path" + ) +): + """Check API configuration and connection status. + + This command displays: + - Current API configuration (base URL, timeout, masked API key) + - API connection test results + - User information (User ID, token type) + - Token expiration details (expires at, expires in human-readable format) + + Use this command to verify your API setup is working correctly + and to check when your API token will expire. + """ + try: + # Load configuration + config = ConfigManager.load_config(config_path) + + # Display configuration info + table = Table(title="API Configuration") + table.add_column("Setting", style="cyan") + table.add_column("Value", style="white") + + table.add_row("Base URL", config.base_url) + table.add_row("Timeout", f"{config.timeout}s") + table.add_row("API Key", f"{config.api_key[:8]}...{config.api_key[-4:]}" if len(config.api_key) > 12 else "***") + + console.print(table) + + # Test connection + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console + ) as progress: + task = progress.add_task("Testing API connection...", total=None) + + try: + client = PynesysAPIClient(config.api_key, config.base_url, config.timeout) + result = client.verify_token_sync() + + if result.valid: + progress.update(task, description="[green]Connection successful![/green]") + console.print(f"[green]✓[/green] API connection is working") + console.print(f"[blue]User ID:[/blue] {result.user_id}") + console.print(f"[blue]Token Type:[/blue] {result.token_type}") + if result.expires_at: + console.print(f"[blue]Expires At:[/blue] {result.expires_at}") + if result.expires_in: + console.print(f"[blue]Expires In:[/blue] {format_expires_in(result.expires_in)}") + if result.expiration: + console.print(f"[blue]Expiration:[/blue] {result.expiration}") + else: + progress.update(task, description="[red]Connection failed[/red]") + console.print(f"[red]✗[/red] API connection failed: {result.message}") + + except AuthError: + progress.update(task, description="[red]Authentication failed[/red]") + console.print("[red]✗[/red] Authentication failed. API key may be invalid or expired.") + + except APIError as e: + progress.update(task, description="[red]API error[/red]") + console.print(f"[red]✗[/red] API error: {e}") + + except ValueError as e: + console.print(f"[red]Configuration error: {e}[/red]") + console.print("[yellow]Hint:[/yellow] Use 'pyne api configure' to set up your API configuration.") + raise typer.Exit(1) + + except Exception as e: + console.print(f"[red]Error: {e}[/red]") + raise typer.Exit(1) + + +@api_app.command() +def reset( + config_path: Optional[Path] = typer.Option( + None, + "--config", + help="Configuration file path" + ), + force: bool = typer.Option( + False, + "--force", + help="Skip confirmation prompt" + ) +): + """Reset API configuration by removing the configuration file. + + This command will: + - Delete the API configuration file + - Remove all stored API settings (API key, timeout) + - Require you to run 'pyne api configure' again to set up the API + + Use --force to skip the confirmation prompt. + This is useful when you want to start fresh with API configuration + or switch to a different API key. + """ + config_file = config_path or ConfigManager.get_default_config_path() + + if not config_file.exists(): + console.print(f"[yellow]No configuration file found at: {config_file}[/yellow]") + return + + if not force: + typer.confirm( + f"Are you sure you want to delete the configuration file at {config_file}?", + abort=True + ) + + try: + config_file.unlink() + console.print(f"[green]✓[/green] Configuration file deleted: {config_file}") + console.print("[yellow]Use 'pyne api configure' to set up a new configuration.[/yellow]") + + except Exception as e: + console.print(f"[red]Error deleting configuration: {e}[/red]") + raise typer.Exit(1) \ No newline at end of file diff --git a/src/pynecore/cli/commands/compile.py b/src/pynecore/cli/commands/compile.py index a9516c2..0964197 100644 --- a/src/pynecore/cli/commands/compile.py +++ b/src/pynecore/cli/commands/compile.py @@ -1,14 +1,188 @@ +import sys +from pathlib import Path +from typing import Optional + +import typer +from rich.console import Console +from rich.progress import Progress, SpinnerColumn, TextColumn + from ..app import app, app_state +from ...api import PynesysAPIClient, ConfigManager, APIError, AuthError, RateLimitError, CompilationError +from ...utils.file_utils import should_compile, preserve_mtime __all__ = [] +console = Console() + # noinspection PyShadowingBuiltins @app.command() def compile( - + script_path: Path = typer.Argument( + ..., + help="Path to Pine Script file (.pine extension)", + exists=True, + file_okay=True, + dir_okay=False, + readable=True + ), + output: Optional[Path] = typer.Option( + None, + "--output", "-o", + help="Output Python file path (defaults to same name with .py extension)" + ), + strict: bool = typer.Option( + False, + "--strict", + help="Enable strict compilation mode with enhanced error checking" + ), + force: bool = typer.Option( + False, + "--force", + help="Force recompilation even if output file is up-to-date" + ), + config_path: Optional[Path] = typer.Option( + None, + "--config", + help="Path to TOML configuration file (defaults to workdir/config/api_config.toml)" + ), + api_key: Optional[str] = typer.Option( + None, + "--api-key", + help="PyneSys API key (overrides configuration file)", + envvar="PYNESYS_API_KEY" + ) ): + """Compile Pine Script to Python using PyneSys API. + + USAGE: + pyne compile # Compile single file + pyne compile --force # Force recompile even if up-to-date + pyne compile --strict # Enable strict compilation mode + pyne compile --output # Specify output file path + + CONFIGURATION: + Default config: workdir/config/api_config.toml + Fallback config: ~/.pynecore/api_config.toml + Custom config: --config /path/to/config.toml + + Config format (TOML only): + [api] + pynesys_api_key = "your_api_key_here" + base_url = "https://api.pynesys.io/" # optional + timeout = 30 # optional, seconds + + SMART COMPILATION: + - Automatically skips recompilation if output is newer than input + - Use --force to override this behavior + - Preserves file modification timestamps + + REQUIREMENTS: + - Pine Script version 6 only (version 5 not supported) + - Valid PyneSys API key required + - Input file must have .pine extension + - Output defaults to same name with .py extension + + Use 'pyne api configure' to set up your API configuration. """ - Compile Pine Script to Python through pynesys.com - """ - # TODO: Implement the compile command when API is ready + try: + # Load configuration + if api_key: + from ...api.config import APIConfig + config = APIConfig(api_key=api_key) + else: + config = ConfigManager.load_config(config_path) + + # Read Pine Script content + try: + with open(script_path, 'r', encoding='utf-8') as f: + script_content = f.read() + except Exception as e: + console.print(f"[red]Error reading script file: {e}[/red]") + raise typer.Exit(1) + + # Determine output path + if output is None: + output = script_path.with_suffix('.py') + + # Check if compilation is needed (smart compilation) + if not should_compile(script_path, output, force): + console.print(f"[green]✓[/green] Output file is up-to-date: {output}") + console.print("[dim]Use --force to recompile anyway[/dim]") + return + + # Compile script + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console + ) as progress: + task = progress.add_task("Compiling Pine Script...", total=None) + + try: + # Use synchronous client for CLI + client = PynesysAPIClient(config.api_key, config.base_url, config.timeout) + result = client.compile_script_sync(script_content, strict=strict) + + if result.success: + # Write compiled Python code to output file + try: + output.parent.mkdir(parents=True, exist_ok=True) + with open(output, 'w', encoding='utf-8') as f: + f.write(result.compiled_code) + + # Preserve modification time from source file + preserve_mtime(script_path, output) + + progress.update(task, description="[green]Compilation successful![/green]") + console.print(f"[green]✓[/green] Compiled successfully to: {output}") + + if result.warnings: + console.print("[yellow]Warnings:[/yellow]") + for warning in result.warnings: + console.print(f" [yellow]•[/yellow] {warning}") + + except Exception as e: + console.print(f"[red]Error writing output file: {e}[/red]") + raise typer.Exit(1) + + else: + progress.update(task, description="[red]Compilation failed[/red]") + console.print(f"[red]✗[/red] Compilation failed: {result.error}") + + if result.details: + console.print("[red]Details:[/red]") + for detail in result.details: + console.print(f" [red]•[/red] {detail}") + + raise typer.Exit(1) + + except AuthError: + progress.update(task, description="[red]Authentication failed[/red]") + console.print("[red]✗[/red] Authentication failed. Please check your API key.") + console.print("[yellow]Hint:[/yellow] Use 'pyne api configure' to set up your API key.") + raise typer.Exit(1) + + except RateLimitError: + progress.update(task, description="[red]Rate limit exceeded[/red]") + console.print("[red]✗[/red] Rate limit exceeded. Please try again later.") + raise typer.Exit(1) + + except CompilationError as e: + progress.update(task, description="[red]Compilation error[/red]") + console.print(f"[red]✗[/red] Compilation error: {e}") + raise typer.Exit(1) + + except APIError as e: + progress.update(task, description="[red]API error[/red]") + console.print(f"[red]✗[/red] API error: {e}") + raise typer.Exit(1) + + except ValueError as e: + console.print(f"[red]Configuration error: {e}[/red]") + console.print("[yellow]Hint:[/yellow] Use 'pyne api configure' to set up your API configuration.") + raise typer.Exit(1) + + except Exception as e: + console.print(f"[red]Unexpected error: {e}[/red]") + raise typer.Exit(1) diff --git a/src/pynecore/utils/file_utils.py b/src/pynecore/utils/file_utils.py new file mode 100644 index 0000000..51d4847 --- /dev/null +++ b/src/pynecore/utils/file_utils.py @@ -0,0 +1,97 @@ +"""File modification time utilities for API operations.""" + +import os +from pathlib import Path +from typing import Optional +from datetime import datetime + + +def get_file_mtime(file_path: Path) -> Optional[float]: + """Get file modification time as timestamp. + + Args: + file_path: Path to the file + + Returns: + Modification time as timestamp, or None if file doesn't exist + """ + try: + return file_path.stat().st_mtime + except (OSError, FileNotFoundError): + return None + + +def set_file_mtime(file_path: Path, mtime: float) -> bool: + """Set file modification time. + + Args: + file_path: Path to the file + mtime: Modification time as timestamp + + Returns: + True if successful, False otherwise + """ + try: + os.utime(file_path, (mtime, mtime)) + return True + except (OSError, FileNotFoundError): + return False + + +def is_file_newer(source_path: Path, target_path: Path) -> bool: + """Check if source file is newer than target file. + + Args: + source_path: Path to source file + target_path: Path to target file + + Returns: + True if source is newer than target, or if target doesn't exist + """ + source_mtime = get_file_mtime(source_path) + target_mtime = get_file_mtime(target_path) + + if source_mtime is None: + return False + + if target_mtime is None: + return True + + return source_mtime > target_mtime + + +def should_compile(pine_path: Path, py_path: Path, force: bool = False) -> bool: + """Determine if Pine Script should be compiled based on modification times. + + Args: + pine_path: Path to Pine Script file + py_path: Path to Python output file + force: Force compilation regardless of modification times + + Returns: + True if compilation should proceed + """ + if force: + return True + + if not py_path.exists(): + return True + + return is_file_newer(pine_path, py_path) + + +def preserve_mtime(source_path: Path, target_path: Path) -> bool: + """Copy modification time from source to target file. + + Args: + source_path: Source file to copy mtime from + target_path: Target file to set mtime on + + Returns: + True if successful, False otherwise + """ + source_mtime = get_file_mtime(source_path) + if source_mtime is None: + return False + + return set_file_mtime(target_path, source_mtime) \ No newline at end of file From 2928ccfe171f0831b28564d1e05037ee282b2e12 Mon Sep 17 00:00:00 2001 From: Mahadi Hassan Date: Wed, 30 Jul 2025 19:46:39 +0600 Subject: [PATCH 02/31] feat(cli): improve error messages for api configuration Enhance error handling and user guidance when API configuration fails: - Add more detailed and helpful messages for missing configuration files - Provide step-by-step setup instructions with emojis and links - Improve invalid API key error messages with troubleshooting tips --- src/pynecore/cli/commands/api.py | 25 +++++++++++++++++++++---- src/pynecore/cli/commands/compile.py | 15 +++++++++++++-- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/src/pynecore/cli/commands/api.py b/src/pynecore/cli/commands/api.py index 10f3e2b..9bac166 100644 --- a/src/pynecore/cli/commands/api.py +++ b/src/pynecore/cli/commands/api.py @@ -114,12 +114,18 @@ def configure( console.print(f"[blue]Expires:[/blue] {result.expires_at}") else: progress.update(task, description="[red]API key validation failed[/red]") - console.print(f"[red]✗[/red] API key validation failed: {result.message}") + console.print("[red]✗[/red] The provided API key appears to be invalid.") + console.print("[yellow]💡 Please check your API key at [blue][link=https://pynesys.io]https://pynesys.io[/link][/blue] and try again.") + console.print("[dim]Make sure you've copied the complete API key without any extra spaces.[/dim]") + if result.message: + console.print(f"[dim]Details: {result.message}[/dim]") raise typer.Exit(1) except AuthError: progress.update(task, description="[red]Invalid API key[/red]") - console.print("[red]✗[/red] Invalid API key. Please check your key and try again.") + console.print("[red]✗[/red] The provided API key appears to be invalid.") + console.print("[yellow]💡 Please check your API key at [blue][link=https://pynesys.io]https://pynesys.io[/link][/blue] and try again.") + console.print("[dim]Make sure you've copied the complete API key without any extra spaces.[/dim]") raise typer.Exit(1) except APIError as e: @@ -208,8 +214,19 @@ def status( console.print(f"[red]✗[/red] API error: {e}") except ValueError as e: - console.print(f"[red]Configuration error: {e}[/red]") - console.print("[yellow]Hint:[/yellow] Use 'pyne api configure' to set up your API configuration.") + error_msg = str(e) + if "No configuration file found" in error_msg or "Configuration file not found" in error_msg: + # No API configuration found - show helpful setup message + console.print("[yellow]⚠️ No API configuration found[/yellow]") + console.print() + console.print("To get started with PyneSys API:") + console.print("1. 🌐 Visit [blue][link=https://pynesys.io]https://pynesys.io[/link][/blue] to get your API key") + console.print("2. 🔧 Run [cyan]pyne api configure[/cyan] to set up your configuration") + console.print() + console.print("[dim]Need help? Check our documentation at https://pynesys.io/docs[/dim]") + else: + console.print(f"[red]Configuration error: {e}[/red]") + console.print("[yellow]Hint:[/yellow] Use 'pyne api configure' to set up your API configuration.") raise typer.Exit(1) except Exception as e: diff --git a/src/pynecore/cli/commands/compile.py b/src/pynecore/cli/commands/compile.py index 0964197..ac73546 100644 --- a/src/pynecore/cli/commands/compile.py +++ b/src/pynecore/cli/commands/compile.py @@ -179,8 +179,19 @@ def compile( raise typer.Exit(1) except ValueError as e: - console.print(f"[red]Configuration error: {e}[/red]") - console.print("[yellow]Hint:[/yellow] Use 'pyne api configure' to set up your API configuration.") + error_msg = str(e) + if "No configuration file found" in error_msg or "Configuration file not found" in error_msg: + # No API configuration found - show helpful setup message + console.print("[yellow]⚠️ No API configuration found[/yellow]") + console.print() + console.print("To get started with PyneSys API:") + console.print("1. 🌐 Visit [blue][link=https://pynesys.io]https://pynesys.io[/link][/blue] to get your API key") + console.print("2. 🔧 Run [cyan]pyne api configure[/cyan] to set up your configuration") + console.print() + console.print("[dim]Need help? Check our documentation at https://pynesys.io/docs[/dim]") + else: + console.print(f"[red]Configuration error: {e}[/red]") + console.print("[yellow]Hint:[/yellow] Use 'pyne api configure' to set up your API configuration.") raise typer.Exit(1) except Exception as e: From 9da94f9991805954835da2a10981f1de68e9ae7d Mon Sep 17 00:00:00 2001 From: Mahadi Hassan Date: Wed, 30 Jul 2025 19:52:00 +0600 Subject: [PATCH 03/31] fix(cli): improve rate limit exceeded error message Provide more helpful information when API rate limit is exceeded, including upgrade options and retry suggestion --- src/pynecore/cli/commands/compile.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pynecore/cli/commands/compile.py b/src/pynecore/cli/commands/compile.py index ac73546..8ac5fdd 100644 --- a/src/pynecore/cli/commands/compile.py +++ b/src/pynecore/cli/commands/compile.py @@ -165,7 +165,9 @@ def compile( except RateLimitError: progress.update(task, description="[red]Rate limit exceeded[/red]") - console.print("[red]✗[/red] Rate limit exceeded. Please try again later.") + console.print("[red]✗[/red] You've reached your API rate limit.") + console.print("[yellow]💡 To increase your limits, consider upgrading your subscription at [blue][link=https://pynesys.io]https://pynesys.io[/link][/blue]") + console.print("[dim]You can also try again in a few minutes/hours when your rate limit resets.[/dim]") raise typer.Exit(1) except CompilationError as e: From 26811a263f1f4d6fb67c4df8dddfd09b3fc63755 Mon Sep 17 00:00:00 2001 From: Mahadi Hassan Date: Wed, 30 Jul 2025 20:09:04 +0600 Subject: [PATCH 04/31] refactor(cli): simplify api configuration by removing redundant options --- src/pynecore/cli/commands/api.py | 50 ++++++++------------------------ 1 file changed, 12 insertions(+), 38 deletions(-) diff --git a/src/pynecore/cli/commands/api.py b/src/pynecore/cli/commands/api.py index 9bac166..3d63cd8 100644 --- a/src/pynecore/cli/commands/api.py +++ b/src/pynecore/cli/commands/api.py @@ -57,40 +57,23 @@ def configure( help="PyneSys API key", prompt="Enter your PyneSys API key", hide_input=True - ), - base_url: str = typer.Option( - "https://api.pynesys.io", - "--base-url", - help="API base URL" - ), - timeout: int = typer.Option( - 30, - "--timeout", - help="Request timeout in seconds" - ), - config_path: Optional[Path] = typer.Option( - None, - "--config", - help="Configuration file path (defaults to ~/.pynecore/config.json)" ) ): """Configure PyneSys API settings and validate your API key. This command sets up your PyneSys API configuration including: - API key for authentication - - Request timeout settings - The configuration is saved to ~/.pynecore/config.json by default. - You can specify a custom config path using --config option. + The configuration is saved to ~/.pynecore/config.json. The API key will be validated during configuration to ensure it's working. """ try: - # Create configuration + # Create configuration with default values config = APIConfig( api_key=api_key, - base_url=base_url, - timeout=timeout + base_url="https://api.pynesys.io", + timeout=30 ) # Test the API key @@ -134,9 +117,9 @@ def configure( raise typer.Exit(1) # Save configuration - ConfigManager.save_config(config, config_path) + ConfigManager.save_config(config, None) - config_file = config_path or ConfigManager.get_default_config_path() + config_file = ConfigManager.get_default_config_path() console.print(f"[green]✓[/green] Configuration saved to: {config_file}") except Exception as e: @@ -145,13 +128,7 @@ def configure( @api_app.command() -def status( - config_path: Optional[Path] = typer.Option( - None, - "--config", - help="Configuration file path" - ) -): +def status(): """Check API configuration and connection status. This command displays: @@ -162,10 +139,12 @@ def status( Use this command to verify your API setup is working correctly and to check when your API token will expire. + + Configuration is loaded from the default location (~/.pynecore/config.json). """ try: # Load configuration - config = ConfigManager.load_config(config_path) + config = ConfigManager.load_config(None) # Display configuration info table = Table(title="API Configuration") @@ -236,11 +215,6 @@ def status( @api_app.command() def reset( - config_path: Optional[Path] = typer.Option( - None, - "--config", - help="Configuration file path" - ), force: bool = typer.Option( False, "--force", @@ -250,7 +224,7 @@ def reset( """Reset API configuration by removing the configuration file. This command will: - - Delete the API configuration file + - Delete the default API configuration file (~/.pynecore/config.json) - Remove all stored API settings (API key, timeout) - Require you to run 'pyne api configure' again to set up the API @@ -258,7 +232,7 @@ def reset( This is useful when you want to start fresh with API configuration or switch to a different API key. """ - config_file = config_path or ConfigManager.get_default_config_path() + config_file = ConfigManager.get_default_config_path() if not config_file.exists(): console.print(f"[yellow]No configuration file found at: {config_file}[/yellow]") From 5ac4db06e113a931db25f386046d6e1723fb059e Mon Sep 17 00:00:00 2001 From: Mahadi Hassan Date: Wed, 30 Jul 2025 20:23:11 +0600 Subject: [PATCH 05/31] feat(cli): add direct api key testing to status command Add support for testing API keys directly without saving to config via --api-key flag. Improves debugging workflow by allowing quick key validation and provides better error messaging when no configuration exists. --- src/pynecore/cli/commands/api.py | 64 +++++++++++++++++++++++--------- 1 file changed, 47 insertions(+), 17 deletions(-) diff --git a/src/pynecore/cli/commands/api.py b/src/pynecore/cli/commands/api.py index 3d63cd8..f7816db 100644 --- a/src/pynecore/cli/commands/api.py +++ b/src/pynecore/cli/commands/api.py @@ -128,7 +128,13 @@ def configure( @api_app.command() -def status(): +def status( + api_key: Optional[str] = typer.Option( + None, + "--api-key", + help="Test a specific API key directly (without saving to config)" + ) +): """Check API configuration and connection status. This command displays: @@ -140,22 +146,45 @@ def status(): Use this command to verify your API setup is working correctly and to check when your API token will expire. - Configuration is loaded from the default location (~/.pynecore/config.json). + If --api-key is provided, it will test that specific key directly without + saving it to the configuration. Otherwise, it loads the saved configuration + from ~/.pynecore/config.json. """ try: - # Load configuration - config = ConfigManager.load_config(None) - - # Display configuration info - table = Table(title="API Configuration") - table.add_column("Setting", style="cyan") - table.add_column("Value", style="white") - - table.add_row("Base URL", config.base_url) - table.add_row("Timeout", f"{config.timeout}s") - table.add_row("API Key", f"{config.api_key[:8]}...{config.api_key[-4:]}" if len(config.api_key) > 12 else "***") - - console.print(table) + if api_key: + # Test the provided API key directly + config = APIConfig( + api_key=api_key, + base_url="https://api.pynesys.io", + timeout=30 + ) + + # Display test configuration info + table = Table(title="API Key Test") + table.add_column("Setting", style="cyan") + table.add_column("Value", style="white") + + table.add_row("Base URL", config.base_url) + table.add_row("Timeout", f"{config.timeout}s") + table.add_row("API Key", f"{config.api_key[:8]}...{config.api_key[-4:]}" if len(config.api_key) > 12 else "***") + table.add_row("Mode", "[yellow]Direct Test (not saved)[/yellow]") + + console.print(table) + else: + # Load configuration from file + config = ConfigManager.load_config(None) + + # Display configuration info + table = Table(title="API Configuration") + table.add_column("Setting", style="cyan") + table.add_column("Value", style="white") + + table.add_row("Base URL", config.base_url) + table.add_row("Timeout", f"{config.timeout}s") + table.add_row("API Key", f"{config.api_key[:8]}...{config.api_key[-4:]}" if len(config.api_key) > 12 else "***") + table.add_row("Mode", "[green]Saved Configuration[/green]") + + console.print(table) # Test connection with Progress( @@ -194,13 +223,14 @@ def status(): except ValueError as e: error_msg = str(e) - if "No configuration file found" in error_msg or "Configuration file not found" in error_msg: - # No API configuration found - show helpful setup message + if not api_key and ("No configuration file found" in error_msg or "Configuration file not found" in error_msg): + # No API configuration found and no API key provided - show helpful setup message console.print("[yellow]⚠️ No API configuration found[/yellow]") console.print() console.print("To get started with PyneSys API:") console.print("1. 🌐 Visit [blue][link=https://pynesys.io]https://pynesys.io[/link][/blue] to get your API key") console.print("2. 🔧 Run [cyan]pyne api configure[/cyan] to set up your configuration") + console.print("3. 🧪 Or test a key directly with [cyan]pyne api status --api-key YOUR_KEY[/cyan]") console.print() console.print("[dim]Need help? Check our documentation at https://pynesys.io/docs[/dim]") else: From 458291e2db1c7112e5e546c0bdd6d04a53f45969 Mon Sep 17 00:00:00 2001 From: Mahadi Hassan Date: Thu, 31 Jul 2025 17:55:47 +0600 Subject: [PATCH 06/31] feat(compiler): implement smart compilation with hash tracking refactor(config): standardize config file naming to api.toml feat(cli): add support for .pine file compilation in run command docs: update help texts to reflect new config file naming --- src/pynecore/api/config.py | 4 +- src/pynecore/cli/commands/api.py | 10 +- src/pynecore/cli/commands/compile.py | 218 +++++++++++++---------- src/pynecore/cli/commands/run.py | 247 ++++++++++++++++++++++++++- src/pynecore/core/compiler.py | 235 +++++++++++++++++++++++++ src/pynecore/utils/hash_utils.py | 143 ++++++++++++++++ 6 files changed, 753 insertions(+), 104 deletions(-) create mode 100644 src/pynecore/core/compiler.py create mode 100644 src/pynecore/utils/hash_utils.py diff --git a/src/pynecore/api/config.py b/src/pynecore/api/config.py index ba670c3..7f961be 100644 --- a/src/pynecore/api/config.py +++ b/src/pynecore/api/config.py @@ -152,8 +152,8 @@ def save_to_file(self, config_path: Path) -> None: class ConfigManager: """Manages API configuration loading and saving.""" - DEFAULT_CONFIG_PATH = Path("workdir/config/api_config.toml") - DEFAULT_FALLBACK_CONFIG_PATH = Path.home() / ".pynecore" / "api_config.toml" + DEFAULT_CONFIG_PATH = Path("workdir/config/api.toml") + DEFAULT_FALLBACK_CONFIG_PATH = Path.home() / ".pynecore" / "api.toml" @classmethod def load_config(cls, config_path: Optional[Path] = None) -> APIConfig: diff --git a/src/pynecore/cli/commands/api.py b/src/pynecore/cli/commands/api.py index f7816db..e998080 100644 --- a/src/pynecore/cli/commands/api.py +++ b/src/pynecore/cli/commands/api.py @@ -64,7 +64,7 @@ def configure( This command sets up your PyneSys API configuration including: - API key for authentication - The configuration is saved to ~/.pynecore/config.json. + The configuration is saved to workdir/config/api.toml. The API key will be validated during configuration to ensure it's working. """ @@ -138,7 +138,7 @@ def status( """Check API configuration and connection status. This command displays: - - Current API configuration (base URL, timeout, masked API key) + - Current API configuration (timeout, masked API key) - API connection test results - User information (User ID, token type) - Token expiration details (expires at, expires in human-readable format) @@ -148,7 +148,7 @@ def status( If --api-key is provided, it will test that specific key directly without saving it to the configuration. Otherwise, it loads the saved configuration - from ~/.pynecore/config.json. + from ~/.pynecore/api.toml. """ try: if api_key: @@ -164,7 +164,6 @@ def status( table.add_column("Setting", style="cyan") table.add_column("Value", style="white") - table.add_row("Base URL", config.base_url) table.add_row("Timeout", f"{config.timeout}s") table.add_row("API Key", f"{config.api_key[:8]}...{config.api_key[-4:]}" if len(config.api_key) > 12 else "***") table.add_row("Mode", "[yellow]Direct Test (not saved)[/yellow]") @@ -179,7 +178,6 @@ def status( table.add_column("Setting", style="cyan") table.add_column("Value", style="white") - table.add_row("Base URL", config.base_url) table.add_row("Timeout", f"{config.timeout}s") table.add_row("API Key", f"{config.api_key[:8]}...{config.api_key[-4:]}" if len(config.api_key) > 12 else "***") table.add_row("Mode", "[green]Saved Configuration[/green]") @@ -254,7 +252,7 @@ def reset( """Reset API configuration by removing the configuration file. This command will: - - Delete the default API configuration file (~/.pynecore/config.json) + - Delete the default API configuration file (workdir/config/api.toml) - Remove all stored API settings (API key, timeout) - Require you to run 'pyne api configure' again to set up the API diff --git a/src/pynecore/cli/commands/compile.py b/src/pynecore/cli/commands/compile.py index 8ac5fdd..aec74eb 100644 --- a/src/pynecore/cli/commands/compile.py +++ b/src/pynecore/cli/commands/compile.py @@ -7,8 +7,9 @@ from rich.progress import Progress, SpinnerColumn, TextColumn from ..app import app, app_state -from ...api import PynesysAPIClient, ConfigManager, APIError, AuthError, RateLimitError, CompilationError -from ...utils.file_utils import should_compile, preserve_mtime +from ...api import ConfigManager, APIError, AuthError, RateLimitError, CompilationError +from ...core.compiler import create_compilation_service +from ...utils.file_utils import preserve_mtime __all__ = [] @@ -44,7 +45,7 @@ def compile( config_path: Optional[Path] = typer.Option( None, "--config", - help="Path to TOML configuration file (defaults to workdir/config/api_config.toml)" + help="Path to TOML configuration file (defaults to workdir/config/api.toml)" ), api_key: Optional[str] = typer.Option( None, @@ -62,8 +63,8 @@ def compile( pyne compile --output # Specify output file path CONFIGURATION: - Default config: workdir/config/api_config.toml - Fallback config: ~/.pynecore/api_config.toml + Default config: workdir/config/api.toml + Fallback config: ~/.pynecore/api.toml Custom config: --config /path/to/config.toml Config format (TOML only): @@ -86,99 +87,138 @@ def compile( Use 'pyne api configure' to set up your API configuration. """ try: - # Load configuration - if api_key: - from ...api.config import APIConfig - config = APIConfig(api_key=api_key) - else: - config = ConfigManager.load_config(config_path) - - # Read Pine Script content - try: - with open(script_path, 'r', encoding='utf-8') as f: - script_content = f.read() - except Exception as e: - console.print(f"[red]Error reading script file: {e}[/red]") - raise typer.Exit(1) + # Create compilation service + compilation_service = create_compilation_service( + api_key=api_key, + config_path=config_path + ) # Determine output path if output is None: output = script_path.with_suffix('.py') # Check if compilation is needed (smart compilation) - if not should_compile(script_path, output, force): + if not compilation_service.needs_compilation(script_path, output) and not force: console.print(f"[green]✓[/green] Output file is up-to-date: {output}") console.print("[dim]Use --force to recompile anyway[/dim]") return # Compile script - with Progress( - SpinnerColumn(), - TextColumn("[progress.description]{task.description}"), - console=console - ) as progress: - task = progress.add_task("Compiling Pine Script...", total=None) + try: + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console + ) as progress: + progress.add_task("Compiling Pine Script...", total=None) + + # Compile the .pine file + compiled_path = compilation_service.compile_file( + script_path, + output, + force=force, + strict=strict + ) + + # Preserve modification time from source file + preserve_mtime(script_path, output) - try: - # Use synchronous client for CLI - client = PynesysAPIClient(config.api_key, config.base_url, config.timeout) - result = client.compile_script_sync(script_content, strict=strict) + console.print(f"[green]Compilation successful![/green] Your Pine Script is now ready: [cyan]{compiled_path}[/cyan]") - if result.success: - # Write compiled Python code to output file - try: - output.parent.mkdir(parents=True, exist_ok=True) - with open(output, 'w', encoding='utf-8') as f: - f.write(result.compiled_code) - - # Preserve modification time from source file - preserve_mtime(script_path, output) - - progress.update(task, description="[green]Compilation successful![/green]") - console.print(f"[green]✓[/green] Compiled successfully to: {output}") - - if result.warnings: - console.print("[yellow]Warnings:[/yellow]") - for warning in result.warnings: - console.print(f" [yellow]•[/yellow] {warning}") - - except Exception as e: - console.print(f"[red]Error writing output file: {e}[/red]") - raise typer.Exit(1) - + except CompilationError as e: + console.print(f"[red]❌ Oops! Compilation encountered an issue:[/red] {str(e)}") + if e.validation_errors: + console.print("[red]Validation errors:[/red]") + for error in e.validation_errors: + console.print(f" [red]• {error}[/red]") + raise typer.Exit(1) + + except AuthError as e: + console.print(f"[red]🔐 Authentication issue:[/red] {str(e)}") + console.print("[yellow]🚀 Quick fix:[/yellow] Run [cyan]'pyne api configure'[/cyan] to set up your API key and get back on track!") + raise typer.Exit(1) + + except RateLimitError as e: + console.print(f"[red]✗[/red] Rate limit exceeded: {str(e)}") + if e.retry_after: + console.print(f"[yellow]Please try again in {e.retry_after} seconds[/yellow]") + console.print("[yellow]💡 To increase your limits, consider upgrading your subscription at [blue][link=https://pynesys.io]https://pynesys.io[/link][/blue]") + raise typer.Exit(1) + + except APIError as e: + error_msg = str(e).lower() + + # Handle specific API error scenarios based on HTTP status codes + if "400" in error_msg or "bad request" in error_msg: + if "compilation fails" in error_msg or "script is too large" in error_msg: + console.print("[red]📝 Script Issue:[/red] Your Pine Script couldn't be compiled") + console.print("[yellow]💡 Common fixes:[/yellow]") + console.print(" • Check if your script is too large (try breaking it into smaller parts)") + console.print(" • Verify your Pine Script syntax is correct") + console.print(" • Make sure you're using Pine Script v6 syntax") else: - progress.update(task, description="[red]Compilation failed[/red]") - console.print(f"[red]✗[/red] Compilation failed: {result.error}") - - if result.details: - console.print("[red]Details:[/red]") - for detail in result.details: - console.print(f" [red]•[/red] {detail}") - - raise typer.Exit(1) + console.print(f"[red]⚠️ Request Error:[/red] {str(e)}") + console.print("[yellow]💡 This usually means there's an issue with the request format[/yellow]") - except AuthError: - progress.update(task, description="[red]Authentication failed[/red]") - console.print("[red]✗[/red] Authentication failed. Please check your API key.") - console.print("[yellow]Hint:[/yellow] Use 'pyne api configure' to set up your API key.") - raise typer.Exit(1) + elif "401" in error_msg or "authentication" in error_msg or "no permission" in error_msg: + console.print("[red]🔐 Authentication Failed:[/red] Your API credentials aren't working") + console.print("[yellow]🚀 Quick fixes:[/yellow]") + console.print(" • Check if your API key is valid and active") + console.print(" • Verify your token type is allowed for compilation") + console.print("[blue]🔑 Get a new API key at [link=https://pynesys.io]https://pynesys.io[/link][/blue]") + console.print("[blue]⚙️ Then run [cyan]'pyne api configure'[/cyan] to update your configuration[/blue]") + + elif "404" in error_msg or "not found" in error_msg: + console.print("[red]🔍 Not Found:[/red] The API endpoint or user wasn't found") + console.print("[yellow]💡 This might indicate:[/yellow]") + console.print(" • Your account may not exist or be accessible") + console.print(" • There might be a temporary service issue") + + elif "422" in error_msg or "validation error" in error_msg: + console.print("[red]📋 Validation Error:[/red] Your request data has validation issues") + console.print("[yellow]💡 Common causes:[/yellow]") + console.print(" • Invalid Pine Script syntax or structure") + console.print(" • Missing required parameters") + console.print(" • Incorrect data format") + console.print(f"[dim]Details: {str(e)}[/dim]") + + elif "429" in error_msg or "rate limit" in error_msg or "too many requests" in error_msg: + console.print("[red]🚦 Rate Limit Exceeded:[/red] You've hit your compilation limit") + console.print("[yellow]⏰ What you can do:[/yellow]") + console.print(" • Wait a bit before trying again") + console.print(" • Consider upgrading your plan for higher limits") + console.print("[blue]💎 Upgrade at [link=https://pynesys.io/pricing]https://pynesys.io/pricing[/link][/blue]") - except RateLimitError: - progress.update(task, description="[red]Rate limit exceeded[/red]") - console.print("[red]✗[/red] You've reached your API rate limit.") - console.print("[yellow]💡 To increase your limits, consider upgrading your subscription at [blue][link=https://pynesys.io]https://pynesys.io[/link][/blue]") - console.print("[dim]You can also try again in a few minutes/hours when your rate limit resets.[/dim]") - raise typer.Exit(1) + elif "500" in error_msg or "server" in error_msg or "internal" in error_msg: + console.print("[red]🔧 Server Error:[/red] Something went wrong on our end") + console.print("[yellow]😅 Don't worry, it's not you![/yellow]") + console.print(" • This is a temporary server issue") + console.print(" • Please try again in a few moments") - except CompilationError as e: - progress.update(task, description="[red]Compilation error[/red]") - console.print(f"[red]✗[/red] Compilation error: {e}") - raise typer.Exit(1) + elif "unsupported pinescript version" in error_msg: + console.print("[red]📌 Version Issue:[/red] Your Pine Script version isn't supported") + if "version 5" in error_msg: + console.print("[yellow]🔄 Pine Script v5 → v6 Migration:[/yellow]") + console.print(" • Update your script to Pine Script version 6") + console.print(" • Most v5 scripts need minimal changes") + console.print("[blue]📖 Migration guide: [link=https://www.tradingview.com/pine-script-docs/en/v6/migration_guides/v5_to_v6_migration_guide.html]Pine Script v5→v6 Guide[/link][/blue]") + else: + console.print("[yellow]💡 Only Pine Script version 6 is currently supported[/yellow]") + + elif "api key" in error_msg: + console.print("[red]🔑 API Key Issue:[/red] There's a problem with your API key") + console.print("[blue]🔑 Get your API key at [link=https://pynesys.io]https://pynesys.io[/link][/blue]") + console.print("[blue]⚙️ Then run [cyan]'pyne api configure'[/cyan] to set up your configuration[/blue]") + + else: + # Generic API error fallback + console.print(f"[red]🌐 API Error:[/red] {str(e)}") + console.print("[yellow]💡 If this persists, please check:[/yellow]") + console.print(" • Your internet connection") + console.print(" • API service status") + console.print("[blue]📞 Need help? [link=https://pynesys.io/support]Contact Support[/link][/blue]") - except APIError as e: - progress.update(task, description="[red]API error[/red]") - console.print(f"[red]✗[/red] API error: {e}") - raise typer.Exit(1) + raise typer.Exit(1) except ValueError as e: error_msg = str(e) @@ -186,16 +226,14 @@ def compile( # No API configuration found - show helpful setup message console.print("[yellow]⚠️ No API configuration found[/yellow]") console.print() - console.print("To get started with PyneSys API:") - console.print("1. 🌐 Visit [blue][link=https://pynesys.io]https://pynesys.io[/link][/blue] to get your API key") - console.print("2. 🔧 Run [cyan]pyne api configure[/cyan] to set up your configuration") + console.print("[bold]Quick setup (takes just few minutes):[/bold]") + console.print("1. 🌐 Get your API key at [blue][link=https://pynesys.io]https://pynesys.io[/link][/blue]") + console.print("2. 🔧 Run [cyan]pyne api configure[/cyan] to save your configuration") console.print() - console.print("[dim]Need help? Check our documentation at https://pynesys.io/docs[/dim]") + console.print("[dim]💬 Need assistance? Our docs are here: https://pynesys.io/docs[/dim]") + elif "this file format isn't supported" in error_msg: + # File format error - show the friendly message directly + console.print(f"[red]{e}[/red]") else: - console.print(f"[red]Configuration error: {e}[/red]") - console.print("[yellow]Hint:[/yellow] Use 'pyne api configure' to set up your API configuration.") - raise typer.Exit(1) - - except Exception as e: - console.print(f"[red]Unexpected error: {e}[/red]") + console.print(f"[red]Attention:[/red] {e}") raise typer.Exit(1) diff --git a/src/pynecore/cli/commands/run.py b/src/pynecore/cli/commands/run.py index 4d2e34f..5518052 100644 --- a/src/pynecore/cli/commands/run.py +++ b/src/pynecore/cli/commands/run.py @@ -4,11 +4,13 @@ import threading import time import sys +from typing import Optional from typer import Option, Argument, secho, Exit from rich.progress import (Progress, SpinnerColumn, TextColumn, BarColumn, ProgressColumn, Task) from rich.text import Text +from rich.console import Console from ..app import app, app_state @@ -17,9 +19,14 @@ from pynecore.core.syminfo import SymInfo from pynecore.core.script_runner import ScriptRunner +from ...core.compiler import create_compilation_service +from ...api.exceptions import APIError, AuthError, RateLimitError, CompilationError +from ...api.config import ConfigManager __all__ = [] +console = Console() + class CustomTimeElapsedColumn(ProgressColumn): """Custom time elapsed column showing milliseconds.""" @@ -53,7 +60,7 @@ def render(self, task: Task) -> Text: @app.command() def run( - script: Path = Argument(..., dir_okay=False, file_okay=True, help="Script to run"), + script: Path = Argument(..., dir_okay=False, file_okay=True, help="Script to run (.py or .pine)"), data: Path = Argument(..., dir_okay=False, file_okay=True, help="Data file to use (*.ohlcv)"), time_from: datetime | None = Option(None, '--from', '-f', @@ -74,9 +81,34 @@ def run( trade_path: Path | None = Option(None, "--trade", "-tp", help="Path to save the trade data", rich_help_panel="Out Path Options"), + force: bool = Option( + False, + "--force", + help="Force recompilation for .pine files (ignore smart compilation)", + rich_help_panel="Compilation Options" + ), + strict: bool = Option( + False, + "--strict", + help="Enable strict compilation mode for .pine files", + rich_help_panel="Compilation Options" + ), + api_key: Optional[str] = Option( + None, + "--api-key", + help="PyneSys API key (overrides configuration file)", + envvar="PYNESYS_API_KEY", + rich_help_panel="Compilation Options" + ), + config_path: Optional[Path] = Option( + None, + "--config", + help="Path to TOML configuration file", + rich_help_panel="Compilation Options" + ), ): """ - Run a script + Run a script (.py or .pine) The system automatically searches for the workdir folder in the current and parent directories. If not found, it creates or uses a workdir folder in the current directory. @@ -85,17 +117,220 @@ def run( Similarly, if [bold]data[/] path is a name without full path, it will be searched in the [italic]"workdir/data"[/] directory. The [bold]plot_path[/], [bold]strat_path[/], and [bold]trade_path[/] work the same way - if they are names without full paths, they will be saved in the [italic]"workdir/output"[/] directory. + + [bold]Pine Script Support:[/bold] + When running a .pine file, it will be automatically compiled to Python before execution. + Use the compilation options (--force, --strict) to control the compilation process. + A valid PyneSys API key is required for Pine Script compilation. + + [bold]Smart Compilation:[/bold] + The system checks for changes in .pine files and only recompiles when necessary. + Use --force to bypass this check and force recompilation. """ # noqa - # Ensure .py extension - if script.suffix != ".py": - script = script.with_suffix(".py") - # Expand script path + # Handle script file extension and path + original_script = script + compiled_file = None + + # Support both .py and .pine files + if script.suffix not in [".py", ".pine"]: + # Check if the file exists with the given extension first + if len(script.parts) == 1: + full_script_path = app_state.scripts_dir / script + else: + full_script_path = script + + if full_script_path.exists(): + # File exists but has unsupported extension + console.print(f"[red]This file format isn't supported:[/red] {script.suffix}") + console.print("[yellow]✨ Currently supported formats:[/yellow] .py, .pine") + if script.suffix in [".ohlcv", ".csv", ".json"]: + console.print(f"[blue]💡 Heads up:[/blue] {script.suffix} files are data files, not executable scripts.") + raise Exit(1) + + # File doesn't exist, try .pine first, then .py + pine_script = script.with_suffix(".pine") + py_script = script.with_suffix(".py") + + if len(script.parts) == 1: + pine_script = app_state.scripts_dir / pine_script + py_script = app_state.scripts_dir / py_script + + if pine_script.exists(): + script = pine_script + elif py_script.exists(): + script = py_script + else: + script = py_script # Default to .py for error message + + # Expand script path if it's just a filename if len(script.parts) == 1: script = app_state.scripts_dir / script + # Check if script exists if not script.exists(): secho(f"Script file '{script}' not found!", fg="red", err=True) raise Exit(1) + + # Handle .pine files - compile them first + if script.suffix == ".pine": + try: + # Create compilation service + compilation_service = create_compilation_service( + api_key=api_key, + config_path=config_path + ) + + # Determine output path for compiled file + compiled_file = script.with_suffix(".py") + + # Check if compilation is needed + if compilation_service.needs_compilation(script, compiled_file) or force: + try: + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console + ) as progress: + progress.add_task("Compiling Pine Script...", total=None) + + # Compile the .pine file + compiled_path = compilation_service.compile_file( + script, + compiled_file, + force=force, + strict=strict + ) + + console.print(f"[green]Compilation successful![/green] Ready to run: [cyan]{compiled_path}[/cyan]") + compiled_file = compiled_path + + except CompilationError as e: + console.print(f"[red]❌ Pine Script compilation encountered an issue:[/red] {str(e)}") + if e.validation_errors: + console.print("[red]Validation errors:[/red]") + for error in e.validation_errors: + console.print(f" [red]• {error}[/red]") + raise Exit(1) + + except AuthError as e: + console.print(f"[red]🔐 Authentication issue:[/red] {str(e)}") + console.print("[yellow]🚀 Quick fix:[/yellow] Run [cyan]'pyne api configure'[/cyan] to set up your API key and get back on track!") + console.print("[blue]Visit https://pynesys.io to get your API key[/blue]") + console.print("[blue]Run 'pyne api configure' to set up your configuration[/blue]") + raise Exit(1) + + except RateLimitError as e: + console.print(f"[red]Rate limit exceeded: {str(e)}[/red]") + if e.retry_after: + console.print(f"[yellow]⏰ Just a moment - please try again in {e.retry_after} seconds[/yellow]") + console.print("[yellow]🚀 Want more compilation? Consider upgrading your subscription at [blue][link=https://pynesys.io]https://pynesys.io[/link][/blue] for higher limits![/yellow]") + raise Exit(1) + + except APIError as e: + error_msg = str(e).lower() + + # Handle specific API error scenarios based on HTTP status codes + if "400" in error_msg or "bad request" in error_msg: + if "compilation fails" in error_msg or "script is too large" in error_msg: + console.print("[red]📝 Script Issue:[/red] Your Pine Script couldn't be compiled") + console.print("[yellow]💡 Common fixes:[/yellow]") + console.print(" • Check if your script is too large (try breaking it into smaller parts)") + console.print(" • Verify your Pine Script syntax is correct") + console.print(" • Make sure you're using Pine Script v6 syntax") + else: + console.print(f"[red]⚠️ Request Error:[/red] {str(e)}") + console.print("[yellow]💡 This usually means there's an issue with the request format[/yellow]") + + elif "401" in error_msg or "authentication" in error_msg or "no permission" in error_msg: + console.print("[red]🔐 Authentication Failed:[/red] Your API credentials aren't working") + console.print("[yellow]🚀 Quick fixes:[/yellow]") + console.print(" • Check if your API key is valid and active") + console.print(" • Verify your token type is allowed for compilation") + console.print("[blue]🔑 Get a new API key at [link=https://pynesys.io]https://pynesys.io[/link][/blue]") + console.print("[blue]⚙️ Then run [cyan]'pyne api configure'[/cyan] to update your configuration[/blue]") + + elif "404" in error_msg or "not found" in error_msg: + console.print("[red]🔍 Not Found:[/red] The API endpoint or user wasn't found") + console.print("[yellow]💡 This might indicate:[/yellow]") + console.print(" • Your account may not exist or be accessible") + console.print(" • There might be a temporary service issue") + console.print("[blue]📞 Contact support if this persists: [link=https://pynesys.io/support]https://pynesys.io/support[/link][/blue]") + + elif "422" in error_msg or "validation error" in error_msg: + console.print("[red]📋 Validation Error:[/red] Your request data has validation issues") + console.print("[yellow]💡 Common causes:[/yellow]") + console.print(" • Invalid Pine Script syntax or structure") + console.print(" • Missing required parameters") + console.print(" • Incorrect data format") + console.print(f"[dim]Details: {str(e)}[/dim]") + + elif "429" in error_msg or "rate limit" in error_msg or "too many requests" in error_msg: + console.print("[red]🚦 Rate Limit Exceeded:[/red] You've hit your compilation limit") + console.print("[yellow]⏰ What you can do:[/yellow]") + console.print(" • Wait a bit before trying again") + console.print(" • Consider upgrading your plan for higher limits") + console.print("[blue]💎 Upgrade at [link=https://pynesys.io/pricing]https://pynesys.io/pricing[/link][/blue]") + + elif "500" in error_msg or "server" in error_msg or "internal" in error_msg: + console.print("[red]🔧 Server Error:[/red] Something went wrong on our end") + console.print("[yellow]😅 Don't worry, it's not you![/yellow]") + console.print(" • This is a temporary server issue") + console.print(" • Please try again in a few moments") + console.print("[blue]📊 Check service status: [link=https://status.pynesys.io]https://status.pynesys.io[/link][/blue]") + + elif "unsupported pinescript version" in error_msg: + console.print("[red]📌 Version Issue:[/red] Your Pine Script version isn't supported") + if "version 5" in error_msg: + console.print("[yellow]🔄 Pine Script v5 → v6 Migration:[/yellow]") + console.print(" • Update your script to Pine Script version 6") + console.print(" • Most v5 scripts need minimal changes") + console.print("[blue]📖 Migration guide: [link=https://www.tradingview.com/pine-script-docs/en/v6/migration_guides/v5_to_v6_migration_guide.html]Pine Script v5→v6 Guide[/link][/blue]") + else: + console.print("[yellow]💡 Only Pine Script version 6 is currently supported[/yellow]") + + elif "api key" in error_msg: + console.print("[red]🔑 API Key Issue:[/red] There's a problem with your API key") + console.print("[blue]🔑 Get your API key at [link=https://pynesys.io]https://pynesys.io[/link][/blue]") + console.print("[blue]⚙️ Then run [cyan]'pyne api configure'[/cyan] to set up your configuration[/blue]") + + else: + # Generic API error fallback + console.print(f"[red]🌐 API Error:[/red] {str(e)}") + console.print("[yellow]💡 If this persists, please check:[/yellow]") + console.print(" • Your internet connection") + console.print(" • API service status") + console.print("[blue]📞 Need help? [link=https://pynesys.io/support]Contact Support[/link][/blue]") + + raise Exit(1) + + else: + console.print(f"[green]⚡ Using cached version:[/green] [cyan]{compiled_file}[/cyan]") + console.print("[dim]Use --force to recompile[/dim]") + + # Update script to point to the compiled file + script = compiled_file + + except ValueError as e: + error_msg = str(e) + if "No configuration file found" in error_msg or "Configuration file not found" in error_msg: + console.print("[yellow]⚠️ No API configuration found[/yellow]") + console.print() + console.print("[bold]Quick setup (takes few minutes):[/bold]") + console.print("1. 🌐 Get your API key at [blue][link=https://pynesys.io]https://pynesys.io[/link][/blue]") + console.print("2. 🔧 Run [cyan]pyne api configure[/cyan] to save your configuration") + console.print() + console.print("[dim]💬 Need assistance? Our docs are here: https://pynesys.io/docs[/dim]") + else: + console.print(f"[red] Attention:[/red] {e}") + raise Exit(1) + + # Ensure we have a .py file at this point + if script.suffix != ".py": + console.print(f"[red]This file format isn't supported:[/red] {script.suffix}") + console.print("[yellow]✨ Currently supported formats:[/yellow] .py, .pine") + if script.suffix in [".ohlcv", ".csv", ".json"]: + console.print(f"[blue]💡 Heads up:[/blue] {script.suffix} files are data files, not executable scripts.") + raise Exit(1) # Check file format and extension if data.suffix == "": diff --git a/src/pynecore/core/compiler.py b/src/pynecore/core/compiler.py new file mode 100644 index 0000000..b3a94b4 --- /dev/null +++ b/src/pynecore/core/compiler.py @@ -0,0 +1,235 @@ +"""Core compilation service for programmatic use. + +The CLI should use this service rather than implementing compilation logic directly. +This ensures all compilation functionality is available programmatically. +""" + +import subprocess +import sys +from pathlib import Path +from typing import Optional, Dict, Any + +from ..api.client import PynesysAPIClient +from ..api.config import APIConfig, ConfigManager +from ..api.exceptions import APIError, AuthError, RateLimitError, CompilationError +from ..utils.hash_utils import ( + file_needs_compilation, + update_hash_after_compilation, + calculate_file_md5, + get_stored_hash, + store_hash +) + + +class CompilationService: + """Core compilation service for programmatic use. + + The CLI should use this service rather than implementing compilation logic directly. + This ensures all compilation functionality is available programmatically. + """ + + def __init__(self, api_client: Optional[PynesysAPIClient] = None, config: Optional[APIConfig] = None): + """Initialize the compilation service. + + Args: + api_client: Optional pre-configured API client + config: Optional API configuration (will load from default if not provided) + """ + if api_client: + self.api_client = api_client + else: + # Load config if not provided + if not config: + config = ConfigManager.load_config() + self.api_client = PynesysAPIClient( + api_key=config.api_key, + base_url=config.base_url, + timeout=config.timeout + ) + + def compile_file( + self, + pine_file_path: Path, + output_file_path: Optional[Path] = None, + force: bool = False, + strict: bool = False + ) -> Path: + """Compile a .pine file to Python. + + Args: + pine_file_path: Path to the .pine file + output_file_path: Optional output path (defaults to .py extension) + force: Force recompilation even if file hasn't changed + strict: Enable strict compilation mode + + Returns: + Path to the compiled .py file + + Raises: + FileNotFoundError: If pine file doesn't exist + CompilationError: If compilation fails + APIError: If API request fails + """ + # Validate input file + if not pine_file_path.exists(): + raise FileNotFoundError(f"Pine file not found: {pine_file_path}") + + if pine_file_path.suffix != '.pine': + raise ValueError(f"This file format isn't supported: {pine_file_path.suffix}. Only .pine files can be compiled! ✨ Try using a .pine file instead.") + + # Determine output path + if output_file_path is None: + output_file_path = pine_file_path.with_suffix('.py') + + # Check if compilation is needed (unless forced) + if not force and not self.needs_compilation(pine_file_path, output_file_path): + return output_file_path + + # Read Pine Script content + try: + with open(pine_file_path, 'r', encoding='utf-8') as f: + script_content = f.read() + except IOError as e: + raise IOError(f"Error reading Pine file {pine_file_path}: {e}") + + # Compile via API + try: + response = self.api_client.compile_script_sync(script_content, strict=strict) + + if not response.success: + raise CompilationError( + f"Compilation failed: {response.error_message}", + status_code=response.status_code, + validation_errors=response.validation_errors + ) + + # Write compiled code to output file + output_file_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_file_path, 'w', encoding='utf-8') as f: + f.write(response.compiled_code) + + # Update hash after successful compilation + update_hash_after_compilation(pine_file_path) + + return output_file_path + + except (APIError, AuthError, RateLimitError, CompilationError) as e: + # Re-raise API-related errors as-is + raise + except Exception as e: + # Wrap unexpected errors + raise APIError(f"Unexpected error during compilation: {e}") + + def needs_compilation(self, pine_file_path: Path, output_file_path: Path) -> bool: + """Check if a .pine file needs compilation using MD5 hash comparison. + + Args: + pine_file_path: Path to the .pine file + output_file_path: Path to the compiled .py file + + Returns: + True if compilation is needed, False otherwise + """ + return file_needs_compilation(pine_file_path, output_file_path) + + def compile_and_run( + self, + pine_file_path: Path, + script_args: Optional[list] = None, + force: bool = False, + strict: bool = False, + output_file_path: Optional[Path] = None + ) -> int: + """Compile a .pine file and run the resulting Python script. + + Args: + pine_file_path: Path to the .pine file + script_args: Arguments to pass to the compiled script + force: Force recompilation even if file hasn't changed + strict: Enable strict compilation mode + output_file_path: Optional output path (defaults to .py extension) + + Returns: + Exit code from the executed script + + Raises: + FileNotFoundError: If pine file doesn't exist + CompilationError: If compilation fails + APIError: If API request fails + """ + # Compile the file + compiled_file = self.compile_file( + pine_file_path=pine_file_path, + output_file_path=output_file_path, + force=force, + strict=strict + ) + + # Prepare command to run the compiled script + cmd = [sys.executable, str(compiled_file)] + if script_args: + cmd.extend(script_args) + + # Run the compiled script + try: + result = subprocess.run(cmd, check=False) + return result.returncode + except Exception as e: + raise RuntimeError(f"Error executing compiled script: {e}") + + def _calculate_file_hash(self, pine_file_path: Path) -> str: + """Calculate MD5 hash of file content. + + Args: + pine_file_path: Path to the .pine file + + Returns: + MD5 hash as hexadecimal string + """ + return calculate_file_md5(pine_file_path) + + def _get_stored_hash(self, pine_file_path: Path) -> Optional[str]: + """Get stored hash from .pine.hash file. + + Args: + pine_file_path: Path to the .pine file + + Returns: + Stored MD5 hash string or None if not found + """ + return get_stored_hash(pine_file_path) + + def _store_hash(self, pine_file_path: Path, hash_value: str) -> None: + """Store hash in .pine.hash file. + + Args: + pine_file_path: Path to the .pine file + hash_value: MD5 hash to store + """ + store_hash(pine_file_path, hash_value) + + +def create_compilation_service( + api_key: Optional[str] = None, + config_path: Optional[Path] = None +) -> CompilationService: + """Factory function to create a CompilationService instance. + + Args: + api_key: Optional API key override + config_path: Optional path to config file + + Returns: + Configured CompilationService instance + + Raises: + ValueError: If no valid configuration found + """ + if api_key: + # Create API client with provided key + api_client = PynesysAPIClient(api_key=api_key) + return CompilationService(api_client=api_client) + else: + # Load configuration from file + config = ConfigManager.load_config(config_path) + return CompilationService(config=config) \ No newline at end of file diff --git a/src/pynecore/utils/hash_utils.py b/src/pynecore/utils/hash_utils.py new file mode 100644 index 0000000..2c0948d --- /dev/null +++ b/src/pynecore/utils/hash_utils.py @@ -0,0 +1,143 @@ +"""Hash utilities for smart compilation of .pine files.""" + +import hashlib +from pathlib import Path +from typing import Optional + + +def calculate_file_md5(file_path: Path, chunk_size: int = 65536) -> str: + """Calculate MD5 hash of file content. + + Uses Python's built-in hashlib module for optimal performance. + MD5 is 2-3x faster than SHA256 for change detection (not security). + Memory efficient with chunk-based reading for large files. + + Args: + file_path: Path to the file to hash + chunk_size: Size of chunks to read (64KB default for optimal performance) + + Returns: + MD5 hash as hexadecimal string + + Raises: + FileNotFoundError: If file doesn't exist + IOError: If file cannot be read + """ + if not file_path.exists(): + raise FileNotFoundError(f"File not found: {file_path}") + + hash_md5 = hashlib.md5() + + try: + with open(file_path, 'rb') as f: + # Read file in chunks for memory efficiency + while chunk := f.read(chunk_size): + hash_md5.update(chunk) + except IOError as e: + raise IOError(f"Error reading file {file_path}: {e}") + + return hash_md5.hexdigest() + + +def get_hash_file_path(pine_file_path: Path) -> Path: + """Get the path for the hash file corresponding to a .pine file. + + Args: + pine_file_path: Path to the .pine file + + Returns: + Path to the corresponding .pine.hash file + """ + return pine_file_path.with_suffix(pine_file_path.suffix + '.hash') + + +def get_stored_hash(pine_file_path: Path) -> Optional[str]: + """Get stored hash from .pine.hash file. + + Args: + pine_file_path: Path to the .pine file + + Returns: + Stored MD5 hash string or None if hash file doesn't exist or is invalid + """ + hash_file_path = get_hash_file_path(pine_file_path) + + if not hash_file_path.exists(): + return None + + try: + with open(hash_file_path, 'r', encoding='utf-8') as f: + stored_hash = f.read().strip() + + # Validate hash format (32 character hexadecimal string) + if len(stored_hash) == 32 and all(c in '0123456789abcdef' for c in stored_hash.lower()): + return stored_hash + else: + return None + + except (IOError, UnicodeDecodeError): + return None + + +def store_hash(pine_file_path: Path, hash_value: str) -> None: + """Store hash in .pine.hash file. + + Args: + pine_file_path: Path to the .pine file + hash_value: MD5 hash to store + + Raises: + IOError: If hash file cannot be written + """ + hash_file_path = get_hash_file_path(pine_file_path) + + try: + # Ensure parent directory exists + hash_file_path.parent.mkdir(parents=True, exist_ok=True) + + with open(hash_file_path, 'w', encoding='utf-8') as f: + f.write(hash_value) + + except IOError as e: + raise IOError(f"Error writing hash file {hash_file_path}: {e}") + + +def file_needs_compilation(pine_file_path: Path, output_file_path: Path) -> bool: + """Check if a .pine file needs compilation based on MD5 hash comparison. + + Args: + pine_file_path: Path to the .pine file + output_file_path: Path to the compiled .py file + + Returns: + True if compilation is needed, False otherwise + """ + # If output file doesn't exist, compilation is needed + if not output_file_path.exists(): + return True + + # Calculate current hash of .pine file + try: + current_hash = calculate_file_md5(pine_file_path) + except (FileNotFoundError, IOError): + # If we can't read the .pine file, assume compilation is needed + return True + + # Get stored hash + stored_hash = get_stored_hash(pine_file_path) + + # If no stored hash or hashes don't match, compilation is needed + return stored_hash != current_hash + + +def update_hash_after_compilation(pine_file_path: Path) -> None: + """Update the hash file after successful compilation. + + Args: + pine_file_path: Path to the .pine file that was compiled + + Raises: + IOError: If hash cannot be calculated or stored + """ + current_hash = calculate_file_md5(pine_file_path) + store_hash(pine_file_path, current_hash) \ No newline at end of file From 057503281c77444d1119c05017aaa075f493d0f8 Mon Sep 17 00:00:00 2001 From: Mahadi Hassan Date: Thu, 31 Jul 2025 18:04:04 +0600 Subject: [PATCH 07/31] style(cli): improve rate limit error messages consistency Update error messages for rate limit exceeded cases to use consistent formatting and emojis across compile and run commands --- src/pynecore/cli/commands/compile.py | 4 ++-- src/pynecore/cli/commands/run.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/pynecore/cli/commands/compile.py b/src/pynecore/cli/commands/compile.py index aec74eb..fb898ba 100644 --- a/src/pynecore/cli/commands/compile.py +++ b/src/pynecore/cli/commands/compile.py @@ -139,9 +139,9 @@ def compile( raise typer.Exit(1) except RateLimitError as e: - console.print(f"[red]✗[/red] Rate limit exceeded: {str(e)}") + console.print("[red]🚦 Rate Limit Exceeded:[/red] You've hit your compilation limit") if e.retry_after: - console.print(f"[yellow]Please try again in {e.retry_after} seconds[/yellow]") + console.print(f"[yellow]⏰ Please try again in {e.retry_after} seconds[/yellow]") console.print("[yellow]💡 To increase your limits, consider upgrading your subscription at [blue][link=https://pynesys.io]https://pynesys.io[/link][/blue]") raise typer.Exit(1) diff --git a/src/pynecore/cli/commands/run.py b/src/pynecore/cli/commands/run.py index 5518052..4dc5a67 100644 --- a/src/pynecore/cli/commands/run.py +++ b/src/pynecore/cli/commands/run.py @@ -220,10 +220,10 @@ def run( raise Exit(1) except RateLimitError as e: - console.print(f"[red]Rate limit exceeded: {str(e)}[/red]") + console.print("[red]🚦 Rate Limit Exceeded:[/red] You've hit your compilation limit") if e.retry_after: - console.print(f"[yellow]⏰ Just a moment - please try again in {e.retry_after} seconds[/yellow]") - console.print("[yellow]🚀 Want more compilation? Consider upgrading your subscription at [blue][link=https://pynesys.io]https://pynesys.io[/link][/blue] for higher limits![/yellow]") + console.print(f"[yellow]⏰ Please try again in {e.retry_after} seconds[/yellow]") + console.print("[yellow]💡 To increase your limits, consider upgrading your subscription at [blue][link=https://pynesys.io]https://pynesys.io[/link][/blue]") raise Exit(1) except APIError as e: From eff37147049e203fcc535c324d33a0d881d05e7e Mon Sep 17 00:00:00 2001 From: Mahadi Hassan Date: Thu, 31 Jul 2025 18:14:58 +0600 Subject: [PATCH 08/31] refactor(cli): centralize api error handling in commands Move API error handling logic from compile.py and run.py into a new api_error_handler.py module. This improves maintainability by eliminating duplicate error handling code and provides consistent error messages across commands. --- src/pynecore/cli/commands/compile.py | 98 +------------- src/pynecore/cli/commands/run.py | 102 +------------- src/pynecore/cli/utils/api_error_handler.py | 141 ++++++++++++++++++++ 3 files changed, 145 insertions(+), 196 deletions(-) create mode 100644 src/pynecore/cli/utils/api_error_handler.py diff --git a/src/pynecore/cli/commands/compile.py b/src/pynecore/cli/commands/compile.py index fb898ba..15fa47d 100644 --- a/src/pynecore/cli/commands/compile.py +++ b/src/pynecore/cli/commands/compile.py @@ -8,6 +8,7 @@ from ..app import app, app_state from ...api import ConfigManager, APIError, AuthError, RateLimitError, CompilationError +from ..utils.api_error_handler import handle_api_errors from ...core.compiler import create_compilation_service from ...utils.file_utils import preserve_mtime @@ -104,7 +105,7 @@ def compile( return # Compile script - try: + with handle_api_errors(console): with Progress( SpinnerColumn(), TextColumn("[progress.description]{task.description}"), @@ -125,101 +126,6 @@ def compile( console.print(f"[green]Compilation successful![/green] Your Pine Script is now ready: [cyan]{compiled_path}[/cyan]") - except CompilationError as e: - console.print(f"[red]❌ Oops! Compilation encountered an issue:[/red] {str(e)}") - if e.validation_errors: - console.print("[red]Validation errors:[/red]") - for error in e.validation_errors: - console.print(f" [red]• {error}[/red]") - raise typer.Exit(1) - - except AuthError as e: - console.print(f"[red]🔐 Authentication issue:[/red] {str(e)}") - console.print("[yellow]🚀 Quick fix:[/yellow] Run [cyan]'pyne api configure'[/cyan] to set up your API key and get back on track!") - raise typer.Exit(1) - - except RateLimitError as e: - console.print("[red]🚦 Rate Limit Exceeded:[/red] You've hit your compilation limit") - if e.retry_after: - console.print(f"[yellow]⏰ Please try again in {e.retry_after} seconds[/yellow]") - console.print("[yellow]💡 To increase your limits, consider upgrading your subscription at [blue][link=https://pynesys.io]https://pynesys.io[/link][/blue]") - raise typer.Exit(1) - - except APIError as e: - error_msg = str(e).lower() - - # Handle specific API error scenarios based on HTTP status codes - if "400" in error_msg or "bad request" in error_msg: - if "compilation fails" in error_msg or "script is too large" in error_msg: - console.print("[red]📝 Script Issue:[/red] Your Pine Script couldn't be compiled") - console.print("[yellow]💡 Common fixes:[/yellow]") - console.print(" • Check if your script is too large (try breaking it into smaller parts)") - console.print(" • Verify your Pine Script syntax is correct") - console.print(" • Make sure you're using Pine Script v6 syntax") - else: - console.print(f"[red]⚠️ Request Error:[/red] {str(e)}") - console.print("[yellow]💡 This usually means there's an issue with the request format[/yellow]") - - elif "401" in error_msg or "authentication" in error_msg or "no permission" in error_msg: - console.print("[red]🔐 Authentication Failed:[/red] Your API credentials aren't working") - console.print("[yellow]🚀 Quick fixes:[/yellow]") - console.print(" • Check if your API key is valid and active") - console.print(" • Verify your token type is allowed for compilation") - console.print("[blue]🔑 Get a new API key at [link=https://pynesys.io]https://pynesys.io[/link][/blue]") - console.print("[blue]⚙️ Then run [cyan]'pyne api configure'[/cyan] to update your configuration[/blue]") - - elif "404" in error_msg or "not found" in error_msg: - console.print("[red]🔍 Not Found:[/red] The API endpoint or user wasn't found") - console.print("[yellow]💡 This might indicate:[/yellow]") - console.print(" • Your account may not exist or be accessible") - console.print(" • There might be a temporary service issue") - - elif "422" in error_msg or "validation error" in error_msg: - console.print("[red]📋 Validation Error:[/red] Your request data has validation issues") - console.print("[yellow]💡 Common causes:[/yellow]") - console.print(" • Invalid Pine Script syntax or structure") - console.print(" • Missing required parameters") - console.print(" • Incorrect data format") - console.print(f"[dim]Details: {str(e)}[/dim]") - - elif "429" in error_msg or "rate limit" in error_msg or "too many requests" in error_msg: - console.print("[red]🚦 Rate Limit Exceeded:[/red] You've hit your compilation limit") - console.print("[yellow]⏰ What you can do:[/yellow]") - console.print(" • Wait a bit before trying again") - console.print(" • Consider upgrading your plan for higher limits") - console.print("[blue]💎 Upgrade at [link=https://pynesys.io/pricing]https://pynesys.io/pricing[/link][/blue]") - - elif "500" in error_msg or "server" in error_msg or "internal" in error_msg: - console.print("[red]🔧 Server Error:[/red] Something went wrong on our end") - console.print("[yellow]😅 Don't worry, it's not you![/yellow]") - console.print(" • This is a temporary server issue") - console.print(" • Please try again in a few moments") - - elif "unsupported pinescript version" in error_msg: - console.print("[red]📌 Version Issue:[/red] Your Pine Script version isn't supported") - if "version 5" in error_msg: - console.print("[yellow]🔄 Pine Script v5 → v6 Migration:[/yellow]") - console.print(" • Update your script to Pine Script version 6") - console.print(" • Most v5 scripts need minimal changes") - console.print("[blue]📖 Migration guide: [link=https://www.tradingview.com/pine-script-docs/en/v6/migration_guides/v5_to_v6_migration_guide.html]Pine Script v5→v6 Guide[/link][/blue]") - else: - console.print("[yellow]💡 Only Pine Script version 6 is currently supported[/yellow]") - - elif "api key" in error_msg: - console.print("[red]🔑 API Key Issue:[/red] There's a problem with your API key") - console.print("[blue]🔑 Get your API key at [link=https://pynesys.io]https://pynesys.io[/link][/blue]") - console.print("[blue]⚙️ Then run [cyan]'pyne api configure'[/cyan] to set up your configuration[/blue]") - - else: - # Generic API error fallback - console.print(f"[red]🌐 API Error:[/red] {str(e)}") - console.print("[yellow]💡 If this persists, please check:[/yellow]") - console.print(" • Your internet connection") - console.print(" • API service status") - console.print("[blue]📞 Need help? [link=https://pynesys.io/support]Contact Support[/link][/blue]") - - raise typer.Exit(1) - except ValueError as e: error_msg = str(e) if "No configuration file found" in error_msg or "Configuration file not found" in error_msg: diff --git a/src/pynecore/cli/commands/run.py b/src/pynecore/cli/commands/run.py index 4dc5a67..14e5a6b 100644 --- a/src/pynecore/cli/commands/run.py +++ b/src/pynecore/cli/commands/run.py @@ -22,6 +22,7 @@ from ...core.compiler import create_compilation_service from ...api.exceptions import APIError, AuthError, RateLimitError, CompilationError from ...api.config import ConfigManager +from ...cli.utils.api_error_handler import handle_api_errors __all__ = [] @@ -185,7 +186,7 @@ def run( # Check if compilation is needed if compilation_service.needs_compilation(script, compiled_file) or force: - try: + with handle_api_errors(console): with Progress( SpinnerColumn(), TextColumn("[progress.description]{task.description}"), @@ -204,105 +205,6 @@ def run( console.print(f"[green]Compilation successful![/green] Ready to run: [cyan]{compiled_path}[/cyan]") compiled_file = compiled_path - except CompilationError as e: - console.print(f"[red]❌ Pine Script compilation encountered an issue:[/red] {str(e)}") - if e.validation_errors: - console.print("[red]Validation errors:[/red]") - for error in e.validation_errors: - console.print(f" [red]• {error}[/red]") - raise Exit(1) - - except AuthError as e: - console.print(f"[red]🔐 Authentication issue:[/red] {str(e)}") - console.print("[yellow]🚀 Quick fix:[/yellow] Run [cyan]'pyne api configure'[/cyan] to set up your API key and get back on track!") - console.print("[blue]Visit https://pynesys.io to get your API key[/blue]") - console.print("[blue]Run 'pyne api configure' to set up your configuration[/blue]") - raise Exit(1) - - except RateLimitError as e: - console.print("[red]🚦 Rate Limit Exceeded:[/red] You've hit your compilation limit") - if e.retry_after: - console.print(f"[yellow]⏰ Please try again in {e.retry_after} seconds[/yellow]") - console.print("[yellow]💡 To increase your limits, consider upgrading your subscription at [blue][link=https://pynesys.io]https://pynesys.io[/link][/blue]") - raise Exit(1) - - except APIError as e: - error_msg = str(e).lower() - - # Handle specific API error scenarios based on HTTP status codes - if "400" in error_msg or "bad request" in error_msg: - if "compilation fails" in error_msg or "script is too large" in error_msg: - console.print("[red]📝 Script Issue:[/red] Your Pine Script couldn't be compiled") - console.print("[yellow]💡 Common fixes:[/yellow]") - console.print(" • Check if your script is too large (try breaking it into smaller parts)") - console.print(" • Verify your Pine Script syntax is correct") - console.print(" • Make sure you're using Pine Script v6 syntax") - else: - console.print(f"[red]⚠️ Request Error:[/red] {str(e)}") - console.print("[yellow]💡 This usually means there's an issue with the request format[/yellow]") - - elif "401" in error_msg or "authentication" in error_msg or "no permission" in error_msg: - console.print("[red]🔐 Authentication Failed:[/red] Your API credentials aren't working") - console.print("[yellow]🚀 Quick fixes:[/yellow]") - console.print(" • Check if your API key is valid and active") - console.print(" • Verify your token type is allowed for compilation") - console.print("[blue]🔑 Get a new API key at [link=https://pynesys.io]https://pynesys.io[/link][/blue]") - console.print("[blue]⚙️ Then run [cyan]'pyne api configure'[/cyan] to update your configuration[/blue]") - - elif "404" in error_msg or "not found" in error_msg: - console.print("[red]🔍 Not Found:[/red] The API endpoint or user wasn't found") - console.print("[yellow]💡 This might indicate:[/yellow]") - console.print(" • Your account may not exist or be accessible") - console.print(" • There might be a temporary service issue") - console.print("[blue]📞 Contact support if this persists: [link=https://pynesys.io/support]https://pynesys.io/support[/link][/blue]") - - elif "422" in error_msg or "validation error" in error_msg: - console.print("[red]📋 Validation Error:[/red] Your request data has validation issues") - console.print("[yellow]💡 Common causes:[/yellow]") - console.print(" • Invalid Pine Script syntax or structure") - console.print(" • Missing required parameters") - console.print(" • Incorrect data format") - console.print(f"[dim]Details: {str(e)}[/dim]") - - elif "429" in error_msg or "rate limit" in error_msg or "too many requests" in error_msg: - console.print("[red]🚦 Rate Limit Exceeded:[/red] You've hit your compilation limit") - console.print("[yellow]⏰ What you can do:[/yellow]") - console.print(" • Wait a bit before trying again") - console.print(" • Consider upgrading your plan for higher limits") - console.print("[blue]💎 Upgrade at [link=https://pynesys.io/pricing]https://pynesys.io/pricing[/link][/blue]") - - elif "500" in error_msg or "server" in error_msg or "internal" in error_msg: - console.print("[red]🔧 Server Error:[/red] Something went wrong on our end") - console.print("[yellow]😅 Don't worry, it's not you![/yellow]") - console.print(" • This is a temporary server issue") - console.print(" • Please try again in a few moments") - console.print("[blue]📊 Check service status: [link=https://status.pynesys.io]https://status.pynesys.io[/link][/blue]") - - elif "unsupported pinescript version" in error_msg: - console.print("[red]📌 Version Issue:[/red] Your Pine Script version isn't supported") - if "version 5" in error_msg: - console.print("[yellow]🔄 Pine Script v5 → v6 Migration:[/yellow]") - console.print(" • Update your script to Pine Script version 6") - console.print(" • Most v5 scripts need minimal changes") - console.print("[blue]📖 Migration guide: [link=https://www.tradingview.com/pine-script-docs/en/v6/migration_guides/v5_to_v6_migration_guide.html]Pine Script v5→v6 Guide[/link][/blue]") - else: - console.print("[yellow]💡 Only Pine Script version 6 is currently supported[/yellow]") - - elif "api key" in error_msg: - console.print("[red]🔑 API Key Issue:[/red] There's a problem with your API key") - console.print("[blue]🔑 Get your API key at [link=https://pynesys.io]https://pynesys.io[/link][/blue]") - console.print("[blue]⚙️ Then run [cyan]'pyne api configure'[/cyan] to set up your configuration[/blue]") - - else: - # Generic API error fallback - console.print(f"[red]🌐 API Error:[/red] {str(e)}") - console.print("[yellow]💡 If this persists, please check:[/yellow]") - console.print(" • Your internet connection") - console.print(" • API service status") - console.print("[blue]📞 Need help? [link=https://pynesys.io/support]Contact Support[/link][/blue]") - - raise Exit(1) - else: console.print(f"[green]⚡ Using cached version:[/green] [cyan]{compiled_file}[/cyan]") console.print("[dim]Use --force to recompile[/dim]") diff --git a/src/pynecore/cli/utils/api_error_handler.py b/src/pynecore/cli/utils/api_error_handler.py new file mode 100644 index 0000000..c3e6a99 --- /dev/null +++ b/src/pynecore/cli/utils/api_error_handler.py @@ -0,0 +1,141 @@ +"""Centralized API error handling utilities for CLI commands.""" + +from rich.console import Console +from typer import Exit + +from pynecore.api.exceptions import APIError, AuthError, RateLimitError, CompilationError + + +def handle_api_errors(console: Console): + """Context manager for handling API errors with user-friendly messages. + + Usage: + with handle_api_errors(console): + # API operations that might raise exceptions + pass + """ + return APIErrorHandler(console) + + +class APIErrorHandler: + """Context manager that provides centralized API error handling.""" + + def __init__(self, console: Console): + self.console = console + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + if exc_type is None: + return False + + if exc_type == CompilationError: + self._handle_compilation_error(exc_value) + elif exc_type == AuthError: + self._handle_auth_error(exc_value) + elif exc_type == RateLimitError: + self._handle_rate_limit_error(exc_value) + elif exc_type == APIError: + self._handle_api_error(exc_value) + else: + return False # Let other exceptions propagate + + raise Exit(1) + + def _handle_compilation_error(self, e: CompilationError): + """Handle compilation-specific errors.""" + self.console.print(f"[red]❌ Oops! Compilation encountered an issue:[/red] {str(e)}") + if e.validation_errors: + self.console.print("[red]Validation errors:[/red]") + for error in e.validation_errors: + self.console.print(f" [red]• {error}[/red]") + + def _handle_auth_error(self, e: AuthError): + """Handle authentication errors.""" + self.console.print(f"[red]🔐 Authentication issue:[/red] {str(e)}") + self.console.print("[yellow]🚀 Quick fix:[/yellow] Run [cyan]'pyne api configure'[/cyan] to set up your API key and get back on track!") + + def _handle_rate_limit_error(self, e: RateLimitError): + """Handle rate limit errors.""" + self.console.print("[red]🚦 Rate Limit Exceeded:[/red] You've hit your compilation limit") + if e.retry_after: + self.console.print(f"[yellow]⏰ Please try again in {e.retry_after} seconds[/yellow]") + self.console.print("[yellow]💡 To increase your limits, consider upgrading your subscription at [blue][link=https://pynesys.io]https://pynesys.io[/link][/blue]") + self.console.print("[blue]💎 Upgrade at [link=https://pynesys.io/pricing]https://pynesys.io/pricing[/link][/blue]") + + def _handle_api_error(self, e: APIError): + """Handle general API errors with specific status code handling.""" + error_msg = str(e).lower() + + # Handle specific API error scenarios based on HTTP status codes + if "400" in error_msg or "bad request" in error_msg: + if "compilation fails" in error_msg or "script is too large" in error_msg: + self.console.print("[red]📝 Script Issue:[/red] Your Pine Script couldn't be compiled") + self.console.print("[yellow]💡 Common fixes:[/yellow]") + self.console.print(" • Check if your script is too large (try breaking it into smaller parts)") + self.console.print(" • Verify your Pine Script syntax is correct") + self.console.print(" • Make sure you're using Pine Script v6 syntax") + else: + self.console.print(f"[red]⚠️ Request Error:[/red] {str(e)}") + self.console.print("[yellow]💡 This usually means there's an issue with the request format[/yellow]") + + elif "401" in error_msg or "authentication" in error_msg or "no permission" in error_msg: + self.console.print("[red]🔐 Authentication Failed:[/red] Your API credentials aren't working") + self.console.print("[yellow]🚀 Quick fixes:[/yellow]") + self.console.print(" • Check if your API key is valid and active") + self.console.print(" • Verify your token type is allowed for compilation") + self.console.print("[blue]🔑 Get a new API key at [link=https://pynesys.io]https://pynesys.io[/link][/blue]") + self.console.print("[blue]⚙️ Then run [cyan]'pyne api configure'[/cyan] to update your configuration[/blue]") + + elif "404" in error_msg or "not found" in error_msg: + self.console.print("[red]🔍 Not Found:[/red] The API endpoint or user wasn't found") + self.console.print("[yellow]💡 This might indicate:[/yellow]") + self.console.print(" • Your account may not exist or be accessible") + self.console.print(" • There might be a temporary service issue") + self.console.print("[blue]📞 Contact support if this persists: [link=https://pynesys.io/support]https://pynesys.io/support[/link][/blue]") + + elif "422" in error_msg or "validation error" in error_msg: + self.console.print("[red]📋 Validation Error:[/red] Your request data has validation issues") + self.console.print("[yellow]💡 Common causes:[/yellow]") + self.console.print(" • Invalid Pine Script syntax or structure") + self.console.print(" • Missing required parameters") + self.console.print(" • Incorrect data format") + self.console.print(f"[dim]Details: {str(e)}[/dim]") + + elif "429" in error_msg or "rate limit" in error_msg or "too many requests" in error_msg: + self.console.print("[red]🚦 Rate Limit Exceeded:[/red] You've hit your compilation limit") + self.console.print("[yellow]⏰ What you can do:[/yellow]") + self.console.print(" • Wait a bit before trying again") + self.console.print(" • Consider upgrading your plan for higher limits") + self.console.print("[blue]💎 Upgrade at [link=https://pynesys.io/pricing]https://pynesys.io/pricing[/link][/blue]") + + elif "500" in error_msg or "server" in error_msg or "internal" in error_msg: + self.console.print("[red]🔧 Server Error:[/red] Something went wrong on our end") + self.console.print("[yellow]😅 Don't worry, it's not you![/yellow]") + self.console.print(" • This is a temporary server issue") + self.console.print(" • Please try again in a few moments") + self.console.print("[blue]📊 Check service status: [link=https://status.pynesys.io]https://status.pynesys.io[/link][/blue]") + + elif "unsupported pinescript version" in error_msg: + self.console.print("[red]📌 Version Issue:[/red] Your Pine Script version isn't supported") + if "version 5" in error_msg: + self.console.print("[yellow]🔄 Pine Script v5 → v6 Migration:[/yellow]") + self.console.print(" • Update your script to Pine Script version 6") + self.console.print(" • Most v5 scripts need minimal changes") + self.console.print("[blue]📖 Migration guide: [link=https://www.tradingview.com/pine-script-docs/en/v6/migration_guides/v5_to_v6_migration_guide.html]Pine Script v5→v6 Guide[/link][/blue]") + else: + self.console.print("[yellow]💡 Only Pine Script version 6 is currently supported[/yellow]") + + elif "api key" in error_msg: + self.console.print("[red]🔑 API Key Issue:[/red] There's a problem with your API key") + self.console.print("[blue]🔑 Get your API key at [link=https://pynesys.io]https://pynesys.io[/link][/blue]") + self.console.print("[blue]⚙️ Then run [cyan]'pyne api configure'[/cyan] to set up your configuration[/blue]") + + else: + # Generic API error fallback + self.console.print(f"[red]🌐 API Error:[/red] {str(e)}") + self.console.print("[yellow]💡 If this persists, please check:[/yellow]") + self.console.print(" • Your internet connection") + self.console.print(" • API service status") + self.console.print("[blue]📞 Need help? [link=https://pynesys.io/support]Contact Support[/link][/blue]") \ No newline at end of file From b81534736fb6513b67b0b559b97cd52bbd36424a Mon Sep 17 00:00:00 2001 From: Mahadi Hassan Date: Thu, 31 Jul 2025 19:07:58 +0600 Subject: [PATCH 09/31] refactor(compiler): replace md5 hash with mtime for compilation check --- src/pynecore/core/compiler.py | 44 +-------- src/pynecore/utils/hash_utils.py | 143 ------------------------------ src/pynecore/utils/mtime_utils.py | 35 ++++++++ 3 files changed, 39 insertions(+), 183 deletions(-) delete mode 100644 src/pynecore/utils/hash_utils.py create mode 100644 src/pynecore/utils/mtime_utils.py diff --git a/src/pynecore/core/compiler.py b/src/pynecore/core/compiler.py index b3a94b4..ab68e32 100644 --- a/src/pynecore/core/compiler.py +++ b/src/pynecore/core/compiler.py @@ -12,13 +12,7 @@ from ..api.client import PynesysAPIClient from ..api.config import APIConfig, ConfigManager from ..api.exceptions import APIError, AuthError, RateLimitError, CompilationError -from ..utils.hash_utils import ( - file_needs_compilation, - update_hash_after_compilation, - calculate_file_md5, - get_stored_hash, - store_hash -) +from ..utils.mtime_utils import file_needs_compilation class CompilationService: @@ -108,8 +102,7 @@ def compile_file( with open(output_file_path, 'w', encoding='utf-8') as f: f.write(response.compiled_code) - # Update hash after successful compilation - update_hash_after_compilation(pine_file_path) + # No need to update tracking info with mtime approach return output_file_path @@ -121,7 +114,7 @@ def compile_file( raise APIError(f"Unexpected error during compilation: {e}") def needs_compilation(self, pine_file_path: Path, output_file_path: Path) -> bool: - """Check if a .pine file needs compilation using MD5 hash comparison. + """Check if a .pine file needs compilation using modification time comparison. Args: pine_file_path: Path to the .pine file @@ -177,36 +170,7 @@ def compile_and_run( except Exception as e: raise RuntimeError(f"Error executing compiled script: {e}") - def _calculate_file_hash(self, pine_file_path: Path) -> str: - """Calculate MD5 hash of file content. - - Args: - pine_file_path: Path to the .pine file - - Returns: - MD5 hash as hexadecimal string - """ - return calculate_file_md5(pine_file_path) - - def _get_stored_hash(self, pine_file_path: Path) -> Optional[str]: - """Get stored hash from .pine.hash file. - - Args: - pine_file_path: Path to the .pine file - - Returns: - Stored MD5 hash string or None if not found - """ - return get_stored_hash(pine_file_path) - - def _store_hash(self, pine_file_path: Path, hash_value: str) -> None: - """Store hash in .pine.hash file. - - Args: - pine_file_path: Path to the .pine file - hash_value: MD5 hash to store - """ - store_hash(pine_file_path, hash_value) + def create_compilation_service( diff --git a/src/pynecore/utils/hash_utils.py b/src/pynecore/utils/hash_utils.py deleted file mode 100644 index 2c0948d..0000000 --- a/src/pynecore/utils/hash_utils.py +++ /dev/null @@ -1,143 +0,0 @@ -"""Hash utilities for smart compilation of .pine files.""" - -import hashlib -from pathlib import Path -from typing import Optional - - -def calculate_file_md5(file_path: Path, chunk_size: int = 65536) -> str: - """Calculate MD5 hash of file content. - - Uses Python's built-in hashlib module for optimal performance. - MD5 is 2-3x faster than SHA256 for change detection (not security). - Memory efficient with chunk-based reading for large files. - - Args: - file_path: Path to the file to hash - chunk_size: Size of chunks to read (64KB default for optimal performance) - - Returns: - MD5 hash as hexadecimal string - - Raises: - FileNotFoundError: If file doesn't exist - IOError: If file cannot be read - """ - if not file_path.exists(): - raise FileNotFoundError(f"File not found: {file_path}") - - hash_md5 = hashlib.md5() - - try: - with open(file_path, 'rb') as f: - # Read file in chunks for memory efficiency - while chunk := f.read(chunk_size): - hash_md5.update(chunk) - except IOError as e: - raise IOError(f"Error reading file {file_path}: {e}") - - return hash_md5.hexdigest() - - -def get_hash_file_path(pine_file_path: Path) -> Path: - """Get the path for the hash file corresponding to a .pine file. - - Args: - pine_file_path: Path to the .pine file - - Returns: - Path to the corresponding .pine.hash file - """ - return pine_file_path.with_suffix(pine_file_path.suffix + '.hash') - - -def get_stored_hash(pine_file_path: Path) -> Optional[str]: - """Get stored hash from .pine.hash file. - - Args: - pine_file_path: Path to the .pine file - - Returns: - Stored MD5 hash string or None if hash file doesn't exist or is invalid - """ - hash_file_path = get_hash_file_path(pine_file_path) - - if not hash_file_path.exists(): - return None - - try: - with open(hash_file_path, 'r', encoding='utf-8') as f: - stored_hash = f.read().strip() - - # Validate hash format (32 character hexadecimal string) - if len(stored_hash) == 32 and all(c in '0123456789abcdef' for c in stored_hash.lower()): - return stored_hash - else: - return None - - except (IOError, UnicodeDecodeError): - return None - - -def store_hash(pine_file_path: Path, hash_value: str) -> None: - """Store hash in .pine.hash file. - - Args: - pine_file_path: Path to the .pine file - hash_value: MD5 hash to store - - Raises: - IOError: If hash file cannot be written - """ - hash_file_path = get_hash_file_path(pine_file_path) - - try: - # Ensure parent directory exists - hash_file_path.parent.mkdir(parents=True, exist_ok=True) - - with open(hash_file_path, 'w', encoding='utf-8') as f: - f.write(hash_value) - - except IOError as e: - raise IOError(f"Error writing hash file {hash_file_path}: {e}") - - -def file_needs_compilation(pine_file_path: Path, output_file_path: Path) -> bool: - """Check if a .pine file needs compilation based on MD5 hash comparison. - - Args: - pine_file_path: Path to the .pine file - output_file_path: Path to the compiled .py file - - Returns: - True if compilation is needed, False otherwise - """ - # If output file doesn't exist, compilation is needed - if not output_file_path.exists(): - return True - - # Calculate current hash of .pine file - try: - current_hash = calculate_file_md5(pine_file_path) - except (FileNotFoundError, IOError): - # If we can't read the .pine file, assume compilation is needed - return True - - # Get stored hash - stored_hash = get_stored_hash(pine_file_path) - - # If no stored hash or hashes don't match, compilation is needed - return stored_hash != current_hash - - -def update_hash_after_compilation(pine_file_path: Path) -> None: - """Update the hash file after successful compilation. - - Args: - pine_file_path: Path to the .pine file that was compiled - - Raises: - IOError: If hash cannot be calculated or stored - """ - current_hash = calculate_file_md5(pine_file_path) - store_hash(pine_file_path, current_hash) \ No newline at end of file diff --git a/src/pynecore/utils/mtime_utils.py b/src/pynecore/utils/mtime_utils.py new file mode 100644 index 0000000..afd3982 --- /dev/null +++ b/src/pynecore/utils/mtime_utils.py @@ -0,0 +1,35 @@ +"""File modification time utilities for smart compilation of .pine files.""" + +import os +from pathlib import Path + + +def file_needs_compilation(pine_file_path: Path, output_file_path: Path) -> bool: + """Check if a .pine file needs compilation based on modification time. + + Args: + pine_file_path: Path to the .pine file + output_file_path: Path to the compiled .py file + + Returns: + True if compilation is needed, False otherwise + """ + # If output file doesn't exist, compilation is needed + if not output_file_path.exists(): + return True + + # If source file doesn't exist, assume compilation is needed + if not pine_file_path.exists(): + return True + + try: + # Get modification times + source_mtime = os.path.getmtime(pine_file_path) + output_mtime = os.path.getmtime(output_file_path) + + # If source is newer than output, compilation is needed + return source_mtime > output_mtime + + except OSError: + # If we can't get modification times, assume compilation is needed + return True \ No newline at end of file From 27a85b02c9746552ccc4c66583823139f6cdc135 Mon Sep 17 00:00:00 2001 From: Adam Wallner Date: Fri, 1 Aug 2025 21:36:40 +0200 Subject: [PATCH 10/31] refactor(api): improve code quality and modernize type hints - Update type hints to use modern Python syntax (PEP 585/604) - Remove unused imports and optimize import statements - Improve code formatting and docstring consistency - Add static method decorators where appropriate - Enhance error handling and exception management - Update .gitignore with development environment entries - Maintain backward compatibility while modernizing codebase --- .gitignore | 17 ++ src/pynecore/api/client.py | 220 ++++++++++---------- src/pynecore/api/config.py | 130 ++++++------ src/pynecore/api/exceptions.py | 21 +- src/pynecore/api/file_manager.py | 295 +++++++++++++-------------- src/pynecore/api/models.py | 36 ++-- src/pynecore/cli/commands/compile.py | 108 +++++----- src/pynecore/cli/commands/run.py | 40 ++-- src/pynecore/core/compiler.py | 181 ++++++++-------- 9 files changed, 504 insertions(+), 544 deletions(-) diff --git a/.gitignore b/.gitignore index 4a6e11c..f46fc36 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ __pycache__/ # Environments .env venv/ +.venv/ # PyPI configuration file .pypirc @@ -23,3 +24,19 @@ dist/ # macOS # .DS_Store + +# +# Editors + AI +# + +.idea/ +.vscode/ +.cursor/ +.windsurf/ +.claude/ +.mcp.json +CLAUDE.md +MEMORY.md + +# Others +uv.lock diff --git a/src/pynecore/api/client.py b/src/pynecore/api/client.py index c446f13..2ef35e3 100644 --- a/src/pynecore/api/client.py +++ b/src/pynecore/api/client.py @@ -1,7 +1,7 @@ """PyneCore API client for PyneSys compiler service.""" import asyncio -from typing import Optional, Dict, Any, TYPE_CHECKING +from typing import Optional, TYPE_CHECKING from datetime import datetime import json @@ -26,63 +26,61 @@ class PynesysAPIClient: """Client for interacting with PyneSys API.""" - + def __init__( - self, - api_key: str, - base_url: str = "https://api.pynesys.io", - timeout: int = 30, + self, + api_key: str, + base_url: str = "https://api.pynesys.io", + timeout: int = 30, ): - """Initialize the API client. - - Args: - api_key: PyneSys API key - base_url: Base URL for the API - timeout: Request timeout in seconds + """ + Initialize the API client. + + :param api_key: PyneSys API key + :param base_url: Base URL for the API + :param timeout: Request timeout in seconds """ if httpx is None: raise ImportError( "httpx is required for API functionality. " "Install it with: pip install httpx" ) - + if not api_key or not api_key.strip(): raise ValueError("API key is required") - + self.api_key = api_key self.base_url = base_url.rstrip("/") self.timeout = timeout self._client: Optional["httpx.AsyncClient"] = None - + def compile_script_sync(self, script: str, strict: bool = False) -> CompileResponse: - """Synchronous wrapper for compile_script. - - Args: - script: Pine Script code to compile - strict: Enable strict compilation mode - - Returns: - CompileResponse with compilation results + """ + Synchronous wrapper for compile_script. + + :param script: Pine Script code to compile + :param strict: Enable strict compilation mode + :return: CompileResponse with compilation results """ return asyncio.run(self.compile_script(script, strict)) - + def verify_token_sync(self) -> TokenValidationResponse: - """Synchronous wrapper for verify_token. - - Returns: - TokenValidationResponse with token validation results + """ + Synchronous wrapper for verify_token. + + :return: TokenValidationResponse with token validation results """ return asyncio.run(self.verify_token()) - + async def __aenter__(self): """Async context manager entry.""" await self._ensure_client() return self - + async def __aexit__(self, exc_type, exc_val, exc_tb): """Async context manager exit.""" await self.close() - + async def _ensure_client(self): """Ensure HTTP client is initialized.""" if self._client is None and httpx is not None: @@ -93,35 +91,33 @@ async def _ensure_client(self): "User-Agent": "PyneCore-API-Client", }, ) - + async def close(self): """Close the HTTP client.""" if self._client: await self._client.aclose() self._client = None - + async def verify_token(self) -> TokenValidationResponse: - """Verify API token validity. - - Returns: - TokenValidationResponse with validation details - - Raises: - AuthError: If token is invalid - NetworkError: If network request fails - APIError: For other API errors + """ + Verify API token validity. + + :return: TokenValidationResponse with validation details + :raises AuthError: If token is invalid + :raises NetworkError: If network request fails + :raises APIError: For other API errors """ await self._ensure_client() - + try: if self._client is None: raise APIError("HTTP client not initialized") - + response = await self._client.get( f"{self.base_url}/auth/verify-token", params={"token": self.api_key} ) - + if response.status_code == 200: data = response.json() return TokenValidationResponse( @@ -138,7 +134,7 @@ async def verify_token(self) -> TokenValidationResponse: self._handle_api_error(response) # This should never be reached due to _handle_api_error raising raise APIError("Unexpected API response") - + except Exception as e: if httpx and isinstance(e, httpx.RequestError): raise NetworkError(f"Network error during token verification: {e}") @@ -146,46 +142,42 @@ async def verify_token(self) -> TokenValidationResponse: raise APIError(f"Unexpected error during token verification: {e}") else: raise - + async def compile_script( - self, - script: str, - strict: bool = False + self, + script: str, + strict: bool = False ) -> CompileResponse: - """Compile Pine Script to Python via API. - - Args: - script: Pine Script code to compile - strict: Whether to use strict compilation mode - - Returns: - CompileResponse with compiled code or error details - - Raises: - AuthError: If authentication fails - RateLimitError: If rate limit is exceeded - CompilationError: If compilation fails - NetworkError: If network request fails - APIError: For other API errors + """ + Compile Pine Script to Python via API. + + :param script: Pine Script code to compile + :param strict: Whether to use strict compilation mode + :return: CompileResponse with compiled code or error details + :raises AuthError: If authentication fails + :raises RateLimitError: If rate limit is exceeded + :raises CompilationError: If compilation fails + :raises NetworkError: If network request fails + :raises APIError: For other API errors """ await self._ensure_client() - + try: # Prepare form data data = { "script": script, "strict": str(strict).lower() } - + if self._client is None: raise APIError("HTTP client not initialized") - + response = await self._client.post( f"{self.base_url}/compiler/compile", data=data, headers={"Content-Type": "application/x-www-form-urlencoded"} ) - + if response.status_code == 200: # Success - return compiled code compiled_code = response.text @@ -197,7 +189,7 @@ async def compile_script( else: # Handle error responses return self._handle_compile_error(response) - + except Exception as e: if httpx and isinstance(e, httpx.RequestError): raise NetworkError(f"Network error during compilation: {e}") @@ -205,24 +197,23 @@ async def compile_script( raise APIError(f"Unexpected error during compilation: {e}") else: raise - - def _handle_api_error(self, response: "httpx.Response") -> None: - """Handle API error responses. - - Args: - response: HTTP response object - - Raises: - Appropriate exception based on status code + + @staticmethod + def _handle_api_error(response: "httpx.Response") -> None: + """ + Handle API error responses. + + :param response: HTTP response object + :raises: Appropriate exception based on status code """ status_code = response.status_code - + try: error_data = response.json() message = error_data.get("message", response.text) except (json.JSONDecodeError, ValueError): message = response.text or f"HTTP {status_code} error" - + if status_code == 401: raise AuthError(message, status_code=status_code) elif status_code == 429: @@ -236,27 +227,23 @@ def _handle_api_error(self, response: "httpx.Response") -> None: raise ServerError(message, status_code=status_code) else: raise APIError(message, status_code=status_code) - + def _handle_compile_error(self, response: "httpx.Response") -> CompileResponse: - """Handle compilation error responses. - - Args: - response: HTTP response object - - Returns: - CompileResponse with error details - - Raises: - CompilationError: For compilation-related errors (422) - Other exceptions: For authentication, rate limiting, etc. + """ + Handle compilation error responses. + + :param response: HTTP response object + :return: CompileResponse with error details + :raises CompilationError: For compilation-related errors (422) + :raises: Other exceptions for authentication, rate limiting, etc. """ status_code = response.status_code - + try: error_data = response.json() except (json.JSONDecodeError, ValueError): error_data = {} - + # Extract error message if "detail" in error_data and isinstance(error_data["detail"], list): # Validation error format (422) @@ -265,14 +252,14 @@ def _handle_compile_error(self, response: "httpx.Response") -> CompileResponse: else: validation_errors = None error_message = error_data.get("message", response.text or f"HTTP {status_code} error") - + # For compilation errors (422), raise CompilationError if status_code == 422: raise CompilationError(error_message, status_code=status_code, validation_errors=validation_errors) - + # For other errors, use the general API error handler self._handle_api_error(response) - + # This should never be reached return CompileResponse( success=False, @@ -281,19 +268,18 @@ def _handle_compile_error(self, response: "httpx.Response") -> CompileResponse: status_code=status_code, raw_response=error_data ) - - def _parse_datetime(self, dt_str: Optional[str]) -> Optional[datetime]: - """Parse datetime string from API response. - - Args: - dt_str: Datetime string from API - - Returns: - Parsed datetime object or None + + @staticmethod + def _parse_datetime(dt_str: Optional[str]) -> Optional[datetime]: + """ + Parse datetime string from API response. + + :param dt_str: Datetime string from API + :return: Parsed datetime object or None """ if not dt_str: return None - + try: # Try common datetime formats for fmt in [ @@ -306,31 +292,31 @@ def _parse_datetime(self, dt_str: Optional[str]) -> Optional[datetime]: return datetime.strptime(dt_str, fmt) except ValueError: continue - + # If no format matches, return None return None - - except Exception: + + except Exception: # noqa return None # Synchronous wrapper for convenience class SyncPynesysAPIClient: """Synchronous wrapper for PynesysAPIClient.""" - + def __init__(self, *args, **kwargs): self._async_client = PynesysAPIClient(*args, **kwargs) - + def verify_token(self) -> TokenValidationResponse: """Synchronous token verification.""" return asyncio.run(self._async_client.verify_token()) - + def compile_script(self, script: str, strict: bool = False) -> CompileResponse: """Synchronous script compilation.""" return asyncio.run(self._async_client.compile_script(script, strict)) - + def __enter__(self): return self - + def __exit__(self, exc_type, exc_val, exc_tb): - asyncio.run(self._async_client.close()) \ No newline at end of file + asyncio.run(self._async_client.close()) diff --git a/src/pynecore/api/config.py b/src/pynecore/api/config.py index 7f961be..a48ad89 100644 --- a/src/pynecore/api/config.py +++ b/src/pynecore/api/config.py @@ -1,9 +1,10 @@ -"""Configuration management for PyneSys API.""" +""" +Configuration management for PyneSys API. +""" import os -import json from pathlib import Path -from typing import Optional, Dict, Any +from typing import Any from dataclasses import dataclass, asdict # Try to import tomllib (Python 3.11+) or tomli for TOML support @@ -15,11 +16,12 @@ except ImportError: tomllib = None + # Simple TOML writer function to avoid tomli_w dependency -def _write_toml(data: Dict[str, Any], file_path: Path) -> None: +def _write_toml(data: dict[str, Any], file_path: Path) -> None: """Write data to TOML file using raw Python.""" lines = [] - + def _format_value(value: Any) -> str: if isinstance(value, str): return f'"{value}"' @@ -29,8 +31,8 @@ def _format_value(value: Any) -> str: return str(value) else: return f'"{str(value)}"' - - def _write_section(section_name: str, section_data: Dict[str, Any]) -> None: + + def _write_section(section_name: str, section_data: dict[str, Any]) -> None: lines.append(f"[{section_name}]") for key, value in section_data.items(): if isinstance(value, str) and "#" in key: @@ -39,19 +41,19 @@ def _write_section(section_name: str, section_data: Dict[str, Any]) -> None: else: lines.append(f"{key} = {_format_value(value)}") lines.append("") # Empty line after section - + # Add header comment lines.append("# PyneCore API Configuration") lines.append("# This is the default configuration file for PyneCore API integration") lines.append("") - + # Write sections - for key, value in data.items(): - if isinstance(value, dict): - _write_section(key, value) + for k, v in data.items(): + if isinstance(v, dict): + _write_section(k, v) else: - lines.append(f"{key} = {_format_value(value)}") - + lines.append(f"{k} = {_format_value(v)}") + # Write to file with open(file_path, 'w', encoding='utf-8') as f: f.write('\n'.join(lines)) @@ -60,13 +62,13 @@ def _write_section(section_name: str, section_data: Dict[str, Any]) -> None: @dataclass class APIConfig: """Configuration for PyneSys API client.""" - + api_key: str base_url: str = "https://api.pynesys.io" timeout: int = 30 - + @classmethod - def from_dict(cls, data: Dict[str, Any]) -> "APIConfig": + def from_dict(cls, data: dict[str, Any]) -> "APIConfig": """Create config from dictionary.""" # Handle both flat format and [api] section format if "api" in data: @@ -78,67 +80,63 @@ def from_dict(cls, data: Dict[str, Any]) -> "APIConfig": api_key = data.get("pynesys_api_key") or data.get("api_key") base_url = data.get("base_url", "https://api.pynesys.io") timeout = data.get("timeout", 30) - + if not api_key: raise ValueError("API key is required (pynesys_api_key or api_key)") - + return cls( api_key=api_key, base_url=base_url, timeout=timeout ) - - def to_dict(self) -> Dict[str, Any]: + + def to_dict(self) -> dict[str, Any]: """Convert config to dictionary.""" return asdict(self) - + @classmethod def from_env(cls) -> "APIConfig": """Create config from environment variables.""" api_key = os.getenv("PYNESYS_API_KEY") if not api_key: raise ValueError("PYNESYS_API_KEY environment variable is required") - + return cls( api_key=api_key, base_url=os.getenv("PYNESYS_BASE_URL", "https://api.pynesys.io"), timeout=int(os.getenv("PYNESYS_TIMEOUT", "30")) ) - + @classmethod def from_file(cls, config_path: Path) -> "APIConfig": - """Load configuration from TOML file. - - Args: - config_path: Path to TOML configuration file - - Returns: - APIConfig instance - - Raises: - ValueError: If file doesn't exist or has invalid format + """ + Load configuration from TOML file. + + :param config_path: Path to TOML configuration file + :return: APIConfig instance + :raises ValueError: If file doesn't exist or has invalid format """ if not config_path.exists(): raise ValueError(f"Configuration file not found: {config_path}") - + try: content = config_path.read_text() import tomllib data = tomllib.loads(content) return cls.from_dict(data) - + except Exception as e: raise ValueError(f"Failed to parse TOML configuration file {config_path}: {e}") - + def save_to_file(self, config_path: Path) -> None: - """Save configuration to TOML file. - - Args: - config_path: Path to save TOML configuration file + """ + Save configuration to TOML file. + + :param config_path: Path to save TOML configuration file """ # Create parent directories if they don't exist config_path.parent.mkdir(parents=True, exist_ok=True) - + # Save as TOML with [api] section using raw Python data = { "api": { @@ -151,63 +149,59 @@ def save_to_file(self, config_path: Path) -> None: class ConfigManager: """Manages API configuration loading and saving.""" - + DEFAULT_CONFIG_PATH = Path("workdir/config/api.toml") DEFAULT_FALLBACK_CONFIG_PATH = Path.home() / ".pynecore" / "api.toml" - + @classmethod - def load_config(cls, config_path: Optional[Path] = None) -> APIConfig: - """Load configuration from various sources. - + def load_config(cls, config_path: Path | None = None) -> APIConfig: + """ + Load configuration from various sources. + Priority order: 1. Provided config_path 2. Environment variables 3. Default config file - - Args: - config_path: Optional path to config file - - Returns: - APIConfig instance - - Raises: - ValueError: If no valid configuration found + + :param config_path: Optional path to config file + :return: APIConfig instance + :raises ValueError: If no valid configuration found """ # Try provided config path first if config_path and config_path.exists(): return APIConfig.from_file(config_path) - + # Try environment variables try: return APIConfig.from_env() except ValueError: pass - + # Try default config file first, then fallback locations if cls.DEFAULT_CONFIG_PATH.exists(): return APIConfig.from_file(cls.DEFAULT_CONFIG_PATH) elif cls.DEFAULT_FALLBACK_CONFIG_PATH.exists(): return APIConfig.from_file(cls.DEFAULT_FALLBACK_CONFIG_PATH) - + raise ValueError( f"No configuration file found. Tried:\n" f" - {cls.DEFAULT_CONFIG_PATH} (default)\n" f" - {cls.DEFAULT_FALLBACK_CONFIG_PATH} (fallback)\n" f"\nUse 'pyne api configure' to set up your API configuration." ) - + @classmethod - def save_config(cls, config: APIConfig, config_path: Optional[Path] = None) -> None: - """Save configuration to file. - - Args: - config: APIConfig instance to save - config_path: Optional path to save config file (defaults to DEFAULT_CONFIG_PATH) + def save_config(cls, config: APIConfig, config_path: Path | None = None) -> None: + """ + Save configuration to file. + + :param config: APIConfig instance to save + :param config_path: Optional path to save config file (defaults to DEFAULT_CONFIG_PATH) """ path = config_path or cls.DEFAULT_CONFIG_PATH config.save_to_file(path) - + @classmethod def get_default_config_path(cls) -> Path: """Get the default configuration file path.""" - return cls.DEFAULT_CONFIG_PATH \ No newline at end of file + return cls.DEFAULT_CONFIG_PATH diff --git a/src/pynecore/api/exceptions.py b/src/pynecore/api/exceptions.py index bab4a46..f3e6f41 100644 --- a/src/pynecore/api/exceptions.py +++ b/src/pynecore/api/exceptions.py @@ -1,12 +1,15 @@ -"""Custom exceptions for PyneCore API client.""" +""" +Custom exceptions for PyneCore API client. +""" -from typing import Optional, Dict, Any +from typing import Any class APIError(Exception): """Base exception for API-related errors.""" - - def __init__(self, message: str = "", status_code: Optional[int] = None, response_data: Optional[Dict[str, Any]] = None): + + def __init__(self, message: str = "", status_code: int | None = None, + response_data: dict[str, Any] | None = None): super().__init__(message) self.status_code = status_code self.response_data = response_data or {} @@ -19,16 +22,16 @@ class AuthError(APIError): class RateLimitError(APIError): """Rate limiting errors (429).""" - - def __init__(self, message: str, retry_after: Optional[int] = None, **kwargs): + + def __init__(self, message: str, retry_after: int | None = None, **kwargs): super().__init__(message, **kwargs) self.retry_after = retry_after class CompilationError(APIError): """Compilation-related errors (400, 422).""" - - def __init__(self, message: str, validation_errors: Optional[list] = None, **kwargs): + + def __init__(self, message: str, validation_errors: list | None = None, **kwargs): super().__init__(message, **kwargs) self.validation_errors = validation_errors or [] @@ -40,4 +43,4 @@ class NetworkError(APIError): class ServerError(APIError): """Server-side errors (500, 502, etc.).""" - pass \ No newline at end of file + pass diff --git a/src/pynecore/api/file_manager.py b/src/pynecore/api/file_manager.py index 24fdf72..2b2ed1e 100644 --- a/src/pynecore/api/file_manager.py +++ b/src/pynecore/api/file_manager.py @@ -1,175 +1,165 @@ -"""File management utilities for API operations.""" +""" +File management utilities for API operations. +""" -import os -import shutil from pathlib import Path from typing import Optional, Dict, Any, List -from datetime import datetime import shutil from datetime import datetime import json class FileManager: - """Manages file operations for API compilation results.""" - + """ + Manages file operations for API compilation results. + """ + def __init__(self, output_dir: Optional[Path] = None, max_log_files: int = 10): - """Initialize file manager. - - Args: - output_dir: Output directory for file operations (defaults to current directory / "output") - max_log_files: Maximum number of log files to keep + """ + Initialize file manager. + + :param output_dir: Output directory for file operations (defaults to current directory / "output") + :param max_log_files: Maximum number of log files to keep """ self.output_dir = output_dir or (Path.cwd() / "output") self.backup_dir = self.output_dir / "backups" self.log_dir = self.output_dir / "logs" self.max_log_files = max_log_files - + def save_compiled_code( - self, - compiled_code: str, - script_path: Path, - custom_output: Optional[Path] = None + self, + compiled_code: str, + script_path: Path, + custom_output: Optional[Path] = None ) -> Path: - """Save compiled Python code to file. - - Args: - compiled_code: The compiled Python code - script_path: Path to the original Pine Script file - custom_output: Optional custom output path - - Returns: - Path to the saved file + """ + Save compiled Python code to file. + + :param compiled_code: The compiled Python code + :param script_path: Path to the original Pine Script file + :param custom_output: Optional custom output path + :return: Path to the saved file """ # Determine output path if custom_output: output_path = custom_output else: output_path = self.generate_output_path(script_path) - + # Ensure output directory exists output_path.parent.mkdir(parents=True, exist_ok=True) - + # Create backup if file exists if output_path.exists(): self.backup_existing_file(output_path) - + # Write compiled code to file try: with open(output_path, 'w', encoding='utf-8') as f: f.write(compiled_code) return output_path except Exception as e: - raise OSError(f"Failed to write compiled code to {output_path}: {e}") - + raise OSError(f"Failed to write compiled code to {output_path}: {e}") + def generate_output_path(self, script_path: Path, custom_output: Optional[Path] = None) -> Path: - """Generate output path for compiled script. - - Args: - script_path: Path to the original Pine Script file - custom_output: Optional custom output path - - Returns: - Path for the output Python file + """ + Generate output path for compiled script. + + :param script_path: Path to the original Pine Script file + :param custom_output: Optional custom output path + :return: Path for the output Python file """ if custom_output: return custom_output - + # Generate output path in output directory filename = script_path.stem + ".py" return self.output_dir / filename - - + @staticmethod def save_compiled_script( - self, - compiled_code: str, - output_path: Path, - original_script_path: Optional[Path] = None, - metadata: Optional[Dict[str, Any]] = None + compiled_code: str, + output_path: Path, + original_script_path: Optional[Path] = None, + metadata: Optional[Dict[str, Any]] = None ) -> Path: - """Save compiled Python code to file. - - Args: - compiled_code: The compiled Python code - output_path: Path where to save the compiled code - original_script_path: Path to the original Pine Script file - metadata: Additional metadata to include in comments - - Returns: - Path to the saved file - - Raises: - OSError: If file cannot be written + """ + Save compiled Python code to file. + + :param compiled_code: The compiled Python code + :param output_path: Path where to save the compiled code + :param original_script_path: Path to the original Pine Script file + :param metadata: Additional metadata to include in comments + :return: Path to the saved file + :raises OSError: If file cannot be written """ # Ensure output directory exists output_path.parent.mkdir(parents=True, exist_ok=True) - + # Prepare file content with metadata header - content_lines = [] - + content_lines = [ + '"""', + 'Compiled Python code from Pine Script', + 'Generated by PyneCore API client', + '' + ] + # Add metadata header as comments - content_lines.append('"""') - content_lines.append('Compiled Python code from Pine Script') - content_lines.append('Generated by PyneCore API client') - content_lines.append('') - + if original_script_path: content_lines.append(f'Original file: {original_script_path}') - + content_lines.append(f'Compiled at: {datetime.now().isoformat()}') - + if metadata: content_lines.append('') content_lines.append('Compilation metadata:') for key, value in metadata.items(): content_lines.append(f' {key}: {value}') - + content_lines.append('"""') content_lines.append('') content_lines.append(compiled_code) - + # Write to file content = '\n'.join(content_lines) - + try: with open(output_path, 'w', encoding='utf-8') as f: f.write(content) return output_path except Exception as e: raise OSError(f"Failed to write compiled script to {output_path}: {e}") - - def backup_existing_file(self, file_path: Path) -> Optional[Path]: - """Create a backup of an existing file. - - Args: - file_path: Path to the file to backup - - Returns: - Path to the backup file, or None if original file doesn't exist + + @staticmethod + def backup_existing_file(file_path: Path) -> Optional[Path]: + """ + Create a backup of an existing file. + + :param file_path: Path to the file to backup + :return: Path to the backup file, or None if original file doesn't exist """ if not file_path.exists(): return None - + # Generate backup filename with timestamp timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") backup_path = file_path.with_suffix(f".{timestamp}.backup{file_path.suffix}") - + try: shutil.copy2(file_path, backup_path) return backup_path except Exception as e: raise OSError(f"Failed to create backup of {file_path}: {e}") - - def get_output_path(self, input_path: Path, output_dir: Optional[Path] = None) -> Path: - """Generate appropriate output path for compiled script. - - Args: - input_path: Path to the input Pine Script file - output_dir: Optional output directory (defaults to same as input) - - Returns: - Path for the output Python file + + @staticmethod + def get_output_path(input_path: Path, output_dir: Optional[Path] = None) -> Path: + """ + Generate appropriate output path for compiled script. + + :param input_path: Path to the input Pine Script file + :param output_dir: Optional output directory (defaults to same as input) + :return: Path for the output Python file """ if output_dir: # Use specified output directory with input filename @@ -177,94 +167,88 @@ def get_output_path(self, input_path: Path, output_dir: Optional[Path] = None) - else: # Use same directory as input, change extension to .py return input_path.with_suffix('.py') - + + @staticmethod def save_compilation_log( - self, - log_data: Dict[str, Any], - log_path: Optional[Path] = None + log_data: Dict[str, Any], + log_path: Optional[Path] = None ) -> Path: - """Save compilation log with metadata. - - Args: - log_data: Dictionary containing compilation log data - log_path: Optional path for log file (defaults to .pynecore/logs/) - - Returns: - Path to the saved log file + """ + Save compilation log with metadata. + + :param log_data: Dictionary containing compilation log data + :param log_path: Optional path for log file (defaults to .pynecore/logs/) + :return: Path to the saved log file """ if log_path is None: log_dir = Path.home() / ".pynecore" / "logs" log_dir.mkdir(parents=True, exist_ok=True) timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") log_path = log_dir / f"compilation_{timestamp}.json" - + # Add timestamp to log data log_data_with_timestamp = { "timestamp": datetime.now().isoformat(), **log_data } - + try: with open(log_path, 'w', encoding='utf-8') as f: json.dump(log_data_with_timestamp, f, indent=2, default=str) return log_path except Exception as e: raise OSError(f"Failed to write compilation log to {log_path}: {e}") - - def clean_old_logs(self, max_age_days: int = 30) -> List[Path]: - """Clean old compilation logs. - - Args: - max_age_days: Maximum age of logs to keep in days - - Returns: - List of paths to deleted log files + + @staticmethod + def clean_old_logs(max_age_days: int = 30) -> List[Path]: + """ + Clean old compilation logs. + + :param max_age_days: Maximum age of logs to keep in days + :return: List of paths to deleted log files """ log_dir = Path.home() / ".pynecore" / "logs" if not log_dir.exists(): return [] - + deleted_files = [] cutoff_time = datetime.now().timestamp() - (max_age_days * 24 * 60 * 60) - + try: for log_file in log_dir.glob("compilation_*.json"): if log_file.stat().st_mtime < cutoff_time: log_file.unlink() deleted_files.append(log_file) - except Exception as e: + except Exception: # noqa # Log cleanup is not critical, so we don't raise pass - + return deleted_files - - def validate_pine_script(self, script_path: Path) -> bool: - """Basic validation of Pine Script file. - - Args: - script_path: Path to Pine Script file - - Returns: - True if file appears to be a valid Pine Script - - Raises: - FileNotFoundError: If script file doesn't exist - OSError: If file cannot be read + + @staticmethod + def validate_pine_script(script_path: Path) -> bool: + """ + Basic validation of Pine Script file. + + :param script_path: Path to Pine Script file + :return: True if file appears to be a valid Pine Script + :raises FileNotFoundError: If script file doesn't exist + :raises OSError: If file cannot be read """ if not script_path.exists(): raise FileNotFoundError(f"Script file not found: {script_path}") - + try: with open(script_path, 'r', encoding='utf-8') as f: content = f.read() - + # Basic Pine Script validation # Check for version directive has_version = any( line.strip().startswith('//@version') or line.strip().startswith('// @version') for line in content.split('\n')[:10] # Check first 10 lines ) - + # Check for common Pine Script functions/keywords pine_keywords = [ 'indicator(', 'strategy(', 'library(', @@ -272,22 +256,21 @@ def validate_pine_script(self, script_path: Path) -> bool: 'ta.', 'math.', 'str.', 'array.', 'matrix.', 'input.', 'request.' ] - + has_pine_syntax = any(keyword in content for keyword in pine_keywords) - + return has_version or has_pine_syntax - + except Exception as e: raise OSError(f"Failed to read script file {script_path}: {e}") - - def get_script_info(self, script_path: Path) -> Dict[str, Any]: - """Extract basic information from Pine Script file. - - Args: - script_path: Path to Pine Script file - - Returns: - Dictionary with script information + + @staticmethod + def get_script_info(script_path: Path) -> Dict[str, Any]: + """ + Extract basic information from Pine Script file. + + :param script_path: Path to Pine Script file + :return: Dictionary with script information """ info = { "path": str(script_path), @@ -297,15 +280,15 @@ def get_script_info(self, script_path: Path) -> Dict[str, Any]: "version": None, "type": "unknown" } - + try: stat = script_path.stat() info["size"] = stat.st_size info["modified"] = datetime.fromtimestamp(stat.st_mtime).isoformat() - + with open(script_path, 'r', encoding='utf-8') as f: content = f.read() - + # Extract version for line in content.split('\n')[:10]: line = line.strip() @@ -313,7 +296,7 @@ def get_script_info(self, script_path: Path) -> Dict[str, Any]: version_part = line.split('version')[-1].strip() info["version"] = version_part break - + # Determine script type if 'indicator(' in content: info["type"] = "indicator" @@ -321,9 +304,9 @@ def get_script_info(self, script_path: Path) -> Dict[str, Any]: info["type"] = "strategy" elif 'library(' in content: info["type"] = "library" - - except Exception: + + except Exception: # noqa # If we can't read the file, return basic info pass - - return info \ No newline at end of file + + return info diff --git a/src/pynecore/api/models.py b/src/pynecore/api/models.py index 8bb3841..afee2b2 100644 --- a/src/pynecore/api/models.py +++ b/src/pynecore/api/models.py @@ -1,7 +1,9 @@ -"""Data models for PyneCore API responses.""" +""" +Data models for PyneCore API responses. +""" from dataclasses import dataclass -from typing import Optional, Dict, Any, List +from typing import Any from datetime import datetime @@ -10,26 +12,26 @@ class TokenValidationResponse: """Response from token validation endpoint.""" valid: bool message: str - user_id: Optional[str] = None - token_type: Optional[str] = None - expiration: Optional[datetime] = None - expires_at: Optional[datetime] = None - expires_in: Optional[int] = None - raw_response: Optional[Dict[str, Any]] = None + user_id: str | None = None + token_type: str | None = None + expiration: datetime | None = None + expires_at: datetime | None = None + expires_in: int | None = None + raw_response: dict[str, Any] | None = None @dataclass class CompileResponse: """Response from script compilation endpoint.""" success: bool - compiled_code: Optional[str] = None - error_message: Optional[str] = None - error: Optional[str] = None - validation_errors: Optional[List[Dict[str, Any]]] = None - warnings: Optional[List[str]] = None - details: Optional[List[str]] = None - status_code: Optional[int] = None - raw_response: Optional[Dict[str, Any]] = None + compiled_code: str | None = None + error_message: str | None = None + error: str | None = None + validation_errors: list[dict[str, Any]] | None = None + warnings: list[str] | None = None + details: list[str] | None = None + status_code: int | None = None + raw_response: dict[str, Any] | None = None @property def has_validation_errors(self) -> bool: @@ -44,4 +46,4 @@ def is_rate_limited(self) -> bool: @property def is_auth_error(self) -> bool: """Check if response indicates authentication error.""" - return self.status_code == 401 \ No newline at end of file + return self.status_code == 401 diff --git a/src/pynecore/cli/commands/compile.py b/src/pynecore/cli/commands/compile.py index 15fa47d..c496563 100644 --- a/src/pynecore/cli/commands/compile.py +++ b/src/pynecore/cli/commands/compile.py @@ -1,4 +1,3 @@ -import sys from pathlib import Path from typing import Optional @@ -6,8 +5,7 @@ from rich.console import Console from rich.progress import Progress, SpinnerColumn, TextColumn -from ..app import app, app_state -from ...api import ConfigManager, APIError, AuthError, RateLimitError, CompilationError +from ..app import app from ..utils.api_error_handler import handle_api_errors from ...core.compiler import create_compilation_service from ...utils.file_utils import preserve_mtime @@ -20,99 +18,94 @@ # noinspection PyShadowingBuiltins @app.command() def compile( - script_path: Path = typer.Argument( - ..., - help="Path to Pine Script file (.pine extension)", - exists=True, - file_okay=True, - dir_okay=False, - readable=True - ), - output: Optional[Path] = typer.Option( - None, - "--output", "-o", - help="Output Python file path (defaults to same name with .py extension)" - ), - strict: bool = typer.Option( - False, - "--strict", - help="Enable strict compilation mode with enhanced error checking" - ), - force: bool = typer.Option( - False, - "--force", - help="Force recompilation even if output file is up-to-date" - ), - config_path: Optional[Path] = typer.Option( - None, - "--config", - help="Path to TOML configuration file (defaults to workdir/config/api.toml)" - ), - api_key: Optional[str] = typer.Option( - None, - "--api-key", - help="PyneSys API key (overrides configuration file)", - envvar="PYNESYS_API_KEY" - ) + script_path: Path = typer.Argument( + ..., + help="Path to Pine Script file (.pine extension)", + exists=True, + file_okay=True, + dir_okay=False, + readable=True + ), + output: Optional[Path] = typer.Option( + None, + "--output", "-o", + help="Output Python file path (defaults to same name with .py extension)" + ), + strict: bool = typer.Option( + False, + "--strict", + help="Enable strict compilation mode with enhanced error checking" + ), + force: bool = typer.Option( + False, + "--force", + help="Force recompilation even if output file is up-to-date" + ), + api_key: Optional[str] = typer.Option( + None, + "--api-key", + help="PyneSys API key (overrides configuration file)", + envvar="PYNESYS_API_KEY" + ) ): - """Compile Pine Script to Python using PyneSys API. - + """ + Compile Pine Script to Python using PyneSys API. + USAGE: pyne compile # Compile single file pyne compile --force # Force recompile even if up-to-date pyne compile --strict # Enable strict compilation mode pyne compile --output # Specify output file path - + CONFIGURATION: Default config: workdir/config/api.toml Fallback config: ~/.pynecore/api.toml - Custom config: --config /path/to/config.toml - + Config format (TOML only): [api] pynesys_api_key = "your_api_key_here" base_url = "https://api.pynesys.io/" # optional timeout = 30 # optional, seconds - + SMART COMPILATION: - Automatically skips recompilation if output is newer than input - Use --force to override this behavior - Preserves file modification timestamps - + REQUIREMENTS: - Pine Script version 6 only (version 5 not supported) - Valid PyneSys API key required - Input file must have .pine extension - Output defaults to same name with .py extension - + Use 'pyne api configure' to set up your API configuration. """ try: # Create compilation service compilation_service = create_compilation_service( api_key=api_key, - config_path=config_path + ) - + # Determine output path if output is None: output = script_path.with_suffix('.py') - + # Check if compilation is needed (smart compilation) if not compilation_service.needs_compilation(script_path, output) and not force: console.print(f"[green]✓[/green] Output file is up-to-date: {output}") console.print("[dim]Use --force to recompile anyway[/dim]") return - + # Compile script with handle_api_errors(console): with Progress( - SpinnerColumn(), - TextColumn("[progress.description]{task.description}"), - console=console + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console ) as progress: progress.add_task("Compiling Pine Script...", total=None) - + # Compile the .pine file compiled_path = compilation_service.compile_file( script_path, @@ -120,12 +113,13 @@ def compile( force=force, strict=strict ) - + # Preserve modification time from source file preserve_mtime(script_path, output) - - console.print(f"[green]Compilation successful![/green] Your Pine Script is now ready: [cyan]{compiled_path}[/cyan]") - + + console.print(f"[green]Compilation successful![/green] Your Pine Script is now ready: " + f"[cyan]{compiled_path}[/cyan]") + except ValueError as e: error_msg = str(e) if "No configuration file found" in error_msg or "Configuration file not found" in error_msg: diff --git a/src/pynecore/cli/commands/run.py b/src/pynecore/cli/commands/run.py index 14e5a6b..649f1dd 100644 --- a/src/pynecore/cli/commands/run.py +++ b/src/pynecore/cli/commands/run.py @@ -20,8 +20,6 @@ from pynecore.core.syminfo import SymInfo from pynecore.core.script_runner import ScriptRunner from ...core.compiler import create_compilation_service -from ...api.exceptions import APIError, AuthError, RateLimitError, CompilationError -from ...api.config import ConfigManager from ...cli.utils.api_error_handler import handle_api_errors __all__ = [] @@ -128,10 +126,7 @@ def run( The system checks for changes in .pine files and only recompiles when necessary. Use --force to bypass this check and force recompilation. """ # noqa - # Handle script file extension and path - original_script = script - compiled_file = None - + # Support both .py and .pine files if script.suffix not in [".py", ".pine"]: # Check if the file exists with the given extension first @@ -139,7 +134,7 @@ def run( full_script_path = app_state.scripts_dir / script else: full_script_path = script - + if full_script_path.exists(): # File exists but has unsupported extension console.print(f"[red]This file format isn't supported:[/red] {script.suffix}") @@ -147,31 +142,31 @@ def run( if script.suffix in [".ohlcv", ".csv", ".json"]: console.print(f"[blue]💡 Heads up:[/blue] {script.suffix} files are data files, not executable scripts.") raise Exit(1) - + # File doesn't exist, try .pine first, then .py pine_script = script.with_suffix(".pine") py_script = script.with_suffix(".py") - + if len(script.parts) == 1: pine_script = app_state.scripts_dir / pine_script py_script = app_state.scripts_dir / py_script - + if pine_script.exists(): script = pine_script elif py_script.exists(): script = py_script else: script = py_script # Default to .py for error message - + # Expand script path if it's just a filename if len(script.parts) == 1: script = app_state.scripts_dir / script - + # Check if script exists if not script.exists(): secho(f"Script file '{script}' not found!", fg="red", err=True) raise Exit(1) - + # Handle .pine files - compile them first if script.suffix == ".pine": try: @@ -180,10 +175,10 @@ def run( api_key=api_key, config_path=config_path ) - + # Determine output path for compiled file compiled_file = script.with_suffix(".py") - + # Check if compilation is needed if compilation_service.needs_compilation(script, compiled_file) or force: with handle_api_errors(console): @@ -193,7 +188,7 @@ def run( console=console ) as progress: progress.add_task("Compiling Pine Script...", total=None) - + # Compile the .pine file compiled_path = compilation_service.compile_file( script, @@ -201,31 +196,32 @@ def run( force=force, strict=strict ) - + console.print(f"[green]Compilation successful![/green] Ready to run: [cyan]{compiled_path}[/cyan]") compiled_file = compiled_path - + else: console.print(f"[green]⚡ Using cached version:[/green] [cyan]{compiled_file}[/cyan]") console.print("[dim]Use --force to recompile[/dim]") - + # Update script to point to the compiled file script = compiled_file - + except ValueError as e: error_msg = str(e) if "No configuration file found" in error_msg or "Configuration file not found" in error_msg: console.print("[yellow]⚠️ No API configuration found[/yellow]") console.print() console.print("[bold]Quick setup (takes few minutes):[/bold]") - console.print("1. 🌐 Get your API key at [blue][link=https://pynesys.io]https://pynesys.io[/link][/blue]") + console.print("1. 🌐 Get your API key at " + "[blue][link=https://pynesys.io]https://pynesys.io[/link][/blue]") console.print("2. 🔧 Run [cyan]pyne api configure[/cyan] to save your configuration") console.print() console.print("[dim]💬 Need assistance? Our docs are here: https://pynesys.io/docs[/dim]") else: console.print(f"[red] Attention:[/red] {e}") raise Exit(1) - + # Ensure we have a .py file at this point if script.suffix != ".py": console.print(f"[red]This file format isn't supported:[/red] {script.suffix}") diff --git a/src/pynecore/core/compiler.py b/src/pynecore/core/compiler.py index ab68e32..c57745e 100644 --- a/src/pynecore/core/compiler.py +++ b/src/pynecore/core/compiler.py @@ -1,13 +1,10 @@ -"""Core compilation service for programmatic use. - -The CLI should use this service rather than implementing compilation logic directly. -This ensures all compilation functionality is available programmatically. +""" +Core compilation service for programmatic use. """ import subprocess import sys from pathlib import Path -from typing import Optional, Dict, Any from ..api.client import PynesysAPIClient from ..api.config import APIConfig, ConfigManager @@ -16,18 +13,19 @@ class CompilationService: - """Core compilation service for programmatic use. - + """ + Core compilation service for programmatic use. + The CLI should use this service rather than implementing compilation logic directly. This ensures all compilation functionality is available programmatically. """ - - def __init__(self, api_client: Optional[PynesysAPIClient] = None, config: Optional[APIConfig] = None): - """Initialize the compilation service. - - Args: - api_client: Optional pre-configured API client - config: Optional API configuration (will load from default if not provided) + + def __init__(self, api_client: PynesysAPIClient | None = None, config: APIConfig | None = None): + """ + Initialize the compilation service. + + :param api_client: Optional pre-configured API client + :param config: Optional API configuration (will load from default if not provided) """ if api_client: self.api_client = api_client @@ -40,115 +38,108 @@ def __init__(self, api_client: Optional[PynesysAPIClient] = None, config: Option base_url=config.base_url, timeout=config.timeout ) - + def compile_file( - self, - pine_file_path: Path, - output_file_path: Optional[Path] = None, - force: bool = False, - strict: bool = False + self, + pine_file_path: Path, + output_file_path: Path | None = None, + force: bool = False, + strict: bool = False ) -> Path: - """Compile a .pine file to Python. - - Args: - pine_file_path: Path to the .pine file - output_file_path: Optional output path (defaults to .py extension) - force: Force recompilation even if file hasn't changed - strict: Enable strict compilation mode - - Returns: - Path to the compiled .py file - - Raises: - FileNotFoundError: If pine file doesn't exist - CompilationError: If compilation fails - APIError: If API request fails + """ + Compile a .pine file to Python. + + :param pine_file_path: Path to the .pine file + :param output_file_path: Optional output path (defaults to .py extension) + :param force: Force recompilation even if file hasn't changed + :param strict: Enable strict compilation mode + :return: Path to the compiled .py file + :raises FileNotFoundError: If pine file doesn't exist + :raises CompilationError: If compilation fails + :raises APIError: If API request fails """ # Validate input file if not pine_file_path.exists(): raise FileNotFoundError(f"Pine file not found: {pine_file_path}") - + if pine_file_path.suffix != '.pine': - raise ValueError(f"This file format isn't supported: {pine_file_path.suffix}. Only .pine files can be compiled! ✨ Try using a .pine file instead.") - + raise ValueError( + f"This file format isn't supported: {pine_file_path.suffix}. " + f"Only .pine files can be compiled! ✨ Try using a .pine file instead.") + # Determine output path if output_file_path is None: output_file_path = pine_file_path.with_suffix('.py') - + # Check if compilation is needed (unless forced) if not force and not self.needs_compilation(pine_file_path, output_file_path): return output_file_path - + # Read Pine Script content try: with open(pine_file_path, 'r', encoding='utf-8') as f: script_content = f.read() except IOError as e: raise IOError(f"Error reading Pine file {pine_file_path}: {e}") - + # Compile via API try: response = self.api_client.compile_script_sync(script_content, strict=strict) - + if not response.success: raise CompilationError( f"Compilation failed: {response.error_message}", status_code=response.status_code, validation_errors=response.validation_errors ) - + # Write compiled code to output file output_file_path.parent.mkdir(parents=True, exist_ok=True) with open(output_file_path, 'w', encoding='utf-8') as f: f.write(response.compiled_code) - + # No need to update tracking info with mtime approach - + return output_file_path - - except (APIError, AuthError, RateLimitError, CompilationError) as e: + + except (APIError, AuthError, RateLimitError, CompilationError): # Re-raise API-related errors as-is raise except Exception as e: # Wrap unexpected errors raise APIError(f"Unexpected error during compilation: {e}") - - def needs_compilation(self, pine_file_path: Path, output_file_path: Path) -> bool: - """Check if a .pine file needs compilation using modification time comparison. - - Args: - pine_file_path: Path to the .pine file - output_file_path: Path to the compiled .py file - - Returns: - True if compilation is needed, False otherwise + + @staticmethod + def needs_compilation(pine_file_path: Path, output_file_path: Path) -> bool: + """ + Check if a .pine file needs compilation using modification time comparison. + + :param pine_file_path: Path to the .pine file + :param output_file_path: Path to the compiled .py file + :return: True if compilation is needed, False otherwise """ return file_needs_compilation(pine_file_path, output_file_path) - + def compile_and_run( - self, - pine_file_path: Path, - script_args: Optional[list] = None, - force: bool = False, - strict: bool = False, - output_file_path: Optional[Path] = None + self, + pine_file_path: Path, + script_args: list | None = None, + force: bool = False, + strict: bool = False, + output_file_path: Path | None = None ) -> int: - """Compile a .pine file and run the resulting Python script. - - Args: - pine_file_path: Path to the .pine file - script_args: Arguments to pass to the compiled script - force: Force recompilation even if file hasn't changed - strict: Enable strict compilation mode - output_file_path: Optional output path (defaults to .py extension) - - Returns: - Exit code from the executed script - - Raises: - FileNotFoundError: If pine file doesn't exist - CompilationError: If compilation fails - APIError: If API request fails + """ + Compile a .pine file and run the resulting Python script. + + :param pine_file_path: Path to the .pine file + :param script_args: Arguments to pass to the compiled script + :param force: Force recompilation even if file hasn't changed + :param strict: Enable strict compilation mode + :param output_file_path: Optional output path (defaults to .py extension) + :return: Exit code from the executed script + :raises FileNotFoundError: If pine file doesn't exist + :raises CompilationError: If compilation fails + :raises APIError: If API request fails """ # Compile the file compiled_file = self.compile_file( @@ -157,37 +148,31 @@ def compile_and_run( force=force, strict=strict ) - + # Prepare command to run the compiled script cmd = [sys.executable, str(compiled_file)] if script_args: cmd.extend(script_args) - + # Run the compiled script try: result = subprocess.run(cmd, check=False) return result.returncode except Exception as e: raise RuntimeError(f"Error executing compiled script: {e}") - - def create_compilation_service( - api_key: Optional[str] = None, - config_path: Optional[Path] = None + api_key: str | None = None, + config_path: Path | None = None ) -> CompilationService: - """Factory function to create a CompilationService instance. - - Args: - api_key: Optional API key override - config_path: Optional path to config file - - Returns: - Configured CompilationService instance - - Raises: - ValueError: If no valid configuration found + """ + Factory function to create a CompilationService instance. + + :param api_key: Optional API key override + :param config_path: Optional path to config file + :return: Configured CompilationService instance + :raises ValueError: If no valid configuration found """ if api_key: # Create API client with provided key @@ -196,4 +181,4 @@ def create_compilation_service( else: # Load configuration from file config = ConfigManager.load_config(config_path) - return CompilationService(config=config) \ No newline at end of file + return CompilationService(config=config) From 2324b1bf58c0bae9bc332cb3d884d3c5d8de9424 Mon Sep 17 00:00:00 2001 From: Adam Wallner Date: Sat, 2 Aug 2025 16:54:14 +0200 Subject: [PATCH 11/31] Massive simplification --- pyproject.toml | 3 - src/pynecore/api/__init__.py | 29 -- src/pynecore/api/client.py | 322 ------------------ src/pynecore/api/config.py | 207 ------------ src/pynecore/api/exceptions.py | 46 --- src/pynecore/api/file_manager.py | 312 ------------------ src/pynecore/api/models.py | 49 --- src/pynecore/cli/commands/__init__.py | 22 +- src/pynecore/cli/commands/api.py | 201 +++--------- src/pynecore/cli/commands/compile.py | 164 ++++------ src/pynecore/cli/commands/run.py | 177 +++------- src/pynecore/cli/utils/api_error_handler.py | 130 ++++---- src/pynecore/pynesys/__init__.py | 0 src/pynecore/pynesys/api.py | 346 ++++++++++++++++++++ src/pynecore/{core => pynesys}/compiler.py | 112 ++----- src/pynecore/utils/file_utils.py | 107 ++---- src/pynecore/utils/mtime_utils.py | 35 -- 17 files changed, 660 insertions(+), 1602 deletions(-) delete mode 100644 src/pynecore/api/__init__.py delete mode 100644 src/pynecore/api/client.py delete mode 100644 src/pynecore/api/config.py delete mode 100644 src/pynecore/api/exceptions.py delete mode 100644 src/pynecore/api/file_manager.py delete mode 100644 src/pynecore/api/models.py create mode 100644 src/pynecore/pynesys/__init__.py create mode 100644 src/pynecore/pynesys/api.py rename src/pynecore/{core => pynesys}/compiler.py (50%) delete mode 100644 src/pynecore/utils/mtime_utils.py diff --git a/pyproject.toml b/pyproject.toml index 82c0c93..65ec398 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,9 +38,6 @@ dependencies = [] # With user-friendly CLI optional-dependencies.cli = ["typer", "rich", "tzdata"] -# For API functionality (PyneSys compiler API) -optional-dependencies.api = ["httpx"] - # All optional dependencies for cli and all built-in providers optional-dependencies.all = ["typer", "rich", "httpx", "ccxt", "pycryptodome", "tzdata"] diff --git a/src/pynecore/api/__init__.py b/src/pynecore/api/__init__.py deleted file mode 100644 index 1046629..0000000 --- a/src/pynecore/api/__init__.py +++ /dev/null @@ -1,29 +0,0 @@ -"""PyneSys API client module.""" - -from .client import PynesysAPIClient -from .exceptions import ( - APIError, - AuthError, - RateLimitError, - CompilationError, - NetworkError, - ServerError -) -from .models import TokenValidationResponse, CompileResponse -from .config import APIConfig, ConfigManager -from .file_manager import FileManager - -__all__ = [ - "PynesysAPIClient", - "APIError", - "AuthError", - "RateLimitError", - "CompilationError", - "NetworkError", - "ServerError", - "TokenValidationResponse", - "CompileResponse", - "APIConfig", - "ConfigManager", - "FileManager" -] \ No newline at end of file diff --git a/src/pynecore/api/client.py b/src/pynecore/api/client.py deleted file mode 100644 index 2ef35e3..0000000 --- a/src/pynecore/api/client.py +++ /dev/null @@ -1,322 +0,0 @@ -"""PyneCore API client for PyneSys compiler service.""" - -import asyncio -from typing import Optional, TYPE_CHECKING -from datetime import datetime -import json - -if TYPE_CHECKING: - import httpx -else: - try: - import httpx - except ImportError: - httpx = None - -from .exceptions import ( - APIError, - AuthError, - RateLimitError, - CompilationError, - NetworkError, - ServerError, -) -from .models import TokenValidationResponse, CompileResponse - - -class PynesysAPIClient: - """Client for interacting with PyneSys API.""" - - def __init__( - self, - api_key: str, - base_url: str = "https://api.pynesys.io", - timeout: int = 30, - ): - """ - Initialize the API client. - - :param api_key: PyneSys API key - :param base_url: Base URL for the API - :param timeout: Request timeout in seconds - """ - if httpx is None: - raise ImportError( - "httpx is required for API functionality. " - "Install it with: pip install httpx" - ) - - if not api_key or not api_key.strip(): - raise ValueError("API key is required") - - self.api_key = api_key - self.base_url = base_url.rstrip("/") - self.timeout = timeout - self._client: Optional["httpx.AsyncClient"] = None - - def compile_script_sync(self, script: str, strict: bool = False) -> CompileResponse: - """ - Synchronous wrapper for compile_script. - - :param script: Pine Script code to compile - :param strict: Enable strict compilation mode - :return: CompileResponse with compilation results - """ - return asyncio.run(self.compile_script(script, strict)) - - def verify_token_sync(self) -> TokenValidationResponse: - """ - Synchronous wrapper for verify_token. - - :return: TokenValidationResponse with token validation results - """ - return asyncio.run(self.verify_token()) - - async def __aenter__(self): - """Async context manager entry.""" - await self._ensure_client() - return self - - async def __aexit__(self, exc_type, exc_val, exc_tb): - """Async context manager exit.""" - await self.close() - - async def _ensure_client(self): - """Ensure HTTP client is initialized.""" - if self._client is None and httpx is not None: - self._client = httpx.AsyncClient( - timeout=self.timeout, - headers={ - "Authorization": f"Bearer {self.api_key}", - "User-Agent": "PyneCore-API-Client", - }, - ) - - async def close(self): - """Close the HTTP client.""" - if self._client: - await self._client.aclose() - self._client = None - - async def verify_token(self) -> TokenValidationResponse: - """ - Verify API token validity. - - :return: TokenValidationResponse with validation details - :raises AuthError: If token is invalid - :raises NetworkError: If network request fails - :raises APIError: For other API errors - """ - await self._ensure_client() - - try: - if self._client is None: - raise APIError("HTTP client not initialized") - - response = await self._client.get( - f"{self.base_url}/auth/verify-token", - params={"token": self.api_key} - ) - - if response.status_code == 200: - data = response.json() - return TokenValidationResponse( - valid=data.get("valid", False), - message=data.get("message", ""), - user_id=data.get("user_id"), - token_type=data.get("token_type"), - expiration=self._parse_datetime(data.get("expiration")), - expires_at=self._parse_datetime(data.get("expires_at")), - expires_in=data.get("expires_in"), - raw_response=data, - ) - else: - self._handle_api_error(response) - # This should never be reached due to _handle_api_error raising - raise APIError("Unexpected API response") - - except Exception as e: - if httpx and isinstance(e, httpx.RequestError): - raise NetworkError(f"Network error during token verification: {e}") - elif not isinstance(e, APIError): - raise APIError(f"Unexpected error during token verification: {e}") - else: - raise - - async def compile_script( - self, - script: str, - strict: bool = False - ) -> CompileResponse: - """ - Compile Pine Script to Python via API. - - :param script: Pine Script code to compile - :param strict: Whether to use strict compilation mode - :return: CompileResponse with compiled code or error details - :raises AuthError: If authentication fails - :raises RateLimitError: If rate limit is exceeded - :raises CompilationError: If compilation fails - :raises NetworkError: If network request fails - :raises APIError: For other API errors - """ - await self._ensure_client() - - try: - # Prepare form data - data = { - "script": script, - "strict": str(strict).lower() - } - - if self._client is None: - raise APIError("HTTP client not initialized") - - response = await self._client.post( - f"{self.base_url}/compiler/compile", - data=data, - headers={"Content-Type": "application/x-www-form-urlencoded"} - ) - - if response.status_code == 200: - # Success - return compiled code - compiled_code = response.text - return CompileResponse( - success=True, - compiled_code=compiled_code, - status_code=200 - ) - else: - # Handle error responses - return self._handle_compile_error(response) - - except Exception as e: - if httpx and isinstance(e, httpx.RequestError): - raise NetworkError(f"Network error during compilation: {e}") - elif not isinstance(e, APIError): - raise APIError(f"Unexpected error during compilation: {e}") - else: - raise - - @staticmethod - def _handle_api_error(response: "httpx.Response") -> None: - """ - Handle API error responses. - - :param response: HTTP response object - :raises: Appropriate exception based on status code - """ - status_code = response.status_code - - try: - error_data = response.json() - message = error_data.get("message", response.text) - except (json.JSONDecodeError, ValueError): - message = response.text or f"HTTP {status_code} error" - - if status_code == 401: - raise AuthError(message, status_code=status_code) - elif status_code == 429: - retry_after = response.headers.get("Retry-After") - raise RateLimitError( - message, - status_code=status_code, - retry_after=int(retry_after) if retry_after else None - ) - elif status_code >= 500: - raise ServerError(message, status_code=status_code) - else: - raise APIError(message, status_code=status_code) - - def _handle_compile_error(self, response: "httpx.Response") -> CompileResponse: - """ - Handle compilation error responses. - - :param response: HTTP response object - :return: CompileResponse with error details - :raises CompilationError: For compilation-related errors (422) - :raises: Other exceptions for authentication, rate limiting, etc. - """ - status_code = response.status_code - - try: - error_data = response.json() - except (json.JSONDecodeError, ValueError): - error_data = {} - - # Extract error message - if "detail" in error_data and isinstance(error_data["detail"], list): - # Validation error format (422) - validation_errors = error_data["detail"] - error_message = "Validation errors occurred" - else: - validation_errors = None - error_message = error_data.get("message", response.text or f"HTTP {status_code} error") - - # For compilation errors (422), raise CompilationError - if status_code == 422: - raise CompilationError(error_message, status_code=status_code, validation_errors=validation_errors) - - # For other errors, use the general API error handler - self._handle_api_error(response) - - # This should never be reached - return CompileResponse( - success=False, - error_message=error_message, - validation_errors=validation_errors, - status_code=status_code, - raw_response=error_data - ) - - @staticmethod - def _parse_datetime(dt_str: Optional[str]) -> Optional[datetime]: - """ - Parse datetime string from API response. - - :param dt_str: Datetime string from API - :return: Parsed datetime object or None - """ - if not dt_str: - return None - - try: - # Try common datetime formats - for fmt in [ - "%Y-%m-%dT%H:%M:%S.%fZ", - "%Y-%m-%dT%H:%M:%SZ", - "%Y-%m-%d %H:%M:%S", - "%Y-%m-%d" - ]: - try: - return datetime.strptime(dt_str, fmt) - except ValueError: - continue - - # If no format matches, return None - return None - - except Exception: # noqa - return None - - -# Synchronous wrapper for convenience -class SyncPynesysAPIClient: - """Synchronous wrapper for PynesysAPIClient.""" - - def __init__(self, *args, **kwargs): - self._async_client = PynesysAPIClient(*args, **kwargs) - - def verify_token(self) -> TokenValidationResponse: - """Synchronous token verification.""" - return asyncio.run(self._async_client.verify_token()) - - def compile_script(self, script: str, strict: bool = False) -> CompileResponse: - """Synchronous script compilation.""" - return asyncio.run(self._async_client.compile_script(script, strict)) - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - asyncio.run(self._async_client.close()) diff --git a/src/pynecore/api/config.py b/src/pynecore/api/config.py deleted file mode 100644 index a48ad89..0000000 --- a/src/pynecore/api/config.py +++ /dev/null @@ -1,207 +0,0 @@ -""" -Configuration management for PyneSys API. -""" - -import os -from pathlib import Path -from typing import Any -from dataclasses import dataclass, asdict - -# Try to import tomllib (Python 3.11+) or tomli for TOML support -try: - import tomllib -except ImportError: - try: - import tomli as tomllib # type: ignore - except ImportError: - tomllib = None - - -# Simple TOML writer function to avoid tomli_w dependency -def _write_toml(data: dict[str, Any], file_path: Path) -> None: - """Write data to TOML file using raw Python.""" - lines = [] - - def _format_value(value: Any) -> str: - if isinstance(value, str): - return f'"{value}"' - elif isinstance(value, bool): - return str(value).lower() - elif isinstance(value, (int, float)): - return str(value) - else: - return f'"{str(value)}"' - - def _write_section(section_name: str, section_data: dict[str, Any]) -> None: - lines.append(f"[{section_name}]") - for key, value in section_data.items(): - if isinstance(value, str) and "#" in key: - # Handle comments - lines.append(f"{key} = {_format_value(value)}") - else: - lines.append(f"{key} = {_format_value(value)}") - lines.append("") # Empty line after section - - # Add header comment - lines.append("# PyneCore API Configuration") - lines.append("# This is the default configuration file for PyneCore API integration") - lines.append("") - - # Write sections - for k, v in data.items(): - if isinstance(v, dict): - _write_section(k, v) - else: - lines.append(f"{k} = {_format_value(v)}") - - # Write to file - with open(file_path, 'w', encoding='utf-8') as f: - f.write('\n'.join(lines)) - - -@dataclass -class APIConfig: - """Configuration for PyneSys API client.""" - - api_key: str - base_url: str = "https://api.pynesys.io" - timeout: int = 30 - - @classmethod - def from_dict(cls, data: dict[str, Any]) -> "APIConfig": - """Create config from dictionary.""" - # Handle both flat format and [api] section format - if "api" in data: - api_data = data["api"] - api_key = api_data.get("pynesys_api_key") or api_data.get("api_key") - base_url = api_data.get("base_url", "https://api.pynesys.io") - timeout = api_data.get("timeout", 30) - else: - api_key = data.get("pynesys_api_key") or data.get("api_key") - base_url = data.get("base_url", "https://api.pynesys.io") - timeout = data.get("timeout", 30) - - if not api_key: - raise ValueError("API key is required (pynesys_api_key or api_key)") - - return cls( - api_key=api_key, - base_url=base_url, - timeout=timeout - ) - - def to_dict(self) -> dict[str, Any]: - """Convert config to dictionary.""" - return asdict(self) - - @classmethod - def from_env(cls) -> "APIConfig": - """Create config from environment variables.""" - api_key = os.getenv("PYNESYS_API_KEY") - if not api_key: - raise ValueError("PYNESYS_API_KEY environment variable is required") - - return cls( - api_key=api_key, - base_url=os.getenv("PYNESYS_BASE_URL", "https://api.pynesys.io"), - timeout=int(os.getenv("PYNESYS_TIMEOUT", "30")) - ) - - @classmethod - def from_file(cls, config_path: Path) -> "APIConfig": - """ - Load configuration from TOML file. - - :param config_path: Path to TOML configuration file - :return: APIConfig instance - :raises ValueError: If file doesn't exist or has invalid format - """ - if not config_path.exists(): - raise ValueError(f"Configuration file not found: {config_path}") - - try: - content = config_path.read_text() - import tomllib - data = tomllib.loads(content) - return cls.from_dict(data) - - except Exception as e: - raise ValueError(f"Failed to parse TOML configuration file {config_path}: {e}") - - def save_to_file(self, config_path: Path) -> None: - """ - Save configuration to TOML file. - - :param config_path: Path to save TOML configuration file - """ - # Create parent directories if they don't exist - config_path.parent.mkdir(parents=True, exist_ok=True) - - # Save as TOML with [api] section using raw Python - data = { - "api": { - "pynesys_api_key": self.api_key, - "timeout": self.timeout - } - } - _write_toml(data, config_path) - - -class ConfigManager: - """Manages API configuration loading and saving.""" - - DEFAULT_CONFIG_PATH = Path("workdir/config/api.toml") - DEFAULT_FALLBACK_CONFIG_PATH = Path.home() / ".pynecore" / "api.toml" - - @classmethod - def load_config(cls, config_path: Path | None = None) -> APIConfig: - """ - Load configuration from various sources. - - Priority order: - 1. Provided config_path - 2. Environment variables - 3. Default config file - - :param config_path: Optional path to config file - :return: APIConfig instance - :raises ValueError: If no valid configuration found - """ - # Try provided config path first - if config_path and config_path.exists(): - return APIConfig.from_file(config_path) - - # Try environment variables - try: - return APIConfig.from_env() - except ValueError: - pass - - # Try default config file first, then fallback locations - if cls.DEFAULT_CONFIG_PATH.exists(): - return APIConfig.from_file(cls.DEFAULT_CONFIG_PATH) - elif cls.DEFAULT_FALLBACK_CONFIG_PATH.exists(): - return APIConfig.from_file(cls.DEFAULT_FALLBACK_CONFIG_PATH) - - raise ValueError( - f"No configuration file found. Tried:\n" - f" - {cls.DEFAULT_CONFIG_PATH} (default)\n" - f" - {cls.DEFAULT_FALLBACK_CONFIG_PATH} (fallback)\n" - f"\nUse 'pyne api configure' to set up your API configuration." - ) - - @classmethod - def save_config(cls, config: APIConfig, config_path: Path | None = None) -> None: - """ - Save configuration to file. - - :param config: APIConfig instance to save - :param config_path: Optional path to save config file (defaults to DEFAULT_CONFIG_PATH) - """ - path = config_path or cls.DEFAULT_CONFIG_PATH - config.save_to_file(path) - - @classmethod - def get_default_config_path(cls) -> Path: - """Get the default configuration file path.""" - return cls.DEFAULT_CONFIG_PATH diff --git a/src/pynecore/api/exceptions.py b/src/pynecore/api/exceptions.py deleted file mode 100644 index f3e6f41..0000000 --- a/src/pynecore/api/exceptions.py +++ /dev/null @@ -1,46 +0,0 @@ -""" -Custom exceptions for PyneCore API client. -""" - -from typing import Any - - -class APIError(Exception): - """Base exception for API-related errors.""" - - def __init__(self, message: str = "", status_code: int | None = None, - response_data: dict[str, Any] | None = None): - super().__init__(message) - self.status_code = status_code - self.response_data = response_data or {} - - -class AuthError(APIError): - """Authentication-related errors (401, invalid token, etc.).""" - pass - - -class RateLimitError(APIError): - """Rate limiting errors (429).""" - - def __init__(self, message: str, retry_after: int | None = None, **kwargs): - super().__init__(message, **kwargs) - self.retry_after = retry_after - - -class CompilationError(APIError): - """Compilation-related errors (400, 422).""" - - def __init__(self, message: str, validation_errors: list | None = None, **kwargs): - super().__init__(message, **kwargs) - self.validation_errors = validation_errors or [] - - -class NetworkError(APIError): - """Network-related errors (timeouts, connection issues).""" - pass - - -class ServerError(APIError): - """Server-side errors (500, 502, etc.).""" - pass diff --git a/src/pynecore/api/file_manager.py b/src/pynecore/api/file_manager.py deleted file mode 100644 index 2b2ed1e..0000000 --- a/src/pynecore/api/file_manager.py +++ /dev/null @@ -1,312 +0,0 @@ -""" -File management utilities for API operations. -""" - -from pathlib import Path -from typing import Optional, Dict, Any, List -import shutil -from datetime import datetime -import json - - -class FileManager: - """ - Manages file operations for API compilation results. - """ - - def __init__(self, output_dir: Optional[Path] = None, max_log_files: int = 10): - """ - Initialize file manager. - - :param output_dir: Output directory for file operations (defaults to current directory / "output") - :param max_log_files: Maximum number of log files to keep - """ - self.output_dir = output_dir or (Path.cwd() / "output") - self.backup_dir = self.output_dir / "backups" - self.log_dir = self.output_dir / "logs" - self.max_log_files = max_log_files - - def save_compiled_code( - self, - compiled_code: str, - script_path: Path, - custom_output: Optional[Path] = None - ) -> Path: - """ - Save compiled Python code to file. - - :param compiled_code: The compiled Python code - :param script_path: Path to the original Pine Script file - :param custom_output: Optional custom output path - :return: Path to the saved file - """ - # Determine output path - if custom_output: - output_path = custom_output - else: - output_path = self.generate_output_path(script_path) - - # Ensure output directory exists - output_path.parent.mkdir(parents=True, exist_ok=True) - - # Create backup if file exists - if output_path.exists(): - self.backup_existing_file(output_path) - - # Write compiled code to file - try: - with open(output_path, 'w', encoding='utf-8') as f: - f.write(compiled_code) - return output_path - except Exception as e: - raise OSError(f"Failed to write compiled code to {output_path}: {e}") - - def generate_output_path(self, script_path: Path, custom_output: Optional[Path] = None) -> Path: - """ - Generate output path for compiled script. - - :param script_path: Path to the original Pine Script file - :param custom_output: Optional custom output path - :return: Path for the output Python file - """ - if custom_output: - return custom_output - - # Generate output path in output directory - filename = script_path.stem + ".py" - return self.output_dir / filename - - @staticmethod - def save_compiled_script( - compiled_code: str, - output_path: Path, - original_script_path: Optional[Path] = None, - metadata: Optional[Dict[str, Any]] = None - ) -> Path: - """ - Save compiled Python code to file. - - :param compiled_code: The compiled Python code - :param output_path: Path where to save the compiled code - :param original_script_path: Path to the original Pine Script file - :param metadata: Additional metadata to include in comments - :return: Path to the saved file - :raises OSError: If file cannot be written - """ - # Ensure output directory exists - output_path.parent.mkdir(parents=True, exist_ok=True) - - # Prepare file content with metadata header - content_lines = [ - '"""', - 'Compiled Python code from Pine Script', - 'Generated by PyneCore API client', - '' - ] - - # Add metadata header as comments - - if original_script_path: - content_lines.append(f'Original file: {original_script_path}') - - content_lines.append(f'Compiled at: {datetime.now().isoformat()}') - - if metadata: - content_lines.append('') - content_lines.append('Compilation metadata:') - for key, value in metadata.items(): - content_lines.append(f' {key}: {value}') - - content_lines.append('"""') - content_lines.append('') - content_lines.append(compiled_code) - - # Write to file - content = '\n'.join(content_lines) - - try: - with open(output_path, 'w', encoding='utf-8') as f: - f.write(content) - return output_path - except Exception as e: - raise OSError(f"Failed to write compiled script to {output_path}: {e}") - - @staticmethod - def backup_existing_file(file_path: Path) -> Optional[Path]: - """ - Create a backup of an existing file. - - :param file_path: Path to the file to backup - :return: Path to the backup file, or None if original file doesn't exist - """ - if not file_path.exists(): - return None - - # Generate backup filename with timestamp - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - backup_path = file_path.with_suffix(f".{timestamp}.backup{file_path.suffix}") - - try: - shutil.copy2(file_path, backup_path) - return backup_path - except Exception as e: - raise OSError(f"Failed to create backup of {file_path}: {e}") - - @staticmethod - def get_output_path(input_path: Path, output_dir: Optional[Path] = None) -> Path: - """ - Generate appropriate output path for compiled script. - - :param input_path: Path to the input Pine Script file - :param output_dir: Optional output directory (defaults to same as input) - :return: Path for the output Python file - """ - if output_dir: - # Use specified output directory with input filename - return output_dir / input_path.with_suffix('.py').name - else: - # Use same directory as input, change extension to .py - return input_path.with_suffix('.py') - - @staticmethod - def save_compilation_log( - log_data: Dict[str, Any], - log_path: Optional[Path] = None - ) -> Path: - """ - Save compilation log with metadata. - - :param log_data: Dictionary containing compilation log data - :param log_path: Optional path for log file (defaults to .pynecore/logs/) - :return: Path to the saved log file - """ - if log_path is None: - log_dir = Path.home() / ".pynecore" / "logs" - log_dir.mkdir(parents=True, exist_ok=True) - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - log_path = log_dir / f"compilation_{timestamp}.json" - - # Add timestamp to log data - log_data_with_timestamp = { - "timestamp": datetime.now().isoformat(), - **log_data - } - - try: - with open(log_path, 'w', encoding='utf-8') as f: - json.dump(log_data_with_timestamp, f, indent=2, default=str) - return log_path - except Exception as e: - raise OSError(f"Failed to write compilation log to {log_path}: {e}") - - @staticmethod - def clean_old_logs(max_age_days: int = 30) -> List[Path]: - """ - Clean old compilation logs. - - :param max_age_days: Maximum age of logs to keep in days - :return: List of paths to deleted log files - """ - log_dir = Path.home() / ".pynecore" / "logs" - if not log_dir.exists(): - return [] - - deleted_files = [] - cutoff_time = datetime.now().timestamp() - (max_age_days * 24 * 60 * 60) - - try: - for log_file in log_dir.glob("compilation_*.json"): - if log_file.stat().st_mtime < cutoff_time: - log_file.unlink() - deleted_files.append(log_file) - except Exception: # noqa - # Log cleanup is not critical, so we don't raise - pass - - return deleted_files - - @staticmethod - def validate_pine_script(script_path: Path) -> bool: - """ - Basic validation of Pine Script file. - - :param script_path: Path to Pine Script file - :return: True if file appears to be a valid Pine Script - :raises FileNotFoundError: If script file doesn't exist - :raises OSError: If file cannot be read - """ - if not script_path.exists(): - raise FileNotFoundError(f"Script file not found: {script_path}") - - try: - with open(script_path, 'r', encoding='utf-8') as f: - content = f.read() - - # Basic Pine Script validation - # Check for version directive - has_version = any( - line.strip().startswith('//@version') or line.strip().startswith('// @version') - for line in content.split('\n')[:10] # Check first 10 lines - ) - - # Check for common Pine Script functions/keywords - pine_keywords = [ - 'indicator(', 'strategy(', 'library(', - 'plot(', 'plotshape(', 'plotchar(', - 'ta.', 'math.', 'str.', 'array.', 'matrix.', - 'input.', 'request.' - ] - - has_pine_syntax = any(keyword in content for keyword in pine_keywords) - - return has_version or has_pine_syntax - - except Exception as e: - raise OSError(f"Failed to read script file {script_path}: {e}") - - @staticmethod - def get_script_info(script_path: Path) -> Dict[str, Any]: - """ - Extract basic information from Pine Script file. - - :param script_path: Path to Pine Script file - :return: Dictionary with script information - """ - info = { - "path": str(script_path), - "name": script_path.name, - "size": 0, - "modified": None, - "version": None, - "type": "unknown" - } - - try: - stat = script_path.stat() - info["size"] = stat.st_size - info["modified"] = datetime.fromtimestamp(stat.st_mtime).isoformat() - - with open(script_path, 'r', encoding='utf-8') as f: - content = f.read() - - # Extract version - for line in content.split('\n')[:10]: - line = line.strip() - if line.startswith('//@version') or line.startswith('// @version'): - version_part = line.split('version')[-1].strip() - info["version"] = version_part - break - - # Determine script type - if 'indicator(' in content: - info["type"] = "indicator" - elif 'strategy(' in content: - info["type"] = "strategy" - elif 'library(' in content: - info["type"] = "library" - - except Exception: # noqa - # If we can't read the file, return basic info - pass - - return info diff --git a/src/pynecore/api/models.py b/src/pynecore/api/models.py deleted file mode 100644 index afee2b2..0000000 --- a/src/pynecore/api/models.py +++ /dev/null @@ -1,49 +0,0 @@ -""" -Data models for PyneCore API responses. -""" - -from dataclasses import dataclass -from typing import Any -from datetime import datetime - - -@dataclass -class TokenValidationResponse: - """Response from token validation endpoint.""" - valid: bool - message: str - user_id: str | None = None - token_type: str | None = None - expiration: datetime | None = None - expires_at: datetime | None = None - expires_in: int | None = None - raw_response: dict[str, Any] | None = None - - -@dataclass -class CompileResponse: - """Response from script compilation endpoint.""" - success: bool - compiled_code: str | None = None - error_message: str | None = None - error: str | None = None - validation_errors: list[dict[str, Any]] | None = None - warnings: list[str] | None = None - details: list[str] | None = None - status_code: int | None = None - raw_response: dict[str, Any] | None = None - - @property - def has_validation_errors(self) -> bool: - """Check if response contains validation errors.""" - return bool(self.validation_errors) - - @property - def is_rate_limited(self) -> bool: - """Check if response indicates rate limiting.""" - return self.status_code == 429 - - @property - def is_auth_error(self) -> bool: - """Check if response indicates authentication error.""" - return self.status_code == 401 diff --git a/src/pynecore/cli/commands/__init__.py b/src/pynecore/cli/commands/__init__.py index 6db4bad..fd3aece 100644 --- a/src/pynecore/cli/commands/__init__.py +++ b/src/pynecore/cli/commands/__init__.py @@ -30,6 +30,16 @@ def setup( "--recreate-demo", help="Recreate demo.py and demo.ohlcv files even if workdir exists", ), + recreate_provider_config: bool = typer.Option( + False, + "--recreate-provider-config", + help="Recreate provider.toml file even if workdir exists", + ), + recreate_api_config: bool = typer.Option( + False, + "--recreate-api-config", + help="Recreate api.toml file even if workdir exists", + ) ): """ Pyne Command Line Interface @@ -211,7 +221,7 @@ def main( # Create providers.toml file for all supported providers (if not exists) providers_file = config_dir / 'providers.toml' - if not providers_file.exists(): + if not providers_file.exists() or recreate_provider_config: with providers_file.open('w') as f: for provider in available_providers: f.write(f"[{provider}]\n") @@ -234,6 +244,16 @@ def main( raise ValueError(f"Unsupported type for {key}: {type(value)}") f.write("\n") + # Create api.toml file for PyneSys API (if not exists) + api_file = config_dir / 'api.toml' + if not api_file.exists() or recreate_api_config: + with api_file.open('w') as f: + f.write("""[api] +api_key = "" +base_url = "https://api.pynesys.io" +timeout = 30 +""") + # Set workdir in app_state app_state.workdir = workdir diff --git a/src/pynecore/cli/commands/api.py b/src/pynecore/cli/commands/api.py index e998080..349c59d 100644 --- a/src/pynecore/cli/commands/api.py +++ b/src/pynecore/cli/commands/api.py @@ -1,34 +1,29 @@ """API configuration and management commands.""" - -import sys -from pathlib import Path -from typing import Optional - import typer from rich.console import Console from rich.table import Table from rich.progress import Progress, SpinnerColumn, TextColumn from ..app import app -from ...api import ConfigManager, APIConfig, PynesysAPIClient, APIError, AuthError +from pynecore.pynesys.api import APIClient, APIError, AuthError def format_expires_in(seconds: int) -> str: """Format expires_in seconds into human-readable format. - + Args: seconds: Number of seconds until expiration - + Returns: Formatted string like "2 days, 5 hours, 30 minutes" """ if seconds <= 0: return "Expired" - + days = seconds // 86400 hours = (seconds % 86400) // 3600 minutes = (seconds % 3600) // 60 - + parts = [] if days > 0: parts.append(f"{days} day{'s' if days != 1 else ''}") @@ -36,116 +31,38 @@ def format_expires_in(seconds: int) -> str: parts.append(f"{hours} hour{'s' if hours != 1 else ''}") if minutes > 0: parts.append(f"{minutes} minute{'s' if minutes != 1 else ''}") - + if not parts: return "Less than 1 minute" - + return ", ".join(parts) + __all__ = [] console = Console() -api_app = typer.Typer(help="PyneSys API configuration and management commands for authentication and connection testing") +api_app = typer.Typer( + help="PyneSys API management commands for connection testing.") app.add_typer(api_app, name="api") -@api_app.command() -def configure( - api_key: Optional[str] = typer.Option( - None, - "--api-key", - help="PyneSys API key", - prompt="Enter your PyneSys API key", - hide_input=True - ) -): - """Configure PyneSys API settings and validate your API key. - - This command sets up your PyneSys API configuration including: - - API key for authentication - - The configuration is saved to workdir/config/api.toml. - - The API key will be validated during configuration to ensure it's working. - """ - try: - # Create configuration with default values - config = APIConfig( - api_key=api_key, - base_url="https://api.pynesys.io", - timeout=30 - ) - - # Test the API key - with Progress( - SpinnerColumn(), - TextColumn("[progress.description]{task.description}"), - console=console - ) as progress: - task = progress.add_task("Validating API key...", total=None) - - try: - client = PynesysAPIClient(config.api_key, config.base_url, config.timeout) - result = client.verify_token_sync() - - if result.valid: - progress.update(task, description="[green]API key validated![/green]") - console.print(f"[green]✓[/green] API key is valid") - console.print(f"[blue]User ID:[/blue] {result.user_id}") - console.print(f"[blue]Token Type:[/blue] {result.token_type}") - if result.expires_at: - console.print(f"[blue]Expires:[/blue] {result.expires_at}") - else: - progress.update(task, description="[red]API key validation failed[/red]") - console.print("[red]✗[/red] The provided API key appears to be invalid.") - console.print("[yellow]💡 Please check your API key at [blue][link=https://pynesys.io]https://pynesys.io[/link][/blue] and try again.") - console.print("[dim]Make sure you've copied the complete API key without any extra spaces.[/dim]") - if result.message: - console.print(f"[dim]Details: {result.message}[/dim]") - raise typer.Exit(1) - - except AuthError: - progress.update(task, description="[red]Invalid API key[/red]") - console.print("[red]✗[/red] The provided API key appears to be invalid.") - console.print("[yellow]💡 Please check your API key at [blue][link=https://pynesys.io]https://pynesys.io[/link][/blue] and try again.") - console.print("[dim]Make sure you've copied the complete API key without any extra spaces.[/dim]") - raise typer.Exit(1) - - except APIError as e: - progress.update(task, description="[red]API error[/red]") - console.print(f"[red]✗[/red] API error: {e}") - raise typer.Exit(1) - - # Save configuration - ConfigManager.save_config(config, None) - - config_file = ConfigManager.get_default_config_path() - console.print(f"[green]✓[/green] Configuration saved to: {config_file}") - - except Exception as e: - console.print(f"[red]Error: {e}[/red]") - raise typer.Exit(1) - - @api_app.command() def status( - api_key: Optional[str] = typer.Option( - None, - "--api-key", - help="Test a specific API key directly (without saving to config)" - ) + api_key: str | None = typer.Option( + None, "--api-key", "-a", + help="Test a specific API key directly (without saving to config)") ): """Check API configuration and connection status. - + This command displays: - Current API configuration (timeout, masked API key) - API connection test results - User information (User ID, token type) - Token expiration details (expires at, expires in human-readable format) - + Use this command to verify your API setup is working correctly and to check when your API token will expire. - + If --api-key is provided, it will test that specific key directly without saving it to the configuration. Otherwise, it loads the saved configuration from ~/.pynecore/api.toml. @@ -158,44 +75,46 @@ def status( base_url="https://api.pynesys.io", timeout=30 ) - + # Display test configuration info table = Table(title="API Key Test") table.add_column("Setting", style="cyan") table.add_column("Value", style="white") - + table.add_row("Timeout", f"{config.timeout}s") - table.add_row("API Key", f"{config.api_key[:8]}...{config.api_key[-4:]}" if len(config.api_key) > 12 else "***") + table.add_row("API Key", + f"{config.api_key[:8]}...{config.api_key[-4:]}" if len(config.api_key) > 12 else "***") table.add_row("Mode", "[yellow]Direct Test (not saved)[/yellow]") - + console.print(table) else: # Load configuration from file config = ConfigManager.load_config(None) - + # Display configuration info table = Table(title="API Configuration") table.add_column("Setting", style="cyan") table.add_column("Value", style="white") - + table.add_row("Timeout", f"{config.timeout}s") - table.add_row("API Key", f"{config.api_key[:8]}...{config.api_key[-4:]}" if len(config.api_key) > 12 else "***") + table.add_row("API Key", + f"{config.api_key[:8]}...{config.api_key[-4:]}" if len(config.api_key) > 12 else "***") table.add_row("Mode", "[green]Saved Configuration[/green]") - + console.print(table) - + # Test connection with Progress( - SpinnerColumn(), - TextColumn("[progress.description]{task.description}"), - console=console + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console ) as progress: task = progress.add_task("Testing API connection...", total=None) - + try: - client = PynesysAPIClient(config.api_key, config.base_url, config.timeout) - result = client.verify_token_sync() - + client = APIClient(config.api_key, config.base_url, config.timeout) + result = client.verify_token() + if result.valid: progress.update(task, description="[green]Connection successful![/green]") console.print(f"[green]✓[/green] API connection is working") @@ -210,15 +129,15 @@ def status( else: progress.update(task, description="[red]Connection failed[/red]") console.print(f"[red]✗[/red] API connection failed: {result.message}") - + except AuthError: progress.update(task, description="[red]Authentication failed[/red]") console.print("[red]✗[/red] Authentication failed. API key may be invalid or expired.") - + except APIError as e: progress.update(task, description="[red]API error[/red]") console.print(f"[red]✗[/red] API error: {e}") - + except ValueError as e: error_msg = str(e) if not api_key and ("No configuration file found" in error_msg or "Configuration file not found" in error_msg): @@ -226,7 +145,8 @@ def status( console.print("[yellow]⚠️ No API configuration found[/yellow]") console.print() console.print("To get started with PyneSys API:") - console.print("1. 🌐 Visit [blue][link=https://pynesys.io]https://pynesys.io[/link][/blue] to get your API key") + console.print( + "1. 🌐 Visit [blue][link=https://pynesys.io]https://pynesys.io[/link][/blue] to get your API key") console.print("2. 🔧 Run [cyan]pyne api configure[/cyan] to set up your configuration") console.print("3. 🧪 Or test a key directly with [cyan]pyne api status --api-key YOUR_KEY[/cyan]") console.print() @@ -235,48 +155,7 @@ def status( console.print(f"[red]Configuration error: {e}[/red]") console.print("[yellow]Hint:[/yellow] Use 'pyne api configure' to set up your API configuration.") raise typer.Exit(1) - + except Exception as e: console.print(f"[red]Error: {e}[/red]") raise typer.Exit(1) - - -@api_app.command() -def reset( - force: bool = typer.Option( - False, - "--force", - help="Skip confirmation prompt" - ) -): - """Reset API configuration by removing the configuration file. - - This command will: - - Delete the default API configuration file (workdir/config/api.toml) - - Remove all stored API settings (API key, timeout) - - Require you to run 'pyne api configure' again to set up the API - - Use --force to skip the confirmation prompt. - This is useful when you want to start fresh with API configuration - or switch to a different API key. - """ - config_file = ConfigManager.get_default_config_path() - - if not config_file.exists(): - console.print(f"[yellow]No configuration file found at: {config_file}[/yellow]") - return - - if not force: - typer.confirm( - f"Are you sure you want to delete the configuration file at {config_file}?", - abort=True - ) - - try: - config_file.unlink() - console.print(f"[green]✓[/green] Configuration file deleted: {config_file}") - console.print("[yellow]Use 'pyne api configure' to set up a new configuration.[/yellow]") - - except Exception as e: - console.print(f"[red]Error deleting configuration: {e}[/red]") - raise typer.Exit(1) \ No newline at end of file diff --git a/src/pynecore/cli/commands/compile.py b/src/pynecore/cli/commands/compile.py index c496563..fcb5672 100644 --- a/src/pynecore/cli/commands/compile.py +++ b/src/pynecore/cli/commands/compile.py @@ -1,14 +1,14 @@ +import tomllib from pathlib import Path -from typing import Optional import typer from rich.console import Console from rich.progress import Progress, SpinnerColumn, TextColumn -from ..app import app -from ..utils.api_error_handler import handle_api_errors -from ...core.compiler import create_compilation_service -from ...utils.file_utils import preserve_mtime +from ..app import app, app_state +from ..utils.api_error_handler import APIErrorHandler +from pynecore.pynesys.compiler import PyneComp +from ...utils.file_utils import copy_mtime __all__ = [] @@ -18,32 +18,24 @@ # noinspection PyShadowingBuiltins @app.command() def compile( - script_path: Path = typer.Argument( - ..., - help="Path to Pine Script file (.pine extension)", - exists=True, - file_okay=True, - dir_okay=False, - readable=True + script: Path = typer.Argument( + ..., file_okay=True, dir_okay=False, + help="Path to Pine Script file (.pine extension)" ), - output: Optional[Path] = typer.Option( - None, - "--output", "-o", + output: Path | None = typer.Option( + None, "--output", "-o", help="Output Python file path (defaults to same name with .py extension)" ), strict: bool = typer.Option( - False, - "--strict", + False, "--strict", '-s', help="Enable strict compilation mode with enhanced error checking" ), force: bool = typer.Option( - False, - "--force", + False, "--force", "-f", help="Force recompilation even if output file is up-to-date" ), - api_key: Optional[str] = typer.Option( - None, - "--api-key", + api_key: str | None = typer.Option( + None, "--api-key", "-a", help="PyneSys API key (overrides configuration file)", envvar="PYNESYS_API_KEY" ) @@ -51,89 +43,65 @@ def compile( """ Compile Pine Script to Python using PyneSys API. - USAGE: - pyne compile # Compile single file - pyne compile --force # Force recompile even if up-to-date - pyne compile --strict # Enable strict compilation mode - pyne compile --output # Specify output file path - CONFIGURATION: Default config: workdir/config/api.toml Fallback config: ~/.pynecore/api.toml - Config format (TOML only): - [api] - pynesys_api_key = "your_api_key_here" - base_url = "https://api.pynesys.io/" # optional - timeout = 30 # optional, seconds - - SMART COMPILATION: - - Automatically skips recompilation if output is newer than input - - Use --force to override this behavior - - Preserves file modification timestamps - REQUIREMENTS: - Pine Script version 6 only (version 5 not supported) - - Valid PyneSys API key required - - Input file must have .pine extension - - Output defaults to same name with .py extension - - Use 'pyne api configure' to set up your API configuration. + - Valid PyneSys API key required (get one at [blue]https://app.pynesys.io[/]) """ - try: - # Create compilation service - compilation_service = create_compilation_service( - api_key=api_key, - ) + # Ensure .py extension + if script.suffix != ".pine": + script = script.with_suffix(".pine") + # Expand script path + if len(script.parts) == 1: + script = app_state.scripts_dir / script - # Determine output path - if output is None: - output = script_path.with_suffix('.py') - - # Check if compilation is needed (smart compilation) - if not compilation_service.needs_compilation(script_path, output) and not force: - console.print(f"[green]✓[/green] Output file is up-to-date: {output}") - console.print("[dim]Use --force to recompile anyway[/dim]") - return - - # Compile script - with handle_api_errors(console): - with Progress( - SpinnerColumn(), - TextColumn("[progress.description]{task.description}"), - console=console - ) as progress: - progress.add_task("Compiling Pine Script...", total=None) - - # Compile the .pine file - compiled_path = compilation_service.compile_file( - script_path, - output, - force=force, - strict=strict - ) - - # Preserve modification time from source file - preserve_mtime(script_path, output) - - console.print(f"[green]Compilation successful![/green] Your Pine Script is now ready: " - f"[cyan]{compiled_path}[/cyan]") - - except ValueError as e: - error_msg = str(e) - if "No configuration file found" in error_msg or "Configuration file not found" in error_msg: - # No API configuration found - show helpful setup message - console.print("[yellow]⚠️ No API configuration found[/yellow]") - console.print() - console.print("[bold]Quick setup (takes just few minutes):[/bold]") - console.print("1. 🌐 Get your API key at [blue][link=https://pynesys.io]https://pynesys.io[/link][/blue]") - console.print("2. 🔧 Run [cyan]pyne api configure[/cyan] to save your configuration") - console.print() - console.print("[dim]💬 Need assistance? Our docs are here: https://pynesys.io/docs[/dim]") - elif "this file format isn't supported" in error_msg: - # File format error - show the friendly message directly - console.print(f"[red]{e}[/red]") - else: - console.print(f"[red]Attention:[/red] {e}") + # Determine output path + if output is None: + output = script.with_suffix('.py') + + # Read api.toml configuration + api_config = {} + try: + with open(app_state.config_dir / 'api.toml', 'rb') as f: + api_config = tomllib.load(f)['api'] + except KeyError: + console.print("[red]Invalid API config file (api.toml)![/red]") raise typer.Exit(1) + except FileNotFoundError: + pass + + # Override API key if provided + if api_key: + api_config['api_key'] = api_key + + # Create the compiler instance + compiler = PyneComp(**api_config) + + # Check if compilation is needed (smart compilation) + if not compiler.needs_compilation(script, output) and not force: + console.print(f"[green]✓[/green] Output file is up-to-date: {output}") + console.print("[dim]Use --force to recompile anyway[/dim]") + return + + # Compile script + with APIErrorHandler(console): + with Progress( + SpinnerColumn(finished_text="[green]✓"), + TextColumn("[progress.description]{task.description}"), + console=console + ) as progress: + task = progress.add_task("Compiling Pine Script...", total=1) + + # Compile the .pine file to .py + out_path = compiler.compile(script, output, force=force, strict=strict) + + progress.update(task, completed=1) + + # Preserve modification time from source file + copy_mtime(script, output) + + console.print(f"The compiled script is located at: [cyan]{out_path}[/cyan]") diff --git a/src/pynecore/cli/commands/run.py b/src/pynecore/cli/commands/run.py index 649f1dd..e873440 100644 --- a/src/pynecore/cli/commands/run.py +++ b/src/pynecore/cli/commands/run.py @@ -1,10 +1,11 @@ -from pathlib import Path -from datetime import datetime import queue import threading import time import sys -from typing import Optional +import tomllib + +from pathlib import Path +from datetime import datetime from typer import Option, Argument, secho, Exit from rich.progress import (Progress, SpinnerColumn, TextColumn, BarColumn, @@ -19,8 +20,8 @@ from pynecore.core.syminfo import SymInfo from pynecore.core.script_runner import ScriptRunner -from ...core.compiler import create_compilation_service -from ...cli.utils.api_error_handler import handle_api_errors +from pynecore.pynesys.compiler import PyneComp +from ...cli.utils.api_error_handler import APIErrorHandler __all__ = [] @@ -80,31 +81,10 @@ def run( trade_path: Path | None = Option(None, "--trade", "-tp", help="Path to save the trade data", rich_help_panel="Out Path Options"), - force: bool = Option( - False, - "--force", - help="Force recompilation for .pine files (ignore smart compilation)", - rich_help_panel="Compilation Options" - ), - strict: bool = Option( - False, - "--strict", - help="Enable strict compilation mode for .pine files", - rich_help_panel="Compilation Options" - ), - api_key: Optional[str] = Option( - None, - "--api-key", - help="PyneSys API key (overrides configuration file)", - envvar="PYNESYS_API_KEY", - rich_help_panel="Compilation Options" - ), - config_path: Optional[Path] = Option( - None, - "--config", - help="Path to TOML configuration file", - rich_help_panel="Compilation Options" - ), + api_key: str | None = Option(None, "--api-key", "-a", + help="PyneSys API key for compilation (overrides configuration file)", + envvar="PYNESYS_API_KEY", + rich_help_panel="Compilation Options"), ): """ Run a script (.py or .pine) @@ -118,50 +98,23 @@ def run( they will be saved in the [italic]"workdir/output"[/] directory. [bold]Pine Script Support:[/bold] - When running a .pine file, it will be automatically compiled to Python before execution. - Use the compilation options (--force, --strict) to control the compilation process. - A valid PyneSys API key is required for Pine Script compilation. - - [bold]Smart Compilation:[/bold] - The system checks for changes in .pine files and only recompiles when necessary. - Use --force to bypass this check and force recompilation. + Also Pine Script (.pine) files could be automatically compiled to Python (.py) before execution, if the + file is newer than the [italic]py[/] file or if the [italic].py[/] file doesn't exist. The compiled [italic].py[/] file will be saved + into the same folder as the original [italic].pine[/] file. + A valid [bold]PyneSys API[/bold] key is required for Pine Script compilation. You can get one at [blue]https://pynesys.io[/blue]. """ # noqa - # Support both .py and .pine files - if script.suffix not in [".py", ".pine"]: - # Check if the file exists with the given extension first - if len(script.parts) == 1: - full_script_path = app_state.scripts_dir / script - else: - full_script_path = script - - if full_script_path.exists(): - # File exists but has unsupported extension - console.print(f"[red]This file format isn't supported:[/red] {script.suffix}") - console.print("[yellow]✨ Currently supported formats:[/yellow] .py, .pine") - if script.suffix in [".ohlcv", ".csv", ".json"]: - console.print(f"[blue]💡 Heads up:[/blue] {script.suffix} files are data files, not executable scripts.") - raise Exit(1) - - # File doesn't exist, try .pine first, then .py - pine_script = script.with_suffix(".pine") - py_script = script.with_suffix(".py") - - if len(script.parts) == 1: - pine_script = app_state.scripts_dir / pine_script - py_script = app_state.scripts_dir / py_script - - if pine_script.exists(): - script = pine_script - elif py_script.exists(): - script = py_script - else: - script = py_script # Default to .py for error message - - # Expand script path if it's just a filename + # Expand script path if len(script.parts) == 1: script = app_state.scripts_dir / script + # If no script suffix, try .pine 1st + if script.suffix == "": + script = script.with_suffix(".pine") + # If doesn't exist, try .py + if not script.exists(): + script = script.with_suffix(".py") + # Check if script exists if not script.exists(): secho(f"Script file '{script}' not found!", fg="red", err=True) @@ -169,66 +122,44 @@ def run( # Handle .pine files - compile them first if script.suffix == ".pine": + # Read api.toml configuration + api_config = {} try: - # Create compilation service - compilation_service = create_compilation_service( - api_key=api_key, - config_path=config_path - ) + with open(app_state.config_dir / 'api.toml', 'rb') as f: + api_config = tomllib.load(f)['api'] + except KeyError: + console.print("[red]Invalid API config file (api.toml)![/red]") + raise Exit(1) + except FileNotFoundError: + pass + + # Override API key if provided + if api_key: + api_config['api_key'] = api_key - # Determine output path for compiled file - compiled_file = script.with_suffix(".py") + # Create the compiler instance + compiler = PyneComp(**api_config) - # Check if compilation is needed - if compilation_service.needs_compilation(script, compiled_file) or force: - with handle_api_errors(console): - with Progress( - SpinnerColumn(), + # Determine output path for compiled file + out_path = script.with_suffix(".py") + + # Check if compilation is needed + if compiler.needs_compilation(script, out_path): + with APIErrorHandler(console): + with Progress( + SpinnerColumn(finished_text="[green]✓"), TextColumn("[progress.description]{task.description}"), console=console - ) as progress: - progress.add_task("Compiling Pine Script...", total=None) - - # Compile the .pine file - compiled_path = compilation_service.compile_file( - script, - compiled_file, - force=force, - strict=strict - ) - - console.print(f"[green]Compilation successful![/green] Ready to run: [cyan]{compiled_path}[/cyan]") - compiled_file = compiled_path - - else: - console.print(f"[green]⚡ Using cached version:[/green] [cyan]{compiled_file}[/cyan]") - console.print("[dim]Use --force to recompile[/dim]") - - # Update script to point to the compiled file - script = compiled_file - - except ValueError as e: - error_msg = str(e) - if "No configuration file found" in error_msg or "Configuration file not found" in error_msg: - console.print("[yellow]⚠️ No API configuration found[/yellow]") - console.print() - console.print("[bold]Quick setup (takes few minutes):[/bold]") - console.print("1. 🌐 Get your API key at " - "[blue][link=https://pynesys.io]https://pynesys.io[/link][/blue]") - console.print("2. 🔧 Run [cyan]pyne api configure[/cyan] to save your configuration") - console.print() - console.print("[dim]💬 Need assistance? Our docs are here: https://pynesys.io/docs[/dim]") - else: - console.print(f"[red] Attention:[/red] {e}") - raise Exit(1) + ) as progress: + task = progress.add_task("Compiling Pine Script...", total=1) - # Ensure we have a .py file at this point - if script.suffix != ".py": - console.print(f"[red]This file format isn't supported:[/red] {script.suffix}") - console.print("[yellow]✨ Currently supported formats:[/yellow] .py, .pine") - if script.suffix in [".ohlcv", ".csv", ".json"]: - console.print(f"[blue]💡 Heads up:[/blue] {script.suffix} files are data files, not executable scripts.") - raise Exit(1) + # Compile the .pine file + compiler.compile(script, out_path) + + progress.update(task, completed=1) + + # Update script to point to the compiled file + script = out_path # Check file format and extension if data.suffix == "": diff --git a/src/pynecore/cli/utils/api_error_handler.py b/src/pynecore/cli/utils/api_error_handler.py index c3e6a99..8e3f4f5 100644 --- a/src/pynecore/cli/utils/api_error_handler.py +++ b/src/pynecore/cli/utils/api_error_handler.py @@ -3,33 +3,24 @@ from rich.console import Console from typer import Exit -from pynecore.api.exceptions import APIError, AuthError, RateLimitError, CompilationError - - -def handle_api_errors(console: Console): - """Context manager for handling API errors with user-friendly messages. - - Usage: - with handle_api_errors(console): - # API operations that might raise exceptions - pass - """ - return APIErrorHandler(console) +from pynecore.pynesys.api import APIError, AuthError, RateLimitError, CompilationError class APIErrorHandler: """Context manager that provides centralized API error handling.""" - - def __init__(self, console: Console): + + def __init__(self, console: Console | None = None): + if not console: + console = Console() self.console = console - + def __enter__(self): return self - + def __exit__(self, exc_type, exc_value, traceback): if exc_type is None: return False - + if exc_type == CompilationError: self._handle_compilation_error(exc_value) elif exc_type == AuthError: @@ -40,102 +31,115 @@ def __exit__(self, exc_type, exc_value, traceback): self._handle_api_error(exc_value) else: return False # Let other exceptions propagate - + raise Exit(1) - + def _handle_compilation_error(self, e: CompilationError): """Handle compilation-specific errors.""" - self.console.print(f"[red]❌ Oops! Compilation encountered an issue:[/red] {str(e)}") + self.console.print(f"[red]❌ Oops! Compilation encountered an issue:[/red] {str(e)}") if e.validation_errors: self.console.print("[red]Validation errors:[/red]") for error in e.validation_errors: self.console.print(f" [red]• {error}[/red]") - + def _handle_auth_error(self, e: AuthError): """Handle authentication errors.""" - self.console.print(f"[red]🔐 Authentication issue:[/red] {str(e)}") - self.console.print("[yellow]🚀 Quick fix:[/yellow] Run [cyan]'pyne api configure'[/cyan] to set up your API key and get back on track!") - + self.console.print(f"[red]🔐 Authentication issue:[/red] {str(e)}") + self.console.print("[yellow]🚀 To fix:[/yellow] Check [cyan]api_key[/cyan] in [cyan]api.toml[/cyan] " + "in your working directory") + def _handle_rate_limit_error(self, e: RateLimitError): """Handle rate limit errors.""" - self.console.print("[red]🚦 Rate Limit Exceeded:[/red] You've hit your compilation limit") + self.console.print("[red]🚦 Rate Limit Exceeded:[/red] You've hit your compilation limit") if e.retry_after: - self.console.print(f"[yellow]⏰ Please try again in {e.retry_after} seconds[/yellow]") - self.console.print("[yellow]💡 To increase your limits, consider upgrading your subscription at [blue][link=https://pynesys.io]https://pynesys.io[/link][/blue]") - self.console.print("[blue]💎 Upgrade at [link=https://pynesys.io/pricing]https://pynesys.io/pricing[/link][/blue]") - + self.console.print(f"[yellow]⏰ Please try again in {e.retry_after} seconds[/yellow]") + self.console.print( + "[yellow]💡 To increase your limits, consider upgrading your subscription at " + "[link=https://pynesys.io]https://pynesys.io[/link]") + self.console.print( + "💎 Upgrade at [link=https://pynesys.io/pricing]https://pynesys.io/pricing[/link]") + def _handle_api_error(self, e: APIError): """Handle general API errors with specific status code handling.""" error_msg = str(e).lower() - + # Handle specific API error scenarios based on HTTP status codes if "400" in error_msg or "bad request" in error_msg: if "compilation fails" in error_msg or "script is too large" in error_msg: - self.console.print("[red]📝 Script Issue:[/red] Your Pine Script couldn't be compiled") - self.console.print("[yellow]💡 Common fixes:[/yellow]") + self.console.print("[red]📝 Script Issue:[/red] Your Pine Script couldn't be compiled") + self.console.print("[yellow]💡 Common fixes:[/yellow]") self.console.print(" • Check if your script is too large (try breaking it into smaller parts)") self.console.print(" • Verify your Pine Script syntax is correct") self.console.print(" • Make sure you're using Pine Script v6 syntax") else: self.console.print(f"[red]⚠️ Request Error:[/red] {str(e)}") - self.console.print("[yellow]💡 This usually means there's an issue with the request format[/yellow]") - + self.console.print("[yellow]💡 This usually means there's an issue with the request format[/yellow]") + elif "401" in error_msg or "authentication" in error_msg or "no permission" in error_msg: self.console.print("[red]🔐 Authentication Failed:[/red] Your API credentials aren't working") - self.console.print("[yellow]🚀 Quick fixes:[/yellow]") + self.console.print("[yellow]🚀 Quick fixes:[/yellow]") self.console.print(" • Check if your API key is valid and active") self.console.print(" • Verify your token type is allowed for compilation") - self.console.print("[blue]🔑 Get a new API key at [link=https://pynesys.io]https://pynesys.io[/link][/blue]") - self.console.print("[blue]⚙️ Then run [cyan]'pyne api configure'[/cyan] to update your configuration[/blue]") - + self.console.print( + "🔑 Get a new API key at [link=https://pynesys.io]https://pynesys.io[/link]") + self.console.print( + "⚙️ Then run [cyan]'pyne api configure'[/cyan] to update your configuration") + elif "404" in error_msg or "not found" in error_msg: - self.console.print("[red]🔍 Not Found:[/red] The API endpoint or user wasn't found") - self.console.print("[yellow]💡 This might indicate:[/yellow]") + self.console.print("[red]🔍 Not Found:[/red] The API endpoint or user wasn't found") + self.console.print("[yellow]💡 This might indicate:[/yellow]") self.console.print(" • Your account may not exist or be accessible") self.console.print(" • There might be a temporary service issue") - self.console.print("[blue]📞 Contact support if this persists: [link=https://pynesys.io/support]https://pynesys.io/support[/link][/blue]") - + self.console.print( + "📞 Contact support if this persists: " + "[link=https://pynesys.io/support]https://pynesys.io/support[/link]") + elif "422" in error_msg or "validation error" in error_msg: - self.console.print("[red]📋 Validation Error:[/red] Your request data has validation issues") - self.console.print("[yellow]💡 Common causes:[/yellow]") + self.console.print("[red]📋 Validation Error:[/red] Your request data has validation issues") + self.console.print("[yellow]💡 Common causes:[/yellow]") self.console.print(" • Invalid Pine Script syntax or structure") self.console.print(" • Missing required parameters") self.console.print(" • Incorrect data format") self.console.print(f"[dim]Details: {str(e)}[/dim]") - + elif "429" in error_msg or "rate limit" in error_msg or "too many requests" in error_msg: - self.console.print("[red]🚦 Rate Limit Exceeded:[/red] You've hit your compilation limit") - self.console.print("[yellow]⏰ What you can do:[/yellow]") + self.console.print("[red]🚦 Rate Limit Exceeded:[/red] You've hit your compilation limit") + self.console.print("[yellow]⏰ What you can do:[/yellow]") self.console.print(" • Wait a bit before trying again") self.console.print(" • Consider upgrading your plan for higher limits") - self.console.print("[blue]💎 Upgrade at [link=https://pynesys.io/pricing]https://pynesys.io/pricing[/link][/blue]") - + self.console.print( + "💎 Upgrade at [link=https://pynesys.io/pricing]https://pynesys.io/pricing[/link]") + elif "500" in error_msg or "server" in error_msg or "internal" in error_msg: - self.console.print("[red]🔧 Server Error:[/red] Something went wrong on our end") - self.console.print("[yellow]😅 Don't worry, it's not you![/yellow]") + self.console.print("[red]🔧 Server Error:[/red] Something went wrong on our end") + self.console.print("[yellow]😅 Don't worry, it's not you![/yellow]") self.console.print(" • This is a temporary server issue") self.console.print(" • Please try again in a few moments") - self.console.print("[blue]📊 Check service status: [link=https://status.pynesys.io]https://status.pynesys.io[/link][/blue]") - + self.console.print( + "📊 Check service status: [link=https://status.pynesys.io]https://status.pynesys.io[/link]") + elif "unsupported pinescript version" in error_msg: - self.console.print("[red]📌 Version Issue:[/red] Your Pine Script version isn't supported") + self.console.print("[red]📌 Version Issue:[/red] Your Pine Script version isn't supported") if "version 5" in error_msg: - self.console.print("[yellow]🔄 Pine Script v5 → v6 Migration:[/yellow]") + self.console.print("[yellow]🔄 Pine Script v5 → v6 Migration:[/yellow]") self.console.print(" • Update your script to Pine Script version 6") self.console.print(" • Most v5 scripts need minimal changes") - self.console.print("[blue]📖 Migration guide: [link=https://www.tradingview.com/pine-script-docs/en/v6/migration_guides/v5_to_v6_migration_guide.html]Pine Script v5→v6 Guide[/link][/blue]") + self.console.print( + "📖 Migration guide: [link=https://www.tradingview.com/pine-script-docs/en/v6/" + "migration_guides/v5_to_v6_migration_guide.html]Pine Script v5→v6 Guide[/link]") else: - self.console.print("[yellow]💡 Only Pine Script version 6 is currently supported[/yellow]") - + self.console.print("[yellow]💡 Only Pine Script version 6 is currently supported[/yellow]") + elif "api key" in error_msg: - self.console.print("[red]🔑 API Key Issue:[/red] There's a problem with your API key") - self.console.print("[blue]🔑 Get your API key at [link=https://pynesys.io]https://pynesys.io[/link][/blue]") - self.console.print("[blue]⚙️ Then run [cyan]'pyne api configure'[/cyan] to set up your configuration[/blue]") - + self.console.print("[red]🔑 API Key Issue:[/red] There's a problem with your API key") + self.console.print("🔑 Get your API key at [link=https://pynesys.io]https://pynesys.io[/link]") + self.console.print( + "⚙️ Then run [cyan]'pyne api configure'[/cyan] to set up your configuration") + else: # Generic API error fallback self.console.print(f"[red]🌐 API Error:[/red] {str(e)}") self.console.print("[yellow]💡 If this persists, please check:[/yellow]") self.console.print(" • Your internet connection") self.console.print(" • API service status") - self.console.print("[blue]📞 Need help? [link=https://pynesys.io/support]Contact Support[/link][/blue]") \ No newline at end of file + self.console.print("📞 Need help? [link=https://pynesys.io/support]Contact Support[/link]") diff --git a/src/pynecore/pynesys/__init__.py b/src/pynecore/pynesys/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/pynecore/pynesys/api.py b/src/pynecore/pynesys/api.py new file mode 100644 index 0000000..43ba9db --- /dev/null +++ b/src/pynecore/pynesys/api.py @@ -0,0 +1,346 @@ +""" +PyneCore API client +""" +from typing import Any + +import json + +from datetime import datetime + +from dataclasses import dataclass + +import urllib.request +import urllib.parse +import urllib.error + + +# +# API Response Models +# + +@dataclass +class TokenValidationResponse: + """Response from token validation endpoint.""" + valid: bool + message: str + user_id: str | None = None + token_type: str | None = None + expiration: datetime | None = None + expires_at: datetime | None = None + expires_in: int | None = None + raw_response: dict[str, Any] | None = None + + +@dataclass +class CompileResponse: + """Response from script compilation endpoint.""" + success: bool + compiled_code: str | None = None + error_message: str | None = None + error: str | None = None + validation_errors: list[dict[str, Any]] | None = None + warnings: list[str] | None = None + details: list[str] | None = None + status_code: int | None = None + raw_response: dict[str, Any] | None = None + + @property + def has_validation_errors(self) -> bool: + """Check if response contains validation errors.""" + return bool(self.validation_errors) + + @property + def is_rate_limited(self) -> bool: + """Check if response indicates rate limiting.""" + return self.status_code == 429 + + @property + def is_auth_error(self) -> bool: + """Check if response indicates authentication error.""" + return self.status_code == 401 + + +# +# Exceptions +# + +class APIError(Exception): + """Base exception for API-related errors.""" + + def __init__(self, message: str = "", status_code: int | None = None, + response_data: dict[str, Any] | None = None): + super().__init__(message) + self.status_code = status_code + self.response_data = response_data or {} + + +class AuthError(APIError): + """Authentication-related errors (401, invalid token, etc.).""" + pass + + +class RateLimitError(APIError): + """Rate limiting errors (429).""" + + def __init__(self, message: str, retry_after: int | None = None, **kwargs): + super().__init__(message, **kwargs) + self.retry_after = retry_after + + +class CompilationError(APIError): + """Compilation-related errors (400, 422).""" + + def __init__(self, message: str, validation_errors: list | None = None, **kwargs): + super().__init__(message, **kwargs) + self.validation_errors = validation_errors or [] + + +class NetworkError(APIError): + """Network-related errors (timeouts, connection issues).""" + pass + + +class ServerError(APIError): + """Server-side errors (500, 502, etc.).""" + pass + + +# +# API Client +# + +class APIClient: + """ + API Client for interacting with PyneSys API + """ + + def __init__(self, api_key: str, base_url: str = "https://api.pynesys.io", timeout: int = 30): + """ + Initialize the API client. + + :param api_key: PyneSys API key + :param base_url: Base URL for the API + :param timeout: Request timeout in seconds + """ + if api_key is None or not api_key.strip(): + raise ValueError("API key is required") + + self.api_key = api_key + self.base_url = base_url.rstrip("/") + self.timeout = timeout + + def _make_request( + self, + method: str, + endpoint: str, + data: dict[str, Any] | None = None, + headers: dict[str, str] | None = None + ) -> urllib.request.Request: + """ + Create a urllib request object. + + :param method: HTTP method (GET, POST, etc.) + :param endpoint: API endpoint + :param data: Request data + :param headers: Additional headers + :return: Configured request object + """ + url = f"{self.base_url}/{endpoint}" + + # Default headers + req_headers = { + "Authorization": f"Bearer {self.api_key}", + "User-Agent": "PyneCore-API-Client", + } + + if headers: + req_headers.update(headers) + + # Handle data encoding + encoded_data = None + if data and method != "GET": + if "Content-Type" in req_headers and "json" in req_headers["Content-Type"]: + encoded_data = json.dumps(data).encode('utf-8') + else: + encoded_data = urllib.parse.urlencode(data).encode('utf-8') + elif data and method == "GET": + # For GET requests, add data as query parameters + query_string = urllib.parse.urlencode(data) + url = f"{url}?{query_string}" + + request = urllib.request.Request( + url, + data=encoded_data, + headers=req_headers, + method=method + ) + + return request + + def verify_token(self) -> TokenValidationResponse: + """ + Verify API token validity. + + :return: TokenValidationResponse with validation details + :raises AuthError: If token is invalid + :raises NetworkError: If network request fails + :raises APIError: For other API errors + """ + try: + request = self._make_request( + "GET", + "auth/verify-token", + data={"token": self.api_key} + ) + + with urllib.request.urlopen(request, timeout=self.timeout) as response: + data = json.loads(response.read().decode('utf-8')) + return TokenValidationResponse( + valid=data.get("valid", False), + message=data.get("message", ""), + user_id=data.get("user_id"), + token_type=data.get("token_type"), + expiration=(datetime.fromisoformat(data["expiration"].replace("Z", "+00:00")) + if data.get("expiration") else None), + expires_at=(datetime.fromisoformat(data["expires_at"].replace("Z", "+00:00")) + if data.get("expires_at") else None), + expires_in=data.get("expires_in"), + raw_response=data, + ) + + except urllib.error.HTTPError as e: + self._handle_http_error(e) + except urllib.error.URLError as e: + raise NetworkError(f"Network error during token verification: {e}") + except Exception as e: + if not isinstance(e, APIError): + raise APIError(f"Unexpected error during token verification: {e}") + else: + raise + + raise APIError("Unexpected error during token verification.") + + def compile_script( + self, + script: str, + strict: bool = False + ) -> CompileResponse: + """ + Compile Pine Script to Python via API. + + :param script: Pine Script code to compile + :param strict: Whether to use strict compilation mode + :return: CompileResponse with compiled code or error details + :raises AuthError: If authentication fails + :raises RateLimitError: If rate limit is exceeded + :raises CompilationError: If compilation fails + :raises NetworkError: If network request fails + :raises APIError: For other API errors + """ + try: + # Prepare form data + data = { + "script": script, + "strict": str(strict).lower() + } + + request = self._make_request( + "POST", + "compiler/compile", + data=data, + headers={"Content-Type": "application/x-www-form-urlencoded"} + ) + + with urllib.request.urlopen(request, timeout=self.timeout) as response: + # Success - return compiled code + compiled_code = response.read().decode('utf-8') + return CompileResponse( + success=True, + compiled_code=compiled_code, + status_code=200 + ) + + except urllib.error.HTTPError as e: + # Handle error responses + return self._handle_compile_http_error(e) + except urllib.error.URLError as e: + raise NetworkError(f"Network error during compilation: {e}") + except Exception as e: + if not isinstance(e, APIError): + raise APIError(f"Unexpected error during compilation: {e}") + else: + raise + + @staticmethod + def _handle_http_error(error: urllib.error.HTTPError) -> None: + """ + Handle HTTP error responses. + + :param error: HTTPError object + :raises: Appropriate exception based on status code + """ + status_code = error.code + + try: + error_content = error.read().decode('utf-8') + error_data = json.loads(error_content) + message = error_data.get("message", error_content) + except (json.JSONDecodeError, ValueError): + message = error.reason or f"HTTP {status_code} error" + + if status_code == 401: + raise AuthError(message, status_code=status_code) + elif status_code == 429: + retry_after = error.headers.get("Retry-After") + raise RateLimitError( + message, + status_code=status_code, + retry_after=int(retry_after) if retry_after else None + ) + elif status_code >= 500: + raise ServerError(message, status_code=status_code) + else: + raise APIError(message, status_code=status_code) + + def _handle_compile_http_error(self, error: urllib.error.HTTPError) -> CompileResponse: + """ + Handle compilation error responses. + + :param error: HTTPError object + :return: CompileResponse with error details + :raises CompilationError: For compilation-related errors (422) + :raises: Other exceptions for authentication, rate limiting, etc. + """ + status_code = error.code + + try: + error_content = error.read().decode('utf-8') + error_data = json.loads(error_content) + except (json.JSONDecodeError, ValueError): + error_data = {} + error_content = error.reason or f"HTTP {status_code} error" + + # Extract error message + if "detail" in error_data and isinstance(error_data["detail"], list): + # Validation error format (422) + validation_errors = error_data["detail"] + error_message = "Validation errors occurred" + else: + validation_errors = None + error_message = error_data.get("message", error_content) + + # For compilation errors (422), raise CompilationError + if status_code == 422: + raise CompilationError(error_message, status_code=status_code, validation_errors=validation_errors) + + # For other errors, use the general error handler + self._handle_http_error(error) + + # This should never be reached + return CompileResponse( + success=False, + error_message=error_message, + validation_errors=validation_errors, + status_code=status_code, + raw_response=error_data + ) diff --git a/src/pynecore/core/compiler.py b/src/pynecore/pynesys/compiler.py similarity index 50% rename from src/pynecore/core/compiler.py rename to src/pynecore/pynesys/compiler.py index c57745e..6289a09 100644 --- a/src/pynecore/core/compiler.py +++ b/src/pynecore/pynesys/compiler.py @@ -6,51 +6,35 @@ import sys from pathlib import Path -from ..api.client import PynesysAPIClient -from ..api.config import APIConfig, ConfigManager -from ..api.exceptions import APIError, AuthError, RateLimitError, CompilationError -from ..utils.mtime_utils import file_needs_compilation +from pynecore.pynesys.api import APIClient +from pynecore.pynesys.api import APIError, AuthError, RateLimitError, CompilationError +from pynecore.utils.file_utils import is_updated -class CompilationService: +class PyneComp: """ - Core compilation service for programmatic use. - - The CLI should use this service rather than implementing compilation logic directly. - This ensures all compilation functionality is available programmatically. + COmpiler through the PyneSys API. """ - def __init__(self, api_client: PynesysAPIClient | None = None, config: APIConfig | None = None): + api_client: APIClient + + def __init__(self, api_key, base_url="https://api.pynesys.io", timeout=30): """ Initialize the compilation service. - :param api_client: Optional pre-configured API client - :param config: Optional API configuration (will load from default if not provided) + :param api_key: PyneSys API key + :param base_url: Base URL for the API + :param timeout: Request timeout in seconds """ - if api_client: - self.api_client = api_client - else: - # Load config if not provided - if not config: - config = ConfigManager.load_config() - self.api_client = PynesysAPIClient( - api_key=config.api_key, - base_url=config.base_url, - timeout=config.timeout - ) - - def compile_file( - self, - pine_file_path: Path, - output_file_path: Path | None = None, - force: bool = False, - strict: bool = False - ) -> Path: + self.api_client = APIClient(api_key=api_key, base_url=base_url, timeout=timeout) + + def compile(self, pine_path: Path, output_path: Path | None = None, + force: bool = False, strict: bool = False) -> Path: """ Compile a .pine file to Python. - :param pine_file_path: Path to the .pine file - :param output_file_path: Optional output path (defaults to .py extension) + :param pine_path: Path to the .pine file + :param output_path: Optional output path (defaults to .py extension) :param force: Force recompilation even if file hasn't changed :param strict: Enable strict compilation mode :return: Path to the compiled .py file @@ -59,32 +43,31 @@ def compile_file( :raises APIError: If API request fails """ # Validate input file - if not pine_file_path.exists(): - raise FileNotFoundError(f"Pine file not found: {pine_file_path}") + if not pine_path.exists(): + raise FileNotFoundError(f"Pine file not found: {pine_path}") - if pine_file_path.suffix != '.pine': - raise ValueError( - f"This file format isn't supported: {pine_file_path.suffix}. " - f"Only .pine files can be compiled! ✨ Try using a .pine file instead.") + if pine_path.suffix != '.pine': + raise ValueError(f"This file format isn't supported: {pine_path.suffix}. " + f"Only .pine files can be compiled!") # Determine output path - if output_file_path is None: - output_file_path = pine_file_path.with_suffix('.py') + if output_path is None: + output_path = pine_path.with_suffix('.py') # Check if compilation is needed (unless forced) - if not force and not self.needs_compilation(pine_file_path, output_file_path): - return output_file_path + if not force and not self.needs_compilation(pine_path, output_path): + return output_path # Read Pine Script content try: - with open(pine_file_path, 'r', encoding='utf-8') as f: + with open(pine_path, 'r', encoding='utf-8') as f: script_content = f.read() except IOError as e: - raise IOError(f"Error reading Pine file {pine_file_path}: {e}") + raise IOError(f"Error reading Pine file {pine_path}: {e}") # Compile via API try: - response = self.api_client.compile_script_sync(script_content, strict=strict) + response = self.api_client.compile_script(script_content, strict=strict) if not response.success: raise CompilationError( @@ -94,13 +77,12 @@ def compile_file( ) # Write compiled code to output file - output_file_path.parent.mkdir(parents=True, exist_ok=True) - with open(output_file_path, 'w', encoding='utf-8') as f: + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, 'w', encoding='utf-8') as f: f.write(response.compiled_code) # No need to update tracking info with mtime approach - - return output_file_path + return output_path except (APIError, AuthError, RateLimitError, CompilationError): # Re-raise API-related errors as-is @@ -118,7 +100,7 @@ def needs_compilation(pine_file_path: Path, output_file_path: Path) -> bool: :param output_file_path: Path to the compiled .py file :return: True if compilation is needed, False otherwise """ - return file_needs_compilation(pine_file_path, output_file_path) + return is_updated(pine_file_path, output_file_path) def compile_and_run( self, @@ -142,9 +124,9 @@ def compile_and_run( :raises APIError: If API request fails """ # Compile the file - compiled_file = self.compile_file( - pine_file_path=pine_file_path, - output_file_path=output_file_path, + compiled_file = self.compile( + pine_path=pine_file_path, + output_path=output_file_path, force=force, strict=strict ) @@ -160,25 +142,3 @@ def compile_and_run( return result.returncode except Exception as e: raise RuntimeError(f"Error executing compiled script: {e}") - - -def create_compilation_service( - api_key: str | None = None, - config_path: Path | None = None -) -> CompilationService: - """ - Factory function to create a CompilationService instance. - - :param api_key: Optional API key override - :param config_path: Optional path to config file - :return: Configured CompilationService instance - :raises ValueError: If no valid configuration found - """ - if api_key: - # Create API client with provided key - api_client = PynesysAPIClient(api_key=api_key) - return CompilationService(api_client=api_client) - else: - # Load configuration from file - config = ConfigManager.load_config(config_path) - return CompilationService(config=config) diff --git a/src/pynecore/utils/file_utils.py b/src/pynecore/utils/file_utils.py index 51d4847..13a3fae 100644 --- a/src/pynecore/utils/file_utils.py +++ b/src/pynecore/utils/file_utils.py @@ -1,97 +1,50 @@ -"""File modification time utilities for API operations.""" - +""" +File utilities +""" import os from pathlib import Path -from typing import Optional -from datetime import datetime -def get_file_mtime(file_path: Path) -> Optional[float]: - """Get file modification time as timestamp. - - Args: - file_path: Path to the file - - Returns: - Modification time as timestamp, or None if file doesn't exist +def copy_mtime(source_path: Path, target_path: Path) -> bool: + """Copy modification time from source to target file. + + :param source_path: Source file to copy mtime from + :param target_path: Target file to set mtime on + :return: True if successful, False otherwise """ try: - return file_path.stat().st_mtime + source_mtime = source_path.stat().st_mtime except (OSError, FileNotFoundError): - return None - + return False -def set_file_mtime(file_path: Path, mtime: float) -> bool: - """Set file modification time. - - Args: - file_path: Path to the file - mtime: Modification time as timestamp - - Returns: - True if successful, False otherwise - """ try: - os.utime(file_path, (mtime, mtime)) + os.utime(target_path, (source_mtime, source_mtime)) return True except (OSError, FileNotFoundError): return False -def is_file_newer(source_path: Path, target_path: Path) -> bool: - """Check if source file is newer than target file. - - Args: - source_path: Path to source file - target_path: Path to target file - - Returns: - True if source is newer than target, or if target doesn't exist +def is_updated(src_path: Path, dest_path: Path) -> bool: """ - source_mtime = get_file_mtime(source_path) - target_mtime = get_file_mtime(target_path) - - if source_mtime is None: - return False - - if target_mtime is None: - return True - - return source_mtime > target_mtime - + Check if a file is updated using modification time comparison. -def should_compile(pine_path: Path, py_path: Path, force: bool = False) -> bool: - """Determine if Pine Script should be compiled based on modification times. - - Args: - pine_path: Path to Pine Script file - py_path: Path to Python output file - force: Force compilation regardless of modification times - - Returns: - True if compilation should proceed + :param src_path: Path to the .pine file + :param dest_path: Path to the compiled .py file + :return: True if compilation is needed, False otherwise + :raises FileNotFoundError: If pine file doesn't exist + :raises OSError: If we can't get modification times """ - if force: - return True - - if not py_path.exists(): + + # If output file doesn't exist, compilation is needed + if not dest_path.exists(): return True - - return is_file_newer(pine_path, py_path) + # If source file doesn't exist, assume compilation is needed + if not src_path.exists(): + raise FileNotFoundError(f"Source file not found: {src_path}") + # Get modification times + src_mtime = os.path.getmtime(src_path) + dst_mtime = os.path.getmtime(dest_path) -def preserve_mtime(source_path: Path, target_path: Path) -> bool: - """Copy modification time from source to target file. - - Args: - source_path: Source file to copy mtime from - target_path: Target file to set mtime on - - Returns: - True if successful, False otherwise - """ - source_mtime = get_file_mtime(source_path) - if source_mtime is None: - return False - - return set_file_mtime(target_path, source_mtime) \ No newline at end of file + # If source is newer than output, compilation is needed + return src_mtime > dst_mtime diff --git a/src/pynecore/utils/mtime_utils.py b/src/pynecore/utils/mtime_utils.py deleted file mode 100644 index afd3982..0000000 --- a/src/pynecore/utils/mtime_utils.py +++ /dev/null @@ -1,35 +0,0 @@ -"""File modification time utilities for smart compilation of .pine files.""" - -import os -from pathlib import Path - - -def file_needs_compilation(pine_file_path: Path, output_file_path: Path) -> bool: - """Check if a .pine file needs compilation based on modification time. - - Args: - pine_file_path: Path to the .pine file - output_file_path: Path to the compiled .py file - - Returns: - True if compilation is needed, False otherwise - """ - # If output file doesn't exist, compilation is needed - if not output_file_path.exists(): - return True - - # If source file doesn't exist, assume compilation is needed - if not pine_file_path.exists(): - return True - - try: - # Get modification times - source_mtime = os.path.getmtime(pine_file_path) - output_mtime = os.path.getmtime(output_file_path) - - # If source is newer than output, compilation is needed - return source_mtime > output_mtime - - except OSError: - # If we can't get modification times, assume compilation is needed - return True \ No newline at end of file From eb3350902a27f9c85b2a72a71eba68ac9ef0eec8 Mon Sep 17 00:00:00 2001 From: Adam Wallner Date: Sat, 2 Aug 2025 23:36:27 +0200 Subject: [PATCH 12/31] feat(api): add local JWT token validation and enhance CLI usage display - Add verify_token_local() method for offline JWT validation without server requests - Support both standard 'exp' and custom 'e' expiration fields in JWT tokens - Add get_usage() and validate_api_key() helper methods to PyneComp class - Enhance compile command to show API usage statistics and key expiration info - Add --usage flag to display statistics after compilation - Remove deprecated api.py CLI command module - Improve error handling with informative user messages --- src/pynecore/cli/commands/__init__.py | 4 +- src/pynecore/cli/commands/api.py | 161 -------------------------- src/pynecore/cli/commands/compile.py | 119 ++++++++++++++++--- src/pynecore/pynesys/api.py | 145 +++++++++++++++++++++++ src/pynecore/pynesys/compiler.py | 62 +++------- 5 files changed, 262 insertions(+), 229 deletions(-) delete mode 100644 src/pynecore/cli/commands/api.py diff --git a/src/pynecore/cli/commands/__init__.py b/src/pynecore/cli/commands/__init__.py index fd3aece..65c0703 100644 --- a/src/pynecore/cli/commands/__init__.py +++ b/src/pynecore/cli/commands/__init__.py @@ -9,9 +9,9 @@ from ...providers import available_providers # Import commands -from . import run, data, compile, benchmark, api +from . import run, data, compile, benchmark -__all__ = ['run', 'data', 'compile', 'benchmark', 'api'] +__all__ = ['run', 'data', 'compile', 'benchmark'] @app.callback() diff --git a/src/pynecore/cli/commands/api.py b/src/pynecore/cli/commands/api.py deleted file mode 100644 index 349c59d..0000000 --- a/src/pynecore/cli/commands/api.py +++ /dev/null @@ -1,161 +0,0 @@ -"""API configuration and management commands.""" -import typer -from rich.console import Console -from rich.table import Table -from rich.progress import Progress, SpinnerColumn, TextColumn - -from ..app import app -from pynecore.pynesys.api import APIClient, APIError, AuthError - - -def format_expires_in(seconds: int) -> str: - """Format expires_in seconds into human-readable format. - - Args: - seconds: Number of seconds until expiration - - Returns: - Formatted string like "2 days, 5 hours, 30 minutes" - """ - if seconds <= 0: - return "Expired" - - days = seconds // 86400 - hours = (seconds % 86400) // 3600 - minutes = (seconds % 3600) // 60 - - parts = [] - if days > 0: - parts.append(f"{days} day{'s' if days != 1 else ''}") - if hours > 0: - parts.append(f"{hours} hour{'s' if hours != 1 else ''}") - if minutes > 0: - parts.append(f"{minutes} minute{'s' if minutes != 1 else ''}") - - if not parts: - return "Less than 1 minute" - - return ", ".join(parts) - - -__all__ = [] - -console = Console() -api_app = typer.Typer( - help="PyneSys API management commands for connection testing.") -app.add_typer(api_app, name="api") - - -@api_app.command() -def status( - api_key: str | None = typer.Option( - None, "--api-key", "-a", - help="Test a specific API key directly (without saving to config)") -): - """Check API configuration and connection status. - - This command displays: - - Current API configuration (timeout, masked API key) - - API connection test results - - User information (User ID, token type) - - Token expiration details (expires at, expires in human-readable format) - - Use this command to verify your API setup is working correctly - and to check when your API token will expire. - - If --api-key is provided, it will test that specific key directly without - saving it to the configuration. Otherwise, it loads the saved configuration - from ~/.pynecore/api.toml. - """ - try: - if api_key: - # Test the provided API key directly - config = APIConfig( - api_key=api_key, - base_url="https://api.pynesys.io", - timeout=30 - ) - - # Display test configuration info - table = Table(title="API Key Test") - table.add_column("Setting", style="cyan") - table.add_column("Value", style="white") - - table.add_row("Timeout", f"{config.timeout}s") - table.add_row("API Key", - f"{config.api_key[:8]}...{config.api_key[-4:]}" if len(config.api_key) > 12 else "***") - table.add_row("Mode", "[yellow]Direct Test (not saved)[/yellow]") - - console.print(table) - else: - # Load configuration from file - config = ConfigManager.load_config(None) - - # Display configuration info - table = Table(title="API Configuration") - table.add_column("Setting", style="cyan") - table.add_column("Value", style="white") - - table.add_row("Timeout", f"{config.timeout}s") - table.add_row("API Key", - f"{config.api_key[:8]}...{config.api_key[-4:]}" if len(config.api_key) > 12 else "***") - table.add_row("Mode", "[green]Saved Configuration[/green]") - - console.print(table) - - # Test connection - with Progress( - SpinnerColumn(), - TextColumn("[progress.description]{task.description}"), - console=console - ) as progress: - task = progress.add_task("Testing API connection...", total=None) - - try: - client = APIClient(config.api_key, config.base_url, config.timeout) - result = client.verify_token() - - if result.valid: - progress.update(task, description="[green]Connection successful![/green]") - console.print(f"[green]✓[/green] API connection is working") - console.print(f"[blue]User ID:[/blue] {result.user_id}") - console.print(f"[blue]Token Type:[/blue] {result.token_type}") - if result.expires_at: - console.print(f"[blue]Expires At:[/blue] {result.expires_at}") - if result.expires_in: - console.print(f"[blue]Expires In:[/blue] {format_expires_in(result.expires_in)}") - if result.expiration: - console.print(f"[blue]Expiration:[/blue] {result.expiration}") - else: - progress.update(task, description="[red]Connection failed[/red]") - console.print(f"[red]✗[/red] API connection failed: {result.message}") - - except AuthError: - progress.update(task, description="[red]Authentication failed[/red]") - console.print("[red]✗[/red] Authentication failed. API key may be invalid or expired.") - - except APIError as e: - progress.update(task, description="[red]API error[/red]") - console.print(f"[red]✗[/red] API error: {e}") - - except ValueError as e: - error_msg = str(e) - if not api_key and ("No configuration file found" in error_msg or "Configuration file not found" in error_msg): - # No API configuration found and no API key provided - show helpful setup message - console.print("[yellow]⚠️ No API configuration found[/yellow]") - console.print() - console.print("To get started with PyneSys API:") - console.print( - "1. 🌐 Visit [blue][link=https://pynesys.io]https://pynesys.io[/link][/blue] to get your API key") - console.print("2. 🔧 Run [cyan]pyne api configure[/cyan] to set up your configuration") - console.print("3. 🧪 Or test a key directly with [cyan]pyne api status --api-key YOUR_KEY[/cyan]") - console.print() - console.print("[dim]Need help? Check our documentation at https://pynesys.io/docs[/dim]") - else: - console.print(f"[red]Configuration error: {e}[/red]") - console.print("[yellow]Hint:[/yellow] Use 'pyne api configure' to set up your API configuration.") - raise typer.Exit(1) - - except Exception as e: - console.print(f"[red]Error: {e}[/red]") - raise typer.Exit(1) diff --git a/src/pynecore/cli/commands/compile.py b/src/pynecore/cli/commands/compile.py index fcb5672..db4925b 100644 --- a/src/pynecore/cli/commands/compile.py +++ b/src/pynecore/cli/commands/compile.py @@ -1,9 +1,11 @@ import tomllib from pathlib import Path +from datetime import datetime import typer from rich.console import Console from rich.progress import Progress, SpinnerColumn, TextColumn +from rich.table import Table from ..app import app, app_state from ..utils.api_error_handler import APIErrorHandler @@ -15,12 +17,78 @@ console = Console() +def _print_usage(compiler: PyneComp): + """Print usage statistics in a nicely formatted table.""" + usage = compiler.get_usage() + + # Create the main usage table + table = Table(title="API Usage Statistics", show_header=True, header_style="bold magenta") + table.add_column("Period", style="cyan", no_wrap=True) + table.add_column("Used", style="bold red", justify="right") + table.add_column("Limit", style="bold green", justify="right") + table.add_column("Remaining", style="bold yellow", justify="right") + table.add_column("Reset At", style="dim") + + # Add daily usage + table.add_row( + "Daily", + str(usage.daily.used), + str(usage.daily.limit), + str(usage.daily.remaining), + usage.daily.reset_at.strftime("%Y-%m-%d %H:%M:%S UTC") + ) + + # Add hourly usage + table.add_row( + "Hourly", + str(usage.hourly.used), + str(usage.hourly.limit), + str(usage.hourly.remaining), + usage.hourly.reset_at.strftime("%Y-%m-%d %H:%M:%S UTC") + ) + + console.print(table) + + # Show API key expiration information + try: + token_info = compiler.validate_api_key() + if token_info.valid and token_info.expiration: + print() + expiry_table = Table(title="API Key Information", show_header=True, header_style="bold magenta") + expiry_table.add_column("Property", style="cyan", no_wrap=True) + expiry_table.add_column("Value", style="white") + + # Calculate days remaining + now = datetime.now() + days_remaining = (token_info.expiration - now).days + hours_remaining = (token_info.expiration - now).total_seconds() / 3600 + + # Format expiration time + expiry_str = token_info.expiration.strftime("%Y-%m-%d %H:%M:%S") + + # Add rows + expiry_table.add_row("Expires At", expiry_str) + + if days_remaining > 0: + expiry_table.add_row("Days Remaining", f"{days_remaining} days") + elif hours_remaining > 0: + expiry_table.add_row("Hours Remaining", f"{hours_remaining:.1f} hours") + else: + expiry_table.add_row("Status", "[red]EXPIRED[/red]") + + console.print(expiry_table) + except (AttributeError, ValueError, TypeError) as e: + print() + console.print(f"[yellow]⚠[/yellow] Could not validate API key expiration: {str(e)}") + console.print("[dim]API key information unavailable[/dim]") + + # noinspection PyShadowingBuiltins @app.command() def compile( script: Path = typer.Argument( - ..., file_okay=True, dir_okay=False, - help="Path to Pine Script file (.pine extension)" + None, file_okay=True, dir_okay=False, + help="Path to Pine Script file (.pine extension) or name of script in [cyan]scripts[/cyan] directory" ), output: Path | None = typer.Option( None, "--output", "-o", @@ -38,30 +106,23 @@ def compile( None, "--api-key", "-a", help="PyneSys API key (overrides configuration file)", envvar="PYNESYS_API_KEY" + ), + show_usage: bool = typer.Option( + False, "--usage", "-u", + help="Print API usage statistics after compilation" ) ): """ Compile Pine Script to Python using PyneSys API. - CONFIGURATION: - Default config: workdir/config/api.toml - Fallback config: ~/.pynecore/api.toml + The system automatically searches for the workdir folder in the current and parent directories. + If not found, it creates or uses a workdir folder in the current directory. - REQUIREMENTS: - - Pine Script version 6 only (version 5 not supported) - - Valid PyneSys API key required (get one at [blue]https://app.pynesys.io[/]) - """ + Only version 6 is supported (no v4 or v5 support). + API key can be provided via --api-key or via config file: [cyan]workdir/config/api.toml[/cyan]. - # Ensure .py extension - if script.suffix != ".pine": - script = script.with_suffix(".pine") - # Expand script path - if len(script.parts) == 1: - script = app_state.scripts_dir / script - - # Determine output path - if output is None: - output = script.with_suffix('.py') + If no script is provided, will print usage statistics and exit. + """ # Read api.toml configuration api_config = {} @@ -81,6 +142,22 @@ def compile( # Create the compiler instance compiler = PyneComp(**api_config) + # Ensure script was provided + if not script: + _print_usage(compiler) + raise typer.Exit(0) + + # Ensure .py extension + if script.suffix != ".pine": + script = script.with_suffix(".pine") + # Expand script path + if len(script.parts) == 1: + script = app_state.scripts_dir / script + + # Determine output path + if output is None: + output = script.with_suffix('.py') + # Check if compilation is needed (smart compilation) if not compiler.needs_compilation(script, output) and not force: console.print(f"[green]✓[/green] Output file is up-to-date: {output}") @@ -105,3 +182,7 @@ def compile( copy_mtime(script, output) console.print(f"The compiled script is located at: [cyan]{out_path}[/cyan]") + + if show_usage: + print() + _print_usage(compiler) diff --git a/src/pynecore/pynesys/api.py b/src/pynecore/pynesys/api.py index 43ba9db..1c1b322 100644 --- a/src/pynecore/pynesys/api.py +++ b/src/pynecore/pynesys/api.py @@ -4,6 +4,7 @@ from typing import Any import json +import base64 from datetime import datetime @@ -31,6 +32,24 @@ class TokenValidationResponse: raw_response: dict[str, Any] | None = None +@dataclass +class UsageLimits: + """Usage limits for daily and hourly periods.""" + limit: int + used: int + remaining: int + reset_at: datetime + + +@dataclass +class UsageResponse: + """Response from account usage endpoint.""" + daily: UsageLimits + hourly: UsageLimits + api_keys: dict[str, Any] + raw_response: dict[str, Any] | None = None + + @dataclass class CompileResponse: """Response from script compilation endpoint.""" @@ -220,6 +239,132 @@ def verify_token(self) -> TokenValidationResponse: raise APIError("Unexpected error during token verification.") + def verify_token_local(self) -> TokenValidationResponse: + """ + Verify JWT token locally without server request. + + :return: TokenValidationResponse with validation details + :raises AuthError: If token format is invalid or expired + """ + try: + # JWT tokens have format: header.payload.signature + parts = self.api_key.split('.') + if len(parts) != 3: + return TokenValidationResponse( + valid=False, + message="Invalid JWT format: must have 3 parts separated by dots" + ) + + header_b64, payload_b64, signature_b64 = parts + + # Decode header + try: + # Add padding if needed + header_b64 += '=' * (4 - len(header_b64) % 4) + header_data = json.loads(base64.urlsafe_b64decode(header_b64).decode('utf-8')) + except (ValueError, json.JSONDecodeError): + return TokenValidationResponse( + valid=False, + message="Invalid JWT header format" + ) + + # Decode payload + try: + # Add padding if needed + payload_b64 += '=' * (4 - len(payload_b64) % 4) + payload_data = json.loads(base64.urlsafe_b64decode(payload_b64).decode('utf-8')) + except (ValueError, json.JSONDecodeError): + return TokenValidationResponse( + valid=False, + message="Invalid JWT payload format" + ) + + # Check expiration - try both 'exp' (standard) and 'e' (custom format) + exp = payload_data.get('exp') or payload_data.get('e') + if exp: + exp_time = datetime.fromtimestamp(exp) + if datetime.now() >= exp_time: + return TokenValidationResponse( + valid=False, + message="Token has expired", + expiration=exp_time, + expires_at=exp_time + ) + + # Extract user info + user_id = payload_data.get('s') # Based on the image, 's' contains user ID + + return TokenValidationResponse( + valid=True, + message="Token is valid", + user_id=user_id, + token_type=header_data.get('typ', 'JWT'), + expiration=datetime.fromtimestamp(exp) if exp else None, + expires_at=datetime.fromtimestamp(exp) if exp else None, + raw_response={ + 'header': header_data, + 'payload': payload_data + } + ) + + except Exception as e: + return TokenValidationResponse( + valid=False, + message=f"Token validation error: {str(e)}" + ) + + def get_usage(self) -> UsageResponse: + """ + Get current usage statistics and limits for the authenticated user. + + :return: UsageResponse with usage details + :raises AuthError: If authentication fails + :raises NetworkError: If network request fails + :raises APIError: For other API errors + """ + try: + request = self._make_request("GET", "account/usage") + + with urllib.request.urlopen(request, timeout=self.timeout) as response: + data = json.loads(response.read().decode('utf-8')) + + # Parse daily usage + daily_data = data["daily"] + daily = UsageLimits( + limit=daily_data["limit"], + used=daily_data["used"], + remaining=daily_data["remaining"], + reset_at=datetime.fromisoformat(daily_data["reset_at"].replace("Z", "+00:00")) + ) + + # Parse hourly usage + hourly_data = data["hourly"] + hourly = UsageLimits( + limit=hourly_data["limit"], + used=hourly_data["used"], + remaining=hourly_data["remaining"], + reset_at=datetime.fromisoformat(hourly_data["reset_at"].replace("Z", "+00:00")) + ) + + return UsageResponse( + daily=daily, + hourly=hourly, + api_keys=data.get("api_keys", {}), + raw_response=data + ) + + except urllib.error.HTTPError as e: + self._handle_http_error(e) + except urllib.error.URLError as e: + raise NetworkError(f"Network error during usage retrieval: {e}") + except Exception as e: + if not isinstance(e, APIError): + raise APIError(f"Unexpected error during usage retrieval: {e}") + else: + raise + + raise APIError("Unexpected error during usage retrieval.") + def compile_script( self, script: str, diff --git a/src/pynecore/pynesys/compiler.py b/src/pynecore/pynesys/compiler.py index 6289a09..0edb197 100644 --- a/src/pynecore/pynesys/compiler.py +++ b/src/pynecore/pynesys/compiler.py @@ -1,19 +1,16 @@ """ Core compilation service for programmatic use. """ - -import subprocess -import sys from pathlib import Path -from pynecore.pynesys.api import APIClient -from pynecore.pynesys.api import APIError, AuthError, RateLimitError, CompilationError +from pynecore.pynesys.api import (APIClient, APIError, AuthError, RateLimitError, CompilationError, UsageResponse, + TokenValidationResponse) from pynecore.utils.file_utils import is_updated class PyneComp: """ - COmpiler through the PyneSys API. + Compiler through the PyneSys API. """ api_client: APIClient @@ -91,6 +88,18 @@ def compile(self, pine_path: Path, output_path: Path | None = None, # Wrap unexpected errors raise APIError(f"Unexpected error during compilation: {e}") + def get_usage(self) -> UsageResponse: + """ + Get current usage statistics and limits for the authenticated user. + """ + return self.api_client.get_usage() + + def validate_api_key(self) -> TokenValidationResponse: + """ + Validate API key. + """ + return self.api_client.verify_token_local() + @staticmethod def needs_compilation(pine_file_path: Path, output_file_path: Path) -> bool: """ @@ -101,44 +110,3 @@ def needs_compilation(pine_file_path: Path, output_file_path: Path) -> bool: :return: True if compilation is needed, False otherwise """ return is_updated(pine_file_path, output_file_path) - - def compile_and_run( - self, - pine_file_path: Path, - script_args: list | None = None, - force: bool = False, - strict: bool = False, - output_file_path: Path | None = None - ) -> int: - """ - Compile a .pine file and run the resulting Python script. - - :param pine_file_path: Path to the .pine file - :param script_args: Arguments to pass to the compiled script - :param force: Force recompilation even if file hasn't changed - :param strict: Enable strict compilation mode - :param output_file_path: Optional output path (defaults to .py extension) - :return: Exit code from the executed script - :raises FileNotFoundError: If pine file doesn't exist - :raises CompilationError: If compilation fails - :raises APIError: If API request fails - """ - # Compile the file - compiled_file = self.compile( - pine_path=pine_file_path, - output_path=output_file_path, - force=force, - strict=strict - ) - - # Prepare command to run the compiled script - cmd = [sys.executable, str(compiled_file)] - if script_args: - cmd.extend(script_args) - - # Run the compiled script - try: - result = subprocess.run(cmd, check=False) - return result.returncode - except Exception as e: - raise RuntimeError(f"Error executing compiled script: {e}") From 2919e7b747812991e08810489b8a97c676295792 Mon Sep 17 00:00:00 2001 From: Adam Wallner Date: Sun, 3 Aug 2025 00:11:04 +0200 Subject: [PATCH 13/31] feat(cli): improve compile command API usage display and error messages - Added optional sleep to _print_usage to ensure usage statistics are saved before display in compile command. - Improved formatting and clarity of API error messages in api_error_handler. - Removed redundant or outdated lines from command init template. --- src/pynecore/cli/commands/__init__.py | 1 - src/pynecore/cli/commands/compile.py | 9 +++++++-- src/pynecore/cli/utils/api_error_handler.py | 14 +++++--------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/pynecore/cli/commands/__init__.py b/src/pynecore/cli/commands/__init__.py index 65c0703..8750238 100644 --- a/src/pynecore/cli/commands/__init__.py +++ b/src/pynecore/cli/commands/__init__.py @@ -250,7 +250,6 @@ def main( with api_file.open('w') as f: f.write("""[api] api_key = "" -base_url = "https://api.pynesys.io" timeout = 30 """) diff --git a/src/pynecore/cli/commands/compile.py b/src/pynecore/cli/commands/compile.py index db4925b..6f830eb 100644 --- a/src/pynecore/cli/commands/compile.py +++ b/src/pynecore/cli/commands/compile.py @@ -1,4 +1,5 @@ import tomllib +import time from pathlib import Path from datetime import datetime @@ -17,7 +18,7 @@ console = Console() -def _print_usage(compiler: PyneComp): +def _print_usage(compiler: PyneComp, sleep: float = 0): """Print usage statistics in a nicely formatted table.""" usage = compiler.get_usage() @@ -51,6 +52,10 @@ def _print_usage(compiler: PyneComp): # Show API key expiration information try: + # Sleep to wait for usage is saved in the background of the DB + if sleep > 0: + time.sleep(sleep) + token_info = compiler.validate_api_key() if token_info.valid and token_info.expiration: print() @@ -185,4 +190,4 @@ def compile( if show_usage: print() - _print_usage(compiler) + _print_usage(compiler, 1.0) diff --git a/src/pynecore/cli/utils/api_error_handler.py b/src/pynecore/cli/utils/api_error_handler.py index 8e3f4f5..3e6f5af 100644 --- a/src/pynecore/cli/utils/api_error_handler.py +++ b/src/pynecore/cli/utils/api_error_handler.py @@ -54,10 +54,10 @@ def _handle_rate_limit_error(self, e: RateLimitError): if e.retry_after: self.console.print(f"[yellow]⏰ Please try again in {e.retry_after} seconds[/yellow]") self.console.print( - "[yellow]💡 To increase your limits, consider upgrading your subscription at " + "[yellow]💡 To increase your limits, consider upgrading your subscription at " "[link=https://pynesys.io]https://pynesys.io[/link]") self.console.print( - "💎 Upgrade at [link=https://pynesys.io/pricing]https://pynesys.io/pricing[/link]") + "💎 Upgrade at [link=https://pynesys.io/pricing]https://pynesys.io/pricing[/link]") def _handle_api_error(self, e: APIError): """Handle general API errors with specific status code handling.""" @@ -112,11 +112,8 @@ def _handle_api_error(self, e: APIError): elif "500" in error_msg or "server" in error_msg or "internal" in error_msg: self.console.print("[red]🔧 Server Error:[/red] Something went wrong on our end") - self.console.print("[yellow]😅 Don't worry, it's not you![/yellow]") self.console.print(" • This is a temporary server issue") self.console.print(" • Please try again in a few moments") - self.console.print( - "📊 Check service status: [link=https://status.pynesys.io]https://status.pynesys.io[/link]") elif "unsupported pinescript version" in error_msg: self.console.print("[red]📌 Version Issue:[/red] Your Pine Script version isn't supported") @@ -125,8 +122,7 @@ def _handle_api_error(self, e: APIError): self.console.print(" • Update your script to Pine Script version 6") self.console.print(" • Most v5 scripts need minimal changes") self.console.print( - "📖 Migration guide: [link=https://www.tradingview.com/pine-script-docs/en/v6/" - "migration_guides/v5_to_v6_migration_guide.html]Pine Script v5→v6 Guide[/link]") + "📖 Migration guide: [link=https://www.tradingview.com/pine-script-docs/en/v6/migration_guides/v5_to_v6_migration_guide.html]Pine Script v5→v6 Guide[/link]") else: self.console.print("[yellow]💡 Only Pine Script version 6 is currently supported[/yellow]") @@ -138,8 +134,8 @@ def _handle_api_error(self, e: APIError): else: # Generic API error fallback - self.console.print(f"[red]🌐 API Error:[/red] {str(e)}") - self.console.print("[yellow]💡 If this persists, please check:[/yellow]") + self.console.print(f"[red]🌐 API Error:[/red] {str(e)}") + self.console.print("[yellow]💡 If this persists, please check:[/yellow]") self.console.print(" • Your internet connection") self.console.print(" • API service status") self.console.print("📞 Need help? [link=https://pynesys.io/support]Contact Support[/link]") From a157507978929598e93977eaa9003d537b01ad97 Mon Sep 17 00:00:00 2001 From: Mahadi Hassan Date: Sun, 3 Aug 2025 18:03:12 +0600 Subject: [PATCH 14/31] fix(api): improve error handling and message formatting - Enhance _handle_http_error to accept pre-extracted error messages - Add support for structured error responses in API error handling - Remove emojis and streamline error messages in CLI output - Improve parsing of JSON error responses with detailed error info --- src/pynecore/cli/utils/api_error_handler.py | 94 ++++++++++++--------- src/pynecore/pynesys/api.py | 41 +++++---- 2 files changed, 76 insertions(+), 59 deletions(-) diff --git a/src/pynecore/cli/utils/api_error_handler.py b/src/pynecore/cli/utils/api_error_handler.py index 3e6f5af..f51108e 100644 --- a/src/pynecore/cli/utils/api_error_handler.py +++ b/src/pynecore/cli/utils/api_error_handler.py @@ -36,7 +36,7 @@ def __exit__(self, exc_type, exc_value, traceback): def _handle_compilation_error(self, e: CompilationError): """Handle compilation-specific errors.""" - self.console.print(f"[red]❌ Oops! Compilation encountered an issue:[/red] {str(e)}") + self.console.print(f"[red]Oops! Compilation encountered an issue:[/red] {str(e)}") if e.validation_errors: self.console.print("[red]Validation errors:[/red]") for error in e.validation_errors: @@ -44,98 +44,108 @@ def _handle_compilation_error(self, e: CompilationError): def _handle_auth_error(self, e: AuthError): """Handle authentication errors.""" - self.console.print(f"[red]🔐 Authentication issue:[/red] {str(e)}") - self.console.print("[yellow]🚀 To fix:[/yellow] Check [cyan]api_key[/cyan] in [cyan]api.toml[/cyan] " + self.console.print(f"[red]Authentication issue:[/red] {str(e)}") + self.console.print("[yellow]To fix:[/yellow] Check [cyan]api_key[/cyan] in [cyan]api.toml[/cyan] " "in your working directory") def _handle_rate_limit_error(self, e: RateLimitError): """Handle rate limit errors.""" - self.console.print("[red]🚦 Rate Limit Exceeded:[/red] You've hit your compilation limit") + self.console.print("[red]Rate Limit Exceeded:[/red] You've hit your compilation limit") if e.retry_after: - self.console.print(f"[yellow]⏰ Please try again in {e.retry_after} seconds[/yellow]") + self.console.print(f"[yellow]Please try again in {e.retry_after} seconds[/yellow]") self.console.print( - "[yellow]💡 To increase your limits, consider upgrading your subscription at " + "[yellow]To increase your limits, consider upgrading your subscription at " "[link=https://pynesys.io]https://pynesys.io[/link]") - self.console.print( - "💎 Upgrade at [link=https://pynesys.io/pricing]https://pynesys.io/pricing[/link]") def _handle_api_error(self, e: APIError): """Handle general API errors with specific status code handling.""" error_msg = str(e).lower() # Handle specific API error scenarios based on HTTP status codes - if "400" in error_msg or "bad request" in error_msg: + if "400" in error_msg or "bad request" in error_msg or ( + '"detail":' in error_msg and '"error":' in error_msg and '"line":' in error_msg): if "compilation fails" in error_msg or "script is too large" in error_msg: - self.console.print("[red]📝 Script Issue:[/red] Your Pine Script couldn't be compiled") - self.console.print("[yellow]💡 Common fixes:[/yellow]") + self.console.print("[red]Script Issue:[/red] Your Pine Script couldn't be compiled") + self.console.print("[yellow]Common fixes:[/yellow]") self.console.print(" • Check if your script is too large (try breaking it into smaller parts)") self.console.print(" • Verify your Pine Script syntax is correct") self.console.print(" • Make sure you're using Pine Script v6 syntax") else: - self.console.print(f"[red]⚠️ Request Error:[/red] {str(e)}") - self.console.print("[yellow]💡 This usually means there's an issue with the request format[/yellow]") + # Try to parse structured error response (JSON-like format) + error_str = str(e) + structured_error_parsed = False + + # Check for structured JSON error format + if "'error':" in error_str and "'line':" in error_str or '"error":' in error_str and '"line":' in error_str: + import re + # Pattern for JSON format: {"detail":{"error":"...","line":...,"file":"..."}} + json_pattern = r'\{"detail":\{"status":"error","error":"([^"]+)","line":(\d+),"file":"([^"]+)"\}\}' + match = re.search(json_pattern, error_str) + if match: + error_msg, line_num, file_name = match.groups() + self.console.print(f"[red]Script Error:[/red] {error_msg}") + self.console.print(f"[yellow]Location:[/yellow] Line {line_num} in {file_name}") + self.console.print("[yellow]Quick fix:[/yellow] Check the variable declaration and spelling") + structured_error_parsed = True + + # Fallback to generic error display if structured parsing failed + if not structured_error_parsed: + self.console.print(f"[red]Script Error:[/red] {str(e)}") + self.console.print("[yellow]Common causes:[/yellow]") + self.console.print(" • Pine Script syntax errors") + self.console.print(" • Unsupported Pine Script features") + self.console.print(" • Incorrect variable declarations or usage") elif "401" in error_msg or "authentication" in error_msg or "no permission" in error_msg: - self.console.print("[red]🔐 Authentication Failed:[/red] Your API credentials aren't working") - self.console.print("[yellow]🚀 Quick fixes:[/yellow]") + self.console.print("[red]Authentication Failed:[/red] Your API credentials aren't working") + self.console.print("[yellow]Quick fixes:[/yellow]") self.console.print(" • Check if your API key is valid and active") self.console.print(" • Verify your token type is allowed for compilation") self.console.print( - "🔑 Get a new API key at [link=https://pynesys.io]https://pynesys.io[/link]") + "Get a new API key at [link=https://pynesys.io]https://pynesys.io[/link]") self.console.print( - "⚙️ Then run [cyan]'pyne api configure'[/cyan] to update your configuration") + "Then run [cyan]'pyne api configure'[/cyan] to update your configuration") elif "404" in error_msg or "not found" in error_msg: - self.console.print("[red]🔍 Not Found:[/red] The API endpoint or user wasn't found") - self.console.print("[yellow]💡 This might indicate:[/yellow]") + self.console.print("[red]Not Found:[/red] The API endpoint or user wasn't found") + self.console.print("[yellow]This might indicate:[/yellow]") self.console.print(" • Your account may not exist or be accessible") self.console.print(" • There might be a temporary service issue") - self.console.print( - "📞 Contact support if this persists: " - "[link=https://pynesys.io/support]https://pynesys.io/support[/link]") elif "422" in error_msg or "validation error" in error_msg: - self.console.print("[red]📋 Validation Error:[/red] Your request data has validation issues") - self.console.print("[yellow]💡 Common causes:[/yellow]") + self.console.print("[red]Validation Error:[/red] Your request data has validation issues") + self.console.print("[yellow]Common causes:[/yellow]") self.console.print(" • Invalid Pine Script syntax or structure") self.console.print(" • Missing required parameters") self.console.print(" • Incorrect data format") self.console.print(f"[dim]Details: {str(e)}[/dim]") elif "429" in error_msg or "rate limit" in error_msg or "too many requests" in error_msg: - self.console.print("[red]🚦 Rate Limit Exceeded:[/red] You've hit your compilation limit") - self.console.print("[yellow]⏰ What you can do:[/yellow]") + self.console.print("[red]Rate Limit Exceeded:[/red] You've hit your compilation limit") + self.console.print("[yellow]What you can do:[/yellow]") self.console.print(" • Wait a bit before trying again") self.console.print(" • Consider upgrading your plan for higher limits") - self.console.print( - "💎 Upgrade at [link=https://pynesys.io/pricing]https://pynesys.io/pricing[/link]") elif "500" in error_msg or "server" in error_msg or "internal" in error_msg: - self.console.print("[red]🔧 Server Error:[/red] Something went wrong on our end") + self.console.print("[red]Server Error:[/red] Something went wrong on our end") self.console.print(" • This is a temporary server issue") self.console.print(" • Please try again in a few moments") elif "unsupported pinescript version" in error_msg: - self.console.print("[red]📌 Version Issue:[/red] Your Pine Script version isn't supported") + self.console.print("[red]Version Issue:[/red] Your Pine Script version isn't supported") if "version 5" in error_msg: - self.console.print("[yellow]🔄 Pine Script v5 → v6 Migration:[/yellow]") + self.console.print("[yellow]Pine Script v5 → v6 Migration:[/yellow]") self.console.print(" • Update your script to Pine Script version 6") self.console.print(" • Most v5 scripts need minimal changes") - self.console.print( - "📖 Migration guide: [link=https://www.tradingview.com/pine-script-docs/en/v6/migration_guides/v5_to_v6_migration_guide.html]Pine Script v5→v6 Guide[/link]") else: - self.console.print("[yellow]💡 Only Pine Script version 6 is currently supported[/yellow]") + self.console.print("[yellow]Only Pine Script version 6 is currently supported[/yellow]") elif "api key" in error_msg: - self.console.print("[red]🔑 API Key Issue:[/red] There's a problem with your API key") - self.console.print("🔑 Get your API key at [link=https://pynesys.io]https://pynesys.io[/link]") + self.console.print("[red]API Key Issue:[/red] There's a problem with your API key") + self.console.print("Get your API key at [link=https://pynesys.io]https://pynesys.io[/link]") self.console.print( - "⚙️ Then run [cyan]'pyne api configure'[/cyan] to set up your configuration") + "Then run [cyan]'pyne api configure'[/cyan] to set up your configuration") else: # Generic API error fallback - self.console.print(f"[red]🌐 API Error:[/red] {str(e)}") - self.console.print("[yellow]💡 If this persists, please check:[/yellow]") - self.console.print(" • Your internet connection") - self.console.print(" • API service status") - self.console.print("📞 Need help? [link=https://pynesys.io/support]Contact Support[/link]") + self.console.print(f"[red]API Error:[/red] {str(e)}") diff --git a/src/pynecore/pynesys/api.py b/src/pynecore/pynesys/api.py index 1c1b322..573c088 100644 --- a/src/pynecore/pynesys/api.py +++ b/src/pynecore/pynesys/api.py @@ -75,7 +75,7 @@ def is_rate_limited(self) -> bool: @property def is_auth_error(self) -> bool: - """Check if response indicates authentication error.""" + """Check if response indicates an authentication error.""" return self.status_code == 401 @@ -254,9 +254,9 @@ def verify_token_local(self) -> TokenValidationResponse: valid=False, message="Invalid JWT format: must have 3 parts separated by dots" ) - + header_b64, payload_b64, signature_b64 = parts - + # Decode header try: # Add padding if needed @@ -267,7 +267,7 @@ def verify_token_local(self) -> TokenValidationResponse: valid=False, message="Invalid JWT header format" ) - + # Decode payload try: # Add padding if needed @@ -278,7 +278,7 @@ def verify_token_local(self) -> TokenValidationResponse: valid=False, message="Invalid JWT payload format" ) - + # Check expiration - try both 'exp' (standard) and 'e' (custom format) exp = payload_data.get('exp') or payload_data.get('e') if exp: @@ -290,10 +290,10 @@ def verify_token_local(self) -> TokenValidationResponse: expiration=exp_time, expires_at=exp_time ) - + # Extract user info user_id = payload_data.get('s') # Based on the image, 's' contains user ID - + return TokenValidationResponse( valid=True, message="Token is valid", @@ -306,7 +306,7 @@ def verify_token_local(self) -> TokenValidationResponse: 'payload': payload_data } ) - + except Exception as e: return TokenValidationResponse( valid=False, @@ -417,21 +417,23 @@ def compile_script( raise @staticmethod - def _handle_http_error(error: urllib.error.HTTPError) -> None: + def _handle_http_error(error: urllib.error.HTTPError, message: str = None) -> None: """ Handle HTTP error responses. :param error: HTTPError object + :param message: Optional pre-extracted error message :raises: Appropriate exception based on status code """ status_code = error.code - try: - error_content = error.read().decode('utf-8') - error_data = json.loads(error_content) - message = error_data.get("message", error_content) - except (json.JSONDecodeError, ValueError): - message = error.reason or f"HTTP {status_code} error" + if message is None: + try: + error_content = error.read().decode('utf-8') + error_data = json.loads(error_content) + message = error_data.get("message", error_content) + except (json.JSONDecodeError, ValueError): + message = error.reason or f"HTTP {status_code} error" if status_code == 401: raise AuthError(message, status_code=status_code) @@ -470,6 +472,10 @@ def _handle_compile_http_error(self, error: urllib.error.HTTPError) -> CompileRe # Validation error format (422) validation_errors = error_data["detail"] error_message = "Validation errors occurred" + elif "detail" in error_data and isinstance(error_data["detail"], dict): + # Structured error format (400) - pass the complete JSON for parsing + validation_errors = None + error_message = error_content # Pass the full JSON response else: validation_errors = None error_message = error_data.get("message", error_content) @@ -478,8 +484,9 @@ def _handle_compile_http_error(self, error: urllib.error.HTTPError) -> CompileRe if status_code == 422: raise CompilationError(error_message, status_code=status_code, validation_errors=validation_errors) - # For other errors, use the general error handler - self._handle_http_error(error) + # For other errors, use the general error handler with the extracted message + else: + self._handle_http_error(error, error_message) # This should never be reached return CompileResponse( From f83a476f5ee88f7b239105802b3cb14902e16ce0 Mon Sep 17 00:00:00 2001 From: Mahadi Hassan Date: Sun, 3 Aug 2025 18:24:02 +0600 Subject: [PATCH 15/31] docs(cli): add compile command documentation and update related files Add comprehensive documentation for the new `compile` command that converts Pine Script to Python using PyneSys API. Update related CLI documentation files to reflect Pine Script support in the `run` command and add cross-references. Update README.md to include basic Pine Script usage examples and add new compile.md documentation file with detailed usage instructions, examples, and troubleshooting information. --- README.md | 17 ++++ docs/cli/README.md | 1 + docs/cli/basics.md | 5 +- docs/cli/compile.md | 210 ++++++++++++++++++++++++++++++++++++++++++++ docs/cli/run.md | 44 +++++++++- 5 files changed, 274 insertions(+), 3 deletions(-) create mode 100644 docs/cli/compile.md diff --git a/README.md b/README.md index 116bf89..62f7f9f 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,23 @@ pyne data download ccxt --symbol "BYBIT:BTC/USDT:USDT" pyne run my_script.py ccxt_BYBIT_BTC_USDT_USDT_1D.ohlcv ``` +### Working with Pine Script Files + +PyneCore also supports running Pine Script files directly with automatic compilation: + +```bash +# Run a Pine Script file directly (requires PyneSys API key) +pyne run my_indicator.pine ccxt_BYBIT_BTC_USDT_USDT_1D.ohlcv --api-key YOUR_API_KEY + +# Or compile Pine Script to Python first +pyne compile my_indicator.pine --output my_indicator.py --api-key YOUR_API_KEY + +# Then run the compiled Python file +pyne run my_indicator.py ccxt_BYBIT_BTC_USDT_USDT_1D.ohlcv +``` + +> **Note**: Pine Script compilation requires a PyneSys API key. Get yours at [pynesys.io](https://pynesys.io) or try it free on our [Discord](https://discord.com/invite/7rhPbSqSG7). + ## Why Choose PyneCore? - **Beyond TradingView Limitations**: No more platform restrictions, code size limits, or subscription fees diff --git a/docs/cli/README.md b/docs/cli/README.md index ac6a074..271f23d 100644 --- a/docs/cli/README.md +++ b/docs/cli/README.md @@ -21,4 +21,5 @@ PyneCore Command Line Interface (CLI) overview and usage - [Basics](./basics.md) - Basic CLI usage and getting started - [Run](./run.md) - Running scripts with the CLI +- [Compile](./compile.md) - Compiling Pine Scripts to Python - [Data](./data.md) - Data management commands diff --git a/docs/cli/basics.md b/docs/cli/basics.md index f0c678a..0bea729 100644 --- a/docs/cli/basics.md +++ b/docs/cli/basics.md @@ -156,13 +156,14 @@ PYNE_WORK_DIR=/path/to/my/workdir pyne run my_script.py my_data.ohlcv The PyneCore CLI provides the following main commands: -- `run`: Run a PyneCore script +- `run`: Run a PyneCore script (.py or .pine) +- `compile`: Compile Pine Script to Python using PyneSys API - `data`: OHLCV related commands -- `compile`: Compile Pine Script to Python through pynesys.com (coming soon) ## Next Steps Now that you understand the basic concepts, you can learn about specific commands: - [Running Scripts](run.md): How to run PyneCore scripts +- [Compiling Pine Scripts](compile.md): How to compile Pine Scripts to Python - [Data Management](data.md): Data download and conversion commands \ No newline at end of file diff --git a/docs/cli/compile.md b/docs/cli/compile.md new file mode 100644 index 0000000..18f8ed2 --- /dev/null +++ b/docs/cli/compile.md @@ -0,0 +1,210 @@ + + +# Compiling Pine Scripts + +The `compile` command allows you to convert Pine Script (.pine) files to Python (.py) files using the PyneSys API. This enables you to migrate your TradingView Pine Script strategies to PyneCore. + +## Basic Usage + +The basic syntax for compiling a Pine Script is: + +```bash +pyne compile [SCRIPT] [OPTIONS] +``` + +Where: +- `SCRIPT`: Path to Pine Script file (.pine extension) or name of script in scripts directory (optional) +- `OPTIONS`: Additional options to customize the compilation + +## Simple Example + +```bash +# Compile a Pine Script file +pyne compile my_strategy.pine +``` + +This command will: +1. Look for `my_strategy.pine` in the current directory or `workdir/scripts/` directory +2. Compile it using the PyneSys API +3. Save the compiled Python code as `my_strategy.py` in the same location + +## API Key Requirements + +To use the compile command, you need a valid PyneSys API key. You can obtain one at [https://pynesys.io](https://pynesys.io). + +The API key can be provided in several ways: + +1. **Command line option**: Use the `--api-key` flag +2. **Environment variable**: Set `PYNESYS_API_KEY` +3. **Configuration file**: Store in `workdir/config/api.toml` + +### Configuration File Setup + +Create a file at `workdir/config/api.toml` with your API key: + +```toml +[pynesys] +api_key = "your-api-key-here" +``` + +## Command Options + +The `compile` command supports several options: + +### Output Options + +- `--output`, `-o`: Specify the output Python file path. Defaults to the same name with `.py` extension. + +Example: +```bash +# Compile with custom output path +pyne compile my_strategy.pine --output ./compiled/my_strategy.py +``` + +### Compilation Options + +- `--strict`, `-s`: Enable strict compilation mode with enhanced error checking +- `--force`, `-f`: Force recompilation even if output file is up-to-date +- `--api-key`, `-a`: PyneSys API key (overrides configuration file) + +Example: +```bash +# Compile with strict mode and force recompilation +pyne compile my_strategy.pine --strict --force +``` + +### Usage Statistics + +- `--usage`, `-u`: Print API usage statistics after compilation + +Example: +```bash +# Show usage statistics +pyne compile my_strategy.pine --usage +``` + +## Pine Script Version Support + +**Important**: Only Pine Script version 6 is supported. Version 4 and 5 scripts are not supported. + +Make sure your Pine Script starts with: +```pine +//@version=6 +``` + +## Usage Statistics + +If you run the compile command without specifying a script, it will display your current API usage statistics and exit: + +```bash +# Display usage statistics +pyne compile +``` + +## Working Directory Integration + +The compile command integrates with PyneCore's working directory system: + +- **Script lookup**: If you provide just a filename, it will be searched in the `workdir/scripts/` directory +- **Output location**: Compiled files are saved in the same directory as the source file +- **Configuration**: API configuration is read from `workdir/config/api.toml` + +## Examples + +### Basic Compilation + +```bash +# Compile a Pine Script in the current directory +pyne compile my_strategy.pine +``` + +### Compilation with API Key + +```bash +# Compile with API key provided via command line +pyne compile my_strategy.pine --api-key "your-api-key" +``` + +### Strict Mode Compilation + +```bash +# Compile with enhanced error checking +pyne compile my_strategy.pine --strict +``` + +### Force Recompilation + +```bash +# Force recompilation even if output is up-to-date +pyne compile my_strategy.pine --force +``` + +### Custom Output Path + +```bash +# Compile to a specific output location +pyne compile my_strategy.pine --output ./strategies/compiled_strategy.py +``` + +### Show Usage Statistics + +```bash +# Compile and show API usage statistics +pyne compile my_strategy.pine --usage +``` + +## Troubleshooting + +### API Key Issues + +``` +Error: No API key provided +``` + +This error occurs when no API key is found. Make sure to: +- Set the `PYNESYS_API_KEY` environment variable, or +- Use the `--api-key` command line option, or +- Create a configuration file at `workdir/config/api.toml` + +### Authentication Errors + +``` +Error: Invalid API key +``` + +This error occurs when the API key is invalid or expired. Check: +- Your API key is correct +- Your subscription is active +- You have remaining API credits + +### Pine Script Version Errors + +``` +Error: Unsupported Pine Script version +``` + +This error occurs when trying to compile Pine Script v4 or v5. Only version 6 is supported. + +### File Not Found + +``` +Script file 'my_strategy.pine' not found! +``` + +This error occurs when the Pine Script file cannot be found. Make sure: +- The file exists in the specified location +- If you provided just a filename, check that it exists in the `workdir/scripts/` directory +- The filename is spelled correctly (case sensitive) \ No newline at end of file diff --git a/docs/cli/run.md b/docs/cli/run.md index c791a79..e5a032c 100644 --- a/docs/cli/run.md +++ b/docs/cli/run.md @@ -26,7 +26,7 @@ pyne run SCRIPT DATA [OPTIONS] ``` Where: -- `SCRIPT`: Path to the PyneCore script (.py) file +- `SCRIPT`: Path to the PyneCore script (.py) or Pine Script (.pine) file - `DATA`: Path to the OHLCV data (.ohlcv) file - `OPTIONS`: Additional options to customize the execution @@ -43,6 +43,44 @@ This command will: 3. Execute the script with the provided data 4. Save outputs to the `workdir/output/` directory +## Pine Script Support + +The `run` command now supports Pine Script (.pine) files in addition to Python (.py) files. When you specify a `.pine` file: + +1. **Automatic Compilation**: The system automatically compiles the Pine Script to Python if: + - The `.py` file doesn't exist, or + - The `.pine` file is newer than the existing `.py` file + +2. **API Key Required**: A valid PyneSys API key is required for Pine Script compilation. You can get one at [https://pynesys.io](https://pynesys.io). + +3. **Output Location**: The compiled `.py` file is saved in the same folder as the original `.pine` file. + +### Pine Script Example + +```bash +# Run a Pine Script directly +pyne run my_strategy.pine eurusd_data.ohlcv +``` + +This command will: +1. Check if `my_strategy.py` exists and is up-to-date +2. If not, compile `my_strategy.pine` to `my_strategy.py` using the PyneSys API +3. Run the compiled Python script with the provided data + +### API Key Configuration + +For Pine Script compilation, you can provide the API key in several ways: + +1. **Command line option**: Use the `--api-key` flag +2. **Environment variable**: Set `PYNESYS_API_KEY` +3. **Configuration file**: Store in `workdir/config/api.toml` + +Example with API key: +```bash +# Run Pine Script with API key +pyne run my_strategy.pine eurusd_data.ohlcv --api-key "your-api-key" +``` + ## Command Arguments The `run` command has two required arguments: @@ -58,6 +96,10 @@ Note: you don't need to write the `.py` and `.ohlcv` extensions in the command. The `run` command supports several options to customize the execution: +### Compilation Options + +- `--api-key`, `-a`: PyneSys API key for Pine Script compilation (overrides configuration file) + ### Date Range Options - `--from`, `-f`: Start date (UTC) in 'YYYY-MM-DD' or 'YYYY-MM-DD HH:MM:SS' format. If not specified, it will use the first date in the data From 7505f71d4f2d716d08d7d991e6c4c6fef05efabc Mon Sep 17 00:00:00 2001 From: Mahadi Hassan Date: Sun, 3 Aug 2025 18:47:14 +0600 Subject: [PATCH 16/31] refactor(api_error_handler): improve error parsing logic and readability Extract error condition checks into descriptive variables for better clarity Use consistent formatting for multi-line strings and conditions Add missing import for regex operations --- src/pynecore/cli/utils/api_error_handler.py | 33 ++++++++++++++++----- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/src/pynecore/cli/utils/api_error_handler.py b/src/pynecore/cli/utils/api_error_handler.py index f51108e..cc1e599 100644 --- a/src/pynecore/cli/utils/api_error_handler.py +++ b/src/pynecore/cli/utils/api_error_handler.py @@ -1,5 +1,7 @@ """Centralized API error handling utilities for CLI commands.""" +import re + from rich.console import Console from typer import Exit @@ -62,8 +64,14 @@ def _handle_api_error(self, e: APIError): error_msg = str(e).lower() # Handle specific API error scenarios based on HTTP status codes - if "400" in error_msg or "bad request" in error_msg or ( - '"detail":' in error_msg and '"error":' in error_msg and '"line":' in error_msg): + has_400_error = "400" in error_msg or "bad request" in error_msg + has_structured_error = ( + '"detail":' in error_msg and + '"error":' in error_msg and + '"line":' in error_msg + ) + + if has_400_error or has_structured_error: if "compilation fails" in error_msg or "script is too large" in error_msg: self.console.print("[red]Script Issue:[/red] Your Pine Script couldn't be compiled") self.console.print("[yellow]Common fixes:[/yellow]") @@ -76,16 +84,27 @@ def _handle_api_error(self, e: APIError): structured_error_parsed = False # Check for structured JSON error format - if "'error':" in error_str and "'line':" in error_str or '"error":' in error_str and '"line":' in error_str: - import re + has_error_and_line = ( + ("'error':" in error_str and "'line':" in error_str) or + ('"error":' in error_str and '"line":' in error_str) + ) + + if has_error_and_line: # Pattern for JSON format: {"detail":{"error":"...","line":...,"file":"..."}} - json_pattern = r'\{"detail":\{"status":"error","error":"([^"]+)","line":(\d+),"file":"([^"]+)"\}\}' + json_pattern = ( + r'\{"detail":\{"status":"error","error":"([^"]+)",' + + r'"line":(\d+),"file":"([^"]+)"\}\}' + ) match = re.search(json_pattern, error_str) if match: error_msg, line_num, file_name = match.groups() self.console.print(f"[red]Script Error:[/red] {error_msg}") - self.console.print(f"[yellow]Location:[/yellow] Line {line_num} in {file_name}") - self.console.print("[yellow]Quick fix:[/yellow] Check the variable declaration and spelling") + self.console.print( + f"[yellow]Location:[/yellow] Line {line_num} in {file_name}" + ) + self.console.print( + "[yellow]Quick fix:[/yellow] Check the variable declaration and spelling" + ) structured_error_parsed = True # Fallback to generic error display if structured parsing failed From 3d7c094581b7892e959b5dae59d3ec3fb36c25a4 Mon Sep 17 00:00:00 2001 From: Adam Wallner Date: Sun, 3 Aug 2025 15:13:23 +0200 Subject: [PATCH 17/31] docs: update Discord invite link and clarify PineComp usage - Updated Discord invite link in pyproject.toml, README.md, and docs. - Removed references to free Discord API keys in README.md. - Updated PineComp description for clarity in ecosystem docs. - Reformatted some long lines for improved readability. --- README.md | 10 +++++----- docs/overview/ecosystem.md | 21 ++++++++++++++------- pyproject.toml | 2 +- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 62f7f9f..d02f8f2 100644 --- a/README.md +++ b/README.md @@ -161,13 +161,13 @@ PyneCore also supports running Pine Script files directly with automatic compila pyne run my_indicator.pine ccxt_BYBIT_BTC_USDT_USDT_1D.ohlcv --api-key YOUR_API_KEY # Or compile Pine Script to Python first -pyne compile my_indicator.pine --output my_indicator.py --api-key YOUR_API_KEY +pyne compile my_indicator.pine --api-key YOUR_API_KEY # Then run the compiled Python file pyne run my_indicator.py ccxt_BYBIT_BTC_USDT_USDT_1D.ohlcv ``` -> **Note**: Pine Script compilation requires a PyneSys API key. Get yours at [pynesys.io](https://pynesys.io) or try it free on our [Discord](https://discord.com/invite/7rhPbSqSG7). +> **Note**: Pine Script compilation requires a PyneSys API key. Get yours at [pynesys.io](https://pynesys.io). ## Why Choose PyneCore? @@ -181,10 +181,10 @@ pyne run my_indicator.py ccxt_BYBIT_BTC_USDT_USDT_1D.ohlcv ## Pine Script Migration Made Easy Have existing Pine Script code you want to run in Python? **PyneSys now offers automatic Pine -Script to PyneCore translation** through our [web platform](https://pynesys.io) and [Discord bot](https://discord.com/invite/7rhPbSqSG7). +Script to PyneCore translation** through our [web platform](https://pynesys.io) and [Discord bot](https://discord.com/invite/jegnhtq6gy). ### 🚀 Get Started Instantly -- **Try for free on Discord**: Use `/pine-help` in our [Discord](https://discord.com/invite/7rhPbSqSG7) for instant conversion - 3 free translations! +- **Try for free on Discord**: Use `/pine-help` in our [Discord](https://discord.com/invite/jegnhtq6gy) for instant conversion - 3 free translations! - **Full service**: Visit [pynesys.io](https://pynesys.io) for subscriptions and higher limits ### 💡 Support the Ecosystem @@ -202,7 +202,7 @@ Love PyneCore and want to see it grow? Consider a **Seed subscription or higher* ### Community - **Discussions**: [GitHub Discussions](https://github.com/pynesys/pynecore/discussions) -- **Discord**: [discord.com/invite/7rhPbSqSG7](https://discord.com/invite/7rhPbSqSG7) +- **Discord**: [discord.com/invite/jegnhtq6gy](https://discord.com/invite/jegnhtq6gy) - **X**: [x.com/pynesys](https://x.com/pynesys) - **Website**: [pynecore.org](https://pynecore.org) diff --git a/docs/overview/ecosystem.md b/docs/overview/ecosystem.md index 283af3d..71698e8 100644 --- a/docs/overview/ecosystem.md +++ b/docs/overview/ecosystem.md @@ -15,7 +15,8 @@ tags: ["ecosystem", "pynecomp", "pynecore", "services", "business-model", "commu # Pyne Ecosystem -The Pyne ecosystem consists of multiple interconnected components that together form a complete solution for TradingView Pine Script compatibility in Python. This page provides an overview of these components and how they work together. +The Pyne ecosystem consists of multiple interconnected components that together form a complete solution for TradingView +Pine Script compatibility in Python. This page provides an overview of these components and how they work together. ## Core Components @@ -23,7 +24,8 @@ The Pyne ecosystem is built around two main components: ### PyneCore (Open Source) -PyneCore is the foundation of the ecosystem - an open-source Python implementation of a Pine Script-like environment. It provides: +PyneCore is the foundation of the ecosystem - an open-source Python implementation of a Pine Script-like environment. It +provides: - A Pine Script compatible runtime in Python - AST transformations that enable Pine Script-like syntax and features @@ -33,24 +35,28 @@ PyneCore is the foundation of the ecosystem - an open-source Python implementati - NA class, which works the same way as Pine Script's NA - Strategy backtesting capabilities, compatible with Pine Script's strategy tester -PyneCore allows you to write Pine Script-like code directly in Python, leveraging Python's ecosystem while maintaining the advantages of Pine Script's execution model. +PyneCore allows you to write Pine Script-like code directly in Python, leveraging Python's ecosystem while maintaining +the advantages of Pine Script's execution model. Learn more about PyneCore in the [What is PyneCore](/docs/overview/what-is-pynecore/) page. ### PyneComp - Pine Script to PyneCore Compiler/Transpiler (SaaS Service) -PyneComp is a compiler service that translates existing Pine Script code into PyneCore-compatible Python code. It offers: +PyneComp is a compiler service that translates existing Pine Script code into PyneCore-compatible Python code. It +offers: - Clean, readable Python code generation (PyneCore) - Strict mode with full scope isolation (it is not needed most of the time) - 100% Pine Script compatibility This service is available through: + - The PyneSys API - The [pynesys.io](https://pynesys.io) web interface - Direct integration with PyneCore CLI (if you have API key) -PyneComp enables a smooth migration path from TradingView Pine Script to Python with minimal effort. PyneCore has all the tools to run the compiled python code. +PyneComp enables a smooth migration path from TradingView Pine Script to Python with minimal effort. PyneCore has all +the tools to run the compiled python code. ## Planned Services @@ -58,7 +64,8 @@ The following services are planned and/or already being developed: ### MetaTrader 4/5 Compiler (Transpiler) -We are planning a compiler which can convert Pine Scripts to MetaTrader 4/5 expert advisors. We have the knowledge and experience to do this. +We are planning a compiler which can convert Pine Scripts to MetaTrader 4/5 expert advisors. We have the knowledge and +experience to do this. ### Strategy Leaderboard @@ -126,5 +133,5 @@ The Pyne ecosystem combines open-source and commercial components: Join our community to get help and share your experiences: - [GitHub Discussions](https://github.com/PyneSys/pynecore/discussions) -- [Discord Server](https://discord.com/invite/7rhPbSqSG7) +- [Discord Server](https://discord.com/invite/jegnhtq6gy) - [Stack Overflow](https://stackoverflow.com/questions/tagged/pynecore) diff --git a/pyproject.toml b/pyproject.toml index 65ec398..5638e9e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,4 +75,4 @@ scripts.pyne = "pynecore.cli:app" # Social "X" = "https://x.com/pynesys" -"Discord" = "https://discord.com/invite/7rhPbSqSG7" +"Discord" = "https://discord.com/invite/jegnhtq6gy" From caeecb602c8a15984b191fd67261600233f0e903 Mon Sep 17 00:00:00 2001 From: Mahadi Hassan Date: Mon, 4 Aug 2025 14:36:01 +0600 Subject: [PATCH 18/31] feat(data): add automatic data file conversion to OHLCV format Implement DataConverter class to automatically detect and convert CSV/TXT/JSON files to OHLCV format when needed. Includes symbol/timeframe auto-detection from filenames and TOML symbol info generation. The conversion is now integrated into the run command, eliminating manual conversion steps. Add support for --symbol and --timeframe parameters in run command for non-OHLCV files. Improve error handling and user feedback during conversion process. Include progress indicators for better UX. --- src/pynecore/cli/commands/data.py | 137 +++++-- src/pynecore/cli/commands/run.py | 70 +++- src/pynecore/core/data_converter.py | 606 ++++++++++++++++++++++++++++ src/pynecore/core/ohlcv_file.py | 2 +- 4 files changed, 777 insertions(+), 38 deletions(-) create mode 100644 src/pynecore/core/data_converter.py diff --git a/src/pynecore/cli/commands/data.py b/src/pynecore/cli/commands/data.py index 34601fb..3510094 100644 --- a/src/pynecore/cli/commands/data.py +++ b/src/pynecore/cli/commands/data.py @@ -15,7 +15,7 @@ from ...providers.provider import Provider from ...utils.rich.date_column import DateColumn -from pynecore.core.ohlcv_file import OHLCVReader, OHLCVWriter +from pynecore.core.ohlcv_file import OHLCVReader __all__ = [] @@ -230,47 +230,130 @@ def convert_to( task = progress.add_task(description="Converting to JSON...", total=1) ohlcv_reader.save_to_json(str(ohlcv_path.with_suffix('.json')), as_datetime=as_datetime) - # Complete task - progress.update(task, completed=1) + # Complete task + progress.update(task, completed=1) + + +def _auto_detect_symbol_timeframe(file_path: Path) -> tuple[str | None, str | None]: + """Auto-detect symbol and timeframe from filename. + + :param file_path: Path to the data file + :return: Tuple of (symbol, timeframe_str) or (None, None) if not detected + """ + filename = file_path.stem # Filename without extension + + # Common patterns for symbol detection + symbol = None + timeframe_str = None + + # Try to extract symbol and timeframe from common filename patterns + # Examples: BTCUSD_1h.csv, AAPL_daily.csv, EUR_USD_4h.csv + parts = filename.replace('-', '_').split('_') + + if len(parts) >= 1: + # The First part is likely the symbol + potential_symbol = parts[0].upper() + if len(potential_symbol) >= 3: # Minimum symbol length + symbol = potential_symbol + + # Look for timeframe indicators + timeframe_map = { + '1m': '1m', '5m': '5m', '15m': '15m', '30m': '30m', + '1h': '1H', '4h': '4H', '1d': '1D', 'daily': '1D', + '1w': '1W', 'weekly': '1W', '1M': '1M', 'monthly': '1M' + } + + for part in parts: + part_lower = part.lower() + if part_lower in timeframe_map: + timeframe_str = timeframe_map[part_lower] + break + + return symbol, timeframe_str @app_data.command() def convert_from( - file_path: Path = Argument(..., help="Path to CSV file to convert"), + file_path: Path = Argument(..., help="Path to CSV/JSON file to convert"), provider: str = Option("custom", '--provider', '-p', help="Data provider, can be any name"), symbol: str | None = Option(None, '--symbol', '-s', show_default=False, - help="Symbol (e.g. BYBIT:BTCUSDT:USDT)"), - timeframe: TimeframeEnum = Option('1D', '--timeframe', '-tf', case_sensitive=False, # type: ignore - help="Timeframe in TradingView fmt"), + help="Symbol (e.g. BTCUSD, auto-detected from filename if not provided)"), + timeframe: TimeframeEnum | None = Option(None, '--timeframe', '-tf', case_sensitive=False, # type: ignore + help="Timeframe (auto-detected from filename if not provided)"), fmt: Enum('Format', {'csv': 'csv', 'json': 'json'}) | None = Option( # noqa # type: ignore None, '--fmt', '-f', case_sensitive=False, - help="Output fmt"), + help="Input format (auto-detected from file extension if not provided)"), tz: str = Option('UTC', '--timezone', '-tz', help="Timezone"), ): """ - Convert data from other sources to pyne's OHLCV format + Convert data from other sources to pyne's OHLCV format with automatic symbol detection """ - with Progress(SpinnerColumn(finished_text="[green]✓"), TextColumn("{task.description}")) as progress: - ohlcv_path = Provider.get_ohlcv_path(symbol, timeframe.value, app_state.data_dir, provider) - if fmt is None: - fmt = file_path.suffix[1:] # noqa + from pynecore.core.data_converter import DataConverter + + # Auto-detect symbol and timeframe from filename if not provided + if symbol is None or timeframe is None: + detected_symbol, detected_timeframe_str = _auto_detect_symbol_timeframe(file_path) + if symbol is None: + symbol = detected_symbol + if timeframe is None and detected_timeframe_str: + try: + timeframe = detected_timeframe_str + except ValueError: + timeframe = None + + # Ensure we have required parameters + if symbol is None: + symbol = "UNKNOWN" # Fallback symbol + if timeframe is None: + timeframe = "1D" # Fallback timeframe + + # Auto-detect format from file extension + if fmt is None: + file_ext = file_path.suffix[1:].lower() + if file_ext in ['csv', 'json']: + fmt = file_ext else: - fmt = fmt.value - # Convert - with OHLCVWriter(ohlcv_path) as ohlcv_writer: - if fmt == 'csv': - task = progress.add_task(description="Converting from CSV...", total=1) - ohlcv_writer.load_from_csv(file_path, tz=tz) - - elif fmt == 'json': - task = progress.add_task(description="Converting from JSON...", total=1) - ohlcv_writer.load_from_json(file_path, tz=tz) - else: - secho(f"Error: Invalid format: {fmt}", err=True, fg=colors.RED) - raise Exit(1) + fmt = 'csv' # Default to CSV + else: + fmt = fmt.value + + # Use the enhanced DataConverter for automatic conversion + converter = DataConverter() + + try: + with Progress(SpinnerColumn(finished_text="[green]✓"), TextColumn("{task.description}")) as progress: + task = progress.add_task(description=f"Converting {fmt.upper() if fmt else 'CSV'} to OHLCV format...", + total=1) + + # Convert timeframe to string value + timeframe_str = "1D" # Default + if timeframe: + if hasattr(timeframe, 'value'): + timeframe_str = timeframe.value + else: + timeframe_str = str(timeframe) + + # Perform conversion with automatic TOML generation + result = converter.convert_if_needed( + file_path=Path(file_path), + provider=provider, + symbol=symbol, + timeframe=timeframe_str, + timezone=tz + ) - # Complete task progress.update(task, completed=1) + + # Show success message with generated files + secho(f"✓ Converted to: {result.ohlcv_path}", fg=colors.GREEN) + toml_path = file_path.with_suffix('.toml') + if toml_path.exists(): + secho(f"✓ Generated symbol info: {toml_path}", fg=colors.GREEN) + secho("⚠️ Please review the auto-generated symbol parameters in the .toml file", fg=colors.YELLOW) + + except Exception as e: + secho(f"Error: {e}", err=True, fg=colors.RED) + raise Exit(1) diff --git a/src/pynecore/cli/commands/run.py b/src/pynecore/cli/commands/run.py index e873440..318b3a1 100644 --- a/src/pynecore/cli/commands/run.py +++ b/src/pynecore/cli/commands/run.py @@ -17,6 +17,7 @@ from ...utils.rich.date_column import DateColumn from pynecore.core.ohlcv_file import OHLCVReader +from pynecore.core.data_converter import DataConverter, DataFormatError, ConversionError from pynecore.core.syminfo import SymInfo from pynecore.core.script_runner import ScriptRunner @@ -85,6 +86,12 @@ def run( help="PyneSys API key for compilation (overrides configuration file)", envvar="PYNESYS_API_KEY", rich_help_panel="Compilation Options"), + symbol: str | None = Option(None, "--symbol", "-s", + help="Symbol name for conversion (required for CSV/TXT/JSON files)", + rich_help_panel="Data Options"), + timeframe: str | None = Option(None, "--timeframe", "-tf", + help="Timeframe for conversion (required for CSV/TXT/JSON files)", + rich_help_panel="Data Options"), ): """ Run a script (.py or .pine) @@ -102,6 +109,9 @@ def run( file is newer than the [italic]py[/] file or if the [italic].py[/] file doesn't exist. The compiled [italic].py[/] file will be saved into the same folder as the original [italic].pine[/] file. A valid [bold]PyneSys API[/bold] key is required for Pine Script compilation. You can get one at [blue]https://pynesys.io[/blue]. + + [bold]Data Support:[/bold] + Supports CSV, TXT, JSON, and OHLCV data files. Non-OHLCV files are automatically converted. Requires [bold]--symbol[/bold] and [bold]--timeframe[/bold] for CSV/TXT/JSON. """ # noqa # Expand script path @@ -161,20 +171,60 @@ def run( # Update script to point to the compiled file script = out_path - # Check file format and extension + # Handle data file format and auto-conversion if data.suffix == "": # No extension, add .ohlcv data = data.with_suffix(".ohlcv") elif data.suffix != ".ohlcv": - # Has extension but not .ohlcv - secho(f"Cannot run with '{data.suffix}' files. The PyneCore runtime requires .ohlcv format.", - fg="red", err=True) - secho("If you're trying to use a different data format, please convert it first:", fg="red") - symbol_placeholder = "YOUR_SYMBOL" - timeframe_placeholder = "YOUR_TIMEFRAME" - secho(f"pyne data convert-from {data} --symbol {symbol_placeholder} --timeframe {timeframe_placeholder}", - fg="yellow") - raise Exit(1) + # Has extension but not .ohlcv - automatically convert + try: + converter = DataConverter() + + # Check if conversion is needed + if converter.is_conversion_required(data): + # Validate required parameters for conversion + if not symbol or not timeframe: + secho(f"Converting '{data.suffix}' file requires symbol and timeframe parameters.", + fg="red", err=True) + secho("Please provide --symbol and --timeframe:", fg="red") + secho(f"pyne run {script} {data} --symbol YOUR_SYMBOL --timeframe YOUR_TIMEFRAME", + fg="yellow") + raise Exit(1) + + with Progress( + SpinnerColumn(finished_text="[green]✓"), + TextColumn("[progress.description]{task.description}"), + console=console + ) as progress: + task = progress.add_task(f"Converting {data.suffix} to OHLCV format...", total=1) + + # Perform conversion + result = converter.convert_if_needed(data, symbol=symbol, timeframe=timeframe) + + progress.update(task, completed=1) + + if result.converted: + console.print(f"[green]✓[/green] Converted {data} to {result.ohlcv_path}") + data = result.ohlcv_path + else: + console.print(f"[blue]ℹ[/blue] Using existing OHLCV file: {result.ohlcv_path}") + data = result.ohlcv_path + else: + # File is already up-to-date, use existing OHLCV file + ohlcv_path = data.with_suffix(".ohlcv") + console.print(f"[blue]ℹ[/blue] Using existing OHLCV file: {ohlcv_path}") + data = ohlcv_path + + except (DataFormatError, ConversionError) as e: + secho(f"Conversion failed: {e}", fg="red", err=True) + secho("Please convert the file manually:", fg="red") + symbol_placeholder = "YOUR_SYMBOL" + timeframe_placeholder = "YOUR_TIMEFRAME" + secho(f"pyne data convert-from {data} " + f"--symbol {symbol_placeholder} " + f"--timeframe {timeframe_placeholder}", + fg="yellow") + raise Exit(1) # Expand data path if len(data.parts) == 1: diff --git a/src/pynecore/core/data_converter.py b/src/pynecore/core/data_converter.py new file mode 100644 index 0000000..9303349 --- /dev/null +++ b/src/pynecore/core/data_converter.py @@ -0,0 +1,606 @@ +"""Automatic data file to OHLCV conversion functionality. + +This module provides automatic detection and conversion of CSV, TXT, and JSON files +to OHLCV format when needed, eliminating the manual step of running pyne data convert. +""" + +from __future__ import annotations + +import json +import struct +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Literal + +from pynecore.core.ohlcv_file import OHLCVWriter, STRUCT_FORMAT +from pynecore.utils.file_utils import copy_mtime, is_updated + + +@dataclass +class ConversionResult: + """Result of a conversion operation. + + :param converted: Whether conversion was performed + :param ohlcv_path: Path to the OHLCV file + :param source_path: Path to the source file + :param was_renamed: Whether the file was renamed from incorrect .ohlcv + """ + converted: bool + ohlcv_path: Path + source_path: Path + was_renamed: bool = False + + +class DataFormatError(Exception): + """Raised when file format cannot be detected or is unsupported.""" + pass + + +class ConversionError(Exception): + """Raised when conversion fails.""" + pass + + +class DataConverter: + """Main class for automatic data file conversion. + + Provides both CLI and programmatic interfaces for converting + CSV, TXT, and JSON files to OHLCV format automatically. + """ + + SUPPORTED_FORMATS = {'csv', 'txt', 'json'} + OHLCV_MAGIC_BYTES = b'\x00\x00\x00\x00' # First 4 bytes pattern for binary OHLCV + + def __init__(self): + pass + + @staticmethod + def detect_format(file_path: Path) -> Literal['csv', 'txt', 'json', 'ohlcv', 'unknown']: + """Detect file format by extension and content inspection. + + :param file_path: Path to the file to analyze + :return: Detected format + :raises FileNotFoundError: If file doesn't exist + :raises DataFormatError: If file cannot be read + """ + if not file_path.exists(): + raise FileNotFoundError(f"File not found: {file_path}") + + # Check extension first + ext = file_path.suffix.lower() + if ext == '.ohlcv': + # Validate if it's actually binary OHLCV format + try: + with open(file_path, 'rb') as f: + # Check if file size is multiple of 24 (OHLCV record size) + file_size = f.seek(0, 2) # Seek to end + f.seek(0) # Back to start + + if file_size == 0: + return 'ohlcv' # Empty OHLCV file is valid + + if file_size % 24 != 0: + # Not a valid OHLCV file, likely renamed + return DataConverter._detect_content_format(file_path) + + # Read first record to validate structure + first_record = f.read(24) + if len(first_record) == 24: + try: + # Try to unpack as OHLCV record + struct.unpack(STRUCT_FORMAT, first_record) + return 'ohlcv' + except struct.error: + # Invalid binary format, likely renamed + return DataConverter._detect_content_format(file_path) + + return 'ohlcv' + except (OSError, IOError) as e: + raise DataFormatError(f"Cannot read file {file_path}: {e}") + + elif ext == '.csv': + return 'csv' + elif ext == '.txt': + return 'txt' + elif ext == '.json': + return 'json' + + # No extension or unknown extension, inspect content + return DataConverter._detect_content_format(file_path) + + @staticmethod + def _detect_content_format(file_path: Path) -> Literal['csv', 'txt', 'json', 'unknown']: + """Detect format by inspecting file content. + + :param file_path: Path to the file + :return: Detected format based on content + """ + try: + with open(file_path, 'r', encoding='utf-8') as f: + # Read first few lines for analysis + first_line = f.readline().strip() + if not first_line: + return 'unknown' + + # Try JSON first (most specific) + f.seek(0) + try: + json.load(f) + return 'json' + except (json.JSONDecodeError, UnicodeDecodeError): + pass + + # Check for CSV patterns + if ',' in first_line: + # Count commas to see if it looks like structured data + comma_count = first_line.count(',') + if comma_count >= 4: # At least OHLC columns + return 'csv' + + # Check for other delimiters (TXT) + if any(delim in first_line for delim in ['\t', ';', '|']): + return 'txt' + + # Default to CSV if it has some structure + if ',' in first_line: + return 'csv' + + return 'unknown' + + except (OSError, IOError, UnicodeDecodeError): + return 'unknown' + + @staticmethod + def is_conversion_required(source_path: Path, ohlcv_path: Path | None = None) -> bool: + """Check if conversion is required based on file freshness. + + :param source_path: Path to the source file + :param ohlcv_path: Path to the OHLCV file (auto-generated if None) + :return: True if conversion is needed + """ + if ohlcv_path is None: + ohlcv_path = source_path.with_suffix('.ohlcv') + + # If OHLCV file doesn't exist, conversion is needed + if not ohlcv_path.exists(): + return True + + # Use existing file utility to check if source is newer + return is_updated(source_path, ohlcv_path) + + def convert_if_needed( + self, + file_path: Path, + *, + force: bool = False, + provider: str = "custom", + symbol: str | None = None, + timeframe: str = "1D", + timezone: str = "UTC" + ) -> ConversionResult: + """Convert file to OHLCV format if needed. + + :param file_path: Path to the data file + :param force: Force conversion even if OHLCV file is up-to-date + :param provider: Data provider name for OHLCV file naming + :param symbol: Symbol for OHLCV file naming + :param timeframe: Timeframe for conversion + :param timezone: Timezone for timestamp conversion + :return: ConversionResult with conversion details + :raises FileNotFoundError: If source file doesn't exist + :raises DataFormatError: If file format is unsupported + :raises ConversionError: If conversion fails + """ + if not file_path.exists(): + raise FileNotFoundError(f"Source file not found: {file_path}") + + # Detect file format + detected_format = self.detect_format(file_path) + + # Handle incorrectly renamed files + was_renamed = False + if file_path.suffix == '.ohlcv' and detected_format in self.SUPPORTED_FORMATS: + # File was incorrectly renamed, fix it + original_path = file_path.with_suffix(f'.{detected_format}') + file_path.rename(original_path) + file_path = original_path + was_renamed = True + + # If it's already OHLCV, no conversion needed + if detected_format == 'ohlcv': + return ConversionResult( + converted=False, + ohlcv_path=file_path, + source_path=file_path, + was_renamed=was_renamed + ) + + # Check if format is supported + if detected_format not in self.SUPPORTED_FORMATS: + raise DataFormatError( + f"Unsupported file format '{detected_format}' for file: {file_path}" + ) + + # Determine OHLCV output path + ohlcv_path = file_path.with_suffix('.ohlcv') + + # Check if conversion is needed + if not force and not self.is_conversion_required(file_path, ohlcv_path): + return ConversionResult( + converted=False, + ohlcv_path=ohlcv_path, + source_path=file_path, + was_renamed=was_renamed + ) + + # Perform conversion + self._convert_file( + source_path=file_path, + ohlcv_path=ohlcv_path, + format_type=detected_format, + provider=provider, + symbol=symbol, + timeframe=timeframe, + timezone=timezone + ) + + return ConversionResult( + converted=True, + ohlcv_path=ohlcv_path, + source_path=file_path, + was_renamed=was_renamed + ) + + def _convert_file( + self, + source_path: Path, + ohlcv_path: Path, + format_type: str, + provider: str, # Currently unused but kept for future extensibility + symbol: str | None, + timeframe: str, + timezone: str + ) -> None: + """Perform the actual file conversion. + + :param source_path: Path to source file + :param ohlcv_path: Path to output OHLCV file + :param format_type: Detected format type + :param provider: Data provider name (reserved for future use) + :param symbol: Symbol name + :param timeframe: Timeframe + :param timezone: Timezone + :raises ConversionError: If conversion fails + """ + # Create temporary file for atomic operation + temp_path = None + try: + with tempfile.NamedTemporaryFile( + mode='wb', + suffix='.ohlcv', + dir=ohlcv_path.parent, + delete=False + ) as temp_file: + temp_path = Path(temp_file.name) + + # Perform conversion using existing OHLCV writer + with OHLCVWriter(temp_path) as ohlcv_writer: + if format_type == 'csv': + ohlcv_writer.load_from_csv(source_path, tz=timezone) + elif format_type == 'json': + ohlcv_writer.load_from_json(source_path, tz=timezone) + elif format_type == 'txt': + # Treat TXT files as CSV with different delimiters + ohlcv_writer.load_from_csv(source_path, tz=timezone) + else: + raise ConversionError(f"Unsupported format for conversion: {format_type}") + + # Atomic rename to final location + temp_path.replace(ohlcv_path) + temp_path = None # Prevent cleanup + + # Copy modification time from source to maintain freshness + copy_mtime(source_path, ohlcv_path) + + # Generate TOML symbol info file if needed + self._create_symbol_info_file( + source_path=source_path, + symbol=symbol, + timeframe=timeframe, + timezone=timezone + ) + + except Exception as e: + # Clean up temporary file on error + if temp_path and temp_path.exists(): + try: + temp_path.unlink() + except OSError: + pass + raise ConversionError(f"Failed to convert {source_path}: {e}") from e + + def _create_symbol_info_file( + self, + source_path: Path, + symbol: str | None, + timeframe: str, + timezone: str + ) -> None: + """Create TOML symbol info file if it doesn't exist or is outdated. + + :param source_path: Path to the source data file + :param symbol: Symbol name (e.g., 'BTCUSD') + :param timeframe: Timeframe (e.g., '1h', '1D') + :param timezone: Timezone for the symbol + """ + toml_path = source_path.with_suffix('.toml') + + # Skip if TOML file exists and is newer than source + if toml_path.exists() and not is_updated(source_path, toml_path): + return + + # Generate symbol info if symbol is provided + if not symbol: + return + + # Analyze price data to calculate trading parameters + price_analysis = self._analyze_price_data(source_path) + + # Determine symbol type based on symbol name patterns + symbol_upper = symbol.upper() + symbol_type, currency, base_currency = self._detect_symbol_type(symbol_upper) + + # Get calculated trading parameters + mintick = price_analysis['tick_size'] + pricescale = price_analysis['price_scale'] + minmove = price_analysis['min_move'] + pointvalue = self._get_default_pointvalue(symbol_type) + + # Create TOML content with warnings + toml_content = f"""# WARNING: This file was auto-generated from price data analysis. +# Please review and adjust the following values according to your broker's specifications: +# - mintick: Minimum price movement (currently: {mintick}) +# - pricescale: Price scale factor (currently: {pricescale}) +# - minmove: Minimum move in price scale units (currently: {minmove}) +# - pointvalue: Market value per price scale unit (currently: {pointvalue}) +# - opening_hours: Trading hours may vary by broker and market + +[symbol] +prefix = "CUSTOM" +description = "{symbol} - Auto-generated symbol info" +ticker = "{symbol_upper}" +currency = "{currency}" +""" + + if base_currency: + toml_content += f'basecurrency = "{base_currency}"\n' + else: + toml_content += '#basecurrency =\n' + + toml_content += f"""period = "{timeframe}" +type = "{symbol_type}" +mintick = {mintick} +pricescale = {pricescale} +minmove = {minmove} +pointvalue = {pointvalue} +timezone = "{timezone}" +volumettype = "base" +#avg_spread = +#taker_fee = +#maker_fee = +#target_price_average = +#target_price_high = +#target_price_low = +#target_price_date = + +# Opening hours (24/7 for crypto, business hours for others) +""" + + if symbol_type == 'crypto': + # 24/7 trading for crypto + for day in range(1, 8): + toml_content += f"""[[opening_hours]] +day = {day} +start = "00:00:00" +end = "23:59:59" + +""" + else: + # Business hours for stocks/forex (Mon-Fri) + for day in range(1, 6): + toml_content += f"""[[opening_hours]] +day = {day} +start = "09:30:00" +end = "16:00:00" + +""" + + toml_content += """# Session starts +[[session_starts]] +day = 1 +time = "00:00:00" + +# Session ends +[[session_ends]] +day = 7 +time = "23:59:59" +""" + + # Write TOML file + try: + with open(toml_path, 'w', encoding='utf-8') as f: + f.write(toml_content) + + # Copy modification time from source to maintain consistency + copy_mtime(source_path, toml_path) + + except (OSError, IOError): + # Don't fail the entire conversion if TOML creation fails + # Just log the issue (could be added later) + pass + + def _analyze_price_data(self, source_path: Path) -> dict[str, float]: + """Analyze price data to calculate trading parameters. + + :param source_path: Path to the source data file + :return: Dictionary with tick_size, price_scale, and min_move + """ + try: + # Try to import pandas for data analysis + try: + import pandas as pd # type: ignore + except ImportError: + # Pandas not available, use basic CSV analysis + return self._analyze_price_data_basic(source_path) + + # Determine file format and read data + if source_path.suffix.lower() == '.csv': + df = pd.read_csv(source_path, nrows=1000) # Sample first 1000 rows + elif source_path.suffix.lower() == '.json': + df = pd.read_json(source_path, lines=True, nrows=1000) + else: + # Default fallback values + return {'tick_size': 0.01, 'price_scale': 100, 'min_move': 1} + + # Find price columns (common names) + price_cols = [] + for col in df.columns: + col_lower = col.lower() + if any(price_name in col_lower for price_name in ['close', 'price', 'high', 'low', 'open']): + price_cols.append(col) + + if not price_cols: + # No price columns found, use defaults + return {'tick_size': 0.01, 'price_scale': 100, 'min_move': 1} + + # Use the first price column for analysis + prices = pd.to_numeric(df[price_cols[0]], errors='coerce').dropna() + + if len(prices) == 0: + return {'tick_size': 0.01, 'price_scale': 100, 'min_move': 1} + + # Calculate decimal places from price data + decimal_places = 0 + for price in prices.head(100): # Check first 100 valid prices + if pd.notna(price): + price_str = str(float(price)) + if '.' in price_str and not price_str.endswith('.0'): + current_decimals = len(price_str.split('.')[1]) + decimal_places = max(decimal_places, current_decimals) + + # Calculate price scale (10^decimal_places) + # Use at least 2 decimal places for reasonable trading precision + decimal_places = max(decimal_places, 2) + price_scale = 10 ** decimal_places + + # Calculate minimum tick size + tick_size = 1.0 / price_scale + + # Min move is typically 1 in price scale units + min_move = 1 + + return { + 'tick_size': tick_size, + 'price_scale': price_scale, + 'min_move': min_move + } + + except (ImportError, ValueError, KeyError, TypeError): + # If analysis fails, return sensible defaults + return {'tick_size': 0.01, 'price_scale': 100, 'min_move': 1} + + @staticmethod + def _analyze_price_data_basic(source_path: Path) -> dict[str, float]: + """Basic price data analysis without pandas dependency. + + :param source_path: Path to the source data file + :return: Dictionary with tick_size, price_scale, and min_move + """ + try: + import csv + + if source_path.suffix.lower() != '.csv': + return {'tick_size': 0.01, 'price_scale': 100, 'min_move': 1} + + with open(source_path, 'r', encoding='utf-8') as f: + reader = csv.DictReader(f) + + # Find price column + price_col = None + for col in reader.fieldnames or []: + col_lower = col.lower() + if any(price_name in col_lower for price_name in ['close', 'price', 'high', 'low', 'open']): + price_col = col + break + + if not price_col: + return {'tick_size': 0.01, 'price_scale': 100, 'min_move': 1} + + # Analyze first 100 rows + decimal_places = 0 + count = 0 + for row in reader: + if count >= 100: + break + + try: + price = float(row[price_col]) + price_str = f"{price:.10f}".rstrip('0') + if '.' in price_str: + current_decimals = len(price_str.split('.')[1]) + decimal_places = max(decimal_places, current_decimals) + count += 1 + except (ValueError, KeyError): + continue + + # Calculate parameters + price_scale = 10 ** decimal_places + tick_size = 1.0 / price_scale if decimal_places > 0 else 1.0 + min_move = 1 + + return { + 'tick_size': tick_size, + 'price_scale': price_scale, + 'min_move': min_move + } + + except (OSError, IOError, ValueError, KeyError, TypeError): + return {'tick_size': 0.01, 'price_scale': 100, 'min_move': 1} + + @staticmethod + def _detect_symbol_type(symbol_upper: str) -> tuple[str, str, str | None]: + """Detect symbol type and extract currency information. + + :param symbol_upper: Uppercase symbol string + :return: Tuple of (symbol_type, currency, base_currency) + """ + if any(crypto in symbol_upper for crypto in ['BTC', 'ETH', 'USD', 'USDT', 'USDC']): + symbol_type = 'crypto' + currency = 'USD' + base_currency = symbol_upper.replace('USD', '').replace('USDT', '').replace('USDC', '') + if not base_currency: # If nothing left after removing USD variants + base_currency = None + elif '/' in symbol_upper: + symbol_type = 'forex' + parts = symbol_upper.split('/') + currency = parts[1] if len(parts) > 1 else 'USD' + base_currency = parts[0] if len(parts) > 0 else None + else: + symbol_type = 'stock' + currency = 'USD' + base_currency = None + + return symbol_type, currency, base_currency + + @staticmethod + def _get_default_pointvalue(symbol_type: str) -> float: + """Get default point value based on symbol type. + + :param symbol_type: Type of symbol (crypto, forex, stock) + :return: Default point value + """ + if symbol_type == 'forex': + return 10.0 + else: # crypto, stock, or unknown + return 1.0 diff --git a/src/pynecore/core/ohlcv_file.py b/src/pynecore/core/ohlcv_file.py index 4496610..4a0b785 100644 --- a/src/pynecore/core/ohlcv_file.py +++ b/src/pynecore/core/ohlcv_file.py @@ -25,7 +25,7 @@ RECORD_SIZE = 24 # 6 * 4 STRUCT_FORMAT = 'Ifffff' # I: uint32, f: float32 -__all__ = ['OHLCVWriter', 'OHLCVReader'] +__all__ = ['OHLCVWriter', 'OHLCVReader', 'STRUCT_FORMAT'] def format_float(value: float) -> str: From 154ed7f93c98425698647c9376304e36e6e88273 Mon Sep 17 00:00:00 2001 From: Mahadi Hassan Date: Tue, 5 Aug 2025 13:43:03 +0600 Subject: [PATCH 19/31] feat(data): add smart defaults for symbol and timeframe detection - Add AUTO timeframe option and auto-detection logic - Make symbol and timeframe optional with default values - Improve help text to show default behaviors - Move data path expansion before conversion logic --- src/pynecore/cli/commands/data.py | 7 +- src/pynecore/cli/commands/run.py | 41 ++++----- src/pynecore/core/data_converter.py | 132 +++++++++++++++++++++++++++- 3 files changed, 152 insertions(+), 28 deletions(-) diff --git a/src/pynecore/cli/commands/data.py b/src/pynecore/cli/commands/data.py index 3510094..b46fb63 100644 --- a/src/pynecore/cli/commands/data.py +++ b/src/pynecore/cli/commands/data.py @@ -26,7 +26,8 @@ AvailableProvidersEnum = Enum('Provider', {name.upper(): name.lower() for name in available_providers}) # Available intervals (The same fmt as described in timeframe.period) -TimeframeEnum = Enum('Timeframe', {name: name for name in ('1', '5', '15', '30', '60', '240', '1D', '1W')}) +# Numeric values represent minutes: 1=1min, 5=5min, 15=15min, 30=30min, 60=1hour, 240=4hours +TimeframeEnum = Enum('Timeframe', {name: name for name in ('1', '5', '15', '30', '60', '240', '1D', '1W', 'AUTO')}) # Trick to avoid type checking errors DateOrDays = datetime if TYPE_CHECKING else str @@ -278,9 +279,9 @@ def convert_from( provider: str = Option("custom", '--provider', '-p', help="Data provider, can be any name"), symbol: str | None = Option(None, '--symbol', '-s', show_default=False, - help="Symbol (e.g. BTCUSD, auto-detected from filename if not provided)"), + help="Symbol (default: auto-detected from filename)"), timeframe: TimeframeEnum | None = Option(None, '--timeframe', '-tf', case_sensitive=False, # type: ignore - help="Timeframe (auto-detected from filename if not provided)"), + help="Timeframe (default: auto-detected from filename)"), fmt: Enum('Format', {'csv': 'csv', 'json': 'json'}) | None = Option( # noqa # type: ignore None, '--fmt', '-f', case_sensitive=False, diff --git a/src/pynecore/cli/commands/run.py b/src/pynecore/cli/commands/run.py index 318b3a1..6114189 100644 --- a/src/pynecore/cli/commands/run.py +++ b/src/pynecore/cli/commands/run.py @@ -87,10 +87,12 @@ def run( envvar="PYNESYS_API_KEY", rich_help_panel="Compilation Options"), symbol: str | None = Option(None, "--symbol", "-s", - help="Symbol name for conversion (required for CSV/TXT/JSON files)", + help="Symbol name for conversion (default: SYMBOL)", + show_default=False, rich_help_panel="Data Options"), timeframe: str | None = Option(None, "--timeframe", "-tf", - help="Timeframe for conversion (required for CSV/TXT/JSON files)", + help="Timeframe for conversion (default: AUTO - detected from data)", + show_default=False, rich_help_panel="Data Options"), ): """ @@ -111,7 +113,7 @@ def run( A valid [bold]PyneSys API[/bold] key is required for Pine Script compilation. You can get one at [blue]https://pynesys.io[/blue]. [bold]Data Support:[/bold] - Supports CSV, TXT, JSON, and OHLCV data files. Non-OHLCV files are automatically converted. Requires [bold]--symbol[/bold] and [bold]--timeframe[/bold] for CSV/TXT/JSON. + Supports CSV, TXT, JSON, and OHLCV data files. Non-OHLCV files are automatically converted. Optional [bold]--symbol[/bold] and [bold]--timeframe[/bold] for CSV/TXT/JSON (smart defaults applied). """ # noqa # Expand script path @@ -171,6 +173,10 @@ def run( # Update script to point to the compiled file script = out_path + # Expand data path first (before conversion logic) + if len(data.parts) == 1: + data = app_state.data_dir / data + # Handle data file format and auto-conversion if data.suffix == "": # No extension, add .ohlcv @@ -182,14 +188,9 @@ def run( # Check if conversion is needed if converter.is_conversion_required(data): - # Validate required parameters for conversion - if not symbol or not timeframe: - secho(f"Converting '{data.suffix}' file requires symbol and timeframe parameters.", - fg="red", err=True) - secho("Please provide --symbol and --timeframe:", fg="red") - secho(f"pyne run {script} {data} --symbol YOUR_SYMBOL --timeframe YOUR_TIMEFRAME", - fg="yellow") - raise Exit(1) + # Use smart defaults - no longer require symbol and timeframe + default_symbol = symbol or "SYMBOL" # Default symbol name + default_timeframe = timeframe or "AUTO" # Auto-detect timeframe from data with Progress( SpinnerColumn(finished_text="[green]✓"), @@ -198,8 +199,12 @@ def run( ) as progress: task = progress.add_task(f"Converting {data.suffix} to OHLCV format...", total=1) - # Perform conversion - result = converter.convert_if_needed(data, symbol=symbol, timeframe=timeframe) + # Perform conversion with smart defaults + result = converter.convert_if_needed( + data, + symbol=default_symbol, + timeframe=default_timeframe + ) progress.update(task, completed=1) @@ -218,17 +223,9 @@ def run( except (DataFormatError, ConversionError) as e: secho(f"Conversion failed: {e}", fg="red", err=True) secho("Please convert the file manually:", fg="red") - symbol_placeholder = "YOUR_SYMBOL" - timeframe_placeholder = "YOUR_TIMEFRAME" - secho(f"pyne data convert-from {data} " - f"--symbol {symbol_placeholder} " - f"--timeframe {timeframe_placeholder}", - fg="yellow") + secho(f"pyne data convert-from {data}", fg="yellow") raise Exit(1) - # Expand data path - if len(data.parts) == 1: - data = app_state.data_dir / data # Check if data exists if not data.exists(): secho(f"Data file '{data}' not found!", fg="red", err=True) diff --git a/src/pynecore/core/data_converter.py b/src/pynecore/core/data_converter.py index 9303349..04b1813 100644 --- a/src/pynecore/core/data_converter.py +++ b/src/pynecore/core/data_converter.py @@ -269,7 +269,7 @@ def _convert_file( :param format_type: Detected format type :param provider: Data provider name (reserved for future use) :param symbol: Symbol name - :param timeframe: Timeframe + :param timeframe: Timeframe (will be auto-detected if "AUTO") :param timezone: Timezone :raises ConversionError: If conversion fails """ @@ -296,6 +296,9 @@ def _convert_file( else: raise ConversionError(f"Unsupported format for conversion: {format_type}") + # Auto-detect timeframe if needed + detected_timeframe = self._detect_timeframe_from_ohlcv(temp_path, timeframe) + # Atomic rename to final location temp_path.replace(ohlcv_path) temp_path = None # Prevent cleanup @@ -307,7 +310,7 @@ def _convert_file( self._create_symbol_info_file( source_path=source_path, symbol=symbol, - timeframe=timeframe, + timeframe=detected_timeframe, timezone=timezone ) @@ -373,10 +376,11 @@ def _create_symbol_info_file( currency = "{currency}" """ + # Use smart defaults for currency fields if base_currency: toml_content += f'basecurrency = "{base_currency}"\n' else: - toml_content += '#basecurrency =\n' + toml_content += 'basecurrency = "EUR" # Default base currency - change if needed\n' toml_content += f"""period = "{timeframe}" type = "{symbol_type}" @@ -604,3 +608,125 @@ def _get_default_pointvalue(symbol_type: str) -> float: return 10.0 else: # crypto, stock, or unknown return 1.0 + + def _detect_timeframe_from_ohlcv(self, ohlcv_path: Path, requested_timeframe: str) -> str: + """Detect timeframe from OHLCV data by analyzing timestamp differences. + + :param ohlcv_path: Path to the OHLCV file + :param requested_timeframe: Requested timeframe ("AUTO" for auto-detection) + :return: Detected or original timeframe + """ + if requested_timeframe.upper() != "AUTO": + return requested_timeframe + + try: + # Read first few records to analyze timestamp differences + with open(ohlcv_path, 'rb') as f: + # Read first 10 records (or less if file is smaller) + timestamps = [] + for _ in range(10): + record = f.read(24) # OHLCV record size + if len(record) < 24: + break + + # Unpack timestamp (first 8 bytes as uint64) + timestamp = struct.unpack(' 0: # Only positive differences + differences.append(diff) + + if not differences: + return "1D" # Default fallback + + # Find the most common difference (mode) + from collections import Counter + diff_counts = Counter(differences) + most_common_diff = diff_counts.most_common(1)[0][0] + + # Convert nanoseconds to timeframe string + return self._nanoseconds_to_timeframe(most_common_diff) + + except (OSError, IOError, struct.error, IndexError, ValueError): + # If detection fails, return default + return "1D" + + @staticmethod + def _nanoseconds_to_timeframe(nanoseconds: int) -> str: + """Convert nanoseconds difference to timeframe string. + + :param nanoseconds: Time difference in nanoseconds + :return: Timeframe string (e.g., '1m', '5m', '1h', '1D') + """ + # Convert to seconds + seconds = nanoseconds / 1_000_000_000 + + # Define common timeframes in seconds + timeframes = [ + (60, "1m"), + (300, "5m"), + (900, "15m"), + (1800, "30m"), + (3600, "1h"), + (14400, "4h"), + (86400, "1D"), + (604800, "1W"), + (2592000, "1M"), # Approximate month + ] + + # Find the closest match + best_match = "1D" # Default + min_diff = float('inf') + + for tf_seconds, tf_string in timeframes: + diff = abs(seconds - tf_seconds) + if diff < min_diff: + min_diff = diff + best_match = tf_string + + return best_match + + def _detect_timeframe_from_data(self, df, requested_timeframe: str) -> str: + """Detect timeframe from DataFrame by analyzing timestamp differences. + + :param df: DataFrame with timestamp column + :param requested_timeframe: Requested timeframe ("AUTO" for auto-detection) + :return: Detected or original timeframe + """ + if requested_timeframe.upper() != "AUTO": + return requested_timeframe + + try: + if len(df) < 2: + return "1D" # Default fallback + + # Calculate differences between consecutive timestamps + timestamps = df['timestamp'].values + differences = [] + + for i in range(1, min(len(timestamps), 10)): # Check first 10 records + diff = timestamps[i] - timestamps[i-1] + if diff > 0: # Only positive differences + differences.append(diff) + + if not differences: + return "1D" # Default fallback + + # Find the most common difference (mode) + from collections import Counter + diff_counts = Counter(differences) + most_common_diff = diff_counts.most_common(1)[0][0] + + # Convert nanoseconds to timeframe string + return self._nanoseconds_to_timeframe(most_common_diff) + + except (KeyError, IndexError, ValueError, TypeError): + # If detection fails, return default + return "1D" From f0db8e2ec659060b6b735028431daf27fcef4fba Mon Sep 17 00:00:00 2001 From: Mahadi Hassan Date: Wed, 6 Aug 2025 21:41:38 +0600 Subject: [PATCH 20/31] refactor(cli): remove timeframe option and simplify conversion logic The timeframe option was removed as it's no longer needed for data conversion. The conversion logic now always uses "AUTO" for timeframe detection. --- src/pynecore/cli/commands/run.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/pynecore/cli/commands/run.py b/src/pynecore/cli/commands/run.py index 6114189..ce5e5b4 100644 --- a/src/pynecore/cli/commands/run.py +++ b/src/pynecore/cli/commands/run.py @@ -90,10 +90,6 @@ def run( help="Symbol name for conversion (default: SYMBOL)", show_default=False, rich_help_panel="Data Options"), - timeframe: str | None = Option(None, "--timeframe", "-tf", - help="Timeframe for conversion (default: AUTO - detected from data)", - show_default=False, - rich_help_panel="Data Options"), ): """ Run a script (.py or .pine) @@ -113,7 +109,7 @@ def run( A valid [bold]PyneSys API[/bold] key is required for Pine Script compilation. You can get one at [blue]https://pynesys.io[/blue]. [bold]Data Support:[/bold] - Supports CSV, TXT, JSON, and OHLCV data files. Non-OHLCV files are automatically converted. Optional [bold]--symbol[/bold] and [bold]--timeframe[/bold] for CSV/TXT/JSON (smart defaults applied). + Supports CSV, TXT, JSON, and OHLCV data files. Non-OHLCV files are automatically converted. Optional [bold]--symbol[/bold] for CSV/TXT/JSON (smart defaults applied). """ # noqa # Expand script path @@ -190,7 +186,7 @@ def run( if converter.is_conversion_required(data): # Use smart defaults - no longer require symbol and timeframe default_symbol = symbol or "SYMBOL" # Default symbol name - default_timeframe = timeframe or "AUTO" # Auto-detect timeframe from data + default_timeframe = "AUTO" # Auto-detect timeframe from data with Progress( SpinnerColumn(finished_text="[green]✓"), From 81b5a96fb001bf94a3be47b2a36235e4275d74c2 Mon Sep 17 00:00:00 2001 From: Mahadi Hassan Date: Wed, 6 Aug 2025 22:40:01 +0600 Subject: [PATCH 21/31] fix(data): handle missing data files and improve path resolution Add file existence check and automatic path resolution for data files in convert_from command Simplify script path handling in run command by removing redundant extension check --- src/pynecore/cli/commands/data.py | 9 +++++++++ src/pynecore/cli/commands/run.py | 6 ++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/pynecore/cli/commands/data.py b/src/pynecore/cli/commands/data.py index b46fb63..9bda76d 100644 --- a/src/pynecore/cli/commands/data.py +++ b/src/pynecore/cli/commands/data.py @@ -294,6 +294,15 @@ def convert_from( """ from pynecore.core.data_converter import DataConverter + # Expand file path if only filename is provided (look in workdir/data) + if len(file_path.parts) == 1: + file_path = app_state.data_dir / file_path + + # Check if file exists + if not file_path.exists(): + secho(f"File '{file_path}' not found!", fg=colors.RED, err=True) + raise Exit(1) + # Auto-detect symbol and timeframe from filename if not provided if symbol is None or timeframe is None: detected_symbol, detected_timeframe_str = _auto_detect_symbol_timeframe(file_path) diff --git a/src/pynecore/cli/commands/run.py b/src/pynecore/cli/commands/run.py index 8248b31..e248e22 100644 --- a/src/pynecore/cli/commands/run.py +++ b/src/pynecore/cli/commands/run.py @@ -111,9 +111,7 @@ def run( [bold]Data Support:[/bold] Supports CSV, TXT, JSON, and OHLCV data files. Non-OHLCV files are automatically converted. Optional [bold]--symbol[/bold] for CSV/TXT/JSON (smart defaults applied). """ # noqa - # Ensure .py extension - if script.suffix != ".py": - script = script.with_suffix(".py") + # Expand script path if len(script.parts) == 1: script = app_state.scripts_dir / script @@ -147,7 +145,7 @@ def run( if api_key: api_config['api_key'] = api_key - if api_config['api_key']: + if api_config.get('api_key'): # Create the compiler instance compiler = PyneComp(**api_config) From 9695e5e700c12eada619e8d44ff1a46811ad81eb Mon Sep 17 00:00:00 2001 From: Mahadi Hassan Date: Sat, 9 Aug 2025 18:00:38 +0600 Subject: [PATCH 22/31] feat(data): improve symbol detection and timeframe handling - Add auto-detect symbol from filename functionality - Enhance timeframe validation with TV-compatible formats - Improve OHLCV file detection and validation - Refactor symbol info generation using SymInfo class - Remove STRUCT_FORMAT from __all__ in ohlcv_file.py --- src/pynecore/cli/commands/data.py | 127 +++-- src/pynecore/cli/commands/run.py | 18 +- src/pynecore/core/data_converter.py | 444 +++++++----------- src/pynecore/core/ohlcv_file.py | 2 +- .../t00_pynecore/data/test_001_ohlcv_file.py | 17 + 5 files changed, 268 insertions(+), 340 deletions(-) diff --git a/src/pynecore/cli/commands/data.py b/src/pynecore/cli/commands/data.py index 9bda76d..4751ed9 100644 --- a/src/pynecore/cli/commands/data.py +++ b/src/pynecore/cli/commands/data.py @@ -13,6 +13,8 @@ from ..app import app, app_state from ...providers import available_providers from ...providers.provider import Provider +from ...lib.timeframe import in_seconds +from ...core.data_converter import DataConverter from ...utils.rich.date_column import DateColumn from pynecore.core.ohlcv_file import OHLCVReader @@ -25,9 +27,31 @@ # Create an enum from it AvailableProvidersEnum = Enum('Provider', {name.upper(): name.lower() for name in available_providers}) -# Available intervals (The same fmt as described in timeframe.period) -# Numeric values represent minutes: 1=1min, 5=5min, 15=15min, 30=30min, 60=1hour, 240=4hours -TimeframeEnum = Enum('Timeframe', {name: name for name in ('1', '5', '15', '30', '60', '240', '1D', '1W', 'AUTO')}) + +# TV-compatible timeframe validation function +def validate_timeframe(value: str) -> str: + """Validate TV-compatible timeframe string. + + :param value: Timeframe string to validate + :return: Validated timeframe string + :raises ValueError: If timeframe is invalid + """ + if value.upper() == 'AUTO': + return value.upper() + + try: + # Test if it's a valid TV timeframe by trying to convert to seconds + in_seconds(value) + return value + except (ValueError, AssertionError): + # Fallback to common timeframes for validation + valid_timeframes = ['1', '2', '3', '5', '10', '15', '30', '45', '60', '120', '180', '240', '360', '480', '720', + '1D', '1W', '1M', '1S', '2S', '5S', '10S', '15S', '30S', 'AUTO'] + if value in valid_timeframes: + return value + raise ValueError( + f"Invalid timeframe: {value}. Must be a valid TradingView timeframe (e.g., '1', '5S', '1D', '1W') or 'AUTO'") + # Trick to avoid type checking errors DateOrDays = datetime if TYPE_CHECKING else str @@ -65,8 +89,8 @@ def download( help="Symbol (e.g. BYBIT:BTC/USDT:USDT)"), list_symbols: bool = Option(False, '--list-symbols', '-ls', help="List available symbols of the provider"), - timeframe: TimeframeEnum = Option('1D', '--timeframe', '-tf', case_sensitive=False, # type: ignore - help="Timeframe in TradingView fmt"), + timeframe: str = Option('1D', '--timeframe', '-tf', callback=validate_timeframe, + help="Timeframe in TradingView format (e.g., '1', '5S', '1D', '1W')"), time_from: DateOrDays = Option("continue", '--from', '-f', # type: ignore callback=parse_date_or_days, formats=[], metavar="[%Y-%m-%d|%Y-%m-%d %H:%M:%S|NUMBER]|continue", @@ -105,7 +129,7 @@ def download( raise Exit(1) # Create provider instance - provider_instance: Provider = provider_class(symbol=symbol, timeframe=timeframe.value, + provider_instance: Provider = provider_class(symbol=symbol, timeframe=timeframe, ohlv_dir=app_state.data_dir) # Download symbol info if not exists @@ -203,8 +227,8 @@ def convert_to( help="Data provider"), symbol: str | None = Option(None, '--symbol', '-s', show_default=False, help="Symbol (e.g. BYBIT:BTCUSDT:USDT)"), - timeframe: TimeframeEnum = Option('1D', '--timeframe', '-tf', case_sensitive=False, # type: ignore - help="Timeframe in TradingView fmt"), + timeframe: str = Option('1D', '--timeframe', '-tf', callback=validate_timeframe, + help="Timeframe in TradingView format (e.g., '1', '5S', '1D', '1W')"), fmt: Enum('Format', {'csv': 'csv', 'json': 'json'}) = Option( # noqa # type: ignore 'csv', '--format', '-f', case_sensitive=False, @@ -218,7 +242,7 @@ def convert_to( # Import provider module from provider_module = __import__(f"pynecore.providers.{provider.value}", fromlist=['']) provider_class = getattr(provider_module, [p for p in dir(provider_module) if p.endswith('Provider')][0]) - ohlcv_path = provider_class.get_ohlcv_path(symbol, timeframe.value, app_state.data_dir) + ohlcv_path = provider_class.get_ohlcv_path(symbol, timeframe, app_state.data_dir) with Progress(SpinnerColumn(finished_text="[green]✓"), TextColumn("{task.description}")) as progress: # Convert @@ -235,44 +259,6 @@ def convert_to( progress.update(task, completed=1) -def _auto_detect_symbol_timeframe(file_path: Path) -> tuple[str | None, str | None]: - """Auto-detect symbol and timeframe from filename. - - :param file_path: Path to the data file - :return: Tuple of (symbol, timeframe_str) or (None, None) if not detected - """ - filename = file_path.stem # Filename without extension - - # Common patterns for symbol detection - symbol = None - timeframe_str = None - - # Try to extract symbol and timeframe from common filename patterns - # Examples: BTCUSD_1h.csv, AAPL_daily.csv, EUR_USD_4h.csv - parts = filename.replace('-', '_').split('_') - - if len(parts) >= 1: - # The First part is likely the symbol - potential_symbol = parts[0].upper() - if len(potential_symbol) >= 3: # Minimum symbol length - symbol = potential_symbol - - # Look for timeframe indicators - timeframe_map = { - '1m': '1m', '5m': '5m', '15m': '15m', '30m': '30m', - '1h': '1H', '4h': '4H', '1d': '1D', 'daily': '1D', - '1w': '1W', 'weekly': '1W', '1M': '1M', 'monthly': '1M' - } - - for part in parts: - part_lower = part.lower() - if part_lower in timeframe_map: - timeframe_str = timeframe_map[part_lower] - break - - return symbol, timeframe_str - - @app_data.command() def convert_from( file_path: Path = Argument(..., help="Path to CSV/JSON file to convert"), @@ -280,8 +266,8 @@ def convert_from( help="Data provider, can be any name"), symbol: str | None = Option(None, '--symbol', '-s', show_default=False, help="Symbol (default: auto-detected from filename)"), - timeframe: TimeframeEnum | None = Option(None, '--timeframe', '-tf', case_sensitive=False, # type: ignore - help="Timeframe (default: auto-detected from filename)"), + timeframe: str | None = Option(None, '--timeframe', '-tf', + help="Timeframe (default: auto-detected from data, e.g., '1', '5S', '1D', '1W')"), fmt: Enum('Format', {'csv': 'csv', 'json': 'json'}) | None = Option( # noqa # type: ignore None, '--fmt', '-f', case_sensitive=False, @@ -292,33 +278,39 @@ def convert_from( """ Convert data from other sources to pyne's OHLCV format with automatic symbol detection """ - from pynecore.core.data_converter import DataConverter # Expand file path if only filename is provided (look in workdir/data) if len(file_path.parts) == 1: file_path = app_state.data_dir / file_path - + # Check if file exists if not file_path.exists(): secho(f"File '{file_path}' not found!", fg=colors.RED, err=True) raise Exit(1) - # Auto-detect symbol and timeframe from filename if not provided - if symbol is None or timeframe is None: - detected_symbol, detected_timeframe_str = _auto_detect_symbol_timeframe(file_path) - if symbol is None: - symbol = detected_symbol - if timeframe is None and detected_timeframe_str: - try: - timeframe = detected_timeframe_str - except ValueError: - timeframe = None + # Auto-detect symbol from filename if not provided + if symbol is None: + symbol = DataConverter.auto_detect_symbol_from_filename(file_path) + + # Use AUTO for timeframe detection from data if not provided + if timeframe is None: + timeframe = "AUTO" + + # Validate timeframe if provided (skip validation for AUTO) + if timeframe is not None and timeframe.upper() != "AUTO": + try: + validate_timeframe(timeframe) + except ValueError as e: + secho(f"Invalid timeframe '{timeframe}': {e}", fg=colors.RED, err=True) + raise Exit(1) # Ensure we have required parameters if symbol is None: - symbol = "UNKNOWN" # Fallback symbol - if timeframe is None: - timeframe = "1D" # Fallback timeframe + secho(f"Error: Could not detect symbol from filename '{file_path.name}'!", fg=colors.RED, err=True) + secho("Please provide a symbol using --symbol option or rename your file to include the symbol.", + fg=colors.YELLOW, err=True) + secho("Example: 'BTCUSD_1h.csv' or use '--symbol BTCUSD'", fg=colors.YELLOW, err=True) + raise Exit(1) # Auto-detect format from file extension if fmt is None: @@ -340,11 +332,8 @@ def convert_from( # Convert timeframe to string value timeframe_str = "1D" # Default - if timeframe: - if hasattr(timeframe, 'value'): - timeframe_str = timeframe.value - else: - timeframe_str = str(timeframe) + if timeframe is not None: + timeframe_str = str(timeframe) # Perform conversion with automatic TOML generation result = converter.convert_if_needed( diff --git a/src/pynecore/cli/commands/run.py b/src/pynecore/cli/commands/run.py index e248e22..fd366df 100644 --- a/src/pynecore/cli/commands/run.py +++ b/src/pynecore/cli/commands/run.py @@ -86,10 +86,7 @@ def run( help="PyneSys API key for compilation (overrides configuration file)", envvar="PYNESYS_API_KEY", rich_help_panel="Compilation Options"), - symbol: str | None = Option(None, "--symbol", "-s", - help="Symbol name for conversion (default: SYMBOL)", - show_default=False, - rich_help_panel="Data Options"), + ): """ Run a script (.py or .pine) @@ -109,7 +106,7 @@ def run( A valid [bold]PyneSys API[/bold] key is required for Pine Script compilation. You can get one at [blue]https://pynesys.io[/blue]. [bold]Data Support:[/bold] - Supports CSV, TXT, JSON, and OHLCV data files. Non-OHLCV files are automatically converted. Optional [bold]--symbol[/bold] for CSV/TXT/JSON (smart defaults applied). + Supports CSV, TXT, JSON, and OHLCV data files. Non-OHLCV files are automatically converted. Symbol is auto-detected from filename. """ # noqa # Expand script path @@ -189,8 +186,15 @@ def run( # Check if conversion is needed if converter.is_conversion_required(data): - # Use smart defaults - no longer require symbol and timeframe - default_symbol = symbol or "SYMBOL" # Default symbol name + # Auto-detect symbol from filename using existing function + detected_symbol = DataConverter.auto_detect_symbol_from_filename(data) + if not detected_symbol: + # Use filename without extension as default symbol + default_symbol = data.stem.upper() + secho(f"Warning: Could not detect symbol from filename '{data.name}'", fg="yellow", err=True) + secho(f"Using default symbol: '{default_symbol}'", fg="yellow") + else: + default_symbol = detected_symbol default_timeframe = "AUTO" # Auto-detect timeframe from data with Progress( diff --git a/src/pynecore/core/data_converter.py b/src/pynecore/core/data_converter.py index 04b1813..ab76714 100644 --- a/src/pynecore/core/data_converter.py +++ b/src/pynecore/core/data_converter.py @@ -9,12 +9,16 @@ import json import struct import tempfile +from collections import Counter from dataclasses import dataclass +from datetime import time from pathlib import Path from typing import Literal -from pynecore.core.ohlcv_file import OHLCVWriter, STRUCT_FORMAT +from pynecore.core.ohlcv_file import OHLCVWriter, OHLCVReader from pynecore.utils.file_utils import copy_mtime, is_updated +from ..lib.timeframe import from_seconds +from .syminfo import SymInfo, SymInfoInterval, SymInfoSession @dataclass @@ -50,11 +54,47 @@ class DataConverter: """ SUPPORTED_FORMATS = {'csv', 'txt', 'json'} - OHLCV_MAGIC_BYTES = b'\x00\x00\x00\x00' # First 4 bytes pattern for binary OHLCV def __init__(self): pass + @staticmethod + def auto_detect_symbol_from_filename(file_path: Path) -> str | None: + """Auto-detect symbol from filename. + + :param file_path: Path to the data file + :return: Symbol name or None if not detected + """ + filename = file_path.stem # Filename without extension + + # Common patterns to try: + # 1. BTCUSD_1h.csv, AAPL_daily.csv, EUR_USD_4h.csv + # 2. BTCUSD-1h.csv, AAPL-daily.csv + # 3. btcusd.csv, aapl.csv (simple symbol only) + # 4. BTC_USD_1D.csv, EUR_USD_4H.csv (with separators) + + # Normalize separators + normalized = filename.replace('-', '_').upper() + parts = normalized.split('_') + + if len(parts) >= 1: + potential_symbol = parts[0] + + # Check if it looks like a valid symbol (3+ chars, alphanumeric) + if len(potential_symbol) >= 3 and potential_symbol.isalnum(): + # Handle common forex patterns like EUR_USD -> EURUSD + if len(parts) >= 2 and len(parts[1]) == 3 and parts[1].isalpha(): + # Likely forex pair: EUR_USD -> EURUSD + return potential_symbol + parts[1] + else: + return potential_symbol + + # Try the whole filename if it's a simple symbol + if len(filename) >= 3 and filename.replace('_', '').replace('-', '').isalnum(): + return filename.upper().replace('_', '').replace('-', '') + + return None + @staticmethod def detect_format(file_path: Path) -> Literal['csv', 'txt', 'json', 'ohlcv', 'unknown']: """Detect file format by extension and content inspection. @@ -70,34 +110,14 @@ def detect_format(file_path: Path) -> Literal['csv', 'txt', 'json', 'ohlcv', 'un # Check extension first ext = file_path.suffix.lower() if ext == '.ohlcv': - # Validate if it's actually binary OHLCV format + # Using OHLCVReader to validate. If it raises an error, it's not a valid OHLCV file. try: - with open(file_path, 'rb') as f: - # Check if file size is multiple of 24 (OHLCV record size) - file_size = f.seek(0, 2) # Seek to end - f.seek(0) # Back to start - - if file_size == 0: - return 'ohlcv' # Empty OHLCV file is valid - - if file_size % 24 != 0: - # Not a valid OHLCV file, likely renamed - return DataConverter._detect_content_format(file_path) - - # Read first record to validate structure - first_record = f.read(24) - if len(first_record) == 24: - try: - # Try to unpack as OHLCV record - struct.unpack(STRUCT_FORMAT, first_record) - return 'ohlcv' - except struct.error: - # Invalid binary format, likely renamed - return DataConverter._detect_content_format(file_path) - + with OHLCVReader(file_path): + # If we can open it successfully, it's a valid OHLCV file return 'ohlcv' - except (OSError, IOError) as e: - raise DataFormatError(f"Cannot read file {file_path}: {e}") + except (ValueError, OSError, IOError): + # Not a valid OHLCV file, likely renamed - detect by content + return DataConverter._detect_content_format(file_path) elif ext == '.csv': return 'csv' @@ -120,8 +140,6 @@ def _detect_content_format(file_path: Path) -> Literal['csv', 'txt', 'json', 'un with open(file_path, 'r', encoding='utf-8') as f: # Read first few lines for analysis first_line = f.readline().strip() - if not first_line: - return 'unknown' # Try JSON first (most specific) f.seek(0) @@ -131,19 +149,19 @@ def _detect_content_format(file_path: Path) -> Literal['csv', 'txt', 'json', 'un except (json.JSONDecodeError, UnicodeDecodeError): pass - # Check for CSV patterns - if ',' in first_line: + # Check for CSV patterns (only if we have content) + if first_line and ',' in first_line: # Count commas to see if it looks like structured data comma_count = first_line.count(',') if comma_count >= 4: # At least OHLC columns return 'csv' # Check for other delimiters (TXT) - if any(delim in first_line for delim in ['\t', ';', '|']): + if first_line and any(delim in first_line for delim in ['\t', ';', '|']): return 'txt' # Default to CSV if it has some structure - if ',' in first_line: + if first_line and ',' in first_line: return 'csv' return 'unknown' @@ -330,7 +348,8 @@ def _create_symbol_info_file( timeframe: str, timezone: str ) -> None: - """Create TOML symbol info file if it doesn't exist or is outdated. + """ + Create TOML symbol info file using SymInfo class if it doesn't exist or is outdated. :param source_path: Path to the source data file :param symbol: Symbol name (e.g., 'BTCUSD') @@ -360,88 +379,60 @@ def _create_symbol_info_file( minmove = price_analysis['min_move'] pointvalue = self._get_default_pointvalue(symbol_type) - # Create TOML content with warnings - toml_content = f"""# WARNING: This file was auto-generated from price data analysis. -# Please review and adjust the following values according to your broker's specifications: -# - mintick: Minimum price movement (currently: {mintick}) -# - pricescale: Price scale factor (currently: {pricescale}) -# - minmove: Minimum move in price scale units (currently: {minmove}) -# - pointvalue: Market value per price scale unit (currently: {pointvalue}) -# - opening_hours: Trading hours may vary by broker and market - -[symbol] -prefix = "CUSTOM" -description = "{symbol} - Auto-generated symbol info" -ticker = "{symbol_upper}" -currency = "{currency}" -""" - - # Use smart defaults for currency fields - if base_currency: - toml_content += f'basecurrency = "{base_currency}"\n' - else: - toml_content += 'basecurrency = "EUR" # Default base currency - change if needed\n' - - toml_content += f"""period = "{timeframe}" -type = "{symbol_type}" -mintick = {mintick} -pricescale = {pricescale} -minmove = {minmove} -pointvalue = {pointvalue} -timezone = "{timezone}" -volumettype = "base" -#avg_spread = -#taker_fee = -#maker_fee = -#target_price_average = -#target_price_high = -#target_price_low = -#target_price_date = - -# Opening hours (24/7 for crypto, business hours for others) -""" - + # Create opening hours based on symbol type + opening_hours = [] if symbol_type == 'crypto': # 24/7 trading for crypto for day in range(1, 8): - toml_content += f"""[[opening_hours]] -day = {day} -start = "00:00:00" -end = "23:59:59" - -""" + opening_hours.append(SymInfoInterval( + day=day, + start=time(0, 0, 0), + end=time(23, 59, 59) + )) else: # Business hours for stocks/forex (Mon-Fri) for day in range(1, 6): - toml_content += f"""[[opening_hours]] -day = {day} -start = "09:30:00" -end = "16:00:00" - -""" - - toml_content += """# Session starts -[[session_starts]] -day = 1 -time = "00:00:00" - -# Session ends -[[session_ends]] -day = 7 -time = "23:59:59" -""" + opening_hours.append(SymInfoInterval( + day=day, + start=time(9, 30, 0), + end=time(16, 0, 0) + )) + + # Create session starts and ends + session_starts = [SymInfoSession(day=1, time=time(0, 0, 0))] + session_ends = [SymInfoSession(day=7, time=time(23, 59, 59))] + + # Create SymInfo instance + syminfo = SymInfo( + prefix="CUSTOM", + description=f"{symbol} - Auto-generated symbol info", + ticker=symbol_upper, + currency=currency, + basecurrency=base_currency or "EUR", + period=timeframe, + type=symbol_type if symbol_type in ["stock", "fund", "dr", "right", "bond", + "warrant", "structured", "index", "forex", + "futures", "spread", "economic", "fundamental", + "crypto", "spot", "swap", "option", "commodity", + "other"] else "other", + mintick=mintick, + pricescale=int(pricescale), + minmove=int(minmove), + pointvalue=pointvalue, + opening_hours=opening_hours, + session_starts=session_starts, + session_ends=session_ends, + timezone=timezone, + volumetype="base" + ) - # Write TOML file + # Save using SymInfo's built-in method try: - with open(toml_path, 'w', encoding='utf-8') as f: - f.write(toml_content) - + syminfo.save_toml(toml_path) # Copy modification time from source to maintain consistency copy_mtime(source_path, toml_path) - except (OSError, IOError): # Don't fail the entire conversion if TOML creation fails - # Just log the issue (could be added later) pass def _analyze_price_data(self, source_path: Path) -> dict[str, float]: @@ -450,126 +441,71 @@ def _analyze_price_data(self, source_path: Path) -> dict[str, float]: :param source_path: Path to the source data file :return: Dictionary with tick_size, price_scale, and min_move """ - try: - # Try to import pandas for data analysis - try: - import pandas as pd # type: ignore - except ImportError: - # Pandas not available, use basic CSV analysis - return self._analyze_price_data_basic(source_path) - - # Determine file format and read data - if source_path.suffix.lower() == '.csv': - df = pd.read_csv(source_path, nrows=1000) # Sample first 1000 rows - elif source_path.suffix.lower() == '.json': - df = pd.read_json(source_path, lines=True, nrows=1000) - else: - # Default fallback values - return {'tick_size': 0.01, 'price_scale': 100, 'min_move': 1} - - # Find price columns (common names) - price_cols = [] - for col in df.columns: - col_lower = col.lower() - if any(price_name in col_lower for price_name in ['close', 'price', 'high', 'low', 'open']): - price_cols.append(col) - - if not price_cols: - # No price columns found, use defaults - return {'tick_size': 0.01, 'price_scale': 100, 'min_move': 1} - - # Use the first price column for analysis - prices = pd.to_numeric(df[price_cols[0]], errors='coerce').dropna() - - if len(prices) == 0: - return {'tick_size': 0.01, 'price_scale': 100, 'min_move': 1} - - # Calculate decimal places from price data - decimal_places = 0 - for price in prices.head(100): # Check first 100 valid prices - if pd.notna(price): - price_str = str(float(price)) - if '.' in price_str and not price_str.endswith('.0'): - current_decimals = len(price_str.split('.')[1]) - decimal_places = max(decimal_places, current_decimals) - - # Calculate price scale (10^decimal_places) - # Use at least 2 decimal places for reasonable trading precision - decimal_places = max(decimal_places, 2) - price_scale = 10 ** decimal_places - - # Calculate minimum tick size - tick_size = 1.0 / price_scale - - # Min move is typically 1 in price scale units - min_move = 1 - - return { - 'tick_size': tick_size, - 'price_scale': price_scale, - 'min_move': min_move - } - - except (ImportError, ValueError, KeyError, TypeError): - # If analysis fails, return sensible defaults - return {'tick_size': 0.01, 'price_scale': 100, 'min_move': 1} + # Always use the basic method for maximum precision without external dependencies + return self._analyze_price_data_basic(source_path) @staticmethod def _analyze_price_data_basic(source_path: Path) -> dict[str, float]: - """Basic price data analysis without pandas dependency. + """Basic price data analysis by converting to OHLCV and using OHLCVReader. :param source_path: Path to the source data file :return: Dictionary with tick_size, price_scale, and min_move """ try: - import csv - - if source_path.suffix.lower() != '.csv': - return {'tick_size': 0.01, 'price_scale': 100, 'min_move': 1} - - with open(source_path, 'r', encoding='utf-8') as f: - reader = csv.DictReader(f) + # Convert file to temporary OHLCV format first + with tempfile.NamedTemporaryFile(suffix='.ohlcv', delete=False) as temp_file: + temp_ohlcv_path = Path(temp_file.name) - # Find price column - price_col = None - for col in reader.fieldnames or []: - col_lower = col.lower() - if any(price_name in col_lower for price_name in ['close', 'price', 'high', 'low', 'open']): - price_col = col - break - - if not price_col: - return {'tick_size': 0.01, 'price_scale': 100, 'min_move': 1} - - # Analyze first 100 rows - decimal_places = 0 - count = 0 - for row in reader: - if count >= 100: - break - - try: - price = float(row[price_col]) - price_str = f"{price:.10f}".rstrip('0') - if '.' in price_str: - current_decimals = len(price_str.split('.')[1]) - decimal_places = max(decimal_places, current_decimals) - count += 1 - except (ValueError, KeyError): - continue - - # Calculate parameters - price_scale = 10 ** decimal_places - tick_size = 1.0 / price_scale if decimal_places > 0 else 1.0 - min_move = 1 - - return { - 'tick_size': tick_size, - 'price_scale': price_scale, - 'min_move': min_move - } - - except (OSError, IOError, ValueError, KeyError, TypeError): + try: + # Use existing conversion logic + with OHLCVWriter(temp_ohlcv_path) as ohlcv_writer: + if source_path.suffix.lower() == '.csv': + ohlcv_writer.load_from_csv(source_path, tz='UTC') + elif source_path.suffix.lower() == '.json': + ohlcv_writer.load_from_json(source_path, tz='UTC') + else: + # Treat as CSV with different delimiters + ohlcv_writer.load_from_csv(source_path, tz='UTC') + + # Now analyze price data using OHLCVReader + with OHLCVReader(temp_ohlcv_path) as reader: + if reader.size == 0: + return {'tick_size': 0.01, 'price_scale': 100, 'min_move': 1} + + # Analyze first 100 records (or all if less) + decimal_places = 0 + sample_size = min(100, reader.size) + + for i in range(sample_size): + ohlcv = reader.read(i) + # Check all price fields for decimal precision + for price in [ohlcv.open, ohlcv.high, ohlcv.low, ohlcv.close]: + # Detect actual decimal precision without artificial limits + if price != int(price): # Has decimal component + # Use high precision to capture actual decimal places + price_str = f"{price:.15f}".rstrip('0').rstrip('.') + if '.' in price_str: + current_decimals = len(price_str.split('.')[1]) + decimal_places = max(decimal_places, current_decimals) + + # Calculate parameters preserving full precision + decimal_places = max(decimal_places, 2) # At least 2 decimal places for safety + price_scale = 10 ** decimal_places + tick_size = 1.0 / price_scale + min_move = 1 + + return { + 'tick_size': tick_size, + 'price_scale': price_scale, + 'min_move': min_move + } + + finally: + # Clean up temporary file + if temp_ohlcv_path.exists(): + temp_ohlcv_path.unlink() + + except (OSError, IOError, ValueError, ConversionError): return {'tick_size': 0.01, 'price_scale': 100, 'min_move': 1} @staticmethod @@ -628,9 +564,9 @@ def _detect_timeframe_from_ohlcv(self, ohlcv_path: Path, requested_timeframe: st record = f.read(24) # OHLCV record size if len(record) < 24: break - - # Unpack timestamp (first 8 bytes as uint64) - timestamp = struct.unpack(' 0: # Only positive differences differences.append(diff) @@ -647,51 +583,34 @@ def _detect_timeframe_from_ohlcv(self, ohlcv_path: Path, requested_timeframe: st return "1D" # Default fallback # Find the most common difference (mode) - from collections import Counter diff_counts = Counter(differences) most_common_diff = diff_counts.most_common(1)[0][0] - # Convert nanoseconds to timeframe string - return self._nanoseconds_to_timeframe(most_common_diff) + # Convert seconds to timeframe string (timestamps are in seconds, not nanoseconds) + return self._seconds_to_timeframe(most_common_diff) except (OSError, IOError, struct.error, IndexError, ValueError): # If detection fails, return default return "1D" @staticmethod - def _nanoseconds_to_timeframe(nanoseconds: int) -> str: - """Convert nanoseconds difference to timeframe string. + def _seconds_to_timeframe(seconds: int) -> str: + """ + Convert seconds difference to timeframe string using TV-compatible timeframe module. - :param nanoseconds: Time difference in nanoseconds + :param seconds: Time difference in seconds :return: Timeframe string (e.g., '1m', '5m', '1h', '1D') """ - # Convert to seconds - seconds = nanoseconds / 1_000_000_000 - - # Define common timeframes in seconds - timeframes = [ - (60, "1m"), - (300, "5m"), - (900, "15m"), - (1800, "30m"), - (3600, "1h"), - (14400, "4h"), - (86400, "1D"), - (604800, "1W"), - (2592000, "1M"), # Approximate month - ] - - # Find the closest match - best_match = "1D" # Default - min_diff = float('inf') - - for tf_seconds, tf_string in timeframes: - diff = abs(seconds - tf_seconds) - if diff < min_diff: - min_diff = diff - best_match = tf_string + # Handle edge cases + if seconds <= 0: + return "1D" # Default fallback - return best_match + # Use TV-compatible timeframe conversion + try: + return from_seconds(seconds) + except (ValueError, AssertionError): + # Fallback to closest standard timeframe if conversion fails + return "1D" def _detect_timeframe_from_data(self, df, requested_timeframe: str) -> str: """Detect timeframe from DataFrame by analyzing timestamp differences. @@ -702,31 +621,30 @@ def _detect_timeframe_from_data(self, df, requested_timeframe: str) -> str: """ if requested_timeframe.upper() != "AUTO": return requested_timeframe - + try: if len(df) < 2: return "1D" # Default fallback - + # Calculate differences between consecutive timestamps timestamps = df['timestamp'].values differences = [] - + for i in range(1, min(len(timestamps), 10)): # Check first 10 records - diff = timestamps[i] - timestamps[i-1] + diff = timestamps[i] - timestamps[i - 1] if diff > 0: # Only positive differences differences.append(diff) - + if not differences: return "1D" # Default fallback - + # Find the most common difference (mode) - from collections import Counter diff_counts = Counter(differences) most_common_diff = diff_counts.most_common(1)[0][0] - - # Convert nanoseconds to timeframe string - return self._nanoseconds_to_timeframe(most_common_diff) - + + # Convert seconds to timeframe string (assuming timestamps are in seconds) + return self._seconds_to_timeframe(most_common_diff) + except (KeyError, IndexError, ValueError, TypeError): # If detection fails, return default return "1D" diff --git a/src/pynecore/core/ohlcv_file.py b/src/pynecore/core/ohlcv_file.py index 90592f3..b583c74 100644 --- a/src/pynecore/core/ohlcv_file.py +++ b/src/pynecore/core/ohlcv_file.py @@ -25,7 +25,7 @@ RECORD_SIZE = 24 # 6 * 4 STRUCT_FORMAT = 'Ifffff' # I: uint32, f: float32 -__all__ = ['OHLCVWriter', 'OHLCVReader', 'STRUCT_FORMAT'] +__all__ = ['OHLCVWriter', 'OHLCVReader'] def format_float(value: float) -> str: diff --git a/tests/t00_pynecore/data/test_001_ohlcv_file.py b/tests/t00_pynecore/data/test_001_ohlcv_file.py index e68e696..0dde804 100644 --- a/tests/t00_pynecore/data/test_001_ohlcv_file.py +++ b/tests/t00_pynecore/data/test_001_ohlcv_file.py @@ -275,6 +275,23 @@ def __test_ohlcv_reader_from_to__(tmp_path): assert candles[-1].timestamp == 1609459500 +def __test_ohlcv_reader_rejects_text_disguised_as_ohlcv__(tmp_path): + """OHLCVReader should raise a clear error when opening a text file renamed to .ohlcv.""" + file_path = tmp_path / "fake_text.ohlcv" + + # Arrange: create a plain-text CSV-like file but with .ohlcv extension + with open(file_path, "w", encoding="utf-8") as f: + f.write("timestamp,open,high,low,close,volume\n") + f.write("1609459200,100,110,90,105,1000\n") + + # Act & Assert: opening via OHLCVReader should fail with a helpful message + with pytest.raises(ValueError) as excinfo: + with OHLCVReader(file_path) as _: + pass # Should not reach here + + assert "Text file detected with .ohlcv extension" in str(excinfo.value) + + def __test_chronological_order_validation__(tmp_path): """Test validation of chronological order in timestamps""" file_path = tmp_path / "test_chronological.ohlcv" From 8cf76ad1c807998752c9193db2d1175f973e307b Mon Sep 17 00:00:00 2001 From: Adam Wallner Date: Sun, 17 Aug 2025 18:34:44 +0200 Subject: [PATCH 23/31] feat(data): add advanced OHLCV analysis and improved symbol handling Major enhancements: - Add comprehensive trading hours detection and analysis in OHLCVWriter - Implement advanced tick size detection with histogram clustering - Add interval auto-correction for files starting with gaps - Extend symbol and provider detection for more filename formats OHLCVWriter improvements: - New trading hours collection and analysis system - Samples existing files on open to detect trading patterns - Smart data sufficiency checks based on timeframe - Detects 24/7 (crypto) vs business hours patterns - Advanced tick size detection with confidence scoring - Automatic interval correction when smaller intervals detected DataConverter updates: - Rename symbol_from_filename to guess_symbol_from_filename (public API) - Rename _detect_symbol_type to guess_symbol_type (public API) - Enhanced filename pattern matching for various formats - Improved provider detection from filename patterns - Better handling of complex patterns like ccxt_BYBIT_BTC_USDT_USDT CLI improvements: - Update data.py and run.py to use new guess_symbol_from_filename API - Remove AUTO timeframe option, simplify validation - Better error handling and user feedback Bug fixes: - Fix timeframe.from_seconds() missing minute format output - Remove redundant symbol type validation Tests: - Add comprehensive DataConverter tests - Add symbol type detection tests - Add timeframe from_seconds tests - Add trading hours detection tests in OHLCV file --- src/pynecore/cli/commands/data.py | 182 ++- src/pynecore/cli/commands/run.py | 34 +- src/pynecore/core/data_converter.py | 1088 +++++++++-------- src/pynecore/core/ohlcv_file.py | 697 ++++++++++- src/pynecore/lib/timeframe.py | 4 +- .../t00_pynecore/data/test_001_ohlcv_file.py | 143 +++ .../data/test_004_data_converter.py | 297 +++++ .../data/test_005_symbol_type_detection.py | 229 ++++ .../t01_timeframe/test_003_from_seconds.py | 34 + 9 files changed, 2033 insertions(+), 675 deletions(-) create mode 100644 tests/t00_pynecore/data/test_004_data_converter.py create mode 100644 tests/t00_pynecore/data/test_005_symbol_type_detection.py create mode 100644 tests/t01_lib/t01_timeframe/test_003_from_seconds.py diff --git a/src/pynecore/cli/commands/data.py b/src/pynecore/cli/commands/data.py index aa9f6b1..7a453b0 100644 --- a/src/pynecore/cli/commands/data.py +++ b/src/pynecore/cli/commands/data.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, TypeAlias from pathlib import Path from enum import Enum from datetime import datetime, timedelta, UTC @@ -14,48 +14,57 @@ from ...providers import available_providers from ...providers.provider import Provider from ...lib.timeframe import in_seconds -from ...core.data_converter import DataConverter +from ...core.data_converter import DataConverter, SupportedFormats as InputFormats +from ...core.ohlcv_file import OHLCVReader from ...utils.rich.date_column import DateColumn -from pynecore.core.ohlcv_file import OHLCVReader __all__ = [] app_data = Typer(help="OHLCV related commands") app.add_typer(app_data, name="data") -# Create an enum from it -AvailableProvidersEnum = Enum('Provider', {name.upper(): name.lower() for name in available_providers}) +# Trick to avoid type checking errors +if TYPE_CHECKING: + DateOrDays: TypeAlias = datetime + + + class AvailableProvidersEnum(Enum): + ... + +else: + # DateOrDays is either a datetime or a number of days + DateOrDays = str + + # Create an enum from available providers + AvailableProvidersEnum = Enum('Provider', {name.upper(): name.lower() for name in available_providers}) + + +# Available output formats +class OutputFormat(Enum): + CSV = 'csv' + JSON = 'json' # TV-compatible timeframe validation function def validate_timeframe(value: str) -> str: """ Validate TV-compatible timeframe string. - + :param value: Timeframe string to validate :return: Validated timeframe string :raises ValueError: If timeframe is invalid """ - if value.upper() == 'AUTO': - return value.upper() - + value = value.upper() try: # Test if it's a valid TV timeframe by trying to convert to seconds in_seconds(value) - return value except (ValueError, AssertionError): - # Fallback to common timeframes for validation - valid_timeframes = ['1', '2', '3', '5', '10', '15', '30', '45', '60', '120', '180', '240', '360', '480', '720', - '1D', '1W', '1M', '1S', '2S', '5S', '10S', '15S', '30S', 'AUTO'] - if value in valid_timeframes: - return value raise ValueError( - f"Invalid timeframe: {value}. Must be a valid TradingView timeframe (e.g., '1', '5S', '1D', '1W') or 'AUTO'") - - -# Trick to avoid type checking errors -DateOrDays = datetime if TYPE_CHECKING else str + f"Invalid timeframe: {value}. Must be a valid timeframe in TradingView format " + f"(e.g. '1', '5', '60', '1D', '1W', '1M')." + ) + return value def parse_date_or_days(value: str) -> datetime | str: @@ -84,7 +93,7 @@ def parse_date_or_days(value: str) -> datetime | str: @app_data.command() def download( - provider: AvailableProvidersEnum = Argument(..., case_sensitive=False, show_default=False, # type: ignore + provider: AvailableProvidersEnum = Argument(..., case_sensitive=False, show_default=False, help="Data provider"), symbol: str | None = Option(None, '--symbol', '-s', show_default=False, help="Symbol (e.g. BYBIT:BTC/USDT:USDT)"), @@ -92,17 +101,18 @@ def download( help="List available symbols of the provider"), timeframe: str = Option('1D', '--timeframe', '-tf', callback=validate_timeframe, help="Timeframe in TradingView format (e.g., '1', '5S', '1D', '1W')"), - time_from: DateOrDays = Option("continue", '--from', '-f', # type: ignore + time_from: DateOrDays = Option("continue", '--from', '-f', callback=parse_date_or_days, formats=[], metavar="[%Y-%m-%d|%Y-%m-%d %H:%M:%S|NUMBER]|continue", help="Start date or days back from now, or 'continue' to resume last download," " or one year if no data"), - time_to: DateOrDays = Option(datetime.now(UTC).replace(second=0, microsecond=0), '--to', '-t', # type: ignore + time_to: DateOrDays = Option(datetime.now(UTC).replace(second=0, microsecond=0), '--to', '-t', callback=parse_date_or_days, formats=[], metavar="[%Y-%m-%d|%Y-%m-%d %H:%M:%S|NUMBER]", help="End date or days from start date"), show_info: bool = Option(False, '--symbol-info', '-si', help="Show symbol info"), - force_save_info: bool = Option(False, '--force-save-info', '-fi', help="Force save symbol info"), + force_save_info: bool = Option(False, '--force-save-info', '-fi', + help="Force save symbol info"), truncate: bool = Option(False, '--truncate', '-tr', help="Truncate file before downloading, all data will be lost"), ): @@ -224,13 +234,9 @@ def cb_progress(current_time: datetime): @app_data.command() def convert_to( - provider: AvailableProvidersEnum = Argument(..., case_sensitive=False, show_default=False, # type: ignore - help="Data provider"), - symbol: str | None = Option(None, '--symbol', '-s', show_default=False, - help="Symbol (e.g. BYBIT:BTCUSDT:USDT)"), - timeframe: str = Option('1D', '--timeframe', '-tf', callback=validate_timeframe, - help="Timeframe in TradingView format (e.g., '1', '5S', '1D', '1W')"), - fmt: Enum('Format', {'csv': 'csv', 'json': 'json'}) = Option( # noqa # type: ignore + ohlcv_path: Path = Argument(..., dir_okay=False, file_okay=True, + help="Data file to convert (*.ohlcv)"), + fmt: OutputFormat = Option( 'csv', '--format', '-f', case_sensitive=False, help="Output format"), @@ -240,120 +246,106 @@ def convert_to( """ Convert downloaded data from pyne's OHLCV format to another format """ - # Import provider module from - provider_module = __import__(f"pynecore.providers.{provider.value}", fromlist=['']) - provider_class = getattr(provider_module, [p for p in dir(provider_module) if p.endswith('Provider')][0]) - ohlcv_path = provider_class.get_ohlcv_path(symbol, timeframe, app_state.data_dir) + # Check file format and extension + if ohlcv_path.suffix == "": + # No extension, add .ohlcv + ohlcv_path = ohlcv_path.with_suffix(".ohlcv") + + # Expand data path + if len(ohlcv_path.parts) == 1: + ohlcv_path = app_state.data_dir / ohlcv_path + # Check if data exists + if not ohlcv_path.exists(): + secho(f"Data file '{ohlcv_path}' not found!", fg="red", err=True) + raise Exit(1) + out_path = None with Progress(SpinnerColumn(finished_text="[green]✓"), TextColumn("{task.description}")) as progress: # Convert with OHLCVReader(str(ohlcv_path)) as ohlcv_reader: - if fmt.value == 'csv': + if fmt.value == OutputFormat.CSV.value: task = progress.add_task(description="Converting to CSV...", total=1) - ohlcv_reader.save_to_csv(str(ohlcv_path.with_suffix('.csv')), as_datetime=as_datetime) + out_path = str(ohlcv_path.with_suffix('.csv')) + ohlcv_reader.save_to_csv(out_path, as_datetime=as_datetime) - elif fmt.value == 'json': + elif fmt.value == OutputFormat.JSON.value: task = progress.add_task(description="Converting to JSON...", total=1) - ohlcv_reader.save_to_json(str(ohlcv_path.with_suffix('.json')), as_datetime=as_datetime) + out_path = str(ohlcv_path.with_suffix('.json')) + ohlcv_reader.save_to_json(out_path, as_datetime=as_datetime) + + else: + raise ValueError(f"Unsupported format: {fmt}") # Complete task progress.update(task, completed=1) + if out_path: + secho(f'Data file converted successfully to "{out_path}"!') + @app_data.command() def convert_from( - file_path: Path = Argument(..., help="Path to CSV/JSON file to convert"), - provider: str = Option("custom", '--provider', '-p', + file_path: Path = Argument(..., help="Path to CSV/JSON/TXT file to convert"), + provider: str = Option(None, '--provider', '-p', help="Data provider, can be any name"), symbol: str | None = Option(None, '--symbol', '-s', show_default=False, - help="Symbol (default: auto-detected from filename)"), - timeframe: str | None = Option(None, '--timeframe', '-tf', - help="Timeframe (default: auto-detected from data, e.g., '1', '5S', '1D', '1W')"), - fmt: Enum('Format', {'csv': 'csv', 'json': 'json'}) | None = Option( # noqa # type: ignore - None, '--fmt', '-f', - case_sensitive=False, - help="Input format (auto-detected from file extension if not provided)"), - tz: str = Option('UTC', '--timezone', '-tz', - help="Timezone"), + help="Symbol (default: from file name)"), + tz: str = Option('UTC', '--timezone', '-tz', help="Timezone"), ): """ - Convert data from other sources to pyne's OHLCV format with automatic symbol detection + Convert data from other sources to pyne's OHLCV format """ - # Expand file path if only filename is provided (look in workdir/data) if len(file_path.parts) == 1: file_path = app_state.data_dir / file_path # Check if file exists if not file_path.exists(): - secho(f"File '{file_path}' not found!", fg=colors.RED, err=True) + secho(f'File "{file_path}" not found!', fg=colors.RED, err=True) raise Exit(1) - # Auto-detect symbol from filename if not provided - if symbol is None: - symbol = DataConverter.auto_detect_symbol_from_filename(file_path) + # Auto-detect symbol and provider from filename if not provided + detected_symbol, detected_provider = DataConverter.guess_symbol_from_filename(file_path) - # Use AUTO for timeframe detection from data if not provided - if timeframe is None: - timeframe = "AUTO" + if symbol is None: + symbol = detected_symbol - # Validate timeframe if provided (skip validation for AUTO) - if timeframe is not None and timeframe.upper() != "AUTO": - try: - validate_timeframe(timeframe) - except ValueError as e: - secho(f"Invalid timeframe '{timeframe}': {e}", fg=colors.RED, err=True) - raise Exit(1) + if provider is None and detected_provider is not None: + provider = detected_provider # Ensure we have required parameters if symbol is None: secho(f"Error: Could not detect symbol from filename '{file_path.name}'!", fg=colors.RED, err=True) - secho("Please provide a symbol using --symbol option or rename your file to include the symbol.", - fg=colors.YELLOW, err=True) - secho("Example: 'BTCUSD_1h.csv' or use '--symbol BTCUSD'", fg=colors.YELLOW, err=True) + secho("Please provide a symbol using --symbol option.", fg=colors.YELLOW, err=True) raise Exit(1) - # Auto-detect format from file extension - if fmt is None: - file_ext = file_path.suffix[1:].lower() - if file_ext in ['csv', 'json']: - fmt = file_ext - else: - fmt = 'csv' # Default to CSV - else: - fmt = fmt.value + # Auto-detect file format + fmt = file_path.suffix[1:].lower() + if fmt not in InputFormats: + raise ValueError(f"Unsupported file format: {file_path}") # Use the enhanced DataConverter for automatic conversion converter = DataConverter() try: with Progress(SpinnerColumn(finished_text="[green]✓"), TextColumn("{task.description}")) as progress: - task = progress.add_task(description=f"Converting {fmt.upper() if fmt else 'CSV'} to OHLCV format...", - total=1) - - # Convert timeframe to string value - timeframe_str = "1D" # Default - if timeframe is not None: - timeframe_str = str(timeframe) + task = progress.add_task(description=f"Converting {fmt.upper()} to OHLCV format...", total=1) # Perform conversion with automatic TOML generation - result = converter.convert_if_needed( + converter.convert_to_ohlcv( file_path=Path(file_path), provider=provider, symbol=symbol, - timeframe=timeframe_str, - timezone=tz + timezone=tz, + force=True ) progress.update(task, completed=1) - # Show success message with generated files - secho(f"✓ Converted to: {result.ohlcv_path}", fg=colors.GREEN) - toml_path = file_path.with_suffix('.toml') - if toml_path.exists(): - secho(f"✓ Generated symbol info: {toml_path}", fg=colors.GREEN) - secho("⚠️ Please review the auto-generated symbol parameters in the .toml file", fg=colors.YELLOW) - except Exception as e: secho(f"Error: {e}", err=True, fg=colors.RED) raise Exit(1) + + secho(f'Data file converted successfully to "{file_path}".') + secho(f'A configuration file was automatically generated for you at "{file_path.with_suffix(".toml")}". ' + f'Please check it and adjust it to match your needs.') diff --git a/src/pynecore/cli/commands/run.py b/src/pynecore/cli/commands/run.py index fd366df..851adcf 100644 --- a/src/pynecore/cli/commands/run.py +++ b/src/pynecore/cli/commands/run.py @@ -186,16 +186,11 @@ def run( # Check if conversion is needed if converter.is_conversion_required(data): - # Auto-detect symbol from filename using existing function - detected_symbol = DataConverter.auto_detect_symbol_from_filename(data) + # Auto-detect symbol and provider from filename + detected_symbol, detected_provider = DataConverter.guess_symbol_from_filename(data) + if not detected_symbol: - # Use filename without extension as default symbol - default_symbol = data.stem.upper() - secho(f"Warning: Could not detect symbol from filename '{data.name}'", fg="yellow", err=True) - secho(f"Using default symbol: '{default_symbol}'", fg="yellow") - else: - default_symbol = detected_symbol - default_timeframe = "AUTO" # Auto-detect timeframe from data + detected_symbol = data.stem.upper() with Progress( SpinnerColumn(finished_text="[green]✓"), @@ -205,25 +200,20 @@ def run( task = progress.add_task(f"Converting {data.suffix} to OHLCV format...", total=1) # Perform conversion with smart defaults - result = converter.convert_if_needed( + converter.convert_to_ohlcv( data, - symbol=default_symbol, - timeframe=default_timeframe + provider=detected_provider, + symbol=detected_symbol, + force=True ) - progress.update(task, completed=1) + # After conversion, the OHLCV file has the same name but .ohlcv extension + data = data.with_suffix(".ohlcv") - if result.converted: - console.print(f"[green]✓[/green] Converted {data} to {result.ohlcv_path}") - data = result.ohlcv_path - else: - console.print(f"[blue]ℹ[/blue] Using existing OHLCV file: {result.ohlcv_path}") - data = result.ohlcv_path + progress.update(task, completed=1) else: # File is already up-to-date, use existing OHLCV file - ohlcv_path = data.with_suffix(".ohlcv") - console.print(f"[blue]ℹ[/blue] Using existing OHLCV file: {ohlcv_path}") - data = ohlcv_path + data = data.with_suffix(".ohlcv") except (DataFormatError, ConversionError) as e: secho(f"Conversion failed: {e}", fg="red", err=True) diff --git a/src/pynecore/core/data_converter.py b/src/pynecore/core/data_converter.py index ab76714..b2afc06 100644 --- a/src/pynecore/core/data_converter.py +++ b/src/pynecore/core/data_converter.py @@ -1,16 +1,13 @@ -"""Automatic data file to OHLCV conversion functionality. +""" +Automatic data file to OHLCV conversion functionality. This module provides automatic detection and conversion of CSV, TXT, and JSON files to OHLCV format when needed, eliminating the manual step of running pyne data convert. """ - from __future__ import annotations import json -import struct -import tempfile -from collections import Counter -from dataclasses import dataclass +from enum import Enum from datetime import time from pathlib import Path from typing import Literal @@ -21,21 +18,6 @@ from .syminfo import SymInfo, SymInfoInterval, SymInfoSession -@dataclass -class ConversionResult: - """Result of a conversion operation. - - :param converted: Whether conversion was performed - :param ohlcv_path: Path to the OHLCV file - :param source_path: Path to the source file - :param was_renamed: Whether the file was renamed from incorrect .ohlcv - """ - converted: bool - ohlcv_path: Path - source_path: Path - was_renamed: bool = False - - class DataFormatError(Exception): """Raised when file format cannot be detected or is unsupported.""" pass @@ -46,133 +28,26 @@ class ConversionError(Exception): pass +class SupportedFormats(Enum): + """Supported data file formats.""" + CSV = 'csv' + TXT = 'txt' + JSON = 'json' + + class DataConverter: - """Main class for automatic data file conversion. - + """ + Main class for automatic data file conversion. + Provides both CLI and programmatic interfaces for converting CSV, TXT, and JSON files to OHLCV format automatically. """ - SUPPORTED_FORMATS = {'csv', 'txt', 'json'} - - def __init__(self): - pass - @staticmethod - def auto_detect_symbol_from_filename(file_path: Path) -> str | None: - """Auto-detect symbol from filename. - - :param file_path: Path to the data file - :return: Symbol name or None if not detected - """ - filename = file_path.stem # Filename without extension - - # Common patterns to try: - # 1. BTCUSD_1h.csv, AAPL_daily.csv, EUR_USD_4h.csv - # 2. BTCUSD-1h.csv, AAPL-daily.csv - # 3. btcusd.csv, aapl.csv (simple symbol only) - # 4. BTC_USD_1D.csv, EUR_USD_4H.csv (with separators) - - # Normalize separators - normalized = filename.replace('-', '_').upper() - parts = normalized.split('_') - - if len(parts) >= 1: - potential_symbol = parts[0] - - # Check if it looks like a valid symbol (3+ chars, alphanumeric) - if len(potential_symbol) >= 3 and potential_symbol.isalnum(): - # Handle common forex patterns like EUR_USD -> EURUSD - if len(parts) >= 2 and len(parts[1]) == 3 and parts[1].isalpha(): - # Likely forex pair: EUR_USD -> EURUSD - return potential_symbol + parts[1] - else: - return potential_symbol - - # Try the whole filename if it's a simple symbol - if len(filename) >= 3 and filename.replace('_', '').replace('-', '').isalnum(): - return filename.upper().replace('_', '').replace('-', '') - - return None - - @staticmethod - def detect_format(file_path: Path) -> Literal['csv', 'txt', 'json', 'ohlcv', 'unknown']: - """Detect file format by extension and content inspection. - - :param file_path: Path to the file to analyze - :return: Detected format - :raises FileNotFoundError: If file doesn't exist - :raises DataFormatError: If file cannot be read - """ - if not file_path.exists(): - raise FileNotFoundError(f"File not found: {file_path}") - - # Check extension first - ext = file_path.suffix.lower() - if ext == '.ohlcv': - # Using OHLCVReader to validate. If it raises an error, it's not a valid OHLCV file. - try: - with OHLCVReader(file_path): - # If we can open it successfully, it's a valid OHLCV file - return 'ohlcv' - except (ValueError, OSError, IOError): - # Not a valid OHLCV file, likely renamed - detect by content - return DataConverter._detect_content_format(file_path) - - elif ext == '.csv': - return 'csv' - elif ext == '.txt': - return 'txt' - elif ext == '.json': - return 'json' - - # No extension or unknown extension, inspect content - return DataConverter._detect_content_format(file_path) - - @staticmethod - def _detect_content_format(file_path: Path) -> Literal['csv', 'txt', 'json', 'unknown']: - """Detect format by inspecting file content. - - :param file_path: Path to the file - :return: Detected format based on content + def is_conversion_required(source_path: Path, ohlcv_path: Path | None = None) -> bool: """ - try: - with open(file_path, 'r', encoding='utf-8') as f: - # Read first few lines for analysis - first_line = f.readline().strip() - - # Try JSON first (most specific) - f.seek(0) - try: - json.load(f) - return 'json' - except (json.JSONDecodeError, UnicodeDecodeError): - pass - - # Check for CSV patterns (only if we have content) - if first_line and ',' in first_line: - # Count commas to see if it looks like structured data - comma_count = first_line.count(',') - if comma_count >= 4: # At least OHLC columns - return 'csv' - - # Check for other delimiters (TXT) - if first_line and any(delim in first_line for delim in ['\t', ';', '|']): - return 'txt' - - # Default to CSV if it has some structure - if first_line and ',' in first_line: - return 'csv' + Check if conversion is required based on file freshness. - return 'unknown' - - except (OSError, IOError, UnicodeDecodeError): - return 'unknown' - - @staticmethod - def is_conversion_required(source_path: Path, ohlcv_path: Path | None = None) -> bool: - """Check if conversion is required based on file freshness. - :param source_path: Path to the source file :param ohlcv_path: Path to the OHLCV file (auto-generated if None) :return: True if conversion is needed @@ -187,25 +62,23 @@ def is_conversion_required(source_path: Path, ohlcv_path: Path | None = None) -> # Use existing file utility to check if source is newer return is_updated(source_path, ohlcv_path) - def convert_if_needed( + def convert_to_ohlcv( self, file_path: Path, *, force: bool = False, - provider: str = "custom", + provider: str | None = None, symbol: str | None = None, - timeframe: str = "1D", timezone: str = "UTC" - ) -> ConversionResult: - """Convert file to OHLCV format if needed. - + ) -> None: + """ + Convert multiple file formats to OHLCV format. + :param file_path: Path to the data file :param force: Force conversion even if OHLCV file is up-to-date :param provider: Data provider name for OHLCV file naming :param symbol: Symbol for OHLCV file naming - :param timeframe: Timeframe for conversion :param timezone: Timezone for timestamp conversion - :return: ConversionResult with conversion details :raises FileNotFoundError: If source file doesn't exist :raises DataFormatError: If file format is unsupported :raises ConversionError: If conversion fails @@ -216,171 +89,217 @@ def convert_if_needed( # Detect file format detected_format = self.detect_format(file_path) - # Handle incorrectly renamed files - was_renamed = False - if file_path.suffix == '.ohlcv' and detected_format in self.SUPPORTED_FORMATS: - # File was incorrectly renamed, fix it - original_path = file_path.with_suffix(f'.{detected_format}') - file_path.rename(original_path) - file_path = original_path - was_renamed = True - # If it's already OHLCV, no conversion needed if detected_format == 'ohlcv': - return ConversionResult( - converted=False, - ohlcv_path=file_path, - source_path=file_path, - was_renamed=was_renamed - ) + raise ConversionError(f"Source file is already in OHLCV format: {file_path}") # Check if format is supported - if detected_format not in self.SUPPORTED_FORMATS: - raise DataFormatError( - f"Unsupported file format '{detected_format}' for file: {file_path}" - ) + if detected_format not in SupportedFormats: + raise DataFormatError(f"Unsupported file format '{detected_format}' for file: {file_path}") # Determine OHLCV output path ohlcv_path = file_path.with_suffix('.ohlcv') # Check if conversion is needed if not force and not self.is_conversion_required(file_path, ohlcv_path): - return ConversionResult( - converted=False, - ohlcv_path=ohlcv_path, - source_path=file_path, - was_renamed=was_renamed - ) - - # Perform conversion - self._convert_file( - source_path=file_path, - ohlcv_path=ohlcv_path, - format_type=detected_format, - provider=provider, - symbol=symbol, - timeframe=timeframe, - timezone=timezone - ) - - return ConversionResult( - converted=True, - ohlcv_path=ohlcv_path, - source_path=file_path, - was_renamed=was_renamed - ) - - def _convert_file( - self, - source_path: Path, - ohlcv_path: Path, - format_type: str, - provider: str, # Currently unused but kept for future extensibility - symbol: str | None, - timeframe: str, - timezone: str - ) -> None: - """Perform the actual file conversion. - - :param source_path: Path to source file - :param ohlcv_path: Path to output OHLCV file - :param format_type: Detected format type - :param provider: Data provider name (reserved for future use) - :param symbol: Symbol name - :param timeframe: Timeframe (will be auto-detected if "AUTO") - :param timezone: Timezone - :raises ConversionError: If conversion fails - """ - # Create temporary file for atomic operation - temp_path = None + return + + # Auto-detect symbol and provider from filename if not provided + if symbol is None or provider is None: + detected_symbol, detected_provider = self.guess_symbol_from_filename(file_path) + if symbol is None: + symbol = detected_symbol + if provider is None and detected_provider is not None: + provider = detected_provider + + # Use default provider if not specified + if provider is None: + provider = "CUSTOM" + + analyzed_tick_size = None + analyzed_price_scale = None + analyzed_min_move = None + detected_timeframe = None + try: - with tempfile.NamedTemporaryFile( - mode='wb', - suffix='.ohlcv', - dir=ohlcv_path.parent, - delete=False - ) as temp_file: - temp_path = Path(temp_file.name) - - # Perform conversion using existing OHLCV writer - with OHLCVWriter(temp_path) as ohlcv_writer: - if format_type == 'csv': - ohlcv_writer.load_from_csv(source_path, tz=timezone) - elif format_type == 'json': - ohlcv_writer.load_from_json(source_path, tz=timezone) - elif format_type == 'txt': - # Treat TXT files as CSV with different delimiters - ohlcv_writer.load_from_csv(source_path, tz=timezone) + # Perform conversion directly to target file with truncate to clear existing data + with OHLCVWriter(ohlcv_path, truncate=True) as ohlcv_writer: + if detected_format == 'csv': + ohlcv_writer.load_from_csv(file_path, tz=timezone) + elif detected_format == 'json': + ohlcv_writer.load_from_json(file_path, tz=timezone) + # TODO: Add support for TXT! + # elif detected_format == 'txt': + # # Treat TXT files as CSV with different delimiters + # ohlcv_writer.load_from_csv(file_path, tz=timezone) else: - raise ConversionError(f"Unsupported format for conversion: {format_type}") + raise ConversionError(f"Unsupported format for conversion: {detected_format}") - # Auto-detect timeframe if needed - detected_timeframe = self._detect_timeframe_from_ohlcv(temp_path, timeframe) + # Get timeframe directly from writer + if ohlcv_writer.interval is None: + raise ConversionError("Cannot determine timeframe from OHLCV file (less than 2 records)") + try: + detected_timeframe = from_seconds(ohlcv_writer.interval) + except (ValueError, AssertionError): + raise ConversionError( + f"Cannot convert interval {ohlcv_writer.interval} seconds to valid timeframe") - # Atomic rename to final location - temp_path.replace(ohlcv_path) - temp_path = None # Prevent cleanup + # Get analyzed tick size data from writer + analyzed_tick_size = ohlcv_writer.analyzed_tick_size + analyzed_price_scale = ohlcv_writer.analyzed_price_scale + analyzed_min_move = ohlcv_writer.analyzed_min_move # Copy modification time from source to maintain freshness - copy_mtime(source_path, ohlcv_path) + copy_mtime(file_path, ohlcv_path) # Generate TOML symbol info file if needed - self._create_symbol_info_file( - source_path=source_path, - symbol=symbol, - timeframe=detected_timeframe, - timezone=timezone - ) + toml_path = file_path.with_suffix('.toml') + + # Skip if TOML file exists and is newer than source (unless force is True) + if symbol and (force or not toml_path.exists() or is_updated(file_path, toml_path)): + # Use analyzed values from OHLCVWriter + if analyzed_tick_size: + mintick = analyzed_tick_size + pricescale = analyzed_price_scale or int(round(1.0 / analyzed_tick_size)) + minmove = analyzed_min_move or 1 + else: + # Fallback to safe defaults if analysis failed + mintick = 0.01 + pricescale = 100 + minmove = 1 + + # Determine symbol type based on symbol name patterns + symbol_upper = symbol.upper() + symbol_type, currency, base_currency = self.guess_symbol_type(symbol_upper) + + # Point value cannot be detected from data, always use 1.0 + # Users can manually adjust in the generated TOML file if needed + pointvalue = 1.0 + + # Get opening hours from OHLCVWriter + analyzed_opening_hours = ohlcv_writer.analyzed_opening_hours + + if analyzed_opening_hours: + # Use automatically detected opening hours + opening_hours = analyzed_opening_hours + else: + # Fallback to default based on symbol type (insufficient data or analysis failed) + opening_hours = self.get_default_opening_hours(symbol_type) + + # Create session starts and ends + session_starts = [SymInfoSession(day=1, time=time(0, 0, 0))] + session_ends = [SymInfoSession(day=7, time=time(23, 59, 59))] + + # Create SymInfo instance + # Use provider as prefix (uppercase), default to "CUSTOM" if not provided + prefix = provider.upper() if provider else "CUSTOM" + syminfo = SymInfo( + prefix=prefix, + description=f"{symbol}", + ticker=symbol_upper, + currency=currency, + basecurrency=base_currency or "USD", + period=detected_timeframe, + type=symbol_type, + mintick=mintick, + pricescale=int(pricescale), + minmove=int(minmove), + pointvalue=pointvalue, + opening_hours=opening_hours, + session_starts=session_starts, + session_ends=session_ends, + timezone=timezone, + ) + + # Save using SymInfo's built-in method + try: + syminfo.save_toml(toml_path) + # Copy modification time from source to maintain consistency + copy_mtime(file_path, toml_path) + except (OSError, IOError): + # Don't fail the entire conversion if TOML creation fails + pass except Exception as e: - # Clean up temporary file on error - if temp_path and temp_path.exists(): + # Clean up output file on error + if ohlcv_path.exists(): try: - temp_path.unlink() + ohlcv_path.unlink() except OSError: pass - raise ConversionError(f"Failed to convert {source_path}: {e}") from e + raise ConversionError(f"Failed to convert {file_path}: {e}") from e - def _create_symbol_info_file( - self, - source_path: Path, - symbol: str | None, - timeframe: str, - timezone: str - ) -> None: + @staticmethod + def detect_format(file_path: Path) -> Literal['csv', 'txt', 'json', 'ohlcv', 'unknown']: """ - Create TOML symbol info file using SymInfo class if it doesn't exist or is outdated. - - :param source_path: Path to the source data file - :param symbol: Symbol name (e.g., 'BTCUSD') - :param timeframe: Timeframe (e.g., '1h', '1D') - :param timezone: Timezone for the symbol + Detect file format by content inspection. + + :param file_path: Path to the file to analyze + :return: Detected format + :raises FileNotFoundError: If file doesn't exist + :raises DataFormatError: If file cannot be read """ - toml_path = source_path.with_suffix('.toml') + if not file_path.exists(): + raise FileNotFoundError(f"File not found: {file_path}") - # Skip if TOML file exists and is newer than source - if toml_path.exists() and not is_updated(source_path, toml_path): - return + # First check if it's a valid OHLCV file (binary format) + try: + with OHLCVReader(file_path): + # If we can open it successfully, it's a valid OHLCV file + return 'ohlcv' + except (ValueError, OSError, IOError): + # Not a valid OHLCV file, detect by content + pass - # Generate symbol info if symbol is provided - if not symbol: - return + # Detect text-based formats by content + try: + with open(file_path, 'r', encoding='utf-8') as f: + # Read first line for initial analysis + first_line = f.readline().strip() - # Analyze price data to calculate trading parameters - price_analysis = self._analyze_price_data(source_path) + # Quick JSON check - look for JSON indicators + if first_line and (first_line.startswith('{') or first_line.startswith('[')): + # Verify it's valid JSON by reading the whole file + f.seek(0) + try: + json.load(f) + return 'json' + except (json.JSONDecodeError, UnicodeDecodeError): + pass + # Reset for further analysis if not JSON + f.seek(0) + first_line = f.readline().strip() + + # Check for CSV patterns + if first_line and ',' in first_line: + # Count commas to see if it looks like structured data + comma_count = first_line.count(',') + if comma_count >= 4: # At least OHLC columns + return 'csv' - # Determine symbol type based on symbol name patterns - symbol_upper = symbol.upper() - symbol_type, currency, base_currency = self._detect_symbol_type(symbol_upper) + # Check for other delimiters (TXT) + if first_line and any(delim in first_line for delim in ['\t', ';', '|']): + return 'txt' - # Get calculated trading parameters - mintick = price_analysis['tick_size'] - pricescale = price_analysis['price_scale'] - minmove = price_analysis['min_move'] - pointvalue = self._get_default_pointvalue(symbol_type) + # Default to CSV if it has any commas + if first_line and ',' in first_line: + return 'csv' + + return 'unknown' + + except (OSError, IOError, UnicodeDecodeError): + return 'unknown' - # Create opening hours based on symbol type + @staticmethod + def get_default_opening_hours(symbol_type: str) -> list[SymInfoInterval]: + """ + Get default opening hours based on symbol type. + + :param symbol_type: Type of symbol ('crypto', 'forex', 'stock', or 'other') + :return: List of SymInfoInterval objects representing default trading hours + """ opening_hours = [] + if symbol_type == 'crypto': # 24/7 trading for crypto for day in range(1, 8): @@ -389,8 +308,17 @@ def _create_symbol_info_file( start=time(0, 0, 0), end=time(23, 59, 59) )) + elif symbol_type == 'forex': + # Forex markets: Sunday 5 PM ET to Friday 5 PM ET (roughly) + # Using Monday-Friday 00:00-23:59 as approximation + for day in range(1, 6): + opening_hours.append(SymInfoInterval( + day=day, + start=time(0, 0, 0), + end=time(23, 59, 59) + )) else: - # Business hours for stocks/forex (Mon-Fri) + # Stock markets and others: typical business hours (Mon-Fri 9:30 AM - 4:00 PM) for day in range(1, 6): opening_hours.append(SymInfoInterval( day=day, @@ -398,253 +326,389 @@ def _create_symbol_info_file( end=time(16, 0, 0) )) - # Create session starts and ends - session_starts = [SymInfoSession(day=1, time=time(0, 0, 0))] - session_ends = [SymInfoSession(day=7, time=time(23, 59, 59))] - - # Create SymInfo instance - syminfo = SymInfo( - prefix="CUSTOM", - description=f"{symbol} - Auto-generated symbol info", - ticker=symbol_upper, - currency=currency, - basecurrency=base_currency or "EUR", - period=timeframe, - type=symbol_type if symbol_type in ["stock", "fund", "dr", "right", "bond", - "warrant", "structured", "index", "forex", - "futures", "spread", "economic", "fundamental", - "crypto", "spot", "swap", "option", "commodity", - "other"] else "other", - mintick=mintick, - pricescale=int(pricescale), - minmove=int(minmove), - pointvalue=pointvalue, - opening_hours=opening_hours, - session_starts=session_starts, - session_ends=session_ends, - timezone=timezone, - volumetype="base" - ) - - # Save using SymInfo's built-in method - try: - syminfo.save_toml(toml_path) - # Copy modification time from source to maintain consistency - copy_mtime(source_path, toml_path) - except (OSError, IOError): - # Don't fail the entire conversion if TOML creation fails - pass - - def _analyze_price_data(self, source_path: Path) -> dict[str, float]: - """Analyze price data to calculate trading parameters. - - :param source_path: Path to the source data file - :return: Dictionary with tick_size, price_scale, and min_move - """ - # Always use the basic method for maximum precision without external dependencies - return self._analyze_price_data_basic(source_path) + return opening_hours @staticmethod - def _analyze_price_data_basic(source_path: Path) -> dict[str, float]: - """Basic price data analysis by converting to OHLCV and using OHLCVReader. - - :param source_path: Path to the source data file - :return: Dictionary with tick_size, price_scale, and min_move + def guess_symbol_from_filename(file_path: Path) -> tuple[str | None, str | None]: """ - try: - # Convert file to temporary OHLCV format first - with tempfile.NamedTemporaryFile(suffix='.ohlcv', delete=False) as temp_file: - temp_ohlcv_path = Path(temp_file.name) - - try: - # Use existing conversion logic - with OHLCVWriter(temp_ohlcv_path) as ohlcv_writer: - if source_path.suffix.lower() == '.csv': - ohlcv_writer.load_from_csv(source_path, tz='UTC') - elif source_path.suffix.lower() == '.json': - ohlcv_writer.load_from_json(source_path, tz='UTC') - else: - # Treat as CSV with different delimiters - ohlcv_writer.load_from_csv(source_path, tz='UTC') - - # Now analyze price data using OHLCVReader - with OHLCVReader(temp_ohlcv_path) as reader: - if reader.size == 0: - return {'tick_size': 0.01, 'price_scale': 100, 'min_move': 1} - - # Analyze first 100 records (or all if less) - decimal_places = 0 - sample_size = min(100, reader.size) - - for i in range(sample_size): - ohlcv = reader.read(i) - # Check all price fields for decimal precision - for price in [ohlcv.open, ohlcv.high, ohlcv.low, ohlcv.close]: - # Detect actual decimal precision without artificial limits - if price != int(price): # Has decimal component - # Use high precision to capture actual decimal places - price_str = f"{price:.15f}".rstrip('0').rstrip('.') - if '.' in price_str: - current_decimals = len(price_str.split('.')[1]) - decimal_places = max(decimal_places, current_decimals) - - # Calculate parameters preserving full precision - decimal_places = max(decimal_places, 2) # At least 2 decimal places for safety - price_scale = 10 ** decimal_places - tick_size = 1.0 / price_scale - min_move = 1 - - return { - 'tick_size': tick_size, - 'price_scale': price_scale, - 'min_move': min_move - } - - finally: - # Clean up temporary file - if temp_ohlcv_path.exists(): - temp_ohlcv_path.unlink() - - except (OSError, IOError, ValueError, ConversionError): - return {'tick_size': 0.01, 'price_scale': 100, 'min_move': 1} + Guess symbol and provider from filename based on common patterns. - @staticmethod - def _detect_symbol_type(symbol_upper: str) -> tuple[str, str, str | None]: - """Detect symbol type and extract currency information. - - :param symbol_upper: Uppercase symbol string - :return: Tuple of (symbol_type, currency, base_currency) - """ - if any(crypto in symbol_upper for crypto in ['BTC', 'ETH', 'USD', 'USDT', 'USDC']): - symbol_type = 'crypto' - currency = 'USD' - base_currency = symbol_upper.replace('USD', '').replace('USDT', '').replace('USDC', '') - if not base_currency: # If nothing left after removing USD variants - base_currency = None - elif '/' in symbol_upper: - symbol_type = 'forex' - parts = symbol_upper.split('/') - currency = parts[1] if len(parts) > 1 else 'USD' - base_currency = parts[0] if len(parts) > 0 else None - else: - symbol_type = 'stock' - currency = 'USD' - base_currency = None - - return symbol_type, currency, base_currency - - @staticmethod - def _get_default_pointvalue(symbol_type: str) -> float: - """Get default point value based on symbol type. - - :param symbol_type: Type of symbol (crypto, forex, stock) - :return: Default point value - """ - if symbol_type == 'forex': - return 10.0 - else: # crypto, stock, or unknown - return 1.0 - - def _detect_timeframe_from_ohlcv(self, ohlcv_path: Path, requested_timeframe: str) -> str: - """Detect timeframe from OHLCV data by analyzing timestamp differences. - - :param ohlcv_path: Path to the OHLCV file - :param requested_timeframe: Requested timeframe ("AUTO" for auto-detection) - :return: Detected or original timeframe + :param file_path: Path to the data file + :return: Tuple of (symbol, provider) or (None, None) if not detected """ - if requested_timeframe.upper() != "AUTO": - return requested_timeframe - - try: - # Read first few records to analyze timestamp differences - with open(ohlcv_path, 'rb') as f: - # Read first 10 records (or less if file is smaller) - timestamps = [] - for _ in range(10): - record = f.read(24) # OHLCV record size - if len(record) < 24: + filename = file_path.stem # Filename without extension + filename_upper = filename.upper() + + # Known provider patterns - these will be detected first + provider_patterns = { + 'capitalcom': ['CAPITALCOM'], + 'capital.com': ['CAPITAL.COM', 'CAPITAL_COM'], + 'ccxt': ['CCXT'], + 'tradingview': ['TRADINGVIEW', 'TV'], + 'mt4': ['MT4', 'METATRADER4'], + 'mt5': ['MT5', 'METATRADER5'], + 'binance': ['BINANCE'], + 'bybit': ['BYBIT'], + 'coinbase': ['COINBASE'], + 'kraken': ['KRAKEN'], + 'oanda': ['OANDA'], + 'ib': ['IB', 'INTERACTIVE_BROKERS'], + } + + # Exchange names for crypto detection + exchange_names = ['BINANCE', 'BYBIT', 'COINBASE', 'KRAKEN', 'BITFINEX', 'HUOBI', 'OKEX', 'FTX'] + + # Crypto bases and quotes for pair detection + crypto_bases = ['BTC', 'ETH', 'XRP', 'ADA', 'DOT', 'LINK', 'LTC', 'BCH', 'UNI', 'MATIC', + 'SOL', 'AVAX', 'LUNA', 'ATOM', 'FTM', 'NEAR', 'ALGO', 'VET', 'FIL', 'ICP'] + # Order matters! Longer suffixes first to avoid false matches (USDT before USD) + crypto_quotes = ['USDT', 'USDC', 'BUSD', 'TUSD', 'DAI', 'USD', 'EUR', 'GBP', 'JPY', 'BTC', 'ETH'] + + detected_provider = None + detected_symbol = None + + # Step 1: Try exchange-based detection first (handles BINANCE_BTC_USDT, BYBIT:BTC:USDT, etc.) + # Clean separators and split + cleaned = filename.replace(':', '_').replace('/', '_').replace(',', '_') + parts = [p.strip() for p in cleaned.split('_') if p.strip()] + + if len(parts) >= 2 and parts[0].upper() in exchange_names: + # Exchange detected at the beginning + detected_provider = parts[0].lower() + + if len(parts) >= 3: + # Could be EXCHANGE_BASE_QUOTE format + if parts[1].upper() in crypto_bases and parts[2].upper() in crypto_quotes: + # Format: BINANCE_BTC_USDT + detected_symbol = f"{parts[1].upper()}/{parts[2].upper()}" + elif parts[1].upper() in crypto_bases: + # Maybe compact format in later parts: CCXT_BYBIT_BTC_USDT_USDT_1 + # Look for quote in remaining parts + for i in range(2, len(parts)): + if parts[i].upper() in crypto_quotes: + detected_symbol = f"{parts[1].upper()}/{parts[i].upper()}" + break + if not detected_symbol: + # No quote found, use the second part as-is + detected_symbol = parts[1].upper() + else: + # Check if second part is a compact pair (BTCUSDT) + potential = parts[1].upper() + for quote in crypto_quotes: + if potential.endswith(quote): + base = potential[:-len(quote)] + if base in crypto_bases: + detected_symbol = f"{base}/{quote}" + break + if not detected_symbol: + # Use second part as-is + detected_symbol = parts[1].upper() + elif len(parts) == 2: + # EXCHANGE_SYMBOL format + potential = parts[1].upper() + # Try to detect compact crypto pair + for quote in crypto_quotes: + if potential.endswith(quote): + base = potential[:-len(quote)] + if base in crypto_bases: + detected_symbol = f"{base}/{quote}" + break + if not detected_symbol: + detected_symbol = potential + + if detected_symbol: + return detected_symbol, detected_provider + + # Step 2: Check for explicit provider patterns (handles CAPITALCOM_EURUSD, TV_BTCUSD, etc.) + # Special case for ccxt_EXCHANGE pattern + if filename_upper.startswith('CCXT_'): + # Remove CCXT_ prefix and try to detect exchange and symbol + temp = filename[5:] # Remove "CCXT_" + temp_parts = temp.replace(':', '_').replace('/', '_').split('_') + if len(temp_parts) >= 2 and temp_parts[0].upper() in exchange_names: + # Format: CCXT_EXCHANGE_... - provider is the exchange name, not 'ccxt' + detected_provider = temp_parts[0].lower() # Use exchange name as provider + # Try to extract symbol from remaining parts + if len(temp_parts) >= 3: + # Try BASE/QUOTE detection + for i in range(1, len(temp_parts) - 1): + if temp_parts[i].upper() in crypto_bases: + for j in range(i + 1, len(temp_parts)): + if temp_parts[j].upper() in crypto_quotes: + detected_symbol = f"{temp_parts[i].upper()}/{temp_parts[j].upper()}" + return detected_symbol, detected_provider + # Fallback to simple extraction + detected_symbol = '_'.join(temp_parts[1:]) if len(temp_parts) > 1 else None + if detected_symbol: + return detected_symbol.upper(), detected_provider + else: + # No recognized exchange after ccxt_, just use ccxt as provider + detected_provider = 'ccxt' + detected_symbol = '_'.join(temp_parts) if temp_parts else None + if detected_symbol: + return detected_symbol.upper(), detected_provider + + for provider, patterns in provider_patterns.items(): + for pattern in patterns: + if pattern in filename_upper: + detected_provider = provider + # Remove provider pattern from filename for symbol detection + temp_filename = filename + for p in patterns: + temp_filename = temp_filename.replace(p, '').replace(p.lower(), '').replace(p.capitalize(), '') + temp_filename = temp_filename.strip('_').strip('-').strip(',').strip().strip() + + # TradingView format might have extra parts like ", 30_cbf9d" + # First remove everything after comma if present + if ',' in temp_filename: + temp_filename = temp_filename.split(',')[0].strip() + + if '_' in temp_filename: + temp_parts = temp_filename.split('_') + # Filter out hash-like strings and pure numbers + symbol_parts = [] + for part in temp_parts: + part = part.strip() + if not part: + continue + # Skip if looks like a hash or timeframe + if len(part) <= 6 and any(c.isdigit() for c in part) and any(c.isalpha() for c in part): + continue + if part.isdigit(): + continue + if part.upper() in ['1M', '5M', '15M', '30M', '60M', '1H', '4H', '1D', '1W', 'DAILY', + 'HOURLY', 'WEEKLY']: + continue + symbol_parts.append(part) + if symbol_parts: + temp_filename = '_'.join(symbol_parts) + + if temp_filename: + # Try to parse the symbol + temp_upper = temp_filename.upper() + + # Check for forex pair (6 chars, all letters) + if len(temp_upper) == 6 and temp_upper.isalpha(): + detected_symbol = temp_upper + # Check for crypto pair + elif any(base in temp_upper for base in crypto_bases): + for quote in crypto_quotes: + if temp_upper.endswith(quote): + base = temp_upper[:-len(quote)] + if base in crypto_bases: + detected_symbol = f"{base}/{quote}" + break + if not detected_symbol: + detected_symbol = temp_upper + else: + detected_symbol = temp_upper + break + if detected_provider: + break + + # Step 3: If no provider detected, try to infer from symbol pattern + if not detected_provider and not detected_symbol: + # Remove common suffixes and prefixes + clean_name = filename + for suffix in ['_1M', '_5M', '_15M', '_30M', '_60M', '_1H', '_4H', '_1D', '_1W', '_DAILY', '_HOURLY', + '_WEEKLY']: + if clean_name.upper().endswith(suffix): + clean_name = clean_name[:len(clean_name) - len(suffix)] + break + + clean_upper = clean_name.upper() + + # First check for crypto patterns (more specific) + for quote in crypto_quotes: + if clean_upper.endswith(quote): + base = clean_upper[:-len(quote)] + if base in crypto_bases: + detected_symbol = f"{base}/{quote}" + detected_provider = 'ccxt' break - # Unpack timestamp (first 4 bytes as uint32) - timestamp = struct.unpack(' 0: # Only positive differences - differences.append(diff) - - if not differences: - return "1D" # Default fallback + # If not crypto, check for 6-letter forex pair + if not detected_symbol and len(clean_upper) == 6 and clean_upper.isalpha(): + detected_symbol = clean_upper + detected_provider = 'forex' + + # If still no match, check for separator-based pairs + if not detected_symbol: + # Try underscore or dash separator + if '_' in clean_name: + parts = clean_name.split('_') + elif '-' in clean_name: + parts = clean_name.split('-') + else: + parts = [] - # Find the most common difference (mode) - diff_counts = Counter(differences) - most_common_diff = diff_counts.most_common(1)[0][0] + if len(parts) == 2: + if len(parts[0]) == 3 and len(parts[1]) == 3 and parts[0].isalpha() and parts[1].isalpha(): + # Likely forex: EUR_USD or EUR-USD + detected_symbol = parts[0].upper() + parts[1].upper() + detected_provider = 'forex' + elif parts[0].upper() in crypto_bases and parts[1].upper() in crypto_quotes: + # Crypto: BTC_USDT or BTC-USDT + detected_symbol = f"{parts[0].upper()}/{parts[1].upper()}" + detected_provider = 'ccxt' - # Convert seconds to timeframe string (timestamps are in seconds, not nanoseconds) - return self._seconds_to_timeframe(most_common_diff) + # Last resort - if it's a known ticker (must have at least one letter) + if not detected_symbol and len(clean_upper) >= 3 and clean_upper.isalnum() and any( + c.isalpha() for c in clean_upper): + detected_symbol = clean_upper - except (OSError, IOError, struct.error, IndexError, ValueError): - # If detection fails, return default - return "1D" + return detected_symbol, detected_provider @staticmethod - def _seconds_to_timeframe(seconds: int) -> str: + def guess_symbol_type(symbol_upper: str) -> tuple[Literal["forex", "crypto", "other"], str, str | None]: """ - Convert seconds difference to timeframe string using TV-compatible timeframe module. - - :param seconds: Time difference in seconds - :return: Timeframe string (e.g., '1m', '5m', '1h', '1D') - """ - # Handle edge cases - if seconds <= 0: - return "1D" # Default fallback + Guess symbol type and extract currency information based on common patterns. - # Use TV-compatible timeframe conversion - try: - return from_seconds(seconds) - except (ValueError, AssertionError): - # Fallback to closest standard timeframe if conversion fails - return "1D" - - def _detect_timeframe_from_data(self, df, requested_timeframe: str) -> str: - """Detect timeframe from DataFrame by analyzing timestamp differences. - - :param df: DataFrame with timestamp column - :param requested_timeframe: Requested timeframe ("AUTO" for auto-detection) - :return: Detected or original timeframe + :param symbol_upper: Uppercase symbol string + :return: Tuple of (symbol_type, currency, base_currency) """ - if requested_timeframe.upper() != "AUTO": - return requested_timeframe - - try: - if len(df) < 2: - return "1D" # Default fallback - - # Calculate differences between consecutive timestamps - timestamps = df['timestamp'].values - differences = [] - - for i in range(1, min(len(timestamps), 10)): # Check first 10 records - diff = timestamps[i] - timestamps[i - 1] - if diff > 0: # Only positive differences - differences.append(diff) + # Common forex pairs - check these first for accurate detection + forex_pairs = { + 'EURUSD', 'GBPUSD', 'USDJPY', 'USDCHF', 'AUDUSD', 'USDCAD', 'NZDUSD', + 'EURGBP', 'EURJPY', 'GBPJPY', 'EURCHF', 'EURAUD', 'EURCAD', 'EURNZD', + 'GBPCHF', 'GBPAUD', 'GBPCAD', 'GBPNZD', 'AUDJPY', 'AUDCHF', 'AUDCAD', + 'AUDNZD', 'CADJPY', 'CADCHF', 'NZDJPY', 'NZDCHF', 'NZDCAD', 'CHFJPY', + 'EUR/USD', 'GBP/USD', 'USD/JPY', 'USD/CHF', 'AUD/USD', 'USD/CAD', 'NZD/USD' + } + + # Common crypto symbols + crypto_symbols = { + 'BTC', 'ETH', 'BNB', 'ADA', 'SOL', 'DOT', 'DOGE', 'AVAX', 'LUNA', 'SHIB', + 'MATIC', 'UNI', 'LINK', 'LTC', 'ALGO', 'BCH', 'XLM', 'VET', 'ATOM', 'FIL', + 'TRX', 'ETC', 'XMR', 'MANA', 'SAND', 'HBAR', 'EGLD', 'THETA', 'FTM', 'XTZ', + 'AAVE', 'AXS', 'CAKE', 'CRO', 'NEAR', 'KSM', 'ENJ', 'CHZ', 'SUSHI', 'SNX' + } + + # Initialize default values + symbol_type: Literal["forex", "crypto", "other"] = 'other' + currency = 'USD' + base_currency: str | None = None + + # Clean up separators + clean_symbol = symbol_upper.replace('_', '').replace('-', '').replace(':', '').strip() + + # Check if it's a direct forex pair match (check both with and without slash) + if clean_symbol in forex_pairs or symbol_upper in forex_pairs or \ + any(pair.replace('/', '') in clean_symbol for pair in forex_pairs): + symbol_type = 'forex' + # Extract currencies from forex pair - more robust extraction + matched = False + for pair in forex_pairs: + clean_pair = pair.replace('/', '') + # Check both versions + if clean_pair in clean_symbol or pair == symbol_upper: + # Found exact match + if '/' in pair: + parts = pair.split('/') + base_currency = parts[0] + currency = parts[1] + else: + base_currency = pair[:3] + currency = pair[3:6] + matched = True + break + + if not matched: + # Fallback extraction for forex + if 'EUR' in clean_symbol: + base_currency = 'EUR' + remaining = clean_symbol.replace('EUR', '') + currency = remaining[:3] if len(remaining) >= 3 else 'USD' + elif 'GBP' in clean_symbol: + base_currency = 'GBP' + remaining = clean_symbol.replace('GBP', '') + currency = remaining[:3] if len(remaining) >= 3 else 'USD' + elif clean_symbol.startswith('USD'): + base_currency = 'USD' + currency = clean_symbol[3:6] if len(clean_symbol) >= 6 else 'EUR' + else: + # Try to extract 3-letter codes + base_currency = clean_symbol[:3] if len(clean_symbol) >= 3 else 'EUR' + currency = clean_symbol[3:6] if len(clean_symbol) >= 6 else 'USD' - if not differences: - return "1D" # Default fallback + # Check if symbol contains '/' separator (explicit format) + elif '/' in symbol_upper: + parts = symbol_upper.split('/') + if len(parts) == 2: + left_part = parts[0].strip() + right_part = parts[1].strip() + + # Check if it's crypto (contains crypto symbols or stable coins) + if any(crypto in left_part for crypto in crypto_symbols) or \ + right_part in ['USDT', 'USDC', 'BUSD', 'DAI', 'UST', 'TUSD']: + symbol_type = 'crypto' + currency = right_part + base_currency = left_part + # Check if it's forex (both parts are 3-letter currency codes) + elif len(left_part) == 3 and len(right_part) == 3 and \ + left_part.isalpha() and right_part.isalpha(): + symbol_type = 'forex' + base_currency = left_part + currency = right_part + else: + # Default to crypto for slash notation + symbol_type = 'crypto' + currency = right_part + base_currency = left_part - # Find the most common difference (mode) - diff_counts = Counter(differences) - most_common_diff = diff_counts.most_common(1)[0][0] + # Check if it's crypto by matching known crypto symbols + elif any(crypto in clean_symbol for crypto in crypto_symbols): + symbol_type = 'crypto' + # Try to extract the quote currency + if 'USDT' in clean_symbol: + currency = 'USDT' + base_currency = clean_symbol.replace('USDT', '') + elif 'USDC' in clean_symbol: + currency = 'USDC' + base_currency = clean_symbol.replace('USDC', '') + elif 'BUSD' in clean_symbol: + currency = 'BUSD' + base_currency = clean_symbol.replace('BUSD', '') + elif 'USD' in clean_symbol: + currency = 'USD' + base_currency = clean_symbol.replace('USD', '') + else: + # Try to find the crypto part + for crypto in crypto_symbols: + if crypto in clean_symbol: + base_currency = crypto + currency = clean_symbol.replace(crypto, '') or 'USDT' + break + else: + currency = 'USDT' + base_currency = clean_symbol + + if not base_currency or base_currency == currency: + base_currency = clean_symbol[:3] if len(clean_symbol) >= 3 else 'BTC' + + # Check if it looks like a forex pair (6 letters, no special chars) + elif len(clean_symbol) == 6 and clean_symbol.isalpha(): + # Could be forex like EURUSD or crypto like BTCUSD + potential_base = clean_symbol[:3] + potential_quote = clean_symbol[3:6] + + # Common forex currencies + forex_currencies = {'EUR', 'USD', 'GBP', 'JPY', 'CHF', 'CAD', 'AUD', 'NZD'} + + if potential_base in forex_currencies and potential_quote in forex_currencies: + symbol_type = 'forex' + base_currency = potential_base + currency = potential_quote + else: + # Default to other for unknown 6-letter symbols + symbol_type = 'other' + currency = 'USD' + base_currency = None - # Convert seconds to timeframe string (assuming timestamps are in seconds) - return self._seconds_to_timeframe(most_common_diff) + else: + # Default to other for everything else + symbol_type = 'other' + currency = 'USD' + base_currency = None - except (KeyError, IndexError, ValueError, TypeError): - # If detection fails, return default - return "1D" + return symbol_type, currency, base_currency diff --git a/src/pynecore/core/ohlcv_file.py b/src/pynecore/core/ohlcv_file.py index b583c74..5b48be5 100644 --- a/src/pynecore/core/ohlcv_file.py +++ b/src/pynecore/core/ohlcv_file.py @@ -11,16 +11,24 @@ The .ohlcv format cannot have gaps in it. All gaps are filled with the previous close price and -1 volume. """ +from typing import Iterator, cast -from typing import Iterator -import os +import csv +import json +import math import mmap +import os import struct -from pathlib import Path +from collections import Counter +from collections.abc import Buffer +from datetime import datetime, time, timedelta, timezone as dt_timezone, UTC from io import BufferedWriter, BufferedRandom -from datetime import datetime, UTC +from math import gcd as math_gcd +from pathlib import Path +from zoneinfo import ZoneInfo from pynecore.types.ohlcv import OHLCV +from ..core.syminfo import SymInfoInterval RECORD_SIZE = 24 # 6 * 4 STRUCT_FORMAT = 'Ifffff' # I: uint32, f: float32 @@ -38,16 +46,31 @@ class OHLCVWriter: Binary OHLCV data writer using direct file operations """ - __slots__ = ('path', '_file', '_size', '_start_timestamp', '_interval', '_current_pos', '_last_timestamp') + __slots__ = ('path', '_file', '_size', '_start_timestamp', '_interval', '_current_pos', '_last_timestamp', + '_price_changes', '_price_decimals', '_last_close', '_analyzed_tick_size', + '_analyzed_price_scale', '_analyzed_min_move', '_confidence', + '_trading_hours', '_analyzed_opening_hours', '_truncate') - def __init__(self, path: str | Path): + def __init__(self, path: str | Path, truncate: bool = False): self.path: str = str(path) self._file: BufferedWriter | BufferedRandom | None = None + self._truncate: bool = truncate self._size: int = 0 self._start_timestamp: int | None = None self._interval: int | None = None self._current_pos: int = 0 self._last_timestamp: int | None = None + # Tick size analysis + self._price_changes: list[float] = [] + self._price_decimals: set[int] = set() + self._last_close: float | None = None + self._analyzed_tick_size: float | None = None + self._analyzed_price_scale: int | None = None + self._analyzed_min_move: int | None = None + self._confidence: float = 0.0 + # Trading hours analysis + self._trading_hours: dict[tuple[int, int], int] = {} # (weekday, hour) -> count + self._analyzed_opening_hours: list | None = None def __enter__(self): self.open() @@ -109,20 +132,72 @@ def interval(self) -> int | None: """ return self._interval + @property + def analyzed_tick_size(self) -> float | None: + """ + Automatically detected tick size from price data + """ + if self._analyzed_tick_size is None and len(self._price_changes) >= 10: + self._analyze_tick_size() + return self._analyzed_tick_size + + @property + def analyzed_price_scale(self) -> int | None: + """ + Automatically detected price scale from price data + """ + if self._analyzed_price_scale is None and len(self._price_changes) >= 10: + self._analyze_tick_size() + return self._analyzed_price_scale + + @property + def analyzed_min_move(self) -> int | None: + """ + Automatically detected min move (usually 1) + """ + if self._analyzed_min_move is None and len(self._price_changes) >= 10: + self._analyze_tick_size() + return self._analyzed_min_move + + @property + def tick_analysis_confidence(self) -> float: + """ + Confidence of tick size analysis (0.0 to 1.0) + """ + if self._confidence == 0.0 and len(self._price_changes) >= 10: + self._analyze_tick_size() + return self._confidence + + @property + def analyzed_opening_hours(self) -> list | None: + """ + Automatically detected opening hours from trading activity + Returns list of SymInfoInterval tuples or None if not enough data + """ + if self._analyzed_opening_hours is None and self._has_enough_data_for_opening_hours(): + self._analyze_opening_hours() + return self._analyzed_opening_hours + def open(self) -> 'OHLCVWriter': """ Open file for writing """ - # Open in rb+ mode to allow both reading and writing - self._file = open(self.path, 'rb+') if os.path.exists(self.path) else open(self.path, 'wb+') + # If truncate is True, always open in write mode to clear existing data + if self._truncate: + self._file = open(self.path, 'wb+') + else: + # Open in rb+ mode to allow both reading and writing + self._file = open(self.path, 'rb+') if os.path.exists(self.path) else open(self.path, 'wb+') self._size = os.path.getsize(self.path) // RECORD_SIZE # Read initial metadata if file exists if self._size >= 2: self._file.seek(0) - first_timestamp = struct.unpack('I', self._file.read(4))[0] + data: Buffer = self._file.read(4) + first_timestamp = struct.unpack('I', data)[0] self._file.seek(RECORD_SIZE) - second_timestamp = struct.unpack('I', self._file.read(4))[0] + data: Buffer = self._file.read(4) + second_timestamp = struct.unpack('I', data)[0] self._start_timestamp = first_timestamp self._interval = second_timestamp - first_timestamp assert self._interval is not None @@ -132,6 +207,10 @@ def open(self) -> 'OHLCVWriter': self._file.seek(0, os.SEEK_END) self._current_pos = self._size + # Collect trading hours from existing data for analysis + if self._size > 0 and not self._truncate: + self._collect_existing_trading_hours() + return self def write(self, candle: OHLCV) -> None: @@ -154,24 +233,23 @@ def write(self, candle: OHLCV) -> None: if self._interval <= 0: raise ValueError(f"Invalid interval: {self._interval}") elif self._size >= 2: # Changed from elif self._size == 2: to properly handle all cases - # For the second candle, validate interval - if self._size == 2: - assert self._last_timestamp is not None and self._interval is not None - current_interval = candle.timestamp - self._last_timestamp - if current_interval > self._interval * 2: - # Truncate and restart - self.truncate() - self._start_timestamp = candle.timestamp - self._interval = None - self._last_timestamp = None - self._current_pos = 0 - self._size = 0 - # Check chronological order if self._last_timestamp is not None and candle.timestamp <= self._last_timestamp: raise ValueError( f"Timestamps must be in chronological order. Got {candle.timestamp} after {self._last_timestamp}") + # Check if we found a smaller interval (indicates initial interval was wrong due to gap) + if self._interval is not None and self._last_timestamp is not None: + current_interval = candle.timestamp - self._last_timestamp + + # If we find a smaller interval, the initial one was wrong (had a gap) + if 0 < current_interval < self._interval: + # Rebuild file with correct interval + self._rebuild_with_correct_interval(current_interval) + # Now write the current candle with the corrected setup + self.write(candle) + return + # Calculate expected timestamp and fill gaps if self._interval is not None and self._last_timestamp is not None: expected_ts = self._last_timestamp + self._interval @@ -180,14 +258,15 @@ def write(self, candle: OHLCV) -> None: if candle.timestamp > expected_ts: # Get previous candle's close price self._file.seek((self._current_pos - 1) * RECORD_SIZE) - prev_data = struct.unpack(STRUCT_FORMAT, self._file.read(RECORD_SIZE)) + data: Buffer = self._file.read(RECORD_SIZE) + prev_data = struct.unpack(STRUCT_FORMAT, data) prev_close = prev_data[4] # 4th index is close price # Fill gap with previous close and -1 volume (gap indicator) while expected_ts < candle.timestamp: - gap_data = struct.pack(STRUCT_FORMAT, - expected_ts, prev_close, prev_close, - prev_close, prev_close, -1.0) + gap_data: Buffer = struct.pack(STRUCT_FORMAT, + expected_ts, prev_close, prev_close, + prev_close, prev_close, -1.0) self._file.seek(self._current_pos * RECORD_SIZE) self._file.write(gap_data) self._current_pos += 1 @@ -196,12 +275,18 @@ def write(self, candle: OHLCV) -> None: # Write actual data self._file.seek(self._current_pos * RECORD_SIZE) - data = struct.pack(STRUCT_FORMAT, - candle.timestamp, candle.open, candle.high, - candle.low, candle.close, candle.volume) + data: Buffer = struct.pack(STRUCT_FORMAT, + candle.timestamp, candle.open, candle.high, + candle.low, candle.close, candle.volume) self._file.write(data) self._file.flush() + # Collect data for tick size analysis + self._collect_price_data(candle) + + # Collect trading hours data + self._collect_trading_hours(candle) + self._last_timestamp = candle.timestamp self._current_pos += 1 self._size = max(self._size, self._current_pos) @@ -260,6 +345,535 @@ def close(self): self._file.close() self._file = None + def _collect_price_data(self, candle: OHLCV) -> None: + """ + Collect price data for tick size analysis during writing. + """ + # Collect price changes + if self._last_close is not None: + change = abs(candle.close - self._last_close) + if change > 0 and len(self._price_changes) < 1000: # Limit to 1000 samples + self._price_changes.append(change) + + # Collect decimal places + for price in [candle.open, candle.high, candle.low, candle.close]: + if price != int(price): # Has decimal component + price_str = f"{price:.15f}".rstrip('0').rstrip('.') + if '.' in price_str: + decimals = len(price_str.split('.')[1]) + self._price_decimals.add(decimals) + + self._last_close = candle.close + + def _analyze_tick_size(self) -> None: + """ + Analyze collected price data to determine tick size using multiple methods. + """ + if not self._price_changes: + # No data, use defaults + self._analyzed_tick_size = 0.01 + self._analyzed_price_scale = 100 + self._analyzed_min_move = 1 + self._confidence = 0.1 + return + + # Try histogram-based method first for better noise handling + histogram_tick = self._calculate_histogram_tick() + + if histogram_tick[0] > 0 and histogram_tick[1] > 0.7: + # High confidence histogram result, use it directly + self._analyzed_tick_size = histogram_tick[0] + self._analyzed_price_scale = int(round(1.0 / histogram_tick[0])) + self._analyzed_min_move = 1 + self._confidence = histogram_tick[1] + return + + # Fall back to other methods + # Method 1: Most frequent small change + freq_tick = self._calculate_frequency_tick() + + # Method 2: Decimal places analysis + decimal_tick = self._calculate_decimal_tick() + + # Combine methods with weighted confidence (no GCD) + tick_size, confidence = self._combine_tick_estimates_no_gcd(freq_tick, decimal_tick) + + # Calculate price scale and min move + if tick_size > 0: + self._analyzed_tick_size = tick_size + self._analyzed_price_scale = int(round(1.0 / tick_size)) + self._analyzed_min_move = 1 + self._confidence = confidence + else: + # Fallback to defaults + self._analyzed_tick_size = 0.01 + self._analyzed_price_scale = 100 + self._analyzed_min_move = 1 + self._confidence = 0.1 + + def _calculate_frequency_tick(self) -> tuple[float, float]: + """ + Calculate tick size based on most frequent small changes. + Returns (tick_size, confidence) + """ + if len(self._price_changes) < 10: + return 0, 0 + + # Apply float32 filtering first + filtered_changes = [] + for c in self._price_changes[:100]: + if c > 0: + # Convert to float32 and back + float32_val = struct.unpack('f', cast(Buffer, struct.pack('f', c)))[0] + # Round to reasonable precision for float32 + rounded = round(float32_val, 6) + if rounded > 0: + filtered_changes.append(rounded) + + if len(filtered_changes) < 5: + return 0, 0 + + # Find most frequent changes + counter = Counter(filtered_changes) + most_common = counter.most_common(10) + + if not most_common: + return 0, 0 + + # Find GCD of frequent changes to get base tick + frequent_changes = [change for change, count in most_common if count >= 2] + if len(frequent_changes) >= 2: + # Convert to integers for GCD + scale = 1000000 # 6 decimal places + int_changes = [int(round(c * scale)) for c in frequent_changes] + + # Calculate GCD + result = int_changes[0] + for val in int_changes[1:]: + result = math_gcd(result, val) + + tick_size = result / scale + + # Confidence based on how many changes match this tick + matches = sum(1 for c in filtered_changes + if abs(round(c / tick_size) * tick_size - c) < tick_size * 0.1) + confidence = min(matches / len(filtered_changes), 1.0) + return tick_size, confidence * 0.7 # Medium weight + + return 0, 0 + + def _calculate_histogram_tick(self) -> tuple[float, float]: + """ + Calculate tick size using histogram-based clustering approach. + This method is robust to float32 noise. + Returns (tick_size, confidence) + """ + if len(self._price_changes) < 10: + return 0, 0 + + # Common tick sizes to test (from 1 to 0.00001) + candidate_ticks = [ + 1.0, 0.5, 0.25, 0.1, 0.05, 0.01, 0.005, 0.001, + 0.0005, 0.0001, 0.00005, 0.00001, 0.000001 + ] + + best_tick = 0 + best_score = 0 + + # Filter out zero changes and convert to float32 precision + changes = [] + for change in self._price_changes[:200]: # Use more samples for histogram + if change > 0: + # Round to float32 precision + float32_val = struct.unpack('f', cast(Buffer, struct.pack('f', change)))[0] + changes.append(float32_val) + + if len(changes) < 5: + return 0, 0 + + # Get min non-zero change to establish scale + min_change = min(changes) + avg_change = sum(changes) / len(changes) + + for tick in candidate_ticks: + # Skip ticks that are too small (less than 1/10 of smallest change) + if tick < min_change * 0.1: + continue + + # Skip ticks that are way too large + if tick > avg_change * 10: + continue + + # Round all changes to this tick size + rounded = [round(c / tick) * tick for c in changes] + + # Calculate how well the rounding fits + errors = [abs(c - r) for c, r in zip(changes, rounded)] + max_error = max(errors) + + # Key insight: if max error is less than tick/2, this tick captures the grid well + if max_error < tick * 0.5: + # Count how many changes are multiples of this tick (within tolerance) + tolerance = tick * 0.1 + multiples = sum(1 for c in changes if abs(round(c / tick) * tick - c) < tolerance) + multiple_ratio = multiples / len(changes) + + # Score based on how many values are clean multiples + if multiple_ratio > 0.7: # Most values are clean multiples + score = multiple_ratio + + # Prefer larger ticks (less precision) when scores are similar + # This helps choose 0.00001 over 0.000001 when both fit + score *= (1.0 + tick * 100) # Small bonus for larger ticks + + if score > best_score: + best_score = score + best_tick = tick + + # If no good tick found with strict criteria, fall back to simple analysis + if best_tick == 0: + # Find the most common order of magnitude in changes + magnitudes = [] + for c in changes: + if c > 0: + # Find order of magnitude + mag = 10 ** math.floor(math.log10(c)) + magnitudes.append(mag) + + if magnitudes: + # Most common magnitude + counter = Counter(magnitudes) + common_mag = counter.most_common(1)[0][0] + # Use tick as 1/10 of common magnitude + best_tick = common_mag / 10 + best_score = 0.5 + + # Calculate confidence based on score + if best_score > 0.8: + confidence = 0.9 + elif best_score > 0.6: + confidence = 0.7 + else: + confidence = best_score + + return best_tick, confidence + + def _calculate_decimal_tick(self) -> tuple[float, float]: + """ + Calculate tick size based on decimal places. + Returns (tick_size, confidence) + """ + if not self._price_decimals: + # No decimals found, probably integer prices + return 1.0, 0.5 + + # Filter out noise from float representation + # If we have 15 decimals, it's likely float noise + valid_decimals = [d for d in self._price_decimals if d <= 10] + + if not valid_decimals: + # All decimals are noise, assume 2 decimal places (cents) + return 0.01, 0.3 + + # Use most common valid decimal places + max_decimals = max(valid_decimals) + tick_size = 10 ** (-max_decimals) + + # Lower confidence for decimal-only method + return tick_size, 0.5 + + @staticmethod + def _combine_tick_estimates_no_gcd(freq: tuple[float, float], + decimal: tuple[float, float]) -> tuple[float, float]: + """ + Combine tick size estimates from frequency and decimal methods only. + Returns (tick_size, confidence) + """ + estimates = [] + + if freq[0] > 0 and freq[1] > 0: + estimates.append(freq) + if decimal[0] > 0 and decimal[1] > 0: + estimates.append(decimal) + + if not estimates: + return 0.01, 0.1 # Default fallback + + # Use highest confidence estimate + best = max(estimates, key=lambda x: x[1]) + return best + + def _collect_trading_hours(self, candle: OHLCV) -> None: + """ + Collect trading hours data from timestamps. + Only collect for candles with actual volume (not gaps). + """ + if candle.volume <= 0: + return # Skip gaps + + # Convert timestamp to datetime + dt = datetime.fromtimestamp(candle.timestamp, tz=None) # Local time + + # Get weekday (1=Monday, 7=Sunday) and hour + weekday = dt.isoweekday() + hour = dt.hour + + # Count occurrences + key = (weekday, hour) + self._trading_hours[key] = self._trading_hours.get(key, 0) + 1 + + def _collect_existing_trading_hours(self) -> None: + """ + Collect trading hours data from existing file for opening hours analysis. + Only samples a subset of data for performance reasons. + """ + if not self._file or self._size == 0: + return + + # Save current position + current_pos = self._file.tell() + + try: + # Sample data: read every Nth record for performance + # For large files, we don't need to read everything + sample_interval = max(1, self._size // 1000) # Sample up to 1000 points + + for i in range(0, self._size, sample_interval): + self._file.seek(i * RECORD_SIZE) + data = self._file.read(RECORD_SIZE) + + if len(data) == RECORD_SIZE: + # Unpack the record + timestamp, open_val, high, low, close, volume = struct.unpack('Ifffff', data) + + # Only collect if volume > 0 (real trading) + if volume > 0: + dt = datetime.fromtimestamp(timestamp, tz=None) + weekday = dt.isoweekday() + hour = dt.hour + key = (weekday, hour) + self._trading_hours[key] = self._trading_hours.get(key, 0) + 1 + + finally: + # Restore file position + self._file.seek(current_pos) + + def _has_enough_data_for_opening_hours(self) -> bool: + """ + Check if we have enough data to analyze opening hours based on timeframe. + """ + if not self._trading_hours or not self._interval: + return False + + # For daily or larger timeframes + if self._interval >= 86400: # >= 1 day + # We need at least a few days to see a pattern + unique_days = len(set(day for day, hour in self._trading_hours.keys())) + return unique_days >= 3 # At least 3 different days + + # For intraday timeframes + # Check if we have at least some meaningful data + # We need enough to see a pattern + data_points = sum(self._trading_hours.values()) + points_per_hour = 3600 / self._interval + hours_covered = data_points / points_per_hour + + # Need at least 2 hours of data to detect any pattern + # This allows even short sessions to be analyzed + return hours_covered >= 2 + + def _analyze_opening_hours(self) -> None: + """ + Analyze collected trading hours to determine opening hours pattern. + Works for both intraday and daily timeframes. + """ + if not self._trading_hours: + self._analyzed_opening_hours = None + return + + # For daily or larger timeframes, analyze which days have trading + if self._interval and self._interval >= 86400: # >= 1 day + self._analyzed_opening_hours = [] + days_with_trading = set(day for day, hour in self._trading_hours.keys()) + + # Check if it's 24/7 (all 7 days have trading) + if len(days_with_trading) == 7: + # 24/7 trading pattern + for day in range(1, 8): + self._analyzed_opening_hours.append(SymInfoInterval( + day=day, + start=time(0, 0, 0), + end=time(23, 59, 59) + )) + elif days_with_trading <= {1, 2, 3, 4, 5}: # Monday-Friday only + # Business days pattern (stock/forex) + for day in range(1, 6): + self._analyzed_opening_hours.append(SymInfoInterval( + day=day, + start=time(9, 30, 0), # Default to US market hours + end=time(16, 0, 0) + )) + else: + # Mixed pattern - include all days that have trading + for day in sorted(days_with_trading): + self._analyzed_opening_hours.append(SymInfoInterval( + day=day, + start=time(0, 0, 0), # Default to full day for daily data + end=time(23, 59, 59) + )) + return + + # For intraday data, analyze hourly patterns + # Check if it's 24/7 trading (crypto pattern) + total_hours = len(self._trading_hours) + if total_hours >= 168 * 0.7: # 70% of all hours in a week (lowered threshold) + # Check if all hours have similar activity + counts = list(self._trading_hours.values()) + avg_count = sum(counts) / len(counts) + variance = sum((c - avg_count) ** 2 for c in counts) / len(counts) + + # If low variance, it's likely 24/7 + if variance < avg_count * 0.5: + self._analyzed_opening_hours = [] + for day in range(1, 8): + self._analyzed_opening_hours.append(SymInfoInterval( + day=day, + start=time(0, 0, 0), + end=time(23, 59, 59) + )) + return + + # Analyze per-day patterns for intraday + self._analyzed_opening_hours = [] + + for day in range(1, 8): # Monday to Sunday + # Get all hours for this day + day_hours = [(hour, count) for (d, hour), count in self._trading_hours.items() if d == day] + + if not day_hours: + continue # No trading on this day + + # Sort by hour + day_hours.sort(key=lambda x: x[0]) + + # Find continuous trading periods + periods = [] + current_start = None + current_end = None + + # Threshold: consider an hour active if it has at least 20% of average activity + total_count = sum(count for _, count in day_hours) + if total_count == 0: + continue + avg_hour_count = total_count / len(day_hours) + threshold = avg_hour_count * 0.2 + + for hour, count in day_hours: + if count >= threshold: + if current_start is None: + current_start = hour + current_end = hour + else: + current_end = hour + else: + if current_start is not None: + periods.append((current_start, current_end)) + current_start = None + current_end = None + + # Add last period if exists + if current_start is not None: + periods.append((current_start, current_end)) + + # Convert periods to SymInfoInterval + for start_hour, end_hour in periods: + self._analyzed_opening_hours.append(SymInfoInterval( + day=day, + start=time(start_hour, 0, 0), + end=time(end_hour, 59, 59) + )) + + # If no opening hours detected, default to business hours + if not self._analyzed_opening_hours: + for day in range(1, 6): # Monday to Friday + self._analyzed_opening_hours.append(SymInfoInterval( + day=day, + start=time(9, 30, 0), + end=time(16, 0, 0) + )) + + def _rebuild_with_correct_interval(self, new_interval: int) -> None: + """ + Rebuild the entire file with the correct interval when a smaller interval is detected. + This happens when initial interval was wrong due to gaps. + + :param new_interval: The correct interval to use + """ + import tempfile + import shutil + + if not self._file or self._size == 0: + return + + # Save current file position and data + current_records = [] + + # Read all existing records + self._file.seek(0) + for i in range(self._size): + offset = i * RECORD_SIZE + self._file.seek(offset) + data = self._file.read(RECORD_SIZE) + if len(cast(bytes, data)) == RECORD_SIZE: + record = struct.unpack(STRUCT_FORMAT, cast(Buffer, data)) + current_records.append(OHLCV(*record, extra_fields={})) + + # Create temp file for rebuilding + temp_fd, temp_path = tempfile.mkstemp(suffix='.ohlcv.tmp', dir=os.path.dirname(self.path)) + try: + # Close temp file descriptor as we'll open it differently + os.close(temp_fd) + + # Create new writer with temp file + with OHLCVWriter(temp_path) as temp_writer: + # Write all records with correct interval + # The writer will now properly handle gaps + for record in current_records: + temp_writer.write(record) + + # Close current file + self._file.close() + + # Replace original with rebuilt file + shutil.move(temp_path, self.path) + + # Reopen the file + self._file = open(self.path, 'rb+') + self._size = os.path.getsize(self.path) // RECORD_SIZE + + # Reset interval to the correct one + self._interval = new_interval + + # Position at end for appending + self._file.seek(0, os.SEEK_END) + self._current_pos = self._size + + # Update last timestamp + if self._size > 0: + self._file.seek((self._size - 1) * RECORD_SIZE) + data: Buffer = self._file.read(4) + self._last_timestamp = struct.unpack('I', data)[0] + self._file.seek(0, os.SEEK_END) + + except Exception as e: + # Clean up temp file on error + if os.path.exists(temp_path): + try: + os.unlink(temp_path) + except OSError: + pass + raise IOError(f"Failed to rebuild file with correct interval: {e}") + def load_from_csv(self, path: str | Path, timestamp_format: str | None = None, timestamp_column: str | None = None, @@ -276,9 +890,6 @@ def load_from_csv(self, path: str | Path, :param time_column: When timestamp is split into date+time columns, time column name :param tz: Timezone name (e.g. 'UTC', 'Europe/London', '+0100') for timestamp conversion """ - import csv - from zoneinfo import ZoneInfo - # Parse timezone timezone = None if tz: @@ -287,7 +898,6 @@ def load_from_csv(self, path: str | Path, sign = 1 if tz.startswith('+') else -1 hours = int(tz[1:3]) minutes = int(tz[3:]) if len(tz) > 3 else 0 - from datetime import timezone as dt_timezone, timedelta timezone = dt_timezone(sign * timedelta(hours=hours, minutes=minutes)) else: # Handle named timezone (e.g. UTC, Europe/London) @@ -348,7 +958,7 @@ def load_from_csv(self, path: str | Path, # Combine date and time ts_str = f"{row[date_idx]} {row[time_idx]}" else: - ts_str = row[timestamp_idx] # type: ignore + ts_str = row[timestamp_idx] # Convert timestamp try: @@ -416,10 +1026,6 @@ def load_from_json(self, path: str | Path, :param tz: Timezone name (e.g. 'UTC', 'Europe/London', '+0100') :param mapping: Optional field mapping, e.g. {'timestamp': 't', 'volume': 'vol'} """ - import json - from datetime import datetime - from zoneinfo import ZoneInfo - # Parse timezone timezone = None if tz: @@ -428,7 +1034,6 @@ def load_from_json(self, path: str | Path, sign = 1 if tz.startswith('+') else -1 hours = int(tz[1:3]) minutes = int(tz[3:]) if len(tz) > 3 else 0 - from datetime import timezone as dt_timezone, timedelta timezone = dt_timezone(sign * timedelta(hours=hours, minutes=minutes)) else: # Handle named timezone @@ -640,8 +1245,8 @@ def open(self) -> 'OHLCVReader': self._size = os.path.getsize(self.path) // RECORD_SIZE if self._size >= 2: - self._start_timestamp = struct.unpack('I', self._mmap[0:4])[0] - second_timestamp = struct.unpack('I', self._mmap[RECORD_SIZE:RECORD_SIZE + 4])[0] + self._start_timestamp = struct.unpack('I', cast(Buffer, self._mmap[0:4]))[0] + second_timestamp = struct.unpack('I', cast(Buffer, self._mmap[RECORD_SIZE:RECORD_SIZE + 4]))[0] self._interval = second_timestamp - self._start_timestamp return self @@ -761,6 +1366,9 @@ def save_to_csv(self, path: str, as_datetime=False) -> None: else: f.write('timestamp,open,high,low,close,volume\n') for candle in self: + # Skip gaps (volume == -1) + if candle.volume == -1: + continue if as_datetime: f.write(f"{datetime.fromtimestamp(candle.timestamp, UTC)},{format_float(candle.open)}," f"{format_float(candle.high)},{format_float(candle.low)},{format_float(candle.close)}," @@ -789,10 +1397,11 @@ def save_to_json(self, path: str, as_datetime: bool = False) -> None: :param path: Path to save the JSON file :param as_datetime: If True, convert timestamps to ISO fmt datetime strings """ - import json - data = [] for candle in self: + # Skip gaps (volume == -1) + if candle.volume == -1: + continue if as_datetime: item = { "time": datetime.fromtimestamp(candle.timestamp, UTC).isoformat(), diff --git a/src/pynecore/lib/timeframe.py b/src/pynecore/lib/timeframe.py index 17df550..99ce038 100644 --- a/src/pynecore/lib/timeframe.py +++ b/src/pynecore/lib/timeframe.py @@ -423,8 +423,8 @@ def from_seconds(seconds: int) -> str: return f"{seconds // (60 * 60 * 24 * 7)}W" if seconds % (60 * 60 * 24) == 0: return f"{seconds // (60 * 60 * 24)}D" - if seconds % (60 * 60) == 0: - return f"{seconds // (60 * 60)}" + if seconds % 60 == 0: + return f"{seconds // 60}" return f"{seconds}S" diff --git a/tests/t00_pynecore/data/test_001_ohlcv_file.py b/tests/t00_pynecore/data/test_001_ohlcv_file.py index 0dde804..2105599 100644 --- a/tests/t00_pynecore/data/test_001_ohlcv_file.py +++ b/tests/t00_pynecore/data/test_001_ohlcv_file.py @@ -413,3 +413,146 @@ def __test_ohlcv_gap_filling_and_skipping__(tmp_path): # Verify the records assert candles[0].timestamp == 1609459260 assert candles[1].timestamp == 1609459380 + + +def __test_opening_hours_detection_intraday__(tmp_path): + """Test opening hours detection for intraday timeframes""" + from datetime import datetime + file_path = tmp_path / "test_opening_hours_intraday.ohlcv" + + with OHLCVWriter(file_path) as writer: + # Simulate stock market data: Monday-Friday 9:30-16:00 + # Start from a Monday 9:30 AM EST (2024-01-08 09:30:00) + base_timestamp = int(datetime(2024, 1, 8, 9, 30).timestamp()) + + # Write data for multiple days with 1-minute intervals + for day in range(5): # Monday to Friday + day_offset = day * 86400 # Seconds in a day + + # Trading hours: 9:30 AM to 4:00 PM (6.5 hours = 390 minutes) + for minute in range(390): + timestamp = base_timestamp + day_offset + (minute * 60) + price = 100.0 + (minute * 0.01) # Gradual price increase + writer.write(OHLCV( + timestamp=timestamp, + open=price, + high=price + 0.5, + low=price - 0.5, + close=price + 0.1, + volume=1000.0 + minute + )) + + # Check opening hours detection + with OHLCVWriter(file_path) as writer: + opening_hours = writer.analyzed_opening_hours + + # Should detect business hours pattern + assert opening_hours is not None, "Opening hours should be detected" + assert len(opening_hours) > 0, "Should have detected some opening hours" + + # Check that we have Monday-Friday entries + days_with_hours = {interval.day for interval in opening_hours} + assert 1 in days_with_hours # Monday + assert 5 in days_with_hours # Friday + assert 6 not in days_with_hours # Saturday should not be present + assert 7 not in days_with_hours # Sunday should not be present + + +def __test_opening_hours_detection_crypto__(tmp_path): + """Test opening hours detection for crypto (24/7) markets""" + from datetime import datetime + file_path = tmp_path / "test_opening_hours_crypto.ohlcv" + + with OHLCVWriter(file_path) as writer: + # Simulate crypto data: 24/7 trading + # Start from a Monday 00:00 UTC (2024-01-08 00:00:00) + base_timestamp = int(datetime(2024, 1, 8, 0, 0).timestamp()) + + # Write data for a full week with 5-minute intervals + for hour in range(168): # 7 days * 24 hours + for five_min in range(12): # 12 five-minute intervals per hour + timestamp = base_timestamp + (hour * 3600) + (five_min * 300) + price = 50000.0 + (hour * 10.0) # BTC-like prices + writer.write(OHLCV( + timestamp=timestamp, + open=price, + high=price + 50, + low=price - 50, + close=price + 10, + volume=100.0 + five_min + )) + + # Check opening hours detection + with OHLCVWriter(file_path) as writer: + opening_hours = writer.analyzed_opening_hours + + # Should detect 24/7 pattern + assert opening_hours is not None, "Opening hours should be detected" + assert len(opening_hours) == 7, "Should have all 7 days for 24/7 trading" + + # Check that all days are 00:00-23:59 + for interval in opening_hours: + assert interval.start.hour == 0 and interval.start.minute == 0 + assert interval.end.hour == 23 and interval.end.minute == 59 + + +def __test_opening_hours_detection_daily__(tmp_path): + """Test opening hours detection for daily timeframes""" + from datetime import datetime + file_path = tmp_path / "test_opening_hours_daily.ohlcv" + + with OHLCVWriter(file_path) as writer: + # Simulate daily stock data: Monday-Friday only + # Start from a Monday (2024-01-08) + base_timestamp = int(datetime(2024, 1, 8, 16, 0).timestamp()) # Daily close at 4 PM + + # Write data for 3 weeks (15 business days) + for week in range(3): + for day in range(5): # Monday to Friday only + timestamp = base_timestamp + (week * 7 * 86400) + (day * 86400) + price = 150.0 + (week * 5) + day + writer.write(OHLCV( + timestamp=timestamp, + open=price, + high=price + 2, + low=price - 2, + close=price + 1, + volume=1000000.0 + )) + + # Check opening hours detection + with OHLCVWriter(file_path) as writer: + opening_hours = writer.analyzed_opening_hours + + # Should detect weekday-only pattern from daily data + assert opening_hours is not None, "Opening hours should be detected" + assert len(opening_hours) == 5, "Should have Monday-Friday for daily stock data" + + # Check that we only have weekdays (1-5) + days = {interval.day for interval in opening_hours} + assert days == {1, 2, 3, 4, 5}, "Should only have Monday-Friday" + + +def __test_opening_hours_insufficient_data__(tmp_path): + """Test opening hours detection with insufficient data""" + file_path = tmp_path / "test_opening_hours_insufficient.ohlcv" + + with OHLCVWriter(file_path) as writer: + # Write only a few data points (less than required minimum) + base_timestamp = 1609459200 + for i in range(5): # Only 5 minutes of data + writer.write(OHLCV( + timestamp=base_timestamp + (i * 60), + open=100.0, + high=101.0, + low=99.0, + close=100.5, + volume=1000.0 + )) + + # Check opening hours detection + with OHLCVWriter(file_path) as writer: + opening_hours = writer.analyzed_opening_hours + + # Should return None for insufficient data + assert opening_hours is None, "Should return None for insufficient data" diff --git a/tests/t00_pynecore/data/test_004_data_converter.py b/tests/t00_pynecore/data/test_004_data_converter.py new file mode 100644 index 0000000..f68baf9 --- /dev/null +++ b/tests/t00_pynecore/data/test_004_data_converter.py @@ -0,0 +1,297 @@ +""" +@pyne +""" +import pytest +from pathlib import Path + +from pynecore.core.data_converter import DataConverter + + +def main(): + """ + Dummy main function to be a valid Pyne script + """ + pass + + +def __test_symbol_provider_detection_ccxt__(): + """Test CCXT-style filename detection""" + dc = DataConverter() + + # Test without ccxt prefix but with exchange + symbol, provider = dc.guess_symbol_from_filename(Path("BINANCE_BTC_USDT.csv")) + assert symbol == "BTC/USDT" + assert provider == "binance" + + # Test exchange with compact symbol + symbol, provider = dc.guess_symbol_from_filename(Path("BINANCE_BTCUSDT.csv")) + assert symbol == "BTC/USDT" + assert provider == "binance" + + # Test with colon separators + symbol, provider = dc.guess_symbol_from_filename(Path("BYBIT:BTC:USDT.csv")) + assert symbol == "BTC/USDT" + assert provider == "bybit" + + # Test ccxt with BYBIT exchange - provider should be bybit, not ccxt + symbol, provider = dc.guess_symbol_from_filename(Path("ccxt_BYBIT_BTC_USDT_USDT_1.csv")) + assert symbol == "BTC/USDT" + assert provider == "bybit" # When ccxt_ prefix, provider is the exchange name + + +def __test_symbol_provider_detection_capitalcom__(): + """Test Capital.com filename detection""" + dc = DataConverter() + + # Test with dots + symbol, provider = dc.guess_symbol_from_filename(Path("capital.com_EURUSD_60.csv")) + assert symbol == "EURUSD" + assert provider == "capital.com" + + # Test with uppercase + symbol, provider = dc.guess_symbol_from_filename(Path("CAPITALCOM_EURUSD.csv")) + assert symbol == "EURUSD" + assert provider == "capitalcom" + + +def __test_symbol_provider_detection_tradingview__(): + """Test TradingView export format detection""" + dc = DataConverter() + + # Test with hash suffix + symbol, provider = dc.guess_symbol_from_filename(Path("CAPITALCOM_EURUSD, 30_cbf9d.csv")) + assert symbol == "EURUSD" + assert provider == "capitalcom" + + # Test TV prefix + symbol, provider = dc.guess_symbol_from_filename(Path("TV_BTCUSD_1h.csv")) + assert symbol == "BTCUSD" + assert provider == "tradingview" + + # Test TradingView prefix + symbol, provider = dc.guess_symbol_from_filename(Path("TRADINGVIEW_AAPL_daily.csv")) + assert symbol == "AAPL" + assert provider == "tradingview" + + +def __test_symbol_provider_detection_metatrader__(): + """Test MetaTrader filename detection""" + dc = DataConverter() + + # Test MT4 format + symbol, provider = dc.guess_symbol_from_filename(Path("MT4_EURUSD_M1.csv")) + assert symbol == "EURUSD" + assert provider == "mt4" + + # Test MT5 format + symbol, provider = dc.guess_symbol_from_filename(Path("MT5_GBPUSD_H1_2024.csv")) + assert symbol == "GBPUSD" + assert provider == "mt5" + + # Test forex pair without explicit provider + symbol, provider = dc.guess_symbol_from_filename(Path("EURUSD.csv")) + assert symbol == "EURUSD" + assert provider == "forex" + + # Test another forex pair + symbol, provider = dc.guess_symbol_from_filename(Path("GBPJPY.csv")) + assert symbol == "GBPJPY" + assert provider == "forex" + + +def __test_symbol_provider_detection_crypto_exchanges__(): + """Test various crypto exchange filename formats""" + dc = DataConverter() + + # Binance + symbol, provider = dc.guess_symbol_from_filename(Path("BINANCE_BTCUSDT.csv")) + assert symbol == "BTC/USDT" + assert provider == "binance" + + # Bybit + symbol, provider = dc.guess_symbol_from_filename(Path("BYBIT_ETH_USDT.csv")) + assert symbol == "ETH/USDT" + assert provider == "bybit" + + # Coinbase + symbol, provider = dc.guess_symbol_from_filename(Path("COINBASE_BTC_USD.csv")) + assert symbol == "BTC/USD" + assert provider == "coinbase" + + # Kraken + symbol, provider = dc.guess_symbol_from_filename(Path("KRAKEN_XRPUSD.csv")) + assert symbol == "XRP/USD" + assert provider == "kraken" + + +def __test_symbol_provider_detection_generic_crypto__(): + """Test generic crypto pair detection without provider""" + dc = DataConverter() + + # Common crypto pairs should be detected + symbol, provider = dc.guess_symbol_from_filename(Path("BTCUSDT.csv")) + assert symbol == "BTC/USDT" + assert provider == "ccxt" + + symbol, provider = dc.guess_symbol_from_filename(Path("ETHUSD.csv")) + assert symbol == "ETH/USD" + assert provider == "ccxt" + + symbol, provider = dc.guess_symbol_from_filename(Path("BTC_USDT.csv")) + assert symbol == "BTC/USDT" + assert provider == "ccxt" + + +def __test_symbol_provider_detection_stock_symbols__(): + """Test stock symbol detection""" + dc = DataConverter() + + # Simple stock symbols + symbol, provider = dc.guess_symbol_from_filename(Path("AAPL.csv")) + assert symbol == "AAPL" + assert provider is None + + symbol, provider = dc.guess_symbol_from_filename(Path("MSFT_daily.csv")) + assert symbol == "MSFT" + assert provider is None + + # With IB provider + symbol, provider = dc.guess_symbol_from_filename(Path("IB_AAPL_1h.csv")) + assert symbol == "AAPL" + assert provider == "ib" + + +def __test_symbol_provider_detection_complex_filenames__(): + """Test complex filename patterns""" + dc = DataConverter() + + # Multiple underscores and timeframe - ccxt_ prefix means provider is exchange + symbol, provider = dc.guess_symbol_from_filename(Path("ccxt_BYBIT_BTC_USDT_USDT_5.csv")) + assert symbol == "BTC/USDT" + assert provider == "bybit" # When ccxt_ prefix, provider is the exchange name + + # Mixed case - Note: Mixed case may not be detected properly + # Using uppercase for consistency + symbol, provider = dc.guess_symbol_from_filename(Path("CAPITAL.COM_EURUSD_60.csv")) + assert symbol == "EURUSD" + assert provider == "capital.com" + + # With date suffix + symbol, provider = dc.guess_symbol_from_filename(Path("MT5_EURUSD_2024_01_01.csv")) + assert symbol == "EURUSD" + assert provider == "mt5" + + +def __test_symbol_provider_detection_edge_cases__(): + """Test edge cases and invalid formats""" + dc = DataConverter() + + # Empty filename + symbol, provider = dc.guess_symbol_from_filename(Path(".csv")) + assert symbol is None + assert provider is None + + # Too short symbol + symbol, provider = dc.guess_symbol_from_filename(Path("XX.csv")) + assert symbol is None + assert provider is None + + # Only provider, no symbol + symbol, provider = dc.guess_symbol_from_filename(Path("CCXT.csv")) + assert symbol is None + assert provider == "ccxt" + + # Numbers only (should not be detected as symbol) + symbol, provider = dc.guess_symbol_from_filename(Path("12345.csv")) + assert symbol is None + assert provider is None + + +def __test_symbol_provider_detection_forex_pairs__(): + """Test various forex pair formats""" + dc = DataConverter() + + # Standard 6-char format + symbol, provider = dc.guess_symbol_from_filename(Path("EURUSD.csv")) + assert symbol == "EURUSD" + assert provider == "forex" + + # With separator + symbol, provider = dc.guess_symbol_from_filename(Path("EUR_USD.csv")) + assert symbol == "EURUSD" + assert provider == "forex" + + # With slash + symbol, provider = dc.guess_symbol_from_filename(Path("EUR-USD.csv")) + assert symbol == "EURUSD" + assert provider == "forex" + + # Less common pairs + symbol, provider = dc.guess_symbol_from_filename(Path("NZDJPY.csv")) + assert symbol == "NZDJPY" + assert provider == "forex" + + +def __test_symbol_provider_detection_our_format__(): + """Test PyneCore own format detection""" + dc = DataConverter() + + # Our format with provider and symbol + symbol, provider = dc.guess_symbol_from_filename(Path("capitalcom_EURUSD_60.ohlcv")) + assert symbol == "EURUSD" + assert provider == "capitalcom" + + # CCXT style with exchange - provider is exchange name + symbol, provider = dc.guess_symbol_from_filename(Path("ccxt_BYBIT_BTC_USDT_USDT_1.ohlcv")) + assert symbol == "BTC/USDT" + assert provider == "bybit" # When ccxt_ prefix, provider is the exchange name + + # Simple format without provider + symbol, provider = dc.guess_symbol_from_filename(Path("BTCUSD_1h.ohlcv")) + assert symbol == "BTC/USD" + assert provider == "ccxt" # Should default to ccxt for crypto + + +# Test runner functions that pytest will find +def test_symbol_provider_detection_ccxt(): + __test_symbol_provider_detection_ccxt__() + + +def test_symbol_provider_detection_capitalcom(): + __test_symbol_provider_detection_capitalcom__() + + +def test_symbol_provider_detection_tradingview(): + __test_symbol_provider_detection_tradingview__() + + +def test_symbol_provider_detection_metatrader(): + __test_symbol_provider_detection_metatrader__() + + +def test_symbol_provider_detection_crypto_exchanges(): + __test_symbol_provider_detection_crypto_exchanges__() + + +def test_symbol_provider_detection_generic_crypto(): + __test_symbol_provider_detection_generic_crypto__() + + +def test_symbol_provider_detection_stock_symbols(): + __test_symbol_provider_detection_stock_symbols__() + + +def test_symbol_provider_detection_complex_filenames(): + __test_symbol_provider_detection_complex_filenames__() + + +def test_symbol_provider_detection_edge_cases(): + __test_symbol_provider_detection_edge_cases__() + + +def test_symbol_provider_detection_forex_pairs(): + __test_symbol_provider_detection_forex_pairs__() + + +def test_symbol_provider_detection_our_format(): + __test_symbol_provider_detection_our_format__() \ No newline at end of file diff --git a/tests/t00_pynecore/data/test_005_symbol_type_detection.py b/tests/t00_pynecore/data/test_005_symbol_type_detection.py new file mode 100644 index 0000000..9d40a62 --- /dev/null +++ b/tests/t00_pynecore/data/test_005_symbol_type_detection.py @@ -0,0 +1,229 @@ +""" +@pyne +""" +import pytest +from pathlib import Path + +from pynecore.core.data_converter import DataConverter + + +def main(): + """ + Dummy main function to be a valid Pyne script + """ + pass + + +def __test_detect_symbol_type_forex__(): + """Test forex pair detection""" + dc = DataConverter() + + # Standard forex pairs + symbol_type, currency, base = dc.guess_symbol_type("EURUSD") + assert symbol_type == "forex" + assert currency == "USD" + assert base == "EUR" + + symbol_type, currency, base = dc.guess_symbol_type("EUR/USD") + assert symbol_type == "forex" + assert currency == "USD" + assert base == "EUR" + + symbol_type, currency, base = dc.guess_symbol_type("GBPUSD") + assert symbol_type == "forex" + assert currency == "USD" + assert base == "GBP" + + symbol_type, currency, base = dc.guess_symbol_type("USDJPY") + assert symbol_type == "forex" + assert currency == "JPY" + assert base == "USD" + + # Less common forex pairs + symbol_type, currency, base = dc.guess_symbol_type("NZDJPY") + assert symbol_type == "forex" + assert currency == "JPY" + assert base == "NZD" + + symbol_type, currency, base = dc.guess_symbol_type("EURCHF") + assert symbol_type == "forex" + assert currency == "CHF" + assert base == "EUR" + + +def __test_detect_symbol_type_crypto__(): + """Test crypto pair detection""" + dc = DataConverter() + + # Common crypto pairs + symbol_type, currency, base = dc.guess_symbol_type("BTC/USDT") + assert symbol_type == "crypto" + assert currency == "USDT" + assert base == "BTC" + + symbol_type, currency, base = dc.guess_symbol_type("BTCUSDT") + assert symbol_type == "crypto" + assert currency == "USDT" + assert base == "BTC" + + symbol_type, currency, base = dc.guess_symbol_type("ETH/USD") + assert symbol_type == "crypto" + assert currency == "USD" + assert base == "ETH" + + symbol_type, currency, base = dc.guess_symbol_type("ETHUSDT") + assert symbol_type == "crypto" + assert currency == "USDT" + assert base == "ETH" + + # Other crypto coins + symbol_type, currency, base = dc.guess_symbol_type("ADAUSDT") + assert symbol_type == "crypto" + assert currency == "USDT" + assert base == "ADA" + + symbol_type, currency, base = dc.guess_symbol_type("DOGEUSDT") + assert symbol_type == "crypto" + assert currency == "USDT" + assert base == "DOGE" + + symbol_type, currency, base = dc.guess_symbol_type("SOL/USDC") + assert symbol_type == "crypto" + assert currency == "USDC" + assert base == "SOL" + + +def __test_detect_symbol_type_other__(): + """Test that unknown symbols default to 'other' type""" + dc = DataConverter() + + # Stock-like symbols should be 'other' now + symbol_type, currency, base = dc.guess_symbol_type("AAPL") + assert symbol_type == "other" + assert currency == "USD" + assert base is None + + symbol_type, currency, base = dc.guess_symbol_type("MSFT") + assert symbol_type == "other" + assert currency == "USD" + assert base is None + + # Unknown patterns + symbol_type, currency, base = dc.guess_symbol_type("UNKNOWN") + assert symbol_type == "other" + assert currency == "USD" + assert base is None + + symbol_type, currency, base = dc.guess_symbol_type("ABC123") + assert symbol_type == "other" + assert currency == "USD" + assert base is None + + # Too short + symbol_type, currency, base = dc.guess_symbol_type("XY") + assert symbol_type == "other" + assert currency == "USD" + assert base is None + + +def __test_detect_symbol_type_edge_cases__(): + """Test edge cases""" + dc = DataConverter() + + # Ambiguous 6-letter that could be forex or other + symbol_type, currency, base = dc.guess_symbol_type("ABCDEF") + assert symbol_type == "other" # Not recognized forex pair + assert currency == "USD" + assert base is None + + # Crypto with USD (not USDT) + symbol_type, currency, base = dc.guess_symbol_type("BTCUSD") + assert symbol_type == "crypto" + assert currency == "USD" + assert base == "BTC" + + # Test with uppercase (method expects uppercase input) + symbol_type, currency, base = dc.guess_symbol_type("BTCUSDT") + assert symbol_type == "crypto" + assert currency == "USDT" + assert base == "BTC" + + # With underscores (cleaned internally) + symbol_type, currency, base = dc.guess_symbol_type("EUR_USD") + assert symbol_type == "forex" + assert currency == "USD" + assert base == "EUR" + + +def __test_detect_symbol_type_forex_with_slash__(): + """Test forex pairs with explicit slash notation""" + dc = DataConverter() + + # These should be detected as forex because both parts are 3-letter currency codes + symbol_type, currency, base = dc.guess_symbol_type("EUR/USD") + assert symbol_type == "forex" + assert currency == "USD" + assert base == "EUR" + + symbol_type, currency, base = dc.guess_symbol_type("GBP/JPY") + assert symbol_type == "forex" + assert currency == "JPY" + assert base == "GBP" + + symbol_type, currency, base = dc.guess_symbol_type("AUD/CAD") + assert symbol_type == "forex" + assert currency == "CAD" + assert base == "AUD" + + +def __test_detect_symbol_type_special_cases__(): + """Test special handling cases""" + dc = DataConverter() + + # BTCUSD vs BTCUSDT - both should be crypto + symbol_type, currency, base = dc.guess_symbol_type("BTCUSD") + assert symbol_type == "crypto" + assert currency == "USD" + assert base == "BTC" + + symbol_type, currency, base = dc.guess_symbol_type("BTCUSDT") + assert symbol_type == "crypto" + assert currency == "USDT" + assert base == "BTC" + + # 6-letter code that happens to have a crypto symbol in it + # but isn't a standard pair + symbol_type, currency, base = dc.guess_symbol_type("BTCXYZ") + assert symbol_type == "crypto" # BTC is detected + # Currency extraction might vary + + # Forex currencies in non-standard order (still detected as forex) + symbol_type, currency, base = dc.guess_symbol_type("JPYEUR") + assert symbol_type == "forex" # Both JPY and EUR are forex currencies + assert currency == "EUR" + assert base == "JPY" + + +# Test runner functions that pytest will find +def test_detect_symbol_type_forex(): + __test_detect_symbol_type_forex__() + + +def test_detect_symbol_type_crypto(): + __test_detect_symbol_type_crypto__() + + +def test_detect_symbol_type_other(): + __test_detect_symbol_type_other__() + + +def test_detect_symbol_type_edge_cases(): + __test_detect_symbol_type_edge_cases__() + + +def test_detect_symbol_type_forex_with_slash(): + __test_detect_symbol_type_forex_with_slash__() + + +def test_detect_symbol_type_special_cases(): + __test_detect_symbol_type_special_cases__() \ No newline at end of file diff --git a/tests/t01_lib/t01_timeframe/test_003_from_seconds.py b/tests/t01_lib/t01_timeframe/test_003_from_seconds.py new file mode 100644 index 0000000..d66ea47 --- /dev/null +++ b/tests/t01_lib/t01_timeframe/test_003_from_seconds.py @@ -0,0 +1,34 @@ +""" +@pyne +""" +from pynecore.lib import script, timeframe + + +@script.indicator(title="Timeframe from_seconds()", shorttitle="tf_fs") +def main(): + # Test basic conversions + assert timeframe.from_seconds(1) == "1S" + assert timeframe.from_seconds(30) == "30S" + assert timeframe.from_seconds(60) == "1" # 1 minute - THIS WAS THE BUG! + assert timeframe.from_seconds(120) == "2" # 2 minutes + assert timeframe.from_seconds(300) == "5" # 5 minutes + assert timeframe.from_seconds(900) == "15" # 15 minutes + assert timeframe.from_seconds(3600) == "60" # 1 hour + assert timeframe.from_seconds(14400) == "240" # 4 hours + assert timeframe.from_seconds(86400) == "1D" # 1 day + assert timeframe.from_seconds(172800) == "2D" # 2 days + assert timeframe.from_seconds(604800) == "1W" # 1 week + assert timeframe.from_seconds(1209600) == "2W" # 2 weeks + assert timeframe.from_seconds(2419200) == "1M" # 1 month (4 weeks) + assert timeframe.from_seconds(4838400) == "2M" # 2 months + + # Edge cases + assert timeframe.from_seconds(59) == "59S" # Not divisible by 60 + assert timeframe.from_seconds(61) == "61S" # Not divisible by 60 + assert timeframe.from_seconds(180) == "3" # 3 minutes + assert timeframe.from_seconds(240) == "4" # 4 minutes + + +def __test_timeframe_from_seconds__(runner, dummy_ohlcv_iter): + """ timeframe.from_seconds() """ + next(runner(dummy_ohlcv_iter).run_iter()) \ No newline at end of file From 204d618489cc5b92830de89b6bb7120aea9f646b Mon Sep 17 00:00:00 2001 From: Adam Wallner Date: Sun, 17 Aug 2025 18:52:48 +0200 Subject: [PATCH 24/31] fix(core): cast buffer in struct.unpack to fix IDE warning --- src/pynecore/core/ohlcv_file.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/pynecore/core/ohlcv_file.py b/src/pynecore/core/ohlcv_file.py index 5b48be5..60acf18 100644 --- a/src/pynecore/core/ohlcv_file.py +++ b/src/pynecore/core/ohlcv_file.py @@ -629,22 +629,23 @@ def _collect_existing_trading_hours(self) -> None: """ if not self._file or self._size == 0: return - + # Save current position current_pos = self._file.tell() - + try: # Sample data: read every Nth record for performance # For large files, we don't need to read everything sample_interval = max(1, self._size // 1000) # Sample up to 1000 points - + for i in range(0, self._size, sample_interval): self._file.seek(i * RECORD_SIZE) data = self._file.read(RECORD_SIZE) - + if len(data) == RECORD_SIZE: # Unpack the record - timestamp, open_val, high, low, close, volume = struct.unpack('Ifffff', data) + timestamp, open_val, high, low, close, volume = \ + struct.unpack('Ifffff', cast(Buffer, data)) # Only collect if volume > 0 (real trading) if volume > 0: @@ -653,31 +654,31 @@ def _collect_existing_trading_hours(self) -> None: hour = dt.hour key = (weekday, hour) self._trading_hours[key] = self._trading_hours.get(key, 0) + 1 - + finally: # Restore file position self._file.seek(current_pos) - + def _has_enough_data_for_opening_hours(self) -> bool: """ Check if we have enough data to analyze opening hours based on timeframe. """ if not self._trading_hours or not self._interval: return False - + # For daily or larger timeframes if self._interval >= 86400: # >= 1 day # We need at least a few days to see a pattern unique_days = len(set(day for day, hour in self._trading_hours.keys())) return unique_days >= 3 # At least 3 different days - + # For intraday timeframes # Check if we have at least some meaningful data # We need enough to see a pattern data_points = sum(self._trading_hours.values()) points_per_hour = 3600 / self._interval hours_covered = data_points / points_per_hour - + # Need at least 2 hours of data to detect any pattern # This allows even short sessions to be analyzed return hours_covered >= 2 @@ -695,7 +696,7 @@ def _analyze_opening_hours(self) -> None: if self._interval and self._interval >= 86400: # >= 1 day self._analyzed_opening_hours = [] days_with_trading = set(day for day, hour in self._trading_hours.keys()) - + # Check if it's 24/7 (all 7 days have trading) if len(days_with_trading) == 7: # 24/7 trading pattern @@ -811,7 +812,7 @@ def _rebuild_with_correct_interval(self, new_interval: int) -> None: """ import tempfile import shutil - + if not self._file or self._size == 0: return From 556712547313458437ad39a985a26534237d11c0 Mon Sep 17 00:00:00 2001 From: Adam Wallner Date: Sun, 17 Aug 2025 19:22:04 +0200 Subject: [PATCH 25/31] docs: improve data conversion docs for smart OHLCV detection and auto config - Update convert-to command description to clarify OHLCV file usage - Add details on automatic extension handling and output file naming - Document auto-detection of symbol, provider, and format from filenames - Provide filename pattern examples for symbol/provider extraction - Explain TOML config generation: symbol type, tick size, trading hours, interval - Update run command docs to support auto conversion from CSV/JSON/TXT - Add section on advanced analysis during conversion (tick size, hours, etc.) - Clarify supported formats and filename detection for run command - Improve examples for automatic conversion and config generation --- docs/cli/data.md | 63 ++++++++++++++++++++++++++++++++++++------------ docs/cli/run.md | 52 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 96 insertions(+), 19 deletions(-) diff --git a/docs/cli/data.md b/docs/cli/data.md index a6392a7..d5a8b3c 100644 --- a/docs/cli/data.md +++ b/docs/cli/data.md @@ -140,25 +140,33 @@ PyneCore uses a binary format (`.ohlcv`) for storing OHLCV data efficiently. How ### Converting to Other Formats -The `convert-to` command converts PyneCore format to CSV or JSON: +The `convert-to` command converts PyneCore OHLCV format to CSV or JSON: ```bash -pyne data convert-to PROVIDER [OPTIONS] +pyne data convert-to OHLCV_FILE [OPTIONS] ``` +Where `OHLCV_FILE` is the path to the OHLCV file to convert. + Options: -- `--symbol`, `-s`: Symbol to convert -- `--timeframe`, `-tf`: Timeframe in TradingView format -- `--format`, `-f`: Output format (csv, json) +- `--format`, `-f`: Output format (csv or json, default: csv) - `--as-datetime`, `-dt`: Save timestamp as datetime instead of UNIX timestamp +The command automatically: +- Adds `.ohlcv` extension if not specified +- Creates output file with the same name but different extension +- Looks in `workdir/data/` if only filename is provided + Example: ```bash -# Convert Bitcoin data to CSV -pyne data convert-to ccxt --symbol "BINANCE:BTC/USDT" --timeframe "1D" --format "csv" +# Convert OHLCV file to CSV +pyne data convert-to BTCUSDT_1D.ohlcv + +# Convert to JSON with human-readable dates +pyne data convert-to BTCUSDT_1D.ohlcv --format json --as-datetime -# Convert with human-readable dates -pyne data convert-to ccxt --symbol "BINANCE:BTC/USDT" --timeframe "1D" --format "csv" --as-datetime +# Short form (extension optional) +pyne data convert-to BTCUSDT_1D -f csv -dt ``` ### Converting from Other Formats @@ -172,21 +180,44 @@ pyne data convert-from FILE_PATH [OPTIONS] Where `FILE_PATH` is the path to the CSV or JSON file to convert. Options: -- `--provider`, `-p`: Data provider name (can be any name, defaults to "custom") -- `--symbol`, `-s`: Symbol name -- `--timeframe`, `-tf`: Timeframe in TradingView format -- `--fmt`, `-f`: Input format (csv, json) - defaults to the file extension if not specified +- `--provider`, `-p`: Data provider name (defaults to auto-detected from filename) +- `--symbol`, `-s`: Symbol name (defaults to auto-detected from filename) - `--timezone`, `-tz`: Timezone of the timestamps (defaults to UTC) +**Automatic Detection Features:** +- **Symbol Detection**: The command automatically detects symbols from common filename patterns +- **Provider Detection**: Recognizes provider names in filenames (BINANCE, BYBIT, CAPITALCOM, etc.) +- **Format Support**: Supports CSV and JSON files, auto-detected from file extension + +**Filename Pattern Examples:** +- `BTCUSDT.csv` → Symbol: BTC/USDT +- `EUR_USD.csv` → Symbol: EUR/USD +- `ccxt_BYBIT_BTC_USDT.csv` → Symbol: BTC/USDT, Provider: bybit +- `BINANCE_ETHUSDT_1h.csv` → Symbol: ETH/USDT, Provider: binance +- `capitalcom_EURUSD.csv` → Symbol: EUR/USD, Provider: capitalcom + Example: ```bash -# Convert CSV to PyneCore format -pyne data convert-from ./data/btcusd.csv --symbol "CUSTOM:BTC/USD" --timeframe "1D" +# Convert CSV with automatic detection +pyne data convert-from ./data/BTCUSDT.csv # Auto-detects BTC/USDT + +# Override auto-detected values if needed +pyne data convert-from ./data/btcusd.csv --symbol "BTC/USD" --provider "kraken" # Convert with timezone specification -pyne data convert-from ./data/eurusd.csv --symbol "CUSTOM:EUR/USD" --timeframe "60" --timezone "Europe/London" +pyne data convert-from ./data/eurusd.csv --timezone "Europe/London" ``` +**Generated TOML Configuration:** + +After conversion, a TOML configuration file is automatically generated with: +- **Smart Symbol Type Detection**: Automatically identifies forex, crypto, or other asset types +- **Tick Size Analysis**: Analyzes price data to determine the minimum price increment +- **Opening Hours Detection**: Detects trading hours from actual trading activity +- **Interval Detection**: Automatically determines the timeframe from timestamp intervals + +The generated TOML file includes all detected information and can be manually adjusted if needed. + ## Data File Structure PyneCore uses a structured approach to store OHLCV data: diff --git a/docs/cli/run.md b/docs/cli/run.md index e5a032c..129be7f 100644 --- a/docs/cli/run.md +++ b/docs/cli/run.md @@ -27,7 +27,7 @@ pyne run SCRIPT DATA [OPTIONS] Where: - `SCRIPT`: Path to the PyneCore script (.py) or Pine Script (.pine) file -- `DATA`: Path to the OHLCV data (.ohlcv) file +- `DATA`: Path to the data file (.ohlcv, .csv, .json, or .txt) - `OPTIONS`: Additional options to customize the execution ## Simple Example @@ -81,15 +81,61 @@ Example with API key: pyne run my_strategy.pine eurusd_data.ohlcv --api-key "your-api-key" ``` +## Automatic Data Conversion + +The `run` command now supports automatic conversion of non-OHLCV data formats. When you provide a CSV, JSON, or TXT file, the system automatically: + +1. **Detects the file format** from the extension +2. **Analyzes the filename** to extract symbol and provider information +3. **Converts the data** to OHLCV format +4. **Generates a TOML configuration** with detected parameters +5. **Runs the script** with the converted data + +### Supported Formats and Detection + +The automatic conversion supports: +- **CSV files**: Standard comma-separated values +- **JSON files**: JSON formatted OHLCV data +- **TXT files**: Tab, semicolon, or pipe-delimited data (coming soon) + +### Filename Pattern Detection + +The system recognizes common filename patterns: +- `BTCUSDT.csv` → Symbol: BTC/USDT +- `EUR_USD.json` → Symbol: EUR/USD +- `ccxt_BYBIT_BTC_USDT.csv` → Symbol: BTC/USDT, Provider: bybit +- `BINANCE_ETHUSDT_1h.csv` → Symbol: ETH/USDT, Provider: binance + +### Example with Automatic Conversion + +```bash +# Run a script with CSV data (automatic conversion) +pyne run my_strategy.py BTCUSDT.csv + +# The system will: +# 1. Detect BTC/USDT as the symbol +# 2. Convert CSV to OHLCV format +# 3. Generate BTCUSDT.toml with symbol info +# 4. Run the script with converted data +``` + +### Advanced Analysis During Conversion + +When converting data, the system performs advanced analysis: +- **Tick Size Detection**: Analyzes price movements to determine minimum price increment +- **Trading Hours Detection**: Identifies when the market is actively trading +- **Interval Auto-Correction**: Detects and fixes incorrect timeframe settings +- **Symbol Type Detection**: Identifies forex, crypto, or other asset types + ## Command Arguments The `run` command has two required arguments: - `SCRIPT`: The script file to run. If only a filename is provided, it will be searched in the `workdir/scripts/` directory. -- `DATA`: The OHLCV data file to use. If only a filename is provided, it will be searched in the `workdir/data/` directory. +- `DATA`: The data file to use. Supports .ohlcv, .csv, .json formats. If only a filename is provided, it will be searched in the `workdir/data/` directory. -Note: you don't need to write the `.py` and `.ohlcv` extensions in the command. +Note: you don't need to write the file extensions in the command. ## Command Options From 6f7a90c0a9bb322789fc1ed221ccdd711dd79acc Mon Sep 17 00:00:00 2001 From: Mahadi Hassan Date: Mon, 18 Aug 2025 16:36:01 +0600 Subject: [PATCH 26/31] feat(data_converter): add support for txt file format in data conversion Implement TXT file support with auto-detected delimiters (tab, semicolon, pipe) Add type checking for price data collection to handle NA values Improve timestamp handling in CSV/TXT parsing with better error handling --- src/pynecore/core/data_converter.py | 6 +- src/pynecore/core/ohlcv_file.py | 183 +++++++++++++++++++++++++++- 2 files changed, 181 insertions(+), 8 deletions(-) diff --git a/src/pynecore/core/data_converter.py b/src/pynecore/core/data_converter.py index b2afc06..8951a35 100644 --- a/src/pynecore/core/data_converter.py +++ b/src/pynecore/core/data_converter.py @@ -128,10 +128,8 @@ def convert_to_ohlcv( ohlcv_writer.load_from_csv(file_path, tz=timezone) elif detected_format == 'json': ohlcv_writer.load_from_json(file_path, tz=timezone) - # TODO: Add support for TXT! - # elif detected_format == 'txt': - # # Treat TXT files as CSV with different delimiters - # ohlcv_writer.load_from_csv(file_path, tz=timezone) + elif detected_format == 'txt': + ohlcv_writer.load_from_txt(file_path, tz=timezone) else: raise ConversionError(f"Unsupported format for conversion: {detected_format}") diff --git a/src/pynecore/core/ohlcv_file.py b/src/pynecore/core/ohlcv_file.py index 60acf18..8d9b4cc 100644 --- a/src/pynecore/core/ohlcv_file.py +++ b/src/pynecore/core/ohlcv_file.py @@ -350,20 +350,24 @@ def _collect_price_data(self, candle: OHLCV) -> None: Collect price data for tick size analysis during writing. """ # Collect price changes - if self._last_close is not None: + if self._last_close is not None and isinstance(candle.close, (int, float)): change = abs(candle.close - self._last_close) if change > 0 and len(self._price_changes) < 1000: # Limit to 1000 samples self._price_changes.append(change) # Collect decimal places for price in [candle.open, candle.high, candle.low, candle.close]: - if price != int(price): # Has decimal component + if isinstance(price, (int, float)) and price != int(price): # Has decimal component price_str = f"{price:.15f}".rstrip('0').rstrip('.') if '.' in price_str: decimals = len(price_str.split('.')[1]) self._price_decimals.add(decimals) - self._last_close = candle.close + # Store last close, handling both float and NA[float] types + if isinstance(candle.close, (int, float)): + self._last_close = float(candle.close) + else: + self._last_close = None def _analyze_tick_size(self) -> None: """ @@ -959,7 +963,7 @@ def load_from_csv(self, path: str | Path, # Combine date and time ts_str = f"{row[date_idx]} {row[time_idx]}" else: - ts_str = row[timestamp_idx] + ts_str = str(row[timestamp_idx]) if timestamp_idx is not None and timestamp_idx < len(row) else "" # Convert timestamp try: @@ -1009,6 +1013,177 @@ def load_from_csv(self, path: str | Path, except (ValueError, IndexError) as e: raise ValueError(f"Invalid data in row: {e}") + def load_from_txt(self, path: str | Path, + timestamp_format: str | None = None, + timestamp_column: str | None = None, + date_column: str | None = None, + time_column: str | None = None, + tz: str | None = None) -> None: + """ + Load OHLCV data from TXT file with auto-detected delimiter. + Supports tab, semicolon, and pipe delimited files. + + :param path: Path to TXT file + :param timestamp_format: Optional datetime fmt for parsing + :param timestamp_column: Column name for timestamp (default tries: timestamp, time, date) + :param date_column: When timestamp is split into date+time columns, date column name + :param time_column: When timestamp is split into date+time columns, time column name + :param tz: Timezone name (e.g. 'UTC', 'Europe/London', '+0100') for timestamp conversion + :raises ValueError: If delimiter cannot be detected or file format is invalid + """ + # Parse timezone + timezone = None + if tz: + if tz.startswith(('+', '-')): + # Handle UTC offset fmt (e.g. +0100, -0500) + sign = 1 if tz.startswith('+') else -1 + hours = int(tz[1:3]) + minutes = int(tz[3:]) if len(tz) > 3 else 0 + timezone = dt_timezone(sign * timedelta(hours=hours, minutes=minutes)) + else: + # Handle named timezone (e.g. UTC, Europe/London) + try: + timezone = ZoneInfo(tz) + except Exception as e: + raise ValueError(f"Invalid timezone {tz}: {e}") + + # Auto-detect delimiter + with open(path, 'r') as f: + first_line = f.readline().strip() + if not first_line: + raise ValueError("File is empty or first line is blank") + + # Check for common delimiters in order of preference + delimiters = ['\t', ';', '|'] + delimiter_counts = {} + + for delim in delimiters: + count = first_line.count(delim) + if count > 0: + delimiter_counts[delim] = count + + if not delimiter_counts: + raise ValueError("No supported delimiter found (tab, semicolon, or pipe)") + + # Use delimiter with highest count + delimiter = max(delimiter_counts, key=lambda x: delimiter_counts[x]) + + # Read TXT file with detected/specified delimiter + with open(path, 'r') as f: + reader = csv.reader(f, delimiter=delimiter) + try: + headers = [h.lower().strip() for h in next(reader)] # Case insensitive + except StopIteration: + raise ValueError("File has no headers") + + if not headers: + raise ValueError("Header row is empty") + + # Find timestamp column + timestamp_idx = None + date_idx = None + time_idx = None + + if date_column and time_column: + try: + date_idx = headers.index(date_column.lower()) + time_idx = headers.index(time_column.lower()) + except ValueError: + raise ValueError(f"Date/time columns not found: {date_column}/{time_column}") + else: + timestamp_col = timestamp_column.lower() if timestamp_column else None + if timestamp_col: + try: + timestamp_idx = headers.index(timestamp_col) + except ValueError: + raise ValueError(f"Timestamp column not found: {timestamp_col}") + else: + # Try common names + for col in ['timestamp', 'time', 'date']: + try: + timestamp_idx = headers.index(col) + break + except ValueError: + continue + + if timestamp_idx is None: + raise ValueError("Timestamp column not found!") + + # Find OHLCV columns + try: + o_idx = headers.index('open') + h_idx = headers.index('high') + l_idx = headers.index('low') + c_idx = headers.index('close') + v_idx = headers.index('volume') + except ValueError as e: + raise ValueError(f"Missing required column: {str(e)}") + + # Process data rows + row_count = 0 + for row in reader: + row_count += 1 + if not row or len(row) != len(headers): + raise ValueError(f"Row {row_count} has incorrect number of columns") + + # Strip whitespace from all fields + row = [field.strip() for field in row] + + # Handle timestamp + try: + if date_idx is not None and time_idx is not None: + # Combine date and time + ts_str = f"{row[date_idx]} {row[time_idx]}" + else: + ts_str = str(row[timestamp_idx]) if timestamp_idx is not None and timestamp_idx < len(row) else "" + + # Convert timestamp + if ts_str.isdigit(): + timestamp = int(ts_str) + else: + if timestamp_format: + dt = datetime.strptime(ts_str, timestamp_format) + else: + # Try common formats + for fmt in [ + '%Y-%m-%d %H:%M:%S%z', # 2024-01-08 19:00:00+0000 + '%Y-%m-%d %H:%M:%S%Z', # 2024-01-08 19:00:00UTC + '%Y-%m-%dT%H:%M:%S%z', # 2024-01-08T19:00:00+0000 + '%Y-%m-%d %H:%M:%S', + '%Y/%m/%d %H:%M:%S', + '%d.%m.%Y %H:%M:%S', + '%Y-%m-%dT%H:%M:%S', + '%Y-%m-%d %H:%M', + '%Y%m%d %H:%M:%S' + ]: + try: + dt = datetime.strptime(ts_str, fmt) + break + except ValueError: + continue + else: + raise ValueError(f"Could not parse timestamp: {ts_str}") + + # Set timezone if specified and convert to timestamp + if timezone: + dt = dt.replace(tzinfo=timezone) + timestamp = int(dt.timestamp()) + except Exception as e: + raise ValueError(f"Failed to parse timestamp '{ts_str}' in row {row_count}: {e}") + + # Write OHLCV data + try: + self.write(OHLCV( + timestamp, + float(row[o_idx]), + float(row[h_idx]), + float(row[l_idx]), + float(row[c_idx]), + float(row[v_idx]) + )) + except (ValueError, IndexError) as e: + raise ValueError(f"Invalid OHLCV data in row {row_count}: {e}") + def load_from_json(self, path: str | Path, timestamp_format: str | None = None, timestamp_field: str | None = None, From b4fe7fbb9333932f0c16824b732f58807ee3d923 Mon Sep 17 00:00:00 2001 From: Mahadi Hassan Date: Tue, 19 Aug 2025 02:06:47 +0600 Subject: [PATCH 27/31] Remove redundant type checks and streamline last close handling --- src/pynecore/core/ohlcv_file.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/pynecore/core/ohlcv_file.py b/src/pynecore/core/ohlcv_file.py index 8d9b4cc..7e02998 100644 --- a/src/pynecore/core/ohlcv_file.py +++ b/src/pynecore/core/ohlcv_file.py @@ -350,24 +350,20 @@ def _collect_price_data(self, candle: OHLCV) -> None: Collect price data for tick size analysis during writing. """ # Collect price changes - if self._last_close is not None and isinstance(candle.close, (int, float)): + if self._last_close is not None: change = abs(candle.close - self._last_close) if change > 0 and len(self._price_changes) < 1000: # Limit to 1000 samples self._price_changes.append(change) # Collect decimal places for price in [candle.open, candle.high, candle.low, candle.close]: - if isinstance(price, (int, float)) and price != int(price): # Has decimal component + if price != int(price): # Has decimal component price_str = f"{price:.15f}".rstrip('0').rstrip('.') if '.' in price_str: decimals = len(price_str.split('.')[1]) self._price_decimals.add(decimals) - # Store last close, handling both float and NA[float] types - if isinstance(candle.close, (int, float)): - self._last_close = float(candle.close) - else: - self._last_close = None + self._last_close = candle.close def _analyze_tick_size(self) -> None: """ @@ -963,7 +959,7 @@ def load_from_csv(self, path: str | Path, # Combine date and time ts_str = f"{row[date_idx]} {row[time_idx]}" else: - ts_str = str(row[timestamp_idx]) if timestamp_idx is not None and timestamp_idx < len(row) else "" + ts_str = row[timestamp_idx] # Convert timestamp try: From f997518a147f7266315ee520741fb80a4d51d0d7 Mon Sep 17 00:00:00 2001 From: Mahadi Hassan Date: Tue, 19 Aug 2025 14:37:15 +0600 Subject: [PATCH 28/31] refactor(ohlcv_file): replace csv module with custom parser for txt files The csv module was replaced with a custom parser implementation to have better control over the parsing process and handle edge cases more reliably. This change improves error handling and supports more flexible input formats while maintaining the same functionality. --- src/pynecore/core/ohlcv_file.py | 300 ++++--- .../t00_pynecore/data/test_001_ohlcv_file.py | 776 +++++++++++++++++- 2 files changed, 941 insertions(+), 135 deletions(-) diff --git a/src/pynecore/core/ohlcv_file.py b/src/pynecore/core/ohlcv_file.py index 7e02998..9283208 100644 --- a/src/pynecore/core/ohlcv_file.py +++ b/src/pynecore/core/ohlcv_file.py @@ -646,7 +646,7 @@ def _collect_existing_trading_hours(self) -> None: # Unpack the record timestamp, open_val, high, low, close, volume = \ struct.unpack('Ifffff', cast(Buffer, data)) - + # Only collect if volume > 0 (real trading) if volume > 0: dt = datetime.fromtimestamp(timestamp, tz=None) @@ -1016,8 +1016,7 @@ def load_from_txt(self, path: str | Path, time_column: str | None = None, tz: str | None = None) -> None: """ - Load OHLCV data from TXT file with auto-detected delimiter. - Supports tab, semicolon, and pipe delimited files. + Load OHLCV data from TXT file using only builtin modules. :param path: Path to TXT file :param timestamp_format: Optional datetime fmt for parsing @@ -1025,7 +1024,6 @@ def load_from_txt(self, path: str | Path, :param date_column: When timestamp is split into date+time columns, date column name :param time_column: When timestamp is split into date+time columns, time column name :param tz: Timezone name (e.g. 'UTC', 'Europe/London', '+0100') for timestamp conversion - :raises ValueError: If delimiter cannot be detected or file format is invalid """ # Parse timezone timezone = None @@ -1048,137 +1046,229 @@ def load_from_txt(self, path: str | Path, first_line = f.readline().strip() if not first_line: raise ValueError("File is empty or first line is blank") - + # Check for common delimiters in order of preference delimiters = ['\t', ';', '|'] delimiter_counts = {} - + for delim in delimiters: count = first_line.count(delim) if count > 0: delimiter_counts[delim] = count - + if not delimiter_counts: raise ValueError("No supported delimiter found (tab, semicolon, or pipe)") - + # Use delimiter with highest count delimiter = max(delimiter_counts, key=lambda x: delimiter_counts[x]) - # Read TXT file with detected/specified delimiter + # Read TXT file with manual parsing for better control with open(path, 'r') as f: - reader = csv.reader(f, delimiter=delimiter) - try: - headers = [h.lower().strip() for h in next(reader)] # Case insensitive - except StopIteration: - raise ValueError("File has no headers") + lines = f.readlines() - if not headers: - raise ValueError("Header row is empty") + if not lines: + raise ValueError("File is empty") - # Find timestamp column - timestamp_idx = None - date_idx = None - time_idx = None + # Parse header line + header_line = lines[0].strip() + if not header_line: + raise ValueError("Header row is empty") - if date_column and time_column: + headers = self._parse_txt_line(header_line, delimiter) + headers = [h.lower().strip() for h in headers] # Case insensitive + + if not headers: + raise ValueError("No headers found") + + # Find timestamp column + timestamp_idx = None + date_idx = None + time_idx = None + + if date_column and time_column: + try: + date_idx = headers.index(date_column.lower()) + time_idx = headers.index(time_column.lower()) + except ValueError: + raise ValueError(f"Date/time columns not found: {date_column}/{time_column}") + else: + timestamp_col = timestamp_column.lower() if timestamp_column else None + if timestamp_col: try: - date_idx = headers.index(date_column.lower()) - time_idx = headers.index(time_column.lower()) + timestamp_idx = headers.index(timestamp_col) except ValueError: - raise ValueError(f"Date/time columns not found: {date_column}/{time_column}") + raise ValueError(f"Timestamp column not found: {timestamp_col}") else: - timestamp_col = timestamp_column.lower() if timestamp_column else None - if timestamp_col: + # Try common names + for col in ['timestamp', 'time', 'date']: try: - timestamp_idx = headers.index(timestamp_col) + timestamp_idx = headers.index(col) + break except ValueError: - raise ValueError(f"Timestamp column not found: {timestamp_col}") - else: - # Try common names - for col in ['timestamp', 'time', 'date']: - try: - timestamp_idx = headers.index(col) - break - except ValueError: - continue + continue - if timestamp_idx is None: - raise ValueError("Timestamp column not found!") + if timestamp_idx is None: + raise ValueError("Timestamp column not found!") - # Find OHLCV columns + # Find OHLCV columns + try: + o_idx = headers.index('open') + h_idx = headers.index('high') + l_idx = headers.index('low') + c_idx = headers.index('close') + v_idx = headers.index('volume') + except ValueError as e: + raise ValueError(f"Missing required column: {str(e)}") + + # Process data rows + for line in lines[1:]: # Skip header + line = line.strip() + if not line: # Skip empty lines + continue + + row = self._parse_txt_line(line, delimiter) + + if len(row) != len(headers): + raise ValueError(f"Row has {len(row)} columns, expected {len(headers)}") + + # Strip whitespace from all fields + row = [field.strip() for field in row] + + # Handle timestamp try: - o_idx = headers.index('open') - h_idx = headers.index('high') - l_idx = headers.index('low') - c_idx = headers.index('close') - v_idx = headers.index('volume') - except ValueError as e: - raise ValueError(f"Missing required column: {str(e)}") + if date_idx is not None and time_idx is not None: + # Combine date and time + ts_str = f"{row[date_idx]} {row[time_idx]}" + else: + ts_str = str(row[timestamp_idx]) if timestamp_idx is not None and timestamp_idx < len(row) else "" - # Process data rows - row_count = 0 - for row in reader: - row_count += 1 - if not row or len(row) != len(headers): - raise ValueError(f"Row {row_count} has incorrect number of columns") - - # Strip whitespace from all fields - row = [field.strip() for field in row] - - # Handle timestamp - try: - if date_idx is not None and time_idx is not None: - # Combine date and time - ts_str = f"{row[date_idx]} {row[time_idx]}" + # Convert timestamp + if ts_str.isdigit(): + timestamp = int(ts_str) + else: + dt = None + if timestamp_format: + dt = datetime.strptime(ts_str, timestamp_format) else: - ts_str = str(row[timestamp_idx]) if timestamp_idx is not None and timestamp_idx < len(row) else "" + # Try common formats + for fmt in [ + '%Y-%m-%d %H:%M:%S%z', # 2025-01-08 19:00:00+0000 + '%Y-%m-%d %H:%M:%S%Z', # 2025-01-08 19:00:00UTC + '%Y-%m-%dT%H:%M:%S%z', # 2025-01-08T19:00:00+0000 + '%Y-%m-%d %H:%M:%S', + '%Y/%m/%d %H:%M:%S', + '%d.%m.%Y %H:%M:%S', + '%Y-%m-%dT%H:%M:%S', + '%Y-%m-%d %H:%M', + '%Y%m%d %H:%M:%S' + ]: + try: + dt = datetime.strptime(ts_str, fmt) + break + except ValueError: + continue - # Convert timestamp - if ts_str.isdigit(): - timestamp = int(ts_str) + if dt is None: + raise ValueError(f"Could not parse timestamp: {ts_str}") + + # Set timezone if specified and convert to timestamp + if timezone and dt is not None: + dt = dt.replace(tzinfo=timezone) + timestamp = int(dt.timestamp()) + except Exception as e: + raise ValueError(f"Failed to parse timestamp '{ts_str}': {e}") + + # Write OHLCV data + try: + self.write(OHLCV( + timestamp, + float(row[o_idx]), + float(row[h_idx]), + float(row[l_idx]), + float(row[c_idx]), + float(row[v_idx]) + )) + except (ValueError, IndexError) as e: + raise ValueError(f"Invalid data in row: {e}") + + @staticmethod + def _parse_txt_line(line: str, delimiter: str) -> list[str]: + """ + Parse a single TXT line with proper handling of quoted fields and escape characters. + + :param line: Line to parse + :param delimiter: Delimiter character + :return: List of parsed fields + :raises ValueError: If line format is invalid + """ + if not line: + return [] + + fields = [] + current_field = "" + in_quotes = False + quote_char = None + i = 0 + + while i < len(line): + char = line[i] + + # Handle escape characters + if char == '\\' and i + 1 < len(line): + next_char = line[i + 1] + if next_char in ['"', "'", '\\', 'n', 't', 'r']: + if next_char == 'n': + current_field += '\n' + elif next_char == 't': + current_field += '\t' + elif next_char == 'r': + current_field += '\r' else: - if timestamp_format: - dt = datetime.strptime(ts_str, timestamp_format) - else: - # Try common formats - for fmt in [ - '%Y-%m-%d %H:%M:%S%z', # 2024-01-08 19:00:00+0000 - '%Y-%m-%d %H:%M:%S%Z', # 2024-01-08 19:00:00UTC - '%Y-%m-%dT%H:%M:%S%z', # 2024-01-08T19:00:00+0000 - '%Y-%m-%d %H:%M:%S', - '%Y/%m/%d %H:%M:%S', - '%d.%m.%Y %H:%M:%S', - '%Y-%m-%dT%H:%M:%S', - '%Y-%m-%d %H:%M', - '%Y%m%d %H:%M:%S' - ]: - try: - dt = datetime.strptime(ts_str, fmt) - break - except ValueError: - continue - else: - raise ValueError(f"Could not parse timestamp: {ts_str}") + current_field += next_char + i += 2 + continue + else: + current_field += char + i += 1 + continue - # Set timezone if specified and convert to timestamp - if timezone: - dt = dt.replace(tzinfo=timezone) - timestamp = int(dt.timestamp()) - except Exception as e: - raise ValueError(f"Failed to parse timestamp '{ts_str}' in row {row_count}: {e}") + # Handle quotes + if char in ['"', "'"] and not in_quotes: + in_quotes = True + quote_char = char + i += 1 + continue + elif char == quote_char and in_quotes: + # Check for escaped quote (double quote) + if i + 1 < len(line) and line[i + 1] == quote_char: + current_field += char + i += 2 + continue + else: + in_quotes = False + quote_char = None + i += 1 + continue - # Write OHLCV data - try: - self.write(OHLCV( - timestamp, - float(row[o_idx]), - float(row[h_idx]), - float(row[l_idx]), - float(row[c_idx]), - float(row[v_idx]) - )) - except (ValueError, IndexError) as e: - raise ValueError(f"Invalid OHLCV data in row {row_count}: {e}") + # Handle delimiter + if char == delimiter and not in_quotes: + fields.append(current_field) + current_field = "" + i += 1 + continue + + # Regular character + current_field += char + i += 1 + + # Add the last field + fields.append(current_field) + + # Validate that quotes are properly closed + if in_quotes: + raise ValueError(f"Unclosed quote in line: {line[:50]}...") + + return fields def load_from_json(self, path: str | Path, timestamp_format: str | None = None, diff --git a/tests/t00_pynecore/data/test_001_ohlcv_file.py b/tests/t00_pynecore/data/test_001_ohlcv_file.py index 2105599..323a389 100644 --- a/tests/t00_pynecore/data/test_001_ohlcv_file.py +++ b/tests/t00_pynecore/data/test_001_ohlcv_file.py @@ -129,8 +129,8 @@ def __test_ohlcv_seek_operations__(tmp_path): with OHLCVWriter(file_path) as writer: for i in range(10): timestamp = 1609459200 + (i * 60) # 1-minute interval - writer.write(OHLCV(timestamp=timestamp, open=100.0+i, high=110.0 + - i, low=90.0+i, close=105.0+i, volume=1000.0+i)) + writer.write(OHLCV(timestamp=timestamp, open=100.0 + i, high=110.0 + i, low=90.0 + i, close=105.0 + i, + volume=1000.0 + i)) # Test seeking to a specific position and direct write # Note: we use low-level file operations to bypass timestamp checks @@ -138,17 +138,20 @@ def __test_ohlcv_seek_operations__(tmp_path): writer.seek(5) # Seek to 6th record # Use direct byte writing to avoid chronological checks + # noinspection PyProtectedMember assert writer._file is not None data = struct.pack( 'Ifffff', 1609459500, # Timestamp - same as the original at position 5 - 200.0, # Open - 210.0, # High - 190.0, # Low - 205.0, # Close - 2000.0 # Volume + 200.0, # Open + 210.0, # High + 190.0, # Low + 205.0, # Close + 2000.0 # Volume ) + # noinspection PyProtectedMember writer._file.write(data) + # noinspection PyProtectedMember writer._file.flush() # Verify seek operation @@ -168,8 +171,8 @@ def __test_ohlcv_truncate__(tmp_path): with OHLCVWriter(file_path) as writer: for i in range(10): timestamp = 1609459200 + (i * 60) - writer.write(OHLCV(timestamp=timestamp, open=100.0+i, high=110.0 + - i, low=90.0+i, close=105.0+i, volume=1000.0+i)) + writer.write(OHLCV(timestamp=timestamp, open=100.0 + i, high=110.0 + i, low=90.0 + i, close=105.0 + i, + volume=1000.0 + i)) # Truncate the file with OHLCVWriter(file_path) as writer: @@ -261,8 +264,8 @@ def __test_ohlcv_reader_from_to__(tmp_path): with OHLCVWriter(file_path) as writer: for i in range(10): timestamp = 1609459200 + (i * 60) - writer.write(OHLCV(timestamp=timestamp, open=100.0+i, high=110.0 + - i, low=90.0+i, close=105.0+i, volume=1000.0+i)) + writer.write(OHLCV(timestamp=timestamp, open=100.0 + i, high=110.0 + i, low=90.0 + i, close=105.0 + i, + volume=1000.0 + i)) # Read specific range with OHLCVReader(file_path) as reader: @@ -419,16 +422,16 @@ def __test_opening_hours_detection_intraday__(tmp_path): """Test opening hours detection for intraday timeframes""" from datetime import datetime file_path = tmp_path / "test_opening_hours_intraday.ohlcv" - + with OHLCVWriter(file_path) as writer: # Simulate stock market data: Monday-Friday 9:30-16:00 # Start from a Monday 9:30 AM EST (2024-01-08 09:30:00) base_timestamp = int(datetime(2024, 1, 8, 9, 30).timestamp()) - + # Write data for multiple days with 1-minute intervals for day in range(5): # Monday to Friday day_offset = day * 86400 # Seconds in a day - + # Trading hours: 9:30 AM to 4:00 PM (6.5 hours = 390 minutes) for minute in range(390): timestamp = base_timestamp + day_offset + (minute * 60) @@ -441,15 +444,15 @@ def __test_opening_hours_detection_intraday__(tmp_path): close=price + 0.1, volume=1000.0 + minute )) - + # Check opening hours detection with OHLCVWriter(file_path) as writer: opening_hours = writer.analyzed_opening_hours - + # Should detect business hours pattern assert opening_hours is not None, "Opening hours should be detected" assert len(opening_hours) > 0, "Should have detected some opening hours" - + # Check that we have Monday-Friday entries days_with_hours = {interval.day for interval in opening_hours} assert 1 in days_with_hours # Monday @@ -462,12 +465,12 @@ def __test_opening_hours_detection_crypto__(tmp_path): """Test opening hours detection for crypto (24/7) markets""" from datetime import datetime file_path = tmp_path / "test_opening_hours_crypto.ohlcv" - + with OHLCVWriter(file_path) as writer: # Simulate crypto data: 24/7 trading # Start from a Monday 00:00 UTC (2024-01-08 00:00:00) base_timestamp = int(datetime(2024, 1, 8, 0, 0).timestamp()) - + # Write data for a full week with 5-minute intervals for hour in range(168): # 7 days * 24 hours for five_min in range(12): # 12 five-minute intervals per hour @@ -481,15 +484,15 @@ def __test_opening_hours_detection_crypto__(tmp_path): close=price + 10, volume=100.0 + five_min )) - + # Check opening hours detection with OHLCVWriter(file_path) as writer: opening_hours = writer.analyzed_opening_hours - + # Should detect 24/7 pattern assert opening_hours is not None, "Opening hours should be detected" assert len(opening_hours) == 7, "Should have all 7 days for 24/7 trading" - + # Check that all days are 00:00-23:59 for interval in opening_hours: assert interval.start.hour == 0 and interval.start.minute == 0 @@ -500,12 +503,12 @@ def __test_opening_hours_detection_daily__(tmp_path): """Test opening hours detection for daily timeframes""" from datetime import datetime file_path = tmp_path / "test_opening_hours_daily.ohlcv" - + with OHLCVWriter(file_path) as writer: # Simulate daily stock data: Monday-Friday only # Start from a Monday (2024-01-08) base_timestamp = int(datetime(2024, 1, 8, 16, 0).timestamp()) # Daily close at 4 PM - + # Write data for 3 weeks (15 business days) for week in range(3): for day in range(5): # Monday to Friday only @@ -519,15 +522,15 @@ def __test_opening_hours_detection_daily__(tmp_path): close=price + 1, volume=1000000.0 )) - + # Check opening hours detection with OHLCVWriter(file_path) as writer: opening_hours = writer.analyzed_opening_hours - + # Should detect weekday-only pattern from daily data assert opening_hours is not None, "Opening hours should be detected" assert len(opening_hours) == 5, "Should have Monday-Friday for daily stock data" - + # Check that we only have weekdays (1-5) days = {interval.day for interval in opening_hours} assert days == {1, 2, 3, 4, 5}, "Should only have Monday-Friday" @@ -536,7 +539,7 @@ def __test_opening_hours_detection_daily__(tmp_path): def __test_opening_hours_insufficient_data__(tmp_path): """Test opening hours detection with insufficient data""" file_path = tmp_path / "test_opening_hours_insufficient.ohlcv" - + with OHLCVWriter(file_path) as writer: # Write only a few data points (less than required minimum) base_timestamp = 1609459200 @@ -549,10 +552,723 @@ def __test_opening_hours_insufficient_data__(tmp_path): close=100.5, volume=1000.0 )) - + # Check opening hours detection with OHLCVWriter(file_path) as writer: opening_hours = writer.analyzed_opening_hours - + # Should return None for insufficient data assert opening_hours is None, "Should return None for insufficient data" + + +def __test_ohlcv_txt_conversion_tab_delimited__(tmp_path): + """Test TXT conversion with tab-delimited format""" + ohlcv_path = tmp_path / "test_txt_tab.ohlcv" + txt_path = tmp_path / "test_input_tab.txt" + + # Create tab-delimited test file + with open(txt_path, 'w') as f: + f.write("timestamp\topen\thigh\tlow\tclose\tvolume\n") + f.write("1609459200\t100.0\t110.0\t90.0\t105.0\t1000.0\n") + f.write("1609459260\t105.0\t115.0\t95.0\t110.0\t1200.0\n") + f.write("1609459320\t110.0\t120.0\t100.0\t115.0\t1400.0\n") + + # Convert TXT to OHLCV + with OHLCVWriter(ohlcv_path) as writer: + writer.load_from_txt(txt_path) + + # Verify converted data + with OHLCVReader(ohlcv_path) as reader: + candles = list(reader) + assert len(candles) == 3 + assert candles[0].timestamp == 1609459200 + assert candles[0].close == 105.0 + assert candles[0].volume == 1000.0 + assert candles[2].timestamp == 1609459320 + assert candles[2].close == 115.0 + + +def __test_ohlcv_txt_conversion_semicolon_delimited__(tmp_path): + """Test TXT conversion with semicolon-delimited format""" + ohlcv_path = tmp_path / "test_txt_semicolon.ohlcv" + txt_path = tmp_path / "test_input_semicolon.txt" + + # Create semicolon-delimited test file + with open(txt_path, 'w') as f: + f.write("timestamp;open;high;low;close;volume\n") + f.write("1609459200;100.0;110.0;90.0;105.0;1000.0\n") + f.write("1609459260;105.0;115.0;95.0;110.0;1200.0\n") + + # Convert TXT to OHLCV + with OHLCVWriter(ohlcv_path) as writer: + writer.load_from_txt(txt_path) + + # Verify converted data + with OHLCVReader(ohlcv_path) as reader: + candles = list(reader) + assert len(candles) == 2 + assert candles[0].timestamp == 1609459200 + assert candles[0].close == 105.0 + assert candles[1].timestamp == 1609459260 + assert candles[1].close == 110.0 + + +def __test_ohlcv_txt_conversion_pipe_delimited__(tmp_path): + """Test TXT conversion with pipe-delimited format""" + ohlcv_path = tmp_path / "test_txt_pipe.ohlcv" + txt_path = tmp_path / "test_input_pipe.txt" + + # Create pipe-delimited test file + with open(txt_path, 'w') as f: + f.write("timestamp|open|high|low|close|volume\n") + f.write("1609459200|100.0|110.0|90.0|105.0|1000.0\n") + f.write("1609459260|105.0|115.0|95.0|110.0|1200.0\n") + + # Convert TXT to OHLCV + with OHLCVWriter(ohlcv_path) as writer: + writer.load_from_txt(txt_path) + + # Verify converted data + with OHLCVReader(ohlcv_path) as reader: + candles = list(reader) + assert len(candles) == 2 + assert candles[0].timestamp == 1609459200 + assert candles[0].close == 105.0 + + +def __test_ohlcv_txt_conversion_quoted_fields__(tmp_path): + """Test TXT conversion with quoted fields""" + ohlcv_path = tmp_path / "test_txt_quoted.ohlcv" + txt_path = tmp_path / "test_input_quoted.txt" + + # Create test file with quoted fields + with open(txt_path, 'w') as f: + f.write('timestamp\topen\thigh\tlow\tclose\tvolume\n') + f.write('1609459200\t"100.0"\t"110.0"\t"90.0"\t"105.0"\t"1000.0"\n') + f.write("1609459260\t'105.0'\t'115.0'\t'95.0'\t'110.0'\t'1200.0'\n") + + # Convert TXT to OHLCV + with OHLCVWriter(ohlcv_path) as writer: + writer.load_from_txt(txt_path) + + # Verify converted data + with OHLCVReader(ohlcv_path) as reader: + candles = list(reader) + assert len(candles) == 2 + assert candles[0].close == 105.0 + assert candles[1].close == 110.0 + + +def __test_ohlcv_txt_conversion_with_timezone__(tmp_path): + """Test TXT conversion with timezone handling""" + ohlcv_path = tmp_path / "test_txt_tz.ohlcv" + txt_path = tmp_path / "test_input_tz.txt" + + # Create test file with datetime strings + with open(txt_path, 'w') as f: + f.write("time\topen\thigh\tlow\tclose\tvolume\n") + f.write("2025-01-01 12:00:00\t100.0\t110.0\t90.0\t105.0\t1000.0\n") + f.write("2025-01-01 12:01:00\t105.0\t115.0\t95.0\t110.0\t1200.0\n") + + # Convert TXT to OHLCV with UTC timezone + with OHLCVWriter(ohlcv_path) as writer: + writer.load_from_txt(txt_path, tz="UTC") + + # Verify converted data + with OHLCVReader(ohlcv_path) as reader: + candles = list(reader) + assert len(candles) == 2 + assert candles[0].close == 105.0 + assert candles[1].close == 110.0 + + +def __test_ohlcv_txt_conversion_date_time_columns__(tmp_path): + """Test TXT conversion with separate date and time columns""" + ohlcv_path = tmp_path / "test_txt_date_time.ohlcv" + txt_path = tmp_path / "test_input_date_time.txt" + + # Create test file with separate date/time columns + with open(txt_path, 'w') as f: + f.write("date\ttime\topen\thigh\tlow\tclose\tvolume\n") + f.write("2025-01-01\t12:00:00\t100.0\t110.0\t90.0\t105.0\t1000.0\n") + f.write("2025-01-01\t12:01:00\t105.0\t115.0\t95.0\t110.0\t1200.0\n") + + # Convert TXT to OHLCV with date/time columns + with OHLCVWriter(ohlcv_path) as writer: + writer.load_from_txt(txt_path, date_column="date", time_column="time") + + # Verify converted data + with OHLCVReader(ohlcv_path) as reader: + candles = list(reader) + assert len(candles) == 2 + assert candles[0].close == 105.0 + + +def __test_ohlcv_txt_conversion_custom_timestamp_format__(tmp_path): + """Test TXT conversion with custom timestamp format""" + ohlcv_path = tmp_path / "test_txt_custom_fmt.ohlcv" + txt_path = tmp_path / "test_input_custom_fmt.txt" + + # Create test file with custom timestamp format + with open(txt_path, 'w') as f: + f.write("timestamp\topen\thigh\tlow\tclose\tvolume\n") + f.write("01.01.2025 12:00:00\t100.0\t110.0\t90.0\t105.0\t1000.0\n") + f.write("01.01.2025 12:01:00\t105.0\t115.0\t95.0\t110.0\t1200.0\n") + + # Convert TXT to OHLCV with custom timestamp format + with OHLCVWriter(ohlcv_path) as writer: + writer.load_from_txt(txt_path, timestamp_format="%d.%m.%Y %H:%M:%S") + + # Verify converted data + with OHLCVReader(ohlcv_path) as reader: + candles = list(reader) + assert len(candles) == 2 + assert candles[0].close == 105.0 + + +def __test_ohlcv_txt_conversion_whitespace_handling__(tmp_path): + """Test TXT conversion with extra whitespace""" + ohlcv_path = tmp_path / "test_txt_whitespace.ohlcv" + txt_path = tmp_path / "test_input_whitespace.txt" + + # Create test file with extra whitespace + with open(txt_path, 'w') as f: + f.write(" timestamp \t open \t high \t low \t close \t volume \n") + f.write(" 1609459200 \t 100.0 \t 110.0 \t 90.0 \t 105.0 \t 1000.0 \n") + f.write(" 1609459260 \t 105.0 \t 115.0 \t 95.0 \t 110.0 \t 1200.0 \n") + + # Convert TXT to OHLCV + with OHLCVWriter(ohlcv_path) as writer: + writer.load_from_txt(txt_path) + + # Verify converted data + with OHLCVReader(ohlcv_path) as reader: + candles = list(reader) + assert len(candles) == 2 + assert candles[0].close == 105.0 + + +def __test_ohlcv_txt_conversion_empty_lines__(tmp_path): + """Test TXT conversion with empty lines""" + ohlcv_path = tmp_path / "test_txt_empty_lines.ohlcv" + txt_path = tmp_path / "test_input_empty_lines.txt" + + # Create test file with empty lines + with open(txt_path, 'w') as f: + f.write("timestamp\topen\thigh\tlow\tclose\tvolume\n") + f.write("\n") # Empty line + f.write("1609459200\t100.0\t110.0\t90.0\t105.0\t1000.0\n") + f.write("\n") # Another empty line + f.write("1609459260\t105.0\t115.0\t95.0\t110.0\t1200.0\n") + f.write("\n") # Final empty line + + # Convert TXT to OHLCV + with OHLCVWriter(ohlcv_path) as writer: + writer.load_from_txt(txt_path) + + # Verify converted data + with OHLCVReader(ohlcv_path) as reader: + candles = list(reader) + assert len(candles) == 2 + assert candles[0].close == 105.0 + + +def __test_ohlcv_txt_conversion_error_cases__(tmp_path): + """Test TXT conversion error handling""" + ohlcv_path = tmp_path / "test_txt_errors.ohlcv" + + # Test empty file + empty_txt = tmp_path / "empty.txt" + with open(empty_txt, 'w') as f: + f.write("") + + with pytest.raises(ValueError, match="File is empty"): + with OHLCVWriter(ohlcv_path) as writer: + writer.load_from_txt(empty_txt) + + # Test file with no delimiter + no_delim_txt = tmp_path / "no_delim.txt" + with open(no_delim_txt, 'w') as f: + f.write("timestamp open high low close volume\n") # Space delimited (not supported) + f.write("1609459200 100.0 110.0 90.0 105.0 1000.0\n") + + with pytest.raises(ValueError, match="No supported delimiter found"): + with OHLCVWriter(ohlcv_path) as writer: + writer.load_from_txt(no_delim_txt) + + # Test file with missing required columns + missing_cols_txt = tmp_path / "missing_cols.txt" + with open(missing_cols_txt, 'w') as f: + f.write("timestamp\topen\thigh\n") # Missing low, close, volume + f.write("1609459200\t100.0\t110.0\n") + + with pytest.raises(ValueError, match="Missing required column"): + with OHLCVWriter(ohlcv_path) as writer: + writer.load_from_txt(missing_cols_txt) + + # Test file with mismatched column count + mismatch_cols_txt = tmp_path / "mismatch_cols.txt" + with open(mismatch_cols_txt, 'w') as f: + f.write("timestamp\topen\thigh\tlow\tclose\tvolume\n") + f.write("1609459200\t100.0\t110.0\t90.0\t105.0\n") # Missing volume + + with pytest.raises(ValueError, match="Row has 5 columns, expected 6"): + with OHLCVWriter(ohlcv_path) as writer: + writer.load_from_txt(mismatch_cols_txt) + + +def __test_ohlcv_txt_conversion_escape_characters__(tmp_path): + """Test TXT conversion with escape characters""" + ohlcv_path = tmp_path / "test_txt_escape.ohlcv" + txt_path = tmp_path / "test_input_escape.txt" + + # Create test file with escape characters (though not commonly used in OHLCV data) + with open(txt_path, 'w') as f: + f.write("timestamp\topen\thigh\tlow\tclose\tvolume\n") + f.write("1609459200\t100.0\t110.0\t90.0\t105.0\t1000.0\n") + f.write("1609459260\t105.0\t115.0\t95.0\t110.0\t1200.0\n") + + # Convert TXT to OHLCV + with OHLCVWriter(ohlcv_path) as writer: + writer.load_from_txt(txt_path) + + # Verify converted data + with OHLCVReader(ohlcv_path) as reader: + candles = list(reader) + assert len(candles) == 2 + assert candles[0].close == 105.0 + + +def __test_ohlcv_txt_mixed_quote_types__(tmp_path): + """Test TXT conversion with mixed quote types""" + ohlcv_path = tmp_path / "test_txt_mixed_quotes.ohlcv" + txt_path = tmp_path / "test_input_mixed_quotes.txt" + + # Create test file with mixed quote types + with open(txt_path, 'w') as f: + f.write("timestamp\topen\thigh\tlow\tclose\tvolume\n") + f.write('1609459200\t"100.0"\t110.0\t"90.0"\t105.0\t"1000.0"\n') + f.write("1609459260\t'105.0'\t115.0\t'95.0'\t110.0\t'1200.0'\n") + + # Convert TXT to OHLCV + with OHLCVWriter(ohlcv_path) as writer: + writer.load_from_txt(txt_path) + + # Verify converted data + with OHLCVReader(ohlcv_path) as reader: + candles = list(reader) + assert len(candles) == 2 + assert candles[0].close == 105.0 + assert candles[1].close == 110.0 + + +def __test_ohlcv_csv_conversion_with_timezone__(tmp_path): + """Test CSV conversion with timezone handling""" + ohlcv_path = tmp_path / "test_csv_tz.ohlcv" + csv_path = tmp_path / "test_input_tz.csv" + + # Create CSV file with datetime strings + with open(csv_path, 'w') as f: + f.write("time,open,high,low,close,volume\n") + f.write("2025-01-01 12:00:00,100.0,110.0,90.0,105.0,1000.0\n") + f.write("2025-01-01 12:01:00,105.0,115.0,95.0,110.0,1200.0\n") + + # Convert CSV to OHLCV with UTC timezone + with OHLCVWriter(ohlcv_path) as writer: + writer.load_from_csv(csv_path, tz="UTC") + + # Verify converted data + with OHLCVReader(ohlcv_path) as reader: + candles = list(reader) + assert len(candles) == 2 + assert candles[0].close == 105.0 + assert candles[1].close == 110.0 + + +def __test_ohlcv_csv_conversion_date_time_columns__(tmp_path): + """Test CSV conversion with separate date and time columns""" + ohlcv_path = tmp_path / "test_csv_date_time.ohlcv" + csv_path = tmp_path / "test_input_date_time.csv" + + # Create CSV file with separate date/time columns + with open(csv_path, 'w') as f: + f.write("date,time,open,high,low,close,volume\n") + f.write("2025-01-01,12:00:00,100.0,110.0,90.0,105.0,1000.0\n") + f.write("2025-01-01,12:01:00,105.0,115.0,95.0,110.0,1200.0\n") + + # Convert CSV to OHLCV with date/time columns + with OHLCVWriter(ohlcv_path) as writer: + writer.load_from_csv(csv_path, date_column="date", time_column="time") + + # Verify converted data + with OHLCVReader(ohlcv_path) as reader: + candles = list(reader) + assert len(candles) == 2 + assert candles[0].close == 105.0 + + +def __test_ohlcv_csv_conversion_custom_timestamp_format__(tmp_path): + """Test CSV conversion with custom timestamp format""" + ohlcv_path = tmp_path / "test_csv_custom_fmt.ohlcv" + csv_path = tmp_path / "test_input_custom_fmt.csv" + + # Create CSV file with custom timestamp format + with open(csv_path, 'w') as f: + f.write("timestamp,open,high,low,close,volume\n") + f.write("01.01.2025 12:00:00,100.0,110.0,90.0,105.0,1000.0\n") + f.write("01.01.2025 12:01:00,105.0,115.0,95.0,110.0,1200.0\n") + + # Convert CSV to OHLCV with custom timestamp format + with OHLCVWriter(ohlcv_path) as writer: + writer.load_from_csv(csv_path, timestamp_format="%d.%m.%Y %H:%M:%S") + + # Verify converted data + with OHLCVReader(ohlcv_path) as reader: + candles = list(reader) + assert len(candles) == 2 + assert candles[0].close == 105.0 + + +def __test_ohlcv_csv_conversion_custom_timestamp_column__(tmp_path): + """Test CSV conversion with custom timestamp column name""" + ohlcv_path = tmp_path / "test_csv_custom_ts.ohlcv" + csv_path = tmp_path / "test_input_custom_ts.csv" + + # Create CSV file with custom timestamp column name + with open(csv_path, 'w') as f: + f.write("datetime,open,high,low,close,volume\n") + f.write("2025-01-01 12:00:00,100.0,110.0,90.0,105.0,1000.0\n") + f.write("2025-01-01 12:01:00,105.0,115.0,95.0,110.0,1200.0\n") + + # Convert CSV to OHLCV with custom timestamp column + with OHLCVWriter(ohlcv_path) as writer: + writer.load_from_csv(csv_path, timestamp_column="datetime") + + # Verify converted data + with OHLCVReader(ohlcv_path) as reader: + candles = list(reader) + assert len(candles) == 2 + assert candles[0].close == 105.0 + + +def __test_ohlcv_csv_conversion_quoted_fields__(tmp_path): + """Test CSV conversion with quoted fields""" + ohlcv_path = tmp_path / "test_csv_quoted.ohlcv" + csv_path = tmp_path / "test_input_quoted.csv" + + # Create CSV file with quoted fields + with open(csv_path, 'w') as f: + f.write("timestamp,open,high,low,close,volume\n") + f.write('1609459200,"100.0","110.0","90.0","105.0","1000.0"\n') + f.write('1609459260,"105.0","115.0","95.0","110.0","1200.0"\n') + + # Convert CSV to OHLCV + with OHLCVWriter(ohlcv_path) as writer: + writer.load_from_csv(csv_path) + + # Verify converted data + with OHLCVReader(ohlcv_path) as reader: + candles = list(reader) + assert len(candles) == 2 + assert candles[0].close == 105.0 + assert candles[1].close == 110.0 + + +def __test_ohlcv_csv_conversion_error_cases__(tmp_path): + """Test CSV conversion error handling""" + ohlcv_path = tmp_path / "test_csv_errors.ohlcv" + + # Test file with missing required columns + missing_cols_csv = tmp_path / "missing_cols.csv" + with open(missing_cols_csv, 'w') as f: + f.write("timestamp,open,high\n") # Missing low, close, volume + f.write("1609459200,100.0,110.0\n") + + with pytest.raises(ValueError, match="Missing required column"): + with OHLCVWriter(ohlcv_path) as writer: + writer.load_from_csv(missing_cols_csv) + + # Test file with invalid timestamp + invalid_ts_csv = tmp_path / "invalid_ts.csv" + with open(invalid_ts_csv, 'w') as f: + f.write("timestamp,open,high,low,close,volume\n") + f.write("invalid-timestamp,100.0,110.0,90.0,105.0,1000.0\n") + + with pytest.raises(ValueError, match="Failed to parse timestamp"): + with OHLCVWriter(ohlcv_path) as writer: + writer.load_from_csv(invalid_ts_csv) + + +def __test_ohlcv_csv_conversion_case_insensitive__(tmp_path): + """Test CSV conversion with case insensitive headers""" + ohlcv_path = tmp_path / "test_csv_case.ohlcv" + csv_path = tmp_path / "test_input_case.csv" + + # Create CSV file with mixed case headers + with open(csv_path, 'w') as f: + f.write("TIMESTAMP,Open,HIGH,low,Close,VOLUME\n") + f.write("1609459200,100.0,110.0,90.0,105.0,1000.0\n") + f.write("1609459260,105.0,115.0,95.0,110.0,1200.0\n") + + # Convert CSV to OHLCV + with OHLCVWriter(ohlcv_path) as writer: + writer.load_from_csv(csv_path) + + # Verify converted data + with OHLCVReader(ohlcv_path) as reader: + candles = list(reader) + assert len(candles) == 2 + assert candles[0].close == 105.0 + + +def __test_ohlcv_json_conversion_with_timezone__(tmp_path): + """Test JSON conversion with timezone handling""" + ohlcv_path = tmp_path / "test_json_tz.ohlcv" + json_path = tmp_path / "test_input_tz.json" + + # Create JSON file with datetime strings + import json + data = [ + {"time": "2025-01-01 12:00:00", "open": 100.0, "high": 110.0, "low": 90.0, "close": 105.0, "volume": 1000.0}, + {"time": "2025-01-01 12:01:00", "open": 105.0, "high": 115.0, "low": 95.0, "close": 110.0, "volume": 1200.0} + ] + with open(json_path, 'w') as f: + json.dump(data, f) + + # Convert JSON to OHLCV with UTC timezone + with OHLCVWriter(ohlcv_path) as writer: + writer.load_from_json(json_path, timestamp_field="time", tz="UTC") + + # Verify converted data + with OHLCVReader(ohlcv_path) as reader: + candles = list(reader) + assert len(candles) == 2 + assert candles[0].close == 105.0 + assert candles[1].close == 110.0 + + +def __test_ohlcv_json_conversion_date_time_fields__(tmp_path): + """Test JSON conversion with separate date and time fields""" + ohlcv_path = tmp_path / "test_json_date_time.ohlcv" + json_path = tmp_path / "test_input_date_time.json" + + # Create JSON file with separate date/time fields + import json + data = [ + {"date": "2025-01-01", "time": "12:00:00", "open": 100.0, "high": 110.0, "low": 90.0, "close": 105.0, + "volume": 1000.0}, + {"date": "2025-01-01", "time": "12:01:00", "open": 105.0, "high": 115.0, "low": 95.0, "close": 110.0, + "volume": 1200.0} + ] + with open(json_path, 'w') as f: + json.dump(data, f) + + # Convert JSON to OHLCV with date/time fields + with OHLCVWriter(ohlcv_path) as writer: + writer.load_from_json(json_path, date_field="date", time_field="time") + + # Verify converted data + with OHLCVReader(ohlcv_path) as reader: + candles = list(reader) + assert len(candles) == 2 + assert candles[0].close == 105.0 + + +def __test_ohlcv_json_conversion_custom_timestamp_format__(tmp_path): + """Test JSON conversion with custom timestamp format""" + ohlcv_path = tmp_path / "test_json_custom_fmt.ohlcv" + json_path = tmp_path / "test_input_custom_fmt.json" + + # Create JSON file with custom timestamp format + import json + data = [ + {"timestamp": "01.01.2025 12:00:00", "open": 100.0, "high": 110.0, "low": 90.0, "close": 105.0, + "volume": 1000.0}, + {"timestamp": "01.01.2025 12:01:00", "open": 105.0, "high": 115.0, "low": 95.0, "close": 110.0, + "volume": 1200.0} + ] + with open(json_path, 'w') as f: + json.dump(data, f) + + # Convert JSON to OHLCV with custom timestamp format + with OHLCVWriter(ohlcv_path) as writer: + writer.load_from_json(json_path, timestamp_format="%d.%m.%Y %H:%M:%S") + + # Verify converted data + with OHLCVReader(ohlcv_path) as reader: + candles = list(reader) + assert len(candles) == 2 + assert candles[0].close == 105.0 + + +def __test_ohlcv_json_conversion_field_mapping__(tmp_path): + """Test JSON conversion with field mapping""" + ohlcv_path = tmp_path / "test_json_mapping.ohlcv" + json_path = tmp_path / "test_input_mapping.json" + + # Create JSON file with custom field names + import json + data = [ + {"t": 1609459200, "o": 100.0, "h": 110.0, "l": 90.0, "c": 105.0, "vol": 1000.0}, + {"t": 1609459260, "o": 105.0, "h": 115.0, "l": 95.0, "c": 110.0, "vol": 1200.0} + ] + with open(json_path, 'w') as f: + json.dump(data, f) + + # Convert JSON to OHLCV with field mapping + mapping = {"timestamp": "t", "open": "o", "high": "h", "low": "l", "close": "c", "volume": "vol"} + with OHLCVWriter(ohlcv_path) as writer: + writer.load_from_json(json_path, mapping=mapping) + + # Verify converted data + with OHLCVReader(ohlcv_path) as reader: + candles = list(reader) + assert len(candles) == 2 + assert candles[0].close == 105.0 + assert candles[1].close == 110.0 + + +def __test_ohlcv_json_conversion_wrapped_data__(tmp_path): + """Test JSON conversion with wrapped data arrays""" + ohlcv_path = tmp_path / "test_json_wrapped.ohlcv" + json_path = tmp_path / "test_input_wrapped.json" + + # Create JSON file with wrapped data array (common API format) + import json + wrapped_data = { + "data": [ + {"timestamp": 1609459200, "open": 100.0, "high": 110.0, "low": 90.0, "close": 105.0, "volume": 1000.0}, + {"timestamp": 1609459260, "open": 105.0, "high": 115.0, "low": 95.0, "close": 110.0, "volume": 1200.0} + ] + } + with open(json_path, 'w') as f: + json.dump(wrapped_data, f) + + # Convert JSON to OHLCV + with OHLCVWriter(ohlcv_path) as writer: + writer.load_from_json(json_path) + + # Verify converted data + with OHLCVReader(ohlcv_path) as reader: + candles = list(reader) + assert len(candles) == 2 + assert candles[0].close == 105.0 + + +def __test_ohlcv_json_conversion_millisecond_timestamps__(tmp_path): + """Test JSON conversion with millisecond timestamps""" + ohlcv_path = tmp_path / "test_json_ms.ohlcv" + json_path = tmp_path / "test_input_ms.json" + + # Create JSON file with millisecond timestamps + import json + data = [ + {"timestamp": 1609459200000, "open": 100.0, "high": 110.0, "low": 90.0, "close": 105.0, "volume": 1000.0}, + {"timestamp": 1609459260000, "open": 105.0, "high": 115.0, "low": 95.0, "close": 110.0, "volume": 1200.0} + ] + with open(json_path, 'w') as f: + json.dump(data, f) + + # Convert JSON to OHLCV (should auto-detect and convert milliseconds) + with OHLCVWriter(ohlcv_path) as writer: + writer.load_from_json(json_path) + + # Verify converted data + with OHLCVReader(ohlcv_path) as reader: + candles = list(reader) + assert len(candles) == 2 + assert candles[0].timestamp == 1609459200 # Should be converted from ms to s + assert candles[1].timestamp == 1609459260 + assert candles[0].close == 105.0 + + +def __test_ohlcv_json_conversion_auto_field_detection__(tmp_path): + """Test JSON conversion with automatic field detection""" + ohlcv_path = tmp_path / "test_json_auto.ohlcv" + json_path = tmp_path / "test_input_auto.json" + + # Create JSON file with 't' timestamp field (should auto-detect) + import json + data = [ + {"t": 1609459200, "open": 100.0, "high": 110.0, "low": 90.0, "close": 105.0, "volume": 1000.0}, + {"t": 1609459260, "open": 105.0, "high": 115.0, "low": 95.0, "close": 110.0, "volume": 1200.0} + ] + with open(json_path, 'w') as f: + json.dump(data, f) + + # Convert JSON to OHLCV (should auto-detect 't' field) + with OHLCVWriter(ohlcv_path) as writer: + writer.load_from_json(json_path) + + # Verify converted data + with OHLCVReader(ohlcv_path) as reader: + candles = list(reader) + assert len(candles) == 2 + assert candles[0].close == 105.0 + + +def __test_ohlcv_json_conversion_error_cases__(tmp_path): + """Test JSON conversion error handling""" + ohlcv_path = tmp_path / "test_json_errors.ohlcv" + + # Test file with missing required fields + missing_fields_json = tmp_path / "missing_fields.json" + import json + data = [ + {"timestamp": 1609459200, "open": 100.0, "high": 110.0} # Missing low, close, volume + ] + with open(missing_fields_json, 'w') as f: + json.dump(data, f) + + with pytest.raises(ValueError, match="Missing field in record"): + with OHLCVWriter(ohlcv_path) as writer: + writer.load_from_json(missing_fields_json) + + # Test file with no timestamp field + no_timestamp_json = tmp_path / "no_timestamp.json" + data = [ + {"open": 100.0, "high": 110.0, "low": 90.0, "close": 105.0, "volume": 1000.0} # No timestamp + ] + with open(no_timestamp_json, 'w') as f: + json.dump(data, f) + + with pytest.raises(ValueError, match="Could not find timestamp field"): + with OHLCVWriter(ohlcv_path) as writer: + writer.load_from_json(no_timestamp_json) + + # Test file with no OHLCV data array + no_data_json = tmp_path / "no_data.json" + data = {"metadata": "some info", "status": "ok"} # No data array + with open(no_data_json, 'w') as f: + json.dump(data, f) + + with pytest.raises(ValueError, match="Could not find OHLCV data array"): + with OHLCVWriter(ohlcv_path) as writer: + writer.load_from_json(no_data_json) + + +def __test_ohlcv_json_conversion_alternative_wrappers__(tmp_path): + """Test JSON conversion with alternative wrapper keys""" + ohlcv_path = tmp_path / "test_json_alt_wrap.ohlcv" + json_path = tmp_path / "test_input_alt_wrap.json" + + # Create JSON file with 'candles' wrapper (alternative to 'data') + import json + wrapped_data = { + "candles": [ + {"timestamp": 1609459200, "open": 100.0, "high": 110.0, "low": 90.0, "close": 105.0, "volume": 1000.0}, + {"timestamp": 1609459260, "open": 105.0, "high": 115.0, "low": 95.0, "close": 110.0, "volume": 1200.0} + ] + } + with open(json_path, 'w') as f: + json.dump(wrapped_data, f) + + # Convert JSON to OHLCV + with OHLCVWriter(ohlcv_path) as writer: + writer.load_from_json(json_path) + + # Verify converted data + with OHLCVReader(ohlcv_path) as reader: + candles = list(reader) + assert len(candles) == 2 + assert candles[0].close == 105.0 From 3378cf8ca524ef35197343c8b5b9207ceee77f60 Mon Sep 17 00:00:00 2001 From: Adam Wallner Date: Tue, 19 Aug 2025 17:48:11 +0200 Subject: [PATCH 29/31] fix(pytest.ini): update ignored glob pattern for test discovery --- pytest.ini | 2 +- src/pynecore/core/ohlcv_file.py | 13 ++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/pytest.ini b/pytest.ini index 8642e04..4665b7e 100644 --- a/pytest.ini +++ b/pytest.ini @@ -6,4 +6,4 @@ log_cli_level = DEBUG log_cli_format = %(asctime)s %(levelname)6s %(module_func_line)30s - %(message)s log_cli_date_format = %Y-%m-%d %H:%M:%S -addopts = --import-mode=importlib -rs -x --spec --ignore-glob="**/data/**" +addopts = --import-mode=importlib -rs -x --spec --ignore-glob="**/data/*modified.py" diff --git a/src/pynecore/core/ohlcv_file.py b/src/pynecore/core/ohlcv_file.py index 9283208..851b412 100644 --- a/src/pynecore/core/ohlcv_file.py +++ b/src/pynecore/core/ohlcv_file.py @@ -1135,13 +1135,12 @@ def load_from_txt(self, path: str | Path, row = [field.strip() for field in row] # Handle timestamp + if date_idx is not None and time_idx is not None: + # Combine date and time + ts_str = f"{row[date_idx]} {row[time_idx]}" + else: + ts_str = str(row[timestamp_idx]) if timestamp_idx is not None and timestamp_idx < len(row) else "" try: - if date_idx is not None and time_idx is not None: - # Combine date and time - ts_str = f"{row[date_idx]} {row[time_idx]}" - else: - ts_str = str(row[timestamp_idx]) if timestamp_idx is not None and timestamp_idx < len(row) else "" - # Convert timestamp if ts_str.isdigit(): timestamp = int(ts_str) @@ -1195,7 +1194,7 @@ def load_from_txt(self, path: str | Path, def _parse_txt_line(line: str, delimiter: str) -> list[str]: """ Parse a single TXT line with proper handling of quoted fields and escape characters. - + :param line: Line to parse :param delimiter: Delimiter character :return: List of parsed fields From 67bb9e5da0d985d9d000052de4d14e186b442d67 Mon Sep 17 00:00:00 2001 From: Adam Wallner Date: Tue, 19 Aug 2025 18:14:10 +0200 Subject: [PATCH 30/31] refactor(ohlcv_file): extract timestamp parsing logic to eliminate code duplication - Created standalone _parse_timestamp() function to centralize timestamp parsing - Eliminates ~80 lines of duplicated code across load_from_csv, load_from_txt, and load_from_json - Also renamed format_float to _format_float and _combine_tick_estimates_no_gcd to _combine_tick_estimates for consistency --- src/pynecore/core/ohlcv_file.py | 191 +++++++++++++------------------- 1 file changed, 74 insertions(+), 117 deletions(-) diff --git a/src/pynecore/core/ohlcv_file.py b/src/pynecore/core/ohlcv_file.py index 851b412..cfe7692 100644 --- a/src/pynecore/core/ohlcv_file.py +++ b/src/pynecore/core/ohlcv_file.py @@ -36,11 +36,63 @@ __all__ = ['OHLCVWriter', 'OHLCVReader'] -def format_float(value: float) -> str: +def _format_float(value: float) -> str: """Format float with max 8 decimal places, removing trailing zeros""" return f"{value:.8g}" +def _parse_timestamp(ts_str: str, timestamp_format: str | None = None, timezone=None) -> int: + """ + Parse timestamp string to Unix timestamp. + + :param ts_str: Timestamp string to parse + :param timestamp_format: Optional specific datetime format for parsing + :param timezone: Optional timezone to apply to the parsed datetime + :return: Unix timestamp as integer + :raises ValueError: If timestamp cannot be parsed + """ + # Handle numeric timestamps + if ts_str.isdigit(): + timestamp = int(ts_str) + # Handle millisecond timestamps (common in JSON APIs) + if timestamp > 253402300799: # 9999-12-31 23:59:59 + timestamp //= 1000 + return timestamp + + # Parse datetime string + dt = None + if timestamp_format: + dt = datetime.strptime(ts_str, timestamp_format) + else: + # Try common formats + for fmt in [ + '%Y-%m-%d %H:%M:%S%z', # 2024-01-08 19:00:00+0000 + '%Y-%m-%d %H:%M:%S%Z', # 2024-01-08 19:00:00UTC + '%Y-%m-%dT%H:%M:%S%z', # 2024-01-08T19:00:00+0000 + '%Y-%m-%d %H:%M:%S', + '%Y/%m/%d %H:%M:%S', + '%d.%m.%Y %H:%M:%S', + '%Y-%m-%dT%H:%M:%S', + '%Y-%m-%dT%H:%M:%SZ', # ISO with Z + '%Y-%m-%d %H:%M', + '%Y%m%d %H:%M:%S' + ]: + try: + dt = datetime.strptime(ts_str, fmt) + break + except ValueError: + continue + + if dt is None: + raise ValueError(f"Could not parse timestamp: {ts_str}") + + # Apply timezone if specified and convert to timestamp + if timezone and dt is not None: + dt = dt.replace(tzinfo=timezone) + + return int(dt.timestamp()) + + class OHLCVWriter: """ Binary OHLCV data writer using direct file operations @@ -396,7 +448,7 @@ def _analyze_tick_size(self) -> None: decimal_tick = self._calculate_decimal_tick() # Combine methods with weighted confidence (no GCD) - tick_size, confidence = self._combine_tick_estimates_no_gcd(freq_tick, decimal_tick) + tick_size, confidence = self._combine_tick_estimates(freq_tick, decimal_tick) # Calculate price scale and min move if tick_size > 0: @@ -583,8 +635,8 @@ def _calculate_decimal_tick(self) -> tuple[float, float]: return tick_size, 0.5 @staticmethod - def _combine_tick_estimates_no_gcd(freq: tuple[float, float], - decimal: tuple[float, float]) -> tuple[float, float]: + def _combine_tick_estimates(freq: tuple[float, float], + decimal: tuple[float, float]) -> tuple[float, float]: """ Combine tick size estimates from frequency and decimal methods only. Returns (tick_size, confidence) @@ -963,36 +1015,7 @@ def load_from_csv(self, path: str | Path, # Convert timestamp try: - if ts_str.isdigit(): - timestamp = int(ts_str) - else: - if timestamp_format: - dt = datetime.strptime(ts_str, timestamp_format) - else: - # Try common formats - for fmt in [ - '%Y-%m-%d %H:%M:%S%z', # 2024-01-08 19:00:00+0000 - '%Y-%m-%d %H:%M:%S%Z', # 2024-01-08 19:00:00UTC - '%Y-%m-%dT%H:%M:%S%z', # 2024-01-08T19:00:00+0000 - '%Y-%m-%d %H:%M:%S', - '%Y/%m/%d %H:%M:%S', - '%d.%m.%Y %H:%M:%S', - '%Y-%m-%dT%H:%M:%S', - '%Y-%m-%d %H:%M', - '%Y%m%d %H:%M:%S' - ]: - try: - dt = datetime.strptime(ts_str, fmt) - break - except ValueError: - continue - else: - raise ValueError(f"Could not parse timestamp: {ts_str}") - - # Set timezone if specified and convert to timestamp - if timezone: - dt = dt.replace(tzinfo=timezone) - timestamp = int(dt.timestamp()) + timestamp = _parse_timestamp(ts_str, timestamp_format, timezone) except Exception as e: raise ValueError(f"Failed to parse timestamp '{ts_str}': {e}") @@ -1142,38 +1165,7 @@ def load_from_txt(self, path: str | Path, ts_str = str(row[timestamp_idx]) if timestamp_idx is not None and timestamp_idx < len(row) else "" try: # Convert timestamp - if ts_str.isdigit(): - timestamp = int(ts_str) - else: - dt = None - if timestamp_format: - dt = datetime.strptime(ts_str, timestamp_format) - else: - # Try common formats - for fmt in [ - '%Y-%m-%d %H:%M:%S%z', # 2025-01-08 19:00:00+0000 - '%Y-%m-%d %H:%M:%S%Z', # 2025-01-08 19:00:00UTC - '%Y-%m-%dT%H:%M:%S%z', # 2025-01-08T19:00:00+0000 - '%Y-%m-%d %H:%M:%S', - '%Y/%m/%d %H:%M:%S', - '%d.%m.%Y %H:%M:%S', - '%Y-%m-%dT%H:%M:%S', - '%Y-%m-%d %H:%M', - '%Y%m%d %H:%M:%S' - ]: - try: - dt = datetime.strptime(ts_str, fmt) - break - except ValueError: - continue - - if dt is None: - raise ValueError(f"Could not parse timestamp: {ts_str}") - - # Set timezone if specified and convert to timestamp - if timezone and dt is not None: - dt = dt.replace(tzinfo=timezone) - timestamp = int(dt.timestamp()) + timestamp = _parse_timestamp(ts_str, timestamp_format, timezone) except Exception as e: raise ValueError(f"Failed to parse timestamp '{ts_str}': {e}") @@ -1356,42 +1348,7 @@ def load_from_json(self, path: str | Path, ts_str = str(record[field_map['timestamp']]) # Convert timestamp - if ts_str.isdigit(): - # Handle millisecond timestamps - ts = int(ts_str) - if ts > 253402300799: # 9999-12-31 23:59:59 - ts //= 1000 - timestamp = ts - else: - dt = None - # Parse datetime string - if timestamp_format: - dt = datetime.strptime(ts_str, timestamp_format) - else: - # Try common formats - for fmt in [ - '%Y-%m-%d %H:%M:%S%z', # 2024-01-08 19:00:00+0000 - '%Y-%m-%d %H:%M:%S%Z', # 2024-01-08 19:00:00UTC - '%Y-%m-%dT%H:%M:%S%z', # 2024-01-08T19:00:00+0000 - '%Y-%m-%d %H:%M:%S', - '%Y/%m/%d %H:%M:%S', - '%Y-%m-%dT%H:%M:%S', - '%Y-%m-%dT%H:%M:%SZ', - '%Y-%m-%d %H:%M', - '%Y%m%d %H:%M:%S' - ]: - try: - dt = datetime.strptime(ts_str, fmt) - break - except ValueError: - continue - else: - raise ValueError(f"Could not parse timestamp: {ts_str}") - - # Set timezone and convert to timestamp - if timezone: - dt = dt.replace(tzinfo=timezone) - timestamp = int(dt.timestamp()) + timestamp = _parse_timestamp(ts_str, timestamp_format, timezone) # Get OHLCV values try: @@ -1631,12 +1588,12 @@ def save_to_csv(self, path: str, as_datetime=False) -> None: if candle.volume == -1: continue if as_datetime: - f.write(f"{datetime.fromtimestamp(candle.timestamp, UTC)},{format_float(candle.open)}," - f"{format_float(candle.high)},{format_float(candle.low)},{format_float(candle.close)}," - f"{format_float(candle.volume)}\n") + f.write(f"{datetime.fromtimestamp(candle.timestamp, UTC)},{_format_float(candle.open)}," + f"{_format_float(candle.high)},{_format_float(candle.low)},{_format_float(candle.close)}," + f"{_format_float(candle.volume)}\n") else: - f.write(f"{candle.timestamp},{format_float(candle.open)},{format_float(candle.high)}," - f"{format_float(candle.low)},{format_float(candle.close)},{format_float(candle.volume)}\n") + f.write(f"{candle.timestamp},{_format_float(candle.open)},{_format_float(candle.high)}," + f"{_format_float(candle.low)},{_format_float(candle.close)},{_format_float(candle.volume)}\n") def save_to_json(self, path: str, as_datetime: bool = False) -> None: """ @@ -1666,20 +1623,20 @@ def save_to_json(self, path: str, as_datetime: bool = False) -> None: if as_datetime: item = { "time": datetime.fromtimestamp(candle.timestamp, UTC).isoformat(), - "open": format_float(candle.open), - "high": format_float(candle.high), - "low": format_float(candle.low), - "close": format_float(candle.close), - "volume": format_float(candle.volume) + "open": _format_float(candle.open), + "high": _format_float(candle.high), + "low": _format_float(candle.low), + "close": _format_float(candle.close), + "volume": _format_float(candle.volume) } else: item = { "timestamp": candle.timestamp, - "open": format_float(candle.open), - "high": format_float(candle.high), - "low": format_float(candle.low), - "close": format_float(candle.close), - "volume": format_float(candle.volume) + "open": _format_float(candle.open), + "high": _format_float(candle.high), + "low": _format_float(candle.low), + "close": _format_float(candle.close), + "volume": _format_float(candle.volume) } data.append(item) From 780ba9297bcfacbaefa63ba287f084eeb317ca99 Mon Sep 17 00:00:00 2001 From: Adam Wallner Date: Tue, 19 Aug 2025 18:53:13 +0200 Subject: [PATCH 31/31] feat: fix formatting of OHLCV CSV output for better readability --- src/pynecore/core/ohlcv_file.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pynecore/core/ohlcv_file.py b/src/pynecore/core/ohlcv_file.py index cfe7692..89e4a03 100644 --- a/src/pynecore/core/ohlcv_file.py +++ b/src/pynecore/core/ohlcv_file.py @@ -1593,7 +1593,8 @@ def save_to_csv(self, path: str, as_datetime=False) -> None: f"{_format_float(candle.volume)}\n") else: f.write(f"{candle.timestamp},{_format_float(candle.open)},{_format_float(candle.high)}," - f"{_format_float(candle.low)},{_format_float(candle.close)},{_format_float(candle.volume)}\n") + f"{_format_float(candle.low)},{_format_float(candle.close)}," + f"{_format_float(candle.volume)}\n") def save_to_json(self, path: str, as_datetime: bool = False) -> None: """