pyproject.toml

[build-system]
requires = [
    "setuptools>=68.2.2",
    "wheel>=0.41.2",
]
build-backend = "setuptools.build_meta"

requires-python = ">= 3.11"

[project]
name = "recurrent-pretraining"
version = "0.1"
dependencies = [
  "numpy<2.0",
  "einops",
  "lightning==2.3.0.dev20240328",
  "jsonargparse[signatures]>=4.27.6",
  "requests>=2.31.0",
  "tensorboard>=2.14.0",
  "torchmetrics>=1.3.1",
  "lm-eval>=0.4.2",
  "wandb",
  "sentencepiece>=0.2.0",
  "tokenizers>=0.15.2",
  "safetensors>=0.4.3",
  "datasets>=2.18.0",
  "transformers>=4.38.0",
  "axonn@git+https://github.com/axonn-ai/axonn@c5ef65d3d4f329c20292a2d26681e8d9cbcccecf",
  "pandas",
  "plotly",
  "packaging",
  "ninja",
  "torchdata",
]

[project.optional-dependencies]
# toggling between different torch versions
torch221 = ["torch==2.2.1"]
torchgte240 = ["torch>=2.4.0"]

# only for testing
dev = [
  "pytest",
  "pytest-rerunfailures",
  "pytest-timeout",
  "transformers>=4.38.0",
  "einops",
  "protobuf",
  "docstring_parser",
  "lightning-cloud",
]

# only for data preproc
data = [
  "litdata>=0.2.2",
  "requests",
  "zstandard",
  "pandas",
  "pyarrow",
]

quant = [
  "bitsandbytes==0.42.0",
  "scipy",
]

[tool.setuptools.packages.find]
include = [
    "recpre",
    "recpre.*",
    "scripts"
]
exclude = []

[tool.setuptools.package-data]
recpre = [
    "LICENSE",
    "README.md",
]

[tool.ruff]
target-version = 'py311'
line-length = 120
fix = true # 😈


exclude = [
    ".git",
    "./.git",
    "/build",
    "notebooks",
    # "scripts",
    "__pycache__",
    "dl/*",
    "log/*",
    "*.pyi",
    "outputs"
]

[tool.ruff.lint]
select = ["E", "F", "B", "C4", "PLC", "G", "PIE", "SIM", "W", "B9"]
ignore = [
    "E203",
    "E305",
    "E402",
    "E501",
    "E721",
    "E741",
    "F405",
    # "F821",
    # "F841",
    "C408",
    "E302",
    "W291",
    "E303",
    "E275",
    "B007",
    "B008",
    "B017",
    "B019",
    "B020",
    "B023",
    "B024",
    "B026",
    "B028",
    "B904",
    "B905",
    "G101",
    "G201",
    "G202",
    # "SIM105",
    # # "SIM108",
    # "SIM110",
    # "SIM113",
    # # "SIM114",
    # "SIM115",
    # "SIM116",
    # "SIM117",
    # "SIM118",
    # "SIM102",
    "SIM103",
    # "SIM112",
    "B006",
    "G004",
    "E731",
    "B011",
    "PT015",
    "E266",
    "E741",
]

# [tool.ruff.lint.per-file-ignores]
# "__init__.py" = ["F401"]

[tool.pyright]
include = ["recpre", "train.py"]
exclude = ["**/__pycache__", "**/*ipynb*"]

pythonVersion = "3.11"
pythonPlatform = "Linux"

# ignores
reportPrivateImportUsage = false
# reportIncompatibleMethodOverride = false
typeCheckingMode = "basic"