Skip to content

Fix Python 3.14t compat and Marlin with Cuda 13.0 #1969

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 12 commits into from
Oct 3, 2025
Merged

Fix Python 3.14t compat and Marlin with Cuda 13.0 #1969

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
f58c719
fix eora import
Qubitium Oct 2, 2025
f262737
make calibration_sort "desc" sane default
Qubitium Oct 2, 2025
d3cb032
fix pypi project name
Qubitium Oct 2, 2025
7a6441d
fix marlin compile
Qubitium Oct 3, 2025
72c4777
dedup
Qubitium Oct 3, 2025
d10bab6
use PYTORCH_ALLOC_CONF
Qubitium Oct 3, 2025
8b8b3b2
use non-deprecated api for tf32 toggle
Qubitium Oct 3, 2025
43a4bb9
fix potential thread state issue
Qubitium Oct 3, 2025
143aa14
dill update
Qubitium Oct 3, 2025
580110c
better default
Qubitium Oct 3, 2025
b83637a
add alloc bench
Qubitium Oct 3, 2025
16a6169
fic compat with dill not ready for 3.14
Qubitium Oct 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/unit_tests.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ env:
CUDA_DEVICE_ORDER: PCI_BUS_ID
CUDA_VISIBLE_DEVICES: 0
TORCH_CUDA_ARCH_LIST: '8.6 8.9 9.0 12.0'
PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True'
PYTORCH_ALLOC_CONF: 'expandable_segments:True'
MAX_JOBS: 4
RUNNER: 10.0.13.31
XEON5: 10.0.14.248
Expand Down
13 changes: 8 additions & 5 deletions MANIFEST.in
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
global-include gptqmodel_ext/**/*.cuh
global-include gptqmodel_ext/**/*.h
global-include gptqmodel_ext/**/*.cpp
global-include gptqmodel_ext/**/*.cu
global-include gptqmodel_ext/**/*.py
recursive-include gptqmodel_ext/awq *.h *.cuh *.cu *.cpp
recursive-include gptqmodel_ext/exllama *.h *.cuh *.cu *.cpp
recursive-include gptqmodel_ext/exllamav2 *.h *.cuh *.cu *.cpp
recursive-include gptqmodel_ext/exllama_eora/eora *.h *.cuh *.cu *.cpp *.py
recursive-include gptqmodel_ext/marlin *.h *.cuh *.cu *.cpp
recursive-include gptqmodel_ext/qqq *.h *.cuh *.cu *.cpp
include gptqmodel_ext/marlin/generate_kernels.py
recursive-exclude gptqmodel_ext __pycache__ *.pyc
prune tests/
prune format/
2 changes: 1 addition & 1 deletion examples/benchmark/perplexity.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@


os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
os.environ["PYTORCH_ALLOC_CONF"] = "expandable_segments:True"

if __name__ == "__main__":
"""
Expand Down
2 changes: 1 addition & 1 deletion examples/eora/eora_generation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@


os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
os.environ["PYTORCH_ALLOC_CONF"] = "expandable_segments:True"
# -- end do not touch


Expand Down
2 changes: 1 addition & 1 deletion examples/eora/eora_load_and_inference.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@


os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
os.environ["PYTORCH_ALLOC_CONF"] = "expandable_segments:True"
# -- end do not touch

from gptqmodel import BACKEND, GPTQModel # noqa: E402
Expand Down
2 changes: 1 addition & 1 deletion examples/eora/evaluation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@


os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
os.environ["PYTORCH_ALLOC_CONF"] = "expandable_segments:True"
# -- end do not touch

from typing import Optional # noqa: E402
Expand Down
2 changes: 1 addition & 1 deletion examples/eora/post_quant_eora_generation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@


os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
os.environ["PYTORCH_ALLOC_CONF"] = "expandable_segments:True"
# -- end do not touch


Expand Down
2 changes: 1 addition & 1 deletion examples/inference/run_with_different_backends.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@


os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
os.environ["PYTORCH_ALLOC_CONF"] = "expandable_segments:True"

pretrained_model_id = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
quantized_model_id = "./TinyLlama/TinyLlama-1.1B-Chat-v1.0-4bit-128g"
Expand Down
2 changes: 1 addition & 1 deletion examples/quantization/basic_usage.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@


os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
os.environ["PYTORCH_ALLOC_CONF"] = "expandable_segments:True"

pretrained_model_id = "/monster/data/model/TinyLlama-1.1B-Chat-v1.0" # "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
quantized_model_id = "TinyLlama-1.1B-Chat-v1.0-4bit-128g"
Expand Down
1 change: 1 addition & 0 deletions gptqmodel/looper/eora_processor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
from ..utils.logger import setup_logger
from ..utils.model import move_to
from ..utils.torch import CPU, DEVICE_0, DEVICE_1, torch_streamCtx, torch_sync
from ..utils.torch import HAS_CUDA, tf32_disable_guard, torch_streamCtx, torch_sync

log = setup_logger()

Expand Down
6 changes: 3 additions & 3 deletions gptqmodel/models/auto.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@
# os.environ["PYTHON_GIL"] = '0'
# log.info("ENV: Auto disable GIL and use free-threading mode when applicable: Python 3.13t+. You must install the -t edition of Python.")

if not os.environ.get("PYTORCH_CUDA_ALLOC_CONF", None):
os.environ["PYTORCH_CUDA_ALLOC_CONF"] = 'expandable_segments:True'
log.info("ENV: Auto setting PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' for memory saving.")
if not os.environ.get("PYTORCH_ALLOC_CONF", None):
os.environ["PYTORCH_ALLOC_CONF"] = 'expandable_segments:True,max_split_size_mb:256,garbage_collection_threshold:0.7'
log.info("ENV: Auto setting PYTORCH_ALLOC_CONF='expandable_segments:True,max_split_size_mb:256,garbage_collection_threshold:0.7' for memory saving.")

if not os.environ.get("CUDA_DEVICE_ORDER", None):
os.environ["CUDA_DEVICE_ORDER"] = 'PCI_BUS_ID'
Expand Down
Loading