Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Introduce OVQuantizationConfig for nncf.quantize() parameters #638

Merged
merged 17 commits into from
Apr 15, 2024
Merged

Introduce OVQuantizationConfig for nncf.quantize() parameters #638

Show file tree
Hide file tree
Changes from 14 commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
22d2d6a
Introduce OVQuantizationConfig for nncf.quantize() parameters
nikita-savelyevv Mar 27, 2024
350bfa9
Ignored scope tweaks
nikita-savelyevv Mar 28, 2024
6861a69
Merge branch 'main' into introduce-ov-quantization-config
nikita-savelyevv Mar 28, 2024
b90fd42
Added **kwargs to quantization call. Added config serialization test.
nikita-savelyevv Apr 2, 2024
bfe982c
Ignored scope changes. Tests pass.
nikita-savelyevv Apr 3, 2024
e7d0d14
Added documentation
nikita-savelyevv Apr 3, 2024
431775e
Linters
nikita-savelyevv Apr 3, 2024
ff2df77
Merge branch 'main' into introduce-ov-quantization-config
nikita-savelyevv Apr 3, 2024
c8edf99
Linters
nikita-savelyevv Apr 3, 2024
77faf7f
Tweak ignored scope serialization
nikita-savelyevv Apr 3, 2024
123e227
Added deprecation errors, tweak docs
nikita-savelyevv Apr 4, 2024
20fd761
Addressed minor comments
nikita-savelyevv Apr 10, 2024
f7fa3a1
Make quantization config contain only serializable properties.
nikita-savelyevv Apr 11, 2024
0e79c09
Small tweaks
nikita-savelyevv Apr 12, 2024
13b2350
Address comments
nikita-savelyevv Apr 12, 2024
f314ba0
Fix ruff
nikita-savelyevv Apr 15, 2024
70ee0ef
Fix ruff 2
nikita-savelyevv Apr 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions optimum/intel/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,7 @@
"OVModelForVision2Seq",
"OVModelForSequenceClassification",
"OVModelForTokenClassification",
"OVQuantizationConfig",
"OVWeightQuantizationConfig",
"OVConfig",
]
Expand Down Expand Up @@ -243,6 +244,7 @@
OVModelForSpeechSeq2Seq,
OVModelForTokenClassification,
OVModelForVision2Seq,
OVQuantizationConfig,
OVWeightQuantizationConfig,
)

Expand Down
2 changes: 1 addition & 1 deletion optimum/intel/openvino/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@
from .trainer import OVTrainer


from .configuration import OVConfig, OVWeightQuantizationConfig
from .configuration import OVConfig, OVQuantizationConfig, OVWeightQuantizationConfig
from .modeling import (
OVModelForAudioClassification,
OVModelForAudioFrameClassification,
Expand Down
321 changes: 222 additions & 99 deletions optimum/intel/openvino/configuration.py
View file
Open in desktop

Large diffs are not rendered by default.

9 changes: 7 additions & 2 deletions optimum/intel/openvino/modeling_base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
from tempfile import TemporaryDirectory, gettempdir
from typing import Dict, Optional, Union

import nncf
nikita-savelyevv marked this conversation as resolved.
Show resolved Hide resolved
import openvino
from huggingface_hub import hf_hub_download
from openvino import Core, convert_model
Expand Down Expand Up @@ -100,7 +101,11 @@ def __init__(
self._openvino_config = OVConfig(quantization_config=quantization_config)

@staticmethod
def load_model(file_name: Union[str, Path], quantization_config: Union[OVWeightQuantizationConfig, Dict] = None):
def load_model(
file_name: Union[str, Path],
quantization_config: Union[OVWeightQuantizationConfig, Dict] = None,
calibration_dataset: Optional[nncf.Dataset] = None,
):
"""
Loads the model.

Expand Down Expand Up @@ -135,7 +140,7 @@ def fix_op_names_duplicates(model: openvino.runtime.Model):

from optimum.intel.openvino.quantization import _weight_only_quantization

model = _weight_only_quantization(model, quantization_config)
model = _weight_only_quantization(model, quantization_config, calibration_dataset=calibration_dataset)

return model

Expand Down
16 changes: 11 additions & 5 deletions optimum/intel/openvino/modeling_decoder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
from tempfile import TemporaryDirectory
from typing import Dict, Optional, Tuple, Union

import nncf
import numpy as np
import openvino
import torch
Expand Down Expand Up @@ -572,7 +573,8 @@ def _from_pretrained(
from_onnx: bool = False,
local_files_only: bool = False,
load_in_8bit: bool = False,
quantization_config: Union[OVWeightQuantizationConfig, Dict] = None,
quantization_config: Optional[Union[OVWeightQuantizationConfig, Dict]] = None,
calibration_dataset: Optional[nncf.Dataset] = None,
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

would prefer to not include a calibration_dataset argument and only support a subset of calibration dataset via the quantization_config when loading a model with the from_pretrained method (and leave the possibility to give any calibration_dataset when applying quantization with the OVQuantizer), what do you think ?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm. There's currently a scenario where custom dataset is provided to .from_pretrained()
method via config: https://github.com/huggingface/optimum-intel/blob/main/tests/openvino/test_quantization.py#L453

Since we decided that .dataset property of the config will now contain only string typed values, it'll look kind of hacky to keep it this way.

How about I remove explicit definition of calibration_dataset argument from .from_pretrained signature, but extract it from **kwargs there? This's still shady, but IMO it's better than passing it through the .dataset property of the config. What do you think?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think for custom dataset we should provide support through the OVQuantizer and not when loading the model with from_pretrained, will open a following PR to add this

**kwargs,
):
model_path = Path(model_id)
Expand All @@ -596,7 +598,11 @@ def _from_pretrained(
quantization_config = cls._prepare_weight_quantization_config(quantization_config, load_in_8bit)

load_in_4bit = quantization_config.bits == 4 if quantization_config else False
model = cls.load_model(model_cache_path, quantization_config=None if load_in_4bit else quantization_config)
model = cls.load_model(
model_cache_path,
quantization_config=None if load_in_4bit else quantization_config,
calibration_dataset=calibration_dataset,
)

model_type = config.model_type.replace("_", "-")
if model_type == "bloom":
Expand Down Expand Up @@ -632,7 +638,7 @@ def _from_pretrained(
f"For the given model, we recommend the following `quantization_config` : {default_config}"
)

if isinstance(quantization_config.dataset, str):
if calibration_dataset is None and isinstance(quantization_config.dataset, str):
tokenizer = quantization_config.tokenizer or AutoTokenizer.from_pretrained(model_id)

from optimum.gptq.data import get_dataset, prepare_dataset
Expand All @@ -644,9 +650,9 @@ def _from_pretrained(
dataset = get_dataset(quantization_config.dataset, tokenizer, seqlen=32, nsamples=nsamples)
dataset = prepare_dataset(dataset)
quantization_config = copy.deepcopy(quantization_config)
quantization_config.dataset = nncf.Dataset(dataset, lambda x: causal_model.prepare_inputs(**x))
calibration_dataset = nncf.Dataset(dataset, lambda x: causal_model.prepare_inputs(**x))

_weight_only_quantization(model, quantization_config)
_weight_only_quantization(model, quantization_config, calibration_dataset)

return causal_model

Expand Down
Loading
Loading