|
14 | 14 | import pytest
|
15 | 15 |
|
16 | 16 | from tests.common import TUNE_PATH
|
17 |
| -from tests.recipes.utils import llama2_test_config |
| 17 | +from tests.recipes.utils import llama2_test_config, write_hf_ckpt_config |
18 | 18 | from tests.test_utils import CKPT_MODEL_PATHS
|
19 | 19 |
|
20 | 20 |
|
@@ -126,6 +126,80 @@ def test_eval_recipe_errors_without_lm_eval(self, capsys, monkeypatch, tmpdir):
|
126 | 126 | in printed_err
|
127 | 127 | )
|
128 | 128 |
|
| 129 | + @pytest.mark.integration_test |
| 130 | + def test_eval_recipe_errors_with_quantization_hf_checkpointer( |
| 131 | + self, capsys, monkeypatch, tmpdir |
| 132 | + ): |
| 133 | + ckpt = "llama2_hf" |
| 134 | + ckpt_path = Path(CKPT_MODEL_PATHS[ckpt]) |
| 135 | + ckpt_dir = ckpt_path.parent |
| 136 | + |
| 137 | + # Config file needed for model conversion. |
| 138 | + write_hf_ckpt_config(ckpt_dir) |
| 139 | + |
| 140 | + cmd = f""" |
| 141 | + tune run eleuther_eval \ |
| 142 | + --config eleuther_evaluation \ |
| 143 | + output_dir={tmpdir} \ |
| 144 | + checkpointer=torchtune.training.FullModelHFCheckpointer \ |
| 145 | + checkpointer.checkpoint_dir='{ckpt_dir}' \ |
| 146 | + checkpointer.checkpoint_files=[{ckpt_path}]\ |
| 147 | + checkpointer.output_dir={tmpdir} \ |
| 148 | + checkpointer.model_type=LLAMA2 \ |
| 149 | + tokenizer.path=/tmp/test-artifacts/tokenizer.model \ |
| 150 | + tokenizer.prompt_template=null \ |
| 151 | + limit=1 \ |
| 152 | + dtype=fp32 \ |
| 153 | + device=cpu \ |
| 154 | + quantizer._component_=torchtune.training.quantization.Int8DynActInt4WeightQuantizer \ |
| 155 | + quantizer.groupsize=256 \ |
| 156 | + """.split() |
| 157 | + |
| 158 | + model_config = llama2_test_config() |
| 159 | + cmd = cmd + model_config |
| 160 | + |
| 161 | + monkeypatch.setattr(sys, "argv", cmd) |
| 162 | + with pytest.raises( |
| 163 | + ValueError, |
| 164 | + match="Quantization is only supported for models quantized and saved with the " |
| 165 | + "FullModelTorchTuneCheckpointer", |
| 166 | + ): |
| 167 | + runpy.run_path(TUNE_PATH, run_name="__main__") |
| 168 | + |
| 169 | + @pytest.mark.integration_test |
| 170 | + def test_eval_recipe_errors_with_qat_quantizer(self, capsys, monkeypatch, tmpdir): |
| 171 | + ckpt = "llama2_tune" |
| 172 | + ckpt_path = Path(CKPT_MODEL_PATHS[ckpt]) |
| 173 | + ckpt_dir = ckpt_path.parent |
| 174 | + |
| 175 | + cmd = f""" |
| 176 | + tune run eleuther_eval \ |
| 177 | + --config eleuther_evaluation \ |
| 178 | + output_dir={tmpdir} \ |
| 179 | + checkpointer=torchtune.training.FullModelTorchTuneCheckpointer \ |
| 180 | + checkpointer.checkpoint_dir='{ckpt_dir}' \ |
| 181 | + checkpointer.checkpoint_files=[{ckpt_path}]\ |
| 182 | + checkpointer.output_dir={tmpdir} \ |
| 183 | + checkpointer.model_type=LLAMA2 \ |
| 184 | + tokenizer.path=/tmp/test-artifacts/tokenizer.model \ |
| 185 | + tokenizer.prompt_template=null \ |
| 186 | + limit=1 \ |
| 187 | + dtype=fp32 \ |
| 188 | + device=cpu \ |
| 189 | + quantizer._component_=torchtune.training.quantization.Int8DynActInt4WeightQATQuantizer \ |
| 190 | + quantizer.groupsize=32\ |
| 191 | + """.split() |
| 192 | + |
| 193 | + model_config = llama2_test_config() |
| 194 | + cmd = cmd + model_config |
| 195 | + |
| 196 | + monkeypatch.setattr(sys, "argv", cmd) |
| 197 | + with pytest.raises( |
| 198 | + ValueError, |
| 199 | + match="QAT quantizers should only be used during quantization aware training", |
| 200 | + ): |
| 201 | + runpy.run_path(TUNE_PATH, run_name="__main__") |
| 202 | + |
129 | 203 | @pytest.mark.integration_test
|
130 | 204 | def test_eval_recipe_errors_with_generate_until_and_mc_tasks(
|
131 | 205 | self, caplog, capsys, monkeypatch, tmpdir
|
|
0 commit comments