fastllm

YerongLi · Jul 11, 2023 · 2e4891c · 2e4891c
1 parent 5745e21
commit 2e4891c
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/cli_demo.py b/src/cli_demo.py
@@ -66,7 +66,7 @@ def main():
     global stop_stream
     model_args, finetuning_args, generating_args = prepare_infer_args()
     model, tokenizer = load_pretrained(model_args, finetuning_args)
-    model = llm.from_hf(model, tokenizer, dtype = "float16") # dtype支持float16, float32, float64, int32, int64, bool
+    model = llm.from_hf(model, tokenizer, dtype = "int4") # dtype支持float16, float32, float64, int32, int64, bool
 
     if torch.cuda.device_count() > 1:
         from accelerate import dispatch_model