fix RLHF & max_length

swift/llm/rlhf.py

@@ -3,7 +3,7 @@
 
 from swift.trainers import TrainerFactory
 from swift.utils import get_logger, get_main, seed_everything
-from .sft import prepare_dataset, prepare_train_model_template, trainer_train
+from .sft import prepare_dataset, prepare_model_template_train, trainer_train
 from .utils import TEMPLATE_MAPPING, RLHFArguments
 
 logger = get_logger()
@@ -18,7 +18,7 @@ def llm_rlhf(args: RLHFArguments) -> Dict[str, Any]:
         logger.warning(f"Please check if args.template_type: '{args.template_type}' is correct.")
 
     msg = {}
-    model, ref_model, template, callbacks = prepare_train_model_template(args)
+    model, ref_model, template, callbacks = prepare_model_template_train(args)
     with TrainerFactory.patch_template(args, template):
         train_dataset, val_dataset = prepare_dataset(args, template, msg)
 

swift/llm/sft.py

@@ -115,7 +115,7 @@ def llm_sft_megatron(args: SftArguments) -> Dict[str, Any]:
     return {}
 
 
-def prepare_train_model_template(args, msg: Optional[Dict[str, Any]] = None):
+def prepare_model_template_train(args, msg: Optional[Dict[str, Any]] = None):
 
     if args.gpu_memory_fraction is not None:
         for device_id in range(torch.cuda.device_count()):
@@ -501,7 +501,7 @@ def llm_sft(args: SftArguments) -> Dict[str, Any]:
     if args.train_backend == 'megatron':
         return llm_sft_megatron(args)
     msg = {}
-    model, template, callbacks = prepare_train_model_template(args, msg)
+    model, template, callbacks = prepare_model_template_train(args, msg)
     train_dataset, val_dataset = prepare_dataset(args, template, msg)
     return trainer_train(args, model, template, train_dataset, val_dataset, callbacks=callbacks, msg=msg)
 

swift/llm/utils/argument.py

@@ -139,7 +139,7 @@ def handle_generation_config(self: Union['SftArguments', 'InferArguments']) -> N
         if self.temperature == 0:
             self.do_sample = False
         if self.do_sample is False and (isinstance(self, InferArguments) and self.infer_backend == 'pt'
-                                        and isinstance(self, SftArguments)):
+                                        or isinstance(self, SftArguments)):
             # fix warning
             self.temperature = 1.
             self.top_p = 1.

swift/llm/utils/template.py

@@ -3340,6 +3340,8 @@ def encode(self: Template,
             chosen_inputs, chosen_tokenizer_kwargs = template_encode(chosen_example)
             rejected_inputs, rejected_tokenizer_kwargs = template_encode(rejected_example)
 
+        if len(chosen_inputs) == 0 or len(rejected_inputs) == 0:
+            return {}, {}
         for suffix, res in zip(['inputs', 'tokenizer_kwargs'], [inputs, tokenizer_kwargs]):
             for prefix in ['chosen', 'rejected']:
                 data = locals()[f'{prefix}_{suffix}']