fix mixed mm inputs and rlhf-v

Former-commit-id: 7c248fac20bf85d57a91132ce7a793c7f84e9218

src/llamafactory/data/processors/supervised.py

@@ -21,6 +21,7 @@ from .processor_utils import greedy_knapsack, infer_seqlen
 
 
 if TYPE_CHECKING:
+    from PIL.Image import Image
     from transformers import PreTrainedTokenizer, ProcessorMixin
 
     from ...hparams import DataArguments
@@ -35,19 +36,18 @@ def _encode_supervised_example(
     response: Sequence[Dict[str, str]],
     system: Optional[str],
     tools: Optional[str],
+    images: Sequence["Image"],
     template: "Template",
     tokenizer: "PreTrainedTokenizer",
     processor: Optional["ProcessorMixin"],
     cutoff_len: int,
     train_on_prompt: bool,
     mask_history: bool,
-) -> Tuple[List[int], List[int]]:
-    messages = prompt + response
-    input_ids, labels = [], []
-    input_ids, labels = template.mm_plugin.process_token_ids(input_ids, labels, tokenizer, processor)
-
+) -> Tuple[List[int], List[int], Dict[str, Any]]:
+    messages = template.mm_plugin.process_messages(prompt + response, images, processor)
+    input_ids, labels = template.mm_plugin.process_token_ids([], [], images, tokenizer, processor)
     encoded_pairs = template.encode_multiturn(tokenizer, messages, system, tools)
-    total_length = 1 if template.efficient_eos else 0
+    total_length = len(input_ids) + (1 if template.efficient_eos else 0)
     if mask_history:
         encoded_pairs = encoded_pairs[::-1]  # high priority for last turns
 
@@ -83,7 +83,10 @@ def _encode_supervised_example(
         input_ids += [tokenizer.eos_token_id]
         labels += [tokenizer.eos_token_id]
 
-    return input_ids, labels
+    extra_inputs = template.mm_plugin.get_mm_inputs(
+        images=images, feature_seqlens={"token_type_ids": len(input_ids)}, processor=processor
+    )
+    return input_ids, labels, extra_inputs
 
 
 def preprocess_supervised_dataset(
@@ -101,12 +104,12 @@ def preprocess_supervised_dataset(
             logger.warning("Dropped invalid example: {}".format(examples["prompt"][i] + examples["response"][i]))
             continue
 
-        prompt = template.mm_plugin.process_messages(examples["prompt"][i], examples["images"][i], processor)
-        input_ids, labels = _encode_supervised_example(
-            prompt=prompt,
+        input_ids, labels, extra_inputs = _encode_supervised_example(
+            prompt=examples["prompt"][i],
             response=examples["response"][i],
             system=examples["system"][i],
             tools=examples["tools"][i],
+            images=examples["images"][i],
             template=template,
             tokenizer=tokenizer,
             processor=processor,
@@ -117,12 +120,8 @@ def preprocess_supervised_dataset(
         model_inputs["input_ids"].append(input_ids)
         model_inputs["attention_mask"].append([1] * len(input_ids))
         model_inputs["labels"].append(labels)
-        template.mm_plugin.process_model_inputs(
-            model_inputs=model_inputs,
-            images=examples["images"][i],
-            feature_seqlens={"token_type_ids": len(input_ids)},
-            processor=processor,
-        )
+        for key, value in extra_inputs.items():
+            model_inputs[key].append(value)
 
     return model_inputs
 
@@ -131,10 +130,15 @@ def preprocess_packed_supervised_dataset(
     examples: Dict[str, List[Any]],
     template: "Template",
     tokenizer: "PreTrainedTokenizer",
+    processor: Optional["ProcessorMixin"],
     data_args: "DataArguments",
 ) -> Dict[str, List[Any]]:
+    # TODO: use `position_ids` to achieve packing
     # build inputs with format `<bos> X1 Y1 <eos> <bos> X2 Y2 <eos>`
     # and labels with format `<ignore> ... <ignore> Y1 <eos> <ignore> ... <ignore> Y2 <eos>`
+    if processor is not None:
+        raise NotImplementedError("`packing` have not been implemented for multimodal datasets.")
+
     valid_num = 0
     batch_input_ids, batch_labels = [], []
     lengths = []
@@ -149,6 +153,7 @@ def preprocess_packed_supervised_dataset(
             response=examples["response"][i],
             system=examples["system"][i],
             tools=examples["tools"][i],
+            images=examples["images"][i],
             template=template,
             tokenizer=tokenizer,
             processor=None,