Truncate sentences longer then the maximum size

C4AI · Jan 7, 2024 · bad0ef8 · bad0ef8
1 parent dcf0bbc
commit bad0ef8
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/triple_extractor_ptbr_pligabue/predicate_extraction/data_formatter.py b/triple_extractor_ptbr_pligabue/predicate_extraction/data_formatter.py
@@ -30,7 +30,7 @@ def split_on_predicate(self, sentence) -> list[str]:
         return trimmed_split
 
     def format_input(self, sentence: str) -> SentenceInput:
-        return bert.tokenizer.encode(sentence, padding="max_length", max_length=self.sentence_size)
+        return bert.tokenizer.encode(sentence, padding="max_length", truncation=True, max_length=self.sentence_size)
 
     def format_inputs(self, sentences: list[str]) -> SentenceInputs:
         return [self.format_input(sentence) for sentence in sentences]