forked from facebookresearch/fairseq
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge internal changes (facebookresearch#483)
Summary: Changelog: - `4889802`: can now remove detokenize sentencepiece output with `--remove-bpe=sentencepiece` (fixes facebookresearch#331). Also added `--sacrebleu` for computing detokenized BLEU. - `0d76427`: fix assertion error when training language model with dataset containing empty sentences - minor bug and style fixes Pull Request resolved: facebookresearch#483 Differential Revision: D13867899 Pulled By: myleott fbshipit-source-id: 25c940b847fe270262ac8f5ac838407b3977fdda
- Loading branch information
1 parent
66ce217
commit 42be3eb
Showing
12 changed files
with
159 additions
and
34 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,80 @@ | ||
# Copyright (c) 2017-present, Facebook, Inc. | ||
# All rights reserved. | ||
# | ||
# This source code is licensed under the license found in the LICENSE file in | ||
# the root directory of this source tree. An additional grant of patent rights | ||
# can be found in the PATENTS file in the same directory. | ||
|
||
import unittest | ||
|
||
import torch | ||
|
||
from fairseq.data import TokenBlockDataset | ||
|
||
import tests.utils as test_utils | ||
|
||
|
||
class TestTokenBlockDataset(unittest.TestCase): | ||
|
||
def _build_dataset(self, data, **kwargs): | ||
sizes = [len(x) for x in data] | ||
underlying_ds = test_utils.TestDataset(data) | ||
return TokenBlockDataset(underlying_ds, sizes, **kwargs) | ||
|
||
def test_eos_break_mode(self): | ||
data = [ | ||
torch.LongTensor([5, 4, 3, 2, 1]), | ||
torch.LongTensor([1]), # this should be filtered | ||
torch.LongTensor([8, 7, 6, 1]), | ||
] | ||
ds = self._build_dataset(data, block_size=None, pad=0, eos=1, break_mode='eos') | ||
self.assertEqual(ds[0].tolist(), [5, 4, 3, 2, 1]) | ||
self.assertEqual(ds[1].tolist(), [1]) | ||
self.assertEqual(ds[2].tolist(), [8, 7, 6, 1]) | ||
|
||
data = [ | ||
torch.LongTensor([5, 4, 3, 2, 1]), | ||
torch.LongTensor([8, 7, 6, 1]), | ||
torch.LongTensor([1]), # this should be filtered | ||
] | ||
ds = self._build_dataset(data, block_size=None, pad=0, eos=1, break_mode='eos') | ||
self.assertEqual(ds[0].tolist(), [5, 4, 3, 2, 1]) | ||
self.assertEqual(ds[1].tolist(), [8, 7, 6, 1]) | ||
self.assertEqual(ds[2].tolist(), [1]) | ||
|
||
def test_block_break_mode(self): | ||
data = [ | ||
torch.LongTensor([5, 4, 3, 2, 1]), | ||
torch.LongTensor([8, 7, 6, 1]), | ||
torch.LongTensor([9, 1]), | ||
] | ||
ds = self._build_dataset(data, block_size=3, pad=0, eos=1, break_mode='none') | ||
self.assertEqual(ds[0].tolist(), [5, 4, 3]) | ||
self.assertEqual(ds[1].tolist(), [2, 1, 8]) | ||
self.assertEqual(ds[2].tolist(), [7, 6, 1]) | ||
self.assertEqual(ds[3].tolist(), [9, 1]) | ||
|
||
def test_complete_break_mode(self): | ||
data = [ | ||
torch.LongTensor([5, 4, 3, 2, 1]), | ||
torch.LongTensor([8, 7, 6, 1]), | ||
torch.LongTensor([9, 1]), | ||
] | ||
ds = self._build_dataset(data, block_size=6, pad=0, eos=1, break_mode='complete') | ||
self.assertEqual(ds[0].tolist(), [5, 4, 3, 2, 1]) | ||
self.assertEqual(ds[1].tolist(), [8, 7, 6, 1, 9, 1]) | ||
|
||
data = [ | ||
torch.LongTensor([4, 3, 2, 1]), | ||
torch.LongTensor([5, 1]), | ||
torch.LongTensor([1]), | ||
torch.LongTensor([6, 1]), | ||
] | ||
ds = self._build_dataset(data, block_size=3, pad=0, eos=1, break_mode='complete') | ||
self.assertEqual(ds[0].tolist(), [4, 3, 2, 1]) | ||
self.assertEqual(ds[1].tolist(), [5, 1, 1]) | ||
self.assertEqual(ds[2].tolist(), [6, 1]) | ||
|
||
|
||
if __name__ == "__main__": | ||
unittest.main() |