Patch recursive import (huggingface#11812)

markurtz · May 21, 2021 · 1b65229 · 1b65229
1 parent bd98716
commit 1b65229
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 2 deletions.
diff --git a/src/transformers/convert_slow_tokenizer.py b/src/transformers/convert_slow_tokenizer.py
@@ -25,7 +25,6 @@
 from tokenizers.models import BPE, Unigram, WordPiece
 
 from .file_utils import requires_backends
-from .models.roformer.tokenization_utils import JiebaPreTokenizer
 
 
 class SentencePieceExtractor:
@@ -299,6 +298,8 @@ def converted(self) -> Tokenizer:
 
 class RoFormerConverter(Converter):
     def converted(self) -> Tokenizer:
+        from .models.roformer.tokenization_utils import JiebaPreTokenizer
+
         vocab = self.original_tokenizer.vocab
         tokenizer = Tokenizer(WordPiece(vocab, unk_token=str(self.original_tokenizer.unk_token)))
 

diff --git a/tests/test_tokenization_utils.py b/tests/test_tokenization_utils.py
@@ -12,6 +12,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+"""
+isort:skip_file
+"""
 import os
 import pickle
 import tempfile
@@ -20,12 +23,14 @@
 
 import numpy as np
 
+# Ensure there are no circular imports when importing the parent class
+from transformers import PreTrainedTokenizerFast
+
 from transformers import (
     BatchEncoding,
     BertTokenizer,
     BertTokenizerFast,
     PreTrainedTokenizer,
-    PreTrainedTokenizerFast,
     TensorType,
     TokenSpan,
     is_tokenizers_available,