diff --git a/zeldarose/datasets/transform.py b/zeldarose/datasets/transform.py index 164d8b0..b048705 100644 --- a/zeldarose/datasets/transform.py +++ b/zeldarose/datasets/transform.py @@ -26,7 +26,10 @@ def encode_dataset( logger.info(f"Loading data from {text_path}") try: full_dataset = datasets.load_dataset("text", data_files=str(text_path), split="train") - except FileNotFoundError as e: + # So far the cleaner way to detect that a dataset is remote??? + # in datasets < 2.14 this was FileNotFoundError, in 2.14 it's the other one + # in the future? Who's to say,,, + except (FileNotFoundError, datasets.builder.DatasetGenerationError) as e: if isinstance(text_path, str): dataset_name, dataset_config, dataset_split = text_path.split(":") full_dataset = datasets.load_dataset(