Download file locally when range request not supported

jvanz · Feb 8, 2022 · 9f90bcb · 9f90bcb
1 parent 72cacbe
commit 9f90bcb
Showing 1 changed file with 9 additions and 1 deletion.
diff --git a/src/datasets/utils/streaming_download_manager.py b/src/datasets/utils/streaming_download_manager.py
@@ -383,7 +383,15 @@ def xopen(file: str, mode="r", *args, use_auth_token: Optional[Union[str, bool]]
     else:
         new_kwargs = {}
     kwargs = {**kwargs, **new_kwargs}
-    file_obj = fsspec.open(file, mode=mode, *args, **kwargs).open()
+    try:
+        file_obj = fsspec.open(file, mode=mode, *args, **kwargs).open()
+    except ValueError as e:
+        if str(e) == "Cannot seek streaming HTTP file":  # Workaround for servers not supporting HTTP range requests
+            file = file.split("::")
+            file = "::".join(file[:-1] + ["simplecache", file[-1]])
+            file_obj = fsspec.open(file, mode=mode, *args, **kwargs).open()
+        else:
+            raise
     _add_retries_to_file_obj_read_method(file_obj)
     return file_obj