Try mkdir for final destination before in WriteFiles (#31690)

* Try mkdir for final destination before in WriteFiles * debug log when fail mkdir
apache · Jun 26, 2024 · 2f51718 · 2f51718
1 parent 46b0a71
commit 2f51718
Showing 1 changed file with 8 additions and 0 deletions.
diff --git a/sdks/python/apache_beam/io/fileio.py b/sdks/python/apache_beam/io/fileio.py
@@ -708,6 +708,14 @@ def process(self, element, w=beam.DoFn.WindowParam):
         self.path.get(),
         move_to)
 
+    try:
+      filesystems.FileSystems.mkdirs(self.path.get())
+    except IOError as e:
+      cause = repr(e)
+      if 'FileExistsError' not in cause:
+        # Usually harmless. Especially if see FileExistsError so no need to log
+        _LOGGER.debug('Fail to create dir for final destination: %s', cause)
+
     try:
       filesystems.FileSystems.rename(
           move_from,