bugy#223: fixed an error on reading huge unicode outputs

src/execution/process_pty.py

@@ -62,15 +62,19 @@ def wait_finish(self):
     def pipe_process_output(self):
         utf8_stream = self.encoding.lower() == 'utf-8'
 
+        buffer = b''
+
         try:
             while True:
                 finished = False
                 wait_new_output = False
 
                 max_read_bytes = 1024
 
+                data = buffer
+                buffer = b''
+
                 if self.is_finished():
-                    data = b""
                     while True:
                         try:
                             chunk = os.read(self.pty_master, max_read_bytes)
@@ -84,9 +88,8 @@ def pipe_process_output(self):
                     finished = True
 
                 else:
-                    data = ""
                     try:
-                        data = os.read(self.pty_master, max_read_bytes)
+                        data += os.read(self.pty_master, max_read_bytes)
                         if data.endswith(b"\r"):
                             data += os.read(self.pty_master, 1)
 
@@ -104,9 +107,17 @@ def pipe_process_output(self):
                     if not data:
                         wait_new_output = True
 
+                if utf8_stream and data:
+                    while data and data[-1] >= 127:
+                        buffer = bytes([data[-1]]) + buffer
+                        data = data[:-1]
+
                 if data:
-                    output_text = data.decode(self.encoding)
-                    self._write_script_output(output_text)
+                    try:
+                        output_text = data.decode(self.encoding)
+                        self._write_script_output(output_text)
+                    except UnicodeDecodeError as e:
+                        print(e)
 
                 if finished:
                     break
@@ -115,7 +126,7 @@ def pipe_process_output(self):
                     time.sleep(0.01)
 
         except:
-            self._write_script_output("Unexpected error occurred. Contact the administrator.")
+            self._write_script_output('\nUnexpected error occurred. Contact the administrator.')
 
             try:
                 self.kill()

src/tests/execution/process_pty_test.py

@@ -0,0 +1,30 @@
+import threading
+import unittest
+
+from execution.process_pty import PtyProcessWrapper
+from react.observable import read_until_closed
+from tests import test_utils
+
+
+class TestPtyProcessWrapper(unittest.TestCase):
+    def test_many_unicode_characters(self):
+        long_unicode_text = ('ΩΨΔ\n' * 100000)
+        test_utils.create_file('test.txt', text=long_unicode_text)
+
+        process_wrapper = PtyProcessWrapper(['cat', 'test.txt'], test_utils.temp_folder)
+        process_wrapper.start()
+
+        thread = threading.Thread(target=process_wrapper.wait_finish, daemon=True)
+        thread.start()
+        thread.join(timeout=0.1)
+
+        output = ''.join(read_until_closed(process_wrapper.output_stream))
+        self.assertEqual(long_unicode_text, output)
+
+        self.assertEqual(0, process_wrapper.get_return_code())
+
+    def setUp(self):
+        test_utils.setup()
+
+    def tearDown(self):
+        test_utils.cleanup()