From a8c57cd201a92f79e85fd15c4098be7838f7c696 Mon Sep 17 00:00:00 2001
From: bugy <buggygm@gmail.com>
Date: Sun, 23 Jun 2019 23:58:22 +0200
Subject: [PATCH] #223: fixed an error on reading huge unicode outputs

---
 src/execution/process_pty.py            | 23 ++++++++++++++-----
 src/tests/execution/__init__.py         |  0
 src/tests/execution/process_pty_test.py | 30 +++++++++++++++++++++++++
 3 files changed, 47 insertions(+), 6 deletions(-)
 create mode 100644 src/tests/execution/__init__.py
 create mode 100644 src/tests/execution/process_pty_test.py

diff --git a/src/execution/process_pty.py b/src/execution/process_pty.py
index 9f110169..181a260b 100644
--- a/src/execution/process_pty.py
+++ b/src/execution/process_pty.py
@@ -62,6 +62,8 @@ def wait_finish(self):
     def pipe_process_output(self):
         utf8_stream = self.encoding.lower() == 'utf-8'
 
+        buffer = b''
+
         try:
             while True:
                 finished = False
@@ -69,8 +71,10 @@ def pipe_process_output(self):
 
                 max_read_bytes = 1024
 
+                data = buffer
+                buffer = b''
+
                 if self.is_finished():
-                    data = b""
                     while True:
                         try:
                             chunk = os.read(self.pty_master, max_read_bytes)
@@ -84,9 +88,8 @@ def pipe_process_output(self):
                     finished = True
 
                 else:
-                    data = ""
                     try:
-                        data = os.read(self.pty_master, max_read_bytes)
+                        data += os.read(self.pty_master, max_read_bytes)
                         if data.endswith(b"\r"):
                             data += os.read(self.pty_master, 1)
 
@@ -104,9 +107,17 @@ def pipe_process_output(self):
                     if not data:
                         wait_new_output = True
 
+                if utf8_stream and data:
+                    while data and data[-1] >= 127:
+                        buffer = bytes([data[-1]]) + buffer
+                        data = data[:-1]
+
                 if data:
-                    output_text = data.decode(self.encoding)
-                    self._write_script_output(output_text)
+                    try:
+                        output_text = data.decode(self.encoding)
+                        self._write_script_output(output_text)
+                    except UnicodeDecodeError as e:
+                        print(e)
 
                 if finished:
                     break
@@ -115,7 +126,7 @@ def pipe_process_output(self):
                     time.sleep(0.01)
 
         except:
-            self._write_script_output("Unexpected error occurred. Contact the administrator.")
+            self._write_script_output('\nUnexpected error occurred. Contact the administrator.')
 
             try:
                 self.kill()
diff --git a/src/tests/execution/__init__.py b/src/tests/execution/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src/tests/execution/process_pty_test.py b/src/tests/execution/process_pty_test.py
new file mode 100644
index 00000000..625bcdc5
--- /dev/null
+++ b/src/tests/execution/process_pty_test.py
@@ -0,0 +1,30 @@
+import threading
+import unittest
+
+from execution.process_pty import PtyProcessWrapper
+from react.observable import read_until_closed
+from tests import test_utils
+
+
+class TestPtyProcessWrapper(unittest.TestCase):
+    def test_many_unicode_characters(self):
+        long_unicode_text = ('ΩΨΔ\n' * 100000)
+        test_utils.create_file('test.txt', text=long_unicode_text)
+
+        process_wrapper = PtyProcessWrapper(['cat', 'test.txt'], test_utils.temp_folder)
+        process_wrapper.start()
+
+        thread = threading.Thread(target=process_wrapper.wait_finish, daemon=True)
+        thread.start()
+        thread.join(timeout=0.1)
+
+        output = ''.join(read_until_closed(process_wrapper.output_stream))
+        self.assertEqual(long_unicode_text, output)
+
+        self.assertEqual(0, process_wrapper.get_return_code())
+
+    def setUp(self):
+        test_utils.setup()
+
+    def tearDown(self):
+        test_utils.cleanup()