diff --git a/ThirdPartyNotices.txt b/ThirdPartyNotices.txt index af80b88206..fea78d6207 100644 --- a/ThirdPartyNotices.txt +++ b/ThirdPartyNotices.txt @@ -320,5 +320,58 @@ LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. +17) fsnotify (MIT) +From https://github.com/fabioz/fsnotify -17) All third party packages in node_modules folders are licensed under the licenses specified in those packages. +Copyright (c) 2021, Fabio Zadrozny + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +18) scandir is released under the new BSD 3-clause license. +From: https://github.com/benhoyt/scandir + +Copyright (c) 2012, Ben Hoyt +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +* Neither the name of Ben Hoyt nor the names of its contributors may be used +to endorse or promote products derived from this software without specific +prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +19) All third party packages in node_modules folders are licensed under the licenses specified in those packages. diff --git a/robocorp-python-ls-core/.pydevproject b/robocorp-python-ls-core/.pydevproject index 026eed8b9a..00ba5b50dc 100644 --- a/robocorp-python-ls-core/.pydevproject +++ b/robocorp-python-ls-core/.pydevproject @@ -1,12 +1,18 @@ -python interpreter -Default - -/${PROJECT_DIR_NAME}/src -/${PROJECT_DIR_NAME}/tests -/${PROJECT_DIR_NAME}/src/robocorp_ls_core/libs/watchdog_lib -/${PROJECT_DIR_NAME}/src/robocorp_ls_core/libs/yaml_lib - -3.6 + + python interpreter + + Default + + + /${PROJECT_DIR_NAME}/src + /${PROJECT_DIR_NAME}/tests + /${PROJECT_DIR_NAME}/src/robocorp_ls_core/libs/watchdog_lib + /${PROJECT_DIR_NAME}/src/robocorp_ls_core/libs/yaml_lib + /${PROJECT_DIR_NAME}/src/robocorp_ls_core/libs/fsnotify_lib + + + 3.6 + diff --git a/robocorp-python-ls-core/src/robocorp_ls_core/libs/fsnotify_lib/README.md b/robocorp-python-ls-core/src/robocorp_ls_core/libs/fsnotify_lib/README.md new file mode 100644 index 0000000000..add6882440 --- /dev/null +++ b/robocorp-python-ls-core/src/robocorp_ls_core/libs/fsnotify_lib/README.md @@ -0,0 +1,9 @@ +The files in this directory contain fsnotify. + +To update, erase it and run: + +pip install fsnotify --target . + +This folder should be automatically added to the PYTHONPATH when needed. + +IMPORTANT: Keep __init__.py in directory \ No newline at end of file diff --git a/robocorp-python-ls-core/src/robocorp_ls_core/libs/fsnotify_lib/__init__.py b/robocorp-python-ls-core/src/robocorp_ls_core/libs/fsnotify_lib/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/robocorp-python-ls-core/src/robocorp_ls_core/libs/fsnotify_lib/fsnotify-0.2.1.dist-info/INSTALLER b/robocorp-python-ls-core/src/robocorp_ls_core/libs/fsnotify_lib/fsnotify-0.2.1.dist-info/INSTALLER new file mode 100644 index 0000000000..a1b589e38a --- /dev/null +++ b/robocorp-python-ls-core/src/robocorp_ls_core/libs/fsnotify_lib/fsnotify-0.2.1.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/robocorp-python-ls-core/src/robocorp_ls_core/libs/fsnotify_lib/fsnotify-0.2.1.dist-info/LICENSE b/robocorp-python-ls-core/src/robocorp_ls_core/libs/fsnotify_lib/fsnotify-0.2.1.dist-info/LICENSE new file mode 100644 index 0000000000..03ff18bf02 --- /dev/null +++ b/robocorp-python-ls-core/src/robocorp_ls_core/libs/fsnotify_lib/fsnotify-0.2.1.dist-info/LICENSE @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2021, Fabio Zadrozny + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/robocorp-python-ls-core/src/robocorp_ls_core/libs/fsnotify_lib/fsnotify-0.2.1.dist-info/METADATA b/robocorp-python-ls-core/src/robocorp_ls_core/libs/fsnotify_lib/fsnotify-0.2.1.dist-info/METADATA new file mode 100644 index 0000000000..970ad10d87 --- /dev/null +++ b/robocorp-python-ls-core/src/robocorp_ls_core/libs/fsnotify_lib/fsnotify-0.2.1.dist-info/METADATA @@ -0,0 +1,79 @@ +Metadata-Version: 2.1 +Name: fsnotify +Version: 0.2.1 +Summary: Simple file watching +Home-page: https://github.com/fabioz/fsnotify +Author: Fabio Zadrozny +Author-email: fabiofz@gmail.com +License: MIT license +Keywords: fsnotify filesystem notifications watchdog watchgod +Platform: UNKNOWN +Classifier: Development Status :: 2 - Pre-Alpha +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Natural Language :: English +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Requires-Python: >=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.* +Description-Content-Type: text/markdown +Provides-Extra: dev +Requires-Dist: twine ; extra == 'dev' +Requires-Dist: wheel ; extra == 'dev' +Requires-Dist: pytest ; extra == 'dev' +Provides-Extra: test +Requires-Dist: pytest ; extra == 'test' + + + + +[![Github Actions Shield](https://img.shields.io/github/workflow/status/fabioz/fsnotify/Run%20Tests)](https://github.com/fabioz/fsnotify/actions) + + + +# fsnotify + +Simple python file watching with throttling when polling. + +- Homepage: https://github.com/fabioz/fsnotify + +# Overview + +The idea for this project is making the fastest possible polling-based file watching +with throttling to avoid high CPU usage when polling. + +# Install + +## Requirements + +**fsnotify** has been developed and tested on [Python 2.7, 3.5, 3.6, 3.7 and 3.8](https://www.python.org/downloads/) + + +## Install from PyPI + +```bash +pip install fsnotify +``` + +This will pull and install the latest stable release from [PyPI](https://pypi.org/). + + +# History + +FSNotify 0.2.0 +--------------- + +Paths to track may now be a List[TrackedPath], indicating whether each path +should be tracked recursively or not. + + +FSNotify 0.1.6 +--------------- + +First stable version. Should be able to properly do recursive file watching +of all passed paths using throttling to avoid high CPU usage when polling + diff --git a/robocorp-python-ls-core/src/robocorp_ls_core/libs/fsnotify_lib/fsnotify-0.2.1.dist-info/RECORD b/robocorp-python-ls-core/src/robocorp_ls_core/libs/fsnotify_lib/fsnotify-0.2.1.dist-info/RECORD new file mode 100644 index 0000000000..05e6c7056c --- /dev/null +++ b/robocorp-python-ls-core/src/robocorp_ls_core/libs/fsnotify_lib/fsnotify-0.2.1.dist-info/RECORD @@ -0,0 +1,11 @@ +fsnotify-0.2.1.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +fsnotify-0.2.1.dist-info/LICENSE,sha256=X47D6i39KocE8tJLUBJoKnELkQVDapxTUn3neBOOheA,1073 +fsnotify-0.2.1.dist-info/METADATA,sha256=7lZBnk7vBbxc9-YVTmjkI_2MGFGe-U0Cj8vI2OMUknM,2456 +fsnotify-0.2.1.dist-info/RECORD,, +fsnotify-0.2.1.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +fsnotify-0.2.1.dist-info/WHEEL,sha256=Z-nyYpwrcSqxfdux5Mbn_DQ525iP7J2DG3JgGvOYyTQ,110 +fsnotify-0.2.1.dist-info/top_level.txt,sha256=Uik-6AdG4ijxEWJNdpPiqvXd6SpTT_vcbob_kjDrolU,9 +fsnotify/__init__.py,sha256=6V5wW_d1PUQUsm5xIr78sVQVNTASkOsj4LJV88qcjt4,13442 +fsnotify/__pycache__/__init__.cpython-37.pyc,, +fsnotify/__pycache__/scandir_vendored.cpython-37.pyc,, +fsnotify/scandir_vendored.py,sha256=KCOwdkBO2IA_hye_IV41JLNzpbYP45rJB0RiqXTAI8s,26249 diff --git a/robocorp-python-ls-core/src/robocorp_ls_core/libs/fsnotify_lib/fsnotify-0.2.1.dist-info/REQUESTED b/robocorp-python-ls-core/src/robocorp_ls_core/libs/fsnotify_lib/fsnotify-0.2.1.dist-info/REQUESTED new file mode 100644 index 0000000000..e69de29bb2 diff --git a/robocorp-python-ls-core/src/robocorp_ls_core/libs/fsnotify_lib/fsnotify-0.2.1.dist-info/WHEEL b/robocorp-python-ls-core/src/robocorp_ls_core/libs/fsnotify_lib/fsnotify-0.2.1.dist-info/WHEEL new file mode 100644 index 0000000000..01b8fc7d4a --- /dev/null +++ b/robocorp-python-ls-core/src/robocorp_ls_core/libs/fsnotify_lib/fsnotify-0.2.1.dist-info/WHEEL @@ -0,0 +1,6 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.36.2) +Root-Is-Purelib: true +Tag: py2-none-any +Tag: py3-none-any + diff --git a/robocorp-python-ls-core/src/robocorp_ls_core/libs/fsnotify_lib/fsnotify-0.2.1.dist-info/top_level.txt b/robocorp-python-ls-core/src/robocorp_ls_core/libs/fsnotify_lib/fsnotify-0.2.1.dist-info/top_level.txt new file mode 100644 index 0000000000..e09c164225 --- /dev/null +++ b/robocorp-python-ls-core/src/robocorp_ls_core/libs/fsnotify_lib/fsnotify-0.2.1.dist-info/top_level.txt @@ -0,0 +1 @@ +fsnotify diff --git a/robocorp-python-ls-core/src/robocorp_ls_core/libs/fsnotify_lib/fsnotify/__init__.py b/robocorp-python-ls-core/src/robocorp_ls_core/libs/fsnotify_lib/fsnotify/__init__.py new file mode 100644 index 0000000000..93be87d6c8 --- /dev/null +++ b/robocorp-python-ls-core/src/robocorp_ls_core/libs/fsnotify_lib/fsnotify/__init__.py @@ -0,0 +1,382 @@ +''' +Sample usage to track changes in a thread. + + import threading + import time + watcher = fsnotify.Watcher() + watcher.accepted_file_extensions = {'.py', '.pyw'} + + # Configure target values to compute throttling. + # Note: internal sleep times will be updated based on + # profiling the actual application runtime to match + # those values. + + watcher.target_time_for_single_scan = 2. + watcher.target_time_for_notification = 4. + + watcher.set_tracked_paths([target_dir]) + + def start_watching(): # Called from thread + for change_enum, change_path in watcher.iter_changes(): + if change_enum == fsnotify.Change.added: + print('Added: ', change_path) + elif change_enum == fsnotify.Change.modified: + print('Modified: ', change_path) + elif change_enum == fsnotify.Change.deleted: + print('Deleted: ', change_path) + + t = threading.Thread(target=start_watching) + t.daemon = True + t.start() + + try: + ... + finally: + watcher.dispose() + + +Note: changes are only reported for files (added/modified/deleted), not directories. +''' +import threading +import sys +try: + from os import scandir +except: + try: + # Search an installed version (which may have speedups). + from scandir import scandir + except: + # If all fails, use our vendored version (which won't have speedups). + from .scandir_vendored import scandir + +try: + from enum import IntEnum +except: + + class IntEnum(object): + pass + +from collections import deque + +import time + +__author__ = 'Fabio Zadrozny' +__email__ = 'fabiofz@gmail.com' +__version__ = '0.2.1' # Version here and in setup.py + +PRINT_SINGLE_POLL_TIME = False + + +class Change(IntEnum): + added = 1 + modified = 2 + deleted = 3 + + +class _SingleVisitInfo(object): + + def __init__(self): + self.count = 0 + self.visited_dirs = set() + self.file_to_mtime = {} + self.last_sleep_time = time.time() + + +class TrackedPath(object): + + __slots__ = ['path', 'recursive'] + + def __init__(self, path, recursive): + self.path = path + self.recursive = recursive + + +class _PathWatcher(object): + ''' + Helper to watch a single path. + ''' + + def __init__(self, root_path, accept_directory, accept_file, single_visit_info, max_recursion_level, sleep_time=.0, recursive=True): + ''' + :type root_path: str + :type accept_directory: Callback[str, bool] + :type accept_file: Callback[str, bool] + :type max_recursion_level: int + :type sleep_time: float + ''' + self.accept_directory = accept_directory + self.accept_file = accept_file + self._max_recursion_level = max_recursion_level + + self._root_path = root_path + self._recursive = recursive + + # Initial sleep value for throttling, it'll be auto-updated based on the + # Watcher.target_time_for_single_scan. + self.sleep_time = sleep_time + + self.sleep_at_elapsed = 1. / 30. + + # When created, do the initial snapshot right away! + old_file_to_mtime = {} + self._check(single_visit_info, lambda _change: None, old_file_to_mtime) + + def __eq__(self, o): + if isinstance(o, _PathWatcher): + return self._root_path == o._root_path + + return False + + def __ne__(self, o): + return not self == o + + def __hash__(self): + return hash(self._root_path) + + def _check_dir(self, dir_path, single_visit_info, append_change, old_file_to_mtime, level): + # This is the actual poll loop + if dir_path in single_visit_info.visited_dirs or level > self._max_recursion_level: + return + single_visit_info.visited_dirs.add(dir_path) + try: + if isinstance(dir_path, bytes): + try: + dir_path = dir_path.decode(sys.getfilesystemencoding()) + except UnicodeDecodeError: + try: + dir_path = dir_path.decode('utf-8') + except UnicodeDecodeError: + return # Ignore if we can't deal with the path. + + new_files = single_visit_info.file_to_mtime + + for entry in scandir(dir_path): + single_visit_info.count += 1 + + # Throttle if needed inside the loop + # to avoid consuming too much CPU. + if single_visit_info.count % 300 == 0: + if self.sleep_time > 0: + t = time.time() + diff = t - single_visit_info.last_sleep_time + if diff > self.sleep_at_elapsed: + time.sleep(self.sleep_time) + single_visit_info.last_sleep_time = time.time() + + if entry.is_dir(): + if self.accept_directory(entry.path): + if self._recursive: + self._check_dir(entry.path, single_visit_info, append_change, old_file_to_mtime, level + 1) + + elif self.accept_file(entry.path): + stat = entry.stat() + mtime = (stat.st_mtime_ns, stat.st_size) + path = entry.path + new_files[path] = mtime + + old_mtime = old_file_to_mtime.pop(path, None) + if not old_mtime: + append_change((Change.added, path)) + elif old_mtime != mtime: + append_change((Change.modified, path)) + + except OSError: + pass # Directory was removed in the meanwhile. + + def _check(self, single_visit_info, append_change, old_file_to_mtime): + self._check_dir(self._root_path, single_visit_info, append_change, old_file_to_mtime, 0) + + +class Watcher(object): + + # By default (if accept_directory is not specified), these will be the + # ignored directories. + ignored_dirs = {u'.git', u'__pycache__', u'.idea', u'node_modules', u'.metadata'} + + # By default (if accept_file is not specified), these will be the + # accepted files. + accepted_file_extensions = () + + # Set to the target value for doing full scan of all files (adds a sleep inside the poll loop + # which processes files to reach the target time). + # Lower values will consume more CPU + # Set to 0.0 to have no sleeps (which will result in a higher cpu load). + target_time_for_single_scan = 2.0 + + # Set the target value from the start of one scan to the start of another scan (adds a + # sleep after a full poll is done to reach the target time). + # Lower values will consume more CPU. + # Set to 0.0 to have a new scan start right away without any sleeps. + target_time_for_notification = 4.0 + + # Set to True to print the time for a single poll through all the paths. + print_poll_time = False + + # This is the maximum recursion level. + max_recursion_level = 10 + + def __init__(self, accept_directory=None, accept_file=None): + ''' + :param Callable[str, bool] accept_directory: + Callable that returns whether a directory should be watched. + Note: if passed it'll override the `ignored_dirs` + + :param Callable[str, bool] accept_file: + Callable that returns whether a file should be watched. + Note: if passed it'll override the `accepted_file_extensions`. + ''' + self._lock = threading.Lock() + + self._path_watchers = set() + self._disposed = threading.Event() + + if accept_directory is None: + from os.path import basename + accept_directory = lambda dir_path: basename(dir_path) not in self.ignored_dirs + if accept_file is None: + accept_file = lambda path_name: \ + not self.accepted_file_extensions or path_name.endswith(self.accepted_file_extensions) + self.accept_file = accept_file + self.accept_directory = accept_directory + self._single_visit_info = _SingleVisitInfo() + + @property + def accept_directory(self): + return self._accept_directory + + @accept_directory.setter + def accept_directory(self, accept_directory): + self._accept_directory = accept_directory + for path_watcher in self._path_watchers: + path_watcher.accept_directory = accept_directory + + @property + def accept_file(self): + return self._accept_file + + @accept_file.setter + def accept_file(self, accept_file): + self._accept_file = accept_file + for path_watcher in self._path_watchers: + path_watcher.accept_file = accept_file + + def dispose(self): + self._disposed.set() + + @property + def path_watchers(self): + return tuple(self._path_watchers) + + def set_tracked_paths(self, paths): + """ + Note: always resets all path trackers to track the passed paths. + :type paths: [str|TrackedPath] + """ + if not isinstance(paths, (list, tuple, set)): + paths = (paths,) + + def key(path_or_str): + if isinstance(path_or_str, TrackedPath): + return -len(path_or_str.path) + return -len(path_or_str) + + # Sort by the path len so that the bigger paths come first (so, + # if there's any nesting we want the nested paths to be visited + # before the parent paths so that the max_recursion_level is correct). + paths = sorted(set(paths), key=key) + path_watchers = set() + + single_visit_info = _SingleVisitInfo() + + for path in paths: + sleep_time = 0. # When collecting the first time, sleep_time should be 0! + path_watcher = _PathWatcher( + path.path if isinstance(path, TrackedPath) else path, + self.accept_directory, + self.accept_file, + single_visit_info, + max_recursion_level=self.max_recursion_level, + sleep_time=sleep_time, + recursive=path.recursive if isinstance(path, TrackedPath) else path + ) + + path_watchers.add(path_watcher) + + with self._lock: + self._single_visit_info = single_visit_info + self._path_watchers = path_watchers + + def iter_changes(self): + ''' + Continuously provides changes (until dispose() is called). + + Changes provided are tuples with the Change enum and filesystem path. + + :rtype: Iterable[Tuple[Change, str]] + ''' + while not self._disposed.is_set(): + + with self._lock: + old_visit_info = self._single_visit_info + old_file_to_mtime = old_visit_info.file_to_mtime + changes = [] + append_change = changes.append + + self._single_visit_info = single_visit_info = _SingleVisitInfo() + path_watchers = self._path_watchers.copy() + + initial_time = time.time() + for path_watcher in self._path_watchers: + path_watcher._check(single_visit_info, append_change, old_file_to_mtime) + + # Note that we pop entries while visiting, so, what remained is what's deleted. + for entry in old_file_to_mtime: + append_change((Change.deleted, entry)) + + for change in changes: + yield change + + actual_time = (time.time() - initial_time) + if self.print_poll_time: + print('--- Total poll time: %.3fs' % actual_time) + + if actual_time > 0: + if self.target_time_for_single_scan <= 0.0: + for path_watcher in path_watchers: + path_watcher.sleep_time = 0.0 + else: + perc = self.target_time_for_single_scan / actual_time + + # Prevent from changing the values too much (go slowly into the right + # direction). + # (to prevent from cases where the user puts the machine on sleep and + # values become too skewed). + if perc > 2.: + perc = 2. + elif perc < 0.5: + perc = 0.5 + + for path_watcher in path_watchers: + if path_watcher.sleep_time <= 0.0: + path_watcher.sleep_time = 0.001 + new_sleep_time = path_watcher.sleep_time * perc + + # Prevent from changing the values too much (go slowly into the right + # direction). + # (to prevent from cases where the user puts the machine on sleep and + # values become too skewed). + diff_sleep_time = new_sleep_time - path_watcher.sleep_time + path_watcher.sleep_time += (diff_sleep_time / (3.0 * len(self._path_watchers))) + + if actual_time > 0: + self._disposed.wait(actual_time) + + if path_watcher.sleep_time < 0.001: + path_watcher.sleep_time = 0.001 + + # print('new sleep time: %s' % path_watcher.sleep_time) + + diff = self.target_time_for_notification - actual_time + if diff > 0.: + self._disposed.wait(diff) + diff --git a/robocorp-python-ls-core/src/robocorp_ls_core/libs/fsnotify_lib/fsnotify/scandir_vendored.py b/robocorp-python-ls-core/src/robocorp_ls_core/libs/fsnotify_lib/fsnotify/scandir_vendored.py new file mode 100644 index 0000000000..e38df9c7d7 --- /dev/null +++ b/robocorp-python-ls-core/src/robocorp_ls_core/libs/fsnotify_lib/fsnotify/scandir_vendored.py @@ -0,0 +1,720 @@ +"""scandir, a better directory iterator and faster os.walk(), now in the Python 3.5 stdlib + +scandir() is a generator version of os.listdir() that returns an +iterator over files in a directory, and also exposes the extra +information most OSes provide while iterating files in a directory +(such as type and stat information). + +This module also includes a version of os.walk() that uses scandir() +to speed it up significantly. + +See README.md or https://github.com/benhoyt/scandir for rationale and +docs, or read PEP 471 (https://www.python.org/dev/peps/pep-0471/) for +more details on its inclusion into Python 3.5 + +scandir is released under the new BSD 3-clause license. + +Copyright (c) 2012, Ben Hoyt +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +* Neither the name of Ben Hoyt nor the names of its contributors may be used +to endorse or promote products derived from this software without specific +prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +""" + +from __future__ import division + +from errno import ENOENT +from os import listdir, lstat, stat, strerror +from os.path import join, islink +from stat import S_IFDIR, S_IFLNK, S_IFREG +import collections +import sys + +try: + import _scandir +except ImportError: + _scandir = None + +try: + import ctypes +except ImportError: + ctypes = None + +if _scandir is None and ctypes is None: + import warnings + warnings.warn("scandir can't find the compiled _scandir C module " + "or ctypes, using slow generic fallback") + +__version__ = '1.10.0' +__all__ = ['scandir', 'walk'] + +# Windows FILE_ATTRIBUTE constants for interpreting the +# FIND_DATA.dwFileAttributes member +FILE_ATTRIBUTE_ARCHIVE = 32 +FILE_ATTRIBUTE_COMPRESSED = 2048 +FILE_ATTRIBUTE_DEVICE = 64 +FILE_ATTRIBUTE_DIRECTORY = 16 +FILE_ATTRIBUTE_ENCRYPTED = 16384 +FILE_ATTRIBUTE_HIDDEN = 2 +FILE_ATTRIBUTE_INTEGRITY_STREAM = 32768 +FILE_ATTRIBUTE_NORMAL = 128 +FILE_ATTRIBUTE_NOT_CONTENT_INDEXED = 8192 +FILE_ATTRIBUTE_NO_SCRUB_DATA = 131072 +FILE_ATTRIBUTE_OFFLINE = 4096 +FILE_ATTRIBUTE_READONLY = 1 +FILE_ATTRIBUTE_REPARSE_POINT = 1024 +FILE_ATTRIBUTE_SPARSE_FILE = 512 +FILE_ATTRIBUTE_SYSTEM = 4 +FILE_ATTRIBUTE_TEMPORARY = 256 +FILE_ATTRIBUTE_VIRTUAL = 65536 + +IS_PY3 = sys.version_info >= (3, 0) + +if IS_PY3: + unicode = str # Because Python <= 3.2 doesn't have u'unicode' syntax + + +class GenericDirEntry(object): + __slots__ = ('name', '_stat', '_lstat', '_scandir_path', '_path') + + def __init__(self, scandir_path, name): + self._scandir_path = scandir_path + self.name = name + self._stat = None + self._lstat = None + self._path = None + + @property + def path(self): + if self._path is None: + self._path = join(self._scandir_path, self.name) + return self._path + + def stat(self, follow_symlinks=True): + if follow_symlinks: + if self._stat is None: + self._stat = stat(self.path) + return self._stat + else: + if self._lstat is None: + self._lstat = lstat(self.path) + return self._lstat + + # The code duplication below is intentional: this is for slightly + # better performance on systems that fall back to GenericDirEntry. + # It avoids an additional attribute lookup and method call, which + # are relatively slow on CPython. + def is_dir(self, follow_symlinks=True): + try: + st = self.stat(follow_symlinks=follow_symlinks) + except OSError as e: + if e.errno != ENOENT: + raise + return False # Path doesn't exist or is a broken symlink + return st.st_mode & 0o170000 == S_IFDIR + + def is_file(self, follow_symlinks=True): + try: + st = self.stat(follow_symlinks=follow_symlinks) + except OSError as e: + if e.errno != ENOENT: + raise + return False # Path doesn't exist or is a broken symlink + return st.st_mode & 0o170000 == S_IFREG + + def is_symlink(self): + try: + st = self.stat(follow_symlinks=False) + except OSError as e: + if e.errno != ENOENT: + raise + return False # Path doesn't exist or is a broken symlink + return st.st_mode & 0o170000 == S_IFLNK + + def inode(self): + st = self.stat(follow_symlinks=False) + return st.st_ino + + def __str__(self): + return '<{0}: {1!r}>'.format(self.__class__.__name__, self.name) + + __repr__ = __str__ + + +def _scandir_generic(path=unicode('.')): + """Like os.listdir(), but yield DirEntry objects instead of returning + a list of names. + """ + for name in listdir(path): + yield GenericDirEntry(path, name) + + +if IS_PY3 and sys.platform == 'win32': + def scandir_generic(path=unicode('.')): + if isinstance(path, bytes): + raise TypeError("os.scandir() doesn't support bytes path on Windows, use Unicode instead") + return _scandir_generic(path) + scandir_generic.__doc__ = _scandir_generic.__doc__ +else: + scandir_generic = _scandir_generic + + +scandir_c = None +scandir_python = None + + +if sys.platform == 'win32': + if ctypes is not None: + from ctypes import wintypes + + # Various constants from windows.h + INVALID_HANDLE_VALUE = ctypes.c_void_p(-1).value + ERROR_FILE_NOT_FOUND = 2 + ERROR_NO_MORE_FILES = 18 + IO_REPARSE_TAG_SYMLINK = 0xA000000C + + # Numer of seconds between 1601-01-01 and 1970-01-01 + SECONDS_BETWEEN_EPOCHS = 11644473600 + + kernel32 = ctypes.windll.kernel32 + + # ctypes wrappers for (wide string versions of) FindFirstFile, + # FindNextFile, and FindClose + FindFirstFile = kernel32.FindFirstFileW + FindFirstFile.argtypes = [ + wintypes.LPCWSTR, + ctypes.POINTER(wintypes.WIN32_FIND_DATAW), + ] + FindFirstFile.restype = wintypes.HANDLE + + FindNextFile = kernel32.FindNextFileW + FindNextFile.argtypes = [ + wintypes.HANDLE, + ctypes.POINTER(wintypes.WIN32_FIND_DATAW), + ] + FindNextFile.restype = wintypes.BOOL + + FindClose = kernel32.FindClose + FindClose.argtypes = [wintypes.HANDLE] + FindClose.restype = wintypes.BOOL + + Win32StatResult = collections.namedtuple('Win32StatResult', [ + 'st_mode', + 'st_ino', + 'st_dev', + 'st_nlink', + 'st_uid', + 'st_gid', + 'st_size', + 'st_atime', + 'st_mtime', + 'st_ctime', + 'st_atime_ns', + 'st_mtime_ns', + 'st_ctime_ns', + 'st_file_attributes', + ]) + + def filetime_to_time(filetime): + """Convert Win32 FILETIME to time since Unix epoch in seconds.""" + total = filetime.dwHighDateTime << 32 | filetime.dwLowDateTime + return total / 10000000 - SECONDS_BETWEEN_EPOCHS + + def find_data_to_stat(data): + """Convert Win32 FIND_DATA struct to stat_result.""" + # First convert Win32 dwFileAttributes to st_mode + attributes = data.dwFileAttributes + st_mode = 0 + if attributes & FILE_ATTRIBUTE_DIRECTORY: + st_mode |= S_IFDIR | 0o111 + else: + st_mode |= S_IFREG + if attributes & FILE_ATTRIBUTE_READONLY: + st_mode |= 0o444 + else: + st_mode |= 0o666 + if (attributes & FILE_ATTRIBUTE_REPARSE_POINT and + data.dwReserved0 == IO_REPARSE_TAG_SYMLINK): + st_mode ^= st_mode & 0o170000 + st_mode |= S_IFLNK + + st_size = data.nFileSizeHigh << 32 | data.nFileSizeLow + st_atime = filetime_to_time(data.ftLastAccessTime) + st_mtime = filetime_to_time(data.ftLastWriteTime) + st_ctime = filetime_to_time(data.ftCreationTime) + + # Some fields set to zero per CPython's posixmodule.c: st_ino, st_dev, + # st_nlink, st_uid, st_gid + return Win32StatResult(st_mode, 0, 0, 0, 0, 0, st_size, + st_atime, st_mtime, st_ctime, + int(st_atime * 1000000000), + int(st_mtime * 1000000000), + int(st_ctime * 1000000000), + attributes) + + class Win32DirEntryPython(object): + __slots__ = ('name', '_stat', '_lstat', '_find_data', '_scandir_path', '_path', '_inode') + + def __init__(self, scandir_path, name, find_data): + self._scandir_path = scandir_path + self.name = name + self._stat = None + self._lstat = None + self._find_data = find_data + self._path = None + self._inode = None + + @property + def path(self): + if self._path is None: + self._path = join(self._scandir_path, self.name) + return self._path + + def stat(self, follow_symlinks=True): + if follow_symlinks: + if self._stat is None: + if self.is_symlink(): + # It's a symlink, call link-following stat() + self._stat = stat(self.path) + else: + # Not a symlink, stat is same as lstat value + if self._lstat is None: + self._lstat = find_data_to_stat(self._find_data) + self._stat = self._lstat + return self._stat + else: + if self._lstat is None: + # Lazily convert to stat object, because it's slow + # in Python, and often we only need is_dir() etc + self._lstat = find_data_to_stat(self._find_data) + return self._lstat + + def is_dir(self, follow_symlinks=True): + is_symlink = self.is_symlink() + if follow_symlinks and is_symlink: + try: + return self.stat().st_mode & 0o170000 == S_IFDIR + except OSError as e: + if e.errno != ENOENT: + raise + return False + elif is_symlink: + return False + else: + return (self._find_data.dwFileAttributes & + FILE_ATTRIBUTE_DIRECTORY != 0) + + def is_file(self, follow_symlinks=True): + is_symlink = self.is_symlink() + if follow_symlinks and is_symlink: + try: + return self.stat().st_mode & 0o170000 == S_IFREG + except OSError as e: + if e.errno != ENOENT: + raise + return False + elif is_symlink: + return False + else: + return (self._find_data.dwFileAttributes & + FILE_ATTRIBUTE_DIRECTORY == 0) + + def is_symlink(self): + return (self._find_data.dwFileAttributes & + FILE_ATTRIBUTE_REPARSE_POINT != 0 and + self._find_data.dwReserved0 == IO_REPARSE_TAG_SYMLINK) + + def inode(self): + if self._inode is None: + self._inode = lstat(self.path).st_ino + return self._inode + + def __str__(self): + return '<{0}: {1!r}>'.format(self.__class__.__name__, self.name) + + __repr__ = __str__ + + def win_error(error, filename): + exc = WindowsError(error, ctypes.FormatError(error)) + exc.filename = filename + return exc + + def _scandir_python(path=unicode('.')): + """Like os.listdir(), but yield DirEntry objects instead of returning + a list of names. + """ + # Call FindFirstFile and handle errors + if isinstance(path, bytes): + is_bytes = True + filename = join(path.decode('mbcs', 'strict'), '*.*') + else: + is_bytes = False + filename = join(path, '*.*') + data = wintypes.WIN32_FIND_DATAW() + data_p = ctypes.byref(data) + handle = FindFirstFile(filename, data_p) + if handle == INVALID_HANDLE_VALUE: + error = ctypes.GetLastError() + if error == ERROR_FILE_NOT_FOUND: + # No files, don't yield anything + return + raise win_error(error, path) + + # Call FindNextFile in a loop, stopping when no more files + try: + while True: + # Skip '.' and '..' (current and parent directory), but + # otherwise yield (filename, stat_result) tuple + name = data.cFileName + if name not in ('.', '..'): + if is_bytes: + name = name.encode('mbcs', 'replace') + yield Win32DirEntryPython(path, name, data) + + data = wintypes.WIN32_FIND_DATAW() + data_p = ctypes.byref(data) + success = FindNextFile(handle, data_p) + if not success: + error = ctypes.GetLastError() + if error == ERROR_NO_MORE_FILES: + break + raise win_error(error, path) + finally: + if not FindClose(handle): + raise win_error(ctypes.GetLastError(), path) + + if IS_PY3: + def scandir_python(path=unicode('.')): + if isinstance(path, bytes): + raise TypeError("os.scandir() doesn't support bytes path on Windows, use Unicode instead") + return _scandir_python(path) + scandir_python.__doc__ = _scandir_python.__doc__ + else: + scandir_python = _scandir_python + + if _scandir is not None: + scandir_c = _scandir.scandir + DirEntry_c = _scandir.DirEntry + + if _scandir is not None: + scandir = scandir_c + DirEntry = DirEntry_c + elif ctypes is not None: + scandir = scandir_python + DirEntry = Win32DirEntryPython + else: + scandir = scandir_generic + DirEntry = GenericDirEntry + + +# Linux, OS X, and BSD implementation +elif sys.platform.startswith(('linux', 'darwin', 'sunos5')) or 'bsd' in sys.platform: + have_dirent_d_type = (sys.platform != 'sunos5') + + if ctypes is not None and have_dirent_d_type: + import ctypes.util + + DIR_p = ctypes.c_void_p + + # Rather annoying how the dirent struct is slightly different on each + # platform. The only fields we care about are d_name and d_type. + class Dirent(ctypes.Structure): + if sys.platform.startswith('linux'): + _fields_ = ( + ('d_ino', ctypes.c_ulong), + ('d_off', ctypes.c_long), + ('d_reclen', ctypes.c_ushort), + ('d_type', ctypes.c_byte), + ('d_name', ctypes.c_char * 256), + ) + elif 'openbsd' in sys.platform: + _fields_ = ( + ('d_ino', ctypes.c_uint64), + ('d_off', ctypes.c_uint64), + ('d_reclen', ctypes.c_uint16), + ('d_type', ctypes.c_uint8), + ('d_namlen', ctypes.c_uint8), + ('__d_padding', ctypes.c_uint8 * 4), + ('d_name', ctypes.c_char * 256), + ) + else: + _fields_ = ( + ('d_ino', ctypes.c_uint32), # must be uint32, not ulong + ('d_reclen', ctypes.c_ushort), + ('d_type', ctypes.c_byte), + ('d_namlen', ctypes.c_byte), + ('d_name', ctypes.c_char * 256), + ) + + DT_UNKNOWN = 0 + DT_DIR = 4 + DT_REG = 8 + DT_LNK = 10 + + Dirent_p = ctypes.POINTER(Dirent) + Dirent_pp = ctypes.POINTER(Dirent_p) + + libc = ctypes.CDLL(ctypes.util.find_library('c'), use_errno=True) + opendir = libc.opendir + opendir.argtypes = [ctypes.c_char_p] + opendir.restype = DIR_p + + readdir_r = libc.readdir_r + readdir_r.argtypes = [DIR_p, Dirent_p, Dirent_pp] + readdir_r.restype = ctypes.c_int + + closedir = libc.closedir + closedir.argtypes = [DIR_p] + closedir.restype = ctypes.c_int + + file_system_encoding = sys.getfilesystemencoding() + + class PosixDirEntry(object): + __slots__ = ('name', '_d_type', '_stat', '_lstat', '_scandir_path', '_path', '_inode') + + def __init__(self, scandir_path, name, d_type, inode): + self._scandir_path = scandir_path + self.name = name + self._d_type = d_type + self._inode = inode + self._stat = None + self._lstat = None + self._path = None + + @property + def path(self): + if self._path is None: + self._path = join(self._scandir_path, self.name) + return self._path + + def stat(self, follow_symlinks=True): + if follow_symlinks: + if self._stat is None: + if self.is_symlink(): + self._stat = stat(self.path) + else: + if self._lstat is None: + self._lstat = lstat(self.path) + self._stat = self._lstat + return self._stat + else: + if self._lstat is None: + self._lstat = lstat(self.path) + return self._lstat + + def is_dir(self, follow_symlinks=True): + if (self._d_type == DT_UNKNOWN or + (follow_symlinks and self.is_symlink())): + try: + st = self.stat(follow_symlinks=follow_symlinks) + except OSError as e: + if e.errno != ENOENT: + raise + return False + return st.st_mode & 0o170000 == S_IFDIR + else: + return self._d_type == DT_DIR + + def is_file(self, follow_symlinks=True): + if (self._d_type == DT_UNKNOWN or + (follow_symlinks and self.is_symlink())): + try: + st = self.stat(follow_symlinks=follow_symlinks) + except OSError as e: + if e.errno != ENOENT: + raise + return False + return st.st_mode & 0o170000 == S_IFREG + else: + return self._d_type == DT_REG + + def is_symlink(self): + if self._d_type == DT_UNKNOWN: + try: + st = self.stat(follow_symlinks=False) + except OSError as e: + if e.errno != ENOENT: + raise + return False + return st.st_mode & 0o170000 == S_IFLNK + else: + return self._d_type == DT_LNK + + def inode(self): + return self._inode + + def __str__(self): + return '<{0}: {1!r}>'.format(self.__class__.__name__, self.name) + + __repr__ = __str__ + + def posix_error(filename): + errno = ctypes.get_errno() + exc = OSError(errno, strerror(errno)) + exc.filename = filename + return exc + + def scandir_python(path=unicode('.')): + """Like os.listdir(), but yield DirEntry objects instead of returning + a list of names. + """ + if isinstance(path, bytes): + opendir_path = path + is_bytes = True + else: + opendir_path = path.encode(file_system_encoding) + is_bytes = False + dir_p = opendir(opendir_path) + if not dir_p: + raise posix_error(path) + try: + result = Dirent_p() + while True: + entry = Dirent() + if readdir_r(dir_p, entry, result): + raise posix_error(path) + if not result: + break + name = entry.d_name + if name not in (b'.', b'..'): + if not is_bytes: + name = name.decode(file_system_encoding) + yield PosixDirEntry(path, name, entry.d_type, entry.d_ino) + finally: + if closedir(dir_p): + raise posix_error(path) + + if _scandir is not None: + scandir_c = _scandir.scandir + DirEntry_c = _scandir.DirEntry + + if _scandir is not None: + scandir = scandir_c + DirEntry = DirEntry_c + elif ctypes is not None and have_dirent_d_type: + scandir = scandir_python + DirEntry = PosixDirEntry + else: + scandir = scandir_generic + DirEntry = GenericDirEntry + + +# Some other system -- no d_type or stat information +else: + scandir = scandir_generic + DirEntry = GenericDirEntry + + +def _walk(top, topdown=True, onerror=None, followlinks=False): + """Like Python 3.5's implementation of os.walk() -- faster than + the pre-Python 3.5 version as it uses scandir() internally. + """ + dirs = [] + nondirs = [] + + # We may not have read permission for top, in which case we can't + # get a list of the files the directory contains. os.walk + # always suppressed the exception then, rather than blow up for a + # minor reason when (say) a thousand readable directories are still + # left to visit. That logic is copied here. + try: + scandir_it = scandir(top) + except OSError as error: + if onerror is not None: + onerror(error) + return + + while True: + try: + try: + entry = next(scandir_it) + except StopIteration: + break + except OSError as error: + if onerror is not None: + onerror(error) + return + + try: + is_dir = entry.is_dir() + except OSError: + # If is_dir() raises an OSError, consider that the entry is not + # a directory, same behaviour than os.path.isdir(). + is_dir = False + + if is_dir: + dirs.append(entry.name) + else: + nondirs.append(entry.name) + + if not topdown and is_dir: + # Bottom-up: recurse into sub-directory, but exclude symlinks to + # directories if followlinks is False + if followlinks: + walk_into = True + else: + try: + is_symlink = entry.is_symlink() + except OSError: + # If is_symlink() raises an OSError, consider that the + # entry is not a symbolic link, same behaviour than + # os.path.islink(). + is_symlink = False + walk_into = not is_symlink + + if walk_into: + for entry in walk(entry.path, topdown, onerror, followlinks): + yield entry + + # Yield before recursion if going top down + if topdown: + yield top, dirs, nondirs + + # Recurse into sub-directories + for name in dirs: + new_path = join(top, name) + # Issue #23605: os.path.islink() is used instead of caching + # entry.is_symlink() result during the loop on os.scandir() because + # the caller can replace the directory entry during the "yield" + # above. + if followlinks or not islink(new_path): + for entry in walk(new_path, topdown, onerror, followlinks): + yield entry + else: + # Yield after recursion if going bottom up + yield top, dirs, nondirs + + +if IS_PY3 or sys.platform != 'win32': + walk = _walk +else: + # Fix for broken unicode handling on Windows on Python 2.x, see: + # https://github.com/benhoyt/scandir/issues/54 + file_system_encoding = sys.getfilesystemencoding() + + def walk(top, topdown=True, onerror=None, followlinks=False): + if isinstance(top, bytes): + top = top.decode(file_system_encoding) + return _walk(top, topdown, onerror, followlinks) diff --git a/robocorp-python-ls-core/src/robocorp_ls_core/libs/yaml_lib/__init__.py b/robocorp-python-ls-core/src/robocorp_ls_core/libs/yaml_lib/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/robocorp-python-ls-core/src/robocorp_ls_core/watchdog_wrapper.py b/robocorp-python-ls-core/src/robocorp_ls_core/watchdog_wrapper.py index 7e6659a9da..9907eb83ec 100644 --- a/robocorp-python-ls-core/src/robocorp_ls_core/watchdog_wrapper.py +++ b/robocorp-python-ls-core/src/robocorp_ls_core/watchdog_wrapper.py @@ -21,6 +21,18 @@ def __init__(self, path, recursive): self.path = path self.recursive = recursive + def __eq__(self, o): + if isinstance(o, PathInfo): + return o.path == self.path and o.recursive == self.recursive + + return False + + def __ne__(self, o): + return not self == o + + def __hash__(self, *args, **kwargs): + return hash(tuple(self.path, self.recursive)) + def _get_watchdog_lib_dir(): parent_dir = os.path.dirname(__file__) @@ -40,6 +52,35 @@ def _import_watchdog(): import watchdog # @UnusedImport +def _get_fsnotify_lib_dir(): + parent_dir = os.path.dirname(__file__) + fsnotify_dir = os.path.join(parent_dir, "libs", "fsnotify_lib") + if not os.path.exists(fsnotify_dir): + msg = "Expected: %s to exist.\nDetails:\n" % (fsnotify_dir,) + check = fsnotify_dir + while True: + dirname = os.path.dirname(check) + exists = os.path.exists(dirname) + msg += f"{dirname} exists: {exists}\n" + if exists: + msg += f"{dirname} contents: {os.listdir(dirname)}\n" + + if not dirname or dirname == check or exists: + break + check = dirname + + raise RuntimeError(msg) + return fsnotify_dir + + +def _import_fsnotify(): + try: + import fsnotify + except ImportError: + sys.path.append(_get_fsnotify_lib_dir()) + import fsnotify # @UnusedImport + + class _Notifier(threading.Thread): def __init__(self, callback, timeout): """ @@ -50,7 +91,7 @@ def __init__(self, callback, timeout): modifications of the same file will be sent as a single notification). """ threading.Thread.__init__(self) - self.name = "Watchdog _Notifier" + self.name = "FS Notifier Thread (_Notifier class)" self.daemon = True self._changes = set() @@ -94,7 +135,101 @@ def create_notifier(callback, timeout): return notifier -class _WatchList(object): +class _FSNotifyWatchList(object): + def __init__(self, new_tracked_paths, new_notifications, observer): + self._new_tracked_paths = new_tracked_paths + self._new_notifications = new_notifications + self._observer = weakref.ref(observer) + + def stop_tracking(self): + observer = self._observer() + if observer is not None and self._new_tracked_paths: + observer._stop_tracking(self._new_tracked_paths, self._new_notifications) + self._new_tracked_paths = [] + self._new_notifications = [] + + +class _FSNotifyObserver(threading.Thread): + def __init__(self, extensions): + threading.Thread.__init__(self) + import fsnotify + + self.name = "_FSNotifyObserver" + self.daemon = True + + self._disposed = threading.Event() + + if extensions is None: + extensions = () + + watcher = self._watcher = fsnotify.Watcher() + watcher.target_time_for_notification = 3.0 + watcher.target_time_for_single_scan = 3.0 + watcher.accepted_file_extensions = extensions + # Could be customizable... + watcher.ignored_dirs = { + ".git", + "__pycache__", + ".idea", + "node_modules", + ".metadata", + } + + self._all_paths_to_track = [] + self._lock = threading.Lock() + self._notifications = [] + + def dispose(self): + if not self._disposed.is_set(): + self._disposed.set() + self._watcher.dispose() + + def run(self): + while not self._disposed.is_set(): + for _change, src_path in self._watcher.iter_changes(): + lower = src_path.lower() + for path, on_change, call_args in self._notifications: + if lower.startswith(path): + on_change(src_path, *call_args) + + def _tracked_paths_set_on_thread(self): + with self._lock: + all_paths_to_track = self._all_paths_to_track[:] + self._watcher.set_tracked_paths(all_paths_to_track) + + def _stop_tracking(self, new_paths_to_track, new_notifications): + with self._lock: + for path in new_paths_to_track: + self._all_paths_to_track.remove(path) + for notification in new_notifications: + self._notifications.remove(notification) + threading.Thread(target=self._tracked_paths_set_on_thread).start() + + def notify_on_any_change(self, paths, on_change, call_args=()): + if self._disposed.is_set(): + return + + import fsnotify + + with self._lock: + new_paths_to_track = [] + new_notifications = [] + for path in paths: + tracked_path = fsnotify.TrackedPath(path.path, path.recursive) + new_paths_to_track.append(tracked_path) + new_notifications.append((path.path.lower(), on_change, call_args)) + + self._notifications.extend(new_notifications) + self._all_paths_to_track.extend(new_paths_to_track) + + threading.Thread(target=self._tracked_paths_set_on_thread).start() + if not self.is_alive(): + self.start() + + return _FSNotifyWatchList(new_paths_to_track, new_notifications, self) + + +class _WatchdogWatchList(object): def __init__(self, watches, observer): self.watches = watches self._observer = weakref.ref(observer) @@ -107,13 +242,13 @@ def stop_tracking(self): self.watches = [] -class _Observer(object): - def __init__(self): - _import_watchdog() +class _WatchdogObserver(object): + def __init__(self, extensions=None): from watchdog.observers import Observer self._observer = Observer() self._started = False + self._extensions = extensions def dispose(self): self._observer.stop() @@ -123,62 +258,6 @@ def _start(self): self._started = True self._observer.start() - def notify_on_extensions_change(self, paths, extensions, on_change, call_args=()): - """ - To be used as: - - notifier = create_notifier(callback=on_file_change, timeout=0.5) - - observer = create_observer() - - watch = observer.notify_on_extensions_change( - [PathInfo('a', recursive=True)], - ['.libspec'], - notifier.on_change - ) - ... - watch.stop_tracking() - - notifier.dispose() - observer.dispose() - - Multiple changes on the same file will be sent as a single change (if - the changes occur during the available timeout). - - :param list(PathInfo) paths: - :param list(str) extensions: - The file extensions that should be tracked. - """ - _import_watchdog() - - from watchdog.events import FileSystemEventHandler - - class _Handler(FileSystemEventHandler): - def __init__(self, extensions): - FileSystemEventHandler.__init__(self) - extensions = [ext.lower() for ext in extensions] - self._extensions = extensions - - def on_any_event(self, event): - if event.is_directory: - return None - for ext in self._extensions: - if event.src_path.lower().endswith(ext): - on_change(event.src_path, *call_args) - break - - handler = _Handler(extensions) - watches = [] - for path_info in paths: - watches.append( - self._observer.schedule( - handler, path_info.path, recursive=path_info.recursive - ) - ) - - self._start() - return _WatchList(watches, self._observer) - def notify_on_any_change(self, paths, on_change, call_args=()): """ To be used as: @@ -187,7 +266,7 @@ def notify_on_any_change(self, paths, on_change, call_args=()): observer = create_observer() - watch = observer.notify_on_extensions_change( + watch = observer.notify_on_any_change( [PathInfo('a', recursive=True)], notifier.on_change ) @@ -204,15 +283,21 @@ def notify_on_any_change(self, paths, on_change, call_args=()): :param list(str) extensions: The file extensions that should be tracked. """ - _import_watchdog() - from watchdog.events import FileSystemEventHandler + extensions = self._extensions + class _Handler(FileSystemEventHandler): def __init__(self,): FileSystemEventHandler.__init__(self) def on_any_event(self, event): + if extensions is not None: + for ext in extensions: + if event.src_path.endswith(ext): + break + else: + return # Note: notify on directory and file changes. on_change(event.src_path, *call_args) @@ -226,8 +311,21 @@ def on_any_event(self, event): ) self._start() - return _WatchList(watches, self._observer) + return _WatchdogWatchList(watches, self._observer) + + +def create_observer(backend, extensions): + """ + :param backend: + The backend to use. + 'fsnotify' or 'watchdog'. + """ + if backend == "watchdog": + _import_watchdog() + return _WatchdogObserver(extensions) + elif backend == "fsnotify": + _import_fsnotify() + return _FSNotifyObserver(extensions) -def create_observer(): - return _Observer() + raise AssertionError(f"Unhandled observer: {backend}") diff --git a/robocorp-python-ls-core/src/robocorp_ls_core/workspace.py b/robocorp-python-ls-core/src/robocorp_ls_core/workspace.py index 87aad569f3..e766cda81b 100644 --- a/robocorp-python-ls-core/src/robocorp_ls_core/workspace.py +++ b/robocorp-python-ls-core/src/robocorp_ls_core/workspace.py @@ -196,9 +196,6 @@ def wait_for_check_done(self, timeout): class _VirtualFS(object): def __init__(self, root_folder_path: str, extensions: List[str]): - # from robocorp_ls_core.watchdog_wrapper import create_notifier, create_observer - # from robocorp_ls_core.watchdog_wrapper import PathInfo - self.root_folder_path = root_folder_path self._dir_to_info: Dict[str, _DirInfo] = {} diff --git a/robocorp-python-ls-core/tests/robocorp_ls_core_tests/test_watchdog_wrapper.py b/robocorp-python-ls-core/tests/robocorp_ls_core_tests/test_watchdog_wrapper.py index 0f792e389d..604b1cf664 100644 --- a/robocorp-python-ls-core/tests/robocorp_ls_core_tests/test_watchdog_wrapper.py +++ b/robocorp-python-ls-core/tests/robocorp_ls_core_tests/test_watchdog_wrapper.py @@ -43,7 +43,8 @@ def test_watchdog_macos(): ) -def test_watchdog_all(tmpdir): +@pytest.mark.parametrize("backend", ["watchdog", "fsnotify"]) +def test_watchdog_all(tmpdir, backend): from robocorp_ls_core import watchdog_wrapper from robocorp_ls_core.watchdog_wrapper import PathInfo from robocorp_ls_core.unittest_tools.fixtures import wait_for_test_condition @@ -58,7 +59,7 @@ def on_change(filepath, *args): assert args == ("foo", "bar") notifier = watchdog_wrapper.create_notifier(on_change, timeout=0.1) - observer = watchdog_wrapper.create_observer() + observer = watchdog_wrapper.create_observer(backend, None) watch = observer.notify_on_any_change( [ @@ -96,7 +97,8 @@ def check1(): observer.dispose() -def test_watchdog_extensions(tmpdir): +@pytest.mark.parametrize("backend", ["watchdog", "fsnotify"]) +def test_watchdog_extensions(tmpdir, backend): from robocorp_ls_core import watchdog_wrapper from robocorp_ls_core.watchdog_wrapper import PathInfo from robocorp_ls_core.unittest_tools.fixtures import wait_for_test_condition @@ -111,14 +113,13 @@ def on_change(filepath, *args): assert args == ("foo", "bar") notifier = watchdog_wrapper.create_notifier(on_change, timeout=0.1) - observer = watchdog_wrapper.create_observer() + observer = watchdog_wrapper.create_observer(backend, (".libspec",)) - watch = observer.notify_on_extensions_change( + watch = observer.notify_on_any_change( [ PathInfo(tmpdir.join("dir_not_rec"), False), PathInfo(tmpdir.join("dir_rec"), True), ], - ["libspec"], notifier.on_change, call_args=("foo", "bar"), ) diff --git a/robotframework-ls/.settings/org.python.pydev.yaml b/robotframework-ls/.settings/org.python.pydev.yaml index 879dc1aa74..4e781fe687 100644 --- a/robotframework-ls/.settings/org.python.pydev.yaml +++ b/robotframework-ls/.settings/org.python.pydev.yaml @@ -15,7 +15,7 @@ MULTI_BLOCK_COMMENT_CHAR: '=' MULTI_BLOCK_COMMENT_SHOW_ONLY_CLASS_NAME: true MULTI_BLOCK_COMMENT_SHOW_ONLY_FUNCTION_NAME: true PYDEV_TEST_RUNNER: '2' -PYDEV_TEST_RUNNER_DEFAULT_PARAMETERS: --capture=no -W ignore::DeprecationWarning -n auto --tb=native -vv --force-regen +PYDEV_TEST_RUNNER_DEFAULT_PARAMETERS: --capture=no -W ignore::DeprecationWarning -n 0 --tb=native -vv --force-regen PYDEV_USE_PYUNIT_VIEW: true SAVE_ACTIONS_ONLY_ON_WORKSPACE_FILES: true SINGLE_BLOCK_COMMENT_ALIGN_RIGHT: true diff --git a/robotframework-ls/src/robotframework_ls/impl/libspec_manager.py b/robotframework-ls/src/robotframework_ls/impl/libspec_manager.py index 36626f6a68..8c0f711f4c 100644 --- a/robotframework-ls/src/robotframework_ls/impl/libspec_manager.py +++ b/robotframework-ls/src/robotframework_ls/impl/libspec_manager.py @@ -410,7 +410,18 @@ def __init__(self, builtin_libspec_dir=None, user_libspec_dir=None): self._main_thread = threading.current_thread() - self._observer = watchdog_wrapper.create_observer() + watch_impl = os.environ.get("ROBOTFRAMEWORK_LS_WATCH_IMPL", "watchdog") + if watch_impl not in ("watchdog", "fsnotify"): + log.info( + f"ROBOTFRAMEWORK_LS_WATCH_IMPL should be 'watchdog' or 'fsnotify'. Found: {watch_impl} (falling back to fsnotify)" + ) + # i.e.: the default is watchdog, so, if a different one is set, + # presumably the default is not ok, so, fall back to watchdog. + watch_impl = "fsnotify" + + self._observer = watchdog_wrapper.create_observer( + watch_impl, (".py", ".libspec") + ) self._file_changes_notifier = watchdog_wrapper.create_notifier( self._on_file_changed, timeout=0.5 diff --git a/robotframework-ls/tests/robotframework_ls_tests/test_libspec_manager.py b/robotframework-ls/tests/robotframework_ls_tests/test_libspec_manager.py index 8c217285fb..c2a4da24d0 100644 --- a/robotframework-ls/tests/robotframework_ls_tests/test_libspec_manager.py +++ b/robotframework-ls/tests/robotframework_ls_tests/test_libspec_manager.py @@ -103,7 +103,7 @@ def disallow_cached_create_libspec(*args, **kwargs): assert library_info is None libspec_manager._cached_create_libspec = original_cached_create_libspec - time.sleep(0.1) + time.sleep(0.2) path = Path(workspace_dir) / "check_lib.py" path.write_text( @@ -116,6 +116,7 @@ def method2(a:int): wait_for_condition( lambda: libspec_manager.get_library_info("check_lib") is not None, msg="Did not recreate library in the available timeout.", + timeout=15, )