diff --git a/inotify/adapters.py b/inotify/adapters.py index faa001f..ab88805 100644 --- a/inotify/adapters.py +++ b/inotify/adapters.py @@ -51,11 +51,12 @@ _STRUCT_HEADER_LENGTH = struct.calcsize(_HEADER_STRUCT_FORMAT) _IS_DEBUG = bool(int(os.environ.get('DEBUG', '0'))) - +#todo: we should have a master exception for the whole adapter class EventTimeoutException(Exception): pass +#todo: we should have a master exception for the whole adapter class TerminalEventException(Exception): def __init__(self, type_name, event): super(TerminalEventException, self).__init__(type_name) @@ -93,8 +94,12 @@ def __del__(self): _LOGGER.debug("Cleaning-up inotify.") os.close(self.__inotify_fd) - def add_watch(self, path_unicode, mask=inotify.constants.IN_ALL_EVENTS): + def add_watch(self, path_unicode, mask=inotify.constants.IN_ALL_EVENTS_WATCH): _LOGGER.debug("Adding watch: [%s]", path_unicode) + # todo: log a warning if user supplies something in mask that is not valid as mask (but silently ignored) + # defined IN_ALL_MASKVALS for that (should encourage user to understand when they are requesting something + # wrong and make it more likely that they log an issue if they have some valid mask value from sources + # which isn't implemented in PyInotify # todo: handle removes for same object (with possible different pathnames) # more outcome-oriented @@ -224,7 +229,7 @@ def _handle_inotify_event(self, wd): def event_gen( self, timeout_s=None, yield_nones=True, filter_predicate=None, - terminal_events=_DEFAULT_TERMINAL_EVENTS): + terminal_events=_DEFAULT_TERMINAL_EVENTS, mask=inotify.constants.IN_ALL_EVENTS): """Yield one event after another. If `timeout_s` is provided, we'll break when no event is received for that many seconds. """ @@ -285,7 +290,8 @@ def event_gen( elif type_name in terminal_events: raise TerminalEventException(type_name, e) - yield e + if header.mask & mask: + yield e if timeout_s is not None: time_since_event_s = time.time() - last_hit_s @@ -306,18 +312,22 @@ def __init__(self, mask=inotify.constants.IN_ALL_EVENTS, # No matter what we actually received as the mask, make sure we have # the minimum that we require to curate our list of watches. - # - # todo: we really should have two masks... the combined one (requested|needed) - # and the user specified mask for the events he wants to receive from tree... - # - # todo: we shouldn't allow the IN_ONESHOT, IN_MASK_* flags here - # while IN_DONT_FOLLOW, IN_EXCL_UNLINK possibly need special implementation + if mask & (inotify.constants.IN_MASK_CREATE | inotify.constants.IN_MASK_ADD | inotify.constants.IN_ONESHOT): + raise ValueError('mask must not contain IN_MASK_CREATE/IN_MASK_ADD/IN_ONESHOT for ' + self.__class__.__name__) + if mask & inotify.constants.IN_DONT_FOLLOW: + _LOGGER.info('IN_DONT_FOLLOW (or the opposite) currently not implemented for ' + self.__class__.__name__) + if mask & inotify.constants.IN_ONLYDIR: + _LOGGER.info('IN_ONLYDIR (or the opposite) currently not implemented for ' + self.__class__.__name__) + # if we would want to give user the opportunity to get only IS_DIR events this would need to be implemented + # in a dedicated way + self._consumer_mask = mask & (~inotify.constants.IN_ISDIR) self._mask = mask | \ - inotify.constants.IN_ISDIR | \ inotify.constants.IN_CREATE | \ inotify.constants.IN_MOVED_TO | \ inotify.constants.IN_DELETE | \ - inotify.constants.IN_MOVED_FROM + inotify.constants.IN_MOVED_FROM | \ + inotify.constants.IN_DELETE_SELF | \ + inotify.constants.IN_MOVE_SELF ignored_dirs_lookup = {} for parent, child in (os.path.split(ignored.rstrip('/')) for ignored in ignored_dirs): @@ -329,8 +339,25 @@ def __init__(self, mask=inotify.constants.IN_ALL_EVENTS, ignored_dirs_lookup[parent] = set((child,)) self._ignored_dirs = ignored_dirs_lookup + self._moved_out_dirs = {} + self._deleted_dirs = {} + self._top_level_watches = {} + self._i = Inotify(block_duration_s=block_duration_s) + def __directory_deleted(self, full_path): + self._i.remove_watch(full_path, superficial=True) + + def __directory_moved_out(self, full_path): + try: + self._i.remove_watch(full_path, superficial=False) + except inotify.calls.InotifyError as ex: + # for the unlikely case the moved diretory is deleted + # and automatically unregistered before we try to + # unregister.... + pass + + def event_gen(self, ignore_missing_new_folders=False, **kwargs): """This is a secondary generator that wraps the principal one, and adds/removes watches as directories are added/removed. @@ -340,6 +367,7 @@ def event_gen(self, ignore_missing_new_folders=False, **kwargs): `ignore_missing_new_folders`. """ + consumer_mask = self._consumer_mask for event in self._i.event_gen(**kwargs): if event is not None: (header, type_names, path, filename) = event @@ -347,12 +375,8 @@ def event_gen(self, ignore_missing_new_folders=False, **kwargs): if header.mask & inotify.constants.IN_ISDIR: full_path = os.path.join(path, filename) - if ( - (header.mask & inotify.constants.IN_MOVED_TO) or - (header.mask & inotify.constants.IN_CREATE) - ) and \ - ( - os.path.exists(full_path) is True or + if (header.mask & inotify.constants.IN_MOVED_TO)\ + or (header.mask & inotify.constants.IN_CREATE): # todo: as long as the "Path already being watche/not in watch list" warnings # instead of exceptions are in place, it should really be default to also log # only a warning if target folder does not exists in tree autodiscover mode. @@ -361,20 +385,15 @@ def event_gen(self, ignore_missing_new_folders=False, **kwargs): # to event_gen but to InotifyTree(s) constructor (at least set default there) # to not steal someones use case to specify this differently for each event_gen # call?? Even more this expression is simply wrong. - ignore_missing_new_folders is False - ) and \ - ( - path not in self._ignored_dirs or - filename not in self._ignored_dirs[path] - ): - _LOGGER.debug("A directory has been created. We're " - "adding a watch on it (because we're " - "being recursive): [%s]", full_path) + if (ignore_missing_new_folders is False or os.path.exists(full_path) is True)\ + and (path not in self._ignored_dirs or filename not in self._ignored_dirs[path]): + _LOGGER.debug("A directory has been created. We're " + "adding a watch on it (because we're " + "being recursive): [%s]", full_path) + self._load_tree(full_path) - self._load_tree(full_path) - - if header.mask & inotify.constants.IN_DELETE: + elif header.mask & inotify.constants.IN_DELETE: _LOGGER.debug("A directory has been removed. We're " "being recursive, but it would have " "automatically been deregistered: [%s]", @@ -385,10 +404,10 @@ def event_gen(self, ignore_missing_new_folders=False, **kwargs): # before the watch on the child disappeared # also we have to take in mind that the subdirectory could be on # ignore list (currently that is handled by the remove_watch but a - # debug message is emitted than what is not fine) + # debug message is emitted then what is not fine) # The watch would've already been cleaned-up internally. - self._i.remove_watch(full_path, superficial=True) + self.__directory_deleted(full_path) elif header.mask & inotify.constants.IN_MOVED_FROM: _LOGGER.debug("A directory has been renamed. We're " "being recursive, we will remove watch " @@ -401,15 +420,11 @@ def event_gen(self, ignore_missing_new_folders=False, **kwargs): # by the move) # also we have to take in mind that the subdirectory could be on # ignore list (currently that is handled by the exception handler) - try: - self._i.remove_watch(full_path, superficial=False) - except inotify.calls.InotifyError as ex: - # for the unlikely case the moved diretory is deleted - # and automatically unregistered before we try to - # unregister.... - pass - - yield event + self.__directory_moved_out(full_path) + if header.mask & consumer_mask: + yield event + else: + yield event @property def inotify(self): @@ -420,29 +435,38 @@ def _load_tree(self, path): # (events that are generated by the implementation and not inotify) for all # found objects so that consumers do not need to scan directories again to # generate themself an overview of the tree - def filter_dirs_add_watches_gen(inotify, mask, dirpath, subdirs, ignored_subdirs): - for subdir in subdirs: - if subdir in ignored_subdirs: - continue - # todo: this add_watch should include the IN_ONLYDIR flag - inotify.add_watch(os.path.join(dirpath, subdir), mask) - yield subdir - - inotify = self._i - mask = self._mask - # todo: this add_watch should include the IN_ONLYDIR flag - inotify.add_watch(path, mask) + + i = self._i + mask = self._mask | inotify.constants.IN_ONLYDIR + wd = i.add_watch(path, mask) + added_watches = [(path, wd)] ignored_dirs = self._ignored_dirs - # todo: check wheter and how to handle symlinks to directories + # todo: check whether and how to handle symlinks to directories for dirpath, subdirs, _f in walk(path): - ignored_subdirs = ignored_dirs.get(dirpath) - if ignored_subdirs: - subdirs[:] = filter_dirs_add_watches_gen(inotify, mask, dirpath, subdirs, ignored_subdirs) - continue - for subdir in subdirs: - # todo: this add_watch should include the IN_ONLYDIR flag - inotify.add_watch(os.path.join(dirpath, subdir), mask) + if subdirs: + num_subdirs = len(subdirs) + pos_subdirs = 0 + ignored_subdirs = ignored_dirs.get(dirpath) + if ignored_subdirs: + while pos_subdirs < num_subdirs: + subdir = subdirs[pos_subdirs] + if subdir in ignored_subdirs: + del subdirs[pos_subdirs] + num_subdirs -= 1 + continue + path = os.path.join(dirpath, subdir) + wd = i.add_watch(path, mask) + added_watches.append((path, wd)) + pos_subdirs += 1 + else: + while pos_subdirs < num_subdirs: + subdir = subdirs[pos_subdirs] + path = os.path.join(dirpath, subdir) + wd = i.add_watch(path, mask) + added_watches.append((path, wd)) + pos_subdirs += 1 + return added_watches class InotifyTree(_BaseTree): """Recursively watch a path.""" @@ -456,7 +480,8 @@ def __init__(self, path, mask=inotify.constants.IN_ALL_EVENTS, def __load_tree(self, path): _LOGGER.debug("Adding initial watches on tree: [%s]", path) - self._load_tree(path) + tl_watch_path, tl_watch_desc = self._load_tree(path)[0] + self._top_level_watches[tl_watch_path] = tl_watch_desc class InotifyTrees(_BaseTree): @@ -472,4 +497,5 @@ def __init__(self, paths, mask=inotify.constants.IN_ALL_EVENTS, def __load_trees(self, paths): _LOGGER.debug("Adding initial watches on trees: [%s]", ",".join(map(str, paths))) for path in paths: - self._load_tree(path) + tl_watch_path, tl_watch_desc = self._load_tree(path)[0] + self._top_level_watches[tl_watch_path] = tl_watch_desc diff --git a/inotify/constants.py b/inotify/constants.py index f350a60..75fa664 100644 --- a/inotify/constants.py +++ b/inotify/constants.py @@ -25,9 +25,9 @@ ## All events which a program can wait on. -IN_ALL_EVENTS = (IN_ACCESS | IN_MODIFY | IN_ATTRIB | IN_CLOSE_WRITE | - IN_CLOSE_NOWRITE | IN_OPEN | IN_MOVED_FROM | IN_MOVED_TO | - IN_CREATE | IN_DELETE | IN_DELETE_SELF | IN_MOVE_SELF) +IN_ALL_EVENTS_WATCH = (IN_ACCESS | IN_MODIFY | IN_ATTRIB | IN_CLOSE_WRITE | + IN_CLOSE_NOWRITE | IN_OPEN | IN_MOVED_FROM | IN_MOVED_TO | + IN_CREATE | IN_DELETE | IN_DELETE_SELF | IN_MOVE_SELF) ## Events sent by kernel. @@ -52,7 +52,11 @@ ## All events sent by kernel -X_IN_ALL_EVENTS = (IN_ALL_EVENTS | IN_UNMOUNT | IN_Q_OVERFLOW | IN_IGNORED) +IN_ALL_EVENTS = (IN_ALL_EVENTS_WATCH | IN_UNMOUNT | IN_Q_OVERFLOW | IN_IGNORED) + +## All flags that can sensefully be used with add_watch + +IN_ALL_MASKVALS = (IN_ALL_EVENTS_WATCH | IN_ONLYDIR | IN_DONT_FOLLOW | IN_EXCL_UNLINK | IN_MASK_CREATE | IN_MASK_ADD | IN_ONESHOT) MASK_LOOKUP = { 0o2000000: 'IN_CLOEXEC', @@ -95,5 +99,5 @@ # things can change... (IN_ISDIR/IN_ACCESS is also "unspecified" # but happens on some (newer?) kernels) MASK_LOOKUP_COMB = dict(((em|dm, [en]+dn) - for em, en in MASK_LOOKUP.items() if em & X_IN_ALL_EVENTS + for em, en in MASK_LOOKUP.items() if em & IN_ALL_EVENTS for dm, dn in ((0, []), (IN_ISDIR, ['IN_ISDIR'])))) diff --git a/tests/test_inotify.py b/tests/test_inotify.py index aefdedd..2ca1c96 100644 --- a/tests/test_inotify.py +++ b/tests/test_inotify.py @@ -356,8 +356,12 @@ def test__cycle(self): else: expected += [ (inotify.adapters._INOTIFY_EVENT(wd=1, mask=1073742336, cookie=0, len=16), ['IN_DELETE', 'IN_ISDIR'], path, 'aa'), - (inotify.adapters._INOTIFY_EVENT(wd=1, mask=1073742336, cookie=0, len=16), ['IN_DELETE', 'IN_ISDIR'], path, 'bb'), + (inotify.adapters._INOTIFY_EVENT(wd=w2, mask=1024, cookie=0, len=0), ['IN_DELETE_SELF'], path1, ''), + (inotify.adapters._INOTIFY_EVENT(wd=w2, mask=32768, cookie=0, len=0), ['IN_IGNORED'], path1, ''), + + (inotify.adapters._INOTIFY_EVENT(wd=w3, mask=1024, cookie=0, len=0), ['IN_DELETE_SELF'], path2, ''), + (inotify.adapters._INOTIFY_EVENT(wd=w3, mask=32768, cookie=0, len=0), ['IN_IGNORED'], path2, ''), ] self.assertEquals(events, expected) @@ -469,6 +473,8 @@ def test__renames(self): else: expected += [ (inotify.adapters._INOTIFY_EVENT(wd=1, mask=1073742336, cookie=0, len=16), ['IN_DELETE', 'IN_ISDIR'], path, 'new_folder'), + (inotify.adapters._INOTIFY_EVENT(wd=3, mask=1024, cookie=0, len=0), ['IN_DELETE_SELF'], new_path, ''), + (inotify.adapters._INOTIFY_EVENT(wd=3, mask=32768, cookie=0, len=0), ['IN_IGNORED'], new_path, ''), ] self.assertEquals(events3, expected)