Skip to content

fix handling disappearing subdirs #29

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Elias481 wants to merge 4 commits into
base: pending-fixes--mergeprep
Choose a base branch
from
Open

fix handling disappearing subdirs #29

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0f1bccf
we want consisten behaviour of tree-watcher (so be able to see the re…
May 4, 2019
615f944
minor improvement of logic
May 5, 2019
3df0eca
preparations (issue not mitigated)
May 8, 2019
8b87b25
adjusted test to expected outcome
May 8, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
152 changes: 89 additions & 63 deletions inotify/adapters.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,11 +51,12 @@
_STRUCT_HEADER_LENGTH = struct.calcsize(_HEADER_STRUCT_FORMAT)
_IS_DEBUG = bool(int(os.environ.get('DEBUG', '0')))


#todo: we should have a master exception for the whole adapter
class EventTimeoutException(Exception):
pass


#todo: we should have a master exception for the whole adapter
class TerminalEventException(Exception):
def __init__(self, type_name, event):
super(TerminalEventException, self).__init__(type_name)
Expand Down Expand Up @@ -93,8 +94,12 @@ def __del__(self):
_LOGGER.debug("Cleaning-up inotify.")
os.close(self.__inotify_fd)

def add_watch(self, path_unicode, mask=inotify.constants.IN_ALL_EVENTS):
def add_watch(self, path_unicode, mask=inotify.constants.IN_ALL_EVENTS_WATCH):
_LOGGER.debug("Adding watch: [%s]", path_unicode)
# todo: log a warning if user supplies something in mask that is not valid as mask (but silently ignored)
# defined IN_ALL_MASKVALS for that (should encourage user to understand when they are requesting something
# wrong and make it more likely that they log an issue if they have some valid mask value from sources
# which isn't implemented in PyInotify

# todo: handle removes for same object (with possible different pathnames)
# more outcome-oriented
Expand Down Expand Up @@ -224,7 +229,7 @@ def _handle_inotify_event(self, wd):

def event_gen(
self, timeout_s=None, yield_nones=True, filter_predicate=None,
terminal_events=_DEFAULT_TERMINAL_EVENTS):
terminal_events=_DEFAULT_TERMINAL_EVENTS, mask=inotify.constants.IN_ALL_EVENTS):
"""Yield one event after another. If `timeout_s` is provided, we'll
break when no event is received for that many seconds.
"""
Expand Down Expand Up @@ -285,7 +290,8 @@ def event_gen(
elif type_name in terminal_events:
raise TerminalEventException(type_name, e)

yield e
if header.mask & mask:
yield e

if timeout_s is not None:
time_since_event_s = time.time() - last_hit_s
Expand All @@ -306,18 +312,22 @@ def __init__(self, mask=inotify.constants.IN_ALL_EVENTS,

# No matter what we actually received as the mask, make sure we have
# the minimum that we require to curate our list of watches.
#
# todo: we really should have two masks... the combined one (requested|needed)
# and the user specified mask for the events he wants to receive from tree...
#
# todo: we shouldn't allow the IN_ONESHOT, IN_MASK_* flags here
# while IN_DONT_FOLLOW, IN_EXCL_UNLINK possibly need special implementation
if mask & (inotify.constants.IN_MASK_CREATE | inotify.constants.IN_MASK_ADD | inotify.constants.IN_ONESHOT):
raise ValueError('mask must not contain IN_MASK_CREATE/IN_MASK_ADD/IN_ONESHOT for ' + self.__class__.__name__)
if mask & inotify.constants.IN_DONT_FOLLOW:
_LOGGER.info('IN_DONT_FOLLOW (or the opposite) currently not implemented for ' + self.__class__.__name__)
if mask & inotify.constants.IN_ONLYDIR:
_LOGGER.info('IN_ONLYDIR (or the opposite) currently not implemented for ' + self.__class__.__name__)
# if we would want to give user the opportunity to get only IS_DIR events this would need to be implemented
# in a dedicated way
self._consumer_mask = mask & (~inotify.constants.IN_ISDIR)
self._mask = mask | \
inotify.constants.IN_ISDIR | \
inotify.constants.IN_CREATE | \
inotify.constants.IN_MOVED_TO | \
inotify.constants.IN_DELETE | \
inotify.constants.IN_MOVED_FROM
inotify.constants.IN_MOVED_FROM | \
inotify.constants.IN_DELETE_SELF | \
inotify.constants.IN_MOVE_SELF

ignored_dirs_lookup = {}
for parent, child in (os.path.split(ignored.rstrip('/')) for ignored in ignored_dirs):
Expand All @@ -329,8 +339,25 @@ def __init__(self, mask=inotify.constants.IN_ALL_EVENTS,
ignored_dirs_lookup[parent] = set((child,))
self._ignored_dirs = ignored_dirs_lookup

self._moved_out_dirs = {}
self._deleted_dirs = {}
self._top_level_watches = {}

self._i = Inotify(block_duration_s=block_duration_s)

def __directory_deleted(self, full_path):
self._i.remove_watch(full_path, superficial=True)

def __directory_moved_out(self, full_path):
try:
self._i.remove_watch(full_path, superficial=False)
except inotify.calls.InotifyError as ex:
# for the unlikely case the moved diretory is deleted
# and automatically unregistered before we try to
# unregister....
pass


def event_gen(self, ignore_missing_new_folders=False, **kwargs):
"""This is a secondary generator that wraps the principal one, and
adds/removes watches as directories are added/removed.
Expand All @@ -340,19 +367,16 @@ def event_gen(self, ignore_missing_new_folders=False, **kwargs):
`ignore_missing_new_folders`.
"""

consumer_mask = self._consumer_mask
for event in self._i.event_gen(**kwargs):
if event is not None:
(header, type_names, path, filename) = event

if header.mask & inotify.constants.IN_ISDIR:
full_path = os.path.join(path, filename)

if (
(header.mask & inotify.constants.IN_MOVED_TO) or
(header.mask & inotify.constants.IN_CREATE)
) and \
(
os.path.exists(full_path) is True or
if (header.mask & inotify.constants.IN_MOVED_TO)\
or (header.mask & inotify.constants.IN_CREATE):
# todo: as long as the "Path already being watche/not in watch list" warnings
# instead of exceptions are in place, it should really be default to also log
# only a warning if target folder does not exists in tree autodiscover mode.
Expand All @@ -361,20 +385,15 @@ def event_gen(self, ignore_missing_new_folders=False, **kwargs):
# to event_gen but to InotifyTree(s) constructor (at least set default there)
# to not steal someones use case to specify this differently for each event_gen
# call?? Even more this expression is simply wrong.
ignore_missing_new_folders is False
) and \
(
path not in self._ignored_dirs or
filename not in self._ignored_dirs[path]
):
_LOGGER.debug("A directory has been created. We're "
"adding a watch on it (because we're "
"being recursive): [%s]", full_path)
if (ignore_missing_new_folders is False or os.path.exists(full_path) is True)\
and (path not in self._ignored_dirs or filename not in self._ignored_dirs[path]):
_LOGGER.debug("A directory has been created. We're "
"adding a watch on it (because we're "
"being recursive): [%s]", full_path)

self._load_tree(full_path)

self._load_tree(full_path)

if header.mask & inotify.constants.IN_DELETE:
elif header.mask & inotify.constants.IN_DELETE:
_LOGGER.debug("A directory has been removed. We're "
"being recursive, but it would have "
"automatically been deregistered: [%s]",
Expand All @@ -385,10 +404,10 @@ def event_gen(self, ignore_missing_new_folders=False, **kwargs):
# before the watch on the child disappeared
# also we have to take in mind that the subdirectory could be on
# ignore list (currently that is handled by the remove_watch but a
# debug message is emitted than what is not fine)
# debug message is emitted then what is not fine)

# The watch would've already been cleaned-up internally.
self._i.remove_watch(full_path, superficial=True)
self.__directory_deleted(full_path)
elif header.mask & inotify.constants.IN_MOVED_FROM:
_LOGGER.debug("A directory has been renamed. We're "
"being recursive, we will remove watch "
Expand All @@ -401,15 +420,11 @@ def event_gen(self, ignore_missing_new_folders=False, **kwargs):
# by the move)
# also we have to take in mind that the subdirectory could be on
# ignore list (currently that is handled by the exception handler)
try:
self._i.remove_watch(full_path, superficial=False)
except inotify.calls.InotifyError as ex:
# for the unlikely case the moved diretory is deleted
# and automatically unregistered before we try to
# unregister....
pass

yield event
self.__directory_moved_out(full_path)
if header.mask & consumer_mask:
yield event
else:
yield event

@property
def inotify(self):
Expand All @@ -420,29 +435,38 @@ def _load_tree(self, path):
# (events that are generated by the implementation and not inotify) for all
# found objects so that consumers do not need to scan directories again to
# generate themself an overview of the tree
def filter_dirs_add_watches_gen(inotify, mask, dirpath, subdirs, ignored_subdirs):
for subdir in subdirs:
if subdir in ignored_subdirs:
continue
# todo: this add_watch should include the IN_ONLYDIR flag
inotify.add_watch(os.path.join(dirpath, subdir), mask)
yield subdir

inotify = self._i
mask = self._mask
# todo: this add_watch should include the IN_ONLYDIR flag
inotify.add_watch(path, mask)

i = self._i
mask = self._mask | inotify.constants.IN_ONLYDIR
wd = i.add_watch(path, mask)
added_watches = [(path, wd)]
ignored_dirs = self._ignored_dirs

# todo: check wheter and how to handle symlinks to directories
# todo: check whether and how to handle symlinks to directories
for dirpath, subdirs, _f in walk(path):
ignored_subdirs = ignored_dirs.get(dirpath)
if ignored_subdirs:
subdirs[:] = filter_dirs_add_watches_gen(inotify, mask, dirpath, subdirs, ignored_subdirs)
continue
for subdir in subdirs:
# todo: this add_watch should include the IN_ONLYDIR flag
inotify.add_watch(os.path.join(dirpath, subdir), mask)
if subdirs:
num_subdirs = len(subdirs)
pos_subdirs = 0
ignored_subdirs = ignored_dirs.get(dirpath)
if ignored_subdirs:
while pos_subdirs < num_subdirs:
subdir = subdirs[pos_subdirs]
if subdir in ignored_subdirs:
del subdirs[pos_subdirs]
num_subdirs -= 1
continue
path = os.path.join(dirpath, subdir)
wd = i.add_watch(path, mask)
added_watches.append((path, wd))
pos_subdirs += 1
else:
while pos_subdirs < num_subdirs:
subdir = subdirs[pos_subdirs]
path = os.path.join(dirpath, subdir)
wd = i.add_watch(path, mask)
added_watches.append((path, wd))
pos_subdirs += 1
return added_watches

class InotifyTree(_BaseTree):
"""Recursively watch a path."""
Expand All @@ -456,7 +480,8 @@ def __init__(self, path, mask=inotify.constants.IN_ALL_EVENTS,

def __load_tree(self, path):
_LOGGER.debug("Adding initial watches on tree: [%s]", path)
self._load_tree(path)
tl_watch_path, tl_watch_desc = self._load_tree(path)[0]
self._top_level_watches[tl_watch_path] = tl_watch_desc


class InotifyTrees(_BaseTree):
Expand All @@ -472,4 +497,5 @@ def __init__(self, paths, mask=inotify.constants.IN_ALL_EVENTS,
def __load_trees(self, paths):
_LOGGER.debug("Adding initial watches on trees: [%s]", ",".join(map(str, paths)))
for path in paths:
self._load_tree(path)
tl_watch_path, tl_watch_desc = self._load_tree(path)[0]
self._top_level_watches[tl_watch_path] = tl_watch_desc
14 changes: 9 additions & 5 deletions inotify/constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,9 @@

## All events which a program can wait on.

IN_ALL_EVENTS = (IN_ACCESS | IN_MODIFY | IN_ATTRIB | IN_CLOSE_WRITE |
IN_CLOSE_NOWRITE | IN_OPEN | IN_MOVED_FROM | IN_MOVED_TO |
IN_CREATE | IN_DELETE | IN_DELETE_SELF | IN_MOVE_SELF)
IN_ALL_EVENTS_WATCH = (IN_ACCESS | IN_MODIFY | IN_ATTRIB | IN_CLOSE_WRITE |
IN_CLOSE_NOWRITE | IN_OPEN | IN_MOVED_FROM | IN_MOVED_TO |
IN_CREATE | IN_DELETE | IN_DELETE_SELF | IN_MOVE_SELF)

## Events sent by kernel.

Expand All @@ -52,7 +52,11 @@

## All events sent by kernel

X_IN_ALL_EVENTS = (IN_ALL_EVENTS | IN_UNMOUNT | IN_Q_OVERFLOW | IN_IGNORED)
IN_ALL_EVENTS = (IN_ALL_EVENTS_WATCH | IN_UNMOUNT | IN_Q_OVERFLOW | IN_IGNORED)

## All flags that can sensefully be used with add_watch

IN_ALL_MASKVALS = (IN_ALL_EVENTS_WATCH | IN_ONLYDIR | IN_DONT_FOLLOW | IN_EXCL_UNLINK | IN_MASK_CREATE | IN_MASK_ADD | IN_ONESHOT)

MASK_LOOKUP = {
0o2000000: 'IN_CLOEXEC',
Expand Down Expand Up @@ -95,5 +99,5 @@
# things can change... (IN_ISDIR/IN_ACCESS is also "unspecified"
# but happens on some (newer?) kernels)
MASK_LOOKUP_COMB = dict(((em|dm, [en]+dn)
for em, en in MASK_LOOKUP.items() if em & X_IN_ALL_EVENTS
for em, en in MASK_LOOKUP.items() if em & IN_ALL_EVENTS
for dm, dn in ((0, []), (IN_ISDIR, ['IN_ISDIR']))))
8 changes: 7 additions & 1 deletion tests/test_inotify.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -356,8 +356,12 @@ def test__cycle(self):
else:
expected += [
(inotify.adapters._INOTIFY_EVENT(wd=1, mask=1073742336, cookie=0, len=16), ['IN_DELETE', 'IN_ISDIR'], path, 'aa'),

(inotify.adapters._INOTIFY_EVENT(wd=1, mask=1073742336, cookie=0, len=16), ['IN_DELETE', 'IN_ISDIR'], path, 'bb'),
(inotify.adapters._INOTIFY_EVENT(wd=w2, mask=1024, cookie=0, len=0), ['IN_DELETE_SELF'], path1, ''),
(inotify.adapters._INOTIFY_EVENT(wd=w2, mask=32768, cookie=0, len=0), ['IN_IGNORED'], path1, ''),

(inotify.adapters._INOTIFY_EVENT(wd=w3, mask=1024, cookie=0, len=0), ['IN_DELETE_SELF'], path2, ''),
(inotify.adapters._INOTIFY_EVENT(wd=w3, mask=32768, cookie=0, len=0), ['IN_IGNORED'], path2, ''),
]

self.assertEquals(events, expected)
Expand Down Expand Up @@ -469,6 +473,8 @@ def test__renames(self):
else:
expected += [
(inotify.adapters._INOTIFY_EVENT(wd=1, mask=1073742336, cookie=0, len=16), ['IN_DELETE', 'IN_ISDIR'], path, 'new_folder'),
(inotify.adapters._INOTIFY_EVENT(wd=3, mask=1024, cookie=0, len=0), ['IN_DELETE_SELF'], new_path, ''),
(inotify.adapters._INOTIFY_EVENT(wd=3, mask=32768, cookie=0, len=0), ['IN_IGNORED'], new_path, ''),
]

self.assertEquals(events3, expected)
Expand Down