Java WatchService, perform action on event using threads

Question

I can monitor a directory by registering cwith a WatchKey (there are tons of examples on the web) however this watcher catches every single event. Eg On windows If am monitoring the d:/temp dir and I create a new.txt file and rename it I get the following events.

ENTRY_CREATE: d:\temp\test\New Text Document.txt
ENTRY_MODIFY: d:\temp\test
ENTRY_DELETE: d:\temp\test\New Text Document.txt
ENTRY_CREATE: d:\temp\test\test.txt
ENTRY_MODIFY: d:\temp\test

I want to perform an action when the new file is created or updated. However I don't want the action to run 5 times in the above example.

My 1st Idea: As I only need to run the action (in this case a push to a private Git server) once every now an then (eg check every 10 seconds only if there are changes to the monitored directory and only then perform the push) I thought of having an object with a boolean parameter that I can get and set from within separate threads.

Now this works kinda ok (unless the gurus can help educated me as to why this is a terrible idea) The problem is that if a file event is caught during the SendToGit thread's operation and this operation completes it sets the "Found" parameter to false. Immediately thereafter one of the other events are caught (as in the example above) they will set the "Found" parameter to true again. This is not ideal as I will then run the SendToGit operation immediately again which will be unnecessary.

My 2nd Idea Investigate pausing the check for changes in the MonitorFolder thread until the SendToGit operation is complete (Ie Keep checking if the ChangesFound Found parameter has been set back to false. When this parameter is false start checking for changes again.

Questions

1. Is this an acceptable way to go or have I gone down a rabbit hole with no hope of return?
2. If I go down the road of my 2nd idea what happens if I am busy with the SendToGit operation and a change is made in the monitoring folder? I suspect that this will not be identified and I may miss changes.

The Rest of the code

ChangesFound.java

package com.acme;

public class ChangesFound {
    
    private boolean found = false;

    public boolean wereFound() {
        return this.found;
    }

    public void setFound(boolean commitToGit) {
        this.found = commitToGit;
    }
}

In my main app I start 2 threads.

1. MonitorFolder.java Monitors the directory and when Watcher events are found set the ChangesFound variable "found" to true.
2. SendToGit.java Every 10 seconds checks if the ChangesFound variable found is true and if it is performs the push. (or in this case just prints a message)

Here is my App that starts the threads:

package com.acme;

import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;

public class App {

    private static ChangesFound chg;
    
    public static void main(String[] args) throws IOException {

        String dirToMonitor = "D:/Temp";
        boolean recursive = true;
        chg = new ChangesFound();

        Runnable r = new SendToGit(chg);
        new Thread(r).start();

        Path dir = Paths.get(dirToMonitor);
        Runnable m = new MonitorFolder(chg, dir, recursive);
        new Thread(m).start();        
    }
}

SendToGit.java

package com.acme;

public class SendToGit implements Runnable {

    private ChangesFound changes;

    public SendToGit(ChangesFound chg) {
        changes = chg;
    }
    
    public void run() {

        while (true) {           
            try {
                Thread.sleep(10000);
            } catch (InterruptedException ex) {
                Thread.currentThread().interrupt();
            }

            System.out.println(java.time.LocalDateTime.now() + " [SendToGit] waking up.");

            if (changes.wereFound()) {
                System.out.println("\t***** CHANGES FOUND push to Git.");
                changes.setFound(false);
            } else {
                System.out.println("\t***** Nothing changed.");
            }
        }
    }
}

MonitorFolder.java (Apologies for the long class I only added this here in case it helps someone else.)

package com.acme;

import java.io.IOException;
import java.nio.file.FileSystems;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import static java.nio.file.LinkOption.NOFOLLOW_LINKS;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import static java.nio.file.StandardWatchEventKinds.ENTRY_CREATE;
import static java.nio.file.StandardWatchEventKinds.ENTRY_DELETE;
import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY;
import static java.nio.file.StandardWatchEventKinds.OVERFLOW;
import java.nio.file.WatchEvent;
import java.nio.file.WatchKey;
import java.nio.file.WatchService;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.HashMap;
import java.util.Map;

public class MonitorFolder implements Runnable  {
    
    private static WatchService watcher;
    private static Map<WatchKey, Path> keys;
    private static boolean recursive;
    private static boolean trace = false;
    private static boolean commitGit = false;
    private static ChangesFound changes;

    @SuppressWarnings("unchecked")
    static <T> WatchEvent<T> cast(WatchEvent<?> event) {
        return (WatchEvent<T>) event;
    }

    /**
     * Creates a WatchService and registers the given directory
     */
    MonitorFolder(ChangesFound chg, Path dir, boolean rec) throws IOException {
        changes = chg;
        watcher = FileSystems.getDefault().newWatchService();
        keys = new HashMap<WatchKey, Path>();
        recursive = rec;

        if (recursive) {
            System.out.format("[MonitorFolder] Scanning %s ...\n", dir);
            registerAll(dir);
            System.out.println("Done.");
        } else {
            register(dir);
        }

        // enable trace after initial registration
        this.trace = true;
    }

    /**
     * Register the given directory with the WatchService
     */
    private static void register(Path dir) throws IOException {
        WatchKey key = dir.register(watcher, ENTRY_CREATE, ENTRY_DELETE, ENTRY_MODIFY);
        if (trace) {
            Path prev = keys.get(key);
            if (prev == null) {
                System.out.format("register: %s\n", dir);
            } else {
                if (!dir.equals(prev)) {
                    System.out.format("update: %s -> %s\n", prev, dir);
                }
            }
        }
        keys.put(key, dir);
    }

    /**
     * Register the given directory, and all its sub-directories, with the
     * WatchService.
     */
    private static void registerAll(final Path start) throws IOException {
        // register directory and sub-directories
        Files.walkFileTree(start, new SimpleFileVisitor<Path>() {
            @Override
            public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)
                    throws IOException {
                register(dir);
                return FileVisitResult.CONTINUE;
            }
        });
    }

    /**
     * Process all events for keys queued to the watcher
     */
    public void run() {
        for (;;) {

            // wait for key to be signalled
            WatchKey key;
            try {
                key = watcher.take();
            } catch (InterruptedException x) {
                return;
            }

            Path dir = keys.get(key);
            if (dir == null) {
                System.err.println("WatchKey not recognized!!");
                continue;
            }

            for (WatchEvent<?> event : key.pollEvents()) {
                WatchEvent.Kind kind = event.kind();

                // TBD - provide example of how OVERFLOW event is handled
                if (kind == OVERFLOW) {
                    System.out.println("Something about Overflow");
                    continue;
                }

                // Context for directory entry event is the file name of entry
                WatchEvent<Path> ev = cast(event);
                Path name = ev.context();
                Path child = dir.resolve(name);

                // print out event and set ChangesFound object Found parameter to True
                System.out.format("[MonitorFolder] " + java.time.LocalDateTime.now() + " - %s: %s\n", event.kind().name(), child);
                changes.setFound(true);

                // if directory is created, and watching recursively, then
                // register it and its sub-directories
                if (recursive && (kind == ENTRY_CREATE)) {
                    try {
                        if (Files.isDirectory(child, NOFOLLOW_LINKS)) {
                            registerAll(child);
                        }
                    } catch (IOException x) {
                        // ignore to keep sample readbale
                    }
                }
            }

            // reset key and remove from set if directory no longer accessible
            boolean valid = key.reset();
            if (!valid) {
                keys.remove(key);

                // all directories are inaccessible
                if (keys.isEmpty()) {
                    System.out.println("keys.isEmpty");
                    break;
                }
            }
        }
    }
}

Answer 1

Both of your strategies will lead to issues because the Watch Service is very verbose and sends many messages when maybe one or two is actually needed to your downstream handling - so sometimes you may do unnecessary work or miss events.

When using WatchService you could collate multiple notifications together and pass on as ONE event listing a sets of recent deletes, creates and updates:

1. DELETE followed by CREATE => sent as UPDATE
2. CREATE followed by MODIFY => sent as CREATE
3. CREATE or MODIFY followed by DELETE => sent as DELETE

Instead of calling WatchService.take() and acting on each message, use WatchService.poll(timeout) and only when nothing is returned act on the union of preceeding set of events as one - not individually after each successful poll.

It is easier to decouple the problems as two components so that you don't repeat the WatchService code the next time you need it:

1. A watch manager which handles watch service + dir registrations and collates the duplicates to send to event listeners as ONE group
2. A Listener class to receive the group of changes and act on the set.

This example may help illustrate - see WatchExample which is the manager which sets up the registrations BUT passes on much fewer events to the callback defined by setListener . You could set up MonitorFolder like WatchExample to reduce the events discovered, and make your code in SendToGit as a Listener which is called on demand with the aggregated set of fileChange(deletes, creates, updates) .

public static void main(String[] args) throws IOException, InterruptedException {

    final List<Path> dirs = Arrays.stream(args).map(Path::of).map(Path::toAbsolutePath).collect(Collectors.toList());
    Kind<?> [] kinds = { StandardWatchEventKinds.ENTRY_CREATE, StandardWatchEventKinds.ENTRY_MODIFY, StandardWatchEventKinds.ENTRY_DELETE};

    // Should launch WatchExample PER Filesystem:
    WatchExample w = new WatchExample();
    w.setListener(WatchExample::fireEvents);

    for(Path dir : dirs)
        w.register(kinds, dir);

    // For 2 or more WatchExample use: new Thread(w[n]::run).start();
    w.run();
}

public class WatchExample implements Runnable {

    private final Set<Path> created = new LinkedHashSet<>();
    private final Set<Path> updated = new LinkedHashSet<>();
    private final Set<Path> deleted = new LinkedHashSet<>();

    private volatile boolean appIsRunning = true;
    // Decide how sensitive the polling is:
    private final int pollmillis = 100;
    private WatchService ws;

    private Listener listener = WatchExample::fireEvents;

    @FunctionalInterface
    interface Listener
    {
        public void fileChange(Set<Path> deleted, Set<Path> created, Set<Path> modified);
    }

    WatchExample() {
    }
    
    public void setListener(Listener listener) {
        this.listener = listener;
    }

    public void shutdown() {
        System.out.println("shutdown()");
        this.appIsRunning = false;
    }

    public void run() {
        System.out.println();
        System.out.println("run() START watch");
        System.out.println();

        try(WatchService autoclose = ws) {

            while(appIsRunning) {

                boolean hasPending = created.size() + updated.size() + deleted.size() > 0;
                System.out.println((hasPending ? "ws.poll("+pollmillis+")" : "ws.take()")+" as hasPending="+hasPending);

                // Use poll if last cycle has some events, as take() may block
                WatchKey wk = hasPending ? ws.poll(pollmillis,TimeUnit.MILLISECONDS) : ws.take();
                if (wk != null)  {
                    for (WatchEvent<?> event : wk.pollEvents()) {
                         Path parent = (Path) wk.watchable();
                         Path eventPath = (Path) event.context();
                         storeEvent(event.kind(), parent.resolve(eventPath));
                     }
                     boolean valid = wk.reset();
                     if (!valid) {
                         System.out.println("Check the path, dir may be deleted "+wk);
                     }
                }

                System.out.println("PENDING: cre="+created.size()+" mod="+updated.size()+" del="+deleted.size());

                // This only sends new notifications when there was NO event this cycle:
                if (wk == null && hasPending) {
                    listener.fileChange(deleted, created, updated);
                    deleted.clear();
                    created.clear();
                    updated.clear();
                }
            }
        }
        catch (InterruptedException e) {
            System.out.println("Watch was interrupted, sending final updates");
            fireEvents(deleted, created, updated);
        }
        catch (IOException e) {
            throw new UncheckedIOException(e);
        }

        System.out.println("run() END watch");
    }

    public void register(Kind<?> [] kinds, Path dir) throws IOException {
        System.out.println("register watch for "+dir);

        // If dirs are from different filesystems WatchService will give errors later
        if (this.ws == null) {
            ws = dir.getFileSystem().newWatchService();
        }
        dir.register(ws, kinds);
    }

    /**
     * Save event for later processing by event kind EXCEPT for:
     * <li>DELETE followed by CREATE           => store as MODIFY
     * <li>CREATE followed by MODIFY           => store as CREATE
     * <li>CREATE or MODIFY followed by DELETE => store as DELETE
     */
    private void
    storeEvent(Kind<?> kind, Path path) {
        System.out.println("STORE "+kind+" path:"+path);

        boolean cre = false;
        boolean mod = false;
        boolean del = kind == StandardWatchEventKinds.ENTRY_DELETE;

        if (kind == StandardWatchEventKinds.ENTRY_CREATE) {
            mod = deleted.contains(path);
            cre = !mod;
        }
        else if (kind == StandardWatchEventKinds.ENTRY_MODIFY) {
            cre = created.contains(path);
            mod = !cre;
        }
        addOrRemove(created, cre, path);
        addOrRemove(updated, mod, path);
        addOrRemove(deleted, del, path);
    }
    // Add or remove from the set:
    private static void addOrRemove(Set<Path> set, boolean add, Path path) {
        if (add) set.add(path);
        else     set.remove(path);
    }

    public static void fireEvents(Set<Path> deleted, Set<Path> created, Set<Path> modified) {
        System.out.println();
        System.out.println("fireEvents START");
        for (Path path : deleted)
            System.out.println("  DELETED: "+path);
        for (Path path : created)
            System.out.println("  CREATED: "+path);
        for (Path path : modified)
            System.out.println("  UPDATED: "+path);
        System.out.println("fireEvents END");
        System.out.println();
    }
}