diff --git a/spyder/plugins/projects/tests/test_plugin.py b/spyder/plugins/projects/tests/test_plugin.py
index 48c3f2007e5..0fa72864a3c 100644
--- a/spyder/plugins/projects/tests/test_plugin.py
+++ b/spyder/plugins/projects/tests/test_plugin.py
@@ -24,7 +24,6 @@
 
 # Local imports
 from spyder.app.cli_options import get_options
-from spyder.config.base import running_in_ci
 from spyder.config.manager import CONF
 import spyder.plugins.base
 from spyder.plugins.preferences.tests.conftest import MainWindowMock
@@ -368,8 +367,6 @@ def test_project_explorer_tree_root(projects, tmpdir, qtbot):
 
 
 @flaky(max_runs=5)
-@pytest.mark.skipif(sys.platform == 'darwin', reason="Fails on Mac")
-@pytest.mark.skipif(not running_in_ci(), reason="Hangs locally sometimes")
 def test_filesystem_notifications(qtbot, projects, tmpdir):
     """
     Test that filesystem notifications are emitted when creating,
@@ -377,6 +374,7 @@ def test_filesystem_notifications(qtbot, projects, tmpdir):
     """
     # Create a directory for the project and some files.
     project_root = tmpdir.mkdir('project0')
+    git_folder = project_root.mkdir('.git')
     folder0 = project_root.mkdir('folder0')
     folder1 = project_root.mkdir('folder1')
     file0 = project_root.join('file0.txt')
@@ -450,15 +448,29 @@ def test_filesystem_notifications(qtbot, projects, tmpdir):
     deleted_folder, is_dir = blocker.args
     assert str(folder0) in deleted_folder
 
-    # For some reason this fails in macOS
-    if not sys.platform == 'darwin':
-        # Test file/folder modification
-        with qtbot.waitSignal(fs_handler.sig_file_modified,
-                              timeout=3000) as blocker:
-            file3.write('abc')
-
-        modified_file, is_dir = blocker.args
-        assert modified_file in str(file3)
+    # Test file/folder modification
+    with qtbot.waitSignal(fs_handler.sig_file_modified,
+                          timeout=3000) as blocker:
+        file3.write('abc')
+
+    modified_file, is_dir = blocker.args
+    assert modified_file in str(file3)
+
+    # Test events in hidden folders are not emitted
+    with qtbot.assertNotEmitted(fs_handler.sig_file_created, wait=2000):
+        git_file = git_folder.join('git_file.txt')
+        git_file.write("Some data")
+
+    # Test events in pycache folders are not emitted
+    with qtbot.assertNotEmitted(fs_handler.sig_file_created, wait=2000):
+        pycache_folder = project_root.mkdir('__pycache__')
+        pycache_file = pycache_folder.join("foo.pyc")
+        pycache_file.write("")
+
+    # Test events for files with binary extensions are not emitted
+    with qtbot.assertNotEmitted(fs_handler.sig_file_created, wait=2000):
+        png_file = project_root.join("binary.png")
+        png_file.write("")
 
 
 def test_loaded_and_closed_signals(create_projects, tmpdir, mocker, qtbot):
diff --git a/spyder/plugins/projects/utils/watcher.py b/spyder/plugins/projects/utils/watcher.py
index f23a2a563cf..5d1d6ea04fe 100644
--- a/spyder/plugins/projects/utils/watcher.py
+++ b/spyder/plugins/projects/utils/watcher.py
@@ -9,6 +9,7 @@
 # Standard lib imports
 import os
 import logging
+from pathlib import Path
 
 # Third-party imports
 from qtpy.QtCore import QObject, Signal
@@ -21,9 +22,20 @@
 from spyder.config.utils import get_edit_extensions
 
 
+# ---- Constants
+# -----------------------------------------------------------------------------
 logger = logging.getLogger(__name__)
 
+EDIT_EXTENSIONS = get_edit_extensions()
 
+FOLDERS_TO_IGNORE = [
+    "__pycache__",
+    "build",
+]
+
+
+# ---- Monkey patches
+# -----------------------------------------------------------------------------
 class BaseThreadWrapper(watchdog.utils.BaseThread):
     """
     Wrapper around watchdog BaseThread class.
@@ -50,6 +62,44 @@ def run_wrapper(self):
 watchdog.utils.BaseThread = BaseThreadWrapper
 
 
+# ---- Auxiliary functions
+# -----------------------------------------------------------------------------
+def ignore_entry(entry: os.DirEntry) -> bool:
+    """Check if an entry should be ignored."""
+    parts = Path(entry.path).parts
+
+    # Ignore files in hidden directories (e.g. .git)
+    if any([p.startswith(".") for p in parts]):
+        return True
+
+    # Ignore specific folders
+    for folder in FOLDERS_TO_IGNORE:
+        if folder in parts:
+            return True
+
+    return False
+
+
+def editable_file(entry: os.DirEntry) -> bool:
+    """Check if an entry file is editable."""
+    if entry.is_file():
+        return (os.path.splitext(entry.path)[1] in EDIT_EXTENSIONS)
+    return True
+
+
+def filter_scandir(path):
+    """
+    Filter entries from os.scandir that we're not interested in tracking in the
+    observer.
+    """
+    return (
+        entry for entry in os.scandir(path)
+        if (not ignore_entry(entry) and editable_file(entry))
+    )
+
+
+# ---- Event handler
+# -----------------------------------------------------------------------------
 class WorkspaceEventHandler(QObject, PatternMatchingEventHandler):
     """
     Event handler for watchdog notifications.
@@ -67,7 +117,7 @@ def __init__(self, parent=None):
         QObject.__init__(self, parent)
         PatternMatchingEventHandler.__init__(
             self,
-            patterns = [f"*{ext}" for ext in get_edit_extensions()],
+            patterns=[f"*{ext}" for ext in EDIT_EXTENSIONS],
         )
 
     def fmt_is_dir(self, is_dir):
@@ -110,6 +160,8 @@ def dispatch(self, event):
             super().dispatch(event)
 
 
+# ---- Watcher
+# -----------------------------------------------------------------------------
 class WorkspaceWatcher(QObject):
     """
     Wrapper class around watchdog observer and notifier.
@@ -148,7 +200,9 @@ def start(self, workspace_folder):
         #   openmsi/openmsistream#56).
         # * There doesn't seem to be issues on Mac, but it's simpler to use a
         #   single observer for all OSes.
-        self.observer = PollingObserverVFS(stat=os.stat, listdir=os.scandir)
+        self.observer = PollingObserverVFS(
+            stat=os.stat, listdir=filter_scandir
+        )
 
         self.observer.schedule(
             self.event_handler, workspace_folder, recursive=True