From 04b273a2280941f449b50e9d1882f5b1d4127376 Mon Sep 17 00:00:00 2001 From: blissful Date: Wed, 18 Oct 2023 10:10:19 -0400 Subject: [PATCH] implement watchdog --- conftest.py | 19 ++++++- flake.nix | 1 + pyproject.toml | 7 +++ rose/__main__.py | 9 ++++ rose/cache.py | 6 +++ rose/cache_test.py | 8 +-- rose/collages_test.py | 102 ++++++++++++------------------------- rose/watcher.py | 114 ++++++++++++++++++++++++++++++++++++++++++ rose/watcher_test.py | 88 ++++++++++++++++++++++++++++++++ 9 files changed, 276 insertions(+), 78 deletions(-) create mode 100644 rose/watcher.py create mode 100644 rose/watcher_test.py diff --git a/conftest.py b/conftest.py index 013a85f..f2a38e4 100644 --- a/conftest.py +++ b/conftest.py @@ -1,5 +1,6 @@ import hashlib import logging +import shutil import sqlite3 from collections.abc import Iterator from pathlib import Path @@ -8,11 +9,17 @@ import pytest from click.testing import CliRunner -from rose.cache import CACHE_SCHEMA_PATH +from rose.cache import CACHE_SCHEMA_PATH, update_cache from rose.config import Config logger = logging.getLogger(__name__) +TESTDATA = Path(__file__).resolve().parent / "testdata" / "cache" +TEST_RELEASE_1 = TESTDATA / "Test Release 1" +TEST_RELEASE_2 = TESTDATA / "Test Release 2" +TEST_RELEASE_3 = TESTDATA / "Test Release 3" +TEST_COLLAGE_1 = TESTDATA / "Collage 1" + @pytest.fixture(autouse=True) def debug_logging() -> None: @@ -130,6 +137,16 @@ def seeded_cache(config: Config) -> None: f.touch() +@pytest.fixture() +def source_dir(config: Config) -> Path: + shutil.copytree(TEST_RELEASE_1, config.music_source_dir / TEST_RELEASE_1.name) + shutil.copytree(TEST_RELEASE_2, config.music_source_dir / TEST_RELEASE_2.name) + shutil.copytree(TEST_RELEASE_3, config.music_source_dir / TEST_RELEASE_3.name) + shutil.copytree(TEST_COLLAGE_1, config.music_source_dir / "!collages") + update_cache(config) + return config.music_source_dir + + def freeze_database_time(conn: sqlite3.Connection) -> None: """ This function freezes the CURRENT_TIMESTAMP function in SQLite3 to diff --git a/flake.nix b/flake.nix index 12373de..6410099 100644 --- a/flake.nix +++ b/flake.nix @@ -34,6 +34,7 @@ setuptools tomli-w uuid6-python + watchdog ]; dev-deps = with python.pkgs; [ black diff --git a/pyproject.toml b/pyproject.toml index 71c5185..c92c055 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,6 +54,10 @@ ignore = [ ] line-length = 100 exclude = [".venv"] +unfixable = [ + # Remove unused variables. + "F841", +] src = ["."] [tool.mypy] @@ -71,6 +75,9 @@ ignore_missing_imports = true module = "send2trash" ignore_missing_imports = true [[tool.mypy.overrides]] +module = "watchdog.*" +ignore_missing_imports = true +[[tool.mypy.overrides]] module = "setuptools" ignore_missing_imports = true diff --git a/rose/__main__.py b/rose/__main__.py index 38ef511..7ee0854 100644 --- a/rose/__main__.py +++ b/rose/__main__.py @@ -15,6 +15,7 @@ from rose.config import Config from rose.releases import dump_releases from rose.virtualfs import mount_virtualfs, unmount_virtualfs +from rose.watcher import start_watchdog @dataclass @@ -56,6 +57,14 @@ def update(ctx: Context, force: bool) -> None: update_cache(ctx.config, force) +@cache.command() +@click.option("--foreground", "-f", is_flag=True, help="Foreground the cache watcher.") +@click.pass_obj +def watch(ctx: Context, foreground: bool) -> None: + """Start a watchdog that will auto-refresh the cache on changes in music_source_dir.""" + start_watchdog(ctx.config, foreground) + + @cli.group() def fs() -> None: """Manage the virtual library.""" diff --git a/rose/cache.py b/rose/cache.py index b43068c..7411bfc 100644 --- a/rose/cache.py +++ b/rose/cache.py @@ -225,6 +225,9 @@ def update_cache_for_releases( for rd in release_dirs: release_id = None files: list[os.DirEntry[str]] = [] + if not rd.is_dir(): + logger.debug(f"Skipping scanning {rd} because it is not a directory") + continue for f in os.scandir(str(rd)): if m := STORED_DATA_FILE_REGEX.match(f.name): release_id = m[1] @@ -837,6 +840,9 @@ def update_cache_for_collages( path = Path(f.path) if path.suffix != ".toml": continue + if not path.is_file(): + logger.debug(f"Skipping processing collage {path.name} because it is not a file") + continue if collage_names is None or path.stem in collage_names: files.append((path.resolve(), path.stem, f)) logger.info(f"Refreshing the read cache for {len(files)} collages") diff --git a/rose/cache_test.py b/rose/cache_test.py index f914f3f..86576ab 100644 --- a/rose/cache_test.py +++ b/rose/cache_test.py @@ -5,6 +5,7 @@ import pytest import tomllib +from conftest import TEST_COLLAGE_1, TEST_RELEASE_1, TEST_RELEASE_2 from rose.cache import ( CACHE_SCHEMA_PATH, STORED_DATA_FILE_REGEX, @@ -62,13 +63,6 @@ def test_migration(config: Config) -> None: assert cursor.fetchone()[0] == 1 -TESTDATA = Path(__file__).resolve().parent.parent / "testdata" / "cache" -TEST_RELEASE_1 = TESTDATA / "Test Release 1" -TEST_RELEASE_2 = TESTDATA / "Test Release 2" -TEST_RELEASE_3 = TESTDATA / "Test Release 3" -TEST_COLLAGE_1 = TESTDATA / "Collage 1" - - def test_update_cache_all(config: Config) -> None: """Test that the update all function works.""" shutil.copytree(TEST_RELEASE_1, config.music_source_dir / TEST_RELEASE_1.name) diff --git a/rose/collages_test.py b/rose/collages_test.py index d038747..c384497 100644 --- a/rose/collages_test.py +++ b/rose/collages_test.py @@ -1,11 +1,10 @@ -import shutil +from pathlib import Path from typing import Any import pytest import tomllib -from rose.cache import connect, update_cache -from rose.cache_test import TEST_COLLAGE_1, TEST_RELEASE_2, TEST_RELEASE_3 +from rose.cache import connect from rose.collages import ( add_release_to_collage, create_collage, @@ -20,20 +19,13 @@ # TODO: Fixture for common setup. -def test_delete_release_from_collage(config: Config) -> None: - # Set up the filesystem that will be updated. - shutil.copytree(TEST_RELEASE_2, config.music_source_dir / TEST_RELEASE_2.name) - shutil.copytree(TEST_RELEASE_3, config.music_source_dir / TEST_RELEASE_3.name) - shutil.copytree(TEST_COLLAGE_1, config.music_source_dir / "!collages") - # Bootstrap initial cache. - update_cache(config) - +def test_delete_release_from_collage(config: Config, source_dir: Path) -> None: delete_release_from_collage( config, "Rose Gold", "Carly Rae Jepsen - 1990. I Love Carly [Pop;Dream Pop] {A Cool Label}" ) # Assert file is updated. - with (config.music_source_dir / "!collages" / "Rose Gold.toml").open("rb") as fp: + with (source_dir / "!collages" / "Rose Gold.toml").open("rb") as fp: diskdata = tomllib.load(fp) assert len(diskdata["releases"]) == 1 assert diskdata["releases"][0]["uuid"] == "ilovenewjeans" @@ -47,94 +39,84 @@ def test_delete_release_from_collage(config: Config) -> None: assert ids == ["ilovenewjeans"] -def test_collage_lifecycle(config: Config) -> None: - # Set up the filesystem that will be updated. - shutil.copytree(TEST_RELEASE_2, config.music_source_dir / TEST_RELEASE_2.name) - shutil.copytree(TEST_RELEASE_3, config.music_source_dir / TEST_RELEASE_3.name) - # Bootstrap initial cache. - update_cache(config) - - filepath = config.music_source_dir / "!collages" / "Rose Gold.toml" +def test_collage_lifecycle(config: Config, source_dir: Path) -> None: + filepath = source_dir / "!collages" / "All Eyes.toml" # Create collage. assert not filepath.exists() - create_collage(config, "Rose Gold") + create_collage(config, "All Eyes") assert filepath.is_file() with connect(config) as conn: - cursor = conn.execute("SELECT EXISTS(SELECT * FROM collages WHERE name = 'Rose Gold')") + cursor = conn.execute("SELECT EXISTS(SELECT * FROM collages WHERE name = 'All Eyes')") assert cursor.fetchone()[0] # Add one release. add_release_to_collage( - config, "Rose Gold", "Carly Rae Jepsen - 1990. I Love Carly [Pop;Dream Pop] {A Cool Label}" + config, "All Eyes", "Carly Rae Jepsen - 1990. I Love Carly [Pop;Dream Pop] {A Cool Label}" ) with filepath.open("rb") as fp: diskdata = tomllib.load(fp) assert {r["uuid"] for r in diskdata["releases"]} == {"ilovecarly"} with connect(config) as conn: cursor = conn.execute( - "SELECT release_id FROM collages_releases WHERE collage_name = 'Rose Gold'" + "SELECT release_id FROM collages_releases WHERE collage_name = 'All Eyes'" ) assert {r["release_id"] for r in cursor} == {"ilovecarly"} # Add another release. add_release_to_collage( - config, "Rose Gold", "NewJeans - 1990. I Love NewJeans [K-Pop;R&B] {A Cool Label}" + config, "All Eyes", "NewJeans - 1990. I Love NewJeans [K-Pop;R&B] {A Cool Label}" ) - with (config.music_source_dir / "!collages" / "Rose Gold.toml").open("rb") as fp: + with (source_dir / "!collages" / "All Eyes.toml").open("rb") as fp: diskdata = tomllib.load(fp) assert {r["uuid"] for r in diskdata["releases"]} == {"ilovecarly", "ilovenewjeans"} with connect(config) as conn: cursor = conn.execute( - "SELECT release_id FROM collages_releases WHERE collage_name = 'Rose Gold'" + "SELECT release_id FROM collages_releases WHERE collage_name = 'All Eyes'" ) assert {r["release_id"] for r in cursor} == {"ilovecarly", "ilovenewjeans"} # Delete one release. delete_release_from_collage( - config, "Rose Gold", "NewJeans - 1990. I Love NewJeans [K-Pop;R&B] {A Cool Label}" + config, "All Eyes", "NewJeans - 1990. I Love NewJeans [K-Pop;R&B] {A Cool Label}" ) with filepath.open("rb") as fp: diskdata = tomllib.load(fp) assert {r["uuid"] for r in diskdata["releases"]} == {"ilovecarly"} with connect(config) as conn: cursor = conn.execute( - "SELECT release_id FROM collages_releases WHERE collage_name = 'Rose Gold'" + "SELECT release_id FROM collages_releases WHERE collage_name = 'All Eyes'" ) assert {r["release_id"] for r in cursor} == {"ilovecarly"} # And delete the collage. - delete_collage(config, "Rose Gold") + delete_collage(config, "All Eyes") assert not filepath.is_file() with connect(config) as conn: - cursor = conn.execute("SELECT EXISTS(SELECT * FROM collages WHERE name = 'Rose Gold')") + cursor = conn.execute("SELECT EXISTS(SELECT * FROM collages WHERE name = 'All Eyes')") assert not cursor.fetchone()[0] -def test_collage_add_duplicate(config: Config) -> None: - shutil.copytree(TEST_RELEASE_3, config.music_source_dir / TEST_RELEASE_3.name) - update_cache(config) - create_collage(config, "Rose Gold") +def test_collage_add_duplicate(config: Config, source_dir: Path) -> None: + create_collage(config, "All Eyes") add_release_to_collage( - config, "Rose Gold", "NewJeans - 1990. I Love NewJeans [K-Pop;R&B] {A Cool Label}" + config, "All Eyes", "NewJeans - 1990. I Love NewJeans [K-Pop;R&B] {A Cool Label}" ) add_release_to_collage( - config, "Rose Gold", "NewJeans - 1990. I Love NewJeans [K-Pop;R&B] {A Cool Label}" + config, "All Eyes", "NewJeans - 1990. I Love NewJeans [K-Pop;R&B] {A Cool Label}" ) - with (config.music_source_dir / "!collages" / "Rose Gold.toml").open("rb") as fp: + with (source_dir / "!collages" / "All Eyes.toml").open("rb") as fp: diskdata = tomllib.load(fp) assert len(diskdata["releases"]) == 1 with connect(config) as conn: - cursor = conn.execute("SELECT * FROM collages_releases WHERE collage_name = 'Rose Gold'") + cursor = conn.execute("SELECT * FROM collages_releases WHERE collage_name = 'All Eyes'") assert len(cursor.fetchall()) == 1 -def test_rename_collage(config: Config) -> None: - shutil.copytree(TEST_COLLAGE_1, config.music_source_dir / "!collages") +def test_rename_collage(config: Config, source_dir: Path) -> None: rename_collage(config, "Rose Gold", "Black Pink") - - assert not (config.music_source_dir / "!collages" / "Rose Gold.toml").exists() - assert (config.music_source_dir / "!collages" / "Black Pink.toml").exists() + assert not (source_dir / "!collages" / "Rose Gold.toml").exists() + assert (source_dir / "!collages" / "Black Pink.toml").exists() with connect(config) as conn: cursor = conn.execute("SELECT EXISTS(SELECT * FROM collages WHERE name = 'Black Pink')") assert cursor.fetchone()[0] @@ -150,19 +132,9 @@ def test_dump_collages(config: Config) -> None: # fmt: on -def test_edit_collages_ordering(monkeypatch: Any, config: Config) -> None: - # Set up the filesystem that will be updated. - shutil.copytree(TEST_RELEASE_2, config.music_source_dir / TEST_RELEASE_2.name) - shutil.copytree(TEST_RELEASE_3, config.music_source_dir / TEST_RELEASE_3.name) - shutil.copytree(TEST_COLLAGE_1, config.music_source_dir / "!collages") - # Bootstrap initial cache. - update_cache(config) - filepath = config.music_source_dir / "!collages" / "Rose Gold.toml" - - def mock_edit(x: str) -> str: - return "\n".join(reversed(x.split("\n"))) - - monkeypatch.setattr("rose.collages.click.edit", mock_edit) +def test_edit_collages_ordering(monkeypatch: Any, config: Config, source_dir: Path) -> None: + filepath = source_dir / "!collages" / "Rose Gold.toml" + monkeypatch.setattr("rose.collages.click.edit", lambda x: "\n".join(reversed(x.split("\n")))) edit_collage_in_editor(config, "Rose Gold") with filepath.open("rb") as fp: @@ -171,19 +143,9 @@ def mock_edit(x: str) -> str: assert data["releases"][1]["uuid"] == "ilovecarly" -def test_edit_collages_delete_release(monkeypatch: Any, config: Config) -> None: - # Set up the filesystem that will be updated. - shutil.copytree(TEST_RELEASE_2, config.music_source_dir / TEST_RELEASE_2.name) - shutil.copytree(TEST_RELEASE_3, config.music_source_dir / TEST_RELEASE_3.name) - shutil.copytree(TEST_COLLAGE_1, config.music_source_dir / "!collages") - # Bootstrap initial cache. - update_cache(config) - filepath = config.music_source_dir / "!collages" / "Rose Gold.toml" - - def mock_edit(x: str) -> str: - return x.split("\n")[0] - - monkeypatch.setattr("rose.collages.click.edit", mock_edit) +def test_edit_collages_delete_release(monkeypatch: Any, config: Config, source_dir: Path) -> None: + filepath = source_dir / "!collages" / "Rose Gold.toml" + monkeypatch.setattr("rose.collages.click.edit", lambda x: x.split("\n")[0]) edit_collage_in_editor(config, "Rose Gold") with filepath.open("rb") as fp: diff --git a/rose/watcher.py b/rose/watcher.py new file mode 100644 index 0000000..930c8c9 --- /dev/null +++ b/rose/watcher.py @@ -0,0 +1,114 @@ +import logging +from dataclasses import dataclass +from pathlib import Path + +from watchdog.events import ( + DirCreatedEvent, + DirDeletedEvent, + FileCreatedEvent, + FileDeletedEvent, + FileModifiedEvent, + FileSystemEventHandler, + FileSystemMovedEvent, +) +from watchdog.observers import Observer +from watchdog.observers.api import BaseObserver + +from rose.cache import ( + update_cache_evict_nonexistent_collages, + update_cache_evict_nonexistent_releases, + update_cache_for_collages, + update_cache_for_releases, +) +from rose.config import Config + +logger = logging.getLogger(__name__) + + +@dataclass +class AffectedEntity: + release: Path | None = None + collage: str | None = None + + +def parse_affected_entity(config: Config, path: str) -> AffectedEntity | None: + relative_path = path.removeprefix(str(config.music_source_dir) + "/") + if relative_path.startswith("!collages/"): + if not relative_path.endswith(".toml"): + return None + collage = relative_path.removeprefix("!collages/").removesuffix(".toml") + logger.debug(f"Parsed change event on collage {collage}") + return AffectedEntity(collage=collage) + try: + release_dir = config.music_source_dir / Path(relative_path).parts[0] + logger.debug(f"Parsed event on release {release_dir}") + return AffectedEntity(release=release_dir) + except IndexError: + return None + + +class EventHandler(FileSystemEventHandler): + def __init__(self, config: Config): + super().__init__() + self.config = config + + def on_created(self, event: FileCreatedEvent | DirCreatedEvent) -> None: + super().on_created(event) # type: ignore + logger.debug(f"Notified of change event for {event.src_path}") + affected = parse_affected_entity(self.config, event.src_path) + if not affected: + return + if affected.collage: + update_cache_for_collages(self.config, [affected.collage]) + elif affected.release: + update_cache_for_releases(self.config, [affected.release]) + + def on_deleted(self, event: FileDeletedEvent | DirDeletedEvent) -> None: + super().on_deleted(event) # type: ignore + logger.debug(f"Notified of change event for {event.src_path}") + affected = parse_affected_entity(self.config, event.src_path) + if not affected: + return + if affected.collage: + update_cache_evict_nonexistent_collages(self.config) + elif affected.release: + update_cache_evict_nonexistent_releases(self.config) + + def on_modified(self, event: FileModifiedEvent) -> None: + super().on_modified(event) # type: ignore + logger.debug(f"Notified of change event for {event.src_path}") + affected = parse_affected_entity(self.config, event.src_path) + if not affected: + return + if affected.collage: + update_cache_for_collages(self.config, [affected.collage]) + elif affected.release: + update_cache_for_releases(self.config, [affected.release]) + + def on_moved(self, event: FileSystemMovedEvent) -> None: + super().on_moved(event) # type: ignore + logger.debug(f"Notified of change event for {event.src_path}") + affected = parse_affected_entity(self.config, event.dest_path) + if not affected: + return + if affected.collage: + update_cache_for_collages(self.config, [affected.collage]) + update_cache_evict_nonexistent_collages(self.config) + elif affected.release: + update_cache_for_releases(self.config, [affected.release]) + update_cache_evict_nonexistent_releases(self.config) + + +def create_watchdog_observer(c: Config) -> BaseObserver: + observer = Observer() + event_handler = EventHandler(c) + observer.schedule(event_handler, c.music_source_dir, recursive=True) # type: ignore + return observer + + +def start_watchdog(c: Config, foreground: bool = False) -> None: # pragma: no cover + logger.info("Starting cache watchdog") + thread = create_watchdog_observer(c) + thread.start() + if foreground: + thread.join() diff --git a/rose/watcher_test.py b/rose/watcher_test.py new file mode 100644 index 0000000..fd87ae2 --- /dev/null +++ b/rose/watcher_test.py @@ -0,0 +1,88 @@ +import shutil +import time +from collections.abc import Iterator +from contextlib import contextmanager + +from conftest import TEST_COLLAGE_1, TEST_RELEASE_2, TEST_RELEASE_3 +from rose.cache import connect +from rose.config import Config +from rose.watcher import create_watchdog_observer + + +@contextmanager +def start_watcher(c: Config) -> Iterator[None]: + observer = create_watchdog_observer(c) + try: + observer.start() + time.sleep(0.05) + yield + finally: + observer.stop() + + +def test_watchdog_events(config: Config) -> None: + src = config.music_source_dir + with start_watcher(config): + # Create release. + shutil.copytree(TEST_RELEASE_2, src / TEST_RELEASE_2.name) + time.sleep(0.05) + with connect(config) as conn: + cursor = conn.execute("SELECT id FROM releases") + assert {r["id"] for r in cursor.fetchall()} == {"ilovecarly"} + + # Create another release. + shutil.copytree(TEST_RELEASE_3, src / TEST_RELEASE_3.name) + time.sleep(0.05) + with connect(config) as conn: + cursor = conn.execute("SELECT id FROM releases") + assert {r["id"] for r in cursor.fetchall()} == {"ilovecarly", "ilovenewjeans"} + + # Create collage. + shutil.copytree(TEST_COLLAGE_1, src / "!collages") + time.sleep(0.05) + with connect(config) as conn: + cursor = conn.execute("SELECT name FROM collages") + assert {r["name"] for r in cursor.fetchall()} == {"Rose Gold"} + cursor = conn.execute("SELECT release_id FROM collages_releases") + assert {r["release_id"] for r in cursor.fetchall()} == {"ilovecarly", "ilovenewjeans"} + + # Create/rename/delete random files; check that they don't interfere with rest of the test. + (src / "hi.nfo").touch() + (src / "hi.nfo").rename(src / "!collages" / "bye.haha") + (src / "!collages" / "bye.haha").unlink() + + # Delete release. + shutil.rmtree(src / TEST_RELEASE_3.name) + time.sleep(0.05) + with connect(config) as conn: + cursor = conn.execute("SELECT id FROM releases") + assert {r["id"] for r in cursor.fetchall()} == {"ilovecarly"} + cursor = conn.execute("SELECT release_id FROM collages_releases") + assert {r["release_id"] for r in cursor.fetchall()} == {"ilovecarly"} + + # Rename release. + (src / TEST_RELEASE_2.name).rename(src / "lalala") + time.sleep(0.05) + with connect(config) as conn: + cursor = conn.execute("SELECT id, source_path FROM releases") + rows = cursor.fetchall() + assert len(rows) == 1 + row = rows[0] + assert row["id"] == "ilovecarly" + assert row["source_path"] == str(src / "lalala") + + # Rename collage. + (src / "!collages" / "Rose Gold.toml").rename(src / "!collages" / "Black Pink.toml") + time.sleep(0.05) + with connect(config) as conn: + cursor = conn.execute("SELECT name FROM collages") + assert {r["name"] for r in cursor.fetchall()} == {"Black Pink"} + cursor = conn.execute("SELECT release_id FROM collages_releases") + assert {r["release_id"] for r in cursor.fetchall()} == {"ilovecarly"} + + # Delete collage. + (src / "!collages" / "Black Pink.toml").unlink() + time.sleep(0.05) + with connect(config) as conn: + cursor = conn.execute("SELECT COUNT(*) FROM collages") + assert cursor.fetchone()[0] == 0