prune dead members on startup (#79)

thumbor · Jul 21, 2023 · db87321 · db87321
1 parent 0888e45
commit db87321
Show file tree

Hide file tree

Showing 3 changed files with 40 additions and 0 deletions.
diff --git a/remotecv/unique_queue.py b/remotecv/unique_queue.py
@@ -91,3 +91,9 @@ def register_worker(self):
             self.resq.redis.expire(
                 f"resque:worker:{str(self)}:started", config.worker_ttl
             )
+
+    def startup(self):
+        if config.prune_dead_members:
+            self.resq.redis.delete("resque:workers")
+
+        super().startup()
diff --git a/remotecv/worker.py b/remotecv/worker.py
@@ -275,6 +275,14 @@ def import_modules():
     type=click.INT,
     help="TTL in seconds for worker",
 )
+@optgroup.option(
+    "--prune-dead-members",
+    envvar="PRUNE_DEAD_MEMBERS",
+    show_envvar=True,
+    is_flag=True,
+    default=False,
+    help="Prune dead members on startup",
+)
 @optgroup.option(
     "--sentry-url",
     envvar="SENTRY_URL",
@@ -323,6 +331,7 @@ def main(**params):
 
     config.timeout = params["timeout"]
     config.worker_ttl = params["worker_ttl"]
+    config.prune_dead_members = params["prune_dead_members"]
     config.server_port = params["server_port"]
     config.log_level = params["level"].upper()
     config.loader = import_module(params["loader"])

diff --git a/tests/test_unique_queue.py b/tests/test_unique_queue.py
@@ -125,3 +125,28 @@ def test_should_create_unique_worker_with_ttl(self):
         expect(
             self.redis.ttl(f"resque:worker:{str(worker)}:started")
         ).Not.to_equal(-1)
+
+    def test_should_prune_dead_members_on_startup(self):
+        config.prune_dead_members = True
+        worker = UniqueWorker(server=self.redis, queues=["Detect"])
+        self.redis.sadd("resque:workers", "teste:1:Detect")
+
+        members = self.redis.smembers("resque:workers")
+        expect(len(members)).to_be_greater_than(1)
+
+        worker.startup()
+
+        members = self.redis.smembers("resque:workers")
+        expect(len(members)).to_equal(1)
+        expect(str(worker).encode() in members).to_be_true()
+
+    def test_should_not_prune_dead_members_on_startup(self):
+        config.prune_dead_members = False
+        worker = UniqueWorker(server=self.redis, queues=["Detect"])
+        self.redis.sadd("resque:workers", "teste:1:Detect")
+
+        worker.startup()
+
+        members = self.redis.smembers("resque:workers")
+        expect(len(members)).to_equal(2)
+        expect(str(worker).encode() in members).to_be_true()