Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added support to Multiplexer and Manager to terminate durable workers #2913

Closed
wants to merge 2 commits into from
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Add terminate_worker method to WorkerMultiplexer and terminate_durabl…
…e_worker method to WorkerManager
  • Loading branch information
John Woolley committed Feb 10, 2024
commit 0f22249b4f8023c6b18633f56ffc0bb6031adcab
30 changes: 29 additions & 1 deletion sanic/worker/manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -408,6 +408,23 @@
logger.info("Removed worker %s", worker.ident)
del worker

def terminate_durable_worker(self, ident: str) -> None:
"""Terminate a durable worker and its processes.

Args:
ident (str): The name of the durable process to terminate.
"""
worker = self.durable.get(ident)
if not worker:
error_logger.error(
f"Durable worker termination failed because {ident} not found."
john-woolley marked this conversation as resolved.
Show resolved Hide resolved
)
return
for process in worker.processes:
process.terminate()
self.remove_worker(worker)
logger.info("Terminated durable worker %s", ident)

Check warning on line 426 in sanic/worker/manager.py

View check run for this annotation

Codecov / codecov/patch

sanic/worker/manager.py#L424-L426

Added lines #L424 - L426 were not covered by tests

@property
def pid(self):
"""Get the process ID of the main process."""
Expand Down Expand Up @@ -452,7 +469,14 @@
elif message == "__TERMINATE__":
self._handle_terminate()
return MonitorCycle.BREAK
elif isinstance(message, tuple) and len(message) == 7:
elif isinstance(message, str) and message.startswith(
"__TERMINATE_WORKER__"
):
self._handle_terminate_worker(

Check warning on line 475 in sanic/worker/manager.py

View check run for this annotation

Codecov / codecov/patch

sanic/worker/manager.py#L475

Added line #L475 was not covered by tests
*message.split(":")[1].split(",")
)
return MonitorCycle.CONTINUE

Check warning on line 478 in sanic/worker/manager.py

View check run for this annotation

Codecov / codecov/patch

sanic/worker/manager.py#L478

Added line #L478 was not covered by tests
elif isinstance(message, tuple) and len(message) == 8:
self._handle_manage(*message) # type: ignore
return MonitorCycle.CONTINUE
elif not isinstance(message, str):
Expand All @@ -466,6 +490,10 @@
def _handle_terminate(self) -> None:
self.shutdown()

def _handle_terminate_worker(self, *args) -> None:
for ident in args:
self.terminate_durable_worker(ident)

Check warning on line 495 in sanic/worker/manager.py

View check run for this annotation

Codecov / codecov/patch

sanic/worker/manager.py#L495

Added line #L495 was not covered by tests

def _handle_message(self, message: str) -> Optional[MonitorCycle]:
logger.debug(
"Incoming monitor message: %s",
Expand Down
8 changes: 8 additions & 0 deletions sanic/worker/multiplexer.py
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,14 @@ def terminate(self, early: bool = False):
message = "__TERMINATE_EARLY__" if early else "__TERMINATE__"
self._monitor_publisher.send(message)

def terminate_worker(self, ident: str):
"""Terminate a worker.

Args:
name (str): The name of the worker to terminate.
"""
john-woolley marked this conversation as resolved.
Show resolved Hide resolved
self._monitor_publisher.send(f"__TERMINATE_WORKER__:{ident}")

@property
def pid(self) -> int:
"""The process ID of the worker."""
Expand Down
18 changes: 17 additions & 1 deletion tests/worker/test_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ def test_terminate(os_mock: Mock):


@patch("sanic.worker.process.os")
def test_shutown(os_mock: Mock):
def test_shutdown(os_mock: Mock):
process = Mock()
process.pid = 1234
process.is_alive.return_value = True
Expand Down Expand Up @@ -417,6 +417,22 @@ def test_remove_worker(manager: WorkerManager, caplog):
assert ("sanic.error", 40, message) in caplog.record_tuples


def test_terminate_durable_worker(manager: WorkerManager, caplog):
worker = manager.manage("TEST", fake_serve, kwargs={})

assert "Sanic-TEST-0" in worker.worker_state
assert len(manager.transient) == 1
assert len(manager.durable) == 1

manager.terminate_durable_worker("WRONG")
message = "Durable worker termination failed because WRONG not found."

assert "Sanic-TEST-0" in worker.worker_state
assert len(manager.transient) == 1
assert len(manager.durable) == 1
assert ("sanic.error", 40, message) in caplog.record_tuples


def test_remove_untracked_worker(manager: WorkerManager, caplog):
caplog.set_level(20)
worker = manager.manage("TEST", fake_serve, kwargs={}, tracked=False)
Expand Down
7 changes: 7 additions & 0 deletions tests/worker/test_multiplexer.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,13 @@ def test_terminate(monitor_publisher: Mock, m: WorkerMultiplexer):
monitor_publisher.send.assert_called_once_with("__TERMINATE__")


def test_terminate_worker(monitor_publisher: Mock, m: WorkerMultiplexer):
m.terminate_worker("foo,bar")
monitor_publisher.send.assert_called_once_with(
"__TERMINATE_WORKER__:foo,bar"
)


def test_scale(monitor_publisher: Mock, m: WorkerMultiplexer):
m.scale(99)
monitor_publisher.send.assert_called_once_with("__SCALE__:99")
Expand Down
Loading