From 8a28ba94b092eacbcfbc259466ad85621209e46c Mon Sep 17 00:00:00 2001 From: Jason Little Date: Sat, 15 Oct 2022 23:20:44 -0500 Subject: [PATCH 1/8] Add templating and comments for missing stream writers to configure_workers_and_start.py. --- docker/configure_workers_and_start.py | 50 ++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 60a5c10ea75c..67405aa72a82 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -50,7 +50,12 @@ MAIN_PROCESS_HTTP_LISTENER_PORT = 8080 - +# Workers with exposed endpoints needs either "client", "federation", or "media" listener_resources +# Watching /_matrix/client needs a "client" listener +# Watching /_matrix/federation needs a "federation" listener +# Watching /_matrix/media and related needs a "media" listener +# Stream Writers require "client" and "replication" listeners because they +# have to attach by instance_map to the master process and have client endpoints. WORKERS_CONFIG: Dict[str, Dict[str, Any]] = { "pusher": { "app": "synapse.app.pusher", @@ -209,6 +214,49 @@ % (MAIN_PROCESS_HTTP_LISTENER_PORT,) ), }, + "account_data": { + "app": "synapse.app.generic_worker", + "listener_resources": ["client", "replication"], + "endpoint_patterns": [ + "^/_matrix/client/(r0|v3|unstable)/.*/tags", + "^/_matrix/client/(r0|v3|unstable)/.*/account_data", + ], + "shared_extra_conf": {}, + "worker_extra_conf": "", + }, + "presence": { + "app": "synapse.app.generic_worker", + "listener_resources": ["client", "replication"], + "endpoint_patterns": ["^/_matrix/client/(api/v1|r0|v3|unstable)/presence/"], + "shared_extra_conf": {}, + "worker_extra_conf": "", + }, + "receipts": { + "app": "synapse.app.generic_worker", + "listener_resources": ["client", "replication"], + "endpoint_patterns": [ + "^/_matrix/client/(r0|v3|unstable)/rooms/.*/receipt", + "^/_matrix/client/(r0|v3|unstable)/rooms/.*/read_markers", + ], + "shared_extra_conf": {}, + "worker_extra_conf": "", + }, + "to_device": { + "app": "synapse.app.generic_worker", + "listener_resources": ["client", "replication"], + "endpoint_patterns": ["^/_matrix/client/(r0|v3|unstable)/sendToDevice/"], + "shared_extra_conf": {}, + "worker_extra_conf": "", + }, + "typing": { + "app": "synapse.app.generic_worker", + "listener_resources": ["client", "replication"], + "endpoint_patterns": [ + "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/typing" + ], + "shared_extra_conf": {}, + "worker_extra_conf": "", + }, } # Templates for sections that may be inserted multiple times in config files From 52628efb1ac7289991d5b594771d54af55288a29 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Sat, 15 Oct 2022 23:23:06 -0500 Subject: [PATCH 2/8] Add warning comment about non-functional stream writers. --- docker/configure_workers_and_start.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 67405aa72a82..f95150acd019 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -56,6 +56,7 @@ # Watching /_matrix/media and related needs a "media" listener # Stream Writers require "client" and "replication" listeners because they # have to attach by instance_map to the master process and have client endpoints. +# BIG WARNING: typing and receipts stream writers are not working correctly at this time. WORKERS_CONFIG: Dict[str, Dict[str, Any]] = { "pusher": { "app": "synapse.app.pusher", From 2f7fedb6c0a54427d5de320ae16288ad56676220 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Sun, 16 Oct 2022 02:24:52 -0500 Subject: [PATCH 3/8] Add stream_writer map and instance_map handling to add_sharding_to_shared_config(). --- docker/configure_workers_and_start.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index f95150acd019..695238f2e802 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -363,6 +363,20 @@ def add_sharding_to_shared_config( "port": worker_port, } + elif worker_type in ["account_data", "presence", "receipts", "to_device", "typing"]: + # Update the list of stream writers + # It's convienent that the name of the worker type is the same as the event stream + shared_config.setdefault("stream_writers", {}).setdefault( + worker_type, [] + ).append(worker_name) + + # Map of stream writer instance names to host/ports combos + # For now, all stream writers need http replication ports + instance_map[worker_name] = { + "host": "localhost", + "port": worker_port, + } + elif worker_type == "media_repository": # The first configured media worker will run the media background jobs shared_config.setdefault("media_instance_running_background_jobs", worker_name) From adb34bcefdc21ed53741076a20704b7ca0902fb3 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Sun, 16 Oct 2022 03:41:20 -0500 Subject: [PATCH 4/8] Remove unnecessary gating for running add_sharding_to_shared_config(). --- docker/configure_workers_and_start.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 695238f2e802..c8307ff41f61 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -509,11 +509,11 @@ def generate_worker_files( # Check if more than one instance of this worker type has been specified worker_type_total_count = worker_types.count(worker_type) - if worker_type_total_count > 1: - # Update the shared config with sharding-related options if necessary - add_sharding_to_shared_config( - shared_config, worker_type, worker_name, worker_port - ) + + # Update the shared config with sharding-related options if necessary + add_sharding_to_shared_config( + shared_config, worker_type, worker_name, worker_port + ) # Enable the worker in supervisord worker_descriptors.append(worker_config) From 92a1345f27ddc8d3477980cb2c364f0c7e3552ad Mon Sep 17 00:00:00 2001 From: Jason Little Date: Sun, 16 Oct 2022 04:35:00 -0500 Subject: [PATCH 5/8] Changelog --- changelog.d/14197.docker | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/14197.docker diff --git a/changelog.d/14197.docker b/changelog.d/14197.docker new file mode 100644 index 000000000000..529ccd99c501 --- /dev/null +++ b/changelog.d/14197.docker @@ -0,0 +1 @@ +Add all Stream Writer worker types to configure_workers_and_start.py. From 0c073d643420ee797f11b9d3cf93f95c424ee16f Mon Sep 17 00:00:00 2001 From: Jason Little Date: Thu, 20 Oct 2022 17:05:42 -0500 Subject: [PATCH 6/8] Rename add_sharding_to_shared_config to process_sharding_and_stream_writers_for_shared_config For the record, I think this function name is to long but seems to get the point across. --- docker/configure_workers_and_start.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index c8307ff41f61..28f8ab90f8a3 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -325,7 +325,7 @@ def convert(src: str, dst: str, **template_vars: object) -> None: outfile.write(rendered) -def add_sharding_to_shared_config( +def process_sharding_and_stream_writers_for_shared_config( shared_config: dict, worker_type: str, worker_name: str, @@ -511,7 +511,7 @@ def generate_worker_files( worker_type_total_count = worker_types.count(worker_type) # Update the shared config with sharding-related options if necessary - add_sharding_to_shared_config( + process_sharding_and_stream_writers_for_shared_config( shared_config, worker_type, worker_name, worker_port ) From 3a03ce725c39b5babaf24df272b3a1887b31a126 Mon Sep 17 00:00:00 2001 From: realtyem Date: Thu, 20 Oct 2022 21:41:57 -0500 Subject: [PATCH 7/8] Removed outdated comment This is no longer accurate --- docker/configure_workers_and_start.py | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 644f51ffcb70..4afc51829696 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -56,7 +56,6 @@ # Watching /_matrix/media and related needs a "media" listener # Stream Writers require "client" and "replication" listeners because they # have to attach by instance_map to the master process and have client endpoints. -# BIG WARNING: typing and receipts stream writers are not working correctly at this time. WORKERS_CONFIG: Dict[str, Dict[str, Any]] = { "pusher": { "app": "synapse.app.pusher", From a229008f9a939817f06e3016dcef05bd41401055 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Fri, 4 Nov 2022 14:50:39 -0500 Subject: [PATCH 8/8] Rename function to reivilibre's suggestion. --- docker/configure_workers_and_start.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 4afc51829696..da259129d1c2 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -319,7 +319,7 @@ def convert(src: str, dst: str, **template_vars: object) -> None: outfile.write(rendered) -def process_sharding_and_stream_writers_for_shared_config( +def add_worker_roles_to_shared_config( shared_config: dict, worker_type: str, worker_name: str, @@ -505,7 +505,7 @@ def generate_worker_files( worker_type_total_count = worker_types.count(worker_type) # Update the shared config with sharding-related options if necessary - process_sharding_and_stream_writers_for_shared_config( + add_worker_roles_to_shared_config( shared_config, worker_type, worker_name, worker_port )