From f2cf4e644bae88569f198e4fba75d4ed408371f9 Mon Sep 17 00:00:00 2001 From: joschrew Date: Wed, 7 Dec 2022 16:25:24 +0100 Subject: [PATCH 001/226] prototype for processing broker --- ocrd/ocrd/cli/__init__.py | 3 + ocrd/ocrd/cli/processing_broker.py | 22 ++ ocrd/ocrd/web/__init__.py | 0 ocrd/ocrd/web/processing_broker/__init__.py | 1 + ocrd/ocrd/web/processing_broker/deployment.py | 217 ++++++++++++++++++ .../processing_broker/processing_broker.py | 52 +++++ 6 files changed, 295 insertions(+) create mode 100644 ocrd/ocrd/cli/processing_broker.py create mode 100644 ocrd/ocrd/web/__init__.py create mode 100644 ocrd/ocrd/web/processing_broker/__init__.py create mode 100644 ocrd/ocrd/web/processing_broker/deployment.py create mode 100644 ocrd/ocrd/web/processing_broker/processing_broker.py diff --git a/ocrd/ocrd/cli/__init__.py b/ocrd/ocrd/cli/__init__.py index c982261e8..8f1fc2472 100644 --- a/ocrd/ocrd/cli/__init__.py +++ b/ocrd/ocrd/cli/__init__.py @@ -31,6 +31,8 @@ def get_help(self, ctx): from ocrd.decorators import ocrd_loglevel from .zip import zip_cli from .log import log_cli +from .processing_broker import processing_broker_cli + @click.group() @click.version_option() @@ -48,3 +50,4 @@ def cli(**kwargs): # pylint: disable=unused-argument cli.add_command(validate_cli) cli.add_command(log_cli) cli.add_command(resmgr_cli) +cli.add_command(processing_broker_cli) diff --git a/ocrd/ocrd/cli/processing_broker.py b/ocrd/ocrd/cli/processing_broker.py new file mode 100644 index 000000000..a66499a24 --- /dev/null +++ b/ocrd/ocrd/cli/processing_broker.py @@ -0,0 +1,22 @@ +""" +OCR-D CLI: start the processing broker + +.. click:: ocrd.cli.processing_broker:zip_cli + :prog: ocrd processing-broker + :nested: full +""" +import click +from ocrd_utils import initLogging +from ocrd.web.processing_broker import ProcessingBroker + + +@click.command('processing-broker') +@click.argument('path_to_config', required=True, type=click.STRING) +def processing_broker_cli(path_to_config, stop=False): + """ + Start and manage processing servers with the processing broker + """ + initLogging() + # Start the broker + app = ProcessingBroker(path_to_config) + app.start() diff --git a/ocrd/ocrd/web/__init__.py b/ocrd/ocrd/web/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ocrd/ocrd/web/processing_broker/__init__.py b/ocrd/ocrd/web/processing_broker/__init__.py new file mode 100644 index 000000000..30ffa35e6 --- /dev/null +++ b/ocrd/ocrd/web/processing_broker/__init__.py @@ -0,0 +1 @@ +from .processing_broker import ProcessingBroker diff --git a/ocrd/ocrd/web/processing_broker/deployment.py b/ocrd/ocrd/web/processing_broker/deployment.py new file mode 100644 index 000000000..80cb24669 --- /dev/null +++ b/ocrd/ocrd/web/processing_broker/deployment.py @@ -0,0 +1,217 @@ + +import uvicorn +import yaml +from fastapi import FastAPI +import docker +import paramiko +import re +import os +from typing import List +import re +from enum import Enum +from dataclasses import dataclass +from ocrd_utils import ( + getLogger +) + + +class Deployer: + """ + Class to wrap the deployment-functionality of the OCR-D Processing-Servers + """ + + def __init__(self, config): + self.log = getLogger("ocrd.processingbroker") + self.log.info("Deployer-init()") + self.config = config + self._started_processing_servers = {} + self._message_queue_id = None + + def _create_docker_client_for_host(self, host_config): + """ + Create a client for the specified host to do the docker deployment. Only if host contains + processors to be deployed with docker + + Returns: + Ready to use docker.DockerClient if host contains processing servers to be + docker-deploed, None otherwise + """ + if not any(x.type == Ptype.docker for x in host_config.processors): + return None + username = host_config.username + hostname = host_config.address + return self._create_docker_client(username, hostname) + + def _create_docker_client(self, username, hostname): + docker_client = docker.DockerClient(base_url=f"ssh://{username}@{hostname}") + return docker_client + + def _create_ssh_client(self, host, user, password, keypath): + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy) + if password: + client.connect(hostname=host, username=user, password=password) + elif keypath: + client.connect(hostname=host, username=user, key_filename=keypath) + else: + # TODO: think about using ~/.ssh/config or leave it like it is and remove this comment + raise Exception("Deploy message queue: password or path_to_privkey must be provided") + return client + + def _close_clients(self, *args): + for client in args: + if hasattr(client, "close") and callable(client.close): + client.close() + + def _deploy_queue(self): + # TODO: use docker-sdk here later + client = self._create_ssh_client( + self.config.message_queue.address, self.config.message_queue.username, + self.config.message_queue.password, self.config.message_queue.path_to_privkey + ) + port = self.config.message_queue.port + + # TODO: use rm here or not? Should queues be reused? + _, stdout, _ = client.exec_command(f"docker run --rm -d -p {port}:5672 rabbitmq") + container_id = stdout.read().decode('utf-8').strip() + self._message_queue_id = container_id + client.close() + self.log.debug("deployed queue") + + def _kill_queue(self): + if not self._message_queue_id: + return + + client = self._create_ssh_client( + self.config.message_queue.address, self.config.message_queue.username, + self.config.message_queue.password, self.config.message_queue.path_to_privkey + ) + # stopping container might take up to 10 Seconds + client.exec_command(f"docker stop {self._message_queue_id}") + self._message_queue_id = None + client.close() + self.log.debug("killed queue") + + def deploy(self): + """ + Deploy the message queue and all processors defined in the config-file + """ + if self._started_processing_servers or self._message_queue_id: + raise Exception("The services have already been deployed") + self._deploy_queue() + for host in self.config.hosts: + ssh_client = self._create_ssh_client(host.address, host.username, host.password, + host.path_to_privkey) + for processor in [x for x in host.processors if x.type is Ptype.native]: + count = processor.number_of_instance + for _ in range(count): + res = self._start_native_processor(ssh_client, processor) + self._add_started_processing_server(host, res) + self._close_clients(ssh_client) + + def kill(self): + for host in self._started_processing_servers: + services = self._started_processing_servers[host] + ssh_client = self._create_ssh_client( + services[0].address, services[0].username, services[0].password, + services[0].path_to_privkey) + for service in services: + if service.type is Ptype.native: + self._kill_native_processor(ssh_client, service) + self._kill_queue() + self._started_processing_servers = {} + + def _add_started_processing_server(self, host, ps_infos): + """ + add infos for a started processing server to the "reminder" + """ + storage = self._started_processing_servers.setdefault(host.name, []) + storage.append(RunningService(host.address, host.username, host.password, + host.path_to_privkey, *ps_infos)) + + def _start_native_processor(self, client, processor): + self.log.debug(f"start native processor: {processor.__dict__}") + assert processor.type == Ptype.native, "expected processor type to be 'native'" + channel = client.invoke_shell() + stdin, stdout = channel.makefile('wb'), channel.makefile('rb') + # TODO: add real command to start processing server here + cmd = "sleep 23s" + stdin.write(f"{cmd} & \n echo xyz$!xyz \n exit \n") + output = stdout.read().decode("utf-8") + stdout.close() + stdin.close() + pid = re.search(r"xyz([0-9]+)xyz", output).group(1) + return Ptype.native, pid + + def _kill_native_processor(self, client, running_service): + assert running_service.type == Ptype.native, "expected processor type to be 'native'" + client.exec_command(f"kill {running_service.identifier}") + + +class Config: + """ + Class to hold the configuration for the ProcessingBroker + + The purpose of this class and its inner classes is to load the config and make its values + accessible + """ + def __init__(self, config_path): + with open(config_path) as fin: + config = yaml.safe_load(fin) + self.message_queue = Config.MessageQueue(**config["message_queue"]) + self.hosts = [] + for d in config.get("hosts", []): + assert len(d.items()) == 1 + for k, v in d.items(): + self.hosts.append(Config.Host(k, **v)) + + class MessageQueue: + def __init__(self, address, port, ssh): + self.address = address + self.port = port + if ssh: + self.username = ssh["username"] + self.password = str(ssh["password"]) if "password" in ssh else None + self.path_to_privkey = ssh.get("path_to_privkey", None) + + class Host: + def __init__(self, name, address, username, password=None, path_to_privkey=None, + deploy_processors=[]): + self.name = name + self.address = address + self.username = username + self.password = str(password) if password is not None else None + self.path_to_privkey = path_to_privkey + self.processors = [self.__class__.Processor(**x) for x in deploy_processors] + + class Processor: + def __init__(self, name, type, number_of_instance=1): + self.name = name + self.number_of_instance = number_of_instance + self.type = Ptype.from_str(type) + + +class Ptype(Enum): + """ + Type of the processing server. It can be started native or with docker + """ + docker = 1 + native = 2 + + @staticmethod + def from_str(label: str): + return Ptype[label.lower()] + + +@dataclass(eq=True, frozen=True) +class RunningService: + """ + (Data-)Class to store all necessary information about a started processing server. Information + is used to stop it later + """ + address: str + username: str + password: str + path_to_privkey: str + type: Ptype + identifier: str diff --git a/ocrd/ocrd/web/processing_broker/processing_broker.py b/ocrd/ocrd/web/processing_broker/processing_broker.py new file mode 100644 index 000000000..6b3ce4c80 --- /dev/null +++ b/ocrd/ocrd/web/processing_broker/processing_broker.py @@ -0,0 +1,52 @@ +import uvicorn +from fastapi import FastAPI +from .deployment import ( + Deployer, + Config +) + + +class ProcessingBroker(FastAPI): + """ + TODO: doc for ProcessingBroker and its methods + """ + + def __init__(self, config_path): + # TODO: set other args: title, description, version, openapi_tags + super().__init__(on_shutdown=[self.on_shutdown]) + # TODO: validate: shema can be used to validate the content of the yaml file. decide if to + # validate here or in Config-Constructor + self.config = Config(config_path) + self.deployer = Deployer(self.config) + self.deployer.deploy() + + self.router.add_api_route( + path='/stop', + endpoint=self.stop_processing_servers, + methods=['POST'], + # tags=['TODO: add a tag'], + # summary='TODO: summary for apidesc', + # TODO: add response model? add a response body at all? + ) + + def start(self): + """ + start processing broker with uvicorn + """ + assert self.config, "config was not parsed correctly" + # TODO: change where to run the processing server: default params, read from config or read + # from cmd? Or do not run at all as fastapi? + # TODO: activate next line again (commented just for testing) + uvicorn.run(self, host='localhost', port=5050) + + async def on_shutdown(self): + # TODO: shutdown docker containers + """ + - hosts and pids should be stored somewhere + - ensure queue is empty or processor is not currently running + - connect to hosts and kill pids + """ + await self.stop_processing_servers() + + async def stop_processing_servers(self): + self.deployer.kill() From 2a4919c5d52d28eddf903d7dd1b517682c394be4 Mon Sep 17 00:00:00 2001 From: joschrew Date: Fri, 16 Dec 2022 15:12:21 +0100 Subject: [PATCH 002/226] refactoring: try to make it easier to read --- ocrd/ocrd/web/processing_broker/deployment.py | 190 +++++++++--------- .../processing_broker/processing_broker.py | 10 +- 2 files changed, 107 insertions(+), 93 deletions(-) diff --git a/ocrd/ocrd/web/processing_broker/deployment.py b/ocrd/ocrd/web/processing_broker/deployment.py index 80cb24669..52904f597 100644 --- a/ocrd/ocrd/web/processing_broker/deployment.py +++ b/ocrd/ocrd/web/processing_broker/deployment.py @@ -13,49 +13,76 @@ from ocrd_utils import ( getLogger ) +from typing import List class Deployer: """ Class to wrap the deployment-functionality of the OCR-D Processing-Servers + + Deployer is the one acting. Config is for represantation of the config-file only. DeployHost is + for managing information, not for actually doing things. """ def __init__(self, config): self.log = getLogger("ocrd.processingbroker") self.log.info("Deployer-init()") self.config = config - self._started_processing_servers = {} + self.hosts = Host.from_config(config) self._message_queue_id = None - def _create_docker_client_for_host(self, host_config): + def deploy(self): """ - Create a client for the specified host to do the docker deployment. Only if host contains - processors to be deployed with docker - - Returns: - Ready to use docker.DockerClient if host contains processing servers to be - docker-deploed, None otherwise + Deploy the message queue and all processors defined in the config-file """ - if not any(x.type == Ptype.docker for x in host_config.processors): - return None - username = host_config.username - hostname = host_config.address - return self._create_docker_client(username, hostname) + self._deploy_queue() + for host in self.hosts: + for p in host.processors_native: + self._deploy_native_processor(p, host) + self._close_clients(host) - def _create_docker_client(self, username, hostname): - docker_client = docker.DockerClient(base_url=f"ssh://{username}@{hostname}") - return docker_client + def kill(self): + self._kill_queue() + for host in self.hosts: + if not hasattr(host, "ssh_client") or not host.ssh_client: + host.ssh_client = self._create_ssh_client(host.address, host.username, + host.password, host.keypath) + for p in host.processors_native: + for pid in p.pids: + host.ssh_client.exec_command(f"kill {pid}") + p.pids = [] + + def _deploy_native_processor(self, processor, host): + """ + - client erstellen fals er nicht existiert (lazy) + - start_native_processor aufrufen + """ + assert not processor.pids, "processors already deployed. Pids are present. Host: " \ + "{host.__dict__}. Processor: {processor.__dict__}" + if not hasattr(host, "ssh_client") or not host.ssh_client: + host.ssh_client = self._create_ssh_client(host.address, host.username, host.password, + host.keypath) + for _ in range(processor.count): + pid = self._start_native_processor(host.ssh_client, processor.name, None, None) + processor.add_started_pid(pid) + + def _start_native_processor(self, client, name, _queue_address, _database_address): + self.log.debug(f"start native processor: {name}") + channel = client.invoke_shell() + stdin, stdout = channel.makefile('wb'), channel.makefile('rb') + # TODO: add real command here to start processing server here + cmd = "sleep 23s" + stdin.write(f"{cmd} & \n echo xyz$!xyz \n exit \n") + output = stdout.read().decode("utf-8") + stdout.close() + stdin.close() + return re.search(r"xyz([0-9]+)xyz", output).group(1) def _create_ssh_client(self, host, user, password, keypath): client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy) - if password: - client.connect(hostname=host, username=user, password=password) - elif keypath: - client.connect(hostname=host, username=user, key_filename=keypath) - else: - # TODO: think about using ~/.ssh/config or leave it like it is and remove this comment - raise Exception("Deploy message queue: password or path_to_privkey must be provided") + assert password or keypath, "password or keypath missing. Should have already been ensured" + client.connect(hostname=host, username=user, password=password, key_filename=keypath) return client def _close_clients(self, *args): @@ -80,6 +107,7 @@ def _deploy_queue(self): def _kill_queue(self): if not self._message_queue_id: + self.log.debug("kill_queue: no queue running") return client = self._create_ssh_client( @@ -92,60 +120,52 @@ def _kill_queue(self): client.close() self.log.debug("killed queue") - def deploy(self): - """ - Deploy the message queue and all processors defined in the config-file - """ - if self._started_processing_servers or self._message_queue_id: - raise Exception("The services have already been deployed") - self._deploy_queue() - for host in self.config.hosts: - ssh_client = self._create_ssh_client(host.address, host.username, host.password, - host.path_to_privkey) - for processor in [x for x in host.processors if x.type is Ptype.native]: - count = processor.number_of_instance - for _ in range(count): - res = self._start_native_processor(ssh_client, processor) - self._add_started_processing_server(host, res) - self._close_clients(ssh_client) - def kill(self): - for host in self._started_processing_servers: - services = self._started_processing_servers[host] - ssh_client = self._create_ssh_client( - services[0].address, services[0].username, services[0].password, - services[0].path_to_privkey) - for service in services: - if service.type is Ptype.native: - self._kill_native_processor(ssh_client, service) - self._kill_queue() - self._started_processing_servers = {} +class Host: + """ + Class to wrap functionality and information to deploy processors to one Host. - def _add_started_processing_server(self, host, ps_infos): - """ - add infos for a started processing server to the "reminder" - """ - storage = self._started_processing_servers.setdefault(host.name, []) - storage.append(RunningService(host.address, host.username, host.password, - host.path_to_privkey, *ps_infos)) + Class Config is for reading/storing the config only. Objects from DeployHost are build from the + config and provide functionality to deploy the containers alongside the config-information - def _start_native_processor(self, client, processor): - self.log.debug(f"start native processor: {processor.__dict__}") - assert processor.type == Ptype.native, "expected processor type to be 'native'" - channel = client.invoke_shell() - stdin, stdout = channel.makefile('wb'), channel.makefile('rb') - # TODO: add real command to start processing server here - cmd = "sleep 23s" - stdin.write(f"{cmd} & \n echo xyz$!xyz \n exit \n") - output = stdout.read().decode("utf-8") - stdout.close() - stdin.close() - pid = re.search(r"xyz([0-9]+)xyz", output).group(1) - return Ptype.native, pid + This class should not do much but hold config information and runtime information. I hope to + make the code better understandable this way. Deployer should still be the class who does things + and this class here should be mostly passive + """ + def __init__(self, config): + self.name = config.name + self.address = config.address + self.username = config.username + self.password = config.password + self.keypath = config.path_to_privkey + self.processors_native = [] + self.processors_docker = [] + for x in config.processors: + if x.type == Ptype.native: + self.processors_native.append( + self.Processor(x.name, x.number_of_instance, Ptype.native) + ) + else: + self.processors_docker.append( + self.Processor(x.name, x.number_of_instance, Ptype.docker) + ) + + @classmethod + def from_config(cls, config): + res = [] + for x in config.hosts: + res.append(cls(x)) + return res + + class Processor: + def __init__(self, name, count, type): + self.name = name + self.count = count + self.type = type + self.pids = [] - def _kill_native_processor(self, client, running_service): - assert running_service.type == Ptype.native, "expected processor type to be 'native'" - client.exec_command(f"kill {running_service.identifier}") + def add_started_pid(self, pid): + self.pids.append(pid) class Config: @@ -153,7 +173,7 @@ class Config: Class to hold the configuration for the ProcessingBroker The purpose of this class and its inner classes is to load the config and make its values - accessible + accessible. This class and its attributes map 1:1 to the yaml-Config file """ def __init__(self, config_path): with open(config_path) as fin: @@ -163,7 +183,7 @@ def __init__(self, config_path): for d in config.get("hosts", []): assert len(d.items()) == 1 for k, v in d.items(): - self.hosts.append(Config.Host(k, **v)) + self.hosts.append(Config.ConfigHost(k, **v)) class MessageQueue: def __init__(self, address, port, ssh): @@ -174,7 +194,7 @@ def __init__(self, address, port, ssh): self.password = str(ssh["password"]) if "password" in ssh else None self.path_to_privkey = ssh.get("path_to_privkey", None) - class Host: + class ConfigHost: def __init__(self, name, address, username, password=None, path_to_privkey=None, deploy_processors=[]): self.name = name @@ -182,9 +202,9 @@ def __init__(self, name, address, username, password=None, path_to_privkey=None, self.username = username self.password = str(password) if password is not None else None self.path_to_privkey = path_to_privkey - self.processors = [self.__class__.Processor(**x) for x in deploy_processors] + self.processors = [self.__class__.ConfigProcessor(**x) for x in deploy_processors] - class Processor: + class ConfigProcessor: def __init__(self, name, type, number_of_instance=1): self.name = name self.number_of_instance = number_of_instance @@ -201,17 +221,3 @@ class Ptype(Enum): @staticmethod def from_str(label: str): return Ptype[label.lower()] - - -@dataclass(eq=True, frozen=True) -class RunningService: - """ - (Data-)Class to store all necessary information about a started processing server. Information - is used to stop it later - """ - address: str - username: str - password: str - path_to_privkey: str - type: Ptype - identifier: str diff --git a/ocrd/ocrd/web/processing_broker/processing_broker.py b/ocrd/ocrd/web/processing_broker/processing_broker.py index 6b3ce4c80..7fc69ca85 100644 --- a/ocrd/ocrd/web/processing_broker/processing_broker.py +++ b/ocrd/ocrd/web/processing_broker/processing_broker.py @@ -4,6 +4,9 @@ Deployer, Config ) +from ocrd_utils import ( + getLogger +) class ProcessingBroker(FastAPI): @@ -19,6 +22,7 @@ def __init__(self, config_path): self.config = Config(config_path) self.deployer = Deployer(self.config) self.deployer.deploy() + self.log = getLogger("ocrd.processingbroker") self.router.add_api_route( path='/stop', @@ -37,7 +41,11 @@ def start(self): # TODO: change where to run the processing server: default params, read from config or read # from cmd? Or do not run at all as fastapi? # TODO: activate next line again (commented just for testing) - uvicorn.run(self, host='localhost', port=5050) + port = 5050 + host = 'localhost' + self.log.debug(f"starting uvicorn. Host: {host}. Port: {port}") + uvicorn.run(self, host=host, port=port) + async def on_shutdown(self): # TODO: shutdown docker containers From d23bb3c70b353cd7ae6751b3c308f854ad177c00 Mon Sep 17 00:00:00 2001 From: joschrew Date: Fri, 16 Dec 2022 16:09:37 +0100 Subject: [PATCH 003/226] adapt code to changed config-file --- ocrd/ocrd/web/processing_broker/deployment.py | 61 +++++++++++++------ 1 file changed, 41 insertions(+), 20 deletions(-) diff --git a/ocrd/ocrd/web/processing_broker/deployment.py b/ocrd/ocrd/web/processing_broker/deployment.py index 52904f597..ede5b57a1 100644 --- a/ocrd/ocrd/web/processing_broker/deployment.py +++ b/ocrd/ocrd/web/processing_broker/deployment.py @@ -82,7 +82,14 @@ def _create_ssh_client(self, host, user, password, keypath): client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy) assert password or keypath, "password or keypath missing. Should have already been ensured" + self.log.debug(f"creating ssh-client with username: '{user}', keypath: '{keypath}'. " + f"host: {host}") + assert bool(password) is not bool(keypath), "expecting either password or keypath " \ + "provided, not both" client.connect(hostname=host, username=user, password=password, key_filename=keypath) + # TODO: connecting could easily fail here: wrong password or wrong path to keyfile. Maybe it + # is better to use except and try to give custom error message + return client def _close_clients(self, *args): @@ -133,21 +140,20 @@ class Host: and this class here should be mostly passive """ def __init__(self, config): - self.name = config.name self.address = config.address self.username = config.username self.password = config.password self.keypath = config.path_to_privkey self.processors_native = [] self.processors_docker = [] - for x in config.processors: - if x.type == Ptype.native: + for x in config.deploy_processors: + if x.deploy_type == DeployType.native: self.processors_native.append( - self.Processor(x.name, x.number_of_instance, Ptype.native) + self.Processor(x.name, x.number_of_instance, DeployType.native) ) else: self.processors_docker.append( - self.Processor(x.name, x.number_of_instance, Ptype.docker) + self.Processor(x.name, x.number_of_instance, DeployType.docker) ) @classmethod @@ -158,10 +164,10 @@ def from_config(cls, config): return res class Processor: - def __init__(self, name, count, type): + def __init__(self, name, count, deploy_type): self.name = name self.count = count - self.type = type + self.deploy_type = deploy_type self.pids = [] def add_started_pid(self, pid): @@ -179,45 +185,60 @@ def __init__(self, config_path): with open(config_path) as fin: config = yaml.safe_load(fin) self.message_queue = Config.MessageQueue(**config["message_queue"]) + self.mongo_db = Config.MongoDb(**config["mongo_db"]) self.hosts = [] for d in config.get("hosts", []): - assert len(d.items()) == 1 - for k, v in d.items(): - self.hosts.append(Config.ConfigHost(k, **v)) + self.hosts.append(Config.ConfigHost(**d)) class MessageQueue: - def __init__(self, address, port, ssh): + def __init__(self, address, port, credentials, ssh): self.address = address self.port = port + if credentials: + self.username = credentials["username"] + self.password = credentials["password"] + if ssh: + self.username = ssh["username"] + self.password = str(ssh["password"]) if "password" in ssh else None + self.path_to_privkey = ssh.get("path_to_privkey", None) + + class MongoDb: + def __init__(self, address, port, credentials, ssh): + self.address = address + self.port = port + if credentials: + self.username = credentials["username"] + self.password = credentials["password"] if ssh: self.username = ssh["username"] self.password = str(ssh["password"]) if "password" in ssh else None self.path_to_privkey = ssh.get("path_to_privkey", None) class ConfigHost: - def __init__(self, name, address, username, password=None, path_to_privkey=None, + def __init__(self, address, username, password=None, path_to_privkey=None, deploy_processors=[]): - self.name = name self.address = address self.username = username self.password = str(password) if password is not None else None self.path_to_privkey = path_to_privkey - self.processors = [self.__class__.ConfigProcessor(**x) for x in deploy_processors] + self.deploy_processors = [ + self.__class__.ConfigDeployProcessor(**x) for x in deploy_processors + ] - class ConfigProcessor: - def __init__(self, name, type, number_of_instance=1): + class ConfigDeployProcessor: + def __init__(self, name, deploy_type, number_of_instance=1): self.name = name self.number_of_instance = number_of_instance - self.type = Ptype.from_str(type) + self.deploy_type = DeployType.from_str(deploy_type) -class Ptype(Enum): +class DeployType(Enum): """ - Type of the processing server. It can be started native or with docker + Deploy-Type of the processing server. It can be started native or with docker """ docker = 1 native = 2 @staticmethod def from_str(label: str): - return Ptype[label.lower()] + return DeployType[label.lower()] From 7e20fef75ad7240c1925e5d81c45f75723076362 Mon Sep 17 00:00:00 2001 From: joschrew Date: Mon, 19 Dec 2022 12:35:38 +0100 Subject: [PATCH 004/226] change config representation class --- ocrd/ocrd/web/processing_broker/deployment.py | 104 +++++++----------- .../processing_broker/processing_broker.py | 2 +- 2 files changed, 40 insertions(+), 66 deletions(-) diff --git a/ocrd/ocrd/web/processing_broker/deployment.py b/ocrd/ocrd/web/processing_broker/deployment.py index ede5b57a1..c3713af0a 100644 --- a/ocrd/ocrd/web/processing_broker/deployment.py +++ b/ocrd/ocrd/web/processing_broker/deployment.py @@ -57,6 +57,7 @@ def _deploy_native_processor(self, processor, host): - client erstellen fals er nicht existiert (lazy) - start_native_processor aufrufen """ + self.log.debug("deploy native processor: '{processor}' on '{host.address}'") assert not processor.pids, "processors already deployed. Pids are present. Host: " \ "{host.__dict__}. Processor: {processor.__dict__}" if not hasattr(host, "ssh_client") or not host.ssh_client: @@ -99,14 +100,15 @@ def _close_clients(self, *args): def _deploy_queue(self): # TODO: use docker-sdk here later + mq_conf = self.config.message_queue client = self._create_ssh_client( - self.config.message_queue.address, self.config.message_queue.username, - self.config.message_queue.password, self.config.message_queue.path_to_privkey + mq_conf.address, mq_conf.ssh.username, + mq_conf.ssh.password if hasattr(mq_conf.ssh, "password") else None, + mq_conf.ssh.path_to_privkey if hasattr(mq_conf.ssh, "path_to_privkey") else None, ) - port = self.config.message_queue.port # TODO: use rm here or not? Should queues be reused? - _, stdout, _ = client.exec_command(f"docker run --rm -d -p {port}:5672 rabbitmq") + _, stdout, _ = client.exec_command(f"docker run --rm -d -p {mq_conf.port}:5672 rabbitmq") container_id = stdout.read().decode('utf-8').strip() self._message_queue_id = container_id client.close() @@ -116,11 +118,18 @@ def _kill_queue(self): if not self._message_queue_id: self.log.debug("kill_queue: no queue running") return + else: + self.log.debug(f"trying to kill queue with id: {self._message_queue_id} now") + # TODO: use docker sdk here later + # TODO: code occures twice. dry + mq_conf = self.config.message_queue client = self._create_ssh_client( - self.config.message_queue.address, self.config.message_queue.username, - self.config.message_queue.password, self.config.message_queue.path_to_privkey + mq_conf.address, mq_conf.ssh.username, + mq_conf.ssh.password if hasattr(mq_conf.ssh, "password") else None, + mq_conf.ssh.path_to_privkey if hasattr(mq_conf.ssh, "path_to_privkey") else None, ) + # stopping container might take up to 10 Seconds client.exec_command(f"docker stop {self._message_queue_id}") self._message_queue_id = None @@ -142,19 +151,22 @@ class Host: def __init__(self, config): self.address = config.address self.username = config.username - self.password = config.password - self.keypath = config.path_to_privkey + self.password = config.password if hasattr(config, "password") else None + self.keypath = config.path_to_privkey if hasattr(config, "path_to_privkey") else None + assert self.password or self.keypath, "Host in configfile with neither password nor keyfile" self.processors_native = [] self.processors_docker = [] for x in config.deploy_processors: - if x.deploy_type == DeployType.native: + if x.deploy_type == 'native': self.processors_native.append( self.Processor(x.name, x.number_of_instance, DeployType.native) ) - else: + elif x.deploy_type == 'docker': self.processors_docker.append( self.Processor(x.name, x.number_of_instance, DeployType.docker) ) + else: + assert False, f"unknown deploy_type: '{x.deploy_type}'" @classmethod def from_config(cls, config): @@ -174,62 +186,24 @@ def add_started_pid(self, pid): self.pids.append(pid) -class Config: - """ - Class to hold the configuration for the ProcessingBroker +class Config(): + def __init__(self, d): + """ + Class-represantation of the configuration-file for the ProcessingBroker + """ + for k, v in d.items(): + if isinstance(v, dict): + setattr(self, k, Config(v)) + elif isinstance(v, list) and len(v) and isinstance(v[0], dict): + setattr(self, k, [Config(x) if isinstance(x, dict) else x for x in v]) + else: + setattr(self, k, v) - The purpose of this class and its inner classes is to load the config and make its values - accessible. This class and its attributes map 1:1 to the yaml-Config file - """ - def __init__(self, config_path): - with open(config_path) as fin: - config = yaml.safe_load(fin) - self.message_queue = Config.MessageQueue(**config["message_queue"]) - self.mongo_db = Config.MongoDb(**config["mongo_db"]) - self.hosts = [] - for d in config.get("hosts", []): - self.hosts.append(Config.ConfigHost(**d)) - - class MessageQueue: - def __init__(self, address, port, credentials, ssh): - self.address = address - self.port = port - if credentials: - self.username = credentials["username"] - self.password = credentials["password"] - if ssh: - self.username = ssh["username"] - self.password = str(ssh["password"]) if "password" in ssh else None - self.path_to_privkey = ssh.get("path_to_privkey", None) - - class MongoDb: - def __init__(self, address, port, credentials, ssh): - self.address = address - self.port = port - if credentials: - self.username = credentials["username"] - self.password = credentials["password"] - if ssh: - self.username = ssh["username"] - self.password = str(ssh["password"]) if "password" in ssh else None - self.path_to_privkey = ssh.get("path_to_privkey", None) - - class ConfigHost: - def __init__(self, address, username, password=None, path_to_privkey=None, - deploy_processors=[]): - self.address = address - self.username = username - self.password = str(password) if password is not None else None - self.path_to_privkey = path_to_privkey - self.deploy_processors = [ - self.__class__.ConfigDeployProcessor(**x) for x in deploy_processors - ] - - class ConfigDeployProcessor: - def __init__(self, name, deploy_type, number_of_instance=1): - self.name = name - self.number_of_instance = number_of_instance - self.deploy_type = DeployType.from_str(deploy_type) + @classmethod + def from_configfile(cls, path): + with open(path) as fin: + x = yaml.safe_load(fin) + return cls(x) class DeployType(Enum): diff --git a/ocrd/ocrd/web/processing_broker/processing_broker.py b/ocrd/ocrd/web/processing_broker/processing_broker.py index 7fc69ca85..562bb98ab 100644 --- a/ocrd/ocrd/web/processing_broker/processing_broker.py +++ b/ocrd/ocrd/web/processing_broker/processing_broker.py @@ -19,7 +19,7 @@ def __init__(self, config_path): super().__init__(on_shutdown=[self.on_shutdown]) # TODO: validate: shema can be used to validate the content of the yaml file. decide if to # validate here or in Config-Constructor - self.config = Config(config_path) + self.config = Config.from_configfile(config_path) self.deployer = Deployer(self.config) self.deployer.deploy() self.log = getLogger("ocrd.processingbroker") From d5a8641703b38306e141491879dbad8c505de8b3 Mon Sep 17 00:00:00 2001 From: joschrew Date: Mon, 19 Dec 2022 13:36:50 +0100 Subject: [PATCH 005/226] add config validation --- ocrd/ocrd/cli/processing_broker.py | 6 +- .../web/processing_broker/config.schema.yml | 144 ++++++++++++++++++ .../processing_broker/processing_broker.py | 14 ++ 3 files changed, 163 insertions(+), 1 deletion(-) create mode 100644 ocrd/ocrd/web/processing_broker/config.schema.yml diff --git a/ocrd/ocrd/cli/processing_broker.py b/ocrd/ocrd/cli/processing_broker.py index a66499a24..cb191e82e 100644 --- a/ocrd/ocrd/cli/processing_broker.py +++ b/ocrd/ocrd/cli/processing_broker.py @@ -8,6 +8,7 @@ import click from ocrd_utils import initLogging from ocrd.web.processing_broker import ProcessingBroker +import sys @click.command('processing-broker') @@ -17,6 +18,9 @@ def processing_broker_cli(path_to_config, stop=False): Start and manage processing servers with the processing broker """ initLogging() - # Start the broker + res = ProcessingBroker.validate_config(path_to_config) + if res: + print(f"config is invalid: {res}") + sys.exit(1) app = ProcessingBroker(path_to_config) app.start() diff --git a/ocrd/ocrd/web/processing_broker/config.schema.yml b/ocrd/ocrd/web/processing_broker/config.schema.yml new file mode 100644 index 000000000..f9d3294ba --- /dev/null +++ b/ocrd/ocrd/web/processing_broker/config.schema.yml @@ -0,0 +1,144 @@ +$schema: https://json-schema.org/draft/2020-12/schema +$id: https://ocr-d.de/spec/web-api/config.schema.yml +description: Schema for the Processing Broker configuration file +type: object +additionalProperties: false +required: + - message_queue +properties: + message_queue: + description: Information about the Message Queue + type: object + additionalProperties: false + required: + - address + - port + properties: + address: + description: The IP address or domain name of the machine where Message Queue is deployed + $ref: "#/$defs/address" + port: + description: The port number of the Message Queue + $ref: "#/$defs/port" + credentials: + description: The credentials for the Message Queue + $ref: "#/$defs/credentials" + ssh: + description: Information required for an SSH connection + $ref: "#/$defs/ssh" + mongo_db: + description: Information about the MongoDB + type: object + additionalProperties: false + required: + - address + - port + properties: + address: + description: The IP address or domain name of the machine where MongoDB is deployed + $ref: "#/$defs/address" + port: + description: The port number of the MongoDB + $ref: "#/$defs/port" + credentials: + description: The credentials for the MongoDB + $ref: "#/$defs/credentials" + ssh: + description: Information required for an SSH connection + $ref: "#/$defs/ssh" + hosts: + description: A list of hosts where Processing Servers will be deployed + type: array + minItems: 1 + items: + description: A host where one or many Processing Servers will be deployed + type: object + additionalProperties: false + required: + - address + - username + - deploy_processors + oneOf: + - required: + - password + - required: + - path_to_privkey + properties: + address: + description: The IP address or domain name of the target machine + $ref: "#/$defs/address" + username: + type: string + password: + type: string + path_to_privkey: + description: Path to private key file + type: string + deploy_processors: + description: List of processors which will be deployed + type: array + minItems: 1 + items: + type: object + additionalProperties: false + required: + - name + properties: + name: + description: Name of the processor + type: string + pattern: "^ocrd-.*$" + examples: + - ocrd-cis-ocropy-binarize + - ocrd-olena-binarize + number_of_instance: + description: Number of instances to be deployed + type: integer + minimum: 1 + default: 1 + deploy_type: + description: Should the processor be deployed natively or with Docker + type: string + enum: + - native + - docker + default: native +$defs: + address: + type: string + anyOf: + - format: hostname + - format: ipv4 + port: + type: integer + minimum: 1 + maximum: 65535 + credentials: + type: object + additionalProperties: false + required: + - username + - password + properties: + username: + type: string + password: + type: string + ssh: + type: object + additionalProperties: false + oneOf: + - required: + - username + - password + - required: + - username + - path_to_privkey + properties: + username: + type: string + password: + type: string + path_to_privkey: + description: Path to private key file + type: string diff --git a/ocrd/ocrd/web/processing_broker/processing_broker.py b/ocrd/ocrd/web/processing_broker/processing_broker.py index 562bb98ab..d39749fa6 100644 --- a/ocrd/ocrd/web/processing_broker/processing_broker.py +++ b/ocrd/ocrd/web/processing_broker/processing_broker.py @@ -7,6 +7,9 @@ from ocrd_utils import ( getLogger ) +import yaml +from jsonschema import validate, ValidationError +from ocrd_utils.package_resources import resource_string class ProcessingBroker(FastAPI): @@ -46,6 +49,17 @@ def start(self): self.log.debug(f"starting uvicorn. Host: {host}. Port: {port}") uvicorn.run(self, host=host, port=port) + @staticmethod + def validate_config(config_path): + with open(config_path) as fin: + obj = yaml.safe_load(fin) + # TODO: move schema to another place?! + schema = yaml.safe_load(resource_string(__name__, 'config.schema.yml')) + try: + validate(obj, schema) + except ValidationError as e: + return f"{e.message}. At {e.json_path}" + return None async def on_shutdown(self): # TODO: shutdown docker containers From fd8e724fbb41273a1e72255f02ef10259a4b0ac4 Mon Sep 17 00:00:00 2001 From: joschrew Date: Mon, 19 Dec 2022 17:16:18 +0100 Subject: [PATCH 006/226] add deployment with docker-sdk --- ocrd/ocrd/web/processing_broker/deployment.py | 116 ++++++++++++++---- .../processing_broker/processing_broker.py | 6 +- 2 files changed, 99 insertions(+), 23 deletions(-) diff --git a/ocrd/ocrd/web/processing_broker/deployment.py b/ocrd/ocrd/web/processing_broker/deployment.py index c3713af0a..172211c01 100644 --- a/ocrd/ocrd/web/processing_broker/deployment.py +++ b/ocrd/ocrd/web/processing_broker/deployment.py @@ -1,19 +1,13 @@ - -import uvicorn import yaml -from fastapi import FastAPI import docker +from docker.transport import SSHHTTPAdapter import paramiko import re -import os -from typing import List -import re from enum import Enum -from dataclasses import dataclass from ocrd_utils import ( getLogger ) -from typing import List +import urllib.parse class Deployer: @@ -38,33 +32,54 @@ def deploy(self): self._deploy_queue() for host in self.hosts: for p in host.processors_native: - self._deploy_native_processor(p, host) + self._deploy_processor(p, host, DeployType.native) + for p in host.processors_docker: + self._deploy_processor(p, host, DeployType.docker) self._close_clients(host) def kill(self): self._kill_queue() for host in self.hosts: + # TODO: provide function in host that does that so it is shorter and better to read if not hasattr(host, "ssh_client") or not host.ssh_client: host.ssh_client = self._create_ssh_client(host.address, host.username, host.password, host.keypath) + + # TODO: provide function in host that does that so it is shorter and better to read + if not hasattr(host, "docker_client") or not host.docker_client: + host.docker_client = self._create_docker_client(host.address, host.username, + host.password, host.keypath) for p in host.processors_native: for pid in p.pids: host.ssh_client.exec_command(f"kill {pid}") p.pids = [] + for p in host.processors_docker: + for pid in p.pids: + self.log.debug(f"trying to kill docker container: {pid}") + # TODO: think about timeout. think about using threads to kill parallelize waiting time + host.docker_client.containers.get(pid).stop() + p.pids = [] - def _deploy_native_processor(self, processor, host): - """ - - client erstellen fals er nicht existiert (lazy) - - start_native_processor aufrufen - """ - self.log.debug("deploy native processor: '{processor}' on '{host.address}'") + def _deploy_processor(self, processor, host, deploy_type): + self.log.debug(f"deploy '{deploy_type}' processor: '{processor}' on '{host.address}'") assert not processor.pids, "processors already deployed. Pids are present. Host: " \ "{host.__dict__}. Processor: {processor.__dict__}" - if not hasattr(host, "ssh_client") or not host.ssh_client: - host.ssh_client = self._create_ssh_client(host.address, host.username, host.password, - host.keypath) + if deploy_type == DeployType.native: + # TODO: provide function in host that does that so it is shorter and better to read + if not hasattr(host, "ssh_client") or not host.ssh_client: + # TODO: add function to host to get params as dict, then `**host.login_dict()` + host.ssh_client = self._create_ssh_client(host.address, host.username, + host.password, host.keypath) + else: + # TODO: provide function in host that does that so it is shorter and better to read + if not hasattr(host, "docker_client") or not host.docker_client: + host.docker_client = self._create_docker_client(host.address, host.username, + host.password, host.keypath) for _ in range(processor.count): - pid = self._start_native_processor(host.ssh_client, processor.name, None, None) + if deploy_type == DeployType.native: + pid = self._start_native_processor(host.ssh_client, processor.name, None, None) + else: + pid = self._start_docker_processor(host.docker_client, processor.name, None, None) processor.add_started_pid(pid) def _start_native_processor(self, client, name, _queue_address, _database_address): @@ -79,20 +94,32 @@ def _start_native_processor(self, client, name, _queue_address, _database_addres stdin.close() return re.search(r"xyz([0-9]+)xyz", output).group(1) - def _create_ssh_client(self, host, user, password, keypath): + def _start_docker_processor(self, client, name, _queue_address, _database_address): + self.log.debug(f"start docker processor: {name}") + # TODO: add real command here to start processing server here + res = client.containers.run("debian", "sleep 31", detach=True, remove=True) + assert res and res.id, "run docker container failed" + return res.id + + def _create_ssh_client(self, address, user, password, keypath): client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy) assert password or keypath, "password or keypath missing. Should have already been ensured" self.log.debug(f"creating ssh-client with username: '{user}', keypath: '{keypath}'. " - f"host: {host}") + f"host: {address}") assert bool(password) is not bool(keypath), "expecting either password or keypath " \ "provided, not both" - client.connect(hostname=host, username=user, password=password, key_filename=keypath) + client.connect(hostname=address, username=user, password=password, key_filename=keypath) # TODO: connecting could easily fail here: wrong password or wrong path to keyfile. Maybe it # is better to use except and try to give custom error message return client + def _create_docker_client(self, address, user, password=None, keypath=None): + assert bool(password) is not bool(keypath), "expecting either password or keypath " \ + "provided, not both" + return CustomDockerClient(user, address, password=password, keypath=keypath) + def _close_clients(self, *args): for client in args: if hasattr(client, "close") and callable(client.close): @@ -216,3 +243,48 @@ class DeployType(Enum): @staticmethod def from_str(label: str): return DeployType[label.lower()] + + +class CustomDockerClient(docker.DockerClient): + """ + Wrapper for docker.DockerClient to use an own SshHttpAdapter. This makes it possible to use + provided password/keyfile for connecting with python-docker-sdk, which otherwise only allows to + use ~/.ssh/config for login + + XXX: inspired by https://github.com/docker/docker-py/issues/2416 . Should be replaced when + docker-sdk provides its own way to make it possible to use custom SSH Credentials. Possible + Problems: APIClient must be given the API-version because it cannot connect prior to read it. I + could imagine this could cause Problems + """ + def __init__(self, user, host, **kwargs): + assert user and host, "user and host must be set" + assert "password" in kwargs or "keypath" in kwargs, "one of password and keyfile is needed" + self.api = docker.APIClient(f"ssh://{host}", use_ssh_client=True, version='1.41') + ssh_adapter = self.CustomSshHttpAdapter(f"ssh://{user}@{host}:22", **kwargs) + self.api.mount('http+docker://ssh', ssh_adapter) + + class CustomSshHttpAdapter(SSHHTTPAdapter): + def __init__(self, base_url, password=None, keypath=None): + self.password = password + self.keypath = keypath + if not self.password and not self.keypath: + raise Exception("either 'password' or 'keypath' must be provided") + super().__init__(base_url) + + def _create_paramiko_client(self, base_url): + """ + this method is called in the superclass constructor. Overwriting allows to set + password/keypath for internal paramiko-client + """ + self.ssh_client = paramiko.SSHClient() + base_url = urllib.parse.urlparse(base_url) + self.ssh_params = { + "hostname": base_url.hostname, + "port": base_url.port, + "username": base_url.username, + } + if self.password: + self.ssh_params["password"] = self.password + elif self.keypath: + self.ssh_params["key_filename"] = self.keypath + self.ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy) diff --git a/ocrd/ocrd/web/processing_broker/processing_broker.py b/ocrd/ocrd/web/processing_broker/processing_broker.py index d39749fa6..bb669c950 100644 --- a/ocrd/ocrd/web/processing_broker/processing_broker.py +++ b/ocrd/ocrd/web/processing_broker/processing_broker.py @@ -68,7 +68,11 @@ async def on_shutdown(self): - ensure queue is empty or processor is not currently running - connect to hosts and kill pids """ - await self.stop_processing_servers() + try: + await self.stop_processing_servers() + except: + self.log.debug("error stopping processing servers: ", exc_info=True) + raise async def stop_processing_servers(self): self.deployer.kill() From 8f0796d440d3d335aeb46dadbb23508868f42afb Mon Sep 17 00:00:00 2001 From: joschrew Date: Tue, 20 Dec 2022 09:47:54 +0100 Subject: [PATCH 007/226] deploy queue and mongodb with docker-sdk --- ocrd/ocrd/web/processing_broker/deployment.py | 72 +++++++++++++++---- 1 file changed, 59 insertions(+), 13 deletions(-) diff --git a/ocrd/ocrd/web/processing_broker/deployment.py b/ocrd/ocrd/web/processing_broker/deployment.py index 172211c01..c12392f32 100644 --- a/ocrd/ocrd/web/processing_broker/deployment.py +++ b/ocrd/ocrd/web/processing_broker/deployment.py @@ -20,16 +20,18 @@ class Deployer: def __init__(self, config): self.log = getLogger("ocrd.processingbroker") - self.log.info("Deployer-init()") + self.log.debug("Deployer-init()") self.config = config self.hosts = Host.from_config(config) self._message_queue_id = None + self._mongodb_id = None def deploy(self): """ Deploy the message queue and all processors defined in the config-file """ self._deploy_queue() + self._deploy_mongodb() for host in self.hosts: for p in host.processors_native: self._deploy_processor(p, host, DeployType.native) @@ -39,6 +41,7 @@ def deploy(self): def kill(self): self._kill_queue() + self._kill_mongodb() for host in self.hosts: # TODO: provide function in host that does that so it is shorter and better to read if not hasattr(host, "ssh_client") or not host.ssh_client: @@ -126,42 +129,85 @@ def _close_clients(self, *args): client.close() def _deploy_queue(self): - # TODO: use docker-sdk here later mq_conf = self.config.message_queue - client = self._create_ssh_client( + client = self._create_docker_client( mq_conf.address, mq_conf.ssh.username, mq_conf.ssh.password if hasattr(mq_conf.ssh, "password") else None, mq_conf.ssh.path_to_privkey if hasattr(mq_conf.ssh, "path_to_privkey") else None, ) - # TODO: use rm here or not? Should queues be reused? - _, stdout, _ = client.exec_command(f"docker run --rm -d -p {mq_conf.port}:5672 rabbitmq") - container_id = stdout.read().decode('utf-8').strip() - self._message_queue_id = container_id + res = client.containers.run( + "rabbitmq", detach=True, remove=True, ports={5672: int(mq_conf.port)} + ) + assert res and res.id, "starting message queue failed" + self._message_queue_id = res.id client.close() self.log.debug("deployed queue") + def _deploy_mongodb(self): + if not hasattr(self.config, "mongo_db"): + self.log.debug("canceled mongo-deploy: no mongo_db in config") + return + conf = self.config.mongo_db + # TODO: shorten this call + client = self._create_docker_client( + conf.address, conf.ssh.username, + conf.ssh.password if hasattr(conf.ssh, "password") else None, + conf.ssh.path_to_privkey if hasattr(conf.ssh, "path_to_privkey") else None, + ) + # TODO: use rm here or not? Should the mongdb be reused? + # TODO: what about the data-dir? Must data be preserved? + res = client.containers.run( + "mongo", detach=True, remove=True, ports={27017: int(conf.port)} + ) + assert res and res.id, "starting mongodb failed" + self._mongodb_id = res.id + client.close() + self.log.debug("deployed mongodb") + + # TODO: create function to stop a container by id. Than use that to stop mongodb and queue def _kill_queue(self): if not self._message_queue_id: - self.log.debug("kill_queue: no queue running") + self.log.debug("kill_queue: queue not running") return else: self.log.debug(f"trying to kill queue with id: {self._message_queue_id} now") - # TODO: use docker sdk here later - # TODO: code occures twice. dry mq_conf = self.config.message_queue - client = self._create_ssh_client( + client = self._create_docker_client( mq_conf.address, mq_conf.ssh.username, mq_conf.ssh.password if hasattr(mq_conf.ssh, "password") else None, mq_conf.ssh.path_to_privkey if hasattr(mq_conf.ssh, "path_to_privkey") else None, ) # stopping container might take up to 10 Seconds - client.exec_command(f"docker stop {self._message_queue_id}") + client.containers.get(self._message_queue_id).stop() self._message_queue_id = None client.close() - self.log.debug("killed queue") + self.log.debug("stopped queue") + + # TODO: see todo _kill_queue + def _kill_mongodb(self): + if not self._mongodb_id: + self.log.debug("kill_mongdb: mongodb not running") + return + else: + self.log.debug(f"trying to kill mongdb with id: {self._mongodb_id} now") + + # TODO: use docker sdk here later + # TODO: code occures twice. dry + conf = self.config.mongo_db + client = self._create_docker_client( + conf.address, conf.ssh.username, + conf.ssh.password if hasattr(conf.ssh, "password") else None, + conf.ssh.path_to_privkey if hasattr(conf.ssh, "path_to_privkey") else None, + ) + + # stopping container might take up to 10 Seconds + client.containers.get(self._mongodb_id).stop() + self._mongodb_id = None + client.close() + self.log.debug("stopped mongodb") class Host: From 65dc60fd3759450626626d34f63a20c95355437c Mon Sep 17 00:00:00 2001 From: joschrew Date: Tue, 20 Dec 2022 13:14:41 +0100 Subject: [PATCH 008/226] refactor code --- ocrd/ocrd/web/processing_broker/deployment.py | 225 ++++++++---------- .../processing_broker/processing_broker.py | 14 +- 2 files changed, 102 insertions(+), 137 deletions(-) diff --git a/ocrd/ocrd/web/processing_broker/deployment.py b/ocrd/ocrd/web/processing_broker/deployment.py index c12392f32..9354f8865 100644 --- a/ocrd/ocrd/web/processing_broker/deployment.py +++ b/ocrd/ocrd/web/processing_broker/deployment.py @@ -1,4 +1,3 @@ -import yaml import docker from docker.transport import SSHHTTPAdapter import paramiko @@ -10,25 +9,27 @@ import urllib.parse +# TODO: remove debug log statements before beta, their purpose is development only class Deployer: - """ - Class to wrap the deployment-functionality of the OCR-D Processing-Servers + """ Class to wrap the deployment-functionality of the OCR-D Processing-Servers Deployer is the one acting. Config is for represantation of the config-file only. DeployHost is for managing information, not for actually doing things. """ def __init__(self, config): + """ + Args: + config (Config): values from config file wrapped into class `Config` + """ self.log = getLogger("ocrd.processingbroker") self.log.debug("Deployer-init()") - self.config = config - self.hosts = Host.from_config(config) - self._message_queue_id = None - self._mongodb_id = None + self.mongo = MongoData(config["mongo_db"]) + self.queue = QueueData(config["message_queue"]) + self.hosts = HostData.from_config(config) def deploy(self): - """ - Deploy the message queue and all processors defined in the config-file + """ Deploy the message queue and all processors defined in the config-file """ self._deploy_queue() self._deploy_mongodb() @@ -43,15 +44,10 @@ def kill(self): self._kill_queue() self._kill_mongodb() for host in self.hosts: - # TODO: provide function in host that does that so it is shorter and better to read - if not hasattr(host, "ssh_client") or not host.ssh_client: - host.ssh_client = self._create_ssh_client(host.address, host.username, - host.password, host.keypath) - - # TODO: provide function in host that does that so it is shorter and better to read - if not hasattr(host, "docker_client") or not host.docker_client: - host.docker_client = self._create_docker_client(host.address, host.username, - host.password, host.keypath) + if host.ssh_client: + host.ssh_client = self._create_ssh_client(host) + if host.docker_client: + host.docker_client = self._create_docker_client(host) for p in host.processors_native: for pid in p.pids: host.ssh_client.exec_command(f"kill {pid}") @@ -59,7 +55,8 @@ def kill(self): for p in host.processors_docker: for pid in p.pids: self.log.debug(f"trying to kill docker container: {pid}") - # TODO: think about timeout. think about using threads to kill parallelize waiting time + # TODO: think about timeout. + # think about using threads to kill parallelized to reduce waiting time host.docker_client.containers.get(pid).stop() p.pids = [] @@ -68,16 +65,11 @@ def _deploy_processor(self, processor, host, deploy_type): assert not processor.pids, "processors already deployed. Pids are present. Host: " \ "{host.__dict__}. Processor: {processor.__dict__}" if deploy_type == DeployType.native: - # TODO: provide function in host that does that so it is shorter and better to read - if not hasattr(host, "ssh_client") or not host.ssh_client: - # TODO: add function to host to get params as dict, then `**host.login_dict()` - host.ssh_client = self._create_ssh_client(host.address, host.username, - host.password, host.keypath) + if not host.ssh_client: + host.ssh_client = self._create_ssh_client(host) else: - # TODO: provide function in host that does that so it is shorter and better to read - if not hasattr(host, "docker_client") or not host.docker_client: - host.docker_client = self._create_docker_client(host.address, host.username, - host.password, host.keypath) + if not host.docker_client: + host.docker_client = self._create_docker_client(host) for _ in range(processor.count): if deploy_type == DeployType.native: pid = self._start_native_processor(host.ssh_client, processor.name, None, None) @@ -104,24 +96,26 @@ def _start_docker_processor(self, client, name, _queue_address, _database_addres assert res and res.id, "run docker container failed" return res.id - def _create_ssh_client(self, address, user, password, keypath): + def _create_ssh_client(self, obj): + address, username, password, keypath = obj.address, obj.username, obj.password, obj.keypath + assert address and username, "address and username are mandatory" + assert bool(password) is not bool(keypath), "expecting either password or keypath, not both" + client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy) - assert password or keypath, "password or keypath missing. Should have already been ensured" - self.log.debug(f"creating ssh-client with username: '{user}', keypath: '{keypath}'. " + self.log.debug(f"creating ssh-client with username: '{username}', keypath: '{keypath}'. " f"host: {address}") - assert bool(password) is not bool(keypath), "expecting either password or keypath " \ - "provided, not both" - client.connect(hostname=address, username=user, password=password, key_filename=keypath) - # TODO: connecting could easily fail here: wrong password or wrong path to keyfile. Maybe it - # is better to use except and try to give custom error message - + # TODO: connecting could easily fail here: wrong password, wrong path to keyfile etc. Maybe + # would be better to use except and try to give custom error message when failing + client.connect(hostname=address, username=username, password=password, key_filename=keypath) return client - def _create_docker_client(self, address, user, password=None, keypath=None): + def _create_docker_client(self, obj): + address, username, password, keypath = obj.address, obj.username, obj.password, obj.keypath + assert address and username, "address and username are mandatory" assert bool(password) is not bool(keypath), "expecting either password or keypath " \ "provided, not both" - return CustomDockerClient(user, address, password=password, keypath=keypath) + return CustomDockerClient(username, address, password=password, keypath=keypath) def _close_clients(self, *args): for client in args: @@ -129,122 +123,92 @@ def _close_clients(self, *args): client.close() def _deploy_queue(self): - mq_conf = self.config.message_queue - client = self._create_docker_client( - mq_conf.address, mq_conf.ssh.username, - mq_conf.ssh.password if hasattr(mq_conf.ssh, "password") else None, - mq_conf.ssh.path_to_privkey if hasattr(mq_conf.ssh, "path_to_privkey") else None, - ) + client = self._create_docker_client(self.queue) # TODO: use rm here or not? Should queues be reused? res = client.containers.run( - "rabbitmq", detach=True, remove=True, ports={5672: int(mq_conf.port)} + "rabbitmq", detach=True, remove=True, ports={5672: self.queue.port} ) assert res and res.id, "starting message queue failed" - self._message_queue_id = res.id + self.queue.pid = res.id client.close() self.log.debug("deployed queue") def _deploy_mongodb(self): - if not hasattr(self.config, "mongo_db"): + if not self.mongo or not self.mongo.address: self.log.debug("canceled mongo-deploy: no mongo_db in config") return - conf = self.config.mongo_db - # TODO: shorten this call - client = self._create_docker_client( - conf.address, conf.ssh.username, - conf.ssh.password if hasattr(conf.ssh, "password") else None, - conf.ssh.path_to_privkey if hasattr(conf.ssh, "path_to_privkey") else None, - ) + client = self._create_docker_client(self.mongo) # TODO: use rm here or not? Should the mongdb be reused? # TODO: what about the data-dir? Must data be preserved? res = client.containers.run( - "mongo", detach=True, remove=True, ports={27017: int(conf.port)} + "mongo", detach=True, remove=True, ports={27017: self.mongo.port} ) assert res and res.id, "starting mongodb failed" - self._mongodb_id = res.id + self.mongo.pid = res.id client.close() self.log.debug("deployed mongodb") - # TODO: create function to stop a container by id. Than use that to stop mongodb and queue def _kill_queue(self): - if not self._message_queue_id: + if not self.queue.pid: self.log.debug("kill_queue: queue not running") return else: - self.log.debug(f"trying to kill queue with id: {self._message_queue_id} now") + self.log.debug(f"trying to kill queue with id: {self.queue.pid} now") - mq_conf = self.config.message_queue - client = self._create_docker_client( - mq_conf.address, mq_conf.ssh.username, - mq_conf.ssh.password if hasattr(mq_conf.ssh, "password") else None, - mq_conf.ssh.path_to_privkey if hasattr(mq_conf.ssh, "path_to_privkey") else None, - ) - - # stopping container might take up to 10 Seconds - client.containers.get(self._message_queue_id).stop() - self._message_queue_id = None + client = self._create_docker_client(self.queue) + client.containers.get(self.queue.pid).stop() + self.queue.pid = None client.close() self.log.debug("stopped queue") - # TODO: see todo _kill_queue def _kill_mongodb(self): - if not self._mongodb_id: + if not self.mongo or not self.mongo.pid: self.log.debug("kill_mongdb: mongodb not running") return else: - self.log.debug(f"trying to kill mongdb with id: {self._mongodb_id} now") - - # TODO: use docker sdk here later - # TODO: code occures twice. dry - conf = self.config.mongo_db - client = self._create_docker_client( - conf.address, conf.ssh.username, - conf.ssh.password if hasattr(conf.ssh, "password") else None, - conf.ssh.path_to_privkey if hasattr(conf.ssh, "path_to_privkey") else None, - ) + self.log.debug(f"trying to kill mongdb with id: {self.mongo.pid} now") - # stopping container might take up to 10 Seconds - client.containers.get(self._mongodb_id).stop() - self._mongodb_id = None + client = self._create_docker_client(self.mongo) + client.containers.get(self.mongo.pid).stop() + self.mongo.pid = None client.close() self.log.debug("stopped mongodb") -class Host: - """ - Class to wrap functionality and information to deploy processors to one Host. - - Class Config is for reading/storing the config only. Objects from DeployHost are build from the - config and provide functionality to deploy the containers alongside the config-information +class HostData: + """Class to wrap information for all processing-server-hosts. - This class should not do much but hold config information and runtime information. I hope to - make the code better understandable this way. Deployer should still be the class who does things - and this class here should be mostly passive + Config information and runtime information is stored here. This class + should not do much but hold config information and runtime information. I + hope to make the code better understandable this way. Deployer should still + be the class who does things and this class here should be mostly passive """ def __init__(self, config): - self.address = config.address - self.username = config.username - self.password = config.password if hasattr(config, "password") else None - self.keypath = config.path_to_privkey if hasattr(config, "path_to_privkey") else None + self.address = config["address"] + self.username = config["username"] + self.password = config.get("password", None) + self.keypath = config.get("path_to_privkey", None) assert self.password or self.keypath, "Host in configfile with neither password nor keyfile" self.processors_native = [] self.processors_docker = [] - for x in config.deploy_processors: - if x.deploy_type == 'native': + for x in config["deploy_processors"]: + if x["deploy_type"] == 'native': self.processors_native.append( - self.Processor(x.name, x.number_of_instance, DeployType.native) + self.Processor(x["name"], x["number_of_instance"], DeployType.native) ) - elif x.deploy_type == 'docker': + elif x["deploy_type"] == 'docker': self.processors_docker.append( - self.Processor(x.name, x.number_of_instance, DeployType.docker) + self.Processor(x["name"], x["number_of_instance"], DeployType.docker) ) else: assert False, f"unknown deploy_type: '{x.deploy_type}'" + self.ssh_client = None + self.docker_client = None @classmethod def from_config(cls, config): res = [] - for x in config.hosts: + for x in config["hosts"]: res.append(cls(x)) return res @@ -259,29 +223,35 @@ def add_started_pid(self, pid): self.pids.append(pid) -class Config(): - def __init__(self, d): - """ - Class-represantation of the configuration-file for the ProcessingBroker - """ - for k, v in d.items(): - if isinstance(v, dict): - setattr(self, k, Config(v)) - elif isinstance(v, list) and len(v) and isinstance(v[0], dict): - setattr(self, k, [Config(x) if isinstance(x, dict) else x for x in v]) - else: - setattr(self, k, v) +class MongoData(): + """ Class to hold information for Mongodb-Docker container + """ - @classmethod - def from_configfile(cls, path): - with open(path) as fin: - x = yaml.safe_load(fin) - return cls(x) + def __init__(self, config): + self.address = config["address"] + self.port = int(config["port"]) + self.username = config["ssh"]["username"] + self.keypath = config["ssh"].get("path_to_privkey", None) + self.password = config["ssh"].get("password", None) + self.credentials = (config["credentials"]["username"], config["credentials"]["password"]) + self.pid = None -class DeployType(Enum): +class QueueData(): + """ Class to hold information for RabbitMQ-Docker container """ - Deploy-Type of the processing server. It can be started native or with docker + def __init__(self, config): + self.address = config["address"] + self.port = int(config["port"]) + self.username = config["ssh"]["username"] + self.keypath = config["ssh"].get("path_to_privkey", None) + self.password = config["ssh"].get("password", None) + self.credentials = (config["credentials"]["username"], config["credentials"]["password"]) + self.pid = None + + +class DeployType(Enum): + """ Deploy-Type of the processing server. """ docker = 1 native = 2 @@ -292,10 +262,11 @@ def from_str(label: str): class CustomDockerClient(docker.DockerClient): - """ - Wrapper for docker.DockerClient to use an own SshHttpAdapter. This makes it possible to use - provided password/keyfile for connecting with python-docker-sdk, which otherwise only allows to - use ~/.ssh/config for login + """Wrapper for docker.DockerClient to use an own SshHttpAdapter. + + This makes it possible to use provided password/keyfile for connecting with + python-docker-sdk, which otherwise only allows to use ~/.ssh/config for + login XXX: inspired by https://github.com/docker/docker-py/issues/2416 . Should be replaced when docker-sdk provides its own way to make it possible to use custom SSH Credentials. Possible diff --git a/ocrd/ocrd/web/processing_broker/processing_broker.py b/ocrd/ocrd/web/processing_broker/processing_broker.py index bb669c950..9eba42e7c 100644 --- a/ocrd/ocrd/web/processing_broker/processing_broker.py +++ b/ocrd/ocrd/web/processing_broker/processing_broker.py @@ -1,12 +1,7 @@ import uvicorn from fastapi import FastAPI -from .deployment import ( - Deployer, - Config -) -from ocrd_utils import ( - getLogger -) +from .deployment import Deployer +from ocrd_utils import getLogger import yaml from jsonschema import validate, ValidationError from ocrd_utils.package_resources import resource_string @@ -20,9 +15,8 @@ class ProcessingBroker(FastAPI): def __init__(self, config_path): # TODO: set other args: title, description, version, openapi_tags super().__init__(on_shutdown=[self.on_shutdown]) - # TODO: validate: shema can be used to validate the content of the yaml file. decide if to - # validate here or in Config-Constructor - self.config = Config.from_configfile(config_path) + with open(config_path) as fin: + self.config = yaml.safe_load(fin) self.deployer = Deployer(self.config) self.deployer.deploy() self.log = getLogger("ocrd.processingbroker") From ad8acc6756d759a5f7856afe3c5684c9f4a5dde3 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Mon, 2 Jan 2023 12:20:06 +0100 Subject: [PATCH 009/226] Add the required dependencies --- ocrd/requirements.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ocrd/requirements.txt b/ocrd/requirements.txt index ad30bc1f8..af5b3fb66 100644 --- a/ocrd/requirements.txt +++ b/ocrd/requirements.txt @@ -10,3 +10,7 @@ pyyaml Deprecated == 1.2.0 memory-profiler >= 0.58.0 sparklines >= 0.4.2 +uvicorn +fastapi +docker +paramiko From 55ba0e4845bcf6aa60ea2ff6247009ed4c4eeea3 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Mon, 2 Jan 2023 12:25:29 +0100 Subject: [PATCH 010/226] Make queue and db deploy more flexible --- ocrd/ocrd/web/processing_broker/deployment.py | 43 ++++++++++++++----- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/ocrd/ocrd/web/processing_broker/deployment.py b/ocrd/ocrd/web/processing_broker/deployment.py index 9354f8865..6999f2df1 100644 --- a/ocrd/ocrd/web/processing_broker/deployment.py +++ b/ocrd/ocrd/web/processing_broker/deployment.py @@ -13,7 +13,7 @@ class Deployer: """ Class to wrap the deployment-functionality of the OCR-D Processing-Servers - Deployer is the one acting. Config is for represantation of the config-file only. DeployHost is + Deployer is the one acting. Config is for representation of the config-file only. DeployHost is for managing information, not for actually doing things. """ @@ -63,7 +63,7 @@ def kill(self): def _deploy_processor(self, processor, host, deploy_type): self.log.debug(f"deploy '{deploy_type}' processor: '{processor}' on '{host.address}'") assert not processor.pids, "processors already deployed. Pids are present. Host: " \ - "{host.__dict__}. Processor: {processor.__dict__}" + "{host.__dict__}. Processor: {processor.__dict__}" if deploy_type == DeployType.native: if not host.ssh_client: host.ssh_client = self._create_ssh_client(host) @@ -114,7 +114,7 @@ def _create_docker_client(self, obj): address, username, password, keypath = obj.address, obj.username, obj.password, obj.keypath assert address and username, "address and username are mandatory" assert bool(password) is not bool(keypath), "expecting either password or keypath " \ - "provided, not both" + "provided, not both" return CustomDockerClient(username, address, password=password, keypath=keypath) def _close_clients(self, *args): @@ -122,26 +122,46 @@ def _close_clients(self, *args): if hasattr(client, "close") and callable(client.close): client.close() - def _deploy_queue(self): + def _deploy_queue(self, image="rabbitmq", detach=True, remove=True, ports=None): client = self._create_docker_client(self.queue) + if ports is None: + # 5672, 5671 - used by AMQP 0-9-1 and AMQP 1.0 clients without and with TLS + # 15672, 15671: HTTP API clients, management UI and rabbitmqadmin, without and with TLS + # 25672: used for internode and CLI tools communication and is allocated from + # a dynamic range (limited to a single port by default, computed as AMQP port + 20000) + ports = { + 5672: self.queue.port, + 15672: 15672, + 25672: 25672 + } # TODO: use rm here or not? Should queues be reused? res = client.containers.run( - "rabbitmq", detach=True, remove=True, ports={5672: self.queue.port} + image=image, + detach=detach, + remove=remove, + ports=ports ) assert res and res.id, "starting message queue failed" self.queue.pid = res.id client.close() self.log.debug("deployed queue") - def _deploy_mongodb(self): + def _deploy_mongodb(self, image="mongo", detach=True, remove=True, ports=None): if not self.mongo or not self.mongo.address: self.log.debug("canceled mongo-deploy: no mongo_db in config") return client = self._create_docker_client(self.mongo) - # TODO: use rm here or not? Should the mongdb be reused? + if ports is None: + ports = { + 27017: self.mongo.port + } + # TODO: use rm here or not? Should the mongodb be reused? # TODO: what about the data-dir? Must data be preserved? res = client.containers.run( - "mongo", detach=True, remove=True, ports={27017: self.mongo.port} + image=image, + detach=detach, + remove=remove, + ports=ports ) assert res and res.id, "starting mongodb failed" self.mongo.pid = res.id @@ -183,6 +203,7 @@ class HostData: hope to make the code better understandable this way. Deployer should still be the class who does things and this class here should be mostly passive """ + def __init__(self, config): self.address = config["address"] self.username = config["username"] @@ -223,7 +244,7 @@ def add_started_pid(self, pid): self.pids.append(pid) -class MongoData(): +class MongoData: """ Class to hold information for Mongodb-Docker container """ @@ -237,9 +258,10 @@ def __init__(self, config): self.pid = None -class QueueData(): +class QueueData: """ Class to hold information for RabbitMQ-Docker container """ + def __init__(self, config): self.address = config["address"] self.port = int(config["port"]) @@ -273,6 +295,7 @@ class CustomDockerClient(docker.DockerClient): Problems: APIClient must be given the API-version because it cannot connect prior to read it. I could imagine this could cause Problems """ + def __init__(self, user, host, **kwargs): assert user and host, "user and host must be set" assert "password" in kwargs or "keypath" in kwargs, "one of password and keyfile is needed" From 22bc5de1d88a24ab9fb065f93c52ab6640cd4968 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Mon, 2 Jan 2023 13:14:30 +0100 Subject: [PATCH 011/226] Refactor deployment of processors --- ocrd/ocrd/web/processing_broker/deployment.py | 59 ++++++++++++------- 1 file changed, 37 insertions(+), 22 deletions(-) diff --git a/ocrd/ocrd/web/processing_broker/deployment.py b/ocrd/ocrd/web/processing_broker/deployment.py index 6999f2df1..929a88483 100644 --- a/ocrd/ocrd/web/processing_broker/deployment.py +++ b/ocrd/ocrd/web/processing_broker/deployment.py @@ -24,19 +24,23 @@ def __init__(self, config): """ self.log = getLogger("ocrd.processingbroker") self.log.debug("Deployer-init()") - self.mongo = MongoData(config["mongo_db"]) - self.queue = QueueData(config["message_queue"]) + self.mongo_data = MongoData(config["mongo_db"]) + self.mq_data = QueueData(config["message_queue"]) self.hosts = HostData.from_config(config) def deploy(self): """ Deploy the message queue and all processors defined in the config-file """ + # Ideally, this should return the address of the RabbitMQ Server self._deploy_queue() + # Ideally, this should return the address of the MongoDB self._deploy_mongodb() for host in self.hosts: for p in host.processors_native: + # Ideally, pass the rabbitmq server and mongodb addresses here self._deploy_processor(p, host, DeployType.native) for p in host.processors_docker: + # Ideally, pass the rabbitmq server and mongodb addresses here self._deploy_processor(p, host, DeployType.docker) self._close_clients(host) @@ -60,7 +64,7 @@ def kill(self): host.docker_client.containers.get(pid).stop() p.pids = [] - def _deploy_processor(self, processor, host, deploy_type): + def _deploy_processor(self, processor, host, deploy_type, rabbitmq_server=None, mongodb=None): self.log.debug(f"deploy '{deploy_type}' processor: '{processor}' on '{host.address}'") assert not processor.pids, "processors already deployed. Pids are present. Host: " \ "{host.__dict__}. Processor: {processor.__dict__}" @@ -72,9 +76,17 @@ def _deploy_processor(self, processor, host, deploy_type): host.docker_client = self._create_docker_client(host) for _ in range(processor.count): if deploy_type == DeployType.native: - pid = self._start_native_processor(host.ssh_client, processor.name, None, None) + pid = self._start_native_processor( + client=host.ssh_client, + name=processor.name, + _queue_address=rabbitmq_server, + _database_address=mongodb) else: - pid = self._start_docker_processor(host.docker_client, processor.name, None, None) + pid = self._start_docker_processor( + client=host.docker_client, + name=processor.name, + _queue_address=rabbitmq_server, + _database_address=mongodb) processor.add_started_pid(pid) def _start_native_processor(self, client, name, _queue_address, _database_address): @@ -87,6 +99,9 @@ def _start_native_processor(self, client, name, _queue_address, _database_addres output = stdout.read().decode("utf-8") stdout.close() stdin.close() + # What does this return and is supposed to return? + # Putting some comments when using patterns is always appreciated + # Since the docker version returns PID, this should also return PID for consistency return re.search(r"xyz([0-9]+)xyz", output).group(1) def _start_docker_processor(self, client, name, _queue_address, _database_address): @@ -123,14 +138,14 @@ def _close_clients(self, *args): client.close() def _deploy_queue(self, image="rabbitmq", detach=True, remove=True, ports=None): - client = self._create_docker_client(self.queue) + client = self._create_docker_client(self.mq_data) if ports is None: # 5672, 5671 - used by AMQP 0-9-1 and AMQP 1.0 clients without and with TLS # 15672, 15671: HTTP API clients, management UI and rabbitmqadmin, without and with TLS # 25672: used for internode and CLI tools communication and is allocated from # a dynamic range (limited to a single port by default, computed as AMQP port + 20000) ports = { - 5672: self.queue.port, + 5672: self.mq_data.port, 15672: 15672, 25672: 25672 } @@ -142,18 +157,18 @@ def _deploy_queue(self, image="rabbitmq", detach=True, remove=True, ports=None): ports=ports ) assert res and res.id, "starting message queue failed" - self.queue.pid = res.id + self.mq_data.pid = res.id client.close() self.log.debug("deployed queue") def _deploy_mongodb(self, image="mongo", detach=True, remove=True, ports=None): - if not self.mongo or not self.mongo.address: + if not self.mongo_data or not self.mongo_data.address: self.log.debug("canceled mongo-deploy: no mongo_db in config") return - client = self._create_docker_client(self.mongo) + client = self._create_docker_client(self.mongo_data) if ports is None: ports = { - 27017: self.mongo.port + 27017: self.mongo_data.port } # TODO: use rm here or not? Should the mongodb be reused? # TODO: what about the data-dir? Must data be preserved? @@ -164,33 +179,33 @@ def _deploy_mongodb(self, image="mongo", detach=True, remove=True, ports=None): ports=ports ) assert res and res.id, "starting mongodb failed" - self.mongo.pid = res.id + self.mongo_data.pid = res.id client.close() self.log.debug("deployed mongodb") def _kill_queue(self): - if not self.queue.pid: + if not self.mq_data.pid: self.log.debug("kill_queue: queue not running") return else: - self.log.debug(f"trying to kill queue with id: {self.queue.pid} now") + self.log.debug(f"trying to kill queue with id: {self.mq_data.pid} now") - client = self._create_docker_client(self.queue) - client.containers.get(self.queue.pid).stop() - self.queue.pid = None + client = self._create_docker_client(self.mq_data) + client.containers.get(self.mq_data.pid).stop() + self.mq_data.pid = None client.close() self.log.debug("stopped queue") def _kill_mongodb(self): - if not self.mongo or not self.mongo.pid: + if not self.mongo_data or not self.mongo_data.pid: self.log.debug("kill_mongdb: mongodb not running") return else: - self.log.debug(f"trying to kill mongdb with id: {self.mongo.pid} now") + self.log.debug(f"trying to kill mongdb with id: {self.mongo_data.pid} now") - client = self._create_docker_client(self.mongo) - client.containers.get(self.mongo.pid).stop() - self.mongo.pid = None + client = self._create_docker_client(self.mongo_data) + client.containers.get(self.mongo_data.pid).stop() + self.mongo_data.pid = None client.close() self.log.debug("stopped mongodb") From efdc5da9a461578200ecfd025bdf6809bfa5ad96 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Mon, 2 Jan 2023 15:17:01 +0100 Subject: [PATCH 012/226] Conceptual integration of the RabbitMQ library --- ocrd/ocrd/web/processing_broker/deployment.py | 35 ++++++++++++-- .../processing_broker/processing_broker.py | 31 +++++++++++++ .../processing_broker/processing_worker.py | 46 +++++++++++++++++++ 3 files changed, 108 insertions(+), 4 deletions(-) create mode 100644 ocrd/ocrd/web/processing_broker/processing_worker.py diff --git a/ocrd/ocrd/web/processing_broker/deployment.py b/ocrd/ocrd/web/processing_broker/deployment.py index 929a88483..21b962ed8 100644 --- a/ocrd/ocrd/web/processing_broker/deployment.py +++ b/ocrd/ocrd/web/processing_broker/deployment.py @@ -32,16 +32,16 @@ def deploy(self): """ Deploy the message queue and all processors defined in the config-file """ # Ideally, this should return the address of the RabbitMQ Server - self._deploy_queue() + rabbitmq_address = self._deploy_queue() # Ideally, this should return the address of the MongoDB - self._deploy_mongodb() + mongodb_address = self._deploy_mongodb() for host in self.hosts: for p in host.processors_native: # Ideally, pass the rabbitmq server and mongodb addresses here - self._deploy_processor(p, host, DeployType.native) + self._deploy_processor(p, host, DeployType.native, rabbitmq_address, mongodb_address) for p in host.processors_docker: # Ideally, pass the rabbitmq server and mongodb addresses here - self._deploy_processor(p, host, DeployType.docker) + self._deploy_processor(p, host, DeployType.docker, rabbitmq_address, mongodb_address) self._close_clients(host) def kill(self): @@ -68,6 +68,10 @@ def _deploy_processor(self, processor, host, deploy_type, rabbitmq_server=None, self.log.debug(f"deploy '{deploy_type}' processor: '{processor}' on '{host.address}'") assert not processor.pids, "processors already deployed. Pids are present. Host: " \ "{host.__dict__}. Processor: {processor.__dict__}" + + # Create the specific RabbitMQ queue here based on the OCR-D processor name (processor.name) + # self.rmq_publisher.create_queue(queue_name=processor.name) + if deploy_type == DeployType.native: if not host.ssh_client: host.ssh_client = self._create_ssh_client(host) @@ -76,12 +80,18 @@ def _deploy_processor(self, processor, host, deploy_type, rabbitmq_server=None, host.docker_client = self._create_docker_client(host) for _ in range(processor.count): if deploy_type == DeployType.native: + # This method should be rather part of the ProcessingWorker + # The Processing Worker can just invoke a static method of ProcessingWorker + # that creates an instance of the ProcessingWorker (Native instance) pid = self._start_native_processor( client=host.ssh_client, name=processor.name, _queue_address=rabbitmq_server, _database_address=mongodb) else: + # This method should be rather part of the ProcessingWorker + # The Processing Worker can just invoke a static method of ProcessingWorker + # that creates an instance of the ProcessingWorker (Docker instance) pid = self._start_docker_processor( client=host.docker_client, name=processor.name, @@ -89,6 +99,7 @@ def _deploy_processor(self, processor, host, deploy_type, rabbitmq_server=None, _database_address=mongodb) processor.add_started_pid(pid) + # Should be part of the ProcessingWorker class def _start_native_processor(self, client, name, _queue_address, _database_address): self.log.debug(f"start native processor: {name}") channel = client.invoke_shell() @@ -104,6 +115,7 @@ def _start_native_processor(self, client, name, _queue_address, _database_addres # Since the docker version returns PID, this should also return PID for consistency return re.search(r"xyz([0-9]+)xyz", output).group(1) + # Should be part of the ProcessingWorker class def _start_docker_processor(self, client, name, _queue_address, _database_address): self.log.debug(f"start docker processor: {name}") # TODO: add real command here to start processing server here @@ -138,6 +150,11 @@ def _close_clients(self, *args): client.close() def _deploy_queue(self, image="rabbitmq", detach=True, remove=True, ports=None): + # This method deploys the RabbitMQ Server. + # Handling of creation of queues, submitting messages to queues, + # and receiving messages from queues is part of the RabbitMQ Library + # Which is part of the OCR-D WebAPI implementation. + client = self._create_docker_client(self.mq_data) if ports is None: # 5672, 5671 - used by AMQP 0-9-1 and AMQP 1.0 clients without and with TLS @@ -161,6 +178,11 @@ def _deploy_queue(self, image="rabbitmq", detach=True, remove=True, ports=None): client.close() self.log.debug("deployed queue") + # Not implemented yet + # Note: The queue address is not just the IP address + queue_address = "RabbitMQ Server address" + return queue_address + def _deploy_mongodb(self, image="mongo", detach=True, remove=True, ports=None): if not self.mongo_data or not self.mongo_data.address: self.log.debug("canceled mongo-deploy: no mongo_db in config") @@ -183,6 +205,11 @@ def _deploy_mongodb(self, image="mongo", detach=True, remove=True, ports=None): client.close() self.log.debug("deployed mongodb") + # Not implemented yet + # Note: The mongodb address is not just the IP address + mongodb_address = "MongoDB Address" + return mongodb_address + def _kill_queue(self): if not self.mq_data.pid: self.log.debug("kill_queue: queue not running") diff --git a/ocrd/ocrd/web/processing_broker/processing_broker.py b/ocrd/ocrd/web/processing_broker/processing_broker.py index 9eba42e7c..5e7989a44 100644 --- a/ocrd/ocrd/web/processing_broker/processing_broker.py +++ b/ocrd/ocrd/web/processing_broker/processing_broker.py @@ -21,6 +21,11 @@ def __init__(self, config_path): self.deployer.deploy() self.log = getLogger("ocrd.processingbroker") + # RMQPublisher object must be created here, reference: RabbitMQ Library (WebAPI Implementation) + # Based on the API calls the ProcessingBroker will send messages to the running instance + # of the RabbitMQ Server (deployed by the Deployer object) through the RMQPublisher object. + self.rmq_publisher = self.configure_publisher(self.config) + self.router.add_api_route( path='/stop', endpoint=self.stop_processing_servers, @@ -30,6 +35,15 @@ def __init__(self, config_path): # TODO: add response model? add a response body at all? ) + """ + Publish messages based on the API calls + Here is a call example to be adopted later + + # The message type is bytes + # Call this method to publish a message + self.rmq_publisher.publish_to_queue(queue_name="queue_name", message="message") + """ + def start(self): """ start processing broker with uvicorn @@ -70,3 +84,20 @@ async def on_shutdown(self): async def stop_processing_servers(self): self.deployer.kill() + + @staticmethod + def configure_publisher(config_file): + rmq_publisher = "RMQPublisher Object" + """ + Here is a template implementation to be adopted later + + rmq_publisher = RMQPublisher(host="localhost", port=5672, vhost="/") + # The credentials are configured inside definitions.json + # when building the RabbitMQ docker image + rmq_publisher.authenticate_and_connect( + username="default-publisher", + password="default-publisher" + ) + rmq_publisher.enable_delivery_confirmations() + """ + return rmq_publisher diff --git a/ocrd/ocrd/web/processing_broker/processing_worker.py b/ocrd/ocrd/web/processing_broker/processing_worker.py new file mode 100644 index 000000000..8436cbbce --- /dev/null +++ b/ocrd/ocrd/web/processing_broker/processing_worker.py @@ -0,0 +1,46 @@ +# Abstraction for the Processing Server unit in this arch: +# https://user-images.githubusercontent.com/7795705/203554094-62ce135a-b367-49ba-9960-ffe1b7d39b2c.jpg + +# Calls to native OCR-D processor should happen through +# the Processing Worker wrapper to hide low level details. +# According to the current requirements, each ProcessingWorker +# is a single OCR-D Processor instance. +class ProcessingWorker: + def __init__(self): + # RMQConsumer object must be created here, reference: RabbitMQ Library (WebAPI Implementation) + # Based on the API calls the ProcessingWorker will receive messages from the running instance + # of the RabbitMQ Server (deployed by the Processing Broker) through the RMQConsumer object. + self.rmq_consumer = self.configure_consumer( + config_file=None, + callback_method=self.on_consumed_message + ) + + @staticmethod + def configure_consumer(config_file, callback_method): + rmq_consumer = "RMQConsumer Object" + """ + Here is a template implementation to be adopted later + + rmq_consumer = RMQConsumer(host="localhost", port=5672, vhost="/") + # The credentials are configured inside definitions.json + # when building the RabbitMQ docker image + rmq_consumer.authenticate_and_connect( + username="default-consumer", + password="default-consumer" + ) + # The callback method is called every time a message is consumed + rmq_consumer.configure_consuming(queue_name="queue_name", callback_method=funcPtr) + + """ + return rmq_consumer + + # Define what happens every time a message is consumed from the queue + def on_consumed_message(self): + pass + + # A separate thread must be created here to listen + # to the queue since this is a blocking action + def start_consuming(self): + # Blocks here and listens for messages coming from the specified queue + # self.rmq_consumer.start_consuming() + pass From 1724662b792af42d24a60a1e51b9ae01188820f2 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Mon, 2 Jan 2023 16:04:44 +0100 Subject: [PATCH 013/226] Refactor: separate deplyoer from deployment utils --- .../{deployment.py => deployer.py} | 113 +++++------------- .../web/processing_broker/deployment_utils.py | 84 +++++++++++++ .../processing_broker/processing_broker.py | 2 +- .../processing_broker/processing_worker.py | 12 +- 4 files changed, 122 insertions(+), 89 deletions(-) rename ocrd/ocrd/web/processing_broker/{deployment.py => deployer.py} (72%) create mode 100644 ocrd/ocrd/web/processing_broker/deployment_utils.py diff --git a/ocrd/ocrd/web/processing_broker/deployment.py b/ocrd/ocrd/web/processing_broker/deployer.py similarity index 72% rename from ocrd/ocrd/web/processing_broker/deployment.py rename to ocrd/ocrd/web/processing_broker/deployer.py index 21b962ed8..8c29cb42e 100644 --- a/ocrd/ocrd/web/processing_broker/deployment.py +++ b/ocrd/ocrd/web/processing_broker/deployer.py @@ -1,12 +1,24 @@ -import docker -from docker.transport import SSHHTTPAdapter -import paramiko import re from enum import Enum from ocrd_utils import ( getLogger ) -import urllib.parse +from .deployment_utils import ( + close_clients, + create_docker_client, + create_ssh_client +) + +# Abstraction of the Deployment functionality +# The Deployer agent is in the middle between +# the ProcessingBroker agent and the ProcessingWorker agents +# ProcessingBroker provides the configuration file to the Deployer agent +# The Deployer agent creates the ProcessingWorker agents. + +# TODO: +# Ideally, the interaction among the agents should happen through +# the defined API calls of the objects to stay loyal to the OOP paradigm +# This would also increase the readability and maintainability of the source code. # TODO: remove debug log statements before beta, their purpose is development only @@ -42,16 +54,16 @@ def deploy(self): for p in host.processors_docker: # Ideally, pass the rabbitmq server and mongodb addresses here self._deploy_processor(p, host, DeployType.docker, rabbitmq_address, mongodb_address) - self._close_clients(host) + close_clients(host) def kill(self): self._kill_queue() self._kill_mongodb() for host in self.hosts: if host.ssh_client: - host.ssh_client = self._create_ssh_client(host) + host.ssh_client = create_ssh_client(host) if host.docker_client: - host.docker_client = self._create_docker_client(host) + host.docker_client = create_docker_client(host) for p in host.processors_native: for pid in p.pids: host.ssh_client.exec_command(f"kill {pid}") @@ -74,10 +86,10 @@ def _deploy_processor(self, processor, host, deploy_type, rabbitmq_server=None, if deploy_type == DeployType.native: if not host.ssh_client: - host.ssh_client = self._create_ssh_client(host) + host.ssh_client = create_ssh_client(host) else: if not host.docker_client: - host.docker_client = self._create_docker_client(host) + host.docker_client = create_docker_client(host) for _ in range(processor.count): if deploy_type == DeployType.native: # This method should be rather part of the ProcessingWorker @@ -123,39 +135,13 @@ def _start_docker_processor(self, client, name, _queue_address, _database_addres assert res and res.id, "run docker container failed" return res.id - def _create_ssh_client(self, obj): - address, username, password, keypath = obj.address, obj.username, obj.password, obj.keypath - assert address and username, "address and username are mandatory" - assert bool(password) is not bool(keypath), "expecting either password or keypath, not both" - - client = paramiko.SSHClient() - client.set_missing_host_key_policy(paramiko.AutoAddPolicy) - self.log.debug(f"creating ssh-client with username: '{username}', keypath: '{keypath}'. " - f"host: {address}") - # TODO: connecting could easily fail here: wrong password, wrong path to keyfile etc. Maybe - # would be better to use except and try to give custom error message when failing - client.connect(hostname=address, username=username, password=password, key_filename=keypath) - return client - - def _create_docker_client(self, obj): - address, username, password, keypath = obj.address, obj.username, obj.password, obj.keypath - assert address and username, "address and username are mandatory" - assert bool(password) is not bool(keypath), "expecting either password or keypath " \ - "provided, not both" - return CustomDockerClient(username, address, password=password, keypath=keypath) - - def _close_clients(self, *args): - for client in args: - if hasattr(client, "close") and callable(client.close): - client.close() - def _deploy_queue(self, image="rabbitmq", detach=True, remove=True, ports=None): # This method deploys the RabbitMQ Server. # Handling of creation of queues, submitting messages to queues, # and receiving messages from queues is part of the RabbitMQ Library # Which is part of the OCR-D WebAPI implementation. - client = self._create_docker_client(self.mq_data) + client = create_docker_client(self.mq_data) if ports is None: # 5672, 5671 - used by AMQP 0-9-1 and AMQP 1.0 clients without and with TLS # 15672, 15671: HTTP API clients, management UI and rabbitmqadmin, without and with TLS @@ -187,7 +173,7 @@ def _deploy_mongodb(self, image="mongo", detach=True, remove=True, ports=None): if not self.mongo_data or not self.mongo_data.address: self.log.debug("canceled mongo-deploy: no mongo_db in config") return - client = self._create_docker_client(self.mongo_data) + client = create_docker_client(self.mongo_data) if ports is None: ports = { 27017: self.mongo_data.port @@ -217,7 +203,7 @@ def _kill_queue(self): else: self.log.debug(f"trying to kill queue with id: {self.mq_data.pid} now") - client = self._create_docker_client(self.mq_data) + client = create_docker_client(self.mq_data) client.containers.get(self.mq_data.pid).stop() self.mq_data.pid = None client.close() @@ -230,13 +216,15 @@ def _kill_mongodb(self): else: self.log.debug(f"trying to kill mongdb with id: {self.mongo_data.pid} now") - client = self._create_docker_client(self.mongo_data) + client = create_docker_client(self.mongo_data) client.containers.get(self.mongo_data.pid).stop() self.mongo_data.pid = None client.close() self.log.debug("stopped mongodb") +# TODO: These should be separated from the Deployer logic +# TODO: Moreover, some of these classes must be just @dataclasses class HostData: """Class to wrap information for all processing-server-hosts. @@ -323,50 +311,3 @@ class DeployType(Enum): @staticmethod def from_str(label: str): return DeployType[label.lower()] - - -class CustomDockerClient(docker.DockerClient): - """Wrapper for docker.DockerClient to use an own SshHttpAdapter. - - This makes it possible to use provided password/keyfile for connecting with - python-docker-sdk, which otherwise only allows to use ~/.ssh/config for - login - - XXX: inspired by https://github.com/docker/docker-py/issues/2416 . Should be replaced when - docker-sdk provides its own way to make it possible to use custom SSH Credentials. Possible - Problems: APIClient must be given the API-version because it cannot connect prior to read it. I - could imagine this could cause Problems - """ - - def __init__(self, user, host, **kwargs): - assert user and host, "user and host must be set" - assert "password" in kwargs or "keypath" in kwargs, "one of password and keyfile is needed" - self.api = docker.APIClient(f"ssh://{host}", use_ssh_client=True, version='1.41') - ssh_adapter = self.CustomSshHttpAdapter(f"ssh://{user}@{host}:22", **kwargs) - self.api.mount('http+docker://ssh', ssh_adapter) - - class CustomSshHttpAdapter(SSHHTTPAdapter): - def __init__(self, base_url, password=None, keypath=None): - self.password = password - self.keypath = keypath - if not self.password and not self.keypath: - raise Exception("either 'password' or 'keypath' must be provided") - super().__init__(base_url) - - def _create_paramiko_client(self, base_url): - """ - this method is called in the superclass constructor. Overwriting allows to set - password/keypath for internal paramiko-client - """ - self.ssh_client = paramiko.SSHClient() - base_url = urllib.parse.urlparse(base_url) - self.ssh_params = { - "hostname": base_url.hostname, - "port": base_url.port, - "username": base_url.username, - } - if self.password: - self.ssh_params["password"] = self.password - elif self.keypath: - self.ssh_params["key_filename"] = self.keypath - self.ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy) diff --git a/ocrd/ocrd/web/processing_broker/deployment_utils.py b/ocrd/ocrd/web/processing_broker/deployment_utils.py new file mode 100644 index 000000000..a63689575 --- /dev/null +++ b/ocrd/ocrd/web/processing_broker/deployment_utils.py @@ -0,0 +1,84 @@ +import docker +from docker.transport import SSHHTTPAdapter +import paramiko +import urllib.parse +from ocrd_utils import ( + getLogger +) + + +def create_ssh_client(obj): + address, username, password, keypath = obj.address, obj.username, obj.password, obj.keypath + assert address and username, "address and username are mandatory" + assert bool(password) is not bool(keypath), "expecting either password or keypath, not both" + + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy) + log = getLogger("ocrd.create_ssh_client") + log.debug(f"creating ssh-client with username: '{username}', keypath: '{keypath}'. " + f"host: {address}") + # TODO: connecting could easily fail here: wrong password, wrong path to keyfile etc. Maybe + # would be better to use except and try to give custom error message when failing + client.connect(hostname=address, username=username, password=password, key_filename=keypath) + return client + + +def create_docker_client(obj): + address, username, password, keypath = obj.address, obj.username, obj.password, obj.keypath + assert address and username, "address and username are mandatory" + assert bool(password) is not bool(keypath), "expecting either password or keypath " \ + "provided, not both" + return CustomDockerClient(username, address, password=password, keypath=keypath) + + +def close_clients(*args): + for client in args: + if hasattr(client, "close") and callable(client.close): + client.close() + + +class CustomDockerClient(docker.DockerClient): + """Wrapper for docker.DockerClient to use an own SshHttpAdapter. + + This makes it possible to use provided password/keyfile for connecting with + python-docker-sdk, which otherwise only allows to use ~/.ssh/config for + login + + XXX: inspired by https://github.com/docker/docker-py/issues/2416 . Should be replaced when + docker-sdk provides its own way to make it possible to use custom SSH Credentials. Possible + Problems: APIClient must be given the API-version because it cannot connect prior to read it. I + could imagine this could cause Problems + """ + + def __init__(self, user, host, **kwargs): + assert user and host, "user and host must be set" + assert "password" in kwargs or "keypath" in kwargs, "one of password and keyfile is needed" + self.api = docker.APIClient(f"ssh://{host}", use_ssh_client=True, version='1.41') + ssh_adapter = self.CustomSshHttpAdapter(f"ssh://{user}@{host}:22", **kwargs) + self.api.mount('http+docker://ssh', ssh_adapter) + + class CustomSshHttpAdapter(SSHHTTPAdapter): + def __init__(self, base_url, password=None, keypath=None): + self.password = password + self.keypath = keypath + if not self.password and not self.keypath: + raise Exception("either 'password' or 'keypath' must be provided") + super().__init__(base_url) + + def _create_paramiko_client(self, base_url): + """ + this method is called in the superclass constructor. Overwriting allows to set + password/keypath for internal paramiko-client + """ + self.ssh_client = paramiko.SSHClient() + base_url = urllib.parse.urlparse(base_url) + self.ssh_params = { + "hostname": base_url.hostname, + "port": base_url.port, + "username": base_url.username, + } + if self.password: + self.ssh_params["password"] = self.password + elif self.keypath: + self.ssh_params["key_filename"] = self.keypath + self.ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy) diff --git a/ocrd/ocrd/web/processing_broker/processing_broker.py b/ocrd/ocrd/web/processing_broker/processing_broker.py index 5e7989a44..c3275a76e 100644 --- a/ocrd/ocrd/web/processing_broker/processing_broker.py +++ b/ocrd/ocrd/web/processing_broker/processing_broker.py @@ -1,6 +1,6 @@ import uvicorn from fastapi import FastAPI -from .deployment import Deployer +from .deployer import Deployer from ocrd_utils import getLogger import yaml from jsonschema import validate, ValidationError diff --git a/ocrd/ocrd/web/processing_broker/processing_worker.py b/ocrd/ocrd/web/processing_broker/processing_worker.py index 8436cbbce..2e3754329 100644 --- a/ocrd/ocrd/web/processing_broker/processing_worker.py +++ b/ocrd/ocrd/web/processing_broker/processing_worker.py @@ -6,7 +6,14 @@ # According to the current requirements, each ProcessingWorker # is a single OCR-D Processor instance. class ProcessingWorker: - def __init__(self): + def __init__(self, processor_arguments, queue_address, database_address): + # Required arguments to run the OCR-D Processor + self.processor_arguments = processor_arguments + # RabbitMQ Address - This contains at least the + # host name, port, and the virtual host + self.rmq_address = queue_address + self.mongodb_address = database_address + # RMQConsumer object must be created here, reference: RabbitMQ Library (WebAPI Implementation) # Based on the API calls the ProcessingWorker will receive messages from the running instance # of the RabbitMQ Server (deployed by the Processing Broker) through the RMQConsumer object. @@ -28,7 +35,8 @@ def configure_consumer(config_file, callback_method): username="default-consumer", password="default-consumer" ) - # The callback method is called every time a message is consumed + + #Note: The queue name here is the processor.name by definition rmq_consumer.configure_consuming(queue_name="queue_name", callback_method=funcPtr) """ From 16bcb4b9b4fce9b577a1377acf3e1e9c302fa0a3 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Mon, 2 Jan 2023 16:27:34 +0100 Subject: [PATCH 014/226] Refactor processing broker/worker --- ocrd/ocrd/web/processing_broker/deployer.py | 92 ++++++++----------- .../processing_broker/processing_broker.py | 11 ++- .../processing_broker/processing_worker.py | 37 +++++++- 3 files changed, 82 insertions(+), 58 deletions(-) diff --git a/ocrd/ocrd/web/processing_broker/deployer.py b/ocrd/ocrd/web/processing_broker/deployer.py index 8c29cb42e..3fd61ea60 100644 --- a/ocrd/ocrd/web/processing_broker/deployer.py +++ b/ocrd/ocrd/web/processing_broker/deployer.py @@ -1,4 +1,3 @@ -import re from enum import Enum from ocrd_utils import ( getLogger @@ -8,6 +7,7 @@ create_docker_client, create_ssh_client ) +from .processing_worker import ProcessingWorker # Abstraction of the Deployment functionality # The Deployer agent is in the middle between @@ -40,43 +40,31 @@ def __init__(self, config): self.mq_data = QueueData(config["message_queue"]) self.hosts = HostData.from_config(config) - def deploy(self): + def deploy_all(self): """ Deploy the message queue and all processors defined in the config-file """ # Ideally, this should return the address of the RabbitMQ Server rabbitmq_address = self._deploy_queue() # Ideally, this should return the address of the MongoDB mongodb_address = self._deploy_mongodb() - for host in self.hosts: - for p in host.processors_native: - # Ideally, pass the rabbitmq server and mongodb addresses here - self._deploy_processor(p, host, DeployType.native, rabbitmq_address, mongodb_address) - for p in host.processors_docker: - # Ideally, pass the rabbitmq server and mongodb addresses here - self._deploy_processor(p, host, DeployType.docker, rabbitmq_address, mongodb_address) - close_clients(host) + self._deploy_processing_workers(self.hosts, rabbitmq_address, mongodb_address) - def kill(self): + def kill_all(self): self._kill_queue() self._kill_mongodb() - for host in self.hosts: - if host.ssh_client: - host.ssh_client = create_ssh_client(host) - if host.docker_client: - host.docker_client = create_docker_client(host) + self._kill_processing_workers() + + def _deploy_processing_workers(self, hosts, rabbitmq_address, mongodb_address): + for host in hosts: for p in host.processors_native: - for pid in p.pids: - host.ssh_client.exec_command(f"kill {pid}") - p.pids = [] + # Ideally, pass the rabbitmq server and mongodb addresses here + self._deploy_processing_worker(p, host, DeployType.native, rabbitmq_address, mongodb_address) for p in host.processors_docker: - for pid in p.pids: - self.log.debug(f"trying to kill docker container: {pid}") - # TODO: think about timeout. - # think about using threads to kill parallelized to reduce waiting time - host.docker_client.containers.get(pid).stop() - p.pids = [] + # Ideally, pass the rabbitmq server and mongodb addresses here + self._deploy_processing_worker(p, host, DeployType.docker, rabbitmq_address, mongodb_address) + close_clients(host) - def _deploy_processor(self, processor, host, deploy_type, rabbitmq_server=None, mongodb=None): + def _deploy_processing_worker(self, processor, host, deploy_type, rabbitmq_server=None, mongodb=None): self.log.debug(f"deploy '{deploy_type}' processor: '{processor}' on '{host.address}'") assert not processor.pids, "processors already deployed. Pids are present. Host: " \ "{host.__dict__}. Processor: {processor.__dict__}" @@ -95,7 +83,7 @@ def _deploy_processor(self, processor, host, deploy_type, rabbitmq_server=None, # This method should be rather part of the ProcessingWorker # The Processing Worker can just invoke a static method of ProcessingWorker # that creates an instance of the ProcessingWorker (Native instance) - pid = self._start_native_processor( + pid = ProcessingWorker.start_native_processor( client=host.ssh_client, name=processor.name, _queue_address=rabbitmq_server, @@ -104,37 +92,13 @@ def _deploy_processor(self, processor, host, deploy_type, rabbitmq_server=None, # This method should be rather part of the ProcessingWorker # The Processing Worker can just invoke a static method of ProcessingWorker # that creates an instance of the ProcessingWorker (Docker instance) - pid = self._start_docker_processor( + pid = ProcessingWorker.start_docker_processor( client=host.docker_client, name=processor.name, _queue_address=rabbitmq_server, _database_address=mongodb) processor.add_started_pid(pid) - # Should be part of the ProcessingWorker class - def _start_native_processor(self, client, name, _queue_address, _database_address): - self.log.debug(f"start native processor: {name}") - channel = client.invoke_shell() - stdin, stdout = channel.makefile('wb'), channel.makefile('rb') - # TODO: add real command here to start processing server here - cmd = "sleep 23s" - stdin.write(f"{cmd} & \n echo xyz$!xyz \n exit \n") - output = stdout.read().decode("utf-8") - stdout.close() - stdin.close() - # What does this return and is supposed to return? - # Putting some comments when using patterns is always appreciated - # Since the docker version returns PID, this should also return PID for consistency - return re.search(r"xyz([0-9]+)xyz", output).group(1) - - # Should be part of the ProcessingWorker class - def _start_docker_processor(self, client, name, _queue_address, _database_address): - self.log.debug(f"start docker processor: {name}") - # TODO: add real command here to start processing server here - res = client.containers.run("debian", "sleep 31", detach=True, remove=True) - assert res and res.id, "run docker container failed" - return res.id - def _deploy_queue(self, image="rabbitmq", detach=True, remove=True, ports=None): # This method deploys the RabbitMQ Server. # Handling of creation of queues, submitting messages to queues, @@ -222,6 +186,30 @@ def _kill_mongodb(self): client.close() self.log.debug("stopped mongodb") + def _kill_processing_workers(self): + for host in self.hosts: + if host.ssh_client: + host.ssh_client = create_ssh_client(host) + if host.docker_client: + host.docker_client = create_docker_client(host) + for p in host.processors_native: + for pid in p.pids: + host.ssh_client.exec_command(f"kill {pid}") + p.pids = [] + for p in host.processors_docker: + for pid in p.pids: + self.log.debug(f"trying to kill docker container: {pid}") + # TODO: think about timeout. + # think about using threads to kill parallelized to reduce waiting time + host.docker_client.containers.get(pid).stop() + p.pids = [] + + # May be good to have more flexibility here + # TODO: Support that functionality as well. + # Then _kill_processing_workers should just call this method in a loop + def _kill_processing_worker(self): + pass + # TODO: These should be separated from the Deployer logic # TODO: Moreover, some of these classes must be just @dataclasses diff --git a/ocrd/ocrd/web/processing_broker/processing_broker.py b/ocrd/ocrd/web/processing_broker/processing_broker.py index c3275a76e..8bc249457 100644 --- a/ocrd/ocrd/web/processing_broker/processing_broker.py +++ b/ocrd/ocrd/web/processing_broker/processing_broker.py @@ -18,7 +18,8 @@ def __init__(self, config_path): with open(config_path) as fin: self.config = yaml.safe_load(fin) self.deployer = Deployer(self.config) - self.deployer.deploy() + # Deploy everything specified in the configuration + self.deployer.deploy_all() self.log = getLogger("ocrd.processingbroker") # RMQPublisher object must be created here, reference: RabbitMQ Library (WebAPI Implementation) @@ -28,7 +29,7 @@ def __init__(self, config_path): self.router.add_api_route( path='/stop', - endpoint=self.stop_processing_servers, + endpoint=self.stop_deployed_agents, methods=['POST'], # tags=['TODO: add a tag'], # summary='TODO: summary for apidesc', @@ -77,13 +78,13 @@ async def on_shutdown(self): - connect to hosts and kill pids """ try: - await self.stop_processing_servers() + await self.stop_deployed_agents() except: self.log.debug("error stopping processing servers: ", exc_info=True) raise - async def stop_processing_servers(self): - self.deployer.kill() + async def stop_deployed_agents(self): + self.deployer.kill_all() @staticmethod def configure_publisher(config_file): diff --git a/ocrd/ocrd/web/processing_broker/processing_worker.py b/ocrd/ocrd/web/processing_broker/processing_worker.py index 2e3754329..4b0f2c2ef 100644 --- a/ocrd/ocrd/web/processing_broker/processing_worker.py +++ b/ocrd/ocrd/web/processing_broker/processing_worker.py @@ -5,10 +5,19 @@ # the Processing Worker wrapper to hide low level details. # According to the current requirements, each ProcessingWorker # is a single OCR-D Processor instance. + +import re +from ocrd_utils import ( + getLogger +) + + class ProcessingWorker: def __init__(self, processor_arguments, queue_address, database_address): + self.log = getLogger("ocrd.processing_worker") + # Required arguments to run the OCR-D Processor - self.processor_arguments = processor_arguments + self.processor_arguments = processor_arguments # processor.name is # RabbitMQ Address - This contains at least the # host name, port, and the virtual host self.rmq_address = queue_address @@ -52,3 +61,29 @@ def start_consuming(self): # Blocks here and listens for messages coming from the specified queue # self.rmq_consumer.start_consuming() pass + + @staticmethod + def start_native_processor(client, name, _queue_address, _database_address): + log = getLogger("ocrd.processing_worker.start_native") + log.debug(f"start native processor: {name}") + channel = client.invoke_shell() + stdin, stdout = channel.makefile('wb'), channel.makefile('rb') + # TODO: add real command here to start processing server here + cmd = "sleep 23s" + stdin.write(f"{cmd} & \n echo xyz$!xyz \n exit \n") + output = stdout.read().decode("utf-8") + stdout.close() + stdin.close() + # What does this return and is supposed to return? + # Putting some comments when using patterns is always appreciated + # Since the docker version returns PID, this should also return PID for consistency + return re.search(r"xyz([0-9]+)xyz", output).group(1) + + @staticmethod + def start_docker_processor(client, name, _queue_address, _database_address): + log = getLogger("ocrd.processing_worker.start_docker") + log.debug(f"start docker processor: {name}") + # TODO: add real command here to start processing server here + res = client.containers.run("debian", "sleep 31", detach=True, remove=True) + assert res and res.id, "run docker container failed" + return res.id From 9901f903e3866e2499f2cad1e626a3b2ad3b9686 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Mon, 2 Jan 2023 16:31:51 +0100 Subject: [PATCH 015/226] Revert requirements.txt to keep track of future conflicting files --- ocrd/requirements.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ocrd/requirements.txt b/ocrd/requirements.txt index af5b3fb66..ad30bc1f8 100644 --- a/ocrd/requirements.txt +++ b/ocrd/requirements.txt @@ -10,7 +10,3 @@ pyyaml Deprecated == 1.2.0 memory-profiler >= 0.58.0 sparklines >= 0.4.2 -uvicorn -fastapi -docker -paramiko From 86979f2b265e92a1fb096aad0862fcf2f27f34f7 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Tue, 3 Jan 2023 09:43:01 +0100 Subject: [PATCH 016/226] Extend requirements --- ocrd/requirements.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ocrd/requirements.txt b/ocrd/requirements.txt index afa64a4c3..b2549ff31 100644 --- a/ocrd/requirements.txt +++ b/ocrd/requirements.txt @@ -11,3 +11,8 @@ Deprecated == 1.2.0 memory-profiler >= 0.58.0 sparklines >= 0.4.2 python-magic +uvicorn +fastapi +docker +paramiko + From 07b1733d35bf372fe05dd28fa82d21e18dc508a4 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Tue, 3 Jan 2023 12:38:27 +0100 Subject: [PATCH 017/226] Adopt useful methods --- .../web/processing_broker/deployment_utils.py | 38 +++++++++++++++++++ ocrd/requirements.txt | 2 +- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/ocrd/ocrd/web/processing_broker/deployment_utils.py b/ocrd/ocrd/web/processing_broker/deployment_utils.py index a63689575..485790231 100644 --- a/ocrd/ocrd/web/processing_broker/deployment_utils.py +++ b/ocrd/ocrd/web/processing_broker/deployment_utils.py @@ -1,5 +1,7 @@ import docker from docker.transport import SSHHTTPAdapter +from frozendict import frozendict +from functools import lru_cache, wraps import paramiko import urllib.parse from ocrd_utils import ( @@ -7,6 +9,42 @@ ) +# Method adopted from Triet's implementation +# https://github.com/OCR-D/core/pull/884/files#diff-8b69cb85b5ffcfb93a053791dec62a2f909a0669ae33d8a2412f246c3b01f1a3R260 +def freeze_args(func): + """ + Transform mutable dictionary into immutable. Useful to be compatible with cache + Code taken from `this post `_ + """ + + @wraps(func) + def wrapped(*args, **kwargs): + args = tuple([frozendict(arg) if isinstance(arg, dict) else arg for arg in args]) + kwargs = {k: frozendict(v) if isinstance(v, dict) else v for k, v in kwargs.items()} + return func(*args, **kwargs) + return wrapped + + +# Method adopted from Triet's implementation +# https://github.com/OCR-D/core/pull/884/files#diff-8b69cb85b5ffcfb93a053791dec62a2f909a0669ae33d8a2412f246c3b01f1a3R260 +@freeze_args +@lru_cache(maxsize=32) +def get_processor(parameter: dict, processor_class=None): + """ + Call this function to get back an instance of a processor. The results are cached based on the parameters. + Args: + parameter (dict): a dictionary of parameters. + processor_class: the concrete `:py:class:~ocrd.Processor` class. + Returns: + When the concrete class of the processor is unknown, `None` is returned. Otherwise, an instance of the + `:py:class:~ocrd.Processor` is returned. + """ + if processor_class: + dict_params = dict(parameter) if parameter else None + return processor_class(workspace=None, parameter=dict_params) + return None + + def create_ssh_client(obj): address, username, password, keypath = obj.address, obj.username, obj.password, obj.keypath assert address and username, "address and username are mandatory" diff --git a/ocrd/requirements.txt b/ocrd/requirements.txt index b2549ff31..f438876b5 100644 --- a/ocrd/requirements.txt +++ b/ocrd/requirements.txt @@ -15,4 +15,4 @@ uvicorn fastapi docker paramiko - +frozendict~=2.3.4 From 6639c7960d1ea0db13a34a4acb3adb0ee306bcef Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Tue, 3 Jan 2023 15:39:02 +0100 Subject: [PATCH 018/226] Create network package by refactoring --- ocrd/ocrd/cli/processing_broker.py | 4 ++-- ocrd/ocrd/network/__init__.py | 21 ++++++++++++++++++ .../config.schema.yml | 0 .../processing_broker => network}/deployer.py | 4 ++-- .../deployment_utils.py | 0 ocrd/ocrd/network/ocrd_network_arch.jpg | Bin 0 -> 319337 bytes .../processing_broker.py | 2 +- .../processing_worker.py | 0 ocrd/ocrd/network/rabbitmq_utils/__init__.py | 2 ++ ocrd/ocrd/network/web_api/__init__.py | 2 ++ ocrd/ocrd/web/__init__.py | 0 ocrd/ocrd/web/processing_broker/__init__.py | 1 - 12 files changed, 30 insertions(+), 6 deletions(-) create mode 100644 ocrd/ocrd/network/__init__.py rename ocrd/ocrd/{web/processing_broker => network}/config.schema.yml (100%) rename ocrd/ocrd/{web/processing_broker => network}/deployer.py (99%) rename ocrd/ocrd/{web/processing_broker => network}/deployment_utils.py (100%) create mode 100644 ocrd/ocrd/network/ocrd_network_arch.jpg rename ocrd/ocrd/{web/processing_broker => network}/processing_broker.py (98%) rename ocrd/ocrd/{web/processing_broker => network}/processing_worker.py (100%) create mode 100644 ocrd/ocrd/network/rabbitmq_utils/__init__.py create mode 100644 ocrd/ocrd/network/web_api/__init__.py delete mode 100644 ocrd/ocrd/web/__init__.py delete mode 100644 ocrd/ocrd/web/processing_broker/__init__.py diff --git a/ocrd/ocrd/cli/processing_broker.py b/ocrd/ocrd/cli/processing_broker.py index cb191e82e..8393e2165 100644 --- a/ocrd/ocrd/cli/processing_broker.py +++ b/ocrd/ocrd/cli/processing_broker.py @@ -7,7 +7,7 @@ """ import click from ocrd_utils import initLogging -from ocrd.web.processing_broker import ProcessingBroker +from ocrd.network import ProcessingBroker import sys @@ -15,7 +15,7 @@ @click.argument('path_to_config', required=True, type=click.STRING) def processing_broker_cli(path_to_config, stop=False): """ - Start and manage processing servers with the processing broker + Start and manage processing servers (workers) with the processing broker """ initLogging() res = ProcessingBroker.validate_config(path_to_config) diff --git a/ocrd/ocrd/network/__init__.py b/ocrd/ocrd/network/__init__.py new file mode 100644 index 000000000..d1b3e7432 --- /dev/null +++ b/ocrd/ocrd/network/__init__.py @@ -0,0 +1,21 @@ +# This network package is supposed to contain all the packages and modules to realize the network architecture: +# https://user-images.githubusercontent.com/7795705/203554094-62ce135a-b367-49ba-9960-ffe1b7d39b2c.jpg +# The architecture is also available as an image: ocrd_network_arch.jpg + +# For reference, currently: +# 1. The WebAPI is available here: +# https://github.com/OCR-D/ocrd-webapi-implementation +# 2. The RabbitMQ Library (i.e., utils) is available here: +# https://github.com/OCR-D/ocrd-webapi-implementation/tree/main/ocrd_webapi/rabbitmq +# 3. Some potentially more useful code to be adopted for the Processing Broker/Worker is available here: +# https://github.com/OCR-D/core/pull/884 +# 4. The Mets Server discussion/implementation is available here: +# https://github.com/OCR-D/core/pull/966 + +# Note: The Mets Server is still not placed on the architecture diagram and probably won't be a part of +# the network package. The reason, Mets Server is tightly coupled with the `OcrdWorkspace`. + +# This package, currently, is under the `core/ocrd` package. +# It could also be a separate package on its own under `core` with the name `ocrd_network`. +# TODO: Correctly identify all current and potential future dependencies. +from .processing_broker import ProcessingBroker diff --git a/ocrd/ocrd/web/processing_broker/config.schema.yml b/ocrd/ocrd/network/config.schema.yml similarity index 100% rename from ocrd/ocrd/web/processing_broker/config.schema.yml rename to ocrd/ocrd/network/config.schema.yml diff --git a/ocrd/ocrd/web/processing_broker/deployer.py b/ocrd/ocrd/network/deployer.py similarity index 99% rename from ocrd/ocrd/web/processing_broker/deployer.py rename to ocrd/ocrd/network/deployer.py index 3fd61ea60..7b849f7e7 100644 --- a/ocrd/ocrd/web/processing_broker/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -2,12 +2,12 @@ from ocrd_utils import ( getLogger ) -from .deployment_utils import ( +from ocrd.network.deployment_utils import ( close_clients, create_docker_client, create_ssh_client ) -from .processing_worker import ProcessingWorker +from ocrd.network.processing_worker import ProcessingWorker # Abstraction of the Deployment functionality # The Deployer agent is in the middle between diff --git a/ocrd/ocrd/web/processing_broker/deployment_utils.py b/ocrd/ocrd/network/deployment_utils.py similarity index 100% rename from ocrd/ocrd/web/processing_broker/deployment_utils.py rename to ocrd/ocrd/network/deployment_utils.py diff --git a/ocrd/ocrd/network/ocrd_network_arch.jpg b/ocrd/ocrd/network/ocrd_network_arch.jpg new file mode 100644 index 0000000000000000000000000000000000000000..db65069a4e83ff83c12590e481068d4fe3eaf83d GIT binary patch literal 319337 zcmeFaXINBQwl2H~5+n(NL@5bM1|>-@5J@5+AUTOhmYicD2uMx>3X*e{oFxZAG6Irw zlu*P1isG)__xrx>{?6&=^gYk+zJ0pEXYoU3&02HJvE~}{9q&8l<;3MYaN~)zoHT%j z4ghH2Kj3l-kOXkBu&}YN;b3E9LYM?a1-L~xd3d>h>jVuK7Z)E7pMrpZf}4(pj{E=k=du|f#v!7` z{fdD`51QE3aA%FrGo&r5q>-0Qc7lMT1HmxiRyDTbq!4|ZKGGlCZ=ZQ77mV1&MvNQ?tX9m0|MU#1xLrc zkBy7}@G&7hBQq;ICpRy@th}PKs=B7OuC=YbqqD2~YtQh===ZVli64^-i%ZKZt842U zoACXE!=vL9#Oc{@<3a;4{`auJ-~W4H|G#k&gX2QS#KgeF{%u@n=q})eL5z8g?g18w zgfh0F-A#J#H#nq^qS8uRa2a@1_Q+n^4_&{-$h&YC{@c+0GP1vIV7~vYk^O%I`+Z!K z06qp9IC&Vv02DYmV@~(Eesx{h;K~R7d3=Bs&tpoban7VRz^Z0urJI`asN?Ck+^i2i zsjs44(S2?O2hXm~?(p2LfIsz_udt>t9ih)kMGt1ax~^>S_xFK>OJH+q-F$TX5>TI$ z6sJWy=f1cEZeDP}j*C2Of?&&PW6-T0n+sq($opFpto;&@H@GO!pIS6HC3A{6+fTd% z#Cg$HVda6ym~BZ^YZ+`kx8Tb-(q#dL*MDcsDbYi#duDlm2RlyYIImAcjh++eIG~rzg8TRZ*afV!%I@`GJ%xe-pYCE((YR?wy1-h%RO@ z5An&CP~D)C5^v5G<;KCTG`G-B3Xd?ldz|G$kW~ot*80R6p{QW!Dk-<;$iiQ`z|5{L zT#3tAWwrmw9_^WJrfD|XQ3(a2pk(80pi`IwuU(_aYN7FdPReItEOuY%?q`aMYTQB| z4&t=EnMk=k=i1z&fTR)~dB^Qah8NwsXPomG-ee=Xx>Dv=9N+eF7J~4jg&ox}7*krm z7$f3gd*#peM(tgtWjT&Q%%62ujm1c=GD%~`0#Pkr*CO1H6@)bt4G@fzNxi!@Dbn$M zHoF30u@Q?BsRRZHO)(NwXN(4Fw)`Aje)(gylP5+6VdlJ_51XpwxZ@o0hO&jbLU(QX z`_FQZxR-q;7JNh{N|I2xNP!Vg19PYRt8j+S;9NtDm7-5A?9^Sz+OhM-}Kfk!A7`??~~ zZ|}NTEuuV6gpX*6Rxo`kmm}eyMA3DJHu1f$Olfy9OaTFz*W=&dgq9oEIa>8ptCkmg zjr+)y_5l)m5ZRO`2vf{>;t!K&3p=BL|{zdyBaBwp8=pvXkh1iQtt5=x!s+fl$7EP)t4k_e@77$ zZTHFhIoe7SF0o@un08$3!qh33BQZ_dO268W_+%Fi357qt(DIU>5+c!%=q-v*%QZ!P zgzn4WZqG-fI68WoJ7F#A6zo_u-ef!a)O!Jb3)C{zgNN%y>W}b4aZNoPr@zUGDp4-+ zd8iFLUN|VlwW(GF?~ExQNU<(^c}3JLKf@-+^Q5o&tTP(eGH1$G?;v5F=|RSD=bSiU zqA^jn24S2hpITJ6VU;~iZD&Ox3ok9N6sK4dEx<#On)-;1tX=}IDUX+_KMX*! zq$8wTbvOunf5z?im^dEjKdl8!Ipx-`y&go9B7JssUD@Eu2d;eJU&jX+?yIr416XpR z1&N~#PU#yzhcf$|n3rB-FrdyhPEE2^l&38?b#8Nzt(76A8X_-&^mT-OrwkJ`lI#W? zRY9mDU(GP(f*6Bo=1h-IJpbt~O4i&4r;A)Xkd4B2mV>52#;Sgb*rgfV$X=hSMN#`t zo$W)lbHBf>dR3>YlaZ00NE5{VQg%jF1m$Lu-@iyN^bKat zV`rtbx)ZgB3QyE38zX+Yyqd>?GC+(b&LRDEJ@EQO%^g+z`)wJ7>2(9w zT;sIvqB=%DeH`>&l!kf|Er=mD^hO`h%>E2#o$xkrXR?zb)zIo8Rh9y-t}7e-xjvw* zrz>M_&i5^cxZQth9m@>E7rv7_9>2F9$xr;?ZVPQ=;JhGG$}s=!v;6x&Gnw|J92Y02 z6K-wX;-jvEO8^?HT|!@Ry82^FIgs%+cx$gLNR`!gY#6F2pghU%ga+xIU*v1O1Dx*V z-6?zJq!}_V)^TJ#8A<`g<75u`&!>n`{yEOA7V;8f^qf5@*As44KbN z)0Jvmw`7s^H58e9_niz!U#u1lir?`Y99HaK;d3$?sj!mc^P-VvvPLwveg zGLE~q`ZMP^AZ{B~)Lwc)gni1jrZ?lkyV6HGK%eRMYEW5S6aE)ZrKRrdXYxLFHL70%5{u4sB2IRf zR3p-TL5fU!(*-~7cvF8-(qH+J#!Y7uk02`J%1X(*y`QdO2oxti-ka9;=Ctlk*gJ^n zR?2B=R7zb}-nj1;ZMd@xY4EN&hO8z0=-@wN;NRg7)_xL6WhtyVoc_HqfxJ1&E$kAg zgSA$K@eVfzxE*CivwgL+y*_|DDY2`bKM=g_-REMH*Lk1f)PAWc4OxDAy%2|iAxm{! zCI01{|LQqc*mjpxZBQgi3uVDJTG0J^?kij@;?AyZKI|68*?5UY)?1f1Ym2ej;ELfy z({i;zzXB-!OdrsmCJQBx#f9JpQlvpQM$A_e%<;{$6YR1R;#8P7j}0cNbdM)aKia445=z-6 zcxKB#D>}(ImF1gDSme`z{p|DlfG^>6KqNkM2^g%Yk$YXAQuaehYHM70F`?eFO%!x< z#Pq@MMNnJ4=c9*H&bwuvJbD{E&)@Z3CFGhRJG`#zrjH{IPE&-4|Rb%gy6Xvwz} zO5^*=C1<2Z5X#FJ^_rCxBB{c28{=o*l}lE&TLub*iEZjlKPF!rcG4OoE9oYuWI19a z3}~0)C%TG4V-2r|H9oQ(c{IktB8}6XIxg{UM1glnfsZ|tcSM0DJ0U=u%95d-ALH4` z6XuJ}+e0ZZrOwKr{BcDxEDr~*9!1vm;Rp4ly$BJ}OCaR6eBLD>BW*g6i=FZ<_^58h zlQLVU3-c`g{k3oXs@_U7OJ|n=>t?}JtEHU$mz?A`H7YgfFLEAGyp?E@ka z>LdMri^3s~_@@*-yNO=Y&IKwpt2PKilJfPZQ|8gI*Disv5(zj{-~t>gX*rr?5t={= ztMei%EoP7z**TOCV&7fbUwOX+xi5G-TxLmMnxD4s;ZSn+E$`vp{D&=hXt{54{PZnM zRi)lW<|rn|-jC9mI8zfDN*Ly&;<0^Nqi{c=WVP&&*KMta%zFqgD(0pFA8yJ;%GigI z>KE7K^GM!PvVB&Lw&Z%2cxz{7pog`s)LO1j((1JnhsEq5erKR(lLX2RF-BsY@hn2s zH4Ah3?6!A^<*_K$7|H#qhYv^hVMzw*!Dk;N9%e^-y^ z^;Wo8!g$=HRbo_>)d9W}+cZpjCcV7q3Ezta&g}R3LJq2Wgd}4N)^vnIhlcAG-Mk5_ zac2rs2HM3|ub>SQNNgl4bGFx6&+0wW_GMmG_Do?M6`p^wF-+W z)Vpq+_*4peH8j>x-`mR1I@FHVS=j!kwwkfzs+KQ`#g)nEj8D5yg6|X0es2J3Z~nW@ z&(n-cK-Iq4o$B=bD{WtTAO7&o=VAl7jt3)4(K4y&yPxfgm&@m*$SJ31 zL!K9tEShP%eNof3QtGTQEDIo1v!9!_R*)=^r5m|t@3us{MxRfbqy7ZXnMKW?cllVx zG<90aAx*1+k&GAdNdgkP#}77Tg|K=N{{zeK@~{V}JW*xk7uCe-p+ zI~FO>$=FT%1A1XG7{?oRT_`~1zC?+17RB?dpP7wg5n_C6iX{~xqeH>6t6{Ig{21_k zSrEmBh7BXHIE*;gzkUs>Q;UT&M&avDlqIhyVlT@al|;2K3%rv*(cecZ<)nQY`~-8` zDfb|KSqfcXsq8unDo4P=coDJ7-PNC6M1ySBuQ|vtb;?b}>|X*r=~~HTYzHWlCBx_C zH-P1s$=bxNjDlI5jDa8Yf=uU~NvxaN@kp**uaZi9PYI4H^vt_usilvr5!v1T;u3grWJlqo-G6L)?fOo@k){L<+;hp`CWJzz7T5Z%#6_w4|?2LpaS-sIuVto>g zg0E^6Zwhk+Fj@qYk+OyZ7=Y^Wh)<4HLG7Dwn{d8>%=pdBjypf@2}2GddJ5SK=&_~o zGhY$ws7lblc~x>>8Tt~~pQyVCvHLH5t9lJ1I~zM(O(^Gi?;bZX=CmZ*KJ-KKWEuMK zb6&zRgY^j8E3}a>Uab8=CcV`9-=H23X9kQffvvG<&-}$;&MtFO{`6^PW^(fZn08ks zN6wt+`?sgBU8EHpG1X0qOsuU=x=?k{kFuLy1p@MVPonE+`;>(NTqgAV7SCwypjR(4 zZRfO0>bw+s9^*UDBfoi!Wirx3stp1ln}%y**%jJJq_{RW!XNlRwWYH>;$M=hIk7h1 z@SqCrIX;o7J6G_Fp}YjN@azjgRH&;+(c)hAVrCIt_To;2?2uD|o_)8Hwe41W)Xf9B zOMpmHTWy)Y+J)-wnZb|bPjV0w{rPi^eDle-R9SB|oclg3-zHKMZ8;%ZT(sSJU-2Wk zn6@sr@Pq@*4}NpJdW^3Ugnx<;xb>HKdMXs3%66Ue56&B9q!@IadReB!lM5%kBaU6= z&J}(<$QJTPF`O;I3px1vUKbA2h0W-mCG51SmK{ed+b(Wi0=<<7oaz3cypRT9ea6a{ zr(UPkE^ys>@>V8Ba6t3B;IPsAcYGu^2^7EIR26?Qjj*4TkB#FtQuyK8^+>dl?U@e% zydd<=wg@XwG&vy)hFVuyX1u;fYm1_aEnqe)Bha0p)aW@7X~x!F6DU$I7)cFv8A`M4 z`uO4WwZlrdU_lz$4OV8g+dy7IhDCi9rh;;%?00IlJM?_i&%=}}wQmfgQHTPD(hlOVWn-C!q>{*)Ngo^L7HDzfJaakB9%nG5n;J_x&7_Asg z79lge$(iASa7eMG{TFX#yO@x554-4S1#kvb?*F-Q-c{WE?`O{vn5`A>RzW1UVbC(h zIFN8F)mA9B>$oeFDMrw@_r&}Na^acMtaemrg(X^bPsM(?_9aXSUgX5LZNHyIHVD7V zX)Y+1*{i~m`+U@A^ubLGrI}}bihT+J0trS`xY;)!i8DM`P%=`EQGa`X)JiiXf9AGT zQ9r)Ep~8oCp3t?UqHMk#4u5<8+{xtM9vcRWuzcy3)S^k9}ZX$!); zKL<$OQ7q!MluxtIN-m`PmqsQfh*XdJ&D{AG%&;t+Q*^b3%7xXg>nu zF%dcDdBr&yzDbKq6Uo8M#TXTT5B%Y(&wn3Wi&gj?$?Bvg<9T$NbXOV$-(6C)X7gm) zvDcwh!;?Efny2=HnL`L{5q_Zq653WqXGqZ=ot?7ShDz$uuP zk0?QmbjEstz6*sPT>?}axmibf$C78Cgv%39y^WVZGUP9x<~r8fMZ&?bE*@!_nI2I% zP)3y3RxjA}vwrS$3G9XoF{1c~ZZfu1fBJ3O?)2+!JiG)B+AaZz0X+LQEr`eV5+S`{ zopp80XQStIUW&JWtZHbC#p&~v?@fOu8?#B}X-5y`2G-&RAe&SBMI-4JOenoe;4ing zgp|Alm?RgV=hx&efz%m|&#GrDirr#4O7#NeX%;dly{5mkGaTgOxdb@pPKjZi_3kVe z&N;6jsyQ>CSoV(dVJo2SksiEZqCh?)uu;k>F|@OOlI6lZ$N0jtZPW3*d|)7;5_WnB zLdQWU-nNA#aCh6Af|62#mq?};Y#j-6ZLLC7a%PfUE&-g*Qqab*Q!;301VWlG;uNIn zb?2e@*54~%q-m6)z8D^{di_|9K`80exGvm!!(c+a%$-f@z9d*8Q~idcDBTxYOMys0*m?0A&+ku9KVS8EWm8v>kpo*^_hh;RUgSX!sq>Go zgHQL3OW^AuLJ8HP0$YEm_LCG)mb$vGZ14~B0kkC)2o1!f?WJ7!W?lk=)&&!6UH@Up zzhdzHnGC+GzWz&p!j3=$q7(yLVaL~FE&++Uk{2ZqI1w_5=q zK5!LQUWLs6d|QUAu=2kQasGAG`u}q(FIwqp$UD4TI)}gYL>edqnUVy;qLr9!LEoq2`qIa_$N1ZbbVt z?CPPXLEj=XqAJS#BA$5t5}@dkcix6Q-y8Qa9(D96w{)&1z7xq>jk$lnIT+YAD|Ttt zj-4%?7bSnz^r6qi&Z30b1OE{*CHs#67Bj{g1k#u1DU4Wvgl3jm7KyC~ghfcxJ_{Rh zF~MN;Fq&;FZ}k=!6frkmTR+L)XU5VoJv9!wSV3hl%vK(8aCd2bj_XpX2;Ll14h&YC zCIVHAvtJw$v3rvS0V6ty^45g6$HyB2s+6$gnQ*%R6r(^V(bA_+fX7L&r;PCelu+MN zWBG&C2puVM_`$++kC@R&UDw_)Wph7S+jx#* zvmDoFLXa)=84uFc)rS>7I-Dhu5;$hZMdris%wQCpB21aR8LstGAx|x3`zowR;$)9Q zSuR?Z9YvC5RK;%(v^Kr7T56rJk-%SG&T44v1H zJLx`x*2$GUn`3*^sPALS10=G3M%#8#wN@c4NF40FRLrpk19<$?VWnW}pWCl1vRM1f z3`5_m-y!Icv*i}wBoZx{@FZS*1rPK)ys&$Dqq3El@x?Mj0oe=DGD6W(t(05R-ed?r z=)n&3kgpK-CM$m7b`1TF9pj2j-T{@*0})9$TL%@GQ(N0hX*)N=kE@ScJbpoeo`Wy# zdYy>!&hp{OvNw~30r6`j-My$KJZhZ+x$Lm?(W3igT5`fV0HMb#t|e_)x&Buh5hang z)X^NLqP9={o1Q_R{PYI%bzeDCP+t~MCO$&Y0LcDRPvJ{IUrx`;-T)uE`~l&DYKX3z z1VHBtp1bK3f#KRKQ&I8YwveWZW$rhLr1)=izA=N_WYD)xuwZX`_}ze_TrG(lR}*Iw zOTE&-{hp+p)zh=>wa~MdwI!i;GSE&NEJ;@OsBWw8LjjT)wi1}`8colz--8n*T_Ivf z?u)!>PSF3^*0d(^8r_l|#YS@r=!BQLs97TwsjRyZp*G z)oa?%SMWB0oi;~Kh>bVDYdRN!MVz~J$=Xa@^tL3M%o=|F3M8e@-4Q!sOGIw3ZLuk& zaG_${Q$B5`1wZ+frEj7RrYQ>`N&t9}2Q|nUFW3<1ZN-Px&CZIpJPYksx}LEqOSbI( zFv$D9>*SBh84^+MuBV){k9p`(hZv#g%`xF0Po*UaAh(wqJ|mRUv=n=W?~W4d3Xwc7 zYrWu^h*Vn8kJ-DRi&z*B7{h;!>Z=xU|Ih*r2aAAer6X0~MFSlKTy5H8DOd@&+cnft z>hiZUqERqV$xSr_lNel*rG2$L~eo$C#hX{K%7>(>wt%}Hl*-4r7OQ4<{ zPw5xjwY*PUzJ^LkoUqr`6EZr7yj)LwOmzlj4doJ;y=If`kiy{5oZ@K<6 zzWJfiiU8SsO1T>WY;jBlhr?Rp_GyBTFjtTP375#j+zEt3Ta~%TA~n*@0$C{jiBpD> zDLE6sQX=cIKye`o_6{dEzcA}TRfbL8GnO{zcTHFC&&C8GB?iLdyK-#rG(#g zN^5B*XT$^_4iF#|<&^@)X355>Sh_1bbo{K_^VGXf<$yzmYHj#b1g_^}gak+5B@j@2 z2~;?WcWN2oHCWwVez#AE%^sM#91I+b5;VM;#jHh*%d5ffhuaa(e9`iRFn^*k66o_h zS(rm8K^JU5&eD%2!%N_8BFRO;x31XG;-Ai2!6!`tvrxZ9tT%V(tIzpX%HI_mVzc5> z;^zz=?A-Ux>1zNHGo}ZN1}B`*L&j7h@p`y){rsH&3DsI~T6LnPA45;|9foqTiLTqV znCBR}iXnNPW^lzQ@^`#jGxT`Ep%wgv@9;Qfmsn+KTwaeCB1+nvlR5Xk?*zGji=56y zf4mur!`ikkzSQU{tb>$Cuo1iJiz>CfJ7u`G_)N*}B%Mk0i*E8E+Aa!WrG|vQoSdY$ z2p~IG=?Y6bUjpmrRum-fX@ppLS}ufGS&ZEa{C4-QfA~}DMh(#mi9?1gN8T&Q;hzt3 z_-CGzzYz}kZF{PMs7*ywE2qefFnq5NG@h{{eFJ$p?>O1gwG^c(v>FeWSq{0k!0*~YVAV5Ou1J9EwxYJmM^!Sr@r8)+u`dhf}rv$Yg& z8Tb8*n~I3ay)hQvu!r&$O@ilz4#XK}8_%|(4Z{ct#-A{N`PhhS^xMh#ZYvM4}>SeXeN)?~WQ1eNORU1kL~ z^y{Sn2k4E%#2J@nq0W!lZELf-l2yzn`8&dIHrvrgY74;yV-hQ8Ati9kUBATY*m7by zHW~WB0H|e%Wpos)oKgl$WaoK8;axr=HZH{o5}aLj>yVo^v8QC=o>kI!B8eX5h&b=6 zw$I`s@|I@kc8|J;jQ!thn?I1JF^V}5?6snx&8!x!zaRr|WEOLTi`NEyebPFOD8|Lg zqO_NGeoE81n-S7HE-6KdwroI*BwM`gu^8cBnZ%=XaCm;pXIg$ux}LwUX73yBQZibJ z9vG_b6TO4tJLQRJkcWv7+tCQM-WZ;WCja)@$UbI&U*OI>t(OJyA}q|s?jjnBV?K6p zF;;7WCR6jR1_gU^tj&xO1-E(22ru!0mR3@#h)Se~OWdb?UZecA`KRK-;k|~OM^%XN z1r9;ahdCRz@~=Z5lM@<6N`om`yjvKQU)%5xMk>8Ox6pBJ?f`55qdM}#zh(Ju{mHz}+LMYZe?mSKYQS+8u0aEaL zB~fS{Ec9hvnV#6h9oLU@eZtTE-c;YdupBYNXf+xEUu?7e92nZ+x5=MOGBngbOV^5Y z!p4_lnqU@16v2FHvB#0rc8f&Y^%$17zxwFJ@Y>s4e>E^|6w%OVH`eG48Vd}S4DhWR zR9(y$Aw;@&iyL2IO(&&W<**$%zi_`fp4Q1m54nka9Zvibb>nl~hPXkcFf;Z=M$n_R zjW^r+15$wf7lV)$g#G${3nX}Q=JVI;m%TCJ9qvYx4acw>o|Z9+iSqHSaSC?y6kjH? zgou6Tw)jmCk4s*s`-q$`FtwdWpdp-HcUQs{q>m*STNz?Q(fprzL`BP;9DiN%RGuH8 zay4E2G|RIW7OEYpp=D0TE~e6hQGhOSCn{2AX(YuV-A6!pE9D(kd|U2L;m@MOIZ{eL zYOC(Ap9PoLWmoLx_KP7}O^B|=Fp{OVaaz){N^oj#s(CKMO?pcj@PE1Bw&eEz%M+us;;U*+Nc=j8AH73S(H2YHo){HLNW;zygMGc!ir=$^X4gH~Jm ziAdE4#_DD0+S|6Z7=dd+rRdzp>ya8Gz`}~`{;?f^-aym>yX}-h_>H%TOo}tj@}`^B z15l6hK~~_-+V0XE_FN_4BK99h#cMMzUEb7*OF01|O0;>@8}~Q3R==f2hgHp6v4sZo za#0L0jFpL)!^gGYq+RlRg(@4zWz})zM2OQ5F+Nygt6pVZ=g+W*hx$mf+U*%WOd0vV zaIWI2Q!3=bLU?&{EpR~P`Zv-d9QRG{uGoB>65Vef+rA0}uQmh!5Fa4hhjkg? zf#MF=Gx>F+Df0rBT3tfSBt}RwH4586Q{xlz^MEHNt8& z+k2c-hEDbr`{Kaw$Vi+v?21-?#ku-N`GD+t83RMEv7s+cH+@_%aLR=u`;@X*4_zSH zpZWMc(rXE3Zt;ImGwSP{yE}gns+xQTLsU(1XusH)~N#H@`*WW9io%-Nz_VW9cEIBYCj#4c&D zm@^5p=Lv|lXPMjcu>>FfR>=7o+qhYgSOEs?=cMS5WrN@UpBc}yLHbK;Fb^4#-wZ9!Jx!JZBS9>15fTA6T21K@zqcpbs!%@C z#6EU{@N_pV0HE-G?Je}=Q27t-8=Zs23IJ4p#;O}R@(#$WLhc2kP{GjO+iMgL$$WnS z`7!D_a0iBFxgH7{=QEFgti2^VZqDZhxeN}Py)QB^GpSBC;gFlT1~)gw@rgvk4E2Mv zV}iOuTRVMK{6yoG>UA!JCby+AhYFTP7z7IVLKf(i{CR(ZqvzVRw5U(tB^`MAsWvrn z(>ItopZ%()#=lQZjlb7icfehc!3hx;_NerR^47V1Uh}4-X^lKYJxWSJ22~4F*4J9F zIi=#AJKJFkAx3H@D?Yqfgm8U3o@7q1SJGt8R$vamy?ZAzSo6&sdP9LCysG#NsnREg zxci<>;!OpMz|yq36Kf{{)96~!JO5kn>yk^Q$W~{0;LxcyWB89-i6($C@>jwD_=svH7!O`@1)p(F=4oW&q9#x<4$)NS!H;|Cl zsHD0V0ZyU|IEmR+1>~*JeOB|iqu#TG$)XJ;&Q1nW;Oq`qeZ3F;-}%76tFiwV5(MVI z>F@Xk4gyRxmkWRb=^g7&ALYkJJkHWU%kH}V%Z(RaW848y`@nTIW zq$6l~2{uc&j~uW957nqR^6`{5Sw}-(XHxR#7(;vdgV~RUAC&^_I%` zt-5oMI&%P)dy0zJ*W4p3&MB>t<+3{LUT;Y*HHI`Y=U`{`&!-H%y;g*&#gcxdn8Oh<#Ur%LFZ(&^BD$TbfK`mFGMo@vV&Jl2fTR#Q^{d z*p?!dCJyhWdve37LwiAAa>Y-l3G9CZ8;wu@9u&y}lIYA`bcwUSoS-S!$oUyd} zxf4FoyansXaa3s>6JbHy`@Jw>#hULn=@pfah6TII!k5plpUBIC9kLabzQD|5W&eS3vX`+`V&6hJ$tRdxoLm3OnlM1Vmw`WkupB*t;`~{r>xeC;xdEk&9@K-Ahgt zALzzLiLiB&eWc)yE)=?9D`mT=mw(Uo^lh!B6bNhatP8Ozu$Q%(_ysB^xXJi@nLYDK6h$`ux990!jHmTFe&v+|d2 zixGqCtg0fo&Kiu4eK$mi#O);~%XW3kphPU~>+_H}$KXm$DE1C~6wAf`{R%f;73=(y z=NyB)V{V_-0tyC~phkioyduRVmi>*6c}s`;_+JYFt{>66Q7k99GOe5@`(Far z4;~%Xf*bLs3k&FizL|bJ^yK6cDALm-8~K?waVS_+UIN+?2#o~t ztbvOQu$Hujh7@hjZNJS5SS(ErlC2MKcnyWR1dc#Y_jgNk2iK0dIaDP`76rTd*XgOC zud-L5;=+1=m|*s~3YlIP6WLDeqCU%Sah3}B{IAxQ-Vc3Q7>?f>CLkeDVBwzf7PkC+ zqzMei_$oEO{9&DNQbiDND5Ge+ZYV43PD`3v^7XZc(A>z7V<=sCjvwvqTWf>Rrp3Ml z(o&SbS&B6Y)RQtb7eYmpJl;1K%F5bjBD7l7m^x}%f)`O;=`#Nty3DK4==ad*KMx)L zW(0!8*DOKxMihQ9_p|%mNERk;j)rWykLV;Bqf5bXj(e+zuHj>@v(j~z{$H8m825?B zKaKHO?DH1Y_P%RBCnvL9s|%BnXDM*`?!1Y*gej zLBh?y=lxu=FXg)k6)bWXXJ!QC1kk}H?3%pYLw_r(-tC%i zLw;G*ML(?gnrcdIziN?%{@4ts~)vPBuBG&ApHdcflnuZB<%>*#HRO9=^TZd2+Lp<+x^f^6FDv7N5@e7rrgZ#o}0UG_UTt z1H#EiKgI(P{z(lod6Yc$+uVd+4{5w(1%hg1Bt@LeRkLf|lf;7H*WorQ96G(xGyVAa z^;)8c`QXFP5|lK-r#p=!k4$ZdHu~&MlGXkhB>3tV{xsI;Q0x8_Zsl#A5(6%&t zDE(lLF*L%Kz`4#Km^aC$t*_YJ1!`HQ`QvMlobB^hdHQLM5tXh=ekc}Ck-arPjtb!v zB7*@2$&l4^M$YnXnZD>(Un%EJKa)8eyN(VbLfR9I6QA&Ikm^o#aH*m7gdVL~GDDvL zU;$km#N1?=l!ByXBz*aX8ubUF*1I-EV90gJZ z;k5|CoL*P=ru|YK^|B*|T2tB3?nIXED@JMax&q)ui9e((Ni*0TEkz$FBi&P$EP?4n z*3t%#Z;Ae4@z4s!t~|k==_@O1YBJ+5@jar)J4rzeL6)bGi{vVoOG+xb7d<2I{LBRn zfz`F3ueN8$N@{A~!n;?CS!%<}fE_D=X{`^siZETbiLMHJ78y)R2uM{Mt&VUAeS} z9d#@%1xu{%hTWnV<1>B5%JXLKP0EwKhyZ=DVH6~5F2GYO?W-kA4g1MGpuZNdijw_` zK|`F}C_IQgNJ|OlOj?{F;V_UlCnyZ<5ow?Nq3YN13Q@mk5a3D1sUs-1p zNu(mg>G3LbY%Ihny*FLPEuAmt*fJnidq`YzCLG&yP4r;w%Zojkb^>fgcy`HR#I1>$ zP4aPtx`_@FS$f8#F2XiUApgOjiI14iUSJ0ozU$cfkBw^X4tH|fsyc=6wlA}?0>^wm z#(VTx;j3>7J*5XlWEMi$8pRfIRkx)zqoF*6_iaNRV%Z5jUVbV1n}DY)@bYg1=>Aph z^k)E%zIig4=0gURhtAK6MohW=*OGI#b~Vg0hhLhDw&od_Q+a1Uo1K&rRI1Z+gM{g& z2gFQMsi+$%lkh$ddz2o^n%j&q9BuZp#(yD!FZvM_B>!M%fHSv4ytS zf>wSpi0^gGnbA79jImKq6c`i}E^IQ({EOo=v)l%ZjtVN2AIYHy>s+%HUbooRSUJ)O@k{I;bWX13qGZ*{YIGIo2D8bLKSq^MilcK;CLY`l;) zpFZ7(Od7cD&q10u9LAT%dw(yr=*BPL!FJ`Z@8DAKy}SIk<_Z{Yb{gz6xcZ@5s-b5r z1>na__K18dUr0-g5pHCYN+p6i*Vvc=^M<>xu$taOnGs$yS^_S` ziqfDd4;f4J_QHuy*=O9g*R;|O210nZ=6o*!l5M0+#$&}m&#wDic)s>~DN4Is3*%Ug zY>Q&iDL2K)ipKoz)V@$Z8+z_Q_pCXm-0*|Z=*#+TN%@f@51Voeb^hwAb3ZqY`-LG< zg9~pcvp$c!<}FaR%+b=(n(~zmp1m7}CaOq^CQd85H??%j-*_QW53l`%#Jn!xlkb#c zsh}2_tG-X{3jmYG5s97VYO@Fl6wxct-I%;R9j~I(vD%OV*pxTt>THUy!BdIoFH#T9ix!4F z4GrJdjjY{h2o_XMf>+NsgPT${Nb?kR$_<2HJ9yyYDCJI;8QmdfZ@X?7m^>)^Ak!x-*W>{q;XVE;jv~_2~J6kNoCRM)UI*7nEST*Kq?U> zVq;-jWlDXh!6CDKMC^mIiJFRMWB4l>0TfL%Q4Ve8s5z9zlL(eA5%8Zg*XK-noTQ;bwtE4i$5n* zB*NmJnWStYE?a0Mk-%z$%dzt%5IG(Kjw|pIcof$-+}}B(Sfyi0t;zAipq`ZYt@0s8 zu%F`KVc6|eFP)~o3cfgZ^IC|=>*|i}~>vb-uwRr;}ubRA~_u88CO%3203ipl(=MA_U%=kPE zP*bD0j{SnQDkIB$vZMM(r7)i51WyX*;@v2REngyDw2lFIMsw7=w?Zwo(dyTu+HCGE z%5@1o!(Oy(Dt7N`_9huIy*cUX@HF^^eKE)TvXqxH5dW*<2>;4-n?Gx8Xd#Qcr(}GP zvy}@eH<`d<`U?29u=X7i<`w*J?r8c2OMuebBZ5X}qF&n)@?skKiV>05t-6)z=Uk)5 zDG2#PqBGpKQ^;NhisS+VhA4tN4yBT#m)qa4`5=dU>AKsJaOm+@XjxGs5E6 zeJWAQd<5>sdEHNF+>%c0ItZ%m4E+UElOi*D^N(T{UwzNNEQ=kB>lZj(pYm}`d!4&% z{`v@@79dfg_Mh^0ZZRODL_XP^w~HBk{0aLJ2RT@>0!bzDNM_Wx&maYt+UESQX+b@= zxR+scCE{htv2h?0F$J5;`t5-&XNRtJPN6Qi5Ni3%WrUi?tYGA)2^$EjzBn`3%Q)w5 zLwQ;??GA%DyLNiWoqy)f#~K^P-idf-ZuTFBQ9=dhX7s*0O2YSC>Vr@I{c9dxuJOUJ zWwX(yt=eK;72Vu}a>iIb>$Ndp78a>GvmGF{~N(CFKVZOX=_N*xFGx&;$StR*nLBIFwWGtTQXl#T0t!9|1n~@%@ zBZ`t#cJV^!>KW z?*AO8i{Z|-ex5g`(@nj`pHbGgVrto&^LIOy?rMn{ zCQCm!JXYXG%{?FCu-T>&ZTu$B{#uC4FiIcShJLH5QfJ`B)Iz2G-6%nf-s*`TAoUzs zfmR6*X**jD%#~bfBAA__t8%jJb&3k+6d1@#|4(@%prIu)s-@An*( z-j@sQOkE4hq3lvodaX_lQal$u>I?J?+O#u=8@JzoaeQgQz92WbHE^OybiUK*r#Q;N zXC*ZEtTf5AvPObYg-cr10E(#WYJ)^L&(gKsi_b$Hdu1?KBFo^@0c}gu&#>k?3ADjV zjPQZL2HXt2KFl)NvKbN1=fyi%b{C`gT@%5_^{ymilWeXBEA4N{?Y=fsP>OD~dE;v< zEe_DQ26mmSdz))hznI=OS$>1sv@*!hsQ;H;YX234(*I0j_`|E~7cy9*(l7br0X$gu zP&UXUv(wd#SZv?7YudeW>TOyzBtbalt=44h?SdN^Bqf*?bYx;#s;;cFT zhqaDg=(EH2A`5Z0gyM{AWZ0pOZbwDeQY%-@mPzt&hD~4^?ek8>-`&o$RuhfjYuxgt zclGF~^1c%%%MiAZgtZ+mG5sZVTteSt-iE+}U#MViws~y*oibTnsn{$?W6kYR30qdY z<;0|TbSsC$WvP#IMAH&WPK)?hPH4%Z+FK}6)G`nA296+hqnlU;XqltamjMKO@+`(F z3}z}012rXT0d)Ap?YMNL%uHXdv8sGZTzzbqul6Z3pH0W`XhmNTPSCMhz@LO)7#Fu@ z#QMp``t*$U!M1*ZKHZ{@IftiwZr7bVAAfXu1PBMNTRc$6T{r&Fubdsd$s;vZd?>$M zQx#QZdJ<*2Vz&;S9&cTp#A%hD%3$=F zk3vpVLz1S<-g9VO@2{d*1t7WY>Hi_QjcvssfELSua#No!nm~K^$>{5QVcXW*K|Qz3 zG0@R4*D%t^j@0b+2@ozB2&ygRn&Gv^*dz38We57t)bhmD*P#vWt3CqtT)yN&0hL%8 zOHk4o%dghn@-Fyjx8J;Mf3pct9+f=*&I`{otFAM%!>O%y#qjU%>(LNc5*T;_7Iw31 zB0z$v^y}x3YDb_IU*^`P6C-%K_p4Ua?85pkW@~qKzR>UOBrYbAcrmH@^;ITPwzRPk zO0xWvxC8`>z(QypX$Y1@1@2oThc@qAjuTX4&V9o9F~-q!F?K^u>TLi|{$UX`bFLq> z@&)HUm?!q3w=h02qsWfeM87r{_Xh(9Go}l+aIT2$2AN_@$m`FQT|W++%B`d@BJpg5 zzgR2L9By(Hhvd_sGVAse*`cG3xbM zdboc?5Fd5O2IuQ4H~vMbed*3=?s|JSL6HOTTVEguGgtW*DwF8vxWioB^JhJCJYKen zVm=xQ{}8F$m9XevKwS0*gv8XpY1wQ9wpCN2`q#EerT_0XX}dyM$~pP-e`W%8>~{?; zri(Yqge%CI`3v)TqSj^H=hEM&XBKvUVi4H>sz>7-G>@mi4d*N_R+x@T8Y=F~Cwl27 zSlk)R0`iC}q!iz?ut{jLwKu5##*+qjS$cgQR=?~wJPlgIY5osav@7n&@7$3qTm1KI z@hj&Brr7B9P}z5%SHo3TKz-eEnopQ?w4yhW$*O?Y_@O&(XG5J*;N+%kPigZe5&m5O z(q99=5fg1E6C07GqQ)S=1`2chS=eSKyU&@;NA3wkNN;+IR?8|;M(Wr(L`5UJnX>uw!!9oEB zv*CdWA4Ur}KHg}5nX^~b_p4+YG{djr=f8dr#TF>zga5muhy60yy5Sk_XK@*#%kOdz z1nmn3)pvifmm`d*ax6+M;|*!DW)BV|_Gzs72IIF)88T&U^QNfvKXRAVM%YyqMmP$i zs@WgBo!RtAsMLR}mL9b;5)w+{<$I-e{O>*ISL@?{ZMQwPIRvT(q|g>2x26y0POnYE z#xcR8U8P)&cabn%*tj~%X~*3L>AMZx@`bI_*L-feDe|d$>7zw3bMeR+D53GWx~^>S z_w#|&NHoTI!b3n=S7E$85yx6zfPFQAPk_YwI`uTslR(rhtLh636OgFwhh*cjQL6n0 ze@{Ls+{5MvzFQMP_+R`-)9(CVGIAzUut9#Ca|EOH6xsLfjT2Ew@gU0P z+b(nQz41kbx8t}p#}1OzG?Wjq@OEUfoSS{ou$!0|$$hGcCQ)+TqdueYr!T%?@Z+FY!N~i0tW|gsp!8nSF0G=Y}VtEi9*@hQL&;-+iN+~?o?+Dz{)zl2^~?EC+wa(O{3e=90t-Mex zFMKWkovGuu*rpW#Z?+0>=?$!wPD2|t9rN!|N^eGQ4!kDkI(*7(*Kv)WNOH2o^)(Pl zG!h-LKyTG;vO)GNzJb8VkEcMsP!ccD@20Nv9MleEwo9RUF+SZyKnYu*VHKm+*B=*K zfC73%t&rnee2|mVy3=skQ$`5&IcW7s-#KV!_Z+k+WS`1GgDYvKXZ`lZPF13xlsYKT?WP9Zdi}#usxgl8JlM{9V;Y$3 z|9LJ~od+L@fPq~4F*fBhob?)SPthOCxP(IYm`tFDN6@1?VQ4uIVZdwh8AvmpT{;IX z1|HzCV7CH>I(qa9P*?AbACN~6`0FBmtd-@r06V)D3qHdbV&ve@lP?Z-am_9sjf>%Q zff|1}I2P%FNs3CbWS?U{QeE7KkPqRngcYUGbQ z#7)|kYrLzW&5&I?@moN@jHj?;f?*J#68%XROn$12xnB9{mOt^y|H6yz_BL>nA9G z0sFrqlK3gNzhL`+()^qY*#GU5W-eg=r=gkF1?>Oah~a|m|3b0le@pSY5P5!U0q8>H z`LzMZANctTk>@w{>|VhB1?>O+=F1DVf5G;Dok8&e_Ag-n??`4VocF z#)k}{q;H<+uSC-8cgBCy1#h-McPc>#c&w3%q}7TXyQ!*hj_n4C<4BG4;k`-Mo9+TB zrZh=Hx2iiBdQY0?J-Jjh-rBR%D1k*94<@#`cP2Xuxt9qPtnICg$-dZ%s>sG!MKuN= zzv{bOtZ}{DN`5dg&~;j^V}e4)D8ki8+ImyIAt(LOv#t+AGC1?is4%@g?9ALTQR!rT zeJWjdW98U6YVwpy8Mia6Zc|`ATFMqU@U__Wd6qjg23mM2D1N2hW*w#^mcY(ugWIo| zv0hZl3h&kPVWml5_PoXn(zFw&>$kKbh@-6WDU(=e4_%@FT)H1^B`En+wGWTC!_!34 zQ?kd#tMId@^cY{A3Eq^TB++nva3c1QTcBh6`VZejkZsh)>#W@N*Et1<#E-XUMyMCM z5+s6PS$tS;0+U|kn~J1Z4DKtiHwDknAQ%^rOz=J|={HkOL63?nZ;y(!zz73=cV-|O!vH(f{G3k3|>N}+H@S} zsyY|mX?dJHC&)RNLe!;s=G5CDH*1I@3lg;wiJO_Ga|i)1 z@NBa|l3<#GmiOH33-nK2PVj16qXbPRSHJcUohHAs28^t;*d}&6 z@+gMU2ltOhze>I<(r&f!c;wU!6GVFRc7qh9otg4bJ92&+?hl%d#kQ;1khDZm)Q6J| zFU3&$xLz1W82S3B$m>?Wo#HP!PNK`Y2|kwC*mSg*Pgv9adRn0#%&P{M@9Rzf2BM17vWut5?n~+YcA3k*>f2ZE zc=E62vws8M+&mSWpJr|{eId4wN!BuJuD{wOFw6{*fk?_Pzyc-&y_URtOqOehnHxeo z#$F?pu$W7gv9rWKVNI$UXeYY8Evy@z>$jgKJdv!dpnoAK%+?&oBCNq6Ufx>cqLkrQ z`6klYV^yE?x$|eOoGWVYAZ(x#X{TLGUnRB)^sOS#7JSAQUlVU#-xJR@zKCYL+M%G7-{Mza9fBO8-oi+`)k!2V*c%T6{D$)(f za2DDh_OPO!VfK|dFgCQHC@|25#>W`h}8YtUW3dL(j-%oJ;KaaqRVIG6=!jVzlMFYo}IBucu%y z?HYv_L!Y9DLN~)J9Va)Dbl^T+NO?^b}K(5l#qv<+B$sA!0$u~+Rh(_ zX}k$t<<|3SVNFIkelsL=GFQ4YZ*n#CozIQn?d#Z>^?cL|8OD1L ztJ{fcZito*IqX|DKP_`$%3Koib_?;5#TQf_7DqcKM844@&j-83PoVH^_7>l)?}YnQ zbm_IFyq=MB(6h(RK3}&siWEu;xE6@CupnZ=7YslQN=p5t=mWUOj)^Es`rMTDgJ$h#D9Xahi<@Eb%Zn&uU z!vIlX*Yx#B(L){GMD4}5MJ*?#-pY*Lsm6n(Ym_5Z8LJ|sJq7O8lsb|(BOMUmQ6I^M z5X_ETe$Y0;}dW;7` zK|^O)tMp~%kQQmkuR$u4cb7Wl3m&~)0vU}9y_~6c9l;H%CPMZqBVQeg)1dgnM{-k$ zn>Vrn6K z3hBOzorZ*Z+d(f+=v z{YG~BJ;}C3RvR1Hl~&;C0`bSIm{rdDW>gEt+6QiXOH77eb+4nxY!FJUwN}98QOqAs z=~19gqs;5(UpjdU5@wz=mx;ZzZrkYhq8-9gXap0w(IlG{po5w?7DYKJ$@8mM^>r!e z;^I&n&!9Bb#_otPe4S^ekzzG7>D%c}llbB3f)-){Qz_z~^KqL-VrKC%LOM>ykB1Eo zm8-*Vn-raL5zVUB+i!$Vj?GsRO(;Ark5TF?aY}jPJ0brf%8y=Qiyd}?sCx_=?h^;0 zUpcv8}GtwBdQ^)zJDRZe=eQ;u{;9HfI%hI7_(0wOBbgRW|Nec3gO? zTNArda}D!8yLiH0bhy@^jgvc<4?=(zOD$(Bf4 zv}xpal$_hw`ok-zFjWbBG_nkideFq4UOIW;P5;(LD5l3E-f1JWWY!5X*At1TdEBX2 zRaa9xO8+KWenL4s{*&(l1;esvWaQ{X`8g07UrxB^C|YQ1N28VzGQF3+<>-A zq2td%FMuXsDVFSj6XhqHSg^-flD{wp;43o*lis-fMhrxw2znvAz)5RG(TR{-irRmc z{QZ@5aFletrEZ1{_=mh>6wEukwr-yLw$qHga80Ga!Q%eYp4~dnT|b1rM>vY9_bJ*h z(@8{8+*P+zV(E2wgSJC>AEC2))hY{iSu$`r{2@1T#lfZrj)D$_(_%yEe1S}O?{E}s z$KF9Gr;syMR`XVb&&&m~+XiKl_hOGa|8X}bb-@WXK?fF8#%iQ z(bCE{SMEOefglC`Kt7k7_|$xW&)rgyOm5JMzxyGN+M{BGMyp28HAct!JDy@9f!~P; z-QEuH>syLZX%BMqu^-7j)vZ-2m}3>Xd~1@9$YtwPWBzJ2JL1chr4y~w774G6&!zF_ z)wlK&2?FOlm``OR5!0n#I9y)Nx)&^}Q9`-j^ktl6KwK(1VG~5slp3E#M5L#eQ*0AGqCew$U5SdjO_aGlnsIo$V+>?WA#{0H34|R-+0|c5@6^6j zMR^YLzWddiiZwraj`wm0>vNwuPo?{#NH`-37J3dU73W-a*)S>V%a41ZYRq!BSatVJ zN3dfL?m$g?Z%Z9A*6Dil_j8cZ%vzms(Px2ae6p@5HbRNlX9+}sFy#gxRi6Cbe!)F` zDk$gc@4O#0d)`56j!WZ68)LVFQm824R`%KgveXk2>e(T0kEwmry943S}OLjAep z0$s?UfTB@1m@w|tC0vAoHtKU|1$(} zt>kbVG%m5!0+dXe_;Sd-?8JAVa8FWK4B6@NaH%*vMY-w`mZE_tpI{er79i>(l`7)Q`@=4!7#|Qb(;(F%6Yr!PS%>O(o== zjo)$IzBKk-W<)Ke)Vb~0sKy#hR<~G!W;i>sI%DNu^MIiGkTg*tdzE^lBZFN=KQxZr z$2WF-0#0Jskf61g+X-2bQm`UK56UMrnm>IeK2Y%SDNFr#2GLMlRalxM^mfU6+(u5F z6J?l-UK6n_pW;WO@L~DO4s5O8;hgBtC~y<(rriSzcBy{u?UIMkn{@JI;uLF`ng||j6NY`&$E^qOhlinMKX6!5c@AcX* zwf77}aL708siSYOk9shy2~@T=bvw+g^S_{pD{6V>-hNYo9eXnT=uI`O`9KHW2oK_Q z8kuM`@4l{LEXV|z`%b7Yzv&2S5~am)8N%F051P*R5-B5QubH|;wIpRFm5INSu#iD` z;ysIyk$uh^*LP*s`y)^)M;+;>jXXQy)jissq!;zkl711ELOdHHjl8ZCqi6wbIR0`q zV{plI_}N(>VY_VKjSS3X^D4CZ+|eo7oD&DPP;Gg5e*V)N4W-m_VajTygAK!9yu6%2t2ZZ&*PO17fH}CH!)h#x*3dCTZu#$6)~}kH6<~;^icIy zuEN%ff(4YYTZOt{%7}b-M5P?9#G2ntN>#bB!IyGQGsm!GW?YaClMCJN9XsuYMrpF@~ayONjRvnGtm{3Zg)`0)Jg8tKio^ zohFZkpZ|iHiSRARfh$ zF0lS3nHhKLji+eJfps6>$^bv7i-LWmif)PpbtvyrqO?(2U&Ivbg^ppdpl|n%FUP5r zWvZTO#8!9={L>;nhM){{%vV|0#3Z zq7}BuDG4`i1w5hpE+Rx(2elvLZ8hGV7IIps*L2E4c1islbm?YxSmr+bYdsza1QG(N zT>5KA_P8OZHVOJ~P8u=SOKk2A!5>8|pr6tNDwlP&Vz3PwtS&p0_Ukr!-3hpKC)7)# zo@9agEn=Lca_^-u$vOz}&mlZN61PziyTy>uE<{Hg&S8J&jb?etM*?(I{nS-4Tay0QaAKLRSW@yAe(ZrK#(pJfLL3PC$;n=Y!*+P4{RZ zz2?H#V|c|^UN+%=#I=&tyIfaP=jXN}5ZhB=aXC)(;Ul`RCtGX#-9HxV>_I~sM3}w# zMhpArUVp!8nh{kd&q{M0yb7XD5a#bdf0;{oKQHF8phM}K_S*g3$)X9PTIvD?C^$5s zWbKm*?zOpIpBWmY{)w4D%krTrotLcxA`pAj1cU`MP&RPRZ&e%QCM0R?A~(#;^=jOS)RS)6B%GG=8kSI%%{+<)x-9?qjWQ zPx8lk_n57pjpkjyP(iDcN<7a3lhe;u&VKHZpnBK(8Tbc_h4M5QAb@3X((9XPHJX{_u7)(d*i}1_g=nZX$l2rwdN} zYb1u?CAhUArme|>?7CrLPMe+ zr=~@>rkR~GqvOHBn}naU!f)4^z-rV40*0)s30sF8ME6#)THR!T>J}kLzmrnfvH0Ty z<=fD9Y;RWc;y2S-IddyY^CM=>R@*8ZI*&KF21OsO25|DG)=JGrQw4N_X zZ%-yfqr9Dx3m)H{=C30`$22?6DP}Tt7eW{*v^!z&HXQeB`R@;`0O^ekm~+2ILRT|4 zq6?x==Cs20$!Z4kE!}vtsAWazXtAaeCSy3# zBg*e;kCN$aDC62OevvKD*B`zMpOSNPB z=o1DXhTaEBC7*-7S)9nMi+n!Xbc5XVWf-#3O9UTI$~CPSAPiKIxKge}*owJ&mn`jF zeLq#%v!rIPpk|H-JG zhnmorhjk_}MnNjq$vtG^sZD{9lk|q22Kl9UX%H!iK%egR?u$^dR?Y&6_QP)q|CPET zMH8RPQ(Q3lFhWr4DG1i`XNxoddbGn=_ToLXvb?AP`+RFI?XUQWbS@*~M@XCni@|9SvO#u-9Qx@?jw zX|tQ?FeK|NSOu-r_xQIbOmDj*7nwh# zlCqb@lKksLtiON${|Pbce?w>cJ-+j2+^n227Em$(VSmOyI2bK?4!TP;?|Ak>786qb zg%T@?&7bnGc~brLAR{nIT#s^TAc(mw-5 z{-o;{Y~T{zv!cxeH>m0Ova_!5Rz=IEi*b#j!3$K~qsj{8Cj+{HrDy`>N0GhqNiXy7 zyv1#xg|WMn^xiGbHFQNoc|5 z*&Ft(Ph$L3hpIct=hj4yBI75+u-_vOMe#z%LQl(sjOuSZUoU#d+8 zu>d0E9G1lOUr4WhQ)k$3@ZFbrr*$WepeC_G)uT7l53p@%19_(w(iER1a3N0(Sdr;z zDGHvJFYX9H8t9g&kk1L+Hs)qS4Cs*`+6UaEs4JsB5x%sKsvDrAoAYelG3u|oFF_mh z&v0DhU28uoeg|!~uwdnTs!h5f9%Gdz()GyLMOfGCEBvsqLrfS~Xt_Z2H7ciBoV$Wj z*)JtVu`s?_u0Zhk(PA9vnfuf+_p%B08k#F_ySlfio4Qv0hgl=+5Pr2|4FY{$3`7ebcel_5Or@)Tx+yi&W7z7Tx6m1NmSOyP$&#vcg{ojJVV~dG zujPz*eWg*=>AgbqGk!5*Wih4?f#m^T!Ixd!^7hmvXkkAIq4ZX~peD@9Y)vgUV+=^l z?lCi9D(N=%4LfU$oz&+o{pMzGxk(PC@fgs^s(t4$dY`(ntmJ4ZdZme5%z2IybS$H7 zn`Fke^|NZy?%m0h!P*Qn zdcQJRlS#UWe^IwKr=RnpVejyz4|hepA$7b8GY!&!Va_W?K2j8lXY zivQ$DU@00WS{P_%q4?F@O!v~$KeC|%O4eS&cR zR7=#wO9D(cn(c;ztUFi<_08aO&Exy@JyaG)J)Hbh*b0zU_1)c(#*ILCEkIr^;%P1Y zu5u1~0%Hg-CkR4w3GFj^isAf1iu8vt)Z~g%-`xRoUH|5fPeQ2uU3kL1Aq;B$5@39E zQPX!P-ETV4K?3q6KV;v>@QJ2)ut}bpQ`Vdugdl6Flu**u?$Ew%f3Rjq7)F#Av*qLv zo;GE!^DQC9q$;0l`I!;zrgh||NFOfHY^kX11Lhmzq?cHkmZ=WPG!;I;C>p+Rbq3NZ z8K$9nT-LXz%gNSnZDoqrn2?GU46_|NGu!{e?6Cim?eUi?4vZBre-0v&o`+vs*WoA! zoB8Cvg?dYN2+q}0B4Oj_poZP{`EFUAI?s#)rHxlwG|X>(Pc>|aP8w@{hDyw7?hNN! zj%lHqsfE1UKk&(?~@Jf(``PmyMP`l-r-OWvj#qj;Y3mmq?Rn-u>`* zg#IhO(qnmDzC?7HQs>p6wAC#Iz94Ed9_>(2B2g63dFNT#DK`WyeVTIiW#tSmw6Iy{ z3O-?SH(yV@1T6Ltf=HJzcO5^Jlbh(Y@FP;WU`HRS8vd+< zi>s@6@W;%g@Lzv*Ul~&@|Cr$v9K0Y3n1y~D6%vOz=v0g>u$no`q;SqXo4B9=a}Aee z_C3hF5o&x6!tG=Z;-@jgT!M{1$P3rAi!Y=?X|zFh@Q!b(4d>bgX;#!$gp#*9h{6)# zv?Cr2cPpw&ku{lxfy<5w=D}{ea2sQIz)hmP+{k z^_R}yb-BcSYg{E!nvwHBTICe{b6D(c#3(X5>*2fE7cyot!z8NDqJNHE=GX4?S4A1w z$BS-?sOxe_w2A7s;kh*uV;V4Lp|o;)s5VD+*S&VyPSa)o_w*8ydo^nVE@1D#+GvpA*L|@{^*)y%D%w z;wd#QrSaL?W+OS)M%L8*_trCnjz>HBlTtTaLIdo$_)cOzw5kww%&N{*n7uIKG)mzQ z(JT{BO0HgL0+YUlALyFnaas~xQaz+PW$(WeEu5l#$KLM6#||5UcRg{5Ta1>e2~DfX zfnL9UeEmcV{=HF) zO^pn#K2D#!$(orEHE-qbqU_fsMHN&97&E{U&w1k*`A&{OVH&cx>sgtZ}o_zuk3|pq+4zecN!zza#Atu>%2&v%L}W zU`AKrbKaszu$cifi*=zA5BsM+4`TD<(GPrEmYuD!Q8mNU*-8b%k!H~pEpta;JrB|W zkABFm5@+4!T{4WkR(wlq0L0<7sk(JZ4ojEZAbg(HcIjk$C)hjD6kJ|HjJtsLr`h1U=p94uq?>( ziv4h`e35SPx&2`qsnOgy=!1QyyMi8Sxyth3Q1D&X^T1~vMMbdtsS_^v!9;qnx67FB zkBrBy{yVAk`NS+L_fMK12wM$b0lhs3H3@6G!@2+wE8!e;A`6EtaZS%YS%IHuNHrJ; zpuk>k4nR6{)S$j@#|8-*kFYI;Re3}2T`D@;!O6>7by_{ZqIldjoFc^Js=uzE?^6zO zq_&M!3=V+6Vno#mm&%We@crZ z$e`#!88pp7F89t=nU5KdTJ9U!tT*Ii(HdZpQy7i%_nu+{iKx03K)yIHbX8+*=x43# z{v?BHOrwpd}4%I~m3fdionJU*_=#@mbsO4DP97n_E-(=-kl@%~`^r1dL9uV0A zY?L~pIYOQ}i|><`-3xMdX>x%XgfxkU|35B?u_qySg7=|>a#xC%72Hnyo?!{4GL~=-AKF^4i`2{Iep!WIE{l9&P#nL@Yt=TZ>5t>-V!s`GM7bbJiU8^YYAR zVE9=v+yKs2{FG@xWNwG*YjPm_oqq@2|Bx{HpF(@~5Y%W$*arq*a?aDqtf9i&5Vp|I z{Hj5bg||%Ei!$qma`&pMqJ*lv{EyO7q}uMPCEeXhR>v{?>+d&KF4SC ze2WEI6NgV&BiyZpT4|yIx9(U7B-uJ!S-<^o$JIgadED+*;WPHIva|R-cT^rAfbuKk zm`t*R*xs+r?tKGkoqds>J!KGS|5!FoFmWWYH(RvQiYTlK3xe%Ft_D_c^~laaQaWpU z?qKCUzD&Muu5r%XIr31xUYd5sU3t;PA@MtX7H*1&lWxib`IOQsiO{}})cEfk=t5^c zVKx3_VjKQ|0OLJ#;+6=L8BD8)o~B79c-PSzEi9Uey_YxfZ~5%l!TKoiD8z>C-LsCI zg{1&zhhsd1n>C7QN2#uSz^>fKrrANP$;KAb24||Q0tT1qcDi$#QN*^5qWwT)g*j(% zn&t#U5em}Wt*$d`^-?h^X#vl}fp(<~p(lJd;BQoxC4aT?o@U@Zi<-tjmYf^^Fq}YI zQixfKbqw{#>!w}NM@NYtmy`zwD&{39m@g%p2furd;^zsBD7ri`cC6kg&Q*fak1LeA z)x<~THa1zhxqw?IrBCWiR6qZ0qd7wP7-D#0YEWaKVUnqsYTqy;_yW*Iq_6y5(ZFAC z9&=k;IB|Y~N>AVR;fTDgJ{GuS@PcB=@wqX@Hy5%%{%24G*Rb8066eUWZ2_udlp))+ zRKT`^?4jJajSp>0&BW8Vxe7z6Ufbf>2%f}B-$l0&U(AAimN}RE{XXZ?t!3Gvn?tHN zX^c?Dxk|uR*UZM4rZ{0TC|i&vEg;PBS@03UZ=H2$upF3e(y^pTsXE9qm^>MKM1~aa zQFHU(dLXE-n0l$`wlFi_Csm1FLp0SOU@$6!o@D_yRSGGXUULqz1bDStjZ1a4GD>?; z86gSW;VXZM_|X}+Mb}y99HbPl^25k&O@@MLgE|9!lV|wRI7$(FI(pwegK>LT zr;U6iimAr49_5Rv9MHr3YSrh8NZxmSU0|xJw9-i2s{LlQU}|m0@})I7(1@-6YnOs{bi5R8mj2 z(U8Qv_C9iYq#)Oyly)tpL@s60M}D^Yec{&i~hI`Az_a^GkyVZpAPs zKGedLnCb;xM~qRAtQ(6FT65Mn-~M_S-ddSUZEsoU+lZ;FgC_V*j7F0}r!3eyfyh}u zk(j{RyXfqN<`{AM*iQMud{0HLOAp&piAg;f7|0h_m})xy=M)L{gi$xEL?Wkk+*VJ^GGoGg8M)2-4RD9p1>lA+*RaIegB*7gWaYD zMaAi>+Dsf71*9F-jl3P(^*y|Sd)y>uUxB!?XKmJnOpB`qU7Wmdimd7%niD*bYE5jOq;U|{@*#tjvD=C8Hk7F*#VK zf04WLhQmh5lqK0Kq3)kYx!;_XqAOg*cMqiX0;s3+m@8)QM+2jn_5PR} zC!pB`rJXSWZA_K5htc4~1r)>T1P~9s&;9$&`3?=$Svi0->6SCRRnBZH4G1lV@ScMf zvSk{_K8kxk}^v&ZvuUY9is zt3QNMJ&b@p1}%w`px2dYoRqh;%z`o}+86bF$TfvJ%4rHAD&{Sx>!&o@mK4p@FvEPA3OnDT;{OCl9 zliMRVNGgXfi`wM7cIuD!q^MuL4%p~(0HC|2JUs@%RZI`Dd)S3*Pn-k+e320BH{OOm z6KannO%m)rVfR;$SVAvlpb`lMMq3QH(XSU=#c*3(0@q`uw`aNb{klBktGqTET9$+t z!gp4|B%&_0Sjdj)CQ8N3sk$h0lFyR~x!s}l9;sSiFaw3sjI|UcPxwcVr7UP3!Z?V! z-XBpKZ_!)Xm3rKc(BkqhYFQ){oEL{ugF8Gq(2h-Dl4{d5sr(uA$jW1d@a4%Sis#zO zbWslzu_xiA>*B~x2gF#NvA+B~D?0*__4cT| z*QFuJY)%tTUuL3bDDp+G%wM%={|-|s*JxF9D_uYVRg`4Y)@y~)P1);$lmg`#+CENj zhkqmCp_q&uU3AN}MXYd7bJ+$1b`-?d<1WPWRNsT2G~Q9i;9lpgGMn#1-dNzC7UOm}&B6W`%fs!mpttSk9rN=AEoEV#B~M^^{~N_tY>Q9i)n?8OgGK z%?3GE%;6QuI{7l8Cb5XI!xC)i1(xw{mkkux+_o>S82WrT{S{m#J#p_EIR$6?R+vj* z6Joac0ek0zZ|ETYi2&6VZ_P*}Cduci(LFM*M2%Houp4Ah6q?#<>at^Hi^XYSPu?k` ztRibKL_uvwJsLEn56JX$?pvvjf&=;enTRWIXYzC`ILT6 zU-nsXh{5{3tgF^h_?slR_Ilh=fxA1`(MI!RA)*hg<{zO$tgY^jEGyV?7QAbIQTj1) z(GJhrpw-II1nX1SLm`N&!L2+e>=xLyhy^`ldif#cQ`xQr zt`Azn3&L91>9S%-7VKG3hP-*w0Ll#!xyUR z2%Wgj?7F$y%D%YZcFobG1#GmRBpQh+m!-Oipcgw0e8+hPsh1}MglDM?! zl{{zT&51R_q8a3rB#GEM1 z=;?aHWkJk&-w{Qv)7#_W zA;6NvID5<{a_|@^Ar%BhJw25JDkw!R`*a>umCy`Ay6yl~#N4$C+Zo?I8hE@$ZNk&l ziX~eCr$pNVvy5?z@@;fMt*bfj!2`{$Q4AKK3CD&|mN`_s7;6PwwDmPmnM~oWqiMHG_@vP4q+1URdHeCIIH%jg8f1J;Xjsyn z_WX5=QtJF2R{05X?1q=E*rM;u{OaA+q>|P#l%F2yKB~T-TaZ6{U;j(wx=mhhI(qe3QyMKmP%QBn;Rbl`I?H;j<-{Y z>*)kr(9wQi1YwN#b0Jqx`BAqk5H##+gS9N{2M0yE^a@TsU!K=D1iySPu7vHM?d&D&zdhm z)#drxllg-NjS_FesSDf>dYuW$RFP?B>-1q?uqNo0*H(3*?mLU6pi*VSp6Ug}X+Q?1 z|9Dg6J(lSA8AgsNOPdp4@ob_M37S`6LJuY#Q7u*!v3&XrrQB;%e5YQzTd=dsn($Um zs0MPyo0FT*b?xmoSS6Gxg2DS~D?P(^>|MAboc=5Yx)?tO6y{NboRM;ZTm*f#Gis5R zevY;b3fAH`z%{L+P%06yvF_B6 zHE`4+;in?wRM5h$Q0P{~Xs8=Uby?k6qMMeZDO#*%*ASL>+ zQ+vlvLm@;5e4`aHO~GW696AEpZnmAhJa>afZY43-&~Jo{(s1im=D7Yk@jBU!Z#*?a zO+>5{8O)Iql(2s^^**lT{x(ZB8SulUGAqfPwQLTisjR=cx zTSptMgU;mgFDc`>X+a5Kw;&;0WGXkPV4K5j`sLGQ*h|x)lhz5mDwqLemwJ=!;#s_6 zBQ;^LThXy6aD)=H^1QoDBh?S$oEtoodNMTR9H~IHU%*>@cFzPGj@DRkcC$rXsi}IZ z|8PR;N}J?|H=rB!4s8r5oN^eIG__%|qtA8;#cg&5vs%6UcS%DX621BJb%%ZVgNTf# zXtbkgrY^9Y3lxdrobq2d19yx)y?M6WV*_DZsO6wT#WaWEqpzhZxH^;OFO^@#_uPVh zx;5w{YR4G72h7^I3=C$tnwY4qf@luZxMw}U9AAz_#BscV7LB$*ThRu~R<9f30B&j^$g~NO* zmC_*t*+6S1BR#j3H1{hcaT*KmBD60(PKpWw)xJM*3*OwPey<+zcK>DoZlvI$pOmV! zyS0hi7ay+p;cS_r(i*jc(3>YoIiWexl<*~JQqFhw&eyiabau=Xh>?<)JmjsPz0cZ6 zA5NHen){S~B&PB9oWbl@IHymK35+9NgJ)=fMBVb1!;sOK=QgVypGNEr(jzq4K9lO^ z$+cGJ&NP1hk1C}2KSmV$=OC{-NB!eTK*D;wzrM@;5AP;+f%F<hXeaW12V^Jw@U{I=n` zK&}oe)7^qL+GZ`zoDG6ZMPP^hPwCY@0QJYTU;$sVsawQil^J+`) z%n%_G=~3JmdcMmhc{d^XKHct{eHwy?eKV%;nrp0p0+Fwi(>tR-vN6=}Y@%S1SyU-y zOM5VNDb9w-Y-6ih*3nG77kV`w_p(rv7Bj(8$xwK4CPO?q&(~GC{0=cklD3bRz)9@D>!({g<4%!%#%loyTz}Ai^mO|J-^Z#;*$1^ zsMqYXMOD}VqC)D>-(%7PBCk4I{bL{~FJT`wZ>z;Q>vr70#SFpq;oLV*HSGq`Dyq_? zUclPJA6JUw-~BE?fL3^-sGwT>Z0Wkg$B01X849>6oE4pCWG|5M+WAdump4O{L3yvv z-!usFd2xbG=d^>AtR}p%-YOeGD!(pv>=PNebfeq-*@)@_RKl6XV09OJu~NQ{#p5DkrjB#{oLicugka4x&jW`4yQVJ zU|s%Vw0dqM*kgv(xQi(Nr+psXVOZ{;l1k!Kw)ZBt>ZRRFQ&m4zYN*Aj$GRU^nuF^Q zVpI^~_C%8qKKCUi-jC(WJ2OacYn5E$98oxPU!|d~KGRiAoKSHX+@@2b6B4R3QM=&9 zeYdKqUhhENjFwcURD|9!%F~JM%roodHJ;6EY%Be_80?vj?XoreM^0qN?w`|%`vKGd z;e`C8G(f;mSRvlb654*OAvl`1f_|_k-i~*|7{KOFCr_oI1uT{u-S7cMpg_NvgD0QsYwIM%!Q&u@tgP5qqt-&-|Tl=;bi#pn{`o7R!zL z2B+Px&3xvX)A8gyUiPx#i7Da(lQJ{G8Kl_7(T>mo7)o4cYflp+Ns3&9q^Be$S zW>6GSU|X@MHNOI4?68p&sli!U6$;FsOaZFOtP4r=4`3LE(0(;jg$nO}$hwZHpW z;}4HM+ABiPIeYU1EPdH85cM78=vmdvn|S+BtG&xymX)QWtmNe}0+zRaJm#w_qz2Zo zfUQ{W>LJt;YnC!2G#E8Qky};Ng-|>$S$xL6>zl60AdvTsT)J&3*B{LMh^%b5 z8tHKEu0)E+9eDD?CJrWADR^uEv4X+E1gjqPS@I%_N;xgyxG9T;ia>>l`4QVYd@?aI|gvb)hSwo=LkE zGNFc=t&D0V8?i3pj02P8-YEwEi-iXn4n%K&+q(1aP zqD@L=^H2@YZid2<8Dz9bitTFciv(>+nyekXe2IvkXb~x9;yk2|?&SASFUjss-lm1O zdbfQBP;t>|*fcbI={gCw&G8z0GC$p9!-YBimFTFe@rpdi6-$K3tPF19bI5@WOkO`e z{@zWXAX#l*|K$X)+!?@VtCe-GUr6e#s=VTPq>=ftFxU~GdqdK`xNbhEjc=7PTLCa5 z-El{B_hUs{YE2kXk<)Giz~t>C8?x1LuZ@~~*oe;Jn&ER(1VG{Bjbk-avDb&AUSO`_nX0EDIVI6J7dO%Z zCCScwr^>iFX7_~{1`bZs)ALxYGJ_@4JuN^!@yZsYc9YoKUSKsOHZn!$jZo`%Vq@6m zElLLR;OmEl(XrxW&+1)PO^BDgV##g}=5;ld@v<3w_;!a|;>!y>mLj2Xb#TO-mjPTg z)Qu7aHWS0U<9>q#r*|Dsvbv;@iR^@@pwHrwo##!_B zlql67)@?E*hciYaz%#AS06Gv|3=gsbE%dboX_;L6HAAo}p3Q83kk)BR=d+&-!Yx1y zk%5*_+fZq>41OU%A-|LGLzWd;0r-gsBgk4@FX72Ri%=0NXL+`JJdP)YkmzCDW$q@i zLiJ6qPbhB$kW=iyLZ)iXT&>a1>yBj|=m!q?uZ^b$Xp%;{hRn3u{x+~tEF_M8?$<)M zQazdKicJ-xhZJ>Q$o9#w-FzzB@A?wuB!es{{Io}}7FStz=umMG+5)lc|IkQG7jc8Y z+t7VO=QFS?h;sY{Q2A?58z*YWD0K~XUJe)zc}su1p%Gcd9A2;Qi(y2DhhYq?D!mMh zgAb3)E>-&S`dt+}!r2_`i@{R~=-K9R8D?`NbPZb#Uyx$v5l~iyjc?dQA5*dL8-D;x zk>#7W2V=Q^IM*et0VZ4D&<65`kQc!oDn)p|$Gi~#632L09^}kI+(zHOs%(&+-dtI? zN7HaA-!U>KpgKLuk(8+RGVWO}yggjMJpy|N<=I26Bm59%Q%?Ai-IGFi&@=PX19036 zu}YPkK>ik^H5G}z0OoXssgUc7;jx;1s0=sD(RJy!k;lU#9V^tBJK!^?aEwBsp-p7} zpx(v9xZKYRC-{-)Hvk4S1bf{267TYv8jYQXt&R?9Ic@miV~`l$rpAbZEUcxwH!V4~ z>QWB%!hztJYv(SxytBiCPm#`>#aMxx8EBohS&6kHT$EK;^*5K?9sY$y<@lczqj$hX zi#_pD;kR(c{{oxx57+m9n(O;72`7G|ap-~krQ#O|VE6kwHv{~Y?_+_WLiE#Bky+&o zI6Td&o=;c8pKZI_l|Q)W?O?_x9;_MHyr#;+=pmfwWCF+tLx@K~ zxg}B}*XipImQ1kRk>U3!%OZ(uSKsF`Bx(>@-F%{vPM@t-ZylTaHECjID)(}EUj3dV z!Awo9?Vpl|Ge2K$jgckD6=Z43^cz6mT(W)rwXWsmz4Xw4AGWK9tEi5grk#(OALyEE@Nq?V83|>&Gd3BM;JyW^sE2-1Cdw7)_(zwDo~wKt@cK3E&) zS9w{mrqfvQ3W$cXfoNzYuO0oM^q-<3SvVOCuZLn(2%rW9DAM2F(^pPjafYn2IlMdw zIjv-wI(1OcYq4upMXw*PH4RFJj!tHUz}jY!WFZ*HdXZ$gopOECZ8Hn5I(f8cH`z#w zRWD0*67OXk(QbOUhK+wGnhJE>yCo9s{5-{zf>5$pd@T@?EfijAWTV9O&5rQS6h)r@ihe5`a>3|sfp#8&Gih7ixbDY+TDb1d3tK|Y(CFVx-8dR*OSIr1h<8!wNvMP!%t8sx?~k2+PIjtV%v!<`>SfQ!@_V^3*miUE z%jj*ZwEcX(rZrK2^$5Vq*y_x1VXlZdMR!V{zOMeHRsg#=V0$Cq02;C*s1uTq zdN#6RIWcsHSUjV`j`>T<*83v|qpryNdDeeAmEEymRC2B#{gHds02qZ4g`dZ8p!#peH4O7qMB;XvTO~d&v7=_dT zvyj?)8Fx~V+Xp!o+08y(oaOPy0-A2VZ_Ps$U#)!14Qn;}2LcrkZ zS4aGT+EFcK|6Ub62Y|$oQ{|7)!A{iB;puj?5Otge?WUG9NOa2cXed z9wBn2T7MdHA?0)6{UPq}fCm20q?$uQqoK#lRRD7u_+qw3C(6$ppiDkZy>f<(H1W6h zJSs(GSpq>qHebd<1Rh6%f$IrXWO~aP3`Y`SH^NDC z^nrUGj;35qod^zktRB9II~bd~h~D3K<4Rw;3q#MiCu*yVeC#BM5Rx(;qT?yB!A<6T z5|zcpq1@b19n4XmoGZF-=iTyBur5)~-H9=;7za&m#Hm!e{C)eqrN7qr(6~I+$KPT* zprMwPUtLL39e36eQH_1Zo{ruBEeHs=Y3xD=85MTDP>Va6@Mj0GoDOPlfRl z4S8=4J*wgI)`R8+K=KwL+TF`Zpv}CBup4%-3>L>nztyh{w@R0tT8&qx?EKlo$Qbe^ z3xW%-_3^)jLsg%BrrS2Xd6jGW(=W&4uwQgkbXfVnM^qB$r%rVHhVwLJud%}T{17tH z6h{jvJFS#JXI%|z&(V$2$@>bc#^DT$!{JWnLn7v|d}lIBUNS5FSjK&)cmdDTVj59t zjpf$y66$Uyd$zE{+h5n1x>CT=lt#6HIO)Dxw&k6V;$iQ88gdM76^zL)3KZ75YwOn@ z1TjD9;X5b0TURjieIzFZNusPZ#mkNT=GOb3A0n_iBjhID*8SsNUAzgY11vetvLHJ7BBPu@`v`tj(lY(z|@N`Zs3-1X`Q1c23Vded~QoQki^ zkZqjq)o#6l{KsI_+Zazkrt)V```L{qU=gSQo3@khf=!&}ss+09Bzes12&=Rm0o|tp zBqejwfRj1cfrHb6%U!Y^*UFhlD>_cC`CZF5Qnc#{pZ&`?L zlht1O&?i?lpNE*AnT<`k`1dU_Z;Z0JQ{5j~iI1Rc%F6j{|FFD$I4*$fJAZep!wk*f zR{xjm7U`F;dfTsKf37%I706g*eZBZm|K=InA?6Qw! zF z=nn*~H$y_7i1bK_h9uOP0B>c=TSFe_Hc3YT&D(?Gz+V4vqunJ-^D_eym>z|{Cos?3 zcQ=5)x+xCmdL3B!ZPA+>_Wr);E6x0_!+k%-!!x934}REc_Z=;h#LWYMpx$!^H{T@5 zgelEjiGas!s=GdspgP1d3Kqs4nV%+zO@7cp^eAxv`a{ zs4m_S_>es!gpD_&lkk#|NMPIuGq`Sk49_&n}Rkr_jAdKw#|Dz z2dta#l$G;ExX(gGz}@FXgmCAI;F=(Ai}qcEUCvtl)GK9wd96}%;*H?!m+BzJ4!R44 zpQWnl^QvCAN`gQjU(H&Nhj$+bZZ{4~M~6&Anm(NfbQTlq)vsURjA-pUsE zAnvy8QS^GY)}b^IQsYlL4&d9+rH?sMEnwIHZInxuf3k@^WzYLLg5vGkQOZ^ik;CuD zyAFtzdC&1ehxq9rFZDE049A=jxj$@ycB0&WH@(8Y<%Ilu?qB5gt{e;7ngD10BX4Yl z$+=xV(g_pelX~SLG;9nlO}&Dh(R9R07d%rO&nWJFUXjT73sef#7@PLt5**?N?4Y>R z?xO}j;H;m8kh5#zBO7#N^<`27Oip=Dsr1X0vH5K|CDZT1-pHyI7lGJJi7~ks(}gPg zg_1Y40I5i+68&b{ggY{>i(A=GWq0KHS#4xQ88g;PtE-8VFp!yyrQCK7*Wv4-~3Jg`#L!D1n=3S9#G;mmZ>Bt$LnCvB8tr?Lx zJdYscK!>9vi?x9}ah|`uxI-6wDrB_9ExKw-?+3VqkYcPB#nX{omL4wT=xvZx4@gm$H=QF1P>U-yh%@r~r0STe5EY+n@9@_E_v8Zdk2{typq z^x!$!Q%HL%886DbaaQ>XQQxJ^1hJf{={j6}bBASj>JmK*E!fc(m&#etHBi8p`%2pW za=Ai>6R&#A=!D&zT|^;{}Agx0}|zBhj` zrX`6^9R&yC$7vmnNe3YVk($<^*P;EK-KSLfoLM7=JtLAs{X=g~P0G!bu(iKH%R)yw zP=dE) z0M_HD3NM4p_~nH^#O$e|PUEg%U)p^ULdKI=DIgQ!QKoPXG}vNFKikVbjgQY|TYWJF zv;-AH^VuJkiwuXJ_6p!W4@???*%n)^LTOHl*WMNzA4HpKme%1X-)qg7t$#q(!pA+_ zRYII(R2kYvy9+z}7WKuxEKXw4@4Hl}uR?gsy*w{lPzv?8E%?^Q3lEGP=ZAOQ@}<0j z3zCyvE7#exEoa)cXwgv?kca53>Af_MRLeYAd2MKo^OM$icQkssi7Pl?PtDbF{{|Zr^uV$WaC=NTI@HZ{~X%myM}M96-3$ za=frwna(rNK2=42?bUH=(xI#I+lTzl)r;G1ODB?dBlj_Sh)F|4vt$9Qrv(|!&FEEV zjf4)JW}58}6$2L}6bKVT-GoNvv5k;rsh&6zc#L^TO=9ht~tFGp^@ON z-R75{GWdKkI(3%MnGk#)ma|#4?IRshWS9DCMYYB|&wLOs=upk}jE~5Vt!nbj$nH7s zA^0viwp^{f+!O7XY@Q3_`F*M+05Wu`-A(7Io*{jOCr

jso{{gYU(%Y0A@TX#_qv z(J<@_`ct|2Z@hvb@b(Litb2ekn$|2f(smn52pzmL)_wp7PA;~JkbdZ~nxrzIz+75+ zb!&PQLjict>1Y#qQEljVg^^Wnnr6|FsR&b>ud zuT?azS=s2by8piQ1EuK8~?I{ZRM z3GP5}q>QB659|G0llWrnYvEv=?KV*J(*NylF?EJxtajOV^i%aA%@ZN0^7zqddTrh0 zL0*>JyaL{?uMPkiT%#&4_*U-T+p`25(+L5Et@6*Rf~SglznOn;5m_BJ=}EfwwT_D* zz-G}53}0?7>;lSPZcsA(EDM^{+-w2aQA2~TrxyyL&7WJvXpA}i{N z1HBFS8NkQzM;h{Pa18%u*X}>~Eb?TdXH9<$XbZe|<+@;|tN%=DUDwIaP?l!bLx}*Y zom*N@Re67?a5Nr2tzVNe@1_s#F4mW{Yu-kj|it<%BXOvr+v}Gc`9+_7kV{i&;jry+r9)$!l8O-|w05eJ_|vqdLwbUzcsi zNFLj)zytUSSGQg`wCZ*NSOA>A(I5D`{hsD26MMIIU^mp})(#eSEFE5JnDh4F3!^5K zs6~wFJbUxmxCP6dY_5bWt}pfVVKdjlmf0fNXfWySks)ME2LB>Hm)uyLAINk*||im+lGiyi#JV^L}!> zpU$-*SOITd1>~q;}R_L#02pD zM>0$fw{(QoOqi{W1sd8y6w+*%dy3M(zbg=-#-rOiR{^=zX#9l;<-hh?2XDPS+L#gn zG$Z>@&C+;QA-RQJ!XIWoWedu--_Q)8Mh&kyQSaTf(YAv+d5hg;YYYVS;Hq(c2ZK&O zkv;yOxuG{C^^kns0Yp<~UuLYb#J1i&4;EF9X=E)rCkPZrbB*w{iF@5xmx(D+AyCCh znRy;y%BTai^tUV4llL}v1r&Blo5Wr@4|UH)f~x-2jZAgzU{TyZwD+L%E`wK^-gUY= zniO8>8=$PnPDFOJU>v_3%wE@l66Rw@hwDMy>vo} zp<({w9pw^(YMe$nQiZBp!yIsqpxUUK+9a#_Wh0r@!aZR2RXb7sXZ;C^Hkj7*$JvJ&SGz~7Pys^By<&^zu+d6q4_0 zOo5C|fIIN2$SEXhhlxoR<5#`6w8x7Zh<%UK{vVj4|G!6^-W&&%z#+4~fbG(N{-Dq) zpH2K{g!~{!6kZpWxfSKU255`(2zawqMPzQ)#6(f#vL%WMbz@$kwDRUq0cw<9Z^rF; z?LSEM@G}+dWH@SJsb7ZliH`c59EAZ@a1}qm%``ovV%H+l_%C9Af{WuSrK~!zitF7GgV}ux6HUjJV*( z+t%2z)wmUlHgO+Qtb4+xXeW07!PINNcbiiNrcy|H3qVn=tE&B<=(B+D1?aQJTYQls zN(cr612z2frsT0Dl04dxML4&bjH=9PqHLK7nV(VfG)CS$cUPKZ>ZG9cO~SC4gbyV~ zu`M|W8Qz6ut4GGB!n4T?7$(i>!J+B{<6j?|dc;19_wHxKzns|)3J|fzK4=YpHdVf$ zJZ)k6u52ME8Es}z#CFEIz%~ZXvD2T)UOQm-f2rAMU?)mo{B&^F8HAUp1jlX3^9xYk z*`VhLsdad|{3tA0^*VEdmVE&G*IytThsR#~7)T+jXs}M24fwU?b(-VFl}umo+Y<>s zQZEy=N$yuqBJb1&xi3$In-FhZ=n zD{~>pc^;InO8D@-1w^Ca=Sjr^{AtiH5TQsnozts#R|OqBC+3Wy{Uea0L^TY)I=BG$ zIzD5_azi=Ux>WW%Kks6%SAAWc*io`=V@GCV&CuJV?Ot$zdZv;Dqo5mrC)~>Yxw2-C zBmsY;22RB6|AvkbW5=+TlIpS+7s7lOsj)1?b!l^Go`ed{;Ri4%{*Lx&av3I$cUZw4 z*y~2&sLwyno31c)6y*gT55V{q1RP9=he!Ht_^AARujO61nt{#9f%YP`uS9GfUa;&@ zu(K>^Hp-73=~1=ja2Kn>*O3<6bgskF(pLAAU%05p%HCmMPzoBAZ#$>F?%)IuNLUY9 z%`|@+sYJn5K6CtawfOP=<}I9>=DZ6!=9s{-Tx-*I;T{RCry6GpXG>&6rV(3{3 zzJ4hIc&{*8Ix5d_SG~2zHBx znQN6~FqazDWS&>sH7+Zk&TW`@!rS)& z_hr#(Q>e|-;YwR$W`uOo?L>o%7v48Aq8~0WyDX%bpAZB}sdZ*?Fh4|0(Zh6yt%&!# z1e1ffOI(^`=fTJ2gPs?Bo95qP9@_=JS`Hk~>&I(+r3A)37GRIR^hv!p=UA>#2Lr5t zceF4qvmY3iQsGzzn>G%-3P^K3046CLh3Tkga$_I`xuU@~>>sDxUaJQ|c@8e7j*w(M z3f(cED!WK5?MaGj$=40>7wTz`rS9X+Vjz_}%k1M0rw+VM!V}KV68UAp*A(|<*BiJ- zW9bP#uJc$aO*!zIV!!_cr_mXVx|@C`_g8ftHFebkq~x&KH>q zI*^Ww4Gy0JN0;b&-$+x9xm`ieVv#p&)wz-SDz0t`qC?eXkpydtB=h}6w{RvZ55PC8 z)6h`D75UeRciFPMF6zTrO9DmZ#?sD6#L*-#RcfbrO#EBFD+@a8mf^5`u7Iv{g!Mv4)*k%g^owBk#whKY=!0p^nCPnPIykz0s)f+!I;*Fq;;p$SMAP-7E zg=&)s4KGeumv8#l&jp#Z_)(j@t|x{89JjK>P?@=2%(~oLv33V9{Al;f!%Xm3DSRyu z;z>J(3LP;5aGfb3z<$|srKexOXqEN}SJ;uZs>3I z1mYEZV=Bb(S-m8gq(>xO?qeMLFUExE+0hjWp*SkcQix$Jo zEn*+tB$iq1U{)0Xfnyus(%EZQntNHylc(DCcIMuWNwj)j%@-d>V;a=-LHN%Wrl^36eq^A>aW=7^MvPwSrvf&Q}!UH_ivQ2ku#^Ma?t72du|rh>cG zX+NbSCh4{yU(^{({+ZoxQb!hvrN@rma0(a&&!3w!TV-9kX3h5k++97{cwVY%IC~;- zxpVr4`Yljn>MszeKh$?pFvGN=L zP^W+~So9f`bge!#Q~MY#n&1C8NDl#}G8qcIoef{A9o`s6LAziTRZc`^+tF0C*fZQJ zyuG-trffVv^)ZY)TU}j_oz%12di#eUVoi5W1^j^AiE_V(6Ao^1VQ6CQs*;N{m&Kuu zH%E8FU#SH&pmB^u(GPjy(9f)<_sx1va|3ri!8X7EQS4wp`nQO(8NRFNrL5}Cm1U&F^ z?LHpbu|_^_b;eVEplycY{CQv^J2wa0N#WPilZiKw31W>Pzzu>03e=cT>DexmhWAVo zR8@QF+g$4e^^_=zVM<3T0;CsOl7{oVbTp^jzIog=m>%K`T47L?^1fS!9fQ67W4J#4 zX}FAjAFjss>wa=b>Y&xDN6KGNdZml7UdM(kh=~1V*z=H^31Mtmg_ZiC(-knP{~5CG zT~pTkqZ;8svv{+0KPwwdKWm8Lc1!?|O&i5bl3d+nMXiHTAUkX!<|7>_gR&u-5#%M* zhq*CX+xn=N`Oc+dP|=Ps%{%r;iFD5=FBsQwEJ5+KSgn`q1aa$eco>bOeza8FS?K#u zg4uUmXy?e2J2r@k!+waOZ<3yOwo@KTs;!gftauryj~_ng;TK_B4ee(v$vU0#s_gZ# z3UPZNl4PDUJzf4JJPvaxHCXgE zi{pcAIzNUY)VI!Jg7^5TDW14%raWx5@TTTc4o(D+!GX~4tCfqxrz8$ z!d~WwE4!&vt_3Jc{)n%_;d@>CEARZkQ@1%KlO7l2N2^CM+j7cIq^L=ytS&oEUFnIc zc+LAAKUpPX!5Z25K1UI}cbXCGm99>l%J@wdqW}k9S;1XdaKqg|v6Z*5nVA<-Zz{gs4W#vC5q4Q&evdQR zdnv!ug}&pt)_mzNt$o289av>z^Z|q=wwo`nxYJS_wzVvuLKv^a82N@yae#F$@^}<> zW0?{$B78hW>-6dvA$>$FiXpMHp^rv8bpVIsE{ zG8^}ze*%)AmwLU^88kQYpH!6w`dYj$DA9*tg18}jFvu0Y77TO&*Q>2>y(;-!-yHm4 z)s@yOUuJ0KN8<)F3FIsgF&kRrz@gGKY45=8u2&&^p*_G#AzyR&DmRwFN%Tms?a@Sq z&?rphWI^SbzpyvvQ>w?tQ51E@hH1zGlkZdj8{%~Aq*+RWQt^W#gpvE!V>I z3%9dZSDh~2LO#g%URFIMr0u2+x~|emaB^;EjBb5|F@tk8Y0<*!hu!35Um_v(~3lnBk4 zSNU57`Z<`yDkUH+FIER#3cMtAS|)SZq{c*VfZyPLVTt&|^Wu?&6tI5n6iiaNJwo;I z@ik1n$hPrx?v8d7bBs;?{L`uhSWng6pxBV&b19`m70@d$j&*_v_%}k1i4C_bM(?v~ zVkKrDmdB*xaT6`o>*tu1c6^|}*x?r_pS@%U$r1VsWUFLfv$m;k{E$M91&=~@i#%vf znpEOdCbH>hpM|=G!+YZ!BtHJz=95Nm;Stqorop~zAfNLXj7z>3r92Hf!iFuOPS$~r zVbc*cYAVKoPS9!LD<55#!d=lyf1a&MgXiWv($T2;-DOfbD|yWrN4Jd^^Q?JRQq7%` z2*fh#&XsxfrveCJSxS zHekHrxyyU!2#aTRALsHI$>V7=JUjYZZBdl#Daml((ika+!pT|#XW+OIk0&Jrw-I*Y z;~DcH(tBb3p*2w#g6+7r;(JSm0mtZt?;WsQ^31AwR<=_tZ*r*_Uqo$t3|b+QEZ9AU z{sEg}MZLk$x8QuD>i6md6OXc=fle#mnUFh3GcU|3yPHYPwODD9iyAe{NN(>L?zc0Z ze2cra@B_`8;9C6noP1%>r?O6vyVH}DboJ2`TofK_mvxm7WYL+{R?vHp<7a1N#6Rdg z!&)FV1v@Ew#N>UCf__SvrCbwgQj8mBE=@ko0s24culxs<$ z58WPBe(bP5{nmN-u<;?#-&hs7-^RQb2j~E(qk?jV@95OlrYRCJf9>g|VL5UFBZ?3F z?lqsiYHWSDwRQxlNoCF8@-{((%u!?DDipj23BWdqgqVDmklS-1j`Q1YpE$`+7RlCh zfv~B@EC<71)9@ZY5v$M=pT%FeY3wgt1c~uxZeCYvkw+wTU}=63UiY#NHFc`!AA>>=`glvo_EAkU@rW(91O-RXJQsnb4{blcY zUfnfybaGb}>k4hT69L9#TvD$jliAlUlS^xm(7{dm0*?)ESL!5g^i~bT>pBq;1ii4awYlzL$r1}edutnDSG6~g$qBJn{X8J zY)uFPBT@?D(NP+&IF*RtnXnmWyYYn@^qw1MkggMsH(PNJb;XyBsDqj2`+b`;02d93 zRiv&-2NhaplLNX2FMnEs$~0T%X54GlniAEq2f6?2=JJ!DFrP* z{dngATfWPC=~$PPL(pxvVURB*%+H-8LXy>c{o&^&7_6IN5LdCIbwG>l0B_Nhd8>s0 z!UF9@CiM>Im_TR^hyyO5E}JL)@Drg^&71NLP5@O^f*i$L#W6E2nGTQ#)#_RHu}7-O zVfGcf&=nw0tJ+t*pdJqt^nv2?Ozp4U= zQWx#i?=Nww%D_TlEglv;aW&72W5N7(Q#0r&#ISPz!ZB?n(ocC^0>M5-`Ffruk+^Jg zban$z`t7VIa#X2yN}6V6NrnF7zOmS+_*l(UEwk*$w8)c_Pb5w(eoyKUR579ZXutEB z^Vk8lps8!Oam7yuMx>Lcr1R~V6Jej<`^T7y5U4I2FU@00yW0jKQBcddEQjMIo05sX z;n8oz(GbP2kQR}VCB0WAya0m&2A?MQ=w@ut{<0ACie9#XQUa`9LgW4=)H`ktRFL*#0bt<%c_lS&C`|KF@ybgJ+UL?xj9{K`s zba!Gnv%AFG1NR*S(|y^!KFPP$drb{)!HGx6I5BRo+r-jB=#2 zIkQjm1wIPuSS1Bc4;~a(DM}yNBD>19m%aE}?Q3a*q}OElL3?4yE7m z^k^!xyDBaGn3&)xIa(JTFA;Ah!(8B7! zu6+4ba)uqe|Nba(Q)=($m>Q^M?8tr;g801anC{men z zi(`n<)O}MGdSl<6=kbct#NjDJ{k)SMkqWC1XLRc{??@XB0fWL*F*@Q&%w186U2n+=QgRxnLcxB5(NsC2aUSNvsp!S+;ZgXYWg7TrY`4X8`-8pxvdC zu5pQ2BL!z3r(rQqRyY1^hM!1C4;ixLr$KgCT0xXqvBJY-_WJmzZvz(_wZPnsKg5Js z`iW?3)grOCCJ%|OJLZNF-pF7o7ke4!M-Mu1V~L>-U`J-wMdVwjzR^?TdMZT7U>kg^ z&SMqfvKTU|I=xF}pf@*w{T+COi1;;)Vj6xc@lW2zM3}D3Op2h4SAtt9>hY#$kWN7J_*W`$_GfM&BETe2z_M~M6n?> z$5JG<{N{5^$B~}}$tOmNH$EOzCsS1l0=H5G6Uek&;%)8(Gq}wYE73#UZ{5GTk2ew^ z=taHP7eF0mBt+bll5xPM9t=*43EX|O5~wiAv=@uW?r%1tc@nOK)J`p`_-ql3i|EU1 z&NmpQ#vAE{u{G!mF_G7NIohP{Q*V(Itz^D|&>I;1lQx)Ein z>pOQkcA6Q9Pb6nEQDp0q*+gZp5vkEN=_q=bwy$Rdo?qD;1vcDAiL{g%vty@>V{*>v z91-_an~mhiW&)+%ra9VCNAPkXd%?X@ClZ@}`&3F0KjF2(gT~;JQ2dk83hC$++U=m# zWUBRqX9DD%FZkufG&w`nabrPv`8*&$7x%T{SF?Sy5K_}LSqJH&4==<6Ne9WUs{Q+S z4onP$BP{&H-R?!_2h|9h(bV_~n;)0c0}ZSI;?FLvzN6LS7w9RHAl=pV^OEwGoGWgu z;;Q(@>W0Hm;`I)UKUfL1^4Tk-hnikGp%OU#2|gn6EY+tP~ayk_p!?v2xXFc zuv$tS@;n$uWHIkV?7|1oe+0b8S!@4G+W5bqf>$!E4E1wySTFbN(buGY(}BJ>I-loh z`lQe;Dj_y|cXV5Zc7@&#$m9~zbeHx=h#1+d>?D}FhJt~<;x=WLBC0JYj-YSlO|QD` z9&}R6UV%Y=d2{>unY}t>DaHj510`q$6xf%mS%P>4uZUDyTU(|W_7Bb!>E#%oSPsHS z1*A<-RK_TyHP0e!`Hn6jyB0rQN zFbo;MV~zI7s@*angKlg{8elhPv9YnOKS00_SmuE>s4@Y|x9 z9b}t;tYrTJDM0$?#!Bk+4& ziwF<@j97_sYkjt&K1nlR!h2i!_T{Ir+rb22XP1m=C@IXgojQPf!K=@gum^n{|4^*QCqUV*4Mfwx- z+my8}D*QWlWOM;vdOP6u zj&+Z0_^>|Eb2?d>Q&%7L{ZgeXl=uz77>;z~9!guf3R$m=Sc$A9p)r`?TQ%?&?sdIR z`n3XQDWsms_jRijxw*`{&zf#G_^%1uc{#qcK=gB>#lu+2L4IqWOne1K!4! zyN~xJ+*Tad0Lufd5k0uqS#OVlhkt=6rCCv~>%8F(`PM%7u4caed=Rv}^dy0Fw^@lN z>jR_bKG(F{F#S&%)Lpa@&?o7?@Z)F1Q`egJ=JnF^{cA>L!YMQR!z3*l)$aELDK9jq zfElZXV%{}nU%gES^w$*hSpL~oK8yqb=}s_KDGEBf54jX_lfM2XToWamyE997%v+Nj z{M;QjSc45KGeNDxy4Qt2uFg~^SMVLAZ93wSyaUCr#mN*pTsR5Q#dND~Mjyq+a1@ zne|KS0G2BI3s0Qn$V4J@=WFwUc(*~|U7RCvecF>KHYsg%iI zjpIuZP2Ysb-XHhPUokbl{-pZ8*!59?x|NQbWDP>&zx9zPD&;}jA=p@-lV|tw zz6S`Sn1A*$)6_Riz-?+tgkVpCTs~?9l7icZ@_bff^K>9N*fD*rlu)YrcY;ewLQ&Q| zFWFp*Da50#Qm!fWDYC}-ob5&&d`i?xQDn8DJZ!$r=1&6h^6Vxd(nDJHM2)^f4`-x! z95l@5#!5{P4O$ z!14Ve7Hj(di@h(8hq7sYey6r~tN5i!XY*>}m3HDjMiwlHHIjPd*OywCGKclZ0=xtHgDdVX(z z^zn(y<(lidzUTKikK;U#bAa5bEx>gu_A-dZqudG5GB7>QeQ-F_QIE))H8?iF({5{f zTmFT6y{+ll22SeBk2%8%&#AD%B9g}89P=Xd2%jqTHi`Z-F1F7uVlMHXKj}^8_0fi@ zETFY+U!qS=yRN`|CDJm|Q03+$q2cg@^I_W}+PoB`7vq36j&lHV5?IJL2=+|@k_#iU zL_PcBU*mHCy;mwfHfveNcqnm)Y5cQdM+mBV@K%JU4y?8`G#~%|EJ-n-G>>MzTeSNt;jC8c@FvZd@PMZ9YO)_U-F_K z8_}8@KDM8g?6ZJz2j_}*94=d>>IHUZ*+vJMD28GG47k8pVkR=1MrLmfHeyf4aWv*> z{p2o8H-&Nq8<=F{%gIfwdj^bkDL$iik@?ke@|4f5uBoCi{4?jno;q>bQD!f2MJUSI zcm0DuVx9p@4NCAOYp9h0r;Qi@91rULoHrT+M>Ok+4Z711$WNDwoSItvop=LB?^2cn zms*FBdoOrp>!);yb67*tnf7FSobT;Ns{ut(2E!xDg%{+#%K0gI7^*|f21@;|>z{i` zp$igC%gBm7MK^u;g5;)DX~1ONa(tZMe*BF0(PC*)#*{l#oTE88(I(tIW)W7)ah2Tr zN{Wcd=0m-4&N;6-`Qk0=m$l=T9;`!(c|pAj+M2Pd^l*Ez*4nc*)S;PI#)^-N7p~X6 zSBx;5Qs%z0dfc-8`13L2gEFh~DSuL?uptE1AIijw)OB~Ko=KdmTnVw6OVd4RGcA=U9tWulE=C$a651;PczNU&*!VK zV^S?zwqONwnp~Wx_5d!h3%gyr=+ zC3L8yL)_u4!Nr1UW6=#GnSF*A2njEDUWfZatgDI{3ENhOk2(;wYYkMfhIPCeHWC6H z7H0MAJYtM7kqACAOjY-wzhK~dN44nv6In^V#C?#tnceOtZ3<>sM6L&~RtJrke1nsi zz*Ey}wA)-)v2P+aXGZNMuN9E*#V-Xu3EsYo)^ZoNQ!9z-H9IsCc!o=2^ws-mwk#HGg$=9>dzQ-sFQx>$#2(mHkY!HXf(Zl~MB}rNP$ap4*5i zW2JI(*Q+1ob#}j#RoLZ!3Rd^oB!i}7?Wz0O#Toh@qQfq6y2QlWL)@?548fk_$g7e# zy>7-!{l^{c_J>;-T?^|@eNDez>U{-g<)5z4Ea%9?_ndy-K zsr72B-}_-pwiPWC4I=`Gz`Io4@=puWjjY}hgj{5Xtw>?L6n*~DI`OZ8VB7wp^FzOi zk{huH3tN_h?b-;++G|u3B|cuir(5!0{yWcJTE(s$3Uf<0%I|eMX<)uYBkn!zdI2*B z4`nHu37l_)??SkCj>NC12=%2*g|q1pN{VO;cnd!wBq(N^v(qnIyEaX~xYL^@{Mi!K z$|767$R+S_JHyTjE){e-X*d4(F?`*9w~U*ed8!pus%!7n*X}85zx9NiH|8dui?J@1 zMly|e6|n3p0pr(?2D%sRnd9`4Jsx0ukK(;&v+-d7*dzR=`PLz+jXK&NuAK!e^u0mtKww8}ln#OpNX5rGhk{%dE&ADF4i5@^8ckiv zUhcL5_{o=i3RXSyxA^_X@zeix?Q1Kx5I|z6Xj$e~0x|ESJ}BnW-o&wvWOchB*djBC zk}<8+ueOTkzw_fLaAe>r5%UC(TJuQ((wuF^$N#~J_4^%oY3znyt=v}-pkRcv(bPE^ zJE&jAh`ASIf8_J&zVJS|?kD<;Luk&+#MJ82qiv}^zBN@xS2*BrUYX10uyB~HUTk3LS2G(*^$WOLJAUa<$S$iS58JsI3mN`5RAUE-z z)*px2e+k>V1J(&+fj1WBb}g(imm_R*=cB97Bovb!J~*<<|@hlc37K za_*mHzF)KazvQpJI_mHWMZjvKnaY*hrXqKL4N`z7uw}7&%N$ zx(5#${2TuZxS^%trMr@LbcnaLnAoH!p;ELbf8LQpSl#yz3%TAn{c%sDA1i1p3^-Up zWu!0|{U$}1)3~{S49RN9K=|eY8u_t+zLN#?gR}cCRm}~~&SaV0lV=mcna6ID-e^*& zCwwzAY7zS5`1*VK+}yv=wZ0yf?Y3h}NcgvSIQ&`1GLM!)i}zHsye}Jx(+>D2 zQ^L}kA9B&ff^x$E^SdRne#^Cij|!k`Ln~36TyYLLCVc^X)?d)#hapdkBe2lwOxM9$ zdZJ)OY*f=r851c>CaFsOJ51a%#+uv^$%hU~@?$R+lkKl$B#k@?mI;D+oi`&gDsvC2 ziXE7u5f7Esx#+sj11wKtGXm~xX}c14of68$-cqzQ$i30UK&rMI$KxA*n6f=B3t1l1 zUCo41wlpf%3CrRZXPQ5p>GO@TIdJno< z%=wr>9-9a1Y~PUh&5>DY1;vJv@e82LKI7IyPbybV{9#Ii;3|Aw$h-+#PmA-De#vTj z%@O(B(`_S*d5FXvUPv!!ha)7RDi;dpAynaUGQNGy5*mxlmPvc2h>Fd z5ah{~$)G^74@{CvBq`zjGlyGP5xwU+7!DS6&Qvcx-CI|G?}>C}aU;Mj}3reRTp)}J_G$g{?ehU|g|xCuqiipADK z&es~eyTcJQlU*dU=Hchepw9V0;oQ?=TbkED)tkEd$alBE*~$CHjoGq8YCO65wVeu8 zqX(oa{bl##rc?H|+K-1iP)w9DFBnd-4ku=(Sr&-cIPVN-jD;5$K4ik-`8*}cX!10t ztcRaRq*>{UI6D6n_dZs;_~ZTZmLW)%e2qcPK-sF8QmRLclVV<3e6^Vi>&~}joaThA zxSQr1IpA@8@lXM7Z ze~F6cR@TE)EX`f=Zgv9-cJ|rIgZDWC`CNhHs74zRi9<1sA?ZKiMM9^wI>KxAzi}+Q zzc_tanji=@c2GZ~>KafrxXiD_0j|KNj2y!JiCeU0X-f9VajTwK>ioGovub^XEhI`N zif4ql+La|g1(zp>9=hGiPdue1nPTIubWm2eVuDJ*|80u9;T!zOXvj>l8pcs&Zy~u6 zOkSPEaz-lScpQCW>i8x3Yi5>(Ji$Be2}-n2X}JzYAU`D#Hw>TQD@W2O>l74mB7vYS zBsar`UoBQq29qVpWj4JW06)ye$!WIV6*V3Fs1gkE3Uy~CIP_MK_jjdqX4?JqWF)d- z>E6BTLUVL#JIA{}cB`+-fgMb?5PYP2Wh=sRqs8W6nx^D%%aYAH(7&QD*&%(eFyJgHj>qQ<9} z(Tb1pL zIpLaJ9DGiD#cwHDYo)_yvXt`XY1`H9w#S^Fh*Epj{{Wv(r6EC4O*%>-5|;c2uvi&v zM<%DL6Q>Kt@!gviWi`YCRq+``^=dsBr-Clb?`2JGri)}v?rF6XJoPE{jM1&N{)qU=s82)Jxs(N?DavPv-LHv^H5xcL^~XsS7TN1N_`_WJf^U_(IdfNE&<=L2 zvcp$Zi^x(Ke2BaTUMBjCm8i?zUskujB8d$iy3^z#ScjS(){m&sg1A20N(IE9am=@H zd=};_F6xi-n?Hd4R>ap0U~e+ZNf=%8(0fZ%mWo1wSg*uLOumloSJ%g9l~I4p8X3rUXo`GH5f=VT{Y+BV_0cM1;DBOLUPNtC z|FisulyPZi;1b(ZZ|?P8nf|t-bx1IozN6TBXud7Cpuo`kcFYZDfw#)0E|9tU ztk1y^f;B>{xI4x#R@u15Qzyx<`tzTT`$_wRU>&!(So;mUBXDiRz|q-~dw$8n!_*izl=zcP`aP%F(f#ays^00Ihf^+N7M^)1@peKcDgZ_IMF}<@Q?;XNz}V;~>yT9CXN1QP*>ZNp7QQ)Uc3_5F z%2tQFlh^T51evqYxIa)>mZqH3tJ>iwm+iM3h891~6tfc|OkR1gULZ(4h<6N&(SYp{ zaAv<)#VyYuK2TB=3>j5m9O{Rc4{$|jyt;93T1-pcYoETNMxBZmgv)EDtGIM#Xf)VX z0hvh*Z#48h9epX~&8t|}tG$grEGpqlS&rQW+c(}+Zmu+v91(?+^3^7o2OVYA2nJ3n zrs9X-fdWr`kOMr>Q;Uim8o8~vt+fOy6=UvMwv1N5E6 zve+38D+8Ys`70&F#Ez6lLE2(BKHVAaDxvun6WdB{vjv6p*Ob~H#ggA{VF7tg1#Fp? zIQd_j$0qR_LGmL3aC1d>9r6UsY5Q*bm44&<@t>`xgD3T~iW9eQ>Fdhee!uYseZ81| zETF9|Am}m;arB=T5R&Al0P^U~1*G^xy7(^S4CZR0U>&Q{5rRmF#EUOjZwbwo9JR6P zhic3E{EG0-h~~Fe09xOL&FYZ(ksr?^$9jQ@>mRLX{ZSUV1&sl)h}3{Z)C4mp0FBtR zSy)VstXKc9|BwCOQ!4(h4t!@{(TYTm;XQ^`iNb{ZXK8$zb{T^~Q>QfE+?zXOt8U9> z-I@`E*AEwV*TtH!SnBrFY_o_%-3a>-Bo;QCb;cE$0ZV|~>G;9a|F3Q8|L6hxqgo1J z>Ve-$#5#lnM6z{=^hSF#5%8t30ZG6xz98HE{{iZ#;6cSIP``N%Nw*A_=ege9Q&f9n z!*f(Xo{D+iG^M)u?6ix$6Ru88er)9hD)N}mFSz5#_88swP-R*3$gnq?!b&T3(N|65I$cgpB2^g2 z!HMoxj3K}N{p^@A>T@0elCz1)1iN!(gsvzoAYNCmb=+QM$?J&~mEU`KFg(d~>A9QJ z=WM^xK&3;(=GtzZ(%6aV*((eztC26@q8B(1Q}Rcgf~+i6De(lpZKloeE@dLho zF9j$+yAJSJ$_I%gLMmVr=Ug{ME^icJ0jIr6gZkX@|A(H(JcpiQA4c+?Z6|4OM`(I4 z8b(tS(%vyHT#g_@&`&cG2;KfMHKnv2xenBiBvh05iPw#F$flHEv>@)P#*?eEM3a^d zCn#eJ_qjfg*pE@9n_{0lSSiU=)>Wk~(iqRC5jd)>ah5P=+bkM!xw}_s2fhuVSrKm= z4w<_WDDer&l-Y=SAF1e-1nf$_lRCJsuj5;6%#U8ypDml;8vTCn@f*vtg}6*WZ7k2f zH4J$pCrQ^ga!V5CKb7gn6&n)Zl>@YozF-n2HQYkiVtW)CAWLJcgad_8qQwlz23nCtv_@6*9LCwiG%V%*|m?_6MEH~-NDOC1cDlq{ghdXYP4;n=e|tGY7F?j)Rf zH?n0OZMB7j<+l_k{h7yptBr47;9HGL%3OKB_f`C_-}}*~v?VrUquOp;SvR6_lNP$M z3BZ}jIkli3+-~hZvCl8ZT{W#VVmQH2H|)5Exw&MeZ|l8dRm*z!zbc(dg5D>?5NAw+ z5~UKmbvmv`h$@bB5c_#a@94y9HS8Of&=%_ue|vnu+IC>1x(;DohwRIuTm)=cbU+$% z!|H{D*#oZ34GMp0aYMkaY?n1^i3}r4o_@4whtD{M-J^d=99EeJV~Wk2QRb=XO# z(CkiVJV3imd7`SsHG(AJ>sDmjf9Ek@^ID5E-%w|~6luN&2*2_FKeYUw=CFOo17sU<^_8Q zoHonMVatbI12$bNK4j~+FU)FJz!)u!SqXgw`%3Eb3kqMmU4IzxF3X6`RxzMn&7;Oc z2-p=4`nHgoTw!F|1M3iM8fDT`ur`TjNB|ieHx2;J)NDv%dN?xBcu&6fXC$*ljn1{e z8M8YiW(|Hkh!W(o;ngx`cgrpbP4@lapC@(SoVWaVm+fd@JcaVU+kcQ3`&SoN{-DqQOeFVbo(s_D+mY`FK}w)U{#%**m(3d{g^`Ds_ zK)DF!Ush8lF&#zWap_Xg{8wkoh3>w1vmiJp{9&qozV1{7fDj-DdhvE0a-?i6ztrog zmlE>l--{sLFj;w2Hv4ED(rZ0N1LjGTa=Y`ixl`X>@imfq_9F7EF!%m_)u~yX4Hb&J z8c4D&=5AOnMy)qeOPE@ewG8CJXyFM{=y4xMg9{^Y4iY7ZKJWl!{>%OS>Eme^{WzPT zdL|CCSBT;B6rTpU^EkM)v9#hJjSc%NkMimsEY3JE>PU#RPgM2THM|aCA=--U8Mkl1 z?m5CGbU;t9B$WObt&g|~bGHXRuG|KPWKPAH@oT(3V3_}99nv2`-md-$ng{Xl{eueb z*NX0+{{J5aK0oJ#zvD(vAOS$h=o(O}Z#TKwpbA^l4}t&ZhKc{KQcTX6l)K~oydiC( zz#fB94{NJOY=2BKq5UdLhIEsNpo0(Z2i4W`5oBp)ZdHQGaLNmUh;}}HI$@-D()8we zode;f+7BQwr(;u&3o^j;5TRL#arPxC+N6wsIDD~#{W=Y$)*;zD3;gqJTss`qar#xU z*Kc%J2sosC?7QV^zglEMG%AC_$F6L+qHq2cZ&epy|1UfkfSUgaOgFp_2n>#7GOJxx z%L*HE_7q4i!dii;urUZ)y9}497@)>j3gAnhwqRSKy3~{JbYPjVn4)n2lFV%SKfMZBU`ILCZ!J}*+%^Gny!Mr zy%p*Z(`Xy0ZZP4hOw@U?qO}WNriz5%-tWt1@2lG4AARoM32hAe+@)6{Sdbt?iQ42e zcCSOY$-p$?RR&VvkC#jQ(Q=F5`unnF9Ejl6O4q$znYqJh2E9^t`}KPd)FK8_qbeqL zjyJ0b%2%#Kn2UL_B~b=enl=c)|Ih8!K&iy|lXhjjvoL$(bMjEwwP4l5V^#a_h0?1( zb2ldlRyZJFHA^fsFK^yhTX-DsxK{i0$_p^ydcaGmZyjO*hg!Ym>E?;kc%$I*xN#R;M-0B#8%(yZP}R@wIJzKzqUhCI zpNW3*msckN<`AfXfYjpTgtIm_LGYBAbkBj;tOtHV2seXihV-81UAf(h*X7k_SRroS zsph~9v5R^N*oz*7g6Wp8&LdZ(3|2h9jh(Pi4ulz*euj0>0(NuNyvMz4aM{>+>O;m_ z-(#~oa*Jq_pM$SA(6F*u{k*|d=XFRMVsdf?vJCoA+2hF1XGqUMl?Mhs{17=|yAFA0 zw+^vECxO`ufeJRm5ya!gCLT#1Ia8?XvtZ)lh>Kf+8hXjWKJ4QprUd!VCm2$a0>Rd} z8#y(&4$;7&(PNY#0{#=`+hCc#hKL6Et}sXZByJk@q#}*u$!9*XQ@pd)FOHf<>hnM0 zEz{ZKp=lyc#OdaID6^Ew6FO#hs10Lnc`(M(CjEf%Eed`-^Cax&L3X2^cn5Mz^$iF4B65xT2XlwYs+i+S-&qE!#7e#J2ed)@P(RpWn^PZlL#!f27@p0pxuN}I* zpv3ds0PZ}^$@0`i5?0{DItb7-*j%`*1r(dfk6@L<0KQSdL01IjMn|MUJxVf}nK~3c z24=9|8zgy__<$A&u$q3H<^SY;A)te$q~k>b%|6(|*Z+o$LoYRw+&7#*kFS);NwHNudkKoaoQTj~1}@ z%w5iII3U{WGL!G6P*fzc2f#$M;s%HO&Rbx*=I~3P}VHuT&mpHG=cHyiSbrKT=0Pj#GngN8~w4&NNq&iN+zQWo_ z_PxDyPWCFUcUuY2_$fa_G4kY7+^BVkY1%qu{$}3%ntR2nIr&+sH(P;^Y&>gp)7k6BX{Sv0&QF>gAYB4!NGcY(0yh30z0&eF_@~_R*^J2Kq3LKOnEa*_a)4f)|xH~dH$am`gNE_vW0;+)U{oL=HILFm0!ng&Yr zf$!YoY+3e&sjiLjecz5pb#3!*jZhlZl*?A_{R0OJgYtHXe-@|rbd2TaRVpd!2K-WG z5+PuXIl*y*Yl`T8b#_y{`q|Y*JsVq!mm8VjKSxTmDVWqaU&1|GRT*W;x-WOhImq^^&9T$Vq1CX*BSG~oXxqHc zAO}oS0JmAy{PG6Q!YzDZ(xHy{`U+DlT@^n5TDGJkih^QC8!lxLh)@Ca9V+|iX(uq+r6Qv%Nz3L!7*_-K7_Oqw4F{cWwoUg8L ze4lZI%a!4ixglW&Zf~m#M5Ue_a<9D#ZFZ>sWZqq1@DcQYA%DCpKi(DaF8_E}Hbn9N z^Kr!yN$1JgG$bE(1B)|?(LF}FVd3%mTIk`!Srps0~vCCq#uhN}diJB)+;?=Qid{C(G zw8#eDDq10LfIc!)Ts3@!wQTmzAehc`7OZhiENpG};IceGIdy`QD~BmRL5b%~fcwlC z7~Ip)izLgcla7c4BIiYrpQg&zA)J8!NPo z04)v@pk%)pQbGYh3HOkd(Zm2?9k4_`1K=j%-IgzFbkJ{bhh&0ybv|Y^C^@|IXv*CF zyh|+VF4X;3&qqQO7MfMT++;*uogk8F41XM}MR~bn>WlyW;sT%HPF_V8Wr`v0fI;r( zUJBOO7Seq+ybS(Z@QO7C9&*_S<~a+5_|)tXqlZT%msc+FhQL!-#`PF0>)bDW^d z@`83tF@TJ+&Ll3q9M>y@+Dg2Ac*RC-`eazH?2c}09}_NR44e5N6UA5&cb*wmh@oA- z{n?z5aE$v+wVhv7xB_{6rv;A{>sDd>(L)hEnNHR>)!k90em740X$V6&xoZq>twUT( zsE12SJg`WJrE6_UeYf+0t~^#~UVj_d)?d-~ey;0%r3&u2zLoaF9gBc&^4Bwq;c zwi0@}meQ1at?|=zf2bK+QS>5gyMp#hBDzw)JHCI;s87}X1Mh2rVfS2_W#wrti*fj@ z$`nX+K;F{pIjYojB~1bY8Zp}g6BLqq8bY^RA5$9pA+rz6pgUj-6HiZ6y4y-sa5wJ}m>jQHb)h6jN?-c$ zrKx7mV{D<0EjPP59l5u4XMu_h$hd1)D&JKs!ayTIt%mIe`{I!PBUq4p@o{us@)F%_;| zXjwRef;}FL&dSLqdRtmMr5fQ2^3y5Ucvv`;so6(PQm-t6f;K*l(!G<>wG)Kmy{}(C z_NO{gLFILbcs6rF=>XC3`Gjj`Lt*j#=g=b;&e;o6ltXTBdwXg*xm-zr&_C+?(p`GU z=7{69YtqhhJ1{S2#dn+=wL7M$G0!&o9A8%>!aU|F<6BuqFZ5Ku@cPN=W9iH{h7AR0 zU&qv{i4ylJbK_K*dbqBJr8qD%v0QbdP^NoY3_C@rt40UOg+Gz@{nVaYBKDyYjbYZL zcgz(P=($QQ1EHB8+y+vFYZ9xuga`T38d5ybgSOofl&H8tMdhSD58YLgt6r+7Xry1t zddA%yrCMY)9<^eM4y8Oo>DxiCZfZf*q=u6SmF2#? zxx?|FD+Plds!@GbesBNL!Ma`o&FOxM7?U=^h)~j!*K8N!6j*z5KAT$pI74PEkE0hP z35s}HsQte`nXIgTxtW9PI)n@CHwpxiHJ_}qd0l-tUX!dr2Sf@x*vi?1x4Gp(lJ9~w zS>0iT9DAPiGl&G4-?A7(__lvz+V|T73d(ONK=&%VdZT;QZe&wg-%VC2<^T6kx8KdZ z6ew8;%TK*xZ{|YNWMj{ve$?i}z0kdSfOH8`xHQzHUP%^xRMo=RA{x0{@51L{w&9)J zsm~Qvm=|DGuz>v1ZLLz$Gm@Hl`o;Gml}(kf>yX!Co~it;bkVC-MS97ZmMb;=Pc=MJ zXI3Rr)2G?sCU(c$^8U$K^V_Ynl%MT5EzGM?xwqq$i(}vSEo&|Y`d<>YvGH&ti8v>z)C=kazcOx5kTT5e*W6NONaB4u=ups_U>eF2; z{6!&q-`>{WE*3LYsAgS5k~JedsxpwpKYy6gt9mnQ;KCIt zlw7!3_RPi(c;2Z!IhyH=WLJ>PAfhXFrCQ{ptlqLly)&EJtE9um7|Y|#L?H!`3?KI( zxn^*!Aj@PC0MRH`UGPSO>rb03XrDJa2EF@g%t58#tP$&%=Ufx^A~6NFUvOylWqK?zgUlDrp~6wK8hP3)3909G7;*D`S&r@_XG&g80d4_3vl zS-&Kn-KL3;Iqrj~sympIkxc7$EB-iJWa=78A z^_T~JN_pJRcHg`~NGK_dNH*ZA9AMf0iq958W%kB9G@I>_uV8&rkVc?y^3`stS)CJMp5Ssqt%iC~_er zXQ5wDLGT&L5KkRoll-b($VpI^^6mJXTwJbA-4j~zxBf#>4MW^#BK+Mcuw_gLvIsk3 z@vux_0Q@XcdcGXp)U)aVmeu)UC3lOuM^CsH7e3 z>on>`XJ`mAlJvH9NT2o^L+7-s-&$$h^L{H< zyVsbmL-BgGx6*g<&oY04Gm`dT{Rv$OIFnt+;`H?0@Y$YA-A;)b@3xFW z#MGAV=#l_6c9!-Q-TIaXi!~W-muYBaIU)#*bzFxjqd$U(O{nrMwT7iza#BhOQ9fHnN&210>1U200mw+Dv4|mupNX2lf!UeM|XGz@qOGwW2qP zGgsrmxzLVH7M_u&^?_cwlqf)SB8b!$*)4YEzYtD&ee@3I{JGi?+7k>-(Y@;X5e@u> zVB~N{xfL3ZtFkI*mb>tVXNmh2!uDP5OXgq@vKc z7tv<;&!{xZGH7mzWXJKe-tj2q^RMI&UM#OwQ>oy_7Gq%aB%AjAQu<14*&Wv@t&B!w zl~S$!)^%0gW`9EPtb}_gau??e!>XDLt61d{ z%do7WR+zW}e=%t6Zd;pb`{@1vN-h^%$Jl=xE`NJp_&vw|n3A!T{t!?-#7gp2(Ap?f z1{;;?-}oZf@(x5CxiS;9+WZNQD_cgL8_N6lgFKfTapoZl%cVTWhZA<6E%i1FmN6LU z(q)ggbg96o+|1yqz@am*tFI~`x+WwO-KGx6UcMe}{cyMM^oeWcV_NW9liAx+gNw}V z$ayyFaZqpLOJ=VF`WQp&I%I}4NV19`n#0(Tt(rjZD+9GPg!C0pz&%kJY1f(Ta@V#q zb3!^ir9GYhd?Y15QOv$_lud?LX1iH7`ekm|DK0a$?2f6~{l(t=%C27T*eI8~o8h?; zq;QPMnr{F()OATa06D8gk_K|;7#wsna7q5of)}O*YqyLN?iEZe?#q%xZv!VIvPD|p zSi#UO>sm>Z03@@5p&eZq7QS~p?3ud2+Xl8qB{|OR=c-*91pCI4XbhC@+dLWTGB8Ry zFwn;AuduLRP0XXo2q>X{X2Ja*IoFR_qyMnJ|CLA#N|gZ;h@7+wQD9OrX zzKx`6Q$kks*-FIuvyS<0Wb+dkWV4ZV#?e?OX9IH6m08*{y0FmuSwej!%62 z<;$&YegS-_GkZK#H=RIWV&pdXth^<1kq)*ZI@zzZkI zy*XfRw-F=AmFSJ_dD9=H2b6LuCxT4#|Mp5$(8Ot1Hknkfq!{mSH4Ar%-Qkbz-fy%R zef)(2qi?AFlA5^j1+LVZjGbLwJB!Vk&W5E&h9m?lQVodCqhF#`tg-!%VeK-}vIer9Q1YNTf&_jpt32-uO&g>9hO-|F$kc zYee;}N*|dIt^(d()WhdKe4o}ozXJ>Wa2eQ=fW6VBp0h;qDh8YBXG=K51`@cek{nNYHeuGH(@BG_8{8+yo;6N4Nm(_I$*q#DGZyQ(u+qz%r zN03{}W<{(w7<%Ly#a}%7>K6qaV4zXNpVu{5dWpba181>7GD^zCU3}xA)NMQ>t3+B&WtUngu z))wG0Sb(2F(aGkX#Im_3xd9sA4pgsfg2vr{vH#@PoeCRQR1KfKt;7f}=sap=Pv`Hq z+khRJ;0AEzZcHtV@j#gL<(`K#Jh+4g89r!wEG*1*sl0_@kNNO>y_|Z5wIfYAvGCcJ znf)uKpgh8>TM}P$-2Dy#^J+WGo9*0ynPDDv4NrE%=Tw(U;HT1G`H-m!0v_9_@g$FS zs7)(9_Y$(U8j#2#`r#gWTU!|HHe~4cPW)--`~gXJ;89bOI;{9b&aOgHc;!>nj=YlU zHhQsRFJCjR(0s~6?Ey-?$g0TYo}!zD4|xjT;83EYp+l?QdB`EjyrD8U?fo!KUqmu%r1*KU)EvTDRiB> zp&(Y0>Hrl7?cLEkLoV|RX#H05c7&O7_t89St>YK1Lu6wux-LBxI4N6EZHgl#(b{1s zt1-`G?4FKGNx%NsYm)2#8GGP3vr)3Gy3BGAZyB?22wqo1uYDnbwRT_V$0&Yr!HhbY z?Tle>hHzY3uF}CI-eg3}DLLD0>IYsk;U{cfpXir9GDQkQK!e=nvD)64u&j(H)?rF< zj#4d0h_vPAH-kRBX?ZU@*$!Aqba4CvXec9dS;fabs1feqvpmc`yY%MVpw6zkQo(l$ zyYW@038VhW-NFi2UTD5Nd9j)IfvERugsDRhPPV?;x<7dp7Nj7#t0Yf7x5FkXr64Zy zLI+ps9GZ@Ru9%G2VR)ICQBl;{l$N6_SU$^qTcz79Jb5j4s;`y05P>}9!C18@cR9}P z&f=?=YFHKj7_Jun+~s<8zDoQ1Crow< zlg_@Ver;??c+ZEC3c^UfOO!L`;k19eCK!g7^3HNM$SouQMfSoARr5uEe?cK7lOy^N z#yTE4h*n5!sejy>JPA)4P z*?qC}3m%c$7|r@67*oa-=YdT9z9 zrd`Pw%yB?!yXt-&pH*@P%qcN+77C5%KXa*kWg8Ty%n^&Cl;qfbqAZ7y_yihwEI^z} z>8yQoJL!aZ=;JkcJ-c%G{xi55PyP_yl6b5|=kb}y*8v0Nk#ck%f<2NVOct(ZLe{cf zUi2RH))qa;K*d1UE4mak&x}RoOuYk>5o?^!`tN<8abTKrsJ{A!mz3ur;v<7@caKz( z9DZT$Bm3f`-h0D^M#SxoUcUAfK-{FPO@wqPv5em$W>gvTb#PnopLNK2wSVar2c?g= z*LmM=G;<7Lw@Ujo+=3;Kz_cWVF@Apa09J~ooXzLR4sSNPHww3#oCopEN>Gqc#})#; za~M!7=PAZL*}Gz@joI6s?rQnW_8&a5)~3#jmu1elt$A<5Pa_xxl$s z!YX{T%9Lf#taPY-h-H_$m3T?Ic9a&Fdqtl2uU#~1WKp^5J9?vb7bGF+b_+iS&Po#Q zEZ#G$cw84h7$OPHQQyTHTd>D#>}10^8EU1TrG-p<*RVtx23EaT(9GhNS{!_U#A+s1 z50^Y=-LCYLRjeGcO3PUA?E`G|G$ZZKy3bJ!oD_pAIy*$XY zwKj0nTDBMiMb6)YH^xIIkQTc$Lf`?B8rJql8Muz^FX9)l&nDJqHbNOh0+e=J7Of48 zhm}Uy?Wc9P=D5#q`)W4lYB$HK>ZGeH!l3cQ>aqM}p|-xA`qA0xulfca9u=Kzd1g!y zy1NqbrO}$X=&hV3izD;d$LTwJ%6N8>4pTnB$9YSd1h1S=JVMY8oWZe*y~_3J^GajF z@o+=xc_7rMN8>*d&t^@^@rN%&1YkNzapP9(Q->lyg{5#hHnU&kyoO_LbpMH{I}GI_ zYS~RY37wT>OR>m$;$u^}c!rAT!)1M@r1M%~(4}pcHQA4` z(K|3x%uFhCBCFp{)+;F?qy{(&aXu=Rr4o^TH~ZemQu@}oj)oZF@HYziZoZpL?PId;){0CiaYa>o*lD+tcD&R?-pY1aeBE1FR>K$Wcj3JA zB#|~Af`sVvx;C2S=};cG!6~hkSNd=w!4v!rb%aOGkE4zN^|COH3% zX#Cq$voB!FJx~fl0YqzD9_fVr3xm)0jJ`>@s};pXW&b{e}lJcC_wt05saw z#m@w-L-IU7tfgOr-&j>@5xnh9Uc|j+t5eZ0K8_!(Bp>N|(wS$Y(B#Y@K*jLdCtU;L zb;~s|uM-R{11=)PVDsD+aaSZ4mbVr&Wo$A-DigQuVJ1(jITmoawHLDwml01W9K7?xf-qqHcum^pEn@?Ehu{#^Ij?p|a3K#pUQm2eZEKjc(S;*L~ z<+cvBG!DK~6+5##KTvp};&^i1(Zh6=0lbvF5!L#SoA4$zu3>N5olFAb3gq;L3|Q2! zmfxvTQhL~|D2Pc*qU92)SW$`^h7wh$rW!3XcUK)KY&^xuRTDVv;OfbS7S1E?P9Jx6 zrXN?5EzNFAm5n|eb^qg?J$aScb?%Bn~L2Fo^ci17D1}GUC}%8hpXS%*ptYjuj{L46QzGU|6_!Ap6Y~TJvYR zYY8@q*twd0(-wL@;?CSFwDn+3Y$LQ%lHL!43%{ zMlJke9Lu8=jJmZYYk!!cTeLRcYN7=tw9W*Yi{u_}pi1KANE2UlODiQ^TVNY2AG!vs zVy--nN7mlKhwSt`II(y9-L~}f`kf}q3eD8{_WDk%Z||)=BMDg*Fl4?Rw{4<#*c;xP zcyi1Y`mV&vORQM(Daaw@hkJ$iryTm%9@0DJ2zroAU%!sj=S>^s>vkI!q4|gqSB_9?zxE~?BW=FtRt4&)g9`-O zR&bcI=JOg#NH|x+tdOvM&RVzmb>j=|T`?n&k!^9*(P|g76=)W0cAqrjs4}<{+7{xM zlN{+Hp74>Y} zLw(px9d?qe+rUGE`3}QEgj4TYo9uPscy#fxbmXIE66ai8-RC`^U-KW1$$vJD@>iVi z2UW8*s%B%4&Hu+9yGhl!0IEh8l#^{zHM{;lqH4Znqrwfx!j|P=zmvWEpZ}2Hwl2Z7kb+&i z*ZfRC=l!!yocIL*+5_P^e}k?cEdMX9Ax(6DU+R>|u=RyE_iY9>?^yUohTiF$RX^V9 zd13Mbj=Ue}-B0kSP3PZM+l-IDsb>H8)MoxWf9FSC=vLBYAXx+1F6#KD|JrljY>a>d zwI|4B#=MACng(Dgg!_w)HPJ8vl#~FyyW!TtPxf`%s6hMq=J2meM`=0GJ1wOX#j*+K z8;*1yP5S9r5{b5kDgp|KC-GwH0ZQFXz1M5AJ6lNm#aV_s&yWA-?ei~ zL2Ik+>EGLNKbrDYzdbw-sNKzjvxh*7!yY#gSnIaYCYtmLrFg-YG><^YR9*^Z zrwI`?B0qzhgT{L>IYuudsQNL;%$lml-Y1BF zQv~7oc>f4D=KdV}La2ZT8~6FueUSF2#dcU|Fc{i~Grv0@!gUHEvHHY%wD$?ewfSp;M%1-ZlAZC9H8-*L8#q%!Zgn{6w-QIB|OYyGf$VyFnY>o%hndhIG z9;Td}y(Jc&#M}TGawb{#xbMQI5Kz8foRUH~U;D(=mN2Nwm@2-FxO$i}QmX9d!yX-Cvt#>@Fa4Z8D?Fwy~nRGfVVBG}04-ON#f zhM<_tIm)y>ODFqw@^%AiXk0+BB5S@SL`k^K*8PlAe(TUI#!X9%-B?Q0JCg20N5l-b z2PCUt2o<^@!b&_dWYxC36f3^0<^U1fuX8?Met~_)TX`SjM52sIaO+3a)ED@NlpZbj z#Gv=I2bw*})TjCh+M)9zRJ|%ng@~?kdFGItXGQ|q;kP+GB}erWk96~{rP$ow-jTXb z^Kie#&X33sjYuWyn6)RlO;-nX^E-s|Oyo}$#@RfJg|5h6><}fLISlar&a2k{-h z=R}+Ou|1=?c*ii3eO&IYe*9?gIwW^~F%J{__9Ujxk{YE9(%=skoA}`08SkOL*lR9I z8`HPhDO@+hzKGUMzocyHclUwvU6~OeqQA5n_>bnp|J$|sf3~#!!Iy89FYhLwNH8Q< zzDADy9WR+IGf0~J*?6gra3;<<6Jc_;{ibkfqqF_4lc{}bwO<@$VSW&iz+#bP1p17W zTvMq2S;tqes%kwh1PFNEpIh|4;(~9)b>@4g8aLRRIUi6Pb1gncC+%H3!Y{kn!hWDA zLyHlY80pi@b)#}|Z)KxI(0=aK1WH()Bzg$$x6l6})|p@zk|)vOl;5BlanS6l=$mXx zeme*RLV88f1ZwUEh9Wt)lok^Tt#8j0V%y;YZi9TYF6n}jEAx+v!v@*#$ znwW`yw^SQ5zLFo}#+8M>Dus@(xzK%Q*dKD=)IptVRbK`;u8^Fu#K*+J>Zm5pz3?8P z1MWxSENNQulpzda%6y@%JAgh>MLxl$POgjqX!PkzlYp{W2X3Kth{NPMME(D<_uX+# zrCYS22nqrgP(g}PM4CvK8W3rs6pdM4HqfU63LzO0UvO zfIxsGybovQ-qGcy1Ot$wH7Q6h8y9_$YBw{!_ zTFLKk

OD_y&-(m^k-#Lm4Q|jNe1{7ZGewu!WSNF2tabywS^!xf|Ec$|VR;n%-hF zyXk198B{7^TE?;u5FrTCvGQT6MLGBMgll2>lG%=r(S_!Pt2BnU;%MYMfav*NfFl{Q zfKf-0VZc(=#d+q>udji#Cnqu+5u9HWP`zE&jO_~P^6LW#8LYw}m2QQERgz_T?}am~ zTI}~NjUGdY80WO(r~!$Opw`fWf3*oo%+i6I!PbZ;STJaQ!X2LEBCJD36S{n$HpA|+ z@VgF@-HuDhj>qekJ>UztCRyM>w;9JtU{F0^wWoijA12YSMEdN+p$wC!yIcYv(-L(} zRx=5<-VMH_jwjWQTnh5hhgKUge2|3hof6&8ppc*AP3Q0Z@HtQ3qwXCLVsY_FRUWAl(+Svteoh3}eet*utV#iM8DY0yjvVo1 z`iY+E-;jRcKE#2&mH4f4X7;o2yIl07jCWQ4joSWqt@vV}le|hPIy}fPSw$`FD$$*N zdoB_08VR2#`#P-jW|aFk3Vl=o5#_=t})>Vp@OoXaB&= zzQrX0%*+m8W;9v=Gs}!!=jQ%tHmikjicj??8~+ZPIrV6rBz(XSpMlmsG+2Gmrc(Mu zW_tFVjeEdF93NT*?Q@Sh*?!t$r(vn_+2?Cclr5-8BpUF$kiO6vCNYAiMF2kbeb|s& zW=4_MO!d*ZTL@Yr&Ica+XD~fw$EZ) zOQpNf2W>tQdEPT{FI=Z#Q0iqU17y-o!H#O)6JwEKrlWVH{Ya=d?Wo{mUs4acrIsp| z`-RN33*of^@e}P+>tlUOjf~sS1tqv=yA6k&!!EPIS8tZN=Qa3YwIEu6Jd0D`L znDm{Gam5~V(HmkHdT4e`6~=ETbf9Y4MBl!Q3)pVBekCl(1#&F}U838LQy~i)iKU_9 zX*skb#2CuYN#s`~4m(HXYUu=@<+3Dkf}bkpmT8SpXDD>fd;(3+avAXIzu?A^pmL(R z$%I6OfDW)*Zzaoerp`y3!phO^^FC5EL+>!+GZ#Xg$4v@vJ{2W9522ywYca=6t*ute z(4Y&e5C_efc^2GY*3U~EL$7N~Ri-z+Oh`FLG||{BE$Qh3yX~elLUnm@9aUp|(CBEz z{kN*jv!S7Cvf^z1w?u}K`$S@Q8M?r<fxVz`jx~YGtP@s_P&IR3?XsvVTCkqjN%ct5!6iF2? z^O6mg6oj*541?tkp{Fmf}^rkXR$~3gr6*_b4lm_R@JEj0qP@Q7hG%ud-e{QV#lQi0#N_ zhDDIaa^8gZ0%<%32&RS6$qz>1B&LZvber?hTkS_;>|~AUi%MPb3*^4ZXrDVXB0UeB zX2*&INocQxAG~s93f`BA@ZZwUnO`3VI^w4h3-SaN<_DNqP=~yhJ>Y_>iT2p2SuJN= zfC34_@tkWaAn7w1&!$=PiK zYfIa|;T~FrQe?4 ziIM;2vopU6sQ;sB4!=jG_(6mCce?kVn(-hFoB=deRiE&0wgmI5|CpARXvI20eXkoebZIC#*8`dZ%P&MUk=%C z?>rtWM8ei{Vm5Mlz^9*9hH~NRqX6~{zG~ORm{S|cisC0$n8@2arQ+c<*pwlbAbtk~ z&GmK*iERYOo>xUrXrp_SA>rJ1-meZwEt*4VEj+}qg_h(#%g7yllM;3f%)`U;Hab1+ zyyqcA<0S0CfPZzA1T+lp-Q=gn@f9!N@03;!oQdS9XjC3M5K5kQSxtJnru?9}ha?sr z*K%UPtN&R-uVLX6h;Q4lI5DCnAaRzBB*Y(~IWVbzlw>?hv9SiufYVBT&__0YYGF2{ zw{}qQ5otj69a%$jQV^)zY!_s{0#x_!FDM1@<7_S9 zbmKnO%UZOj%Ww4Lu!#=d+LMq>{sBzx(YubY+|w^eOH>)0F4MM(z9n7!l3G{tf|2%J zF7g!3*?L6ML>$eNQhD;AEH{UE>?TBKp{K}-f!&w!anVybZx$SdAC??>JQitv{us1+tA)~fJfR?PS+fYyML0 z#C^XV=PD3P#DJFbwFOK^RTWCY_`&sk3b}fHA4@S~i*gg&{kYFgC?s4ea62@>@w^84?_QBiFc{;pABBPYMi8Vsm%ZO+B`xg~ZBDxI zzILG4#@0WydS_V3?A&<`qR(5X-*XJ>lg`u^ES>oWFCqTliYq^AtZ%HbY=N+%E7(g9 z`lfVruH@#6H1F#ODj-_my{-V|cTEj)#M^J^$!dP5*ZdaS`}^`LGAh(P;iwjRJ#6}w zhf|L<4jjI8WY=M8aWK?5%}|)}`s?)JzZ~KE9P#=aAB+Fc$^CtWE%{bdFrgW`sLl`4 zEgnvSpI@1C`Q*ATcoMd#Hjn)2IZtViZ?Vkp^_u@Gs4Ml^46^ICuR?;JQjWujYJOTL z)7oMkpCKTK8(V1bOgr*Wb$y#4waNiTipEZy1v;!!$1-(VVqX;By(`Y4tUeS%116*H z$~BtUhROZ4vD;s>akD1n3JFv|3`JE$dT|<9pP@3zSNd7^^rMr;Ftd4&-Wv2Yyx3)Q z`q2&3>yYu7mvPivpB9i+>O2(QdC9YgcGZapgg6drK=IVZDy@Pui5rp;~-Pv47GS8Lu|ni zla_%^6QNREoV|SAEyq?;c`x4)+fZ9TQJjR+z(=2AtqHFRfOrJ2ZrCS{vR3kPUCEIL z$X$Bab5kQ#`zptEbkw<7Z@-{~^zM?i4OF(Je7!7oF#; zsgAbEe9t+t%{ef7n1u-kLm>dzCt?zJT{Iw1<-p;3_Sx5r5r?jL%Cfv68OY$NUVKe{ z5A)>mt-w<`(IKsaX}fbsnQKc8&ZFF17w+mmcem4(In-(uWg_JnabWt6vu+(5`(0ca z&@0vG$5LvS6a=e~t{b)~jwJA$Qig_r!JZY54h)smUVQ`m{mFto$i{e|&SHg~>=oU(HKLu)k$vc8nD!*U4 zeIe2Ny+rWuIR1k>{Y^APHhk^6ZivUv{-5h(OaKX2XJIQUgwPBJQ2{>l@mWu4m+$5A zzvX@YyHVk*ou7<49MF2=lplg_Y*PUfkL7w5*WQ^*ajvPEJTAmg|j4_K;NcC#MUHsT)LMgpPXdYMhWVs{89Sw=r<9%@R! zD;OhO&8VMUvb}1P6!+3MKA5CaZx>wNcz22$H#TC3C#sQ#(kZZwNpN!U%C+uy6Fbh` z%P=X9>AOzM@8?*6f->U{@7&8CYM^>8jQi<~iS0|0`8z;-Fcbd_IT{IAz?T4Mx<7M! zIjMO8Gf%6Cg^_;u^bBgZk-3nO@~}$c1;-q@JhUg|^=t7aN3g3|iYe7;q>?LfGEBe5 z=2)mDinpOe_M)>kr&M4EQOd$9L8S$VrOzC+1~iTuXXwoACgd<;Kak!4`*8SN#k}&0 zmZ3W)8MGT~m)^gxAq}ACQ04Yit|w=7(cS%fyavsj zbggr@iEIkUE3NBMmlh&fV=3}TLtJ{o?)L7#43#%D8mNTRIvG?u>LaVltooY|W=`x+l)`3sVw)iW6uO$_t+-KI4(#- zV;56ZKj=Y1Tto@K*UyDEz0U`xEF(wv>#V#Jc;CG&HJY}jeV%b=NT|CjDs%c4KYXZ9 zNxk6R3qyTXQrDLn7byJKsjrhXe74~w-J>n6vYhcP-)m;qzLL}y$&(ienfa{ROeQlnVTx3A=N1eU&yE?97Y= zYPEx$>xJtKi|X%N2r7N^crTy?`_@)BtC5hl3F#9Vjm?e4g=MsCPynYGKO9E`G=KlT z!IIBw@PF>GfL3QFJYU##7TWA2(`n7lZ0`ShoeSl<j>z1UkLI{zYj$TCp&x}m6+0I@W_)%-0{&W}1e7sPk?Y4KAee@4F#Yca z9ijiL%W=N&>31E5sE#bv>d0_E;gCprNVZ*1<=%GrKwac|ZPwKPjAr?t(L4W7zxzW6 z{TqB0F#r}v`qfv}7ZIHJXz1wEKKMH|7#QGjsRj!E^ZXk=Yg_mPa#76+Im5|McmQ_m z-TVeKCH`Ko`G*xszt=4PCm;XO=>J``Xm>BXE)O}!GG?r~0mOG+ix+`7^l&Q? zm?(S|E~5c%YQ4s}AG)$90(9wtJz0G(Tltpv`LEj3@vmwf7=N{YdLHt&A-Jg zzSnF1tL7#32fIDrE$jU|Gwg5ttRD^2Z&<(KJx`Dt|JiIliXj|X0xm`oh9{Q5sJ37i zV+lotiwItPG?=xDY5_ngJ$!ll1WDKRZ+P$jl$QU|=l^1D^RGh(Bwwc-{MOpPsVVWl zrg`wK-{*%m!tZlzh_;R6oO4; z7YttK)q=liJ!;X2G!0ri0=B!pI|;*Yj63C03uf?ia{^utl=i)75^H$#D07!eDYLytJBwNuA?Na|H*}Y{``M6MftO{lEgss z33oReJ`H&JY{r*1qyTIo5=N*5{$+y8I5rhtFFH+liCh%1y-uUIujj)3rZ?zHj$lP^MGE>B1^PK#Ee-Oo zt7L(~{qy;K$k9~G>{$eU+@~jd;E&`6J0Xs_f8eiQyDEzO35RCc%ACavFZS6CRR`(L& z+joN||D^T-u**D96G;X^H5*f7TZZ|`jF>o$RYZ7r90jCq7S0cH+nmgq4pLmMU@cEU zgnLM1VHL(b27K_OxMVU%a&4qqNt9aI+2~MEztLX1K@V^>n8Pu@h{= z8hk=nT`@{UT{7fV_={a0EJE-K1&I6MA%2>gA!`b=V%`(btM_=G6U3xSJa|>I(*L=H zlHcp^{2-isgJp70!4@e&r#c(()#DgN=)4onvQK zfuzV=`nnyI%@&8BU)s4knSwQ002WYINQdf=j;K_2gUw2buaJr7nak_Emc5Of8 zf474Hf+Z_QV#S8wSVa%!_iH(`rN~XlnOL@vmsAh1$NHsJ5c@VEUKa-z%HTuvi|Q+; z(JReQ-6Q5rHX&x?UFE4j@96+Z94#d8Z#=8tinx5Q*Zn|lz5%(>)dEXR&yM45;CND0 zq(@8a;+YmaTow1IjOYB;k@=sG5dM9{Q(EU60{rhe%U>U-W*NrgKRj6|yioeQB&KZ& z%66qy*~H-u`mQ&l{~e_fLm97ymRg1ZpS&gm!oHXynw~7yqQzXH8~wKn#+&JDZJ$!i z$cTa*{xAwYJFp2k&6a()B6d+-N@NQ&;OsbjHpD2u3K1J_2X^+b>sXvHy_REA9~-k;JDQHx);tO7xEn7`D-4!v12O8Yj%Z>u}@8( z+Bcp3apQ0K%75s^{o(0;ifun!)Sq%urqjrsxJ&U)u3DYQA+zZ)ckecXSRSEc`1=M{ zZX^C9^sf*$+3-3z$TxmzRdf&qE!-h2X<-c#jA4B(jt-o$!SKVO2LtPaU{rMnLKGhh9eo0H zrIwMeE!rGlYb5=f5L9XGPlw>6PvBceu9I|r4w3)a$AFg*fLv5|7}#(GgsA(|abPo5 zSPK>W9XAT(v3#KQa}YinJdIqL2YjgAD7iuPHO?3YK)%yP5d8EZ6My#e|0%!E6Z;uc z%%1KuYMCcs#(cspYvpMn?0~;M+=`q9IsIvNgzVoU=K2F6`UZrEYieMTtQ^R?IYR;b zYrP=@3Ohj-%m)9W{&Ii;K^8TDndcx&aWK-E*F*yp8*Jw{ddMPoHrr01E6y@ zj|Xtoc;&oG>0E#?%ySI)8@n1hyjH1Q+yN0$0A1`Bk&$gKo$d)V8OCT3UEe**_L72 zZ9w6#1=QE^06w`IN2B!(f%T8HZAn@PUqf{ATX2?JWRPqSmyv{rN=D?jp`$TQ7KB$i z$aSvkByr#Q>->?g{uE}A~ED5fs^Dt~IUD}#>=?5A z&EN0~WRTDM=zk&6>(8c4NkH|jo<`#r8*|tIkmQz;;iA@c<1L_z#2wJ-ZlJ1f#wT@XFW7B#2A+cb4p6hNBC?)d7*u3M9drO!myATbE}Dv9V1bkR+0_;Um_ddT}K5 zfhKLd#Dvri(5zXif~`>G1J;SEEYqLmJ^f>T|MND(9grA&a-(aIe|o38Yy+9aC*kn- zmrwm6^2V28{~zNP-!h7^jmd%_s~?1S1q1J(J=yXeY{^i$2gj_5EhFT+041Cx6$Hmj zBq;u*uteaVujB{Dx#kDKF#L}H`(_XJ^ZNi!(pOE)@ayTRr^Fxqz<=!PKTkJ}wtZ%k zanzqglVu7}S$|p%{w<;gKZrr!&^M%D7{Gaou&iUYs6Zm)NCuWWcd zG7Y|Eeneoo4GA#6a+dXtkLLHi@7^Eg_pScyhxvU5DgHNgONhV0#J={Lop?ur0$!C+ zb`_9ts6e7p!Lr5ymRX^C2j1w$Ss)kr&Ec3yFxz3#LV$qW^w?cO`3J&fPwA6?@`FGB zQZ!I(53p|^Bd}5kjz1b$OhXKREwzBc-^qe6pF=^vo}R{0-}&nI;o3p5gB>CfCQGSl z*e7R7qHZUsu*W~QsDiE*8#`qR8UsROkMjktSoIaXiq;Y!!L%%AXH(v{ zdUhc$oISu>J9v~~?R23q%|7#f#>XPzuNHHcPT~?gm@?v>6s#SBGaO~Y<6na2iE^7m zU$Eix7o}1+f^nJd0k=Hw2vW;sbDh+@75ZoX!mpndt&V&HCf3eu>ueKmfxaNl3CY&* z72>f=)BSvJagxl|*>EQpd6qZh3kWeZwC2Hstd(wt1FREPjBa%klDbjqQYaze_J&2} zI^O1+XK!g$7G}P=s?8UaB+O4|#?0#D?p4p<&$(vg5no_2Ch7WQmh-~wm{XEgxcv#% zGX5%-eb??@EhzRrKXGiATh)|gTbh@Jp84+dnDOUtwT1C&joSGJ4-VcR_bqwWGOjZn zl3{h@OF-WbaE3=#aJ|4b%aE}4ZB6q>XMWgJ^> z*9Hnfjz$;&uiG)u>?tktRaf@M`LutXYfVaTS<_ zo1&j=LLOR(7SV>fwZD$a>c)^+EVx|GrRbTRdvrUG57?3OD?f8$TD zmq8LN_lm9`YclMelsHq^al0&!(Ta1`Po``W!Z3I}^(mW(+2J&ioaGO`!_xfK`zkGE zCa*wi=!YVvcy!9ObIXd_=X8eKoDV;7kzi!Nud^aoipbq}Y0-7Ed7*4Li5&a$7r_OdkQ#AVIU=67^s z<($|&wGAU;DQfQza^M~bLT7QcJsc=LlbBF65jpNFjHEH3cc2WH#|Z>pD8EyB7YNqu z#YmgDm&X=2w4%eIrI+ShJ?LtT`oSg0uj!o^ZtCnE?VuY_r8;1axwa>@zcP2<J7=tU>zECFK4{KZ1jHw6_=hKYM;WlFVBt0EO1_85 zng@B`t;L1;z^clubsIEV_(~gj1*L9sHN0TlZ5%Wd`(VFL6kmRx*skThacih4NfbC$ z`W4+Gt}#st=h{VFsjhWe-8g0vBZP0SUk~iORm4!>{|hp?cCt-MhJnu3!AU}ae!{)Q96hkxsS$*`lv%CEvrCJw=qrN|S@v30 z4K6bvke^zH@5Sll<_uUS94&roa}8?i5tBdKot>L%p{qEeQRMt|`Sp;;-F%JslF7CJ zX@3qezDmENSt?tFhp=$8wPx#_S8eRgg+d#eh5^m~GK(rG-3`QwfdvKMF&bjcRdD;9 zM(gu%KQz(0nIOg#lo9_(JmCVXgZ-u8hMQC}~X+94{`VQ8{hjhJFpu7Ev3dZN)@5FCe+$s9U zZsXq$mzg@tA`% zYMvqfb$;L}bjFO%y(Q`7VlKMXg+kWctb6Q(ul+IMjWo8$a2s68^fXLMx85Wbf#C5u zP;hP81-yl!;;S!Qo^QcRe|Zg=Ov$i2PgU0k)s1kSy2d?=v&&$9@d)-wx`o2*jLM5V zsY~k3RlO@Xp75^htHoVbqmhasxGs8=m=CF*>Zg|{FO5(cXMPA*T$WY2tim4F5qbJ( zcG&Sx&*;gM{$YL)eWOaj0|jm32c|WI@ee&jcCbGU3sa_vBhZ>h=p*hWyQanHc+bWw z+J(}x?h?WrDM?=nx-+nNqz?>EgP^)>=rDm9?b1nYe(81;+#L1)Y8C!yrG456{M=1$ z?e8KS-h_lbTZqNqNd)PT)&FoStqCl~d!UcNJA>XeNVW+?0IlFF0Iu>0a+!NxN&##p zYtGn&7~8=o$X=>@#xAN_!lyr2<0(9)MSpEj-{I%KBH^YH$EIAhOl3OOec_buPLAU_ z6Uy&Ha2d7vOfploup`$W3rOvLTOK|G+b8^5gs$xk^|tDhI;WZ`X~It6c^c+0jh(ss zA~fe7iLG6zdvijHVa3Mp*hA+$urA>Qcaf`ZV_IViUsFJjjqHhr=YtE-%B-FNd0Q{e z#<12Hn$~vqj@!}0CH@k7QH*|mmI7DRMVqP+t9JES(&3oI`LPagVg=&m$Rl}J4^Gai zN3cH9#PM~wQ<+uS(p7#C9d*#^&@tD28xVi`Xndxe0^4!TQ1ktdW5|hv_vHDNxvZb3 zViS9wtobn!X^P!)!SP^3?4t)hy=GkrT@;^)#cV>z3En(Yrw41@l=~lTrLrR;zlQ>! zK>7yj_c^(A_fM(ws@TViEn5ZzO&cFfpVZa2pbOujwQA%{WjuhES0}VB^p#Q`QrXse zeF=Ap&35ft=}k+0LT$4F<;%vAgKo_kcc=Ty+!!7t7UeCSun5GTo1H)K+wW*ai3wQgn*@g;eyiTdI_g3X1bA6W8$8n@@Km1q% za*2WTvR`fBhYvNlLh|cV0eBv@{<(u8i%f_bNXPea^5o5t{r3y^i-{Fm zIOjY$=w_1s^eO7F-OeV)8qF8weLRSc+4($a9oI$`%AV3pM{2q8XW5S0UDsk%t}1tu zIqo8NS>MKB>AKE*gAtLOT6gzcC^Xc@NC`g7GNP~fn(=ip=T4~R`RYe5_p9%|j_R-% z(YH{I+P_5m^p-Ym{q`TxM6i$a;?7ZW5 zG#wK6)FHKBqm=sW$!3GJL{h|YMt&L0j>~oIA3{H_(;;(3zOr813;NpA_@BZ-=Qwm5 za`Y_FOF)2`!?(7m1l=$g)_=;U61-ui-@k6EA4vCS3C}+o>2BE#ug}SVqn-||6J_ro zo(MJW~Uhv8`pxURX*1M89Y!lwSiOFO{VI`gc|W`zv_9OZ*RP z`b||zbNPw~n)nK{saqBhBFL^;>&&-vmP$QrmzIdCb}}p@{Z$bNzR~Lu59BRv#dILMFRlMplPnoG6XX1P2D0nO*szgV)R(6g+dkMaRT~ho@KATE8SKakiZ&l> zPvz;#Hj+1S6i|+B45o>Yo(kj_uA9$Y(lT;5hDl2&T%oLgBz=;{YY|~WUa%leca2i- zV&kcy&LXN=4eQ(vMMh_#^%ap;tVjD91A5;4eT(CQj{+|`y&}PJUrFL{yjHRad7Q*S zXLo!=>b&}KM(VTn5yo`Ia^-EOUivQ|8bt963TwSS=r5RaPI|k@!}u;R`oQdubbcr8@|eN<8C!Rw{fOSvD%YBFvQ;Gg-pu7cti`-y7lK-;;7v+q+@*UJg^B`1KZ- zmk`R^nr2M_dY~&xVT7|l1X7_v3B=3!MSMiv`v_!8Rr=4jqiEQ0h0*R z?Skgy4-21XQR%9dv=_K1OF#34+zqx6d|{SFZ7t!xNI zgg6$=(X(Df;o(TaZ6>f61&qAjZ%+MD+59VLf2ocA&#r9VpgD1V>qxva7*C927%C?Q zl?^Y*+!4N@vYC?t%q7+W1gLDZwkjLACvx%h49Inct?@fs$&~lflt-pP( z#RT^1)j8GN_YuW&?2-0%h$xdo(kpxGvtyN8y_4drSdEBW#CEXS&DpMBF1m2Tink@& zYOFnbjF8`E)_pC(A@NnZLG+uodjBEs9V~50i7`33IBCtjE@rHz3|*l+ouBMaVG$(I zHS21z_&V^MzivvbCa_WSDqBA{ym*`B^wuDO$}e37qQb4n#U%W%SVreoGC2Rq6Mra_ z!TlKSP8QQWd`0rCgpxI8N)s#`r8-tgXhP&o-ZNo)ZK{93GMl;P;t7k57m{=Znl)EFO z9gmFZSQ9&HBtdtnN!8WF!BQ-W{x!tZRP*fFlRV)dV$n~QQipfU(t}J3Nv1RX5^>eq zDOpvFqHjb5IW$Xa&Tzw*uO-4IbXV$l`k)D>$?nBun}Y8w`_kT2ZfClZ_({}y@?%ua zaWmH7XBsxW5WDp@GtPO%md0zR21*i+l@;VC2)w&`3L_t$?PBW888XUVBEYMjZIR>y z>$D#kQ=`0H`(8J;xv25UnbmU*ij+l{$%~pydF(0NP@SyYN8{r_(Cl=R?#OpPrtP;# zvMb;{2~n}O)~k_cB1?5RerpzZs~TYL69G;z1!TS7FCpGY)irJsWfy!cri+X5wtwzX zPK=0)Ck4;gz+Rjgx;$#Q>2f7X^ETfsBH5cmH0LRI5#&ms6Wt_;Gg6k>PVHwqle^~F zqGyu>Ot|(W6OqQ;jR6{H_FtN_30OTisd^TUv0N4=TAHot49s_s&cg2MMSm1S#l5hWA=WVUu8R4rJJX4R za%M}KV^o}1W=F3Ii$z+@ojmOlD3|6{lN=}79M9!?VqT-p(cyCc`GyP7;!Yt0{ee4O zZ`i_$88iuc@In4%Q4hGYXmpCOetz*;u6-{=nH8jn=8dFVeE2QZV=wVPjh0um>1;H! z>1R@7lI}5Y=#jIM%??TEvZt3hHw@KW;!4YNIf9fIFM=svor=(1;zt!!0|Gs7aN&hPYVO}A!8!>B5?mWZR zYjh$(>u6Al%8CplbNTzwN?l;&(?Wx6^@nx`I4ePKf9Nv&b9EWq(CEEGb-uzZ#OC_Z zn~-X0U7dWNhJmXm!mbpB)0zrC9}IS34xb^A{`r9uW+ik6-Xnl8Gtk7-n20P6Y?+xB zayG~=41yJv381PXXR3FlLzYgykfSrjc3@LN%pZiPqMaH}G^qkXAF4v8zF| zu!K+20k*I+ozpV=I=oWV!eKXq*PW6P9}S+s?544Wxp@D%EQ@1;vV(9h+wi?e9c~q# z_gjJI?#1|@usIUzSI9+)>#>-P{affb_nPb0;|b)}<9j?)3`^+PJz_~fHI%O2{FAom z7oxgzORf6az8moA1^5J&u^@l8pU2Q-I$C~V$Smx4x3}j!nYgL z=7kSqjGi1j`uuIbSx}J**OWL};c`^;scxo|P0Yg7nhiGvAyPrrI;0HuRt8EQS_c*9 z+q)boJQ7VQ5=nAnPPMiw;2r(=V%GTt>s$J_W=w@=^P8M->a>$%Q#vZgpuS4btecL! zK@LK-C7RKGh`1Iy6doblNoI9PG?zKaiLTx-)51Zp+U~$D3U@`PB+1nx+R(G__51+z z+w=>pu>&;PgYIua`spxDZUNSt5EcP=DKy&M&hSLD_t(&MVV25(JzbzO>9SULDG!0Z4k%acY2Ad(;|6f% z<+1UeT$B4k&DQz69%g27l$F{F7~f0xw4JRn1gXYaV>Jxork&vwrQMN*p`yQ3QX1*h zrINIhuJ<|l?vSS}`pd9;P?~%g#B4p0A*<$LizBzhLEUSRqld=htovAa;5g=u=_%pU$=Jasq^Ko_HIfRADXTZ^m( zgQYqH`YUUZf&RlU$}0I(IMlh(B#70V`gkKrw&oXME|>}_zr@=+FG_?=495gWXIT4U zF-%k<22b?Uk+B+M1+{@u4i2`CbWcvU3sf@uQ#tHgCnThZg*dWA-n|nV2$%SfDZ&vt z$3^OTto4JKW(VrR{`<4pt@0WAJS+VPkv)+ghuu(G3=cL671hd8FTmK3bffBn_rPv9 zUDoL@+xN(qaooh>M!>$IJ97Ky<~ez(8({f_2#ynwu^@Jz23a5S zF3TbZkjDQh=kxuM^AUebKlQgeCjLn|Un#(4Sd+JMJ|U3lla(5*fX0FDFgy!E=tP*I zp*TO67Q9mn5E1Aug>4U^R;I){HfuAG(2|9n{-FM}h18U~&4U zx&H1GX*@!Ru&Q;3QOjeo9?xu}1MU1q%m>-++1KUS!7w2AlQKKt>W3x2No zx<*h%bQfgw?m9)bULiZ?ow)%?@`9=lI@wPNM*v@S2&Rc@GzN)BNTVVs?ngntr^a2? z@I=|$GQP*(7BcHnLhIp{Drx-&WtW-b@lbj1A5(v40h_lr^9+1%)`aj6jLiJkvASi z+^W3Je6D(x90EW6LB=pTu^Qj8EaD0ZjUX+;8sTI*RhrS>t&@{4aV#v2Mo7JEcg}dD zM`yiZ&1m1*eeySM7BMZD+bD}OnP=cv*1KMqEwk>n?d!Fq)jW6oV@hz(i=LA>az59Z zNty1#)8bI|1+k?xs|r}2&FXb>Q{wVsL@)fua?0tueMVs4OQ4Zr1F7^0Dd?Lkmv+m0!?) z$PX&#Z*G_01c4O_^k=(g|A(oZSFi~L1wKSL1j|FGz_dfEP%IA$# zUS4%%?5L9TILQ&Tk&^ms`;DS^ZZ%YO)I6uF2vP|+c=R=wNY<`^wwsE}Ivb9knD0$d{h zq$4r?ZNWn%xp3}1)MWDtBa>QM%-S%!koC3~z0`MK$jDApV+GGVom z0??)PyPFVZIt-o7FoE4#6;xe3!T*AWe*y76|9=VYk6b*a2S*JM>N|(QxyTTijirZ= z8f_uSNPI{}_1ISR)Lyv>+4&&@eE`LR#yZ!r#j0%+Uk?(uaaGRiiQm?~?BTq|q+rwz9qHJFl*5D5*%;SuXqF_{i@2WoIUIy*_A=LWr-KI2 z!71}j?J&Fr{_J(-?xX21&Rn4GCo&}DMLloa)^Xf$QH`rp8;+t`pMZz!Ka;#oJFwV- zN1svNp;;2Ry>3K@D~L5teDwCyb7oc*IoBYiJ|tMw8VW_Xcr0dYeuO;Fk9-n<;><$N z@{c`Jpw- z@IQqIW)L1g6fEU=A5m$W)evI6_>u@!yh=MZKU&zykZ}LmfnLoGG4qgO0>;KwqT}!b z>!3oVd)$Dv7Fp-x*c0DKw#-_j< zL0BJxnolKsg(3{hh8H6)eojAm!69IvKOKt&$DFZYgkpnD2-i`P{?tD~n)uuiZ9QK4x^5XTys!0hzpN^_ z$O7{JwFJ=H5Qj&i!dh09y2@Z69C%E!V47hc^2he#-(nEDG&uH~9oQ$nAQxf*BPZqX zXaT*nhEOV*5rWLsO^6%t7fnzHQmPuoTvBweHhY?(F&kE;* zWU#2|_i9`kQEVgd2`d03fWbG|bp5u9f_xk}1tG5LAS590!6?78zbX{0p(B%oyK5Al zd#L4JrOp5R=#I}z-_pU6J=yBlm@Jh3FVE^k;K4;LHE+_1 z!n?)JvyAJ93gv4lzi%aKO%}p-Hk>?(xVY^|wm=3$8fO}(@cM(MMlTZVb2#^-d!#pW zj2=ttk!ZW5;E0gX`r)cK>aLL`=D!GaK6k*x?5Y*r`72LjPe1a!6`2UDRi5S^Sw>_} z*SK*kgf2R>mUZ>^C?*Q_AO@hY(<|vi)y;xtO3%uKyM{n~;QFPb{W+uHn;xGBTXCni zg2MTHx)z+vS;7dMdlw6zF}PItx{tNOzkQwzIe zi7HP{OqF{l#OEB%zoF4zt+7j}J}bTI1&EyWg#04?s>>G`Hz8A~Y3->u;Qt8xRF)2F z>EM&&WNHmJtC|f^BBEq=An8|=csiV1#RwsRTs+rqVmWbOHARa!O2A5GrXzia6}jy> zc9#Ctk!d5Cw;NTaQ>d(^$x)O=cOD& zkKMT9sU+wlZ`5CPyyTw)VRW_Oxm8;H;#}1ANxv6c2=2BY(c(YeE&u3fq2vdUQT2bv z1SE(FPS~KIVgk>*QeEIZZh;E;;u;_@8|VgkG_fqwpjq$^ch{sLt~LJz5Q)*cnuZMN zcjMWkF4@9@XQlbp#e*YkJxNZY3CaEM`VOi_nX$dp={DhMwwNkL@OdSo1bQ1S_qMR~ zh6L|}JN^V!ybQM5}=oO5$c(>J^@&_wrUkNlCvTg8Kz^l%L7eko^yQ+!8NS~pL@ z@fQf{Sb5`v!}IqT%Dp}qnLX*?^?@G657XEVOz|k|_!qYNkOhflHEuzCLwnNsxhfh{~sebq}NV*%Le)(=@K2M_ZcJU&GvqL*Vz53LTrCVNJW z{2Y~H@Y17A$d!@FbG3pH$7i$N1P3$i=sW3;Oe!E(A}8Gh2G5#QXD4^l9BGG&R3}i# z#N^K!AHcdMIGO2a?kX;2GJR-Djt}Y1iN4<6e#`LUsI!k(Ii0ynb7QossHo}FN$G@t={mfAyH;DLTaq zRin@Up0xZt8v2BiIqJD0O`*urTzBdNU%dzXp0SO^$^m-TlI({m7N-g3KK6x?kf`H=OQ~>tn0E)m>7W5MT#`s!Sbx0@Y zEqnHuwAilfPNzW*G6GreZfr;Wbp_I<<={;S_X{$$ZI)QqN{K?>W1?m&DUB^HLOWRr zBC|64u$L0AA3k=hqP&c8&UTS&g=OS|TAtZ_k>=fyF~UBqMW)le#HGWzFQnF9Q0u{8 zNTTwvcCU(|-j|3QdWEsTw~w`8J?L5?kctDuenV--TFYYov?oDFN%c=N6_hk48u|`g zY%ABe363o;d4R$sfYwAqlLRFD5=IQeoL>( zaSG_I@!nfFo~ly$A!(DU{fbPd-;U6)T_rg>Ag_Ni@oEk=AN?a3dX1&4kcYaF0A*Y6 z8nbVI*DTKoVl8qWemYAh1;kT_kJ%k}h7RkFzT4b12kkWBEI7*b^KG>)oJ^u4BA2@w z44!lybc(rPxR5TWl4_wDTq&^Uu70tiPwI5x?Du{|Lg#Z3x5bC97eWZkkcdG=UFf;6k8Of9_sKcwoNB@;-qe7jqckSFP{~ep%+( z?_3>z|B32f>Q{35;TG>m7C-F|0+f;?ZNt0$V^@4!7MipaqN<9YuAfb^Nn|zyF z4-KKih}e|#VlB*u4;DF4T6hB+DgexdT08{Y5A&OTHq|rxzhz@ z3;XZeN4={q;OBL%M)Wb1ow5%O zjVaR~0`f@;6s4i71wv_*Sv0Za`gg`36d?z!$BmM67N)Bu!=n3u~zJl8eN&P`VxR zriw5=WXl_^kY_r^G7qX6FP9@hag)VVsfF!X=jiHq$=?iU_2*^yC3Bx^lQ^gtX%J#y zAZJP}XGx*3XD2u7AjKm;o`=5+QGN+2_}X*aWf-sIB=Z$i0lJ0KpEl$scMUDku#6a1 z(=19eaxl+A3=a1t}VOc1PS{}c0)@A^Wlz^eJ5)MuZA64qDoI- zvgnlJTWC>kC04hIUs<**ms?C*q02wL;PYhnUDL--2179j#KV}ev*97*Z-U+4NPD>| z`IH_`bTWq&h*mTOMLLf4pl1y#E?%^ce`&7!aP?jPaZM1Mb9le>XaAsOe%`kKN4Mw& zN&&dWx;621@&B;*=7CV|?f>{7O=T;QY%>Zek|adTDA|%|(a2gULX0(I9ZT7laBxDF zkjj>jecz)9k$sGPpRtT##`Jr4p6~PdoTqc1b9&a#d7fYYq`B{VX70J~`+8s3^}1fy z>!O4MKyDnHuTAwSR|ZrRK*QSr!J9HCRDK7Sw9gC?`REMz9{fHVofle9ka* z_dfuX`?crzKYiPGJNz%RLla)^z}P7WeObUUs&c~w4d$S7_<@k=KU%IW^Qa#Mg=_SG-u+G;3?Qle$#-TYRLdyAX zW(PbtaWVZ)0N+C=VFOqnH>@6|^4G)@Dn!MFfni-S(_$#`1?g3#a{D1W50c zalu-it+n@KXopI;cwRqNm{H!|eC+%(gvO@>RsP*57lUqu?gac9e>AwhH=E<+1P%G zIuFxJT{g!3iyR?MqNZYxcf%^{JD?^i?rWo`;lak@gf<>h{Me~Tx$Cucw)L`a8{AaY zF6mdX0g2F;5&nW5g%L*Dj~}hAbau82rtJIfvCls)>cD&r2?3 zgzbf(->hucSPq~Rw~y4!8BIXG4s*Y2M*bIRM((dX4w6z4uui^m`Y9>J_GQwPT-UeU z8vvUqZY_L2SRo^Ffwqy%pO#3qpO#2*XIiD{pO(nbKk^a@m&1f#ef^=zlCxWy_d!v? zgT~v@n}=np*}8)|$VEhy3X#iiqz=a8;>5j_7NQ|!cU*T@jZco!(fXIG`x~uJ*T?CG z6;X7KJ$Ye}-H@2Yo_T7(&8uyV*K=`ElCY$+>tFKk?WrM8QAQas(9h)D$6isON~%O|?~i zn-{~M8FEXDgjx%fIKoArhIZZD?I|X|?93G<&&Q%o}wBHyQEwmSokYi{q`qcLs;!Q0?)vq*- zhWFbHT z&KTou_xEJz%DQ*#LR?|3cme8SiNL|ogPq}@tEsXu*qPW{ z*YAb0!X(}`3=3-8o!jv!dqao*9YOmtV!yQ+n#0l>?^GnUa^w@7x4fcnXrWBerQtr7 zk$T4k{~5AfvODd>$fvd&k)|PUiy8GOINau#;@sm-1U;F{pi^A>G{1pCWdE#iqzu>Y zRl$tSoYa(c&S@Bw?3PBRXKHrLjGu{Ih#pzE80wSk9s07efDZrC0FQ0RHdiA`1mrl* zX|x}wdFI5O$ktve6JbJu$;Cssve3=L+CJa~OFQEFwgewmx%T9& zDt;<=2i3yiP=@vCnf@%_$v9yazBFg2we@XolO^zn2tlIELn7r$zlP`6h{n*>Ga!^#;VtzHn? zYg*aZtorX4rjcVW#q2j|yPI~$F@aMMbj&8K=Oi_bu&qkKuBXYz$s1O)MLF%HS6>cJOO&f>Va(Q8rS^_^NrT$Yz5Z(&`x-szERKqGG|ShR9LH+m}j3o0ux z7L7D)6_OU_3L%=%7ll$av*viV`@bACsB;pqYVC~(EDF6|d?k^|X8QO=dGEN2F>R(< za6b!;;QF2J$_|b;LTITrMuro(M|C3huN2wR9h35lsD9-pG%9kFqtKib4Va>byg^O< z1fX3X^inja9~SnnOJuDeZVNCHC}MvK$D38FAg>9a0x`1HnL!2sV#~t42 zjj1Gi$(hc|AUK8)OP($lm8Vn-a zaLg-)>y52<(Vmi+Ydv^Lp}i_-z5v$~@qXJpRV!l86Sy9%YCb6LDqz|n5;7sY>?@z7 z(&nZ0135+l}uJq!;L-bu|4Zt=ga*=6%r14ux+CWR=6zHc`yNIw7VewiI7#1 zjQ%#+?FR8aZA(WVUft0oD8Tq$$~z%}XrfBar=1&epw$2d%JiFDsV&gSH?9JRYx6=Q zo1*d%bcnP@B@(A8{i%EQCuAXrl7*m*G_df2LqxS0Urvx`YqWksVn?n61YV*(Vw(qrD_OpKAdDCDy*RBh_u%>mQ zhxVavV6gsmV_nKMZZOUr%dBuc_imzMIoTO+Ra6lvUfby`{3E9kxn(?4q)n>15fMw1 zbwAThm5D*XJ8ol~#MV||cN>jNg`QtsHItiq-o%fYv>rEdCS|(J^SaCAoO>-I;BueB zizFapo|4D2)7*6l!)3N%Buzf?gxh;L8Xl|Rs>g=CKx>(ogOQ$!a}GK63a!^^u#syp zZ|{J3poG zGfZyx_=$)|DlR4qY&R80>WYsYMBOf^3cIev$0#}%MScYjg*Xhu$r{heU3IpYC}ZNa zw9>*&k7pxRcMh_=jZ|!8<+7n`KvVY+%`#u8WRk56pDao^2GXa!ANcXv`Dl?BHO3iv zq#Se&oRP{2*4G^tn{O@Sn&M9?845Wx1WfSB|E7+>_qu-IDou9uG zu19EQcL%hF_(l|RB_3vIEq8dKjc*Ut26ZHpc(o4P01u-_P@5@BR8(l&MLNY-h{|Bx z$isa!CgP6nET@ID_<@$Y@3LzcHBQytzF`-Bam}g;%XAVBi*T@Qw|Jpiwr`P{U1^$M zn;DApm}{j9KF6evCN=O%Y@6lrC;2_T|27~#STu87?3v4jJ#Sa%=I=73Y$Ypby|j?o zJQii@CLUTyQMa)|((R7`HIyllv7>a)4sE&;6Man%7KtuFh-k>*}6a%^3t5SjmZ|7_{L;;tEIse8*w zUTxd&H$UMGq1GiJ}Vd2*#jK5EEN>vX7! zmxW;-Lu+`LRe29uHJvb~Q?{+TC*^pxc{Q445sY^}G$smDsH1gKS8ikGaP6_lh;-yQ;*gJSYwfaFOp(vIYdg0Ji zHI)V9@3)%mHWrru!m_k6&&l+uLCZt~+GgVZMjIb-$R`PCoO2l?CIqR1h$;*U>7#-wqXs(chXE=IkAlrA8XL zZ8cY|+M}EeMH{Z6h{irPeD$+Ur26aR4!Rac~MA_L(~E`d~cwY%W7W?7%m z4+>F7+`R?qYQ~aRb4?2iit^rs^>^;bUKr!OeE5;*^Hqo~&_vP>;E4@__@FIpvu3qC%YK z5z^UT$kph~;k2_v`R$^5dy1W4t8^>b>BD>j{ zj@{3(^)$P;w;jSjPLv=ZiU={a5_>VIdepMP8PV)_6*bLtVRQoSS!uW%At51d8Wj1r zaW)lr5B8;XXL7*Cz2hUL<|w$gQ_lKWSPJ?MA@ho#`6c0j24+!gV0nY-hG-u3FoOm{ zpkKB&m#ozWZKX*3coRy~U+l|A#FN`Iq{k6#gxo-cWZ`fHUVFFknP-8_ml}3f+shr` zT6`I@R?|Y&vZ{G%+(J20en5ETItwlwzT9SrroIXN*y-86S@C45F0JdS6Gb;RY@;?T+V{AbvlKIxlZ|0wH#dT#>!r)D0ozw zdOCkeN%P45J0i1oO1#5IItnFx-wH6pJ_T+a^h=-+c<8HH~LLrK3`q z9J8cCzKPAYV7}%Yc7VCKJx1sbZ)LHKBGE9HDi!VvT&i#mc7%*2e{{u?#JverSN^tJ zll2DKnL^}4=XP-=CJ+ZIAQ+^#=YESs1wyy$L{o_L5vM>V?s|vq$7QscSq&7-<+$Iu zJqymz03bMXzr~WB_-f4kQRcSeozeWRiP}Bo%8gaf90g8m2Yl}KIh4%PcNcOQnsq-w zjl$%^Nd`p@#_@!VhS)?&f^cQ}3r5OqdxgE~sdpdcAnrMGZ!_mJN1lfevD+*xh`xp& z4-vJ4&<856v7qlTr`Tr*&9g)lV)ipsYUDZ+Bz(esPEyJaWE##&YRNw4E{2~pg;>uz zbSY+r5_7AwC5+1AuMxKTItxWikJYrXZaFj^)>u5D_%gQ!&TZ8nlah%sF?q81tP6ES zA+)5|>fUWL+A%vHm$}|ojACcKB9NAz+J+H2oq)@}E74a-e&SLsz?^S}xZrN4e_dQ4*% zv6x#&`z^O#q@ta1HesEn(e4MrAzn0S4Qy~h`e4T=6*OrRO|%C;&^h3Xg4TvioAx2u z+@&2!kr#A~Eyjsnr6>1?MJH`mOr1{nWk0USOOo|hs2R4^ z?@zs#D%~-%Jp%Z#Xj6cqUqct(Y=-$*k0k=gb@AUt~hEiv&J+X z^}fi_Px?l?*ojB;U-q5zGAM+knH z^X+FT<^zcLw-MOriN(Q*%EgP42~IgdL6EmZE`P@nj_Ifw`7D@q)~ENZ+B*~1L=$z&1y z^Xx{Tv%$o(b+VB6oMe`l^=jYb4K|zTnH7xYg$lxL7+3eB;p-|^{3R00`S)}fZado0 zWrH_2e0eiJtHwb>dw6cvy)^qq`Yag=HH!i6zH5=X6W3a`!(e!;%mk>p=kgnCEYdYs z3o;ySs>*ii)`5m=6Wr=!4$S;zcvo&tXu8}>z2L6=6!UUPwc_rzeR|Qa zY8=9Jy_ZI$lTxVr<7w;$9@vIq!Qx(82PM6djE7RUkLXb#+yQ z7|cbWOh7^PdG-WVI>%O-?Ux;DZr9fLuO0?*?*pmYYpz@ah(40TffZ`5#QiDpb$h#z zv24eyI#33}+-L?>K(@4@iBvT38B%bbzshX6ydu4QJsyc2NY^AvjP_EGYVtCn7fj~C zsBHC48wxVM%9dgOVDXI6I}SR-GaUN1MG2|35%(Sy^PcziO&~S6F!ok!L#?`=XsY%c zJCOglXj@^#p#V3*`|!(FpK`Lt$_wK94L2NAKU8-4`W&~#@+@m$&Cx6yS=LUWw}wS1 z9|CoVGf%48Ptr|o4tpo-Q?CIc)v}~j|$!%+aVAS#``CKjYv&5H7rR6DDqD8&;slpAV9PgOF4NC z(N40anYA!&EsSx$m*@FnBD-7*!%5FVJu)^DQT^HDzHI|#HWOkA&W%=&$!HvsjOyon z|9C5%^cjK}A9P7ni!Kt)46#8kuC=?)KTl@P-Ggz~B3=wcM7^w;?0*-68;$lq@B;}D zjuKGOb%ZPRaks@Yrbl)!Eu72{S*PQ>2hE``hK}ri=cdI$*N9;leG`FW((r6vHZmE6 zh?MnxZt)_cC8nxo6BX*H+9j>`DcPE+$z4}ZZk#}dE7xRi0s|5E+<{5D z5b|jzX^(`pBVKzEe|<0a+;Y~BwClNA9SVY0X_!M2%tmkfqhE!oPrCGOzcJs|$Nd>{ zcm0cT6JtTnH{JhW9mYzGUW{UX7iFVttH>nA@@XN0+TReJ103$ZVEWS8*ncm8=4L zh*!oNIqN4G^r&92f14)CE}5utR0(GfS)DPCSAp1iBFN58_^q=|SGvlT9a2sDR6jHc ziC#P-H_NR)GiO+j+*ZhMa<#C87#qB}7Vl=dY;e;{@zh|OEvsC0%UZ7KQ?J9b`j#fs z&T;z3y^o0}4qZnbU^rsv-E?6!2R&Nl^%*j1au+3UPKq6%Qlji8U0~M+U*<@bRF(BA z&b@x>bqdbrev(#0j>sfeQ4HE?9qi$+x8l*U6(1{UV+qzxXL<@f1u(L-y}8W%ybGTp zH``hyo{Vo)IV+gjuC3)f3xc!~hC>0guw)t_v)z^(nkt8J2^F=5pLGq|jO^cowqk~i zs|I;(Z5Q(+goB8W34_>jV!E3nfsXods@!%`YBm`{w_bx-8~NM4nVzu}VT39?%wE+| z*-O$cd>MYabw$S730)SER2Eh-cRvU+rNA_*O+H&LVwK`iw2-KL+GQ9OTjsEjQgW)% zuBTKU=)i0r8_h@}n!)+8R@Td2@LLoa2TCe@WN0a=%xH~62O5WI#cgE-NRQ(Ad9M|B ztR*bE;I{eSGriAne zijRi%Sw{-#ac;YHJoT=j>HgIQ4Cw^{KEK%=m+?SB!yzSq^&tcQ<5|v+-c0ijYq@ux zdMQw%vMoz@=ZZDYh^9jp~ZXsYLDEpVUmK6lIE@e8uzGK#8@4Fca(Pa?Z+(X?^rOXgoM$;C=Txe=ywuGkMee z)N1a<59MOdrePc=Y!+{c{YSN6c>8%h{x)rUjrPwF{i4IkY+U}h_TZDnp)ujaeVZfN zf_EQe=G4qHj4e0Z2i03Xiz>Exb4{1=vdWU0;?juwF-=OTWK^VURJ-wy^V7e1SM*JG zQ#rvJy5twDM(d!k4m?PDx9mPrx}#B6V!gsD>#3KNyJ8v$o5&7YoeG~J$oXp@1~$rM zmim*;xlg^9zuhbH6qr{N`SA;K6UE>?#rBa$)cnAGw5aD7>ssZvsstuj#ZQ*19hSB- zLh2W5dYq%THa-$9GLWbQJso?hxrQqCe$C`TB@5Xl#bmDkS&HKPhqL(iypb=nB`@!0 zvBdH=Y!D(nUi2#!ugk(Nt8KTs#x!AMRaIR{2i9)=<@rd zj_2z5P}`T#8=?*0^MhZDfB%zb8}9d%0{H>gm;9gv)I+t5uT!J}&{mRz?d``-bIy7K z1|8UqBr*SKx4}=li?Tue6px+epj+tv8wtE$vmU;F?eg36(ZN}SRxPVLjZmCFLplTI zk_jTzsCE|YCsELXry>sdqPqJR%nHs4V`1{h&Gfm>yK(zMopMdiH3;onC{$)mvCT5z zn__^i&0fwdIF|ROSM+RA%DmMi+IRSHmnWQ@q1UN!Y0vA@Vy6c~>LtsMneOP3F00GU zRJZeav8fYs8?g&Zocd(vON55SXg7O3k;jK{$J0xgU#bl$hPtLJ`E;(`C_Tei8=lDR z$HyG@(xYZpES|U??zjY*4tB$YD%Q7M?(bt|&f0!a`wri2O1M5`b~<-YDLJ3Wpi&ro z9Y79zJVGHQcg(KuhA**SR4h6?6|}a%c0HfS8fczrv1r`0C@FM1WX~*Hu^ebRme0F@~16bp2}k`yQ8fYTAg`DBQ&GCC~(wb;zk>PoOnb1980%t+U4| z*uId9uhkwL<~E3Qc#qTW^ioU=N_lGe%Bqwzv)5 zZV;895_&{8maQ_14%*BlS4!+6cbzp7yj%=1XyHE1A7*=812+coalv~j8V8L5?Ckwm zbtR{faet@S3!aGs%M-V@@0VbO z35j(jko8?h9_5!y76cc*^{MN?Cn@o9g&-s_rZsUPrPecL#Z`e<3-%GFY(^tuZb8q= z??oYCm`#ZQdF!=RdAB|_YrFB6TU?rQ4iLjsq;*R3NqmT=P<)RPvJ~YTrm9_>%sO^~ zJae>kA}l(Kd|Ib-qWtlJ(8RMYX1iFqxx%bISwnOIav!AuBAo@T|UBoKrjW` z4S)NboZq1D?CXw{$$OFSBKhbA4|92rt0-lSN_Ud3NCo3Yy-t46kD>})BJ&(bW?YfY zm-I*)e(zAkx7Zi9CDvUI$Ctr*Fg7e^)hC1=q(vZP&yBrI@PrQTzRTbmM6}i%E1MoZ z)_c!N~odrh=b>rW8;_*pfQ= zVJcZ!y$e@fTG4NA!51zhNm|uu^T<FY#Vb3ZrmJHe@vggQn1GF zbQi-ds2TWdOVm7fO>giK?@_KuMB&7>5x%vB`?Wm-b)NfVH(SLNJ71PHKnBl-9lxjq z74nh_bA!00bH9Y!>UT>8}c_;)Y;&NV{N!XD5q&<=~aG8ZvT6Y9uSKsp`R^=vd)G!ioueK~n^3|3h< zYpZWyppc73b7)B|MW-sI4}8q)ro<7F`Y&2R!_O4xqCUztS08TO5w@MpecucadS3cL zmxWy{;3SJYZ`)S7_^s>tTZClUxwxvoPD=a+E)t(exaW~++O|kBxtlSRE02|h)3(ub zB+$PbP#E*^$PCAZugkloS{ZNe&Ju7~ZthHonCFp(uH=|D%KK9fU<)pjeGT*7LJ?;p z>uVgAUCt`DChk;)hNJC_x1L${%LWyAJKS`zey-Pc+|Cca_&U-5Q%J}5A8dpJg}CW{ zINn|%|Fk%f_w;oek%(J99LE-$=s1`oJ-l-H;amy^c&oB!1c?VKq$l?%`jgWURaFf3 zULLP6N=Cdt%+NQT108Lue@?X{weq&^>ne=jON+3q$A#b`5>?r@!5##q5clG-oai%BjU2q#Hjt`(;FxgzC$Cjh+ct8a(>XJ! ztpKnb`V)3hMoo3v{NZqn#TvIj*zhcOKX;0%5bHxw02c_a=Yh;m=OW0lL>}z6_OwqH zZ}yDLx`sTw483){1md`;=Y{YqpgN2VNI=N0VhiVy@+52|ubu(yw%6@t*Tah*O@MN0 zq=E_UZAc2>5zo?pUxLHnJ_y}u6+6)k5${`?M@>ZoZF^FYU7(=Q zjc^^0nq~yv>*V3w9r;_yi&GSjT&iI0*cJ9HnSDKN?}+<9F;$OG5n<7wf7XyF zQ1W zo}Ms1XI;o?lsBRTv76rScyr!m^!3D^mbQCF`izpWc~24a8OlNvd!B|vC&*mF!%fdV)tG7Ucd64u-Ke_kC;iZ$Gg-f z(~<;l_(lski5oujZ-jHrkRtl5OxOxF)`X0nsYj~Ur#TP28Y2jKh;^28WzETxd`Sxq zYSxElp52Ja*>Q7!Ld?NN666 zhr$)y!WGpUOoK}9PuqcJH^eRK`XM9c6 z@A4?X;Djx|2y5E2PR0N}Q#*zj#P~Q3jBQ}64OMG8KFc$-^q|0suJ#2Flxl_PWM4gZ-_<7#=fhN4@4tdFPkxeT~yO41X^d^9ykzG@5m8L}XqoeLvR)w?>8(vxRs^dF@) zEySsXRYmL^r@Imu1AKk#dW$SqlKlEVLqZU+8Lk*OA;$^G`V?qSp_k2q(Qmg)PRO|G zKsyVr%0wwPBR$hdcU~z{hNd6Z8j4gC}An zQdUWFoSnUa8br571gv{;0c6;WMurAy?_U`zb41u^H9@2A*hqG9&^1`@^Ashs;H_M; z(Vl&&MkNPa_PpPIBB!brx5ozwWNw!FMJ`)yN!4C9SfucGYTBkHY!@~iNV45ebAkqF z2Up6>@i-;GV8tJRyvX1>mjJO0@e%NgUdOs(?6Y1aeMZVvRBE&sl4 z`Dw4h<>h4E(_{xt!-9KCw%G9Iq2(ga>YkE=t2dv_lUm0c9Cq7HGUDjZzH*>-D9{Q) zU`Ujyz`MB(WLwp}o{mU;UKnOQFr+RgVhbDc*M}@8F?`Y=rgeG>x&3(g+8ehQqk~d; z^+73>#iqtU4(B@y5Liq~EWY`Cyj!<;?k+PDSdo4a3I9Mg?|2s3Ftu*tb=M9C8C+p|or z!?OjHdtm&{f4o`1LKXHuvzYhLgv@z|YE8IUtY8Hz86A?Rja$lO^S6IWcG>mHD1`U! zDdq>PuDIS;?(&2VLXx-RJMk#%=J0nVYbz#`H#{E?tkQ$8R<9)9B@f}P+F2~F#f9ZW z4IS|#R_|#PW_qNVAz*D>u@hgUfKCBf$lAl~XPKkoGQw|uXyNT2JeM7LTvgv}jLLYA zJXeDdCNzcnhr4s#Lo@q3eT<`T(*t3cz(jPhf;}5j({xNW=vXR6F{#e_>d01S?j*;R zCCV$Tg2!EkWb&g}Nh(V>J5k0+JcRyE^C{bw;@h?^@6Hf-K-Mzq6y0|GskZacTm>)t ziQ9L$5!G}+&nz1m@Mbe6kKlhP@aUna%f*F(aXmX)i#?jJgE`J&SW$G%JH)ys`DS{* z<-VwX!_*7TD)A42WPuM=j}C}IFmy8PE|Sl#KG@j-)B6l@$!or} zxe6Q10OiqH+09SHrm!bD(t&!K=(}UV*u@zBfcdJrk;Fu|EH)l&?O-WUvE8>&OV|3? zpnhY|6&Hiu(IGFGI&ACswH4zU&TW|&(;6m|F_Eqx{Ie`iY~}|=9|UtaZ;To7Oh8@`qD01 zP33Z6PS;}@3)Rcdj1FDaaBLCY6Jt}EB5WMO$|m?3!hb4o>l2TWE3D9?TU@oy_hk<0##jX>JQQ+l-3?)oYZ4j@ zKM$EN_rRk7$Ynn8#J*D|c%g@|5yZAx{`TeAp!vb#Tz{UV1~Im6*PBMM%obd(S5AiW z*LNOv;dNm)SZ{3>|b(4G6v_+<^ZH119bK@|VXoosL0G%O*L;jrl8yBQ-6@ z9Nnr|2CZC=JZk0W@eX>S_|fM{@>ui@xhgrh**M2B2~qYQ|PZccV?FIvg)8yY@EL|mL?U|>je(}5PG zU257@8}YP0^OeAUoeIhD^8^Y1EKfKU-QzBN;E3t-DAEmATjmJtSbFT7i4rQ%-I)-U z?c$M&iLL8#j^%m4GOj^7{CM1upIFIZ?ptg=sKbZ^te}PlG8p!gBfE66V#mEA#jDZ6 zrcEDN+EG8V(bq~f2hNN;&KXA#OlmKa4XPvJ z2b_H`RXrNi<_ps8a8wakKEB=7b)Ph93|g1j&eeZ?wp5U@2RyRH=_HpI2{>3AZ$#G% zQZlQvanhJ}K!Mm0Y`v!c_?xnT#!0zH$Dze-v>{;`HxaDR>5l%m@%2yvuuwzdb{%EV3eON-Vdp%Z%|z5CWp=L)bLH1adW9!I#7uwrpTsA5x|6wE5dalX1fbPA!Az^6A+x?FD7yv^gV)>%{ign?l z7hK1&PK{^Q+n+yQbl13p9|^16*>jI~{Yqa$$<7uq)~$rC7&$gvv3|Yt=-L8o>3sJ= z_WmJN(+H1nyXiAE&^*o+P$JZJ5i-3yO3*;*A%l{~i=9`!S+Bu0m*iR>bK8%XAi~oEd zXi$EW51L9c0pD*I$T)bh%@6TN3L}8Wr=R6YZpCBAul6A4$k=ImFU6d%+q3_{*Z+#i z@$~=TIsGBu56U_GfWE9il-Tmgsam~)Is;^U!a##bA|{qrgEHAX@|+=wEDX4t{%1c! zio&-JM^@06e}>>>s)} zOgPReQ(9aj5ccvk3M7zqr^rZ}kN(uwXNW(jE}~vtQi34?lX?sX1PyjtC9TuoG-w5{NJCD@3!#!Ysku>OhWvbJaTIa z86j4oKC@2JvO5}j&?Uv|$I**zrt4PSunfW?pkO7q!-2GlxC0F=Sb!q&oXAFf!Uo1p z%yWv+e4UJIzOPw7>!tZ_d*5yE_igXH|Ni^pp72pLOCMRSE+&ruK~#&CRh8-c>XO0v zDIth_-h#oOGs=JQU%s#Mgv_tfgi^M@Oe~NDO_~PlLqLv#Z(b6Hxc(vo{%hy^pPd=s zz>LvA77>V?=lUrxDIfSL;6{S>xSs;`1f1YAPKvU1tZ@W4~peO+{`VO)-t>!XzT2^mrjYs0 z)Ij~*tq0n);PViY<+om^KlpJ9zdyd6^zXr4{@stjpTF-r-t!H35JM7i1n{qaW-z_; zK#$;{D$xJGc?b~7?RuY*SAGtRl`oOd0P*3y0+i9Bu+z_{8T$Tw#sAa)e#6q2`RVW< z4YK=Zp3g5H2LSSIs~UeaZ9)PUC!HO2W~^#MBoW}buXKB9L6%dro!FKG9aRe;6hAMN z%%=1WkK9k^oBO-o+)L5tH==?6%jkvGttn|)Tj76QNK!tQ8W5MX^S2k$$xz!$UpOD{;NNW;rD^=+#CNDvFBfV+E}q) zf{Fiq656R?A&pc{!45Rhe$X(2{zW0=>jcM3@vnXCFSh?ji+;~5Tia(Ve13UL|Dx;Z?B(6Sk1r4PNo%{L4k0kzKXP~^h*!hUsmRRpqu{I zU-pYkCG*oSs&P8de`rGL&uIw%Q&w0 z|JlzC%5YEni;;azG>r*RcYAmnT(?U)(A*Dy$iV@-8#|r=1}E3V!Z(lhvQPj{*&<&KY%(or^O)$O=K*Ys98qM; z*9N(IJqJ#51QD_W@b@?6u&aiSbaOxalj$Pg8Y}#>kNbu)gYNv_Kbidx>ZpJ2-09lM zzeS-EOa2OK`U+dnfhK?JkNlcX_3I(1L~MOL5{ik{=#(hf7*;&@+@R2wU7@n;k;|As zQnq~(y|`I-RK4f%a+B3i{SBX~*FJinsWY6+s6kPlPO>2=zRBj9G?E$LhgKio(55L) zbU@QF;YM!=?f&Ss_(E}(G}W^&Z4Fmi5&;&*=zClLzt{~ioq z7C$B+&Ll?}P>R^ODTtgU$07o8$4zkryni3Jum7T~qyk-2_a=&o5NObl#IArE3p{hTyw3Wr-hGkq`hx;M zk^Y*R_d)Hwv=4~_mEEZKR@+M=8Ka)03+YAqPG=AVbfB7SO=WIY&##yeo@J&DT==9h z&gh4%@Oi1eX->?&o>%dvJL^Wp>n9>_o@|Ra5A>#150)$iY-;7?5mtG=PigoNmijaX8L~0eMNj*Xa%SUA=YOThdE4}F*xrgty%~4s^fy(Ko;T!|p zA@2-JPs!dwu~w%z55?H|fks{KzU5DAycosTK>qVMw&idy=%|RqPS+j%)hR_ihHs!{iQ_B~kYHhdWJ8RE>4WAJJ&Ov``?cTWR~Q=O((IY{k|tL%s5*V~2UMhWy2T}L^!~xy zvz>Xf^KMbJFFLcxCWKjN?Cr)@Y*xg6+q7PLR1g%&1 z7d|QMZsBk~)`V_Tg8slzl?~@}=gh_g^`7rJc`9yl?1K5oJ>O$BL66>^euRqXp6bse zE02AQUg&m31s=cCmGMg3=6>yM{>xV8Z!MThT=)B|E&2-0y z07OZL=SUAaQag`k6m%{w7V>5{&8v?|LK;*jYsSX}Aegx@sc!chV6oQ*b;@22ke>wV znb0|kpXeH0%gN(RGu5eZIoG@GPQZ2Uv%a5hurfmQ3CPA!-bS9+Gkg2qrO!$viOT88 zH>(LN-?%~S@HR3DDL2U=3!agGlge5DY9p9EZ5NZ&ZuMEEd8>xX)A|jm?#hJ8k{dp! z`(7=Y;>sH53M-3GwZF=}M>Gj=lwh_HyxcnxC=jM}i#fh8Z6VKfac*_eOQ8oFo`3zN z;V=fSThuk=#dShX$MdlgNYuSiFQ<7X!UG*oYnf<0YO%17e-z;(SF`QDfDc<|xkDp% z??^);{m0~21e=1*dxVIs6a|A@N;?iLpExA|D|8-xmWqg{C|II8qyn>VR$zkGu0bBD zb*w1y!j6zfQtryv&G0xy_vB-F_m3Oc>@l=OISVBvOPki3aHbRC0Sl{|OH7vXT5Z#d zp|%%CqBRtCwbADR%4*KS{$7*C#3V*KlG48KLz%s)QA#{4zKD5CZ1^xkG_fjt6-_ro z9JI7q)|NT9?^Q_4G5JIc#QepURd7Ce#R8U&M&sF7EcL|;YqmbN_1z3#+I^MJL&{Jj zY%Q~$d>1X}Xcc;I#!nKRq4uObz;_&yg)x)pSj}b&fQb@O;R+`l13kIs?&k<=+$xrA zJrELB3{#)o$?w(y{jlrpbWRAj-tFD?JJNGZMyo5e{ayK7qbnLJ#;?^LIaqu)@>u89 z+b}f<>R+Ev@b%V%Q~)Lf;*fiQ21=JBLIU;kNC)uG%_AW4a2jw|2CwZv-F$mHV1V-A zfym#62p62h9t(OB!@Lx2{-E&lFVMl?InVEf^FLDIT|E8SD(0nB)d58EoLQ18 z0C0#6H_5$EdlTK22#Cxv2@bMFeN5fCu4gNY5rh7Z7Pom7*Nocrl0zYmF#3+FQC`Ba z@N|xWqcXEyY@2sSqyTZb&2lQp@eYygp|sWkuXjWFjSoonh%00nwo(069q8MtZL4J! z&Ujd2(Zl}Uvwj+pvV5uyVbESTKuz@ORiT`{Za7Ul+^L{;Yk@gd=C!^;VE67F)VIxQ zqbBD-H=t}Sob5BDR*}%F`LOHWx+bZm!dauaz1S#~kSj}6jL9Tq;@gI$-@J-jIor+s zt|gqTP#1gIjlcSMJJdU|aFvCF4p%8j?y|tQvuTfx?6h!=ZcKTRQiQmzWUV^twM#Ej zRqcjULT?K72*I7kL{Qkrm0)71-is35ae&{8*Iphc8lGeI+OMC_F(k*TZ%ZtivAXPOrDzXhglPEK+Jce5}EHTbS}~TaQzV9#;4#p~LBfc7uYv zH*u4b>W_qur#PEV9>K9{(?6#k?6kS(xz8fbGyA5{UNNmngQ0V}UMEVpufZPhKvqsA zu8)~CoMyQ^T#Fy5bG>BPa+u+O-WxXSHrwMnt0xPA+-8N8P7FWvT38>Lz#|)B!$Ggh z#3AtJ^zJlss?}gL=k9}tbr~RZUWWya1Jw`EJqp{KMZF3FQt-4CA-D=7iC8ExoY3AnK6d%&$aX0Vcm6m zkB~;fg32x_vd$wF-m)fk!IIY3&l$t3vLWstN8efLkvZ{hTM~v{86_WbMvfd+a;Mnu zQ*XgVvWcoID$0SDA!e#sr@IM0Q*p0)rxSZV^u*vGqe`&lKE5OO#zXAY^j8A~HB2R& zgQi_GnSHf>`c6q9jje8-)YW7*c5?1$sKZ%eMQptZ>wsd{nS$YSsdh%bHtKxQw>K`z zn<|n!{7J!te7&qFi_(`yW0T&3s=O5pHry~Ww7%O!x^U8+0*}=%h zVr4>zz}#;|QH0(kS)Rr3^jv0+XrD-GEWG=T?3{mqbt0D=fL*?*0LWxF`hapYgCS~u z3Bc~dSjvu8-~iQ(GU~NA4$;1NOpv1kk10h0Ic+3bcI&|%!1`hLQZ)M=PxH5WrvIzs zzk9?#l1KcbvPLF74+11h_6u~C0iJ&ryQ=(lYLo?b{(uC{C%VsF<%@BE@DZx@C43yK zGEdq#^cwr=tjQMRVTO;>Uv;1TQGey{ofmL!zeflB&WZWXiD5Ox%TlAz>|kOrdJEEf zfTDVu`M!0_?GV^up}*ecs?MVsR9P`1~#$-n)$7;-yi(A??p?U z??ubsEwk}e)nhjf4GV%W%Y9^XXO1Sx9XJv*?A)fwcwrA!4ZN@deeTeAz^lZK1Hflw zF^i@XWn^3>WAdqIu{nSNJ2c5PIG=4)70&9)&-#P$x3 zwCO;ffA~%r`)_fp-zj6iwt)V88((gH?#7`+5PjwD#6~+HA=vTQy3Y_32dt!#Y<<=L z+r^EHb})fdjmMq=yRKm6A*-AJ0PyzBt#|*1@A%#p_Ps6a>F>9nFhr9d1CP-TJjR&F zat;~?r$!E-H#dRW&e@!Byg4-rxd+=DlJvG@#4Bl8(;7QBiQd}fr6}_qA^z6``Tvgf zZX6P`S;6`@dqN{4L-6ow@Rzxf1ldYRTiChe2L)9ONY~MXOm@Z|t&0LDl9Ak``*&OHK!7 zQ&*2%n;vPqrd{2>A=HoBa{S^>Dj=@n-`&Y~ck=(6cKc@cE5MzY;NhFA=t(g#9GV(w zB(k*uxRDM$xedq-;V-ghxysarJMOIDK#eqP6_OU_3Ryk>Zvbll9U#wlF!(zd{4HQ` zIQa!|8dYDM2KL9Sc5k2%ufB?12du~s?fO_wKy32KRQfy>;)tu!6D4;TA^-t0mZ{W zv$QLx%cG)Oi#d2U>p1oaL{{@-!4_H@s4ZP6_gn1IE<37K?4cN*H(8!rk6d;{>;+t_ zt4W(IA2GLiiUJY)F_oL0wVRr5!PonbT*-2PV2)<^>=<{R3c0qJPP{++8G_#)KE~NO zsyT$H$&O<6gcm(GdM_Fw%du*w@#e&k#<|1dBo$ zSQKW#qOjzHTvjj8&TgWGM;0K&ur0@kY+J5LTRU{wzU*;Cei@_jVfk<9@%}@TJ@sHG zD?b)g?Haap6>Soph}=qTiEuvJTwa<8x_*UB6TA)Ay(^EA-5PyPqopWFT^p8!d4vOa zJ_k}gP9eGhX6#S9=wFK^?TP|qjnZHCuLJEcPw@VlTbkhg%KsmG?;Y1vyQT|=CQ4NV z5d@+F(m}e^fJ&FHbcljT6ObmIAP6W@1O%i8q=OKZUPBK>q!;PEmxLN131|84nSJ)2 zGvArBfAhUF?>oQ!2fC8AR`O)6_1xvUuIIiVS&*et$ph;ZHLrcbO9AJv(i@Nm?UXy} zH&^(imwiQf`yg$1t;~xkB)PMYQ$UL-juHHSYgLHf6Jz{N1<6?Nf7lxe?;V^rd~(iV>$ zm$!eGvaI9^Zz!-$ol1K_x1{TVrDA!}!s9|ei{ApD7HSs#q6&>RKsl4Zl%CM(;c=(LyiAXQu?<7hySW=7SA^4W6p+O5i%725t!wFZI;u&`)DT| z2oW@XEwHtiIlWy7B?TG+{i-1eR2Eg~RA9yb4@fi5cQL7dA-xnt>w_(t2Zx$Avq=xR zx9U}FaHZJ8*5X{NPTWrl5amU!NLvIYzVz-l4F9EN(<{4z=UFY?cH{P)AhYg7vz2Uo zqa}zfq2=L{R(T6=mx{6n?xh-abqS_FVj$gFnbRs{|(<9cc!`<>p; z@jm`Wh5}HvfFBKvr)(j<^TrFUl|755vC{X$mf6=QVU64*6AX9rAAvx(L0cOCF|_t) zCE)(6$TxP8PYIR`DjabGfmiC7_vWT5;FzUyhcE9})sf?EGq3mDBH>WcWmJx5Iu{n6 zyLN4Bf74k*9`txDqkz*!adH#Zx0%+(-wKNR*4$v&EVKnqv%6U z$O*M(e(T=a9_niMw~OS(4^ng;X?P^!M84O6?(9J@YutP8v^#`vhLFQ+U(Y~-r9cIk zQnkZt$m0Vz0iV1pkAxn>&iTOfvv4ypeaAKAM`s{o9@#09(=*WL_+EVY4#6J~CLo_h ze>Fb?^?sf?+U!5Tsq7-E!-iJRK$45@Y)-H6XP^=FfBzgXiJP1Sylk=Q7_C{+=B~{oA&Ys#?t#Z^|V~Th48D*6T<3dbKM6}D0KaZ0@f{n56 zcSDNX-w&@c(T<`y)%AsN+_n$A*aeJ!vxY=hAm>DPmnRP88VrBJ zA-1O}gdN=8!bXoLc)a2aG+BKHYP5qi2MXyvgq0T~tATJXZ~ztp9v_+cf8V$D*YkMS zM;@$4@+p6RLHhNZ4dH}1Q1E|<68B%7$$x=yHXd)N&a-`)yP#98`oWzP*R#*C2w30( z|NNbb0s3*%blE||*RcI!yZ(61w;zT)<=9&TBnD)qn2eX{CKr~1L=$lSd?-JSg)M07 zgZ?Dy^2{|V6(+Wode;Hf-Hrj(&21Ft$Hinv&%u^<=ItNe4_yIVuIBtNaE_HFn*JS! zQ^&e1lh%2}U371o242#-;&blyn9DNlK{%w?dy3)m>u1Fr;{`V>30xKPi>EAf*y1D z{KcvpV}ZHDQ3P1^CKzDVf#&JR#h6v-Hn7AgD?5Yc@J~k0QJun%sf}ROLWFL^J>w%A z0+$KwG!C{|?kfhW&hEKoC#pc1{9xc&oET_x;8{xYgFcc{wPYvYZJM52E}FNJH9>0i z$U2y`w(zfiBa-h9fzCj`ZU6NhTjIZLYKxzN8fMazHN}!ai9N8L$`wr;!4Qs?tQU`t z&t=_upA1S-e-@>-C1P?Xs7@(L)u6W<%K!=GQ~y>rn5Mo%sA|J47MuKsjL{$EA^LwA z$L|S6h0iZjsxmvg)#Lu~v^15-!e$=!?sikd=v_trH0qY<5B}snIhcfH>xRMN)~K7d zN#P_-Qrmppd^MqLg1maS3v~0InhuG}Oc|_|IG1pw8*H!S4Gt9Q#q$m68QNFgyZ9-b zC-Cp0m=juk-LkHFDRD{}h(1=21c`+fnYAsLvdZ(lmG6NhEvvvk%u75$4N)9*9e2-p z;yyU?5hh@(d5A2BA8j}o`=`WaAii@={v!lm1lDa=2#`SM8ufKgwA_0KhtsC0F zZ*>Ev{)<$q;cE+Lpv(0#glfR9eH%{F(YUPK8rZ_3L()=2RY2+!oUH5tt!uQ2($o!i z(3BSvC}Av7vwF0>BR28n6+*{RE)wMMET!JlaZ=Kw-MjFf!50vbCz$QP$b9V+deXmK zl5HZN#3T5Dqmb3r)_h1HU&GkfC;gj4XG+KMFZ2>xs90p+Xk zdsq3Hvo;oMt=OwHYyduc~NuRG^uk>-<($x!Y;%P)sI*N7W)t{juK+WUz~BR>qft*+E?@ zFYJaB=SGAwO)nGra38~m5E<`k9-y;7|4OKY|EqA8-}8HZ&#Jj>Rv6{x8NDI?$0BdG zTlxD}vZ(eE*O6ap%2PTLZqdTE$~RyPWu4B>o6pK&`){6bT))F%laCOsA9ZFP7TCVX zl*K{*q<6t$gh6RAT0RV}+dVFCu4SRN{k$ag@fO#1eot2x-msi$$5bDdBl4T<;@nFxC35l8 z8uV}iC@?MOe+??-2ZTMb?{e^f6ue&>syv$Z4m+AzFZ3F4WlQ4!6)f(5^c{Di-vMZx zx(C3?&BITG?IFRCT{SWA5wK{szgO?K5{>Yt3j z>KdXE|Emplw8xH{_=QnTK0JwOFV6*jq!jYdq#Xm)c!KP}B5e$`E`CV7Z>KJMz|J_F zMKzUZ*+nEb)ehhd6!$!DE^xwjUJ)4RBJ>-{^AYj!#y!&yPV)Qn$gJtrx!xZ^PiJ8v z+|{lV0l=0%D36Tk2e)3L%bk1QIQHO7$FQ9k^*B+QnWK)}S6US0iVst^@4fb=5s7C3 z_*1{|7p}^{d5e0ECRPTB4PQP7b%w~0i=3d`V^5>ZES)#vlM6Uu$k6#RTKl3gNp}QdSqllT%K- zrFDTM6mcJ$1M|q;@bSY4u2OM6)$iN0@CMj)D>D@9;{Enk9Qt&$HVDb#4W0HLM!qdP zjj~sJzLWhl-mk!|h33F`t7QLZ@1~{xY4;$e=(6#IMQwT6D-QAx;^u-*`$xM=ditlu zgXrQs+%N>qd6H1qFmGK`-f&Iuy$Y8NgZ+8Bsk;TFFfY%-TuMy^r;ykBF!!SpgvQqa zXR!_e>%`5Y{*n==?{-=u2_?O~)grNPUwh?SICUS7;CDiui({sUdoGf0>yK9#w&hl` zSTosw%#L-neoyy^83S21(Y2V*yy2fTvH@wF#sph((B#~6*Geh@|#go85CX&YD^W9)X2PU zJtB27z^uJ_{`5(dCxzR&iQOnWEb;qJI{ey2f1^jXmAAHnf?uVVy_Ued+8wx7+x>Pw*?qK>P2$fXI^?gC(dzF3 zv1ZBHCk?CE!Z168a%B(Yt@F5bi<^nA@yu!O#{OdZKEL&N8>@Ha&av*O`~lO5lQvrE z7>p_KN`4m8^3cZfH}iJ}s^=nv4i_5{t~gG!R9H=RXyXrue|TlhAH9|pXyEZ6^7pFq zPwu9>Egw=^4!aDuKS_oW%RkF&|FQi|e23mP;A+LGa(ThT&rGIAF%6zTOrr9ACJ>S8 zr;9ml9shw9^IGWN(!Q33Uu=anD3i|;9U+8bO}DIcsrlokEhg{i{AgF>uzYb&n`8LB zzxAAR_=N@dV_pHC@bj``1I^pMggMEeJ(?W~AnDC~B6;09Z`(BOa;v)LE_bQ#7G;Xj# z#s{Mvl3=$$`^JI`({}PULQ;9W#q*}{x?J;fFq&O~zWfvsA-(hZz;|L*Z0efYDOImr zDfb>5>+5f`+mG8^rN-UXjFGgxZN6Y4yU}T8X~)Cp(?PO`OuweL(3fsysb8#)qM^<8 zMgP|xE0?@3&u6o74K4P0q{(Z7TW_y;e6A*bK$TX#$l_LZxOn-8+?6`tg^I{SIfy2v zB$ZUUB;Mny-NVPV9>U?d!v#CtFD0` zOx|PFzgt^oegDPbwrdIPx%x@w`a5giy@#I{);pf0Zv99-9d}TZj5w4He{K?kit}YV zSc{dSk8ASG`!R5X({0!r9I0AqkhtL3^=4^vzQSU&80>NAD{|tH-zl~fiLRGGws@%7p*P)O5EG;vyX$oPk`O@v&d( zH1-6(9K46IoBI50F9k&C)eu6bkp;Um7I3#E!N<&~OE?}MNXG;%&ejL<{kzh*ur^OE z%Y4lFCo$R`7UjDgCkd!m>IHk-J@-d1sT`%+^ttIsWF<^62MDMQMR;23e5g9Qw<%9u~Ca#Fh2dZ^?}vpEpEd-HZ}bf3Of##nPZ`OtDcIxG1-)reEw z7*KQ9%0ote3wucXzK72v{tP7jnEmDSKJKAZ=Qhf=_`Q8<2+BuS-5+SsN(rW>a=^i+@8 znhCXooMk^5@+w#XHvsfe^u~h;wJh82d-^(PN!+dJ^JV3P$szYm?_kGT!n$@Ea)j#| zQrLoVK02sD2t9rLsf;-%d2hwodW_cnv;|G-J391B+!kX$y1_+=updX%{0iUa*ZrPj z$>L+ezwkEB2HjRHV)kztx0e`1dCH}z+=Q2zu6%z7ud8B@IRKlSLRx2N{et!suaGQ1W3>wrm3#Mj)pxZSqF$ z7sm63J|6CRJ_|ckHbMS;{ANB^`8|FoF@ zkSD374Pzq%91d*mG)?GKWoy|Xw_4Pj%|PT$txI+3kz49fNpm>pKS?}hV348Y5p_5G zGw_H_xIIwN5VqubDm)?dsCP?sb(L-8Qi&ai_YOSi}4SyfML3 z@Xm`iIy4mCj*}ZtgwiPPQ3< z;^`0et^HQer3Cd`Df@{4jwbHUWbsmH!6J0O0U(U|{umOExd0JZ=dDrXMk?$?oaYQg z4!~v~Y{3U@fHu!+P#)OH`D-Wix1Hx9iyv11WuFeY^PjA|9yz2W)j=A13ZQfBOYv`lpseg(bE8yj<2m};p?Ci@HIzZvej5+K~Bn!-jV_mC#&Jq=I)Vw94o1j>=r zl~;i$@A?Z-J?{CWl*bos?kuO1U7tc;ozEdoO0i-9t@2EWNYNKmt+3M{k_wUBN_*qV zd?oHN*PZ1Y9aX068fNoXtWzu_Qd+QpLLnA_Pz#gvq||G$VVH0b-3K9;kRsK z0>1{^w=x*^Z-cz@0fQlLLjdATjpeQp7!-2=AV1PpWh>t?G$gu&_+QI?oO2#FxEk_k8Y@3@VNAbM$Y0F2AOud)dB& z{AN|c#8PEKE6u8d#UMcr=xgw@Ebmql!}aXXKXO+TzNU|ByrW#=E;O}bUgVwQAPG`o z&?<}5e#)&>Jk=b|UBLfKm%~Fzwr(%=l&5hc$W13!!xi=h#+%TL@e-WYrLE+pbRyv>l>T z%=Z0`;KNsa3y6_wj9Ub^X6L?*79Cxh7kANmcyl&!CLI@IP5ZJ6erKTJ`l&&ZbohIU z62Y5x}A0fSn^Td z7lQX0-VDgFlzx@Z9;(iE)F$UCsZHULC+3R-5XEW{yb zayoA>%gqP=Bie2el8;=*(X1BV4xg6<>nks;+a9LhGZNYfAUD4KYX!JFOjMJ&+6=C* ztX+^b?tSG8d|J-88M-TA3PTD3NXrD`98Ke6VQ{4Bw_)e~tE1h`0%%uIFQE#)VqqhR#ybqk1CLxslO13}f%V7B zZZO)BdV6q-3J2kqW$sQH>u37Kg90r{zs5GM{8f9F{G`etWo`9T8|SMD5Uu%GqBtAp zoy!70Q9dJO08*ujocf%A`Y>&Zs~v0eRX73=`>)`hTZDjqIsXG6DeL)GxV4<01Pbaa06z0 zIq2nwKN(-D==nVH<<{k+H8Z7|S2yh5-7UR)q4T0J1&k)BcAZC^SAD>cSLB2F{HAv= z?z#=Z7w{l6P`o{4js&tdeo6pr+=gM6uJI`aw5wqrUk_b|`N}R3E{X!(;1IAK1fg7S z7zY*zO%ng^X6~$6H9hIt=<74;pYIjgtz9GWqf~=8(CB{BM0(Pq0bBjNsYcHgFZRkB z1IsYMK%!96qrTh4q>g%S`Dga_V8Dl+fewlWxc}~8!CevuU%6`j2YMoOQ(81q`St`r9nO9Zpah_L z0P*oG->`X@z)^2}R0azJ-)9*061_WE?j>Y#B0|tL&z3SHrt$ITiz%6Dg6xuvLN?%p zuHoTCWsz1+aSz1e53?@4D5feIIf&R}2WiNm;Tz33;wLueM0_(LivbKd^Nw^V&CyK( z0n4YMk8_$_-4fUf4vSU@yv|hPQ_;z1Agve!sDYLgt{O)z6bV~&7(h;Gh+{+Cgkrj(-$-a9DL-TnOVi*m)4?P%t*3S*x9K>V$UCy z@Vj#)$7gjNy8KKVqlmHBt*cI0H_a^5Rq|fR@SLH3J5dmUrcwc>dC6{-P$+T_i2W;| zr*fXKY0TQuFjv$S^e#Je^1Bd2Pi9ILI(*|0LtW@ zJ=1c4P{%d_k39nd53qd{d&WM9zdEibYJ{&1xiQA?m$az);TzC(yK)jvyDLgEiL~o7 zBRzRI9~HvZ_a(qofqMbDM^kx@a7_?(5s&R_*ESM4~k1UJ( z-umKQa^kXR9#)+K3rr2t$A9%wcC+BnQd>s!26JTNsRhtt^x?nrZ*N?kPK5yrkU5R9K&rw~+5#A-P1ydeLL@V=%5Y<5 z8+aots|ZE`Sxmsf2=&NAD+Sg+V-pdE5~g@9L0P1K#t3!Wa95WChOQk_yLf{TpHpC@ z;c<=Mm4OSNdImDttJi2`t{+~|V9Q?5tNb|z;2rx!)tS%a~PUsQ-df+8>gclqy*1fxsD5hoN70*T}qh*5#T zL#M@{c7{7X*}&@h)tR`|S)b*Ckpo1axR)e(-QKHJ=BY$@$pQ}0@S#H-yc;X;?bI1| zSnjT)3nwkd{iDA%EgN|cgnx*&b8t&9iEc?g$c@WgH#oY$loiNVo98_2z8q0=l(ub`ds#bt+gWO_$FK;{&liL62WK9^Y zjJ0KxD_S}*=vc`wrv2*}>0!;3C4e~pJaT8XAfDf>@YGJ<@x=DdNKY)KsZhYYQyqP0M}#uB&gkoC8NQ^8%Y*}%W}W;$`l5aN*4{YYfp5(OaM%cZjFBq1NEme(lvQfyC#w zG6!7M@(1<}NOHx#OlD``$))joDN)mE82_mB3=|4jt{SaB0$A~nr2GH&*x$42E&!s0 z%)a$gqq7)8o;>AMN$|OI24Y5U)JyMD_S*@R6}J)s#CN1n~uOS8{pIO*NS?Iq?d-6rvT(2 z+UN{4Rg&zR)bR!GixyW#s2rVvNN|Ob_Ntvm+0R?@3m0Q?@OeVK z7AX>|+D(J2SX?TKQP;Y!tM;dj&GJUXnQMm-*f!YMt=->kj`**9cx*N1iEu#o2JSmk zwIir*`hIqE;qJuA@iC1?%1K(c)2MvJn$AgN7HrlGcb1mRzBvOeWzpu@9|~M+IeGTV zUSNOY;ait(EB+aM6L&*u^?~$dOwb!Z+Y|!?Qh$gLiku9S2UPN^C2T@Yq@}CTzzgRv zG_l?%>}Bw!g>r?-wlpHd+-E?hG2nHYBCHHeFac{S`?RT;j&=SD+!e z;Tx!3?CR*CU8!_k!gs4@7GVqrn%+m@4UFR(hWgO4$rRsetrf$it7jnf^_G#PmvMbv z*+O6Ebzz^u_i%+FFB1buF;`N&zINtyUgrq$YOsbxddOeF%4>BaE~>p6GHLe)nvyeb z#@i}cEUQzmdbfwo)f=`--ei>?UR+sryXvC2!Abct_<=-dWfB~A$E``Gv284Uk>-#r^3 z4(nte4R{YE;8>sl8bD5R64(GFM-kVfgB=FK<#*;Mia>n4P)r+w1nMjU+C&QnTo1@& z=#Px?Jn}sqbDxr4CoIO0m1&E;rg~p^LDrw_v6lvF8jPtNSAOTdD_0@gu^5pB^0`Q> z+Xiyq*ApeZi^^aU@5O4g*GNVd3ksGQTgUQwle#job1SmTt&(K^f6@R|U?9}Ew21X| z%qkvBD~kE`bslXI?>8w)cRj7bssX5x!goW?c>7Kpz}d=V^)>{bal>+WIK8mF9&E zT1k5ja@CGaZ15SfKRZG7=Evq`)h^aXlD^a18Pw~1z1#AfYUFKWvefv591aj3(YqM1 z+Ib{#YE|q*_AfQUjMBL=PVZ*%Vo(Repz~u}k)X+4$xrzkxG*2k#HYGgp>eLJZ`w=c zB?x94HY?LiD1t_N%_1w;UShl<(V@Ut%GWeM!S-W;-lI^(rbQjH!au1W`1gDlkWaG$ zG>|Mt2p?2}+}AgRpfw0;G$CgoXYh$Gr#qVrAUgJ*hn|v|!&Y;FMlXrDSXN5HG-Ur1 zPyzGb`@}!ndw0N*Kyk0~nQZWhymTKH3RnYF3-VwYIZ{dx2QJM`z;3`+u^TFLzxosa znwdgq@NF?*``Mz^p5y(Ylpi)J!6AZYaofemd%HhP*b^*c_5%sT2>&sqg(-D=` zIj*W+k9J8PK0y54;c+-_bX#JC{CVi=XcndTey)^o9Q`YOhv-aPyNVC!P=-~n*R;=a zXp@s}e6nd-d7+9OnaO&7)1d?Q40(K|9C?D%#uI-Hpu?sgH8tEEk_UUeK32^3sNnZf z`+B5E*oe5a6TM@A+MFgpL&`^+WSlUe#St(F{V}NeJE{tz2Jg&42i11()%i! zt51&|BK&63Kri8JvJU=Pb{Jb9AA5R~PKuj0>Xt3Bk-HzDU)Ux386>Cq@{tc+A(Xd3w=-f<_`Jl zXG3%bWeptVC}tl!=H+~ENc`yCj@omI%mcm9e>M53NbXT{wSkma(nrodLcfMlA$GPZ0KO^$1LFr_`&oqtSYbU8gBzS$mDb|sG3 zq$DoOeBG1IG<)!qD#K%bB5os+Skd5ia)g-%<^&3vb1Z98KTiBoL96@0&0MJ2pG2!) zp3RQV*tAg7*MYUCwdy$AQ)d#cm`XIJL!KTmAmUx468dza()at=epimq(HmJ!02*(h zEn1J=4B~El2=*~#f2}(SCmkD<=b3k`dTqeDBY7FuE#$*K}z);c@-}woCS2euDYkUfz15p|R@HdW`6JBW$3ZP!%z5zhQ3)mW; z;TJDNTa9+GnBmZYceqkoOna~p-@=WDKA3&h4b%>B=H_RhJQesveHf0J(MflH+rTy$ znL2uSh;HK@%XNFv%R^NZ=>0gno^~Km`Ze|pM5RsEo;fFAFsPmu7X!z1B;?{k+J#DUS}6x4ntir$1iD(fsHKIksu#VzYR=-?G}OG>!qVB z6K7xS)ug_sp}KtDSsRhnY#$MyiYVIxF81SWveRUYD)uh2_IHInkC8lzuZ6KsGJ`Zs z!4%k(yCKD&%WCn0_gfX8C`ZsL2=B(MH8DgkKp4kxf@ngrzDlyAjOR})BNpy3%6OKO zKo~Ek*r7RH49nx=QB0}pP|Q+AcN+|>BzqQ}_vOiPkc&m0n~O6hcy{W01=`zFzqQSh zOWypu&2^GhRhAQRkY}=To^!G7p-)cMv*~U6WQf`AAGGzZqi{O@S!|5+Y>~r-Z3G8} zbrK6{7_ZUFY850B5Im5|I+9hk^q36leLx)29HV`X`6(a!yl)DdCzG5tNO37~kOm${ zan7E$#U3h-!_GaSn-!xlD#e8WB}8wZo)$a2GJQv88&^>dv(51`iUZpHx?a*&Q1*$(s1m@<3+B_%c9Qj4CYFa~aBHC9ujpTFanIn}^mxI0XE_#BkwtUbSBD9_=*#)l2Dc>>1qIXY`smByzDl8UOU$8?>6 z3e8-JHpT6Vg=W4#CPAg9^AKFDo=!kX_lg? z8_F5J0ZHnN>?h+f>x!ay3fd>TE3Cb9ffxKlF{3Rt-x*g7LN%Fv?|d(2>8-m%PRawK z`+425-FrcHfeC3_OJ5VYh;GkFej5F{#t!t1X)&bz!r_$Jbk^LKq!~Vc+r9hLg)^?B zRPExd45f7(+_D93;m{W#W2%6p5M+3wWW*KR^+<=~o}r#%8ctvD>PbXGgx3|nZ`Id{M?4<^ES ze#XY>j)-l!=_VXLe9INs8BrMY=!Et#wo+aKTlo-u7?W*eYp7?so*X%SDXV=x{I)`G zk+cqZLI0!88_6@;6+!dQ(!J&%#u{v}gNUpSWQYI4lJR^p`q{--;4-YZs9)6hD-G}L z@T$ajBk-)OAm63`Yb8&^B)_NFSFqXoTkX9n6ylm{Nmd{7op}tIuCr0nHBw7h(uRLI37FHb^U7U5CS-|c}@0WLBL>qe-v zB0MA-{sR|TBzn6bbbP4}z*!zo!146MO@v@jzJT$Bs-(B|i<~jBnU`O%y%UM;;!~&-`@c z@Pzj5W@prV*FvWV_u}QfOdN$I{=gJYyiZ%|Clh;uk?i^0%Q3*X-j8pj9i@WJK3e~v zL*60JhOS?lX`_UZ{MbIclKeSV!bRFvpV1);Ve~E*(JYYwt;Es(3OY}s0s6YaHC%UHH`GQxs8uC?fj!mW4d&w>ohpPg+)QwkCJ%+3n@_kUXjIXDMtUB%fvcJF!cru_6iem$m4 zD=TvLrF_=cl{{??fNMuzbc|vxvDV!O1smOl7u7MZ5@2Xj__K}+2 zN>JC-=Udd0sjOS-sUf0YflV|wpx=aO@PmW&vn-@1;~K>BuU&R5-p(Sg zs4U2lpITg_e0L|JhneFH1h27yX_;l4U@1fg7i`%8}`KwD22gdZbagiV|$!K{Iy8r=n zD}e<)l9F#eW3QkT_S!G&dMY!%Y~DLI~{kH31)=YB;(W+z*Ew+n3CR2ha(T z^wFA}I5$nRMK?EG8Wc0aq}VFNgke;@-LPN9mwcQ;ioLoQhQ<%SX3v#aOWTOU_gz*_1=W4Crx8Oi669j69eAe~fSS|m>3}Es z4s3Co3uvlv26`Uy63PZyjEBNTy&(I-7V%0T7vyhZ4d36y8s_nTKIY{=&!Uir5&FmV z&p_6;$v7rs0YT<-9_y0ST`=OXy#o2$*sN2-xhr9Z7S^Nb=XOV{DC?q|4k@+?ytXqF zaBA_|N9F@4qN?aocN-X3Cp$oVZzXWYN%qNnG%@z2wSaS$27k5O`DtsZ!{bM`Bw6Ug zWF!FZ*=n$I(oC2Zxd(!GoZrBt3@Q(h*v}9Q!)iL(p`N!T*{5Lp%W?n1jHKEB;0h6?Gg+mF8X>Zkh{GGR#cFk(=)@!pn1QU8kjQG8yNY2g+u zS0)AnHJ^vy~#YL4;oZiooJprm?Pkir5j#S+R0T3Zx79e^ zLLSlOsv$Bqmh5oskVv4P$f#O*t%F1Kep1%T1$O>|*RVq^7%|b70;_fTbUCIJMYiQ6 zxW9AWE;^`RDl+%5V|@#d(nq`XX8UQ52;w-=-8!}Xd-Uk+7k3xn+@icrD^=vd8{xA7 zRM4d%?pSu65-?`rA?*x@%XNb7Cs1^Ho^wOxUcrWK6JZG4g^N;R8_ zhSnZ*42ay3RARksUiNaX3+>DwYFa}g=B0wKUgHWS7f2UPjaaUqX*tnfO$UD+SgiG} zEDy1iq@xo1xa|%3Q774sBrlO=aCDnToZB#z_|)?NLDKQ4pWHl3I{MJG(#;|bo!6Gw zAdD7h$jT2ned=0Y_lysGDwr;d3vHN2F?pUA)Ugk%{ z??M=`;5CeS*dZ8C!r~-v&qAvos6A(qh_NJ4L*_#3_P#|+Og07rH9o7yLAX7v4ygqh zhI1#X;cD@E*Y)A2p{6z;smM}^Ng~|Wi%}0oZw3%cWI7ExSO^c79Nc2sD~&fe5$1?) z4PZ!`HMiergq99Z*S+7aN?x1($w_X<-+dLLSO)j=9gaTdMX3YmsKPQZLg6^`&zgsQ zJ*@S`jggn`UFs!$DGL%-pu>@lmfJa%?@+HN#+nPJc0IAX`c*qA{=?-TlYt6by~X!1 zp6w?Gqwk7k$>+QmE34p^W*SW6+XIgaLs<$#uE)yxlS?5%H$x{-^1PKnnwa(!E6bn! zpY(`&115*HFZY5&A$GNn`ud#w%LzPJjLC^iE?wC?;Un1xD6+nQ-VT8TyE^kiYMuNU zS(vA1Us@}yvB+v39^ov{ilx>1`lVXLe)N1rzq#~cV7O#wFgzvCZ1?AuMaFq+s_cv2 zq@IX#_?+jre4(`VMCQfw-kGk_e7El+@7}FkQlOM{adW1z%2SEIrZRxT|;5p+(X@T8C2*y0HW zD7mx}>F%IW&qZCZ*4A}bYewoClMbwoB1m+{0b@})eY~FLwha+H*cx!S!jswgxwu*5al)3{%KdxiA_V6ZS-lv# zaGGd~ka=fH=#;%q8}(F9X*V^R$U>x0mGsC$ihMMJ&Eb2)MQROW?7am6R;p-lJpapZ zspIKeUD1{=F@*ku3{vz+bP?`F{jhkkr%Bzn{*084iqi7OAc!vurjajJkt2PqX!=s! zFo6YU4kekkk6UM!J2_V&%JP9zO;2<%o>yV^pv@J0A%N8jimH&rtS6pCFz0;EIF{IL z&uCR>dGbDWdy-Z@y?p%y=tfOt%e<6OY?tpq?P4R+!`v;Vq(*+{P6SJU7*Q{we{}Sv z2`sFqcnEzZ!GHQ;h+DHqn??ps2*dSu$Nda+#jCPaw9V&M_&OiyT>}mSs*6is>SzL8 zCgYOx{tg7*et8DMY%!h&16f9h*ImM16O>I{n`*|q_ZB>V`&20H#^Y-IqEYAmB-~`9 z;f_}hPHL7%p!k{XPPr6WozgX`g5j|Mun_LNq4=r7+U1|}VGDAQ#hR~aPdrItf*VBl zsjk8rL#!h8AQ2FzGDQmE{{8N!)iLL1FOpP247tO8+LDx34GT6f_+2Vb_l)(oy#GjF zhOw=+`Z>#?Zo14C@OS}6@`5;+*~^KR&S9E&DA53^T{f}xJX^6^Po=7|Qqjrd^Z`+Q!{xE66jlNSa```}W(4qWro1el^ohBoyq7=RK zea10{J$zbV#WFFBtYUwb?x&n^@&WKAe0?~w0<#%AL!XENvwMp*Q>qbF9a7hHz31z; ze+%z>{}SFSuqFl)7$&jWQ0c?WdA>+(o#d%cA64`0_#Z9`TWSfQSsO5!0HkyT&jG=$oaR}T8AnTu@MlL4*O6`ATSV|xdvHym#(*N^2 z|KV0+|M?bP5~>46<5Kh@cX+#eF8TtnmpoR=c@{!9=|EyAIWcQ%`DVg9z~~vdOl4{+ zl{>4Zk3T(82Tlpi54|WWb0y&jY3WsR#U0+ccU$4HvLsDrz5UE9KX%$6SsuqvN-l>9 zk!g#7a${LkSZX3J>bCS@Nfb~3!q0lPo(sh+boeqqN-dF>6-kx{wR1(*4LQeUfD1q zJ}V~q;657iWrr&Zb9r#gGcYU6!ErYbYisggE!sp$xs0o}o+UffUn*c@90%e5oHJ#v;o} zY|3z+bL&TZ)s>K#u`jG&dZTmQqSX(ysj;5g=#pT^thJbL6AE_(U4Nirzx&2In1q9l z!%@u!1X1kLe2;`f_2y#awxGcFV~CjE=X{3xlk__$;ae%Qp~asqOrsgR0J8bfmrB2E zs^SE&fm>O6Qj`3_lr12~-S@xTUK1T&Y*X)Win$)o5%a*x_neG^j!*u&rbii3SP@o3 zZAA74S14z2*PsbQsMwp?R>o@-D^=Io{G|{AgAFA1356%_4N?u}7vOF`C-{^($@AI0 zbyRnP_YqD2&Jm}H6%Y!Ob)hMrLtO06Gdg!qATy#?nm8wm-*njk6_B(OzTxMM-c!XK zrm--#&DI7(Y--2oFV^6vx&j6WSISM_8cgk1K6@i|9_84};VwtM zq`>sv?=t40!Gad>I79{{n?!!PI-($P@_g?$d8PXPaUEx_1y+QXa$VJ#|5JI?Wa6=L z1vk+*X-Uw?0dV>IfRky$IAXkPP@^LfdQw(I2G{%v21L;fWS#+}XzQ8R0y_ayh$#n2 z!zcvD5oq!8rB?}7Gb1(Y+#rM;HWEFs&^~$+kY!b(mNPw;#s54eE;V)K#p8zrkJX1d zG_he-YD(YHwWDd`w0C{fp;Kp;s2|>?+$D7Tek!OQ4|mbDFRV~f*`#}O#}o) zg;1oIh;$JU0jVJh0#YL&ARr*U69MT>kS1MPkREy`p#}(XKF^$)XU;vpd*3sa6FhVSlY!;m90;Pqp7!ze9jssJVQ(ZhB zv4(N#HJf1Xj8A;$UL5GJS>ZiZw;NNQX)Yf=$1hRVY%Oi!8ACzle%g`sNCKu*>khln>spFj0$itha2LrqY47fS;Q-!1TXtGu)4Q}0obhS7-s@vYTum8^>!AjAGr)=GGn>sT86HgBi&uyvK zN&{#k0lC9q_~#AvzwevIP`ZY>M1m8k5qJPgr(VfC{Vzn=`e&n8{d=w#c%1(?$Y{Xf z^8$cJ_Yq*EMs4KMZQV7dS<*FtlaPu_Irhr{axKRkijhAelKx%q717T?Bd&bY3%1B^ zZy1fnmcH`I=+%~oDG(ASaj4)zqHby%;_^ke13@n8W8-}yy_P~0np$fB_Q(k^O=SaI zNdB%?nVd*`1;Fg9`9Xxo5v2_h^9{(;cSZzx?lUf_K}u-+^pIathFH zhmbA*T{r)qERp;F>DvEJBd>omdz-(FsPv1BM+pl(q5?!rpC!bd%^Y176$Nwsu z_&;EkQUGNlBw(7VZ^dCK#k1%A*tfxl2-#>OuYlw7xa?kpCvU_grQn)3JszDvG@S89 zGudmK^}Y{yQI1=DJ3xfF0R8{6BHKcn^F&G};uB)i-SVG`O-*)KRwh-*g68DOmZ2OS^IB>5=H+IsBE~2i* z08K%#pS8x3K8cSlBWDa>Oc>zLrOX@x#qAW#lvre>W^dZJCrq|30*AmdR_ObWAF3X? z7FuWwudDYYdNOt0Z(&KDXK~Zxc!W~&zr*g-I(5J;vv~|Gae%Mtjs<*0>1qKA>*G$K zAXgUfzpigB4kr(4RFE$l&^z3;f)r{*wv>~nvvTNGo@<|_?s;));#jD4;(48xJdo=%YTYW_S zyOq$t`|$k7-!X}6hF$`c8xq)DHs+Sy;U^Vsy7E&vP%a~b_EUxzvF+O_A@xEsMOm&7 zZn{|6PSx{yL}?u}XJa5ki;Nf5xIacKYi;X3?I;+Z1O9-RFxr`*HM5;(w|KWM?sjbc z2Sb#o^71Zx9;IZ)p)X!|e^zx!pzNH=n0!qTTxo^An3Z}}EfJevEx}mr1}RF*u^la| zuSy?X(aRRmc>QW4yE^6h3n)#4HiFA8{}T}NSpF5C^S@~krr!l;0uMv35H+Js`vezC zP`dCh{-AN2-c5$r*!pLZ^!qIcSu-bmO%ou`jr4(3Rt3sq5)6+_Ea~o(AGP#mO(l*4 zCm3Nob8Dy*tB>H&7j^BfQG3x_5M#x2N|HY#BV<#*@=${#YG=}F39FvjgxmM}d?Xy^ zU=HS#m!>pQ-@4slQK^kc-<0%CT5aR==9k{Ra^!u*pev5~Vu9Axm9922U+Y(ppyQM; zvNgTyaMqQQZfEzGQ&xd?Bc&x_%+HdmuHAvIMVieTnz8j262-8EP?hq=h<+y2bC$^o z!{QZ#Xu@#eKWI+;qvqDh@BGo4`};wxKl;^w`QBe;-uWYcl~-s|fBuiGra$_Qeh-p? zsOScO_OSVS2L~}H`qajQM7%VS4DEo=Dscs!eahTEX3cQ$Jo#vBo*LJKq&bL6kSpjY zjLNE04B^d_d-dmphGK|B>+z<|#pM%#fZTf3>y+FD*&>0daT058TUnvGo$2;^Uhk!+ zoOpeJ4mF^+?L_;Px7HCeXBLC~Rk?yxI2u_`XC^P0>L;Xtmmnd$8D4*+<@n#er|xRV z5IHC+{Eh#-pFAx9DG*R?$!q}Q2cz=?=kpZ4bq=Tx=nm)Ub2OfRPbXnSx#df)-@}f)&@REfe?p);Y;#fqmnO z@`0^iWV4y}i!ysiKYit$B)tJSWf?&9fw&1ex_-mms=v(MU0wt5mz19dVg^Mg(JtL% zYqeQlUxs67ba`H9>0Ot}MKM$m4iyqVW(oR+4WCLq`+YNNHW4giet2 zBEwG>zka>vPYe+EWI5DGSPV-pC<16Knl8t6xyWhYPlh8Hu;Es63fLMBwR;n{EV?L4 zBUpSNW4bPVyI$LSK1chml$Ct`knef>nj3X}nZE3ZG%|z=1$`$qPW)%=soHe?SeVO* zZCXh5_@D5aITDx0>sop2&3Qi^j)}$#cRHRwN-luB;zT%`T6qYy)VUu$XgQbNeBcjL z58^l<-icHou-VSH>h3*CG_IqwAzh*ecy@i>+5m$h4scGW%3Spo9Y*)JX$+OAZGh4Q z2I%C~c*wjH7Vd#_TD;(mno#WZLAuk*$>e}KxlpVnwOc#QKYDPT1X}jxAINNgT(3;l zbwjmpJNk?>)V!3IS~qQBe3YW3k&w;#s0mRm zV)VcESpk%Dy71qbm3q2|@1Qt((G!!l?#%wMGXY?~O&jGapA&<8j_tb=Tk3gKEfMZI zdOCht^NrNo=6=yTh<$EjoCF@DDJ z3^^*TLWwYaq;CZm@<4r)i3`}JFL}xkc3b4(#|-~L(DHB%J-N?=l>4y)V#e`VpQQ{I zEn@u#cmQu2q5e=Rb{FkEx0saOsts{%e*Uy-VJq`@3hi96k;}VPvg%>=i>v z%1V$6L-4@%Bcq36pjcr*z)D>vL_CYc=YyL$MZjR4;d_7|Xo+4$lLKT-6aVS9)-Auu ziG^?jOKPe+QhVb_w!_njmgb)a3hCS>QSoEVB%un^t*4_ULirtpdyk(h zdxM(b%$+%khg8JO`Pa9{WZ9$q{W6Z6QBIk(r!QT(`QYc^_%0jJoo?kX4q_(<>wgF^ z@n7`d>@$y-2qh{Fz3^fI)Kd*=CNVFejAC52&A~Q`L6JqBSVbrNG0c7Z> ztC>gU`pe!%GnkFGmN1MeYeIF~=IkT;-0WQCUv}z*$`;O_zeBpd^bJpmYOULB_F^V- zL+Jt%1XtgbN7%}{=lPKxhM`*PU9uu>S>3*$_t9$w;ou~vulK`55jzt@kxsXwqL9nq zOS)*^FzS0*#s?k0v-h)ChjkvAL!|PeQd{rO$x~2w1GY(jC0WymRN z6B5%<&F`RW2Mu=ziU3JrzPG-$ofD6ccoIITXDU|ROSKi@Zh6^@5!My7e(Cj+D5I|@ z^i>v-rSEfG!rJ#vWcpWFHdQal4=RJ8=Vv$~KC}~vT}<|9WmzX$KTIlT_4|NVUJn_S}kKZR{SihQPRG7on%o#IWk1W6zXhTDu)-USyq+o;QL~OB(?I z56%F`GBlH5nlMB?CUF5e2!70Sv_7_sSyo`;Q~Btbu@=!0axkDrh?_wejtCr-zD)*y zxgr8=B}Y?*HbqG@6L&p3?Fvo&d*kOjzLLb(p3Aigj50hsbPPE_0r`L#DAIj=Wmq8j zvR))P@ru8xdk{(Fsa^@U(fB#<)6cELC?gR#h|bRJs&Yu6g?f#;!QKjAL}Y=1eYKa_ zGUl8dZL2C3ER{nce=uTaKwm+bse_Q-c6d9GLitvTT|GspfrbIH)=Ox6?~~5Q$5p`P z=7r$Sp+n7hQna?Q$^7<@zaGbPsj&1acAfpcJBvfx5Bex%EgM}O+SoW{B+8$<0ta8^ z-B6ch(QeVn_jZ-wMW4n~w;)_}lT!N|X5h~>7_Uz7BaADs2MLSTrFg_v{6aZ~N2m7# z{`c(~GUe;`ude<{WD zVL+o#ACi(*|2P)JX!}^5@97?y>&uHOPzWCpbkWDu;Y_eZJWWL4y>yp@l4#{%!}og8 zp>Rz5p&~GPAadI<{Sr5Ms9{yc&v4@oal!sau+e_?x>x}6t4o4Rjs1;{hkhit7P*7n34+ZtKEaX?9~kxQEJgK zy4gTNWE1em-m`RtUhABwiv|jP_rA(y_{!0@AKMlOJX4o}UK-qI<6pRjm?=bzWE|EV z^m2)Dzw)JceSZ%k(BJlPcyuVo%%?l*6DVAkJ!Mq>5eF)p7oGG489aHTKk05PO3q!r6tIX_Y&*}mmuen?Xv({U`7uVoxx}~Gz|e* z58!aju?LhC7Ew1GXcvd1SOCrk;L7iKo}e|Ndb)b{MA2;*_E#=f_RLA1OxuE)+2eLH zF)ab3&yuNmXP3vlIm>nfJla~rIm`#2xxK}JY<13OAU!>GcCP|CAbA$*w#cVUdfN(> zlqR9e5`*;E)$g~}PHmQ1%e~)73DR0mxxah=KwoKy`-eoQzcsrA)$grSjB3;1D|m6f%HfMxS1NG&&K&1sK`q1AZ1- zK=G#VOc-DZGdNC=0PZiT?bxG%6d1Fyiv^u-m~*KFVhg~@>FckX{C&gJ|NVv!ev!>g zBp4AE)Gk=83#;;5U}Aoe{j>*v)fcj#E#$l_CGZ>!Rq04s)F<(-XstkGY8Zqq%atqI zcg>8qxKT~0K(>boDdFqfG!G>`Da^#kGkKC8{ZHlcb&n(M%Qohq19t*pTgxgCgt*YlouY&K+SYtoMdY)MRL4A%6*(Cca)uxp=q+Jw6#SPELUPBg;IQqUKRa) z26SPS%`|N!sr;Skgz6Q!48(DPO8^I*Ofr4v9gqq^Rbxneiq5#L2;Pav>pWVmGA*GD z|E^2%D!-5Xb*po?qdpYiT}}aI*d=>_vVgMyrW=zgk;fN!O+^Dmw#G0hQJ2h>@#J$S z<8m?&PS(dsJUEGyC;P(5etD7~oa9X>dE!Z4{QuT@;ct0HMzW=z@a z<_83|rDhv~<42mml2KSjd9;`lui|Y2cX|V=C6WM=k$C=t9O5yq*PGq3Dp^r9W6tsd zl8tOPgg*r^THkC0`ZG!_AqgTt>w}PV(xnD!;FyX@{Y5t23L^2nlA$^I*#FT10f3^v zq*5<|a7hFxDXd-ZnOTvjZJo&qu3JAkm_!cesk_TA>ROXR8dM#^ubu+%x`5+nR|{>)|}jKGa`Qg`Q@@^mAiX zwbZ^Y8B%lz`^s|K4V_baGn3;|)9B0O&S0)Ltkg#~3r>Rx4|haI%<|(bp8T#CfBjv= zM90fLbD2fhkm{!nFavJe%;9uC_Jv5yT*@Is7D`Nv0A4|W0kV2KXB(+*7I@0aJ_mSa zMN%U^pkF1?p$OT3t?drW78KxbJgN8ZN6yh(06-S-z?Qfw(V+>fs|_Q7W2h)(%)4R7)1}W5dXsns6llNHalm+yPN9)$ zXUcE4aRx;56E0?rNHzhGv>zVO6nJZ(dF;@S0pF}+N!5GlcHED1K9!0rk!@-efoGCy zjw7+G0_i>`@RV!LgFN_@P85L&+@CL@LFbe@L%KS+h}RDlbWGE8ul}fd%6%*;2YSm_ zV#o1yMoX%#q>-fo*Q9WPD*m<}G?*I`O62Zq_(IflJF6pXw_0nH`VQ9+$Ae?qeDt$1 zZoomNWFke(W67dTruW#V%neS3oU;Lp2hm~xrc(G~Ql14rTnxCmQ6JE(K+YlCe;|R; zQ3cuo86oJ{pqofZ?*K-v{)|J407CSv3T`-|Tu<(04+uuL1L!h~RHpyX7QN zr^hdAWw)_m`StsE?kzQF{0%gr#>fHL!i<@(Q0Zq18kN8z6;g5~mQtv8?rv?Ceb;23 z*t1nz|9T6l3rf;!3y_vG1IJ}Axsfj6=wh&;9^=h%G7+NtqJsAzZ|NsMg>LV)w7oUo zJg$0D@U}?cQ_bt?szB*f=Rh?^(Z~&24@@&Wh`~-6Qks@}ImE@;0RtGq*&%=Ka?~fu z=f#U_jVp1Be)7oBjwldiVeq3p?Y5-y<=H;O8kZ8{BzH4EbD~O!!raDR?q0);EQnNx z;4w!y<&G$L+g**CbfpWE^*^>>5Mo&7QNlBk5?<(?4M~;1&-sXSS3l2(WjJB>Vcm~R zpG50RiuC(eHt9df0VV(3((bQ%@A+gzN%1SXO&%!h%jlx<$(3`?ZcN9mCX&;dl2Yw$ z8uIovKrwSLlO|+9AEI0oD4XQe>TaYG7`}i=7NI;N%EahqeMtU&=Tz19Ag0*o@{h=8 zo4dJ;tQ@3n2Ko17-3L((F**<=t@=>zcLe9671jyR9$qekaWN(AgKDcIU(D^d?W=w! zx|j+8Q~~8PD(7PP+82lx*mFWmcG@a+k%Upna9>J3~Aw z_X%v50#}q4=!WH0NfIAkQ_*$hU12qN>?e8#eU%W0I~?urQFm8)bN8)_i=+QM8Nk_y zn4Hb_6}BJY%d8A7%d0QZ_N2AZD=7^QxcnYCe0+!ETB^C>PB4yXPPtlRCC?V`G zmlfSLV};mz>rD%{zie7$mpumR{1f_z1R6enBFwkTdD^C{T}I=Pm$X(SYxQ`A#$ixd1Z z${e~T=)NAPdO>|jD}v59h#H5^T(TzcZ4^SiU75D^n?VO4H{>C(WtD^+>eW9b00(C9 z1{&a*H0)_9iu|djs^LG~MvS-(-<<|TL83DIMmV%TXiI7WiTQoHZeN|SwT^ekLCk-b zt^6usq2+)Juv~_VlQX+N{yN}qys?TRq}?HS=28rP0L()gaur3I^MAypL)_3=L;^id z!SuUr+cd)tKxN3o73dCAM#RV8#fcvT{XBzZPQK(>&FYZROSVg#Dqn9I(m|EXT7Y9< zFW_0eBx?n?wCjUrQbNl`jMN;k$J^tTe1fJl8NSn;YDAIU zA{!DTCK$Om3lyXqdVi5M>s|pwu)ci7;Yki$)w-*Do(j;&MreQlEuVf%UP2Y__CA;k9#L8;kzp}b5+eb(mUPqG{hsV<~D zNIl~a6GBLzwh8rtnAx-F8RH8z6p+EHM*%41#rpC{peGf zmOy6;YOfa%lD@zgm-C(XT2A#?ICFb*vmVe5lWlN_jgwE-C*gVgSZdez*r}&!fKh1KdMZUsUcPcf`5$5@m2?x$^YKopuwiKI$}p?;l0* z{6(g?Ptq7i;ya*NH6LJqN%aO~M~+a!ylXPUpqu)GZ?RkUu@^^HihUP0cuXj`aEAA$ z^W7*dUBd9j@5vW|f=Wg`C$g^&Q214Vh%ZnCZ8$x2$qbMHj-g1>X0L&P`9(&&DHN2l zBZQn=Z~H~Yo=#$n`__BN+_3c$NUb9DKWW|J1QGb4!&=uJ> z!*8&qd8X(qM>RhH;SzLIQ$A=d2FXnLs-l3#qpE>k$i}-pb-8NCW13J&5Y`0UOu0q$ zETA_mJ;;F$*9iIDOq{TW-J1c$JK>7-XP!PI1m#6EHzTE@;&iHkbErk=_AfHAsI(B7 z0V$0NZ0jqxP3AFb_2FTF$ZeCwNKD5+^JPV++oK3BSQl&qZEB@@1;!)RC*l#3#+PFE zMh#6Yf{q4Nl3BC{aNSi6e&4!&7pI6cKC27>p-TYhLZv@MF&ax>F>CPT>cN0YN<7;WD)z|AE>v=sBLnD*CqH-qh5Jr5xhFqP59#iUesHTGg zK>YTj534zAJhL7|4R6Iz0)0JpbqId+v!?rvh3;#kWy7KTq>W{t5d{Fs4w0rYe z;Ba}OHU?QP2N$SyO7te5yCHL#Bz6Fl7|5B{5-`E*YjRA24HSW@6&H;cYorcT7gT%$ zYr&0P{Diocz48GHZsV?f?(O$x8ot>S_>71(get^HbH!W#Tll%7bB%lXTn<%$lIUKd zyn^{xA~Pt$zA!zL5vBN%kt?klBu*e&w0X51yOaE?;eJzF;!{XWBrt6KRSj>!obtoa z)e2Gve+EGS0KIhgk3DY#gjQ2}y11jM?&Mj+`fHW;Fxsbayw86a-Csd8UF5%CTC3oQ zNxm{9@XjjpOAMp;`>T(V*ET!Wd>d3z2%lC^JI%ppeK5gTrW@;1~=A7C425|p0z z5#}wZgV!?@<(&fo)~p>wP@~ji1_>L@~fa-9P%AHb9Dg)c$Bp77a%c>B*j0|o_FZ|p!j*fvLAD= zAzD8l^WKL6;DZa;kgAo533XWgu*-`$v%UglWB4Ag?Ks4+hNMGl?|=0&zDC)? z8W5DKt-94nw9i4O)N^>uc^GXxkX`T2G7+}s6Mf>F_-aEW3}wFTzbUCdoDiO8^3z1w znd0~-`0>}io`UNhgIRHp400SdEJ{j~SFFbvMZQEJ$XF$~qqw!cM9HM_c8al+$#;9~ zH>2O`VTMtuO9N-0xP&IYKPsTc-Q18@2OI6qs?a^$sVYjZ!TJOT2YG5|{nBpmyfKLSqftE6=}ND5g=-@7n1A8G701 z#GX+sqSzDEnJSMTVX4Q^qPAN zW7ifMcnn|2ijpbiKVn)XSqC-dT!yiX-}`o~8y#~Pt3V7!zeJxUNXWFzvd&tN%Dabr zQ{Vj}yJ|m?V`VkfcmNcv?rqRueFeUj5H8w)KiGt~VPt<}wD|#q3YnH^HO*w}%5Z9o zY{Q-rLm&@9bO(Z@^p9*r7r3tayJwD?hg>0NhzK#)Y=^*gdVw-WE*tQvy91Pfk$E12sTkm|i@X9+^lTelhsOL-6q&N2Cyf!as(vuiRfE^{`)9%dqWf4ck}!3qTPA#FYhC|2C^Z}@>snKgi% z!I}1kx(O;NXDh0X^0bj;8rDGr=q%sh($b30rGfp!*F(-c;^NvN(;WFMW#+zqy@aT? z1ngF0|IJArnQxMpWmFnI;WWp?(U%z zU`F;%9(C*u0QsLnsvfx-YCD{RMM<6ycVN}yX#Js|%lY1y(aIxQMIHIAqS|6sOy%Vz zqM)?S!E3l+=cPum7QXk9EcrKA&P<0KPS#S))CU|#&$QMCi#X*7eH7`tC*^zqRLj2p zmL=wz@ue-f?3-P$sm3M61&yb04ZFXW^uva0$p@^&tFf%ge!<)p_Oaf}UbjCr#aqS(_ZZvlUx|rv7 zb9Vy@GQuVDcTQZ=4Oy8;KG$g{Dk}Ua;uETiLAMI4A$0cb!tx&<@;FaV-FPVwpFU`n#H4CLL^-MkC+23brOGIJed&rc592+xVU8=i?%T8m=22rK{({> z6EE_6d-PF%kb?Pac_kcP!pi1*oF3#qrERE0uT+^br~E%Mvz$N7%$zu6zRxC^NBYx$ z`P{zC&Styq0>z?r`s&XBD>Jqf*s!=r>Sky&7)}|pR^8!v#lP8kJrHe>n#3YBt$Mh^ z)JD9AaXkh%Z7X?F>Wi8BvtQmi?}w~K#1#QH4wC6P#>+--EyrVPew2cXBVOK^Zcbi2 z6DD}Ks_fjyyp&j$(nvNser#E}skGCBC4}rI?Y0g;YkH8QK<0LGFn4P+Nm(AeG+%FAbVCA2|NX z1KfJ;JEI323mFRME=uZ&t<1R|x&iIVC=!e(Y;-)~T~)Q!(L5Bl)?kmJG2$p8P%uiU zB%*M`!Gm7e!M*je2ktMAC5IpSdU-`V2wUui9qAuRSU zjX5?22-u}e<{DVtTPlt2!R?N7DXouGN zO(yOs=1^p(<#14cMr+`hoAx6N%A77Vk(-E1s`tu=0_;ac_i~v=dL_H%x zVFw{y7CN#wX~xR{rn3k|FN}jeaqcb+0u(RCT$T5XTugsKeL!dYKBDz9^+vdPUVFa0 zxIRwLN=9+$ug)%=g2UwY`2Yl#$evkJEgZl-?(cgYOkpD`tWpVeRt@i!DRJVQ_Rn+F zw9oE<1c3%01PKT6S1PHtK=z*1NfH+=SymkxJ$~?B7q)IHOG2(AH@5)m5$mUB;<$yF zOY%MFrv~)9&Qg)RM5-5d96iqW*Ao(AQ=KyHDsJz?%FlUpRvo_NT+H+-^mg6HV3=S0 zA}dJNi>WVo0Yq|=48h-dChPLcoW0Q}lG9Fud!2fPdRT44tF$hd9hl7 z=YkPxq@N6<@KN+g*#&k zN0s`q@}?*~$lM?&UrloFw{3}nz?%zE{I2hjonqV}i4C+rbLEZ1j{_J}6phggOMCA0 zP7*{e%cG@?V%UE1M_&oA;f>M52}c1zuH?%{C3ms!NnDlc-kQ-()qZxPZsV~pEi^oz zL`X4w)8No`FV=qEQpFts(=z6ZTCTs|Z?)CW>SmBEyJT?`_(&bwR{2fP+fSZ~oUJ!L zdtjGT8v!KYpO*e-_HtrB{pz23ESm7J0HDze({k1@{^k96;c?!Bbf*=0RpfAm?WcHU zwHKWhO*;r1ZGtX(+|DTD)J)Ig>s(x-UIA`A7?!Jt+IFKV*?8mT(x(++<<&lSRa&I^ z(%*%ycz)->;POFisbez&GCqhS^_1vh0O!fiB}mZ6O&AW?F-ZNXCgH{4_+@APp>(=OP;On zd>qdoc~M~W>dlA`n8C9`DW$9PVT(&l#nO?_?+PBk{wNZQyhXF4@7vXgEXs4pQ_*KgodtoV48%^jnf(-o*oY0i<)@&+%rdCa9o6ShZTh5+cV<* z6vC*>^5SX{^UUd#nWKYs>cGtSYYd-(ZubTyFGL3Y2P7u66x?p+Mb0};2KB7tYFf{? zB&utxeZR2)LE=}U#<1w{f{lwaqwU|sZEkYAJLliryOzIn|K|7%sXKE5z*fP*ING?- zg1H7D=}0y-HQ~2F{Ygf#*OE-)wgT8e{6FtCG%sfs0!SS%h$H~-$AFw!m6FVw0~ zDMYXOw`oZ|GL^i#apH+`)Gz{ac7ktqoefblrXI!|vL(Uo1xk!G`;L`zvqzr)*-|7{ zo>4=b*$XKhfqc47mdgdM|+@~%7X+^?9 zuEg>xje7ryIoWMA{P1ZkPe_rqD?uNIc}8;-W+sX?D?xTmg|W-YI(`PYW*Xd z_q1AYtj@vB8};wgLKEal;#7Ly-*UUMeq#-0;U-bA;=A+g(J?)`4^!bIPY43IjjW`u z!Mz(<`S0CW48N|nS#zd@yFgip@=kUxc7fNv~e1^n9AWCX#8o zZ*yis(^K$pV#HBWB=#YRX=k2s;|0=2;Qa(D*%cX3e`AQ_;c&kB+12ZvkAIO(fm3>! zR(=0OcGtb6Lfd`7VjoWp{;7&XVJ54qd8>Y?Qs0lR9{LSuuF8FDt%jl`Xnj|YF5AD( zY=4`V>spMezWv;Qv#RI{iOB}YvX)p<2$w)We1_lE(pY4ANGlVy5N-3ARc9d$+%)Y_>0)b19#lN(;Ewt^5hp$+T(hs4apV0RQ#1m=Rx*fz<4c4dO zScSpE=)n{heclW=?Qa8_xT|?yMM-ppStbTOhT2xG^D7zbft=H?RQrpx6!O*b#Ud&P z78B;a7c4Q$gkye=4vrg+^M=^=nxt2+^Z55oN6?@8A|B4QG}(yoO};B)f_|P2`=`$ZkFfSgVtGPWfdjm59|#r0?)Kw zlp5sA**g{A%`U4uVZ71M9t*IWU4iiM4{?6`oi(xZw6wnSxO^vm)DcX-vv@dD%hsdo zw|CaWN?ZAf{OfLn!i)eL^n!=)_Jel;+XFOh1cMjZxU2eIUa?8;87r3zCYu2NurWi} zHs`0qA#h7q>UAwx>h24f_K>BSOj;184muE|2 zp2RU=IV`7~J))8kpRGBvR@DjmdJ0LSuA?(3_n=UI{yQLIVxhXL?L5MVf<4`H3Y?@s z(kr^Dt5NU0|6T@Oox8RIos>V8@fv#O(ILeAcHsf6Z-?)9EmHB%N$07P*H0&7!26FM zgU>z>u(aS7Nvsf-%aSExMlAReenURoNlQUA%(C#htA}qc&c>kPrGwSFuJsvKNY@k3gZ~iCyQjJv~HllHBP%{2>pyK!J r8VkSj@3WVJg7QCKi*E9hYt}p&gOhoCG6zoPz{wo=+jD^W*U$e36F{y) literal 0 HcmV?d00001 diff --git a/ocrd/ocrd/web/processing_broker/processing_broker.py b/ocrd/ocrd/network/processing_broker.py similarity index 98% rename from ocrd/ocrd/web/processing_broker/processing_broker.py rename to ocrd/ocrd/network/processing_broker.py index 8bc249457..e9784f7d9 100644 --- a/ocrd/ocrd/web/processing_broker/processing_broker.py +++ b/ocrd/ocrd/network/processing_broker.py @@ -1,6 +1,6 @@ import uvicorn from fastapi import FastAPI -from .deployer import Deployer +from ocrd.network.deployer import Deployer from ocrd_utils import getLogger import yaml from jsonschema import validate, ValidationError diff --git a/ocrd/ocrd/web/processing_broker/processing_worker.py b/ocrd/ocrd/network/processing_worker.py similarity index 100% rename from ocrd/ocrd/web/processing_broker/processing_worker.py rename to ocrd/ocrd/network/processing_worker.py diff --git a/ocrd/ocrd/network/rabbitmq_utils/__init__.py b/ocrd/ocrd/network/rabbitmq_utils/__init__.py new file mode 100644 index 000000000..506e33044 --- /dev/null +++ b/ocrd/ocrd/network/rabbitmq_utils/__init__.py @@ -0,0 +1,2 @@ +# This rabbitmq_utils package is supposed to contain the code that is currently available under: +# https://github.com/OCR-D/ocrd-webapi-implementation/tree/main/ocrd_webapi/rabbitmq diff --git a/ocrd/ocrd/network/web_api/__init__.py b/ocrd/ocrd/network/web_api/__init__.py new file mode 100644 index 000000000..f3265b436 --- /dev/null +++ b/ocrd/ocrd/network/web_api/__init__.py @@ -0,0 +1,2 @@ +# This web_api package is supposed to contain the code that is currently available under: +# https://github.com/OCR-D/ocrd-webapi-implementation diff --git a/ocrd/ocrd/web/__init__.py b/ocrd/ocrd/web/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/ocrd/ocrd/web/processing_broker/__init__.py b/ocrd/ocrd/web/processing_broker/__init__.py deleted file mode 100644 index 30ffa35e6..000000000 --- a/ocrd/ocrd/web/processing_broker/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .processing_broker import ProcessingBroker From 76c40a321124e8d99fb6a3abdf7daf591643c183 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Tue, 3 Jan 2023 15:51:03 +0100 Subject: [PATCH 019/226] Add example broker configuration --- ocrd/ocrd/network/example-broker-config.yml | 33 +++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 ocrd/ocrd/network/example-broker-config.yml diff --git a/ocrd/ocrd/network/example-broker-config.yml b/ocrd/ocrd/network/example-broker-config.yml new file mode 100644 index 000000000..265d1b241 --- /dev/null +++ b/ocrd/ocrd/network/example-broker-config.yml @@ -0,0 +1,33 @@ +message_queue: + address: localhost + port: 5672 + credentials: + username: admin + password: admin + ssh: + username: mm + path_to_privkey: /home/mm/.ssh/cloud.key +mongo_db: + address: localhost + port: 27018 + credentials: + username: admin + password: admin + ssh: + username: mm + path_to_privkey: /home/mm/.ssh/cloud.key +hosts: + - address: localhost + username: mm + path_to_privkey: /home/mm/.ssh/cloud.key + deploy_processors: + - name: ocrd-cis-ocropy-binarize + number_of_instance: 1 + deploy_type: native + - address: localhost + username: mm + path_to_privkey: /home/mm/.ssh/cloud.key + deploy_processors: + - name: ocrd-gibt-es-nicht-test + number_of_instance: 1 + deploy_type: docker From 3e4e76c89d1bdb8b56c4f4dbc9cfb64bf7db28e1 Mon Sep 17 00:00:00 2001 From: joschrew Date: Fri, 6 Jan 2023 13:47:07 +0100 Subject: [PATCH 020/226] Change logger names --- ocrd/ocrd/network/deployer.py | 2 +- ocrd/ocrd/network/deployment_utils.py | 2 +- ocrd/ocrd/network/processing_broker.py | 6 +++--- ocrd/ocrd/network/processing_worker.py | 12 ++++++------ 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index 7b849f7e7..16380a505 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -34,7 +34,7 @@ def __init__(self, config): Args: config (Config): values from config file wrapped into class `Config` """ - self.log = getLogger("ocrd.processingbroker") + self.log = getLogger(__name__) self.log.debug("Deployer-init()") self.mongo_data = MongoData(config["mongo_db"]) self.mq_data = QueueData(config["message_queue"]) diff --git a/ocrd/ocrd/network/deployment_utils.py b/ocrd/ocrd/network/deployment_utils.py index 485790231..2d43ae462 100644 --- a/ocrd/ocrd/network/deployment_utils.py +++ b/ocrd/ocrd/network/deployment_utils.py @@ -52,7 +52,7 @@ def create_ssh_client(obj): client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy) - log = getLogger("ocrd.create_ssh_client") + log = getLogger(__name__) log.debug(f"creating ssh-client with username: '{username}', keypath: '{keypath}'. " f"host: {address}") # TODO: connecting could easily fail here: wrong password, wrong path to keyfile etc. Maybe diff --git a/ocrd/ocrd/network/processing_broker.py b/ocrd/ocrd/network/processing_broker.py index e9784f7d9..60af2c250 100644 --- a/ocrd/ocrd/network/processing_broker.py +++ b/ocrd/ocrd/network/processing_broker.py @@ -20,7 +20,7 @@ def __init__(self, config_path): self.deployer = Deployer(self.config) # Deploy everything specified in the configuration self.deployer.deploy_all() - self.log = getLogger("ocrd.processingbroker") + self.log = getLogger(__name__) # RMQPublisher object must be created here, reference: RabbitMQ Library (WebAPI Implementation) # Based on the API calls the ProcessingBroker will send messages to the running instance @@ -39,7 +39,7 @@ def __init__(self, config_path): """ Publish messages based on the API calls Here is a call example to be adopted later - + # The message type is bytes # Call this method to publish a message self.rmq_publisher.publish_to_queue(queue_name="queue_name", message="message") @@ -91,7 +91,7 @@ def configure_publisher(config_file): rmq_publisher = "RMQPublisher Object" """ Here is a template implementation to be adopted later - + rmq_publisher = RMQPublisher(host="localhost", port=5672, vhost="/") # The credentials are configured inside definitions.json # when building the RabbitMQ docker image diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index 4b0f2c2ef..f958b194a 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -14,7 +14,7 @@ class ProcessingWorker: def __init__(self, processor_arguments, queue_address, database_address): - self.log = getLogger("ocrd.processing_worker") + self.log = getLogger(__name__) # Required arguments to run the OCR-D Processor self.processor_arguments = processor_arguments # processor.name is @@ -36,7 +36,7 @@ def configure_consumer(config_file, callback_method): rmq_consumer = "RMQConsumer Object" """ Here is a template implementation to be adopted later - + rmq_consumer = RMQConsumer(host="localhost", port=5672, vhost="/") # The credentials are configured inside definitions.json # when building the RabbitMQ docker image @@ -44,10 +44,10 @@ def configure_consumer(config_file, callback_method): username="default-consumer", password="default-consumer" ) - + #Note: The queue name here is the processor.name by definition rmq_consumer.configure_consuming(queue_name="queue_name", callback_method=funcPtr) - + """ return rmq_consumer @@ -64,7 +64,7 @@ def start_consuming(self): @staticmethod def start_native_processor(client, name, _queue_address, _database_address): - log = getLogger("ocrd.processing_worker.start_native") + log = getLogger(__name__) log.debug(f"start native processor: {name}") channel = client.invoke_shell() stdin, stdout = channel.makefile('wb'), channel.makefile('rb') @@ -81,7 +81,7 @@ def start_native_processor(client, name, _queue_address, _database_address): @staticmethod def start_docker_processor(client, name, _queue_address, _database_address): - log = getLogger("ocrd.processing_worker.start_docker") + log = getLogger(__name__) log.debug(f"start docker processor: {name}") # TODO: add real command here to start processing server here res = client.containers.run("debian", "sleep 31", detach=True, remove=True) From f1c27f3c8f22737e202867e04339c89e53ab8678 Mon Sep 17 00:00:00 2001 From: joschrew Date: Fri, 6 Jan 2023 14:30:24 +0100 Subject: [PATCH 021/226] Replace double quoted strings with single quoted We decided to use only single quotes for strings to make it consistent. I kept docstrings in triple double quotes because of PEP 257. --- ocrd/ocrd/network/deployer.py | 94 +++++++++++++------------- ocrd/ocrd/network/deployment_utils.py | 36 +++++----- ocrd/ocrd/network/processing_broker.py | 18 ++--- ocrd/ocrd/network/processing_worker.py | 26 +++---- 4 files changed, 87 insertions(+), 87 deletions(-) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index 16380a505..833770a32 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -35,9 +35,9 @@ def __init__(self, config): config (Config): values from config file wrapped into class `Config` """ self.log = getLogger(__name__) - self.log.debug("Deployer-init()") - self.mongo_data = MongoData(config["mongo_db"]) - self.mq_data = QueueData(config["message_queue"]) + self.log.debug('Deployer-init()') + self.mongo_data = MongoData(config['mongo_db']) + self.mq_data = QueueData(config['message_queue']) self.hosts = HostData.from_config(config) def deploy_all(self): @@ -65,9 +65,9 @@ def _deploy_processing_workers(self, hosts, rabbitmq_address, mongodb_address): close_clients(host) def _deploy_processing_worker(self, processor, host, deploy_type, rabbitmq_server=None, mongodb=None): - self.log.debug(f"deploy '{deploy_type}' processor: '{processor}' on '{host.address}'") - assert not processor.pids, "processors already deployed. Pids are present. Host: " \ - "{host.__dict__}. Processor: {processor.__dict__}" + self.log.debug(f'deploy "{deploy_type}" processor: "{processor}" on "{host.address}"') + assert not processor.pids, 'processors already deployed. Pids are present. Host: ' \ + '{host.__dict__}. Processor: {processor.__dict__}' # Create the specific RabbitMQ queue here based on the OCR-D processor name (processor.name) # self.rmq_publisher.create_queue(queue_name=processor.name) @@ -99,7 +99,7 @@ def _deploy_processing_worker(self, processor, host, deploy_type, rabbitmq_serve _database_address=mongodb) processor.add_started_pid(pid) - def _deploy_queue(self, image="rabbitmq", detach=True, remove=True, ports=None): + def _deploy_queue(self, image='rabbitmq', detach=True, remove=True, ports=None): # This method deploys the RabbitMQ Server. # Handling of creation of queues, submitting messages to queues, # and receiving messages from queues is part of the RabbitMQ Library @@ -123,19 +123,19 @@ def _deploy_queue(self, image="rabbitmq", detach=True, remove=True, ports=None): remove=remove, ports=ports ) - assert res and res.id, "starting message queue failed" + assert res and res.id, 'starting message queue failed' self.mq_data.pid = res.id client.close() - self.log.debug("deployed queue") + self.log.debug('deployed queue') # Not implemented yet # Note: The queue address is not just the IP address - queue_address = "RabbitMQ Server address" + queue_address = 'RabbitMQ Server address' return queue_address - def _deploy_mongodb(self, image="mongo", detach=True, remove=True, ports=None): + def _deploy_mongodb(self, image='mongo', detach=True, remove=True, ports=None): if not self.mongo_data or not self.mongo_data.address: - self.log.debug("canceled mongo-deploy: no mongo_db in config") + self.log.debug('canceled mongo-deploy: no mongo_db in config') return client = create_docker_client(self.mongo_data) if ports is None: @@ -150,41 +150,41 @@ def _deploy_mongodb(self, image="mongo", detach=True, remove=True, ports=None): remove=remove, ports=ports ) - assert res and res.id, "starting mongodb failed" + assert res and res.id, 'starting mongodb failed' self.mongo_data.pid = res.id client.close() - self.log.debug("deployed mongodb") + self.log.debug('deployed mongodb') # Not implemented yet # Note: The mongodb address is not just the IP address - mongodb_address = "MongoDB Address" + mongodb_address = 'MongoDB Address' return mongodb_address def _kill_queue(self): if not self.mq_data.pid: - self.log.debug("kill_queue: queue not running") + self.log.debug('kill_queue: queue not running') return else: - self.log.debug(f"trying to kill queue with id: {self.mq_data.pid} now") + self.log.debug(f'trying to kill queue with id: {self.mq_data.pid} now') client = create_docker_client(self.mq_data) client.containers.get(self.mq_data.pid).stop() self.mq_data.pid = None client.close() - self.log.debug("stopped queue") + self.log.debug('stopped queue') def _kill_mongodb(self): if not self.mongo_data or not self.mongo_data.pid: - self.log.debug("kill_mongdb: mongodb not running") + self.log.debug('kill_mongdb: mongodb not running') return else: - self.log.debug(f"trying to kill mongdb with id: {self.mongo_data.pid} now") + self.log.debug(f'trying to kill mongdb with id: {self.mongo_data.pid} now') client = create_docker_client(self.mongo_data) client.containers.get(self.mongo_data.pid).stop() self.mongo_data.pid = None client.close() - self.log.debug("stopped mongodb") + self.log.debug('stopped mongodb') def _kill_processing_workers(self): for host in self.hosts: @@ -194,11 +194,11 @@ def _kill_processing_workers(self): host.docker_client = create_docker_client(host) for p in host.processors_native: for pid in p.pids: - host.ssh_client.exec_command(f"kill {pid}") + host.ssh_client.exec_command(f'kill {pid}') p.pids = [] for p in host.processors_docker: for pid in p.pids: - self.log.debug(f"trying to kill docker container: {pid}") + self.log.debug(f'trying to kill docker container: {pid}') # TODO: think about timeout. # think about using threads to kill parallelized to reduce waiting time host.docker_client.containers.get(pid).stop() @@ -223,31 +223,31 @@ class HostData: """ def __init__(self, config): - self.address = config["address"] - self.username = config["username"] - self.password = config.get("password", None) - self.keypath = config.get("path_to_privkey", None) - assert self.password or self.keypath, "Host in configfile with neither password nor keyfile" + self.address = config['address'] + self.username = config['username'] + self.password = config.get('password', None) + self.keypath = config.get('path_to_privkey', None) + assert self.password or self.keypath, 'Host in configfile with neither password nor keyfile' self.processors_native = [] self.processors_docker = [] - for x in config["deploy_processors"]: - if x["deploy_type"] == 'native': + for x in config['deploy_processors']: + if x['deploy_type'] == 'native': self.processors_native.append( - self.Processor(x["name"], x["number_of_instance"], DeployType.native) + self.Processor(x['name'], x['number_of_instance'], DeployType.native) ) - elif x["deploy_type"] == 'docker': + elif x['deploy_type'] == 'docker': self.processors_docker.append( - self.Processor(x["name"], x["number_of_instance"], DeployType.docker) + self.Processor(x['name'], x['number_of_instance'], DeployType.docker) ) else: - assert False, f"unknown deploy_type: '{x.deploy_type}'" + assert False, f'unknown deploy_type: "{x.deploy_type}"' self.ssh_client = None self.docker_client = None @classmethod def from_config(cls, config): res = [] - for x in config["hosts"]: + for x in config['hosts']: res.append(cls(x)) return res @@ -267,12 +267,12 @@ class MongoData: """ def __init__(self, config): - self.address = config["address"] - self.port = int(config["port"]) - self.username = config["ssh"]["username"] - self.keypath = config["ssh"].get("path_to_privkey", None) - self.password = config["ssh"].get("password", None) - self.credentials = (config["credentials"]["username"], config["credentials"]["password"]) + self.address = config['address'] + self.port = int(config['port']) + self.username = config['ssh']['username'] + self.keypath = config['ssh'].get('path_to_privkey', None) + self.password = config['ssh'].get('password', None) + self.credentials = (config['credentials']['username'], config['credentials']['password']) self.pid = None @@ -281,12 +281,12 @@ class QueueData: """ def __init__(self, config): - self.address = config["address"] - self.port = int(config["port"]) - self.username = config["ssh"]["username"] - self.keypath = config["ssh"].get("path_to_privkey", None) - self.password = config["ssh"].get("password", None) - self.credentials = (config["credentials"]["username"], config["credentials"]["password"]) + self.address = config['address'] + self.port = int(config['port']) + self.username = config['ssh']['username'] + self.keypath = config['ssh'].get('path_to_privkey', None) + self.password = config['ssh'].get('password', None) + self.credentials = (config['credentials']['username'], config['credentials']['password']) self.pid = None diff --git a/ocrd/ocrd/network/deployment_utils.py b/ocrd/ocrd/network/deployment_utils.py index 2d43ae462..1ba265570 100644 --- a/ocrd/ocrd/network/deployment_utils.py +++ b/ocrd/ocrd/network/deployment_utils.py @@ -47,14 +47,14 @@ def get_processor(parameter: dict, processor_class=None): def create_ssh_client(obj): address, username, password, keypath = obj.address, obj.username, obj.password, obj.keypath - assert address and username, "address and username are mandatory" - assert bool(password) is not bool(keypath), "expecting either password or keypath, not both" + assert address and username, 'address and username are mandatory' + assert bool(password) is not bool(keypath), 'expecting either password or keypath, not both' client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy) log = getLogger(__name__) - log.debug(f"creating ssh-client with username: '{username}', keypath: '{keypath}'. " - f"host: {address}") + log.debug(f'creating ssh-client with username: "{username}", keypath: "{keypath}". ' + f'host: {address}') # TODO: connecting could easily fail here: wrong password, wrong path to keyfile etc. Maybe # would be better to use except and try to give custom error message when failing client.connect(hostname=address, username=username, password=password, key_filename=keypath) @@ -63,15 +63,15 @@ def create_ssh_client(obj): def create_docker_client(obj): address, username, password, keypath = obj.address, obj.username, obj.password, obj.keypath - assert address and username, "address and username are mandatory" - assert bool(password) is not bool(keypath), "expecting either password or keypath " \ - "provided, not both" + assert address and username, 'address and username are mandatory' + assert bool(password) is not bool(keypath), 'expecting either password or keypath ' \ + 'provided, not both' return CustomDockerClient(username, address, password=password, keypath=keypath) def close_clients(*args): for client in args: - if hasattr(client, "close") and callable(client.close): + if hasattr(client, 'close') and callable(client.close): client.close() @@ -89,10 +89,10 @@ class CustomDockerClient(docker.DockerClient): """ def __init__(self, user, host, **kwargs): - assert user and host, "user and host must be set" - assert "password" in kwargs or "keypath" in kwargs, "one of password and keyfile is needed" - self.api = docker.APIClient(f"ssh://{host}", use_ssh_client=True, version='1.41') - ssh_adapter = self.CustomSshHttpAdapter(f"ssh://{user}@{host}:22", **kwargs) + assert user and host, 'user and host must be set' + assert 'password' in kwargs or 'keypath' in kwargs, 'one of password and keyfile is needed' + self.api = docker.APIClient(f'ssh://{host}', use_ssh_client=True, version='1.41') + ssh_adapter = self.CustomSshHttpAdapter(f'ssh://{user}@{host}:22', **kwargs) self.api.mount('http+docker://ssh', ssh_adapter) class CustomSshHttpAdapter(SSHHTTPAdapter): @@ -100,7 +100,7 @@ def __init__(self, base_url, password=None, keypath=None): self.password = password self.keypath = keypath if not self.password and not self.keypath: - raise Exception("either 'password' or 'keypath' must be provided") + raise Exception('either "password" or "keypath" must be provided') super().__init__(base_url) def _create_paramiko_client(self, base_url): @@ -111,12 +111,12 @@ def _create_paramiko_client(self, base_url): self.ssh_client = paramiko.SSHClient() base_url = urllib.parse.urlparse(base_url) self.ssh_params = { - "hostname": base_url.hostname, - "port": base_url.port, - "username": base_url.username, + 'hostname': base_url.hostname, + 'port': base_url.port, + 'username': base_url.username, } if self.password: - self.ssh_params["password"] = self.password + self.ssh_params['password'] = self.password elif self.keypath: - self.ssh_params["key_filename"] = self.keypath + self.ssh_params['key_filename'] = self.keypath self.ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy) diff --git a/ocrd/ocrd/network/processing_broker.py b/ocrd/ocrd/network/processing_broker.py index 60af2c250..7bd10e590 100644 --- a/ocrd/ocrd/network/processing_broker.py +++ b/ocrd/ocrd/network/processing_broker.py @@ -42,20 +42,20 @@ def __init__(self, config_path): # The message type is bytes # Call this method to publish a message - self.rmq_publisher.publish_to_queue(queue_name="queue_name", message="message") + self.rmq_publisher.publish_to_queue(queue_name='queue_name', message='message') """ def start(self): """ start processing broker with uvicorn """ - assert self.config, "config was not parsed correctly" + assert self.config, 'config was not parsed correctly' # TODO: change where to run the processing server: default params, read from config or read # from cmd? Or do not run at all as fastapi? # TODO: activate next line again (commented just for testing) port = 5050 host = 'localhost' - self.log.debug(f"starting uvicorn. Host: {host}. Port: {port}") + self.log.debug(f'starting uvicorn. Host: {host}. Port: {port}') uvicorn.run(self, host=host, port=port) @staticmethod @@ -67,7 +67,7 @@ def validate_config(config_path): try: validate(obj, schema) except ValidationError as e: - return f"{e.message}. At {e.json_path}" + return f'{e.message}. At {e.json_path}' return None async def on_shutdown(self): @@ -80,7 +80,7 @@ async def on_shutdown(self): try: await self.stop_deployed_agents() except: - self.log.debug("error stopping processing servers: ", exc_info=True) + self.log.debug('error stopping processing servers: ', exc_info=True) raise async def stop_deployed_agents(self): @@ -88,16 +88,16 @@ async def stop_deployed_agents(self): @staticmethod def configure_publisher(config_file): - rmq_publisher = "RMQPublisher Object" + rmq_publisher = 'RMQPublisher Object' """ Here is a template implementation to be adopted later - rmq_publisher = RMQPublisher(host="localhost", port=5672, vhost="/") + rmq_publisher = RMQPublisher(host='localhost', port=5672, vhost='/') # The credentials are configured inside definitions.json # when building the RabbitMQ docker image rmq_publisher.authenticate_and_connect( - username="default-publisher", - password="default-publisher" + username='default-publisher', + password='default-publisher' ) rmq_publisher.enable_delivery_confirmations() """ diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index f958b194a..04a6a2363 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -33,20 +33,20 @@ def __init__(self, processor_arguments, queue_address, database_address): @staticmethod def configure_consumer(config_file, callback_method): - rmq_consumer = "RMQConsumer Object" + rmq_consumer = 'RMQConsumer Object' """ Here is a template implementation to be adopted later - rmq_consumer = RMQConsumer(host="localhost", port=5672, vhost="/") + rmq_consumer = RMQConsumer(host='localhost', port=5672, vhost='/') # The credentials are configured inside definitions.json # when building the RabbitMQ docker image rmq_consumer.authenticate_and_connect( - username="default-consumer", - password="default-consumer" + username='default-consumer', + password='default-consumer' ) #Note: The queue name here is the processor.name by definition - rmq_consumer.configure_consuming(queue_name="queue_name", callback_method=funcPtr) + rmq_consumer.configure_consuming(queue_name='queue_name', callback_method=funcPtr) """ return rmq_consumer @@ -65,25 +65,25 @@ def start_consuming(self): @staticmethod def start_native_processor(client, name, _queue_address, _database_address): log = getLogger(__name__) - log.debug(f"start native processor: {name}") + log.debug(f'start native processor: {name}') channel = client.invoke_shell() stdin, stdout = channel.makefile('wb'), channel.makefile('rb') # TODO: add real command here to start processing server here - cmd = "sleep 23s" - stdin.write(f"{cmd} & \n echo xyz$!xyz \n exit \n") - output = stdout.read().decode("utf-8") + cmd = 'sleep 23s' + stdin.write(f'{cmd} & \n echo xyz$!xyz \n exit \n') + output = stdout.read().decode('utf-8') stdout.close() stdin.close() # What does this return and is supposed to return? # Putting some comments when using patterns is always appreciated # Since the docker version returns PID, this should also return PID for consistency - return re.search(r"xyz([0-9]+)xyz", output).group(1) + return re.search(r'xyz([0-9]+)xyz', output).group(1) @staticmethod def start_docker_processor(client, name, _queue_address, _database_address): log = getLogger(__name__) - log.debug(f"start docker processor: {name}") + log.debug(f'start docker processor: {name}') # TODO: add real command here to start processing server here - res = client.containers.run("debian", "sleep 31", detach=True, remove=True) - assert res and res.id, "run docker container failed" + res = client.containers.run('debian', 'sleep 31', detach=True, remove=True) + assert res and res.id, 'run docker container failed' return res.id From 4fcaeb5b7a2df5ba1de33a50e8ab1b8ba5cee920 Mon Sep 17 00:00:00 2001 From: joschrew Date: Mon, 9 Jan 2023 08:21:04 +0100 Subject: [PATCH 022/226] Add typehints for network files --- ocrd/ocrd/network/deployer.py | 53 +++++++++++++++----------- ocrd/ocrd/network/deployment_utils.py | 29 +++++++------- ocrd/ocrd/network/processing_broker.py | 14 ++++--- ocrd/ocrd/network/processing_worker.py | 23 +++++++---- 4 files changed, 71 insertions(+), 48 deletions(-) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index 833770a32..a6662a8f1 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -1,3 +1,4 @@ +from __future__ import annotations from enum import Enum from ocrd_utils import ( getLogger @@ -8,6 +9,8 @@ create_ssh_client ) from ocrd.network.processing_worker import ProcessingWorker +from typing import List, Dict, Union + # Abstraction of the Deployment functionality # The Deployer agent is in the middle between @@ -29,7 +32,7 @@ class Deployer: for managing information, not for actually doing things. """ - def __init__(self, config): + def __init__(self, config: Dict) -> None: """ Args: config (Config): values from config file wrapped into class `Config` @@ -40,7 +43,7 @@ def __init__(self, config): self.mq_data = QueueData(config['message_queue']) self.hosts = HostData.from_config(config) - def deploy_all(self): + def deploy_all(self) -> None: """ Deploy the message queue and all processors defined in the config-file """ # Ideally, this should return the address of the RabbitMQ Server @@ -49,12 +52,13 @@ def deploy_all(self): mongodb_address = self._deploy_mongodb() self._deploy_processing_workers(self.hosts, rabbitmq_address, mongodb_address) - def kill_all(self): + def kill_all(self) -> None: self._kill_queue() self._kill_mongodb() self._kill_processing_workers() - def _deploy_processing_workers(self, hosts, rabbitmq_address, mongodb_address): + def _deploy_processing_workers(self, hosts: List[HostData], rabbitmq_address: str, + mongodb_address: str) -> None: for host in hosts: for p in host.processors_native: # Ideally, pass the rabbitmq server and mongodb addresses here @@ -64,7 +68,8 @@ def _deploy_processing_workers(self, hosts, rabbitmq_address, mongodb_address): self._deploy_processing_worker(p, host, DeployType.docker, rabbitmq_address, mongodb_address) close_clients(host) - def _deploy_processing_worker(self, processor, host, deploy_type, rabbitmq_server=None, mongodb=None): + def _deploy_processing_worker(self, processor, host: HostData, deploy_type: DeployType, + rabbitmq_server: str = '', mongodb: str = '') -> None: self.log.debug(f'deploy "{deploy_type}" processor: "{processor}" on "{host.address}"') assert not processor.pids, 'processors already deployed. Pids are present. Host: ' \ '{host.__dict__}. Processor: {processor.__dict__}' @@ -80,6 +85,7 @@ def _deploy_processing_worker(self, processor, host, deploy_type, rabbitmq_serve host.docker_client = create_docker_client(host) for _ in range(processor.count): if deploy_type == DeployType.native: + assert host.ssh_client # to satisfy mypy # This method should be rather part of the ProcessingWorker # The Processing Worker can just invoke a static method of ProcessingWorker # that creates an instance of the ProcessingWorker (Native instance) @@ -89,6 +95,7 @@ def _deploy_processing_worker(self, processor, host, deploy_type, rabbitmq_serve _queue_address=rabbitmq_server, _database_address=mongodb) else: + assert host.docker_client # to satisfy mypy # This method should be rather part of the ProcessingWorker # The Processing Worker can just invoke a static method of ProcessingWorker # that creates an instance of the ProcessingWorker (Docker instance) @@ -99,14 +106,15 @@ def _deploy_processing_worker(self, processor, host, deploy_type, rabbitmq_serve _database_address=mongodb) processor.add_started_pid(pid) - def _deploy_queue(self, image='rabbitmq', detach=True, remove=True, ports=None): + def _deploy_queue(self, image: str = 'rabbitmq', detach: bool = True, remove: bool = True, + ports: Union[Dict, None] = None) -> str: # This method deploys the RabbitMQ Server. # Handling of creation of queues, submitting messages to queues, # and receiving messages from queues is part of the RabbitMQ Library # Which is part of the OCR-D WebAPI implementation. client = create_docker_client(self.mq_data) - if ports is None: + if not ports: # 5672, 5671 - used by AMQP 0-9-1 and AMQP 1.0 clients without and with TLS # 15672, 15671: HTTP API clients, management UI and rabbitmqadmin, without and with TLS # 25672: used for internode and CLI tools communication and is allocated from @@ -133,12 +141,13 @@ def _deploy_queue(self, image='rabbitmq', detach=True, remove=True, ports=None): queue_address = 'RabbitMQ Server address' return queue_address - def _deploy_mongodb(self, image='mongo', detach=True, remove=True, ports=None): + def _deploy_mongodb(self, image: str = 'mongo', detach: bool = True, remove: bool = True, + ports: Union[Dict, None] = None) -> str: if not self.mongo_data or not self.mongo_data.address: self.log.debug('canceled mongo-deploy: no mongo_db in config') - return + return "" client = create_docker_client(self.mongo_data) - if ports is None: + if not ports: ports = { 27017: self.mongo_data.port } @@ -160,7 +169,7 @@ def _deploy_mongodb(self, image='mongo', detach=True, remove=True, ports=None): mongodb_address = 'MongoDB Address' return mongodb_address - def _kill_queue(self): + def _kill_queue(self) -> None: if not self.mq_data.pid: self.log.debug('kill_queue: queue not running') return @@ -173,7 +182,7 @@ def _kill_queue(self): client.close() self.log.debug('stopped queue') - def _kill_mongodb(self): + def _kill_mongodb(self) -> None: if not self.mongo_data or not self.mongo_data.pid: self.log.debug('kill_mongdb: mongodb not running') return @@ -186,7 +195,7 @@ def _kill_mongodb(self): client.close() self.log.debug('stopped mongodb') - def _kill_processing_workers(self): + def _kill_processing_workers(self) -> None: for host in self.hosts: if host.ssh_client: host.ssh_client = create_ssh_client(host) @@ -207,7 +216,7 @@ def _kill_processing_workers(self): # May be good to have more flexibility here # TODO: Support that functionality as well. # Then _kill_processing_workers should just call this method in a loop - def _kill_processing_worker(self): + def _kill_processing_worker(self) -> None: pass @@ -222,7 +231,7 @@ class HostData: be the class who does things and this class here should be mostly passive """ - def __init__(self, config): + def __init__(self, config: dict) -> None: self.address = config['address'] self.username = config['username'] self.password = config.get('password', None) @@ -245,20 +254,20 @@ def __init__(self, config): self.docker_client = None @classmethod - def from_config(cls, config): + def from_config(cls, config: Dict) -> List: res = [] for x in config['hosts']: res.append(cls(x)) return res class Processor: - def __init__(self, name, count, deploy_type): + def __init__(self, name: str, count: int, deploy_type: DeployType) -> None: self.name = name self.count = count self.deploy_type = deploy_type - self.pids = [] + self.pids: List = [] - def add_started_pid(self, pid): + def add_started_pid(self, pid) -> None: self.pids.append(pid) @@ -266,7 +275,7 @@ class MongoData: """ Class to hold information for Mongodb-Docker container """ - def __init__(self, config): + def __init__(self, config: Dict) -> None: self.address = config['address'] self.port = int(config['port']) self.username = config['ssh']['username'] @@ -280,7 +289,7 @@ class QueueData: """ Class to hold information for RabbitMQ-Docker container """ - def __init__(self, config): + def __init__(self, config: Dict) -> None: self.address = config['address'] self.port = int(config['port']) self.username = config['ssh']['username'] @@ -297,5 +306,5 @@ class DeployType(Enum): native = 2 @staticmethod - def from_str(label: str): + def from_str(label: str) -> DeployType: return DeployType[label.lower()] diff --git a/ocrd/ocrd/network/deployment_utils.py b/ocrd/ocrd/network/deployment_utils.py index 1ba265570..f73d81333 100644 --- a/ocrd/ocrd/network/deployment_utils.py +++ b/ocrd/ocrd/network/deployment_utils.py @@ -1,3 +1,4 @@ +from __future__ import annotations import docker from docker.transport import SSHHTTPAdapter from frozendict import frozendict @@ -7,18 +8,19 @@ from ocrd_utils import ( getLogger ) +from typing import Callable, Union, Any # Method adopted from Triet's implementation # https://github.com/OCR-D/core/pull/884/files#diff-8b69cb85b5ffcfb93a053791dec62a2f909a0669ae33d8a2412f246c3b01f1a3R260 -def freeze_args(func): +def freeze_args(func: Callable) -> Callable: """ Transform mutable dictionary into immutable. Useful to be compatible with cache Code taken from `this post `_ """ @wraps(func) - def wrapped(*args, **kwargs): + def wrapped(*args, **kwargs) -> Callable: args = tuple([frozendict(arg) if isinstance(arg, dict) else arg for arg in args]) kwargs = {k: frozendict(v) if isinstance(v, dict) else v for k, v in kwargs.items()} return func(*args, **kwargs) @@ -29,7 +31,7 @@ def wrapped(*args, **kwargs): # https://github.com/OCR-D/core/pull/884/files#diff-8b69cb85b5ffcfb93a053791dec62a2f909a0669ae33d8a2412f246c3b01f1a3R260 @freeze_args @lru_cache(maxsize=32) -def get_processor(parameter: dict, processor_class=None): +def get_processor(parameter: dict, processor_class: type) -> Union[type, None]: """ Call this function to get back an instance of a processor. The results are cached based on the parameters. Args: @@ -45,7 +47,7 @@ def get_processor(parameter: dict, processor_class=None): return None -def create_ssh_client(obj): +def create_ssh_client(obj: Any) -> paramiko.SSHClient: address, username, password, keypath = obj.address, obj.username, obj.password, obj.keypath assert address and username, 'address and username are mandatory' assert bool(password) is not bool(keypath), 'expecting either password or keypath, not both' @@ -61,7 +63,7 @@ def create_ssh_client(obj): return client -def create_docker_client(obj): +def create_docker_client(obj: Any) -> CustomDockerClient: address, username, password, keypath = obj.address, obj.username, obj.password, obj.keypath assert address and username, 'address and username are mandatory' assert bool(password) is not bool(keypath), 'expecting either password or keypath ' \ @@ -69,7 +71,7 @@ def create_docker_client(obj): return CustomDockerClient(username, address, password=password, keypath=keypath) -def close_clients(*args): +def close_clients(*args) -> None: for client in args: if hasattr(client, 'close') and callable(client.close): client.close() @@ -88,7 +90,7 @@ class CustomDockerClient(docker.DockerClient): could imagine this could cause Problems """ - def __init__(self, user, host, **kwargs): + def __init__(self, user: str, host: str, **kwargs) -> None: assert user and host, 'user and host must be set' assert 'password' in kwargs or 'keypath' in kwargs, 'one of password and keyfile is needed' self.api = docker.APIClient(f'ssh://{host}', use_ssh_client=True, version='1.41') @@ -96,24 +98,25 @@ def __init__(self, user, host, **kwargs): self.api.mount('http+docker://ssh', ssh_adapter) class CustomSshHttpAdapter(SSHHTTPAdapter): - def __init__(self, base_url, password=None, keypath=None): + def __init__(self, base_url, password: Union[str, None] = None, + keypath: Union[str, None] = None) -> None: self.password = password self.keypath = keypath if not self.password and not self.keypath: raise Exception('either "password" or "keypath" must be provided') super().__init__(base_url) - def _create_paramiko_client(self, base_url): + def _create_paramiko_client(self, base_url: str) -> None: """ this method is called in the superclass constructor. Overwriting allows to set password/keypath for internal paramiko-client """ self.ssh_client = paramiko.SSHClient() - base_url = urllib.parse.urlparse(base_url) + parsed_base_url = urllib.parse.urlparse(base_url) self.ssh_params = { - 'hostname': base_url.hostname, - 'port': base_url.port, - 'username': base_url.username, + 'hostname': parsed_base_url.hostname, + 'port': parsed_base_url.port, + 'username': parsed_base_url.username, } if self.password: self.ssh_params['password'] = self.password diff --git a/ocrd/ocrd/network/processing_broker.py b/ocrd/ocrd/network/processing_broker.py index 7bd10e590..dcb31019e 100644 --- a/ocrd/ocrd/network/processing_broker.py +++ b/ocrd/ocrd/network/processing_broker.py @@ -5,6 +5,7 @@ import yaml from jsonschema import validate, ValidationError from ocrd_utils.package_resources import resource_string +from typing import Union class ProcessingBroker(FastAPI): @@ -12,7 +13,7 @@ class ProcessingBroker(FastAPI): TODO: doc for ProcessingBroker and its methods """ - def __init__(self, config_path): + def __init__(self, config_path: str) -> None: # TODO: set other args: title, description, version, openapi_tags super().__init__(on_shutdown=[self.on_shutdown]) with open(config_path) as fin: @@ -45,7 +46,7 @@ def __init__(self, config_path): self.rmq_publisher.publish_to_queue(queue_name='queue_name', message='message') """ - def start(self): + def start(self) -> None: """ start processing broker with uvicorn """ @@ -59,7 +60,7 @@ def start(self): uvicorn.run(self, host=host, port=port) @staticmethod - def validate_config(config_path): + def validate_config(config_path: str) -> Union[str, None]: with open(config_path) as fin: obj = yaml.safe_load(fin) # TODO: move schema to another place?! @@ -70,7 +71,7 @@ def validate_config(config_path): return f'{e.message}. At {e.json_path}' return None - async def on_shutdown(self): + async def on_shutdown(self) -> None: # TODO: shutdown docker containers """ - hosts and pids should be stored somewhere @@ -83,11 +84,12 @@ async def on_shutdown(self): self.log.debug('error stopping processing servers: ', exc_info=True) raise - async def stop_deployed_agents(self): + async def stop_deployed_agents(self) -> None: self.deployer.kill_all() + # TODO: add correct typehint if RMQPublisher is available here in core @staticmethod - def configure_publisher(config_file): + def configure_publisher(config_file: str) -> 'RMQPublisher': rmq_publisher = 'RMQPublisher Object' """ Here is a template implementation to be adopted later diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index 04a6a2363..3b51b4c23 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -10,10 +10,14 @@ from ocrd_utils import ( getLogger ) +from typing import Callable +from ocrd.network.deployment_utils import CustomDockerClient +from paramiko import SSHClient class ProcessingWorker: - def __init__(self, processor_arguments, queue_address, database_address): + def __init__(self, processor_arguments: dict, queue_address: str, + database_address: str) -> None: self.log = getLogger(__name__) # Required arguments to run the OCR-D Processor @@ -27,12 +31,13 @@ def __init__(self, processor_arguments, queue_address, database_address): # Based on the API calls the ProcessingWorker will receive messages from the running instance # of the RabbitMQ Server (deployed by the Processing Broker) through the RMQConsumer object. self.rmq_consumer = self.configure_consumer( - config_file=None, + config_file="", callback_method=self.on_consumed_message ) + # TODO: change typehint for return if class is finally part of core(ocrd_network) @staticmethod - def configure_consumer(config_file, callback_method): + def configure_consumer(config_file: str, callback_method: Callable) -> 'RMQConsumer': rmq_consumer = 'RMQConsumer Object' """ Here is a template implementation to be adopted later @@ -52,18 +57,19 @@ def configure_consumer(config_file, callback_method): return rmq_consumer # Define what happens every time a message is consumed from the queue - def on_consumed_message(self): + def on_consumed_message(self) -> None: pass # A separate thread must be created here to listen # to the queue since this is a blocking action - def start_consuming(self): + def start_consuming(self) -> None: # Blocks here and listens for messages coming from the specified queue # self.rmq_consumer.start_consuming() pass @staticmethod - def start_native_processor(client, name, _queue_address, _database_address): + def start_native_processor(client: SSHClient, name: str, _queue_address: str, + _database_address: str) -> str: log = getLogger(__name__) log.debug(f'start native processor: {name}') channel = client.invoke_shell() @@ -77,10 +83,13 @@ def start_native_processor(client, name, _queue_address, _database_address): # What does this return and is supposed to return? # Putting some comments when using patterns is always appreciated # Since the docker version returns PID, this should also return PID for consistency + # TODO: mypy error: ignore or fix. Problem: re.search returns Optional (can be None, causes + # error if try to call) return re.search(r'xyz([0-9]+)xyz', output).group(1) @staticmethod - def start_docker_processor(client, name, _queue_address, _database_address): + def start_docker_processor(client: CustomDockerClient, name: str, _queue_address: str, + _database_address: str) -> str: log = getLogger(__name__) log.debug(f'start docker processor: {name}') # TODO: add real command here to start processing server here From d700522b4b518966a71258ea289bb6bfb7af4388 Mon Sep 17 00:00:00 2001 From: joschrew Date: Mon, 9 Jan 2023 10:04:58 +0100 Subject: [PATCH 023/226] Add address-option to processing broker --- ocrd/ocrd/cli/processing_broker.py | 10 ++++++++-- ocrd/ocrd/network/processing_broker.py | 13 +++++-------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/ocrd/ocrd/cli/processing_broker.py b/ocrd/ocrd/cli/processing_broker.py index 8393e2165..fb8dba547 100644 --- a/ocrd/ocrd/cli/processing_broker.py +++ b/ocrd/ocrd/cli/processing_broker.py @@ -13,7 +13,8 @@ @click.command('processing-broker') @click.argument('path_to_config', required=True, type=click.STRING) -def processing_broker_cli(path_to_config, stop=False): +@click.option('-a', '--address', help='Host name/IP, port to bind the Processing-Broker to') +def processing_broker_cli(path_to_config, address: str): """ Start and manage processing servers (workers) with the processing broker """ @@ -22,5 +23,10 @@ def processing_broker_cli(path_to_config, stop=False): if res: print(f"config is invalid: {res}") sys.exit(1) - app = ProcessingBroker(path_to_config) + try: + host, port = address.split(":") + port_int = int(port) + except ValueError: + raise click.UsageError('The --adddress option must have the format IP:PORT') + app = ProcessingBroker(path_to_config, host, port_int) app.start() diff --git a/ocrd/ocrd/network/processing_broker.py b/ocrd/ocrd/network/processing_broker.py index dcb31019e..994e2784b 100644 --- a/ocrd/ocrd/network/processing_broker.py +++ b/ocrd/ocrd/network/processing_broker.py @@ -13,9 +13,11 @@ class ProcessingBroker(FastAPI): TODO: doc for ProcessingBroker and its methods """ - def __init__(self, config_path: str) -> None: + def __init__(self, config_path: str, host: str, port: int) -> None: # TODO: set other args: title, description, version, openapi_tags super().__init__(on_shutdown=[self.on_shutdown]) + self.hostname = host + self.port = port with open(config_path) as fin: self.config = yaml.safe_load(fin) self.deployer = Deployer(self.config) @@ -51,13 +53,8 @@ def start(self) -> None: start processing broker with uvicorn """ assert self.config, 'config was not parsed correctly' - # TODO: change where to run the processing server: default params, read from config or read - # from cmd? Or do not run at all as fastapi? - # TODO: activate next line again (commented just for testing) - port = 5050 - host = 'localhost' - self.log.debug(f'starting uvicorn. Host: {host}. Port: {port}') - uvicorn.run(self, host=host, port=port) + self.log.debug(f'starting uvicorn. Host: {self.host}. Port: {self.port}') + uvicorn.run(self, host=self.hostname, port=self.port) @staticmethod def validate_config(config_path: str) -> Union[str, None]: From 019e94e088e29c0dfce54a2f6f6245cf961eff1b Mon Sep 17 00:00:00 2001 From: joschrew Date: Mon, 9 Jan 2023 10:21:48 +0100 Subject: [PATCH 024/226] Remove informational non-code files again They were added to give additional information but are not needed here any more in this place --- ocrd/ocrd/network/example-broker-config.yml | 33 -------------------- ocrd/ocrd/network/ocrd_network_arch.jpg | Bin 319337 -> 0 bytes 2 files changed, 33 deletions(-) delete mode 100644 ocrd/ocrd/network/example-broker-config.yml delete mode 100644 ocrd/ocrd/network/ocrd_network_arch.jpg diff --git a/ocrd/ocrd/network/example-broker-config.yml b/ocrd/ocrd/network/example-broker-config.yml deleted file mode 100644 index 265d1b241..000000000 --- a/ocrd/ocrd/network/example-broker-config.yml +++ /dev/null @@ -1,33 +0,0 @@ -message_queue: - address: localhost - port: 5672 - credentials: - username: admin - password: admin - ssh: - username: mm - path_to_privkey: /home/mm/.ssh/cloud.key -mongo_db: - address: localhost - port: 27018 - credentials: - username: admin - password: admin - ssh: - username: mm - path_to_privkey: /home/mm/.ssh/cloud.key -hosts: - - address: localhost - username: mm - path_to_privkey: /home/mm/.ssh/cloud.key - deploy_processors: - - name: ocrd-cis-ocropy-binarize - number_of_instance: 1 - deploy_type: native - - address: localhost - username: mm - path_to_privkey: /home/mm/.ssh/cloud.key - deploy_processors: - - name: ocrd-gibt-es-nicht-test - number_of_instance: 1 - deploy_type: docker diff --git a/ocrd/ocrd/network/ocrd_network_arch.jpg b/ocrd/ocrd/network/ocrd_network_arch.jpg deleted file mode 100644 index db65069a4e83ff83c12590e481068d4fe3eaf83d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 319337 zcmeFaXINBQwl2H~5+n(NL@5bM1|>-@5J@5+AUTOhmYicD2uMx>3X*e{oFxZAG6Irw zlu*P1isG)__xrx>{?6&=^gYk+zJ0pEXYoU3&02HJvE~}{9q&8l<;3MYaN~)zoHT%j z4ghH2Kj3l-kOXkBu&}YN;b3E9LYM?a1-L~xd3d>h>jVuK7Z)E7pMrpZf}4(pj{E=k=du|f#v!7` z{fdD`51QE3aA%FrGo&r5q>-0Qc7lMT1HmxiRyDTbq!4|ZKGGlCZ=ZQ77mV1&MvNQ?tX9m0|MU#1xLrc zkBy7}@G&7hBQq;ICpRy@th}PKs=B7OuC=YbqqD2~YtQh===ZVli64^-i%ZKZt842U zoACXE!=vL9#Oc{@<3a;4{`auJ-~W4H|G#k&gX2QS#KgeF{%u@n=q})eL5z8g?g18w zgfh0F-A#J#H#nq^qS8uRa2a@1_Q+n^4_&{-$h&YC{@c+0GP1vIV7~vYk^O%I`+Z!K z06qp9IC&Vv02DYmV@~(Eesx{h;K~R7d3=Bs&tpoban7VRz^Z0urJI`asN?Ck+^i2i zsjs44(S2?O2hXm~?(p2LfIsz_udt>t9ih)kMGt1ax~^>S_xFK>OJH+q-F$TX5>TI$ z6sJWy=f1cEZeDP}j*C2Of?&&PW6-T0n+sq($opFpto;&@H@GO!pIS6HC3A{6+fTd% z#Cg$HVda6ym~BZ^YZ+`kx8Tb-(q#dL*MDcsDbYi#duDlm2RlyYIImAcjh++eIG~rzg8TRZ*afV!%I@`GJ%xe-pYCE((YR?wy1-h%RO@ z5An&CP~D)C5^v5G<;KCTG`G-B3Xd?ldz|G$kW~ot*80R6p{QW!Dk-<;$iiQ`z|5{L zT#3tAWwrmw9_^WJrfD|XQ3(a2pk(80pi`IwuU(_aYN7FdPReItEOuY%?q`aMYTQB| z4&t=EnMk=k=i1z&fTR)~dB^Qah8NwsXPomG-ee=Xx>Dv=9N+eF7J~4jg&ox}7*krm z7$f3gd*#peM(tgtWjT&Q%%62ujm1c=GD%~`0#Pkr*CO1H6@)bt4G@fzNxi!@Dbn$M zHoF30u@Q?BsRRZHO)(NwXN(4Fw)`Aje)(gylP5+6VdlJ_51XpwxZ@o0hO&jbLU(QX z`_FQZxR-q;7JNh{N|I2xNP!Vg19PYRt8j+S;9NtDm7-5A?9^Sz+OhM-}Kfk!A7`??~~ zZ|}NTEuuV6gpX*6Rxo`kmm}eyMA3DJHu1f$Olfy9OaTFz*W=&dgq9oEIa>8ptCkmg zjr+)y_5l)m5ZRO`2vf{>;t!K&3p=BL|{zdyBaBwp8=pvXkh1iQtt5=x!s+fl$7EP)t4k_e@77$ zZTHFhIoe7SF0o@un08$3!qh33BQZ_dO268W_+%Fi357qt(DIU>5+c!%=q-v*%QZ!P zgzn4WZqG-fI68WoJ7F#A6zo_u-ef!a)O!Jb3)C{zgNN%y>W}b4aZNoPr@zUGDp4-+ zd8iFLUN|VlwW(GF?~ExQNU<(^c}3JLKf@-+^Q5o&tTP(eGH1$G?;v5F=|RSD=bSiU zqA^jn24S2hpITJ6VU;~iZD&Ox3ok9N6sK4dEx<#On)-;1tX=}IDUX+_KMX*! zq$8wTbvOunf5z?im^dEjKdl8!Ipx-`y&go9B7JssUD@Eu2d;eJU&jX+?yIr416XpR z1&N~#PU#yzhcf$|n3rB-FrdyhPEE2^l&38?b#8Nzt(76A8X_-&^mT-OrwkJ`lI#W? zRY9mDU(GP(f*6Bo=1h-IJpbt~O4i&4r;A)Xkd4B2mV>52#;Sgb*rgfV$X=hSMN#`t zo$W)lbHBf>dR3>YlaZ00NE5{VQg%jF1m$Lu-@iyN^bKat zV`rtbx)ZgB3QyE38zX+Yyqd>?GC+(b&LRDEJ@EQO%^g+z`)wJ7>2(9w zT;sIvqB=%DeH`>&l!kf|Er=mD^hO`h%>E2#o$xkrXR?zb)zIo8Rh9y-t}7e-xjvw* zrz>M_&i5^cxZQth9m@>E7rv7_9>2F9$xr;?ZVPQ=;JhGG$}s=!v;6x&Gnw|J92Y02 z6K-wX;-jvEO8^?HT|!@Ry82^FIgs%+cx$gLNR`!gY#6F2pghU%ga+xIU*v1O1Dx*V z-6?zJq!}_V)^TJ#8A<`g<75u`&!>n`{yEOA7V;8f^qf5@*As44KbN z)0Jvmw`7s^H58e9_niz!U#u1lir?`Y99HaK;d3$?sj!mc^P-VvvPLwveg zGLE~q`ZMP^AZ{B~)Lwc)gni1jrZ?lkyV6HGK%eRMYEW5S6aE)ZrKRrdXYxLFHL70%5{u4sB2IRf zR3p-TL5fU!(*-~7cvF8-(qH+J#!Y7uk02`J%1X(*y`QdO2oxti-ka9;=Ctlk*gJ^n zR?2B=R7zb}-nj1;ZMd@xY4EN&hO8z0=-@wN;NRg7)_xL6WhtyVoc_HqfxJ1&E$kAg zgSA$K@eVfzxE*CivwgL+y*_|DDY2`bKM=g_-REMH*Lk1f)PAWc4OxDAy%2|iAxm{! zCI01{|LQqc*mjpxZBQgi3uVDJTG0J^?kij@;?AyZKI|68*?5UY)?1f1Ym2ej;ELfy z({i;zzXB-!OdrsmCJQBx#f9JpQlvpQM$A_e%<;{$6YR1R;#8P7j}0cNbdM)aKia445=z-6 zcxKB#D>}(ImF1gDSme`z{p|DlfG^>6KqNkM2^g%Yk$YXAQuaehYHM70F`?eFO%!x< z#Pq@MMNnJ4=c9*H&bwuvJbD{E&)@Z3CFGhRJG`#zrjH{IPE&-4|Rb%gy6Xvwz} zO5^*=C1<2Z5X#FJ^_rCxBB{c28{=o*l}lE&TLub*iEZjlKPF!rcG4OoE9oYuWI19a z3}~0)C%TG4V-2r|H9oQ(c{IktB8}6XIxg{UM1glnfsZ|tcSM0DJ0U=u%95d-ALH4` z6XuJ}+e0ZZrOwKr{BcDxEDr~*9!1vm;Rp4ly$BJ}OCaR6eBLD>BW*g6i=FZ<_^58h zlQLVU3-c`g{k3oXs@_U7OJ|n=>t?}JtEHU$mz?A`H7YgfFLEAGyp?E@ka z>LdMri^3s~_@@*-yNO=Y&IKwpt2PKilJfPZQ|8gI*Disv5(zj{-~t>gX*rr?5t={= ztMei%EoP7z**TOCV&7fbUwOX+xi5G-TxLmMnxD4s;ZSn+E$`vp{D&=hXt{54{PZnM zRi)lW<|rn|-jC9mI8zfDN*Ly&;<0^Nqi{c=WVP&&*KMta%zFqgD(0pFA8yJ;%GigI z>KE7K^GM!PvVB&Lw&Z%2cxz{7pog`s)LO1j((1JnhsEq5erKR(lLX2RF-BsY@hn2s zH4Ah3?6!A^<*_K$7|H#qhYv^hVMzw*!Dk;N9%e^-y^ z^;Wo8!g$=HRbo_>)d9W}+cZpjCcV7q3Ezta&g}R3LJq2Wgd}4N)^vnIhlcAG-Mk5_ zac2rs2HM3|ub>SQNNgl4bGFx6&+0wW_GMmG_Do?M6`p^wF-+W z)Vpq+_*4peH8j>x-`mR1I@FHVS=j!kwwkfzs+KQ`#g)nEj8D5yg6|X0es2J3Z~nW@ z&(n-cK-Iq4o$B=bD{WtTAO7&o=VAl7jt3)4(K4y&yPxfgm&@m*$SJ31 zL!K9tEShP%eNof3QtGTQEDIo1v!9!_R*)=^r5m|t@3us{MxRfbqy7ZXnMKW?cllVx zG<90aAx*1+k&GAdNdgkP#}77Tg|K=N{{zeK@~{V}JW*xk7uCe-p+ zI~FO>$=FT%1A1XG7{?oRT_`~1zC?+17RB?dpP7wg5n_C6iX{~xqeH>6t6{Ig{21_k zSrEmBh7BXHIE*;gzkUs>Q;UT&M&avDlqIhyVlT@al|;2K3%rv*(cecZ<)nQY`~-8` zDfb|KSqfcXsq8unDo4P=coDJ7-PNC6M1ySBuQ|vtb;?b}>|X*r=~~HTYzHWlCBx_C zH-P1s$=bxNjDlI5jDa8Yf=uU~NvxaN@kp**uaZi9PYI4H^vt_usilvr5!v1T;u3grWJlqo-G6L)?fOo@k){L<+;hp`CWJzz7T5Z%#6_w4|?2LpaS-sIuVto>g zg0E^6Zwhk+Fj@qYk+OyZ7=Y^Wh)<4HLG7Dwn{d8>%=pdBjypf@2}2GddJ5SK=&_~o zGhY$ws7lblc~x>>8Tt~~pQyVCvHLH5t9lJ1I~zM(O(^Gi?;bZX=CmZ*KJ-KKWEuMK zb6&zRgY^j8E3}a>Uab8=CcV`9-=H23X9kQffvvG<&-}$;&MtFO{`6^PW^(fZn08ks zN6wt+`?sgBU8EHpG1X0qOsuU=x=?k{kFuLy1p@MVPonE+`;>(NTqgAV7SCwypjR(4 zZRfO0>bw+s9^*UDBfoi!Wirx3stp1ln}%y**%jJJq_{RW!XNlRwWYH>;$M=hIk7h1 z@SqCrIX;o7J6G_Fp}YjN@azjgRH&;+(c)hAVrCIt_To;2?2uD|o_)8Hwe41W)Xf9B zOMpmHTWy)Y+J)-wnZb|bPjV0w{rPi^eDle-R9SB|oclg3-zHKMZ8;%ZT(sSJU-2Wk zn6@sr@Pq@*4}NpJdW^3Ugnx<;xb>HKdMXs3%66Ue56&B9q!@IadReB!lM5%kBaU6= z&J}(<$QJTPF`O;I3px1vUKbA2h0W-mCG51SmK{ed+b(Wi0=<<7oaz3cypRT9ea6a{ zr(UPkE^ys>@>V8Ba6t3B;IPsAcYGu^2^7EIR26?Qjj*4TkB#FtQuyK8^+>dl?U@e% zydd<=wg@XwG&vy)hFVuyX1u;fYm1_aEnqe)Bha0p)aW@7X~x!F6DU$I7)cFv8A`M4 z`uO4WwZlrdU_lz$4OV8g+dy7IhDCi9rh;;%?00IlJM?_i&%=}}wQmfgQHTPD(hlOVWn-C!q>{*)Ngo^L7HDzfJaakB9%nG5n;J_x&7_Asg z79lge$(iASa7eMG{TFX#yO@x554-4S1#kvb?*F-Q-c{WE?`O{vn5`A>RzW1UVbC(h zIFN8F)mA9B>$oeFDMrw@_r&}Na^acMtaemrg(X^bPsM(?_9aXSUgX5LZNHyIHVD7V zX)Y+1*{i~m`+U@A^ubLGrI}}bihT+J0trS`xY;)!i8DM`P%=`EQGa`X)JiiXf9AGT zQ9r)Ep~8oCp3t?UqHMk#4u5<8+{xtM9vcRWuzcy3)S^k9}ZX$!); zKL<$OQ7q!MluxtIN-m`PmqsQfh*XdJ&D{AG%&;t+Q*^b3%7xXg>nu zF%dcDdBr&yzDbKq6Uo8M#TXTT5B%Y(&wn3Wi&gj?$?Bvg<9T$NbXOV$-(6C)X7gm) zvDcwh!;?Efny2=HnL`L{5q_Zq653WqXGqZ=ot?7ShDz$uuP zk0?QmbjEstz6*sPT>?}axmibf$C78Cgv%39y^WVZGUP9x<~r8fMZ&?bE*@!_nI2I% zP)3y3RxjA}vwrS$3G9XoF{1c~ZZfu1fBJ3O?)2+!JiG)B+AaZz0X+LQEr`eV5+S`{ zopp80XQStIUW&JWtZHbC#p&~v?@fOu8?#B}X-5y`2G-&RAe&SBMI-4JOenoe;4ing zgp|Alm?RgV=hx&efz%m|&#GrDirr#4O7#NeX%;dly{5mkGaTgOxdb@pPKjZi_3kVe z&N;6jsyQ>CSoV(dVJo2SksiEZqCh?)uu;k>F|@OOlI6lZ$N0jtZPW3*d|)7;5_WnB zLdQWU-nNA#aCh6Af|62#mq?};Y#j-6ZLLC7a%PfUE&-g*Qqab*Q!;301VWlG;uNIn zb?2e@*54~%q-m6)z8D^{di_|9K`80exGvm!!(c+a%$-f@z9d*8Q~idcDBTxYOMys0*m?0A&+ku9KVS8EWm8v>kpo*^_hh;RUgSX!sq>Go zgHQL3OW^AuLJ8HP0$YEm_LCG)mb$vGZ14~B0kkC)2o1!f?WJ7!W?lk=)&&!6UH@Up zzhdzHnGC+GzWz&p!j3=$q7(yLVaL~FE&++Uk{2ZqI1w_5=q zK5!LQUWLs6d|QUAu=2kQasGAG`u}q(FIwqp$UD4TI)}gYL>edqnUVy;qLr9!LEoq2`qIa_$N1ZbbVt z?CPPXLEj=XqAJS#BA$5t5}@dkcix6Q-y8Qa9(D96w{)&1z7xq>jk$lnIT+YAD|Ttt zj-4%?7bSnz^r6qi&Z30b1OE{*CHs#67Bj{g1k#u1DU4Wvgl3jm7KyC~ghfcxJ_{Rh zF~MN;Fq&;FZ}k=!6frkmTR+L)XU5VoJv9!wSV3hl%vK(8aCd2bj_XpX2;Ll14h&YC zCIVHAvtJw$v3rvS0V6ty^45g6$HyB2s+6$gnQ*%R6r(^V(bA_+fX7L&r;PCelu+MN zWBG&C2puVM_`$++kC@R&UDw_)Wph7S+jx#* zvmDoFLXa)=84uFc)rS>7I-Dhu5;$hZMdris%wQCpB21aR8LstGAx|x3`zowR;$)9Q zSuR?Z9YvC5RK;%(v^Kr7T56rJk-%SG&T44v1H zJLx`x*2$GUn`3*^sPALS10=G3M%#8#wN@c4NF40FRLrpk19<$?VWnW}pWCl1vRM1f z3`5_m-y!Icv*i}wBoZx{@FZS*1rPK)ys&$Dqq3El@x?Mj0oe=DGD6W(t(05R-ed?r z=)n&3kgpK-CM$m7b`1TF9pj2j-T{@*0})9$TL%@GQ(N0hX*)N=kE@ScJbpoeo`Wy# zdYy>!&hp{OvNw~30r6`j-My$KJZhZ+x$Lm?(W3igT5`fV0HMb#t|e_)x&Buh5hang z)X^NLqP9={o1Q_R{PYI%bzeDCP+t~MCO$&Y0LcDRPvJ{IUrx`;-T)uE`~l&DYKX3z z1VHBtp1bK3f#KRKQ&I8YwveWZW$rhLr1)=izA=N_WYD)xuwZX`_}ze_TrG(lR}*Iw zOTE&-{hp+p)zh=>wa~MdwI!i;GSE&NEJ;@OsBWw8LjjT)wi1}`8colz--8n*T_Ivf z?u)!>PSF3^*0d(^8r_l|#YS@r=!BQLs97TwsjRyZp*G z)oa?%SMWB0oi;~Kh>bVDYdRN!MVz~J$=Xa@^tL3M%o=|F3M8e@-4Q!sOGIw3ZLuk& zaG_${Q$B5`1wZ+frEj7RrYQ>`N&t9}2Q|nUFW3<1ZN-Px&CZIpJPYksx}LEqOSbI( zFv$D9>*SBh84^+MuBV){k9p`(hZv#g%`xF0Po*UaAh(wqJ|mRUv=n=W?~W4d3Xwc7 zYrWu^h*Vn8kJ-DRi&z*B7{h;!>Z=xU|Ih*r2aAAer6X0~MFSlKTy5H8DOd@&+cnft z>hiZUqERqV$xSr_lNel*rG2$L~eo$C#hX{K%7>(>wt%}Hl*-4r7OQ4<{ zPw5xjwY*PUzJ^LkoUqr`6EZr7yj)LwOmzlj4doJ;y=If`kiy{5oZ@K<6 zzWJfiiU8SsO1T>WY;jBlhr?Rp_GyBTFjtTP375#j+zEt3Ta~%TA~n*@0$C{jiBpD> zDLE6sQX=cIKye`o_6{dEzcA}TRfbL8GnO{zcTHFC&&C8GB?iLdyK-#rG(#g zN^5B*XT$^_4iF#|<&^@)X355>Sh_1bbo{K_^VGXf<$yzmYHj#b1g_^}gak+5B@j@2 z2~;?WcWN2oHCWwVez#AE%^sM#91I+b5;VM;#jHh*%d5ffhuaa(e9`iRFn^*k66o_h zS(rm8K^JU5&eD%2!%N_8BFRO;x31XG;-Ai2!6!`tvrxZ9tT%V(tIzpX%HI_mVzc5> z;^zz=?A-Ux>1zNHGo}ZN1}B`*L&j7h@p`y){rsH&3DsI~T6LnPA45;|9foqTiLTqV znCBR}iXnNPW^lzQ@^`#jGxT`Ep%wgv@9;Qfmsn+KTwaeCB1+nvlR5Xk?*zGji=56y zf4mur!`ikkzSQU{tb>$Cuo1iJiz>CfJ7u`G_)N*}B%Mk0i*E8E+Aa!WrG|vQoSdY$ z2p~IG=?Y6bUjpmrRum-fX@ppLS}ufGS&ZEa{C4-QfA~}DMh(#mi9?1gN8T&Q;hzt3 z_-CGzzYz}kZF{PMs7*ywE2qefFnq5NG@h{{eFJ$p?>O1gwG^c(v>FeWSq{0k!0*~YVAV5Ou1J9EwxYJmM^!Sr@r8)+u`dhf}rv$Yg& z8Tb8*n~I3ay)hQvu!r&$O@ilz4#XK}8_%|(4Z{ct#-A{N`PhhS^xMh#ZYvM4}>SeXeN)?~WQ1eNORU1kL~ z^y{Sn2k4E%#2J@nq0W!lZELf-l2yzn`8&dIHrvrgY74;yV-hQ8Ati9kUBATY*m7by zHW~WB0H|e%Wpos)oKgl$WaoK8;axr=HZH{o5}aLj>yVo^v8QC=o>kI!B8eX5h&b=6 zw$I`s@|I@kc8|J;jQ!thn?I1JF^V}5?6snx&8!x!zaRr|WEOLTi`NEyebPFOD8|Lg zqO_NGeoE81n-S7HE-6KdwroI*BwM`gu^8cBnZ%=XaCm;pXIg$ux}LwUX73yBQZibJ z9vG_b6TO4tJLQRJkcWv7+tCQM-WZ;WCja)@$UbI&U*OI>t(OJyA}q|s?jjnBV?K6p zF;;7WCR6jR1_gU^tj&xO1-E(22ru!0mR3@#h)Se~OWdb?UZecA`KRK-;k|~OM^%XN z1r9;ahdCRz@~=Z5lM@<6N`om`yjvKQU)%5xMk>8Ox6pBJ?f`55qdM}#zh(Ju{mHz}+LMYZe?mSKYQS+8u0aEaL zB~fS{Ec9hvnV#6h9oLU@eZtTE-c;YdupBYNXf+xEUu?7e92nZ+x5=MOGBngbOV^5Y z!p4_lnqU@16v2FHvB#0rc8f&Y^%$17zxwFJ@Y>s4e>E^|6w%OVH`eG48Vd}S4DhWR zR9(y$Aw;@&iyL2IO(&&W<**$%zi_`fp4Q1m54nka9Zvibb>nl~hPXkcFf;Z=M$n_R zjW^r+15$wf7lV)$g#G${3nX}Q=JVI;m%TCJ9qvYx4acw>o|Z9+iSqHSaSC?y6kjH? zgou6Tw)jmCk4s*s`-q$`FtwdWpdp-HcUQs{q>m*STNz?Q(fprzL`BP;9DiN%RGuH8 zay4E2G|RIW7OEYpp=D0TE~e6hQGhOSCn{2AX(YuV-A6!pE9D(kd|U2L;m@MOIZ{eL zYOC(Ap9PoLWmoLx_KP7}O^B|=Fp{OVaaz){N^oj#s(CKMO?pcj@PE1Bw&eEz%M+us;;U*+Nc=j8AH73S(H2YHo){HLNW;zygMGc!ir=$^X4gH~Jm ziAdE4#_DD0+S|6Z7=dd+rRdzp>ya8Gz`}~`{;?f^-aym>yX}-h_>H%TOo}tj@}`^B z15l6hK~~_-+V0XE_FN_4BK99h#cMMzUEb7*OF01|O0;>@8}~Q3R==f2hgHp6v4sZo za#0L0jFpL)!^gGYq+RlRg(@4zWz})zM2OQ5F+Nygt6pVZ=g+W*hx$mf+U*%WOd0vV zaIWI2Q!3=bLU?&{EpR~P`Zv-d9QRG{uGoB>65Vef+rA0}uQmh!5Fa4hhjkg? zf#MF=Gx>F+Df0rBT3tfSBt}RwH4586Q{xlz^MEHNt8& z+k2c-hEDbr`{Kaw$Vi+v?21-?#ku-N`GD+t83RMEv7s+cH+@_%aLR=u`;@X*4_zSH zpZWMc(rXE3Zt;ImGwSP{yE}gns+xQTLsU(1XusH)~N#H@`*WW9io%-Nz_VW9cEIBYCj#4c&D zm@^5p=Lv|lXPMjcu>>FfR>=7o+qhYgSOEs?=cMS5WrN@UpBc}yLHbK;Fb^4#-wZ9!Jx!JZBS9>15fTA6T21K@zqcpbs!%@C z#6EU{@N_pV0HE-G?Je}=Q27t-8=Zs23IJ4p#;O}R@(#$WLhc2kP{GjO+iMgL$$WnS z`7!D_a0iBFxgH7{=QEFgti2^VZqDZhxeN}Py)QB^GpSBC;gFlT1~)gw@rgvk4E2Mv zV}iOuTRVMK{6yoG>UA!JCby+AhYFTP7z7IVLKf(i{CR(ZqvzVRw5U(tB^`MAsWvrn z(>ItopZ%()#=lQZjlb7icfehc!3hx;_NerR^47V1Uh}4-X^lKYJxWSJ22~4F*4J9F zIi=#AJKJFkAx3H@D?Yqfgm8U3o@7q1SJGt8R$vamy?ZAzSo6&sdP9LCysG#NsnREg zxci<>;!OpMz|yq36Kf{{)96~!JO5kn>yk^Q$W~{0;LxcyWB89-i6($C@>jwD_=svH7!O`@1)p(F=4oW&q9#x<4$)NS!H;|Cl zsHD0V0ZyU|IEmR+1>~*JeOB|iqu#TG$)XJ;&Q1nW;Oq`qeZ3F;-}%76tFiwV5(MVI z>F@Xk4gyRxmkWRb=^g7&ALYkJJkHWU%kH}V%Z(RaW848y`@nTIW zq$6l~2{uc&j~uW957nqR^6`{5Sw}-(XHxR#7(;vdgV~RUAC&^_I%` zt-5oMI&%P)dy0zJ*W4p3&MB>t<+3{LUT;Y*HHI`Y=U`{`&!-H%y;g*&#gcxdn8Oh<#Ur%LFZ(&^BD$TbfK`mFGMo@vV&Jl2fTR#Q^{d z*p?!dCJyhWdve37LwiAAa>Y-l3G9CZ8;wu@9u&y}lIYA`bcwUSoS-S!$oUyd} zxf4FoyansXaa3s>6JbHy`@Jw>#hULn=@pfah6TII!k5plpUBIC9kLabzQD|5W&eS3vX`+`V&6hJ$tRdxoLm3OnlM1Vmw`WkupB*t;`~{r>xeC;xdEk&9@K-Ahgt zALzzLiLiB&eWc)yE)=?9D`mT=mw(Uo^lh!B6bNhatP8Ozu$Q%(_ysB^xXJi@nLYDK6h$`ux990!jHmTFe&v+|d2 zixGqCtg0fo&Kiu4eK$mi#O);~%XW3kphPU~>+_H}$KXm$DE1C~6wAf`{R%f;73=(y z=NyB)V{V_-0tyC~phkioyduRVmi>*6c}s`;_+JYFt{>66Q7k99GOe5@`(Far z4;~%Xf*bLs3k&FizL|bJ^yK6cDALm-8~K?waVS_+UIN+?2#o~t ztbvOQu$Hujh7@hjZNJS5SS(ErlC2MKcnyWR1dc#Y_jgNk2iK0dIaDP`76rTd*XgOC zud-L5;=+1=m|*s~3YlIP6WLDeqCU%Sah3}B{IAxQ-Vc3Q7>?f>CLkeDVBwzf7PkC+ zqzMei_$oEO{9&DNQbiDND5Ge+ZYV43PD`3v^7XZc(A>z7V<=sCjvwvqTWf>Rrp3Ml z(o&SbS&B6Y)RQtb7eYmpJl;1K%F5bjBD7l7m^x}%f)`O;=`#Nty3DK4==ad*KMx)L zW(0!8*DOKxMihQ9_p|%mNERk;j)rWykLV;Bqf5bXj(e+zuHj>@v(j~z{$H8m825?B zKaKHO?DH1Y_P%RBCnvL9s|%BnXDM*`?!1Y*gej zLBh?y=lxu=FXg)k6)bWXXJ!QC1kk}H?3%pYLw_r(-tC%i zLw;G*ML(?gnrcdIziN?%{@4ts~)vPBuBG&ApHdcflnuZB<%>*#HRO9=^TZd2+Lp<+x^f^6FDv7N5@e7rrgZ#o}0UG_UTt z1H#EiKgI(P{z(lod6Yc$+uVd+4{5w(1%hg1Bt@LeRkLf|lf;7H*WorQ96G(xGyVAa z^;)8c`QXFP5|lK-r#p=!k4$ZdHu~&MlGXkhB>3tV{xsI;Q0x8_Zsl#A5(6%&t zDE(lLF*L%Kz`4#Km^aC$t*_YJ1!`HQ`QvMlobB^hdHQLM5tXh=ekc}Ck-arPjtb!v zB7*@2$&l4^M$YnXnZD>(Un%EJKa)8eyN(VbLfR9I6QA&Ikm^o#aH*m7gdVL~GDDvL zU;$km#N1?=l!ByXBz*aX8ubUF*1I-EV90gJZ z;k5|CoL*P=ru|YK^|B*|T2tB3?nIXED@JMax&q)ui9e((Ni*0TEkz$FBi&P$EP?4n z*3t%#Z;Ae4@z4s!t~|k==_@O1YBJ+5@jar)J4rzeL6)bGi{vVoOG+xb7d<2I{LBRn zfz`F3ueN8$N@{A~!n;?CS!%<}fE_D=X{`^siZETbiLMHJ78y)R2uM{Mt&VUAeS} z9d#@%1xu{%hTWnV<1>B5%JXLKP0EwKhyZ=DVH6~5F2GYO?W-kA4g1MGpuZNdijw_` zK|`F}C_IQgNJ|OlOj?{F;V_UlCnyZ<5ow?Nq3YN13Q@mk5a3D1sUs-1p zNu(mg>G3LbY%Ihny*FLPEuAmt*fJnidq`YzCLG&yP4r;w%Zojkb^>fgcy`HR#I1>$ zP4aPtx`_@FS$f8#F2XiUApgOjiI14iUSJ0ozU$cfkBw^X4tH|fsyc=6wlA}?0>^wm z#(VTx;j3>7J*5XlWEMi$8pRfIRkx)zqoF*6_iaNRV%Z5jUVbV1n}DY)@bYg1=>Aph z^k)E%zIig4=0gURhtAK6MohW=*OGI#b~Vg0hhLhDw&od_Q+a1Uo1K&rRI1Z+gM{g& z2gFQMsi+$%lkh$ddz2o^n%j&q9BuZp#(yD!FZvM_B>!M%fHSv4ytS zf>wSpi0^gGnbA79jImKq6c`i}E^IQ({EOo=v)l%ZjtVN2AIYHy>s+%HUbooRSUJ)O@k{I;bWX13qGZ*{YIGIo2D8bLKSq^MilcK;CLY`l;) zpFZ7(Od7cD&q10u9LAT%dw(yr=*BPL!FJ`Z@8DAKy}SIk<_Z{Yb{gz6xcZ@5s-b5r z1>na__K18dUr0-g5pHCYN+p6i*Vvc=^M<>xu$taOnGs$yS^_S` ziqfDd4;f4J_QHuy*=O9g*R;|O210nZ=6o*!l5M0+#$&}m&#wDic)s>~DN4Is3*%Ug zY>Q&iDL2K)ipKoz)V@$Z8+z_Q_pCXm-0*|Z=*#+TN%@f@51Voeb^hwAb3ZqY`-LG< zg9~pcvp$c!<}FaR%+b=(n(~zmp1m7}CaOq^CQd85H??%j-*_QW53l`%#Jn!xlkb#c zsh}2_tG-X{3jmYG5s97VYO@Fl6wxct-I%;R9j~I(vD%OV*pxTt>THUy!BdIoFH#T9ix!4F z4GrJdjjY{h2o_XMf>+NsgPT${Nb?kR$_<2HJ9yyYDCJI;8QmdfZ@X?7m^>)^Ak!x-*W>{q;XVE;jv~_2~J6kNoCRM)UI*7nEST*Kq?U> zVq;-jWlDXh!6CDKMC^mIiJFRMWB4l>0TfL%Q4Ve8s5z9zlL(eA5%8Zg*XK-noTQ;bwtE4i$5n* zB*NmJnWStYE?a0Mk-%z$%dzt%5IG(Kjw|pIcof$-+}}B(Sfyi0t;zAipq`ZYt@0s8 zu%F`KVc6|eFP)~o3cfgZ^IC|=>*|i}~>vb-uwRr;}ubRA~_u88CO%3203ipl(=MA_U%=kPE zP*bD0j{SnQDkIB$vZMM(r7)i51WyX*;@v2REngyDw2lFIMsw7=w?Zwo(dyTu+HCGE z%5@1o!(Oy(Dt7N`_9huIy*cUX@HF^^eKE)TvXqxH5dW*<2>;4-n?Gx8Xd#Qcr(}GP zvy}@eH<`d<`U?29u=X7i<`w*J?r8c2OMuebBZ5X}qF&n)@?skKiV>05t-6)z=Uk)5 zDG2#PqBGpKQ^;NhisS+VhA4tN4yBT#m)qa4`5=dU>AKsJaOm+@XjxGs5E6 zeJWAQd<5>sdEHNF+>%c0ItZ%m4E+UElOi*D^N(T{UwzNNEQ=kB>lZj(pYm}`d!4&% z{`v@@79dfg_Mh^0ZZRODL_XP^w~HBk{0aLJ2RT@>0!bzDNM_Wx&maYt+UESQX+b@= zxR+scCE{htv2h?0F$J5;`t5-&XNRtJPN6Qi5Ni3%WrUi?tYGA)2^$EjzBn`3%Q)w5 zLwQ;??GA%DyLNiWoqy)f#~K^P-idf-ZuTFBQ9=dhX7s*0O2YSC>Vr@I{c9dxuJOUJ zWwX(yt=eK;72Vu}a>iIb>$Ndp78a>GvmGF{~N(CFKVZOX=_N*xFGx&;$StR*nLBIFwWGtTQXl#T0t!9|1n~@%@ zBZ`t#cJV^!>KW z?*AO8i{Z|-ex5g`(@nj`pHbGgVrto&^LIOy?rMn{ zCQCm!JXYXG%{?FCu-T>&ZTu$B{#uC4FiIcShJLH5QfJ`B)Iz2G-6%nf-s*`TAoUzs zfmR6*X**jD%#~bfBAA__t8%jJb&3k+6d1@#|4(@%prIu)s-@An*( z-j@sQOkE4hq3lvodaX_lQal$u>I?J?+O#u=8@JzoaeQgQz92WbHE^OybiUK*r#Q;N zXC*ZEtTf5AvPObYg-cr10E(#WYJ)^L&(gKsi_b$Hdu1?KBFo^@0c}gu&#>k?3ADjV zjPQZL2HXt2KFl)NvKbN1=fyi%b{C`gT@%5_^{ymilWeXBEA4N{?Y=fsP>OD~dE;v< zEe_DQ26mmSdz))hznI=OS$>1sv@*!hsQ;H;YX234(*I0j_`|E~7cy9*(l7br0X$gu zP&UXUv(wd#SZv?7YudeW>TOyzBtbalt=44h?SdN^Bqf*?bYx;#s;;cFT zhqaDg=(EH2A`5Z0gyM{AWZ0pOZbwDeQY%-@mPzt&hD~4^?ek8>-`&o$RuhfjYuxgt zclGF~^1c%%%MiAZgtZ+mG5sZVTteSt-iE+}U#MViws~y*oibTnsn{$?W6kYR30qdY z<;0|TbSsC$WvP#IMAH&WPK)?hPH4%Z+FK}6)G`nA296+hqnlU;XqltamjMKO@+`(F z3}z}012rXT0d)Ap?YMNL%uHXdv8sGZTzzbqul6Z3pH0W`XhmNTPSCMhz@LO)7#Fu@ z#QMp``t*$U!M1*ZKHZ{@IftiwZr7bVAAfXu1PBMNTRc$6T{r&Fubdsd$s;vZd?>$M zQx#QZdJ<*2Vz&;S9&cTp#A%hD%3$=F zk3vpVLz1S<-g9VO@2{d*1t7WY>Hi_QjcvssfELSua#No!nm~K^$>{5QVcXW*K|Qz3 zG0@R4*D%t^j@0b+2@ozB2&ygRn&Gv^*dz38We57t)bhmD*P#vWt3CqtT)yN&0hL%8 zOHk4o%dghn@-Fyjx8J;Mf3pct9+f=*&I`{otFAM%!>O%y#qjU%>(LNc5*T;_7Iw31 zB0z$v^y}x3YDb_IU*^`P6C-%K_p4Ua?85pkW@~qKzR>UOBrYbAcrmH@^;ITPwzRPk zO0xWvxC8`>z(QypX$Y1@1@2oThc@qAjuTX4&V9o9F~-q!F?K^u>TLi|{$UX`bFLq> z@&)HUm?!q3w=h02qsWfeM87r{_Xh(9Go}l+aIT2$2AN_@$m`FQT|W++%B`d@BJpg5 zzgR2L9By(Hhvd_sGVAse*`cG3xbM zdboc?5Fd5O2IuQ4H~vMbed*3=?s|JSL6HOTTVEguGgtW*DwF8vxWioB^JhJCJYKen zVm=xQ{}8F$m9XevKwS0*gv8XpY1wQ9wpCN2`q#EerT_0XX}dyM$~pP-e`W%8>~{?; zri(Yqge%CI`3v)TqSj^H=hEM&XBKvUVi4H>sz>7-G>@mi4d*N_R+x@T8Y=F~Cwl27 zSlk)R0`iC}q!iz?ut{jLwKu5##*+qjS$cgQR=?~wJPlgIY5osav@7n&@7$3qTm1KI z@hj&Brr7B9P}z5%SHo3TKz-eEnopQ?w4yhW$*O?Y_@O&(XG5J*;N+%kPigZe5&m5O z(q99=5fg1E6C07GqQ)S=1`2chS=eSKyU&@;NA3wkNN;+IR?8|;M(Wr(L`5UJnX>uw!!9oEB zv*CdWA4Ur}KHg}5nX^~b_p4+YG{djr=f8dr#TF>zga5muhy60yy5Sk_XK@*#%kOdz z1nmn3)pvifmm`d*ax6+M;|*!DW)BV|_Gzs72IIF)88T&U^QNfvKXRAVM%YyqMmP$i zs@WgBo!RtAsMLR}mL9b;5)w+{<$I-e{O>*ISL@?{ZMQwPIRvT(q|g>2x26y0POnYE z#xcR8U8P)&cabn%*tj~%X~*3L>AMZx@`bI_*L-feDe|d$>7zw3bMeR+D53GWx~^>S z_w#|&NHoTI!b3n=S7E$85yx6zfPFQAPk_YwI`uTslR(rhtLh636OgFwhh*cjQL6n0 ze@{Ls+{5MvzFQMP_+R`-)9(CVGIAzUut9#Ca|EOH6xsLfjT2Ew@gU0P z+b(nQz41kbx8t}p#}1OzG?Wjq@OEUfoSS{ou$!0|$$hGcCQ)+TqdueYr!T%?@Z+FY!N~i0tW|gsp!8nSF0G=Y}VtEi9*@hQL&;-+iN+~?o?+Dz{)zl2^~?EC+wa(O{3e=90t-Mex zFMKWkovGuu*rpW#Z?+0>=?$!wPD2|t9rN!|N^eGQ4!kDkI(*7(*Kv)WNOH2o^)(Pl zG!h-LKyTG;vO)GNzJb8VkEcMsP!ccD@20Nv9MleEwo9RUF+SZyKnYu*VHKm+*B=*K zfC73%t&rnee2|mVy3=skQ$`5&IcW7s-#KV!_Z+k+WS`1GgDYvKXZ`lZPF13xlsYKT?WP9Zdi}#usxgl8JlM{9V;Y$3 z|9LJ~od+L@fPq~4F*fBhob?)SPthOCxP(IYm`tFDN6@1?VQ4uIVZdwh8AvmpT{;IX z1|HzCV7CH>I(qa9P*?AbACN~6`0FBmtd-@r06V)D3qHdbV&ve@lP?Z-am_9sjf>%Q zff|1}I2P%FNs3CbWS?U{QeE7KkPqRngcYUGbQ z#7)|kYrLzW&5&I?@moN@jHj?;f?*J#68%XROn$12xnB9{mOt^y|H6yz_BL>nA9G z0sFrqlK3gNzhL`+()^qY*#GU5W-eg=r=gkF1?>Oah~a|m|3b0le@pSY5P5!U0q8>H z`LzMZANctTk>@w{>|VhB1?>O+=F1DVf5G;Dok8&e_Ag-n??`4VocF z#)k}{q;H<+uSC-8cgBCy1#h-McPc>#c&w3%q}7TXyQ!*hj_n4C<4BG4;k`-Mo9+TB zrZh=Hx2iiBdQY0?J-Jjh-rBR%D1k*94<@#`cP2Xuxt9qPtnICg$-dZ%s>sG!MKuN= zzv{bOtZ}{DN`5dg&~;j^V}e4)D8ki8+ImyIAt(LOv#t+AGC1?is4%@g?9ALTQR!rT zeJWjdW98U6YVwpy8Mia6Zc|`ATFMqU@U__Wd6qjg23mM2D1N2hW*w#^mcY(ugWIo| zv0hZl3h&kPVWml5_PoXn(zFw&>$kKbh@-6WDU(=e4_%@FT)H1^B`En+wGWTC!_!34 zQ?kd#tMId@^cY{A3Eq^TB++nva3c1QTcBh6`VZejkZsh)>#W@N*Et1<#E-XUMyMCM z5+s6PS$tS;0+U|kn~J1Z4DKtiHwDknAQ%^rOz=J|={HkOL63?nZ;y(!zz73=cV-|O!vH(f{G3k3|>N}+H@S} zsyY|mX?dJHC&)RNLe!;s=G5CDH*1I@3lg;wiJO_Ga|i)1 z@NBa|l3<#GmiOH33-nK2PVj16qXbPRSHJcUohHAs28^t;*d}&6 z@+gMU2ltOhze>I<(r&f!c;wU!6GVFRc7qh9otg4bJ92&+?hl%d#kQ;1khDZm)Q6J| zFU3&$xLz1W82S3B$m>?Wo#HP!PNK`Y2|kwC*mSg*Pgv9adRn0#%&P{M@9Rzf2BM17vWut5?n~+YcA3k*>f2ZE zc=E62vws8M+&mSWpJr|{eId4wN!BuJuD{wOFw6{*fk?_Pzyc-&y_URtOqOehnHxeo z#$F?pu$W7gv9rWKVNI$UXeYY8Evy@z>$jgKJdv!dpnoAK%+?&oBCNq6Ufx>cqLkrQ z`6klYV^yE?x$|eOoGWVYAZ(x#X{TLGUnRB)^sOS#7JSAQUlVU#-xJR@zKCYL+M%G7-{Mza9fBO8-oi+`)k!2V*c%T6{D$)(f za2DDh_OPO!VfK|dFgCQHC@|25#>W`h}8YtUW3dL(j-%oJ;KaaqRVIG6=!jVzlMFYo}IBucu%y z?HYv_L!Y9DLN~)J9Va)Dbl^T+NO?^b}K(5l#qv<+B$sA!0$u~+Rh(_ zX}k$t<<|3SVNFIkelsL=GFQ4YZ*n#CozIQn?d#Z>^?cL|8OD1L ztJ{fcZito*IqX|DKP_`$%3Koib_?;5#TQf_7DqcKM844@&j-83PoVH^_7>l)?}YnQ zbm_IFyq=MB(6h(RK3}&siWEu;xE6@CupnZ=7YslQN=p5t=mWUOj)^Es`rMTDgJ$h#D9Xahi<@Eb%Zn&uU z!vIlX*Yx#B(L){GMD4}5MJ*?#-pY*Lsm6n(Ym_5Z8LJ|sJq7O8lsb|(BOMUmQ6I^M z5X_ETe$Y0;}dW;7` zK|^O)tMp~%kQQmkuR$u4cb7Wl3m&~)0vU}9y_~6c9l;H%CPMZqBVQeg)1dgnM{-k$ zn>Vrn6K z3hBOzorZ*Z+d(f+=v z{YG~BJ;}C3RvR1Hl~&;C0`bSIm{rdDW>gEt+6QiXOH77eb+4nxY!FJUwN}98QOqAs z=~19gqs;5(UpjdU5@wz=mx;ZzZrkYhq8-9gXap0w(IlG{po5w?7DYKJ$@8mM^>r!e z;^I&n&!9Bb#_otPe4S^ekzzG7>D%c}llbB3f)-){Qz_z~^KqL-VrKC%LOM>ykB1Eo zm8-*Vn-raL5zVUB+i!$Vj?GsRO(;Ark5TF?aY}jPJ0brf%8y=Qiyd}?sCx_=?h^;0 zUpcv8}GtwBdQ^)zJDRZe=eQ;u{;9HfI%hI7_(0wOBbgRW|Nec3gO? zTNArda}D!8yLiH0bhy@^jgvc<4?=(zOD$(Bf4 zv}xpal$_hw`ok-zFjWbBG_nkideFq4UOIW;P5;(LD5l3E-f1JWWY!5X*At1TdEBX2 zRaa9xO8+KWenL4s{*&(l1;esvWaQ{X`8g07UrxB^C|YQ1N28VzGQF3+<>-A zq2td%FMuXsDVFSj6XhqHSg^-flD{wp;43o*lis-fMhrxw2znvAz)5RG(TR{-irRmc z{QZ@5aFletrEZ1{_=mh>6wEukwr-yLw$qHga80Ga!Q%eYp4~dnT|b1rM>vY9_bJ*h z(@8{8+*P+zV(E2wgSJC>AEC2))hY{iSu$`r{2@1T#lfZrj)D$_(_%yEe1S}O?{E}s z$KF9Gr;syMR`XVb&&&m~+XiKl_hOGa|8X}bb-@WXK?fF8#%iQ z(bCE{SMEOefglC`Kt7k7_|$xW&)rgyOm5JMzxyGN+M{BGMyp28HAct!JDy@9f!~P; z-QEuH>syLZX%BMqu^-7j)vZ-2m}3>Xd~1@9$YtwPWBzJ2JL1chr4y~w774G6&!zF_ z)wlK&2?FOlm``OR5!0n#I9y)Nx)&^}Q9`-j^ktl6KwK(1VG~5slp3E#M5L#eQ*0AGqCew$U5SdjO_aGlnsIo$V+>?WA#{0H34|R-+0|c5@6^6j zMR^YLzWddiiZwraj`wm0>vNwuPo?{#NH`-37J3dU73W-a*)S>V%a41ZYRq!BSatVJ zN3dfL?m$g?Z%Z9A*6Dil_j8cZ%vzms(Px2ae6p@5HbRNlX9+}sFy#gxRi6Cbe!)F` zDk$gc@4O#0d)`56j!WZ68)LVFQm824R`%KgveXk2>e(T0kEwmry943S}OLjAep z0$s?UfTB@1m@w|tC0vAoHtKU|1$(} zt>kbVG%m5!0+dXe_;Sd-?8JAVa8FWK4B6@NaH%*vMY-w`mZE_tpI{er79i>(l`7)Q`@=4!7#|Qb(;(F%6Yr!PS%>O(o== zjo)$IzBKk-W<)Ke)Vb~0sKy#hR<~G!W;i>sI%DNu^MIiGkTg*tdzE^lBZFN=KQxZr z$2WF-0#0Jskf61g+X-2bQm`UK56UMrnm>IeK2Y%SDNFr#2GLMlRalxM^mfU6+(u5F z6J?l-UK6n_pW;WO@L~DO4s5O8;hgBtC~y<(rriSzcBy{u?UIMkn{@JI;uLF`ng||j6NY`&$E^qOhlinMKX6!5c@AcX* zwf77}aL708siSYOk9shy2~@T=bvw+g^S_{pD{6V>-hNYo9eXnT=uI`O`9KHW2oK_Q z8kuM`@4l{LEXV|z`%b7Yzv&2S5~am)8N%F051P*R5-B5QubH|;wIpRFm5INSu#iD` z;ysIyk$uh^*LP*s`y)^)M;+;>jXXQy)jissq!;zkl711ELOdHHjl8ZCqi6wbIR0`q zV{plI_}N(>VY_VKjSS3X^D4CZ+|eo7oD&DPP;Gg5e*V)N4W-m_VajTygAK!9yu6%2t2ZZ&*PO17fH}CH!)h#x*3dCTZu#$6)~}kH6<~;^icIy zuEN%ff(4YYTZOt{%7}b-M5P?9#G2ntN>#bB!IyGQGsm!GW?YaClMCJN9XsuYMrpF@~ayONjRvnGtm{3Zg)`0)Jg8tKio^ zohFZkpZ|iHiSRARfh$ zF0lS3nHhKLji+eJfps6>$^bv7i-LWmif)PpbtvyrqO?(2U&Ivbg^ppdpl|n%FUP5r zWvZTO#8!9={L>;nhM){{%vV|0#3Z zq7}BuDG4`i1w5hpE+Rx(2elvLZ8hGV7IIps*L2E4c1islbm?YxSmr+bYdsza1QG(N zT>5KA_P8OZHVOJ~P8u=SOKk2A!5>8|pr6tNDwlP&Vz3PwtS&p0_Ukr!-3hpKC)7)# zo@9agEn=Lca_^-u$vOz}&mlZN61PziyTy>uE<{Hg&S8J&jb?etM*?(I{nS-4Tay0QaAKLRSW@yAe(ZrK#(pJfLL3PC$;n=Y!*+P4{RZ zz2?H#V|c|^UN+%=#I=&tyIfaP=jXN}5ZhB=aXC)(;Ul`RCtGX#-9HxV>_I~sM3}w# zMhpArUVp!8nh{kd&q{M0yb7XD5a#bdf0;{oKQHF8phM}K_S*g3$)X9PTIvD?C^$5s zWbKm*?zOpIpBWmY{)w4D%krTrotLcxA`pAj1cU`MP&RPRZ&e%QCM0R?A~(#;^=jOS)RS)6B%GG=8kSI%%{+<)x-9?qjWQ zPx8lk_n57pjpkjyP(iDcN<7a3lhe;u&VKHZpnBK(8Tbc_h4M5QAb@3X((9XPHJX{_u7)(d*i}1_g=nZX$l2rwdN} zYb1u?CAhUArme|>?7CrLPMe+ zr=~@>rkR~GqvOHBn}naU!f)4^z-rV40*0)s30sF8ME6#)THR!T>J}kLzmrnfvH0Ty z<=fD9Y;RWc;y2S-IddyY^CM=>R@*8ZI*&KF21OsO25|DG)=JGrQw4N_X zZ%-yfqr9Dx3m)H{=C30`$22?6DP}Tt7eW{*v^!z&HXQeB`R@;`0O^ekm~+2ILRT|4 zq6?x==Cs20$!Z4kE!}vtsAWazXtAaeCSy3# zBg*e;kCN$aDC62OevvKD*B`zMpOSNPB z=o1DXhTaEBC7*-7S)9nMi+n!Xbc5XVWf-#3O9UTI$~CPSAPiKIxKge}*owJ&mn`jF zeLq#%v!rIPpk|H-JG zhnmorhjk_}MnNjq$vtG^sZD{9lk|q22Kl9UX%H!iK%egR?u$^dR?Y&6_QP)q|CPET zMH8RPQ(Q3lFhWr4DG1i`XNxoddbGn=_ToLXvb?AP`+RFI?XUQWbS@*~M@XCni@|9SvO#u-9Qx@?jw zX|tQ?FeK|NSOu-r_xQIbOmDj*7nwh# zlCqb@lKksLtiON${|Pbce?w>cJ-+j2+^n227Em$(VSmOyI2bK?4!TP;?|Ak>786qb zg%T@?&7bnGc~brLAR{nIT#s^TAc(mw-5 z{-o;{Y~T{zv!cxeH>m0Ova_!5Rz=IEi*b#j!3$K~qsj{8Cj+{HrDy`>N0GhqNiXy7 zyv1#xg|WMn^xiGbHFQNoc|5 z*&Ft(Ph$L3hpIct=hj4yBI75+u-_vOMe#z%LQl(sjOuSZUoU#d+8 zu>d0E9G1lOUr4WhQ)k$3@ZFbrr*$WepeC_G)uT7l53p@%19_(w(iER1a3N0(Sdr;z zDGHvJFYX9H8t9g&kk1L+Hs)qS4Cs*`+6UaEs4JsB5x%sKsvDrAoAYelG3u|oFF_mh z&v0DhU28uoeg|!~uwdnTs!h5f9%Gdz()GyLMOfGCEBvsqLrfS~Xt_Z2H7ciBoV$Wj z*)JtVu`s?_u0Zhk(PA9vnfuf+_p%B08k#F_ySlfio4Qv0hgl=+5Pr2|4FY{$3`7ebcel_5Or@)Tx+yi&W7z7Tx6m1NmSOyP$&#vcg{ojJVV~dG zujPz*eWg*=>AgbqGk!5*Wih4?f#m^T!Ixd!^7hmvXkkAIq4ZX~peD@9Y)vgUV+=^l z?lCi9D(N=%4LfU$oz&+o{pMzGxk(PC@fgs^s(t4$dY`(ntmJ4ZdZme5%z2IybS$H7 zn`Fke^|NZy?%m0h!P*Qn zdcQJRlS#UWe^IwKr=RnpVejyz4|hepA$7b8GY!&!Va_W?K2j8lXY zivQ$DU@00WS{P_%q4?F@O!v~$KeC|%O4eS&cR zR7=#wO9D(cn(c;ztUFi<_08aO&Exy@JyaG)J)Hbh*b0zU_1)c(#*ILCEkIr^;%P1Y zu5u1~0%Hg-CkR4w3GFj^isAf1iu8vt)Z~g%-`xRoUH|5fPeQ2uU3kL1Aq;B$5@39E zQPX!P-ETV4K?3q6KV;v>@QJ2)ut}bpQ`Vdugdl6Flu**u?$Ew%f3Rjq7)F#Av*qLv zo;GE!^DQC9q$;0l`I!;zrgh||NFOfHY^kX11Lhmzq?cHkmZ=WPG!;I;C>p+Rbq3NZ z8K$9nT-LXz%gNSnZDoqrn2?GU46_|NGu!{e?6Cim?eUi?4vZBre-0v&o`+vs*WoA! zoB8Cvg?dYN2+q}0B4Oj_poZP{`EFUAI?s#)rHxlwG|X>(Pc>|aP8w@{hDyw7?hNN! zj%lHqsfE1UKk&(?~@Jf(``PmyMP`l-r-OWvj#qj;Y3mmq?Rn-u>`* zg#IhO(qnmDzC?7HQs>p6wAC#Iz94Ed9_>(2B2g63dFNT#DK`WyeVTIiW#tSmw6Iy{ z3O-?SH(yV@1T6Ltf=HJzcO5^Jlbh(Y@FP;WU`HRS8vd+< zi>s@6@W;%g@Lzv*Ul~&@|Cr$v9K0Y3n1y~D6%vOz=v0g>u$no`q;SqXo4B9=a}Aee z_C3hF5o&x6!tG=Z;-@jgT!M{1$P3rAi!Y=?X|zFh@Q!b(4d>bgX;#!$gp#*9h{6)# zv?Cr2cPpw&ku{lxfy<5w=D}{ea2sQIz)hmP+{k z^_R}yb-BcSYg{E!nvwHBTICe{b6D(c#3(X5>*2fE7cyot!z8NDqJNHE=GX4?S4A1w z$BS-?sOxe_w2A7s;kh*uV;V4Lp|o;)s5VD+*S&VyPSa)o_w*8ydo^nVE@1D#+GvpA*L|@{^*)y%D%w z;wd#QrSaL?W+OS)M%L8*_trCnjz>HBlTtTaLIdo$_)cOzw5kww%&N{*n7uIKG)mzQ z(JT{BO0HgL0+YUlALyFnaas~xQaz+PW$(WeEu5l#$KLM6#||5UcRg{5Ta1>e2~DfX zfnL9UeEmcV{=HF) zO^pn#K2D#!$(orEHE-qbqU_fsMHN&97&E{U&w1k*`A&{OVH&cx>sgtZ}o_zuk3|pq+4zecN!zza#Atu>%2&v%L}W zU`AKrbKaszu$cifi*=zA5BsM+4`TD<(GPrEmYuD!Q8mNU*-8b%k!H~pEpta;JrB|W zkABFm5@+4!T{4WkR(wlq0L0<7sk(JZ4ojEZAbg(HcIjk$C)hjD6kJ|HjJtsLr`h1U=p94uq?>( ziv4h`e35SPx&2`qsnOgy=!1QyyMi8Sxyth3Q1D&X^T1~vMMbdtsS_^v!9;qnx67FB zkBrBy{yVAk`NS+L_fMK12wM$b0lhs3H3@6G!@2+wE8!e;A`6EtaZS%YS%IHuNHrJ; zpuk>k4nR6{)S$j@#|8-*kFYI;Re3}2T`D@;!O6>7by_{ZqIldjoFc^Js=uzE?^6zO zq_&M!3=V+6Vno#mm&%We@crZ z$e`#!88pp7F89t=nU5KdTJ9U!tT*Ii(HdZpQy7i%_nu+{iKx03K)yIHbX8+*=x43# z{v?BHOrwpd}4%I~m3fdionJU*_=#@mbsO4DP97n_E-(=-kl@%~`^r1dL9uV0A zY?L~pIYOQ}i|><`-3xMdX>x%XgfxkU|35B?u_qySg7=|>a#xC%72Hnyo?!{4GL~=-AKF^4i`2{Iep!WIE{l9&P#nL@Yt=TZ>5t>-V!s`GM7bbJiU8^YYAR zVE9=v+yKs2{FG@xWNwG*YjPm_oqq@2|Bx{HpF(@~5Y%W$*arq*a?aDqtf9i&5Vp|I z{Hj5bg||%Ei!$qma`&pMqJ*lv{EyO7q}uMPCEeXhR>v{?>+d&KF4SC ze2WEI6NgV&BiyZpT4|yIx9(U7B-uJ!S-<^o$JIgadED+*;WPHIva|R-cT^rAfbuKk zm`t*R*xs+r?tKGkoqds>J!KGS|5!FoFmWWYH(RvQiYTlK3xe%Ft_D_c^~laaQaWpU z?qKCUzD&Muu5r%XIr31xUYd5sU3t;PA@MtX7H*1&lWxib`IOQsiO{}})cEfk=t5^c zVKx3_VjKQ|0OLJ#;+6=L8BD8)o~B79c-PSzEi9Uey_YxfZ~5%l!TKoiD8z>C-LsCI zg{1&zhhsd1n>C7QN2#uSz^>fKrrANP$;KAb24||Q0tT1qcDi$#QN*^5qWwT)g*j(% zn&t#U5em}Wt*$d`^-?h^X#vl}fp(<~p(lJd;BQoxC4aT?o@U@Zi<-tjmYf^^Fq}YI zQixfKbqw{#>!w}NM@NYtmy`zwD&{39m@g%p2furd;^zsBD7ri`cC6kg&Q*fak1LeA z)x<~THa1zhxqw?IrBCWiR6qZ0qd7wP7-D#0YEWaKVUnqsYTqy;_yW*Iq_6y5(ZFAC z9&=k;IB|Y~N>AVR;fTDgJ{GuS@PcB=@wqX@Hy5%%{%24G*Rb8066eUWZ2_udlp))+ zRKT`^?4jJajSp>0&BW8Vxe7z6Ufbf>2%f}B-$l0&U(AAimN}RE{XXZ?t!3Gvn?tHN zX^c?Dxk|uR*UZM4rZ{0TC|i&vEg;PBS@03UZ=H2$upF3e(y^pTsXE9qm^>MKM1~aa zQFHU(dLXE-n0l$`wlFi_Csm1FLp0SOU@$6!o@D_yRSGGXUULqz1bDStjZ1a4GD>?; z86gSW;VXZM_|X}+Mb}y99HbPl^25k&O@@MLgE|9!lV|wRI7$(FI(pwegK>LT zr;U6iimAr49_5Rv9MHr3YSrh8NZxmSU0|xJw9-i2s{LlQU}|m0@})I7(1@-6YnOs{bi5R8mj2 z(U8Qv_C9iYq#)Oyly)tpL@s60M}D^Yec{&i~hI`Az_a^GkyVZpAPs zKGedLnCb;xM~qRAtQ(6FT65Mn-~M_S-ddSUZEsoU+lZ;FgC_V*j7F0}r!3eyfyh}u zk(j{RyXfqN<`{AM*iQMud{0HLOAp&piAg;f7|0h_m})xy=M)L{gi$xEL?Wkk+*VJ^GGoGg8M)2-4RD9p1>lA+*RaIegB*7gWaYD zMaAi>+Dsf71*9F-jl3P(^*y|Sd)y>uUxB!?XKmJnOpB`qU7Wmdimd7%niD*bYE5jOq;U|{@*#tjvD=C8Hk7F*#VK zf04WLhQmh5lqK0Kq3)kYx!;_XqAOg*cMqiX0;s3+m@8)QM+2jn_5PR} zC!pB`rJXSWZA_K5htc4~1r)>T1P~9s&;9$&`3?=$Svi0->6SCRRnBZH4G1lV@ScMf zvSk{_K8kxk}^v&ZvuUY9is zt3QNMJ&b@p1}%w`px2dYoRqh;%z`o}+86bF$TfvJ%4rHAD&{Sx>!&o@mK4p@FvEPA3OnDT;{OCl9 zliMRVNGgXfi`wM7cIuD!q^MuL4%p~(0HC|2JUs@%RZI`Dd)S3*Pn-k+e320BH{OOm z6KannO%m)rVfR;$SVAvlpb`lMMq3QH(XSU=#c*3(0@q`uw`aNb{klBktGqTET9$+t z!gp4|B%&_0Sjdj)CQ8N3sk$h0lFyR~x!s}l9;sSiFaw3sjI|UcPxwcVr7UP3!Z?V! z-XBpKZ_!)Xm3rKc(BkqhYFQ){oEL{ugF8Gq(2h-Dl4{d5sr(uA$jW1d@a4%Sis#zO zbWslzu_xiA>*B~x2gF#NvA+B~D?0*__4cT| z*QFuJY)%tTUuL3bDDp+G%wM%={|-|s*JxF9D_uYVRg`4Y)@y~)P1);$lmg`#+CENj zhkqmCp_q&uU3AN}MXYd7bJ+$1b`-?d<1WPWRNsT2G~Q9i;9lpgGMn#1-dNzC7UOm}&B6W`%fs!mpttSk9rN=AEoEV#B~M^^{~N_tY>Q9i)n?8OgGK z%?3GE%;6QuI{7l8Cb5XI!xC)i1(xw{mkkux+_o>S82WrT{S{m#J#p_EIR$6?R+vj* z6Joac0ek0zZ|ETYi2&6VZ_P*}Cduci(LFM*M2%Houp4Ah6q?#<>at^Hi^XYSPu?k` ztRibKL_uvwJsLEn56JX$?pvvjf&=;enTRWIXYzC`ILT6 zU-nsXh{5{3tgF^h_?slR_Ilh=fxA1`(MI!RA)*hg<{zO$tgY^jEGyV?7QAbIQTj1) z(GJhrpw-II1nX1SLm`N&!L2+e>=xLyhy^`ldif#cQ`xQr zt`Azn3&L91>9S%-7VKG3hP-*w0Ll#!xyUR z2%Wgj?7F$y%D%YZcFobG1#GmRBpQh+m!-Oipcgw0e8+hPsh1}MglDM?! zl{{zT&51R_q8a3rB#GEM1 z=;?aHWkJk&-w{Qv)7#_W zA;6NvID5<{a_|@^Ar%BhJw25JDkw!R`*a>umCy`Ay6yl~#N4$C+Zo?I8hE@$ZNk&l ziX~eCr$pNVvy5?z@@;fMt*bfj!2`{$Q4AKK3CD&|mN`_s7;6PwwDmPmnM~oWqiMHG_@vP4q+1URdHeCIIH%jg8f1J;Xjsyn z_WX5=QtJF2R{05X?1q=E*rM;u{OaA+q>|P#l%F2yKB~T-TaZ6{U;j(wx=mhhI(qe3QyMKmP%QBn;Rbl`I?H;j<-{Y z>*)kr(9wQi1YwN#b0Jqx`BAqk5H##+gS9N{2M0yE^a@TsU!K=D1iySPu7vHM?d&D&zdhm z)#drxllg-NjS_FesSDf>dYuW$RFP?B>-1q?uqNo0*H(3*?mLU6pi*VSp6Ug}X+Q?1 z|9Dg6J(lSA8AgsNOPdp4@ob_M37S`6LJuY#Q7u*!v3&XrrQB;%e5YQzTd=dsn($Um zs0MPyo0FT*b?xmoSS6Gxg2DS~D?P(^>|MAboc=5Yx)?tO6y{NboRM;ZTm*f#Gis5R zevY;b3fAH`z%{L+P%06yvF_B6 zHE`4+;in?wRM5h$Q0P{~Xs8=Uby?k6qMMeZDO#*%*ASL>+ zQ+vlvLm@;5e4`aHO~GW696AEpZnmAhJa>afZY43-&~Jo{(s1im=D7Yk@jBU!Z#*?a zO+>5{8O)Iql(2s^^**lT{x(ZB8SulUGAqfPwQLTisjR=cx zTSptMgU;mgFDc`>X+a5Kw;&;0WGXkPV4K5j`sLGQ*h|x)lhz5mDwqLemwJ=!;#s_6 zBQ;^LThXy6aD)=H^1QoDBh?S$oEtoodNMTR9H~IHU%*>@cFzPGj@DRkcC$rXsi}IZ z|8PR;N}J?|H=rB!4s8r5oN^eIG__%|qtA8;#cg&5vs%6UcS%DX621BJb%%ZVgNTf# zXtbkgrY^9Y3lxdrobq2d19yx)y?M6WV*_DZsO6wT#WaWEqpzhZxH^;OFO^@#_uPVh zx;5w{YR4G72h7^I3=C$tnwY4qf@luZxMw}U9AAz_#BscV7LB$*ThRu~R<9f30B&j^$g~NO* zmC_*t*+6S1BR#j3H1{hcaT*KmBD60(PKpWw)xJM*3*OwPey<+zcK>DoZlvI$pOmV! zyS0hi7ay+p;cS_r(i*jc(3>YoIiWexl<*~JQqFhw&eyiabau=Xh>?<)JmjsPz0cZ6 zA5NHen){S~B&PB9oWbl@IHymK35+9NgJ)=fMBVb1!;sOK=QgVypGNEr(jzq4K9lO^ z$+cGJ&NP1hk1C}2KSmV$=OC{-NB!eTK*D;wzrM@;5AP;+f%F<hXeaW12V^Jw@U{I=n` zK&}oe)7^qL+GZ`zoDG6ZMPP^hPwCY@0QJYTU;$sVsawQil^J+`) z%n%_G=~3JmdcMmhc{d^XKHct{eHwy?eKV%;nrp0p0+Fwi(>tR-vN6=}Y@%S1SyU-y zOM5VNDb9w-Y-6ih*3nG77kV`w_p(rv7Bj(8$xwK4CPO?q&(~GC{0=cklD3bRz)9@D>!({g<4%!%#%loyTz}Ai^mO|J-^Z#;*$1^ zsMqYXMOD}VqC)D>-(%7PBCk4I{bL{~FJT`wZ>z;Q>vr70#SFpq;oLV*HSGq`Dyq_? zUclPJA6JUw-~BE?fL3^-sGwT>Z0Wkg$B01X849>6oE4pCWG|5M+WAdump4O{L3yvv z-!usFd2xbG=d^>AtR}p%-YOeGD!(pv>=PNebfeq-*@)@_RKl6XV09OJu~NQ{#p5DkrjB#{oLicugka4x&jW`4yQVJ zU|s%Vw0dqM*kgv(xQi(Nr+psXVOZ{;l1k!Kw)ZBt>ZRRFQ&m4zYN*Aj$GRU^nuF^Q zVpI^~_C%8qKKCUi-jC(WJ2OacYn5E$98oxPU!|d~KGRiAoKSHX+@@2b6B4R3QM=&9 zeYdKqUhhENjFwcURD|9!%F~JM%roodHJ;6EY%Be_80?vj?XoreM^0qN?w`|%`vKGd z;e`C8G(f;mSRvlb654*OAvl`1f_|_k-i~*|7{KOFCr_oI1uT{u-S7cMpg_NvgD0QsYwIM%!Q&u@tgP5qqt-&-|Tl=;bi#pn{`o7R!zL z2B+Px&3xvX)A8gyUiPx#i7Da(lQJ{G8Kl_7(T>mo7)o4cYflp+Ns3&9q^Be$S zW>6GSU|X@MHNOI4?68p&sli!U6$;FsOaZFOtP4r=4`3LE(0(;jg$nO}$hwZHpW z;}4HM+ABiPIeYU1EPdH85cM78=vmdvn|S+BtG&xymX)QWtmNe}0+zRaJm#w_qz2Zo zfUQ{W>LJt;YnC!2G#E8Qky};Ng-|>$S$xL6>zl60AdvTsT)J&3*B{LMh^%b5 z8tHKEu0)E+9eDD?CJrWADR^uEv4X+E1gjqPS@I%_N;xgyxG9T;ia>>l`4QVYd@?aI|gvb)hSwo=LkE zGNFc=t&D0V8?i3pj02P8-YEwEi-iXn4n%K&+q(1aP zqD@L=^H2@YZid2<8Dz9bitTFciv(>+nyekXe2IvkXb~x9;yk2|?&SASFUjss-lm1O zdbfQBP;t>|*fcbI={gCw&G8z0GC$p9!-YBimFTFe@rpdi6-$K3tPF19bI5@WOkO`e z{@zWXAX#l*|K$X)+!?@VtCe-GUr6e#s=VTPq>=ftFxU~GdqdK`xNbhEjc=7PTLCa5 z-El{B_hUs{YE2kXk<)Giz~t>C8?x1LuZ@~~*oe;Jn&ER(1VG{Bjbk-avDb&AUSO`_nX0EDIVI6J7dO%Z zCCScwr^>iFX7_~{1`bZs)ALxYGJ_@4JuN^!@yZsYc9YoKUSKsOHZn!$jZo`%Vq@6m zElLLR;OmEl(XrxW&+1)PO^BDgV##g}=5;ld@v<3w_;!a|;>!y>mLj2Xb#TO-mjPTg z)Qu7aHWS0U<9>q#r*|Dsvbv;@iR^@@pwHrwo##!_B zlql67)@?E*hciYaz%#AS06Gv|3=gsbE%dboX_;L6HAAo}p3Q83kk)BR=d+&-!Yx1y zk%5*_+fZq>41OU%A-|LGLzWd;0r-gsBgk4@FX72Ri%=0NXL+`JJdP)YkmzCDW$q@i zLiJ6qPbhB$kW=iyLZ)iXT&>a1>yBj|=m!q?uZ^b$Xp%;{hRn3u{x+~tEF_M8?$<)M zQazdKicJ-xhZJ>Q$o9#w-FzzB@A?wuB!es{{Io}}7FStz=umMG+5)lc|IkQG7jc8Y z+t7VO=QFS?h;sY{Q2A?58z*YWD0K~XUJe)zc}su1p%Gcd9A2;Qi(y2DhhYq?D!mMh zgAb3)E>-&S`dt+}!r2_`i@{R~=-K9R8D?`NbPZb#Uyx$v5l~iyjc?dQA5*dL8-D;x zk>#7W2V=Q^IM*et0VZ4D&<65`kQc!oDn)p|$Gi~#632L09^}kI+(zHOs%(&+-dtI? zN7HaA-!U>KpgKLuk(8+RGVWO}yggjMJpy|N<=I26Bm59%Q%?Ai-IGFi&@=PX19036 zu}YPkK>ik^H5G}z0OoXssgUc7;jx;1s0=sD(RJy!k;lU#9V^tBJK!^?aEwBsp-p7} zpx(v9xZKYRC-{-)Hvk4S1bf{267TYv8jYQXt&R?9Ic@miV~`l$rpAbZEUcxwH!V4~ z>QWB%!hztJYv(SxytBiCPm#`>#aMxx8EBohS&6kHT$EK;^*5K?9sY$y<@lczqj$hX zi#_pD;kR(c{{oxx57+m9n(O;72`7G|ap-~krQ#O|VE6kwHv{~Y?_+_WLiE#Bky+&o zI6Td&o=;c8pKZI_l|Q)W?O?_x9;_MHyr#;+=pmfwWCF+tLx@K~ zxg}B}*XipImQ1kRk>U3!%OZ(uSKsF`Bx(>@-F%{vPM@t-ZylTaHECjID)(}EUj3dV z!Awo9?Vpl|Ge2K$jgckD6=Z43^cz6mT(W)rwXWsmz4Xw4AGWK9tEi5grk#(OALyEE@Nq?V83|>&Gd3BM;JyW^sE2-1Cdw7)_(zwDo~wKt@cK3E&) zS9w{mrqfvQ3W$cXfoNzYuO0oM^q-<3SvVOCuZLn(2%rW9DAM2F(^pPjafYn2IlMdw zIjv-wI(1OcYq4upMXw*PH4RFJj!tHUz}jY!WFZ*HdXZ$gopOECZ8Hn5I(f8cH`z#w zRWD0*67OXk(QbOUhK+wGnhJE>yCo9s{5-{zf>5$pd@T@?EfijAWTV9O&5rQS6h)r@ihe5`a>3|sfp#8&Gih7ixbDY+TDb1d3tK|Y(CFVx-8dR*OSIr1h<8!wNvMP!%t8sx?~k2+PIjtV%v!<`>SfQ!@_V^3*miUE z%jj*ZwEcX(rZrK2^$5Vq*y_x1VXlZdMR!V{zOMeHRsg#=V0$Cq02;C*s1uTq zdN#6RIWcsHSUjV`j`>T<*83v|qpryNdDeeAmEEymRC2B#{gHds02qZ4g`dZ8p!#peH4O7qMB;XvTO~d&v7=_dT zvyj?)8Fx~V+Xp!o+08y(oaOPy0-A2VZ_Ps$U#)!14Qn;}2LcrkZ zS4aGT+EFcK|6Ub62Y|$oQ{|7)!A{iB;puj?5Otge?WUG9NOa2cXed z9wBn2T7MdHA?0)6{UPq}fCm20q?$uQqoK#lRRD7u_+qw3C(6$ppiDkZy>f<(H1W6h zJSs(GSpq>qHebd<1Rh6%f$IrXWO~aP3`Y`SH^NDC z^nrUGj;35qod^zktRB9II~bd~h~D3K<4Rw;3q#MiCu*yVeC#BM5Rx(;qT?yB!A<6T z5|zcpq1@b19n4XmoGZF-=iTyBur5)~-H9=;7za&m#Hm!e{C)eqrN7qr(6~I+$KPT* zprMwPUtLL39e36eQH_1Zo{ruBEeHs=Y3xD=85MTDP>Va6@Mj0GoDOPlfRl z4S8=4J*wgI)`R8+K=KwL+TF`Zpv}CBup4%-3>L>nztyh{w@R0tT8&qx?EKlo$Qbe^ z3xW%-_3^)jLsg%BrrS2Xd6jGW(=W&4uwQgkbXfVnM^qB$r%rVHhVwLJud%}T{17tH z6h{jvJFS#JXI%|z&(V$2$@>bc#^DT$!{JWnLn7v|d}lIBUNS5FSjK&)cmdDTVj59t zjpf$y66$Uyd$zE{+h5n1x>CT=lt#6HIO)Dxw&k6V;$iQ88gdM76^zL)3KZ75YwOn@ z1TjD9;X5b0TURjieIzFZNusPZ#mkNT=GOb3A0n_iBjhID*8SsNUAzgY11vetvLHJ7BBPu@`v`tj(lY(z|@N`Zs3-1X`Q1c23Vded~QoQki^ zkZqjq)o#6l{KsI_+Zazkrt)V```L{qU=gSQo3@khf=!&}ss+09Bzes12&=Rm0o|tp zBqejwfRj1cfrHb6%U!Y^*UFhlD>_cC`CZF5Qnc#{pZ&`?L zlht1O&?i?lpNE*AnT<`k`1dU_Z;Z0JQ{5j~iI1Rc%F6j{|FFD$I4*$fJAZep!wk*f zR{xjm7U`F;dfTsKf37%I706g*eZBZm|K=InA?6Qw! zF z=nn*~H$y_7i1bK_h9uOP0B>c=TSFe_Hc3YT&D(?Gz+V4vqunJ-^D_eym>z|{Cos?3 zcQ=5)x+xCmdL3B!ZPA+>_Wr);E6x0_!+k%-!!x934}REc_Z=;h#LWYMpx$!^H{T@5 zgelEjiGas!s=GdspgP1d3Kqs4nV%+zO@7cp^eAxv`a{ zs4m_S_>es!gpD_&lkk#|NMPIuGq`Sk49_&n}Rkr_jAdKw#|Dz z2dta#l$G;ExX(gGz}@FXgmCAI;F=(Ai}qcEUCvtl)GK9wd96}%;*H?!m+BzJ4!R44 zpQWnl^QvCAN`gQjU(H&Nhj$+bZZ{4~M~6&Anm(NfbQTlq)vsURjA-pUsE zAnvy8QS^GY)}b^IQsYlL4&d9+rH?sMEnwIHZInxuf3k@^WzYLLg5vGkQOZ^ik;CuD zyAFtzdC&1ehxq9rFZDE049A=jxj$@ycB0&WH@(8Y<%Ilu?qB5gt{e;7ngD10BX4Yl z$+=xV(g_pelX~SLG;9nlO}&Dh(R9R07d%rO&nWJFUXjT73sef#7@PLt5**?N?4Y>R z?xO}j;H;m8kh5#zBO7#N^<`27Oip=Dsr1X0vH5K|CDZT1-pHyI7lGJJi7~ks(}gPg zg_1Y40I5i+68&b{ggY{>i(A=GWq0KHS#4xQ88g;PtE-8VFp!yyrQCK7*Wv4-~3Jg`#L!D1n=3S9#G;mmZ>Bt$LnCvB8tr?Lx zJdYscK!>9vi?x9}ah|`uxI-6wDrB_9ExKw-?+3VqkYcPB#nX{omL4wT=xvZx4@gm$H=QF1P>U-yh%@r~r0STe5EY+n@9@_E_v8Zdk2{typq z^x!$!Q%HL%886DbaaQ>XQQxJ^1hJf{={j6}bBASj>JmK*E!fc(m&#etHBi8p`%2pW za=Ai>6R&#A=!D&zT|^;{}Agx0}|zBhj` zrX`6^9R&yC$7vmnNe3YVk($<^*P;EK-KSLfoLM7=JtLAs{X=g~P0G!bu(iKH%R)yw zP=dE) z0M_HD3NM4p_~nH^#O$e|PUEg%U)p^ULdKI=DIgQ!QKoPXG}vNFKikVbjgQY|TYWJF zv;-AH^VuJkiwuXJ_6p!W4@???*%n)^LTOHl*WMNzA4HpKme%1X-)qg7t$#q(!pA+_ zRYII(R2kYvy9+z}7WKuxEKXw4@4Hl}uR?gsy*w{lPzv?8E%?^Q3lEGP=ZAOQ@}<0j z3zCyvE7#exEoa)cXwgv?kca53>Af_MRLeYAd2MKo^OM$icQkssi7Pl?PtDbF{{|Zr^uV$WaC=NTI@HZ{~X%myM}M96-3$ za=frwna(rNK2=42?bUH=(xI#I+lTzl)r;G1ODB?dBlj_Sh)F|4vt$9Qrv(|!&FEEV zjf4)JW}58}6$2L}6bKVT-GoNvv5k;rsh&6zc#L^TO=9ht~tFGp^@ON z-R75{GWdKkI(3%MnGk#)ma|#4?IRshWS9DCMYYB|&wLOs=upk}jE~5Vt!nbj$nH7s zA^0viwp^{f+!O7XY@Q3_`F*M+05Wu`-A(7Io*{jOCr

jso{{gYU(%Y0A@TX#_qv z(J<@_`ct|2Z@hvb@b(Litb2ekn$|2f(smn52pzmL)_wp7PA;~JkbdZ~nxrzIz+75+ zb!&PQLjict>1Y#qQEljVg^^Wnnr6|FsR&b>ud zuT?azS=s2by8piQ1EuK8~?I{ZRM z3GP5}q>QB659|G0llWrnYvEv=?KV*J(*NylF?EJxtajOV^i%aA%@ZN0^7zqddTrh0 zL0*>JyaL{?uMPkiT%#&4_*U-T+p`25(+L5Et@6*Rf~SglznOn;5m_BJ=}EfwwT_D* zz-G}53}0?7>;lSPZcsA(EDM^{+-w2aQA2~TrxyyL&7WJvXpA}i{N z1HBFS8NkQzM;h{Pa18%u*X}>~Eb?TdXH9<$XbZe|<+@;|tN%=DUDwIaP?l!bLx}*Y zom*N@Re67?a5Nr2tzVNe@1_s#F4mW{Yu-kj|it<%BXOvr+v}Gc`9+_7kV{i&;jry+r9)$!l8O-|w05eJ_|vqdLwbUzcsi zNFLj)zytUSSGQg`wCZ*NSOA>A(I5D`{hsD26MMIIU^mp})(#eSEFE5JnDh4F3!^5K zs6~wFJbUxmxCP6dY_5bWt}pfVVKdjlmf0fNXfWySks)ME2LB>Hm)uyLAINk*||im+lGiyi#JV^L}!> zpU$-*SOITd1>~q;}R_L#02pD zM>0$fw{(QoOqi{W1sd8y6w+*%dy3M(zbg=-#-rOiR{^=zX#9l;<-hh?2XDPS+L#gn zG$Z>@&C+;QA-RQJ!XIWoWedu--_Q)8Mh&kyQSaTf(YAv+d5hg;YYYVS;Hq(c2ZK&O zkv;yOxuG{C^^kns0Yp<~UuLYb#J1i&4;EF9X=E)rCkPZrbB*w{iF@5xmx(D+AyCCh znRy;y%BTai^tUV4llL}v1r&Blo5Wr@4|UH)f~x-2jZAgzU{TyZwD+L%E`wK^-gUY= zniO8>8=$PnPDFOJU>v_3%wE@l66Rw@hwDMy>vo} zp<({w9pw^(YMe$nQiZBp!yIsqpxUUK+9a#_Wh0r@!aZR2RXb7sXZ;C^Hkj7*$JvJ&SGz~7Pys^By<&^zu+d6q4_0 zOo5C|fIIN2$SEXhhlxoR<5#`6w8x7Zh<%UK{vVj4|G!6^-W&&%z#+4~fbG(N{-Dq) zpH2K{g!~{!6kZpWxfSKU255`(2zawqMPzQ)#6(f#vL%WMbz@$kwDRUq0cw<9Z^rF; z?LSEM@G}+dWH@SJsb7ZliH`c59EAZ@a1}qm%``ovV%H+l_%C9Af{WuSrK~!zitF7GgV}ux6HUjJV*( z+t%2z)wmUlHgO+Qtb4+xXeW07!PINNcbiiNrcy|H3qVn=tE&B<=(B+D1?aQJTYQls zN(cr612z2frsT0Dl04dxML4&bjH=9PqHLK7nV(VfG)CS$cUPKZ>ZG9cO~SC4gbyV~ zu`M|W8Qz6ut4GGB!n4T?7$(i>!J+B{<6j?|dc;19_wHxKzns|)3J|fzK4=YpHdVf$ zJZ)k6u52ME8Es}z#CFEIz%~ZXvD2T)UOQm-f2rAMU?)mo{B&^F8HAUp1jlX3^9xYk z*`VhLsdad|{3tA0^*VEdmVE&G*IytThsR#~7)T+jXs}M24fwU?b(-VFl}umo+Y<>s zQZEy=N$yuqBJb1&xi3$In-FhZ=n zD{~>pc^;InO8D@-1w^Ca=Sjr^{AtiH5TQsnozts#R|OqBC+3Wy{Uea0L^TY)I=BG$ zIzD5_azi=Ux>WW%Kks6%SAAWc*io`=V@GCV&CuJV?Ot$zdZv;Dqo5mrC)~>Yxw2-C zBmsY;22RB6|AvkbW5=+TlIpS+7s7lOsj)1?b!l^Go`ed{;Ri4%{*Lx&av3I$cUZw4 z*y~2&sLwyno31c)6y*gT55V{q1RP9=he!Ht_^AARujO61nt{#9f%YP`uS9GfUa;&@ zu(K>^Hp-73=~1=ja2Kn>*O3<6bgskF(pLAAU%05p%HCmMPzoBAZ#$>F?%)IuNLUY9 z%`|@+sYJn5K6CtawfOP=<}I9>=DZ6!=9s{-Tx-*I;T{RCry6GpXG>&6rV(3{3 zzJ4hIc&{*8Ix5d_SG~2zHBx znQN6~FqazDWS&>sH7+Zk&TW`@!rS)& z_hr#(Q>e|-;YwR$W`uOo?L>o%7v48Aq8~0WyDX%bpAZB}sdZ*?Fh4|0(Zh6yt%&!# z1e1ffOI(^`=fTJ2gPs?Bo95qP9@_=JS`Hk~>&I(+r3A)37GRIR^hv!p=UA>#2Lr5t zceF4qvmY3iQsGzzn>G%-3P^K3046CLh3Tkga$_I`xuU@~>>sDxUaJQ|c@8e7j*w(M z3f(cED!WK5?MaGj$=40>7wTz`rS9X+Vjz_}%k1M0rw+VM!V}KV68UAp*A(|<*BiJ- zW9bP#uJc$aO*!zIV!!_cr_mXVx|@C`_g8ftHFebkq~x&KH>q zI*^Ww4Gy0JN0;b&-$+x9xm`ieVv#p&)wz-SDz0t`qC?eXkpydtB=h}6w{RvZ55PC8 z)6h`D75UeRciFPMF6zTrO9DmZ#?sD6#L*-#RcfbrO#EBFD+@a8mf^5`u7Iv{g!Mv4)*k%g^owBk#whKY=!0p^nCPnPIykz0s)f+!I;*Fq;;p$SMAP-7E zg=&)s4KGeumv8#l&jp#Z_)(j@t|x{89JjK>P?@=2%(~oLv33V9{Al;f!%Xm3DSRyu z;z>J(3LP;5aGfb3z<$|srKexOXqEN}SJ;uZs>3I z1mYEZV=Bb(S-m8gq(>xO?qeMLFUExE+0hjWp*SkcQix$Jo zEn*+tB$iq1U{)0Xfnyus(%EZQntNHylc(DCcIMuWNwj)j%@-d>V;a=-LHN%Wrl^36eq^A>aW=7^MvPwSrvf&Q}!UH_ivQ2ku#^Ma?t72du|rh>cG zX+NbSCh4{yU(^{({+ZoxQb!hvrN@rma0(a&&!3w!TV-9kX3h5k++97{cwVY%IC~;- zxpVr4`Yljn>MszeKh$?pFvGN=L zP^W+~So9f`bge!#Q~MY#n&1C8NDl#}G8qcIoef{A9o`s6LAziTRZc`^+tF0C*fZQJ zyuG-trffVv^)ZY)TU}j_oz%12di#eUVoi5W1^j^AiE_V(6Ao^1VQ6CQs*;N{m&Kuu zH%E8FU#SH&pmB^u(GPjy(9f)<_sx1va|3ri!8X7EQS4wp`nQO(8NRFNrL5}Cm1U&F^ z?LHpbu|_^_b;eVEplycY{CQv^J2wa0N#WPilZiKw31W>Pzzu>03e=cT>DexmhWAVo zR8@QF+g$4e^^_=zVM<3T0;CsOl7{oVbTp^jzIog=m>%K`T47L?^1fS!9fQ67W4J#4 zX}FAjAFjss>wa=b>Y&xDN6KGNdZml7UdM(kh=~1V*z=H^31Mtmg_ZiC(-knP{~5CG zT~pTkqZ;8svv{+0KPwwdKWm8Lc1!?|O&i5bl3d+nMXiHTAUkX!<|7>_gR&u-5#%M* zhq*CX+xn=N`Oc+dP|=Ps%{%r;iFD5=FBsQwEJ5+KSgn`q1aa$eco>bOeza8FS?K#u zg4uUmXy?e2J2r@k!+waOZ<3yOwo@KTs;!gftauryj~_ng;TK_B4ee(v$vU0#s_gZ# z3UPZNl4PDUJzf4JJPvaxHCXgE zi{pcAIzNUY)VI!Jg7^5TDW14%raWx5@TTTc4o(D+!GX~4tCfqxrz8$ z!d~WwE4!&vt_3Jc{)n%_;d@>CEARZkQ@1%KlO7l2N2^CM+j7cIq^L=ytS&oEUFnIc zc+LAAKUpPX!5Z25K1UI}cbXCGm99>l%J@wdqW}k9S;1XdaKqg|v6Z*5nVA<-Zz{gs4W#vC5q4Q&evdQR zdnv!ug}&pt)_mzNt$o289av>z^Z|q=wwo`nxYJS_wzVvuLKv^a82N@yae#F$@^}<> zW0?{$B78hW>-6dvA$>$FiXpMHp^rv8bpVIsE{ zG8^}ze*%)AmwLU^88kQYpH!6w`dYj$DA9*tg18}jFvu0Y77TO&*Q>2>y(;-!-yHm4 z)s@yOUuJ0KN8<)F3FIsgF&kRrz@gGKY45=8u2&&^p*_G#AzyR&DmRwFN%Tms?a@Sq z&?rphWI^SbzpyvvQ>w?tQ51E@hH1zGlkZdj8{%~Aq*+RWQt^W#gpvE!V>I z3%9dZSDh~2LO#g%URFIMr0u2+x~|emaB^;EjBb5|F@tk8Y0<*!hu!35Um_v(~3lnBk4 zSNU57`Z<`yDkUH+FIER#3cMtAS|)SZq{c*VfZyPLVTt&|^Wu?&6tI5n6iiaNJwo;I z@ik1n$hPrx?v8d7bBs;?{L`uhSWng6pxBV&b19`m70@d$j&*_v_%}k1i4C_bM(?v~ zVkKrDmdB*xaT6`o>*tu1c6^|}*x?r_pS@%U$r1VsWUFLfv$m;k{E$M91&=~@i#%vf znpEOdCbH>hpM|=G!+YZ!BtHJz=95Nm;Stqorop~zAfNLXj7z>3r92Hf!iFuOPS$~r zVbc*cYAVKoPS9!LD<55#!d=lyf1a&MgXiWv($T2;-DOfbD|yWrN4Jd^^Q?JRQq7%` z2*fh#&XsxfrveCJSxS zHekHrxyyU!2#aTRALsHI$>V7=JUjYZZBdl#Daml((ika+!pT|#XW+OIk0&Jrw-I*Y z;~DcH(tBb3p*2w#g6+7r;(JSm0mtZt?;WsQ^31AwR<=_tZ*r*_Uqo$t3|b+QEZ9AU z{sEg}MZLk$x8QuD>i6md6OXc=fle#mnUFh3GcU|3yPHYPwODD9iyAe{NN(>L?zc0Z ze2cra@B_`8;9C6noP1%>r?O6vyVH}DboJ2`TofK_mvxm7WYL+{R?vHp<7a1N#6Rdg z!&)FV1v@Ew#N>UCf__SvrCbwgQj8mBE=@ko0s24culxs<$ z58WPBe(bP5{nmN-u<;?#-&hs7-^RQb2j~E(qk?jV@95OlrYRCJf9>g|VL5UFBZ?3F z?lqsiYHWSDwRQxlNoCF8@-{((%u!?DDipj23BWdqgqVDmklS-1j`Q1YpE$`+7RlCh zfv~B@EC<71)9@ZY5v$M=pT%FeY3wgt1c~uxZeCYvkw+wTU}=63UiY#NHFc`!AA>>=`glvo_EAkU@rW(91O-RXJQsnb4{blcY zUfnfybaGb}>k4hT69L9#TvD$jliAlUlS^xm(7{dm0*?)ESL!5g^i~bT>pBq;1ii4awYlzL$r1}edutnDSG6~g$qBJn{X8J zY)uFPBT@?D(NP+&IF*RtnXnmWyYYn@^qw1MkggMsH(PNJb;XyBsDqj2`+b`;02d93 zRiv&-2NhaplLNX2FMnEs$~0T%X54GlniAEq2f6?2=JJ!DFrP* z{dngATfWPC=~$PPL(pxvVURB*%+H-8LXy>c{o&^&7_6IN5LdCIbwG>l0B_Nhd8>s0 z!UF9@CiM>Im_TR^hyyO5E}JL)@Drg^&71NLP5@O^f*i$L#W6E2nGTQ#)#_RHu}7-O zVfGcf&=nw0tJ+t*pdJqt^nv2?Ozp4U= zQWx#i?=Nww%D_TlEglv;aW&72W5N7(Q#0r&#ISPz!ZB?n(ocC^0>M5-`Ffruk+^Jg zban$z`t7VIa#X2yN}6V6NrnF7zOmS+_*l(UEwk*$w8)c_Pb5w(eoyKUR579ZXutEB z^Vk8lps8!Oam7yuMx>Lcr1R~V6Jej<`^T7y5U4I2FU@00yW0jKQBcddEQjMIo05sX z;n8oz(GbP2kQR}VCB0WAya0m&2A?MQ=w@ut{<0ACie9#XQUa`9LgW4=)H`ktRFL*#0bt<%c_lS&C`|KF@ybgJ+UL?xj9{K`s zba!Gnv%AFG1NR*S(|y^!KFPP$drb{)!HGx6I5BRo+r-jB=#2 zIkQjm1wIPuSS1Bc4;~a(DM}yNBD>19m%aE}?Q3a*q}OElL3?4yE7m z^k^!xyDBaGn3&)xIa(JTFA;Ah!(8B7! zu6+4ba)uqe|Nba(Q)=($m>Q^M?8tr;g801anC{men z zi(`n<)O}MGdSl<6=kbct#NjDJ{k)SMkqWC1XLRc{??@XB0fWL*F*@Q&%w186U2n+=QgRxnLcxB5(NsC2aUSNvsp!S+;ZgXYWg7TrY`4X8`-8pxvdC zu5pQ2BL!z3r(rQqRyY1^hM!1C4;ixLr$KgCT0xXqvBJY-_WJmzZvz(_wZPnsKg5Js z`iW?3)grOCCJ%|OJLZNF-pF7o7ke4!M-Mu1V~L>-U`J-wMdVwjzR^?TdMZT7U>kg^ z&SMqfvKTU|I=xF}pf@*w{T+COi1;;)Vj6xc@lW2zM3}D3Op2h4SAtt9>hY#$kWN7J_*W`$_GfM&BETe2z_M~M6n?> z$5JG<{N{5^$B~}}$tOmNH$EOzCsS1l0=H5G6Uek&;%)8(Gq}wYE73#UZ{5GTk2ew^ z=taHP7eF0mBt+bll5xPM9t=*43EX|O5~wiAv=@uW?r%1tc@nOK)J`p`_-ql3i|EU1 z&NmpQ#vAE{u{G!mF_G7NIohP{Q*V(Itz^D|&>I;1lQx)Ein z>pOQkcA6Q9Pb6nEQDp0q*+gZp5vkEN=_q=bwy$Rdo?qD;1vcDAiL{g%vty@>V{*>v z91-_an~mhiW&)+%ra9VCNAPkXd%?X@ClZ@}`&3F0KjF2(gT~;JQ2dk83hC$++U=m# zWUBRqX9DD%FZkufG&w`nabrPv`8*&$7x%T{SF?Sy5K_}LSqJH&4==<6Ne9WUs{Q+S z4onP$BP{&H-R?!_2h|9h(bV_~n;)0c0}ZSI;?FLvzN6LS7w9RHAl=pV^OEwGoGWgu z;;Q(@>W0Hm;`I)UKUfL1^4Tk-hnikGp%OU#2|gn6EY+tP~ayk_p!?v2xXFc zuv$tS@;n$uWHIkV?7|1oe+0b8S!@4G+W5bqf>$!E4E1wySTFbN(buGY(}BJ>I-loh z`lQe;Dj_y|cXV5Zc7@&#$m9~zbeHx=h#1+d>?D}FhJt~<;x=WLBC0JYj-YSlO|QD` z9&}R6UV%Y=d2{>unY}t>DaHj510`q$6xf%mS%P>4uZUDyTU(|W_7Bb!>E#%oSPsHS z1*A<-RK_TyHP0e!`Hn6jyB0rQN zFbo;MV~zI7s@*angKlg{8elhPv9YnOKS00_SmuE>s4@Y|x9 z9b}t;tYrTJDM0$?#!Bk+4& ziwF<@j97_sYkjt&K1nlR!h2i!_T{Ir+rb22XP1m=C@IXgojQPf!K=@gum^n{|4^*QCqUV*4Mfwx- z+my8}D*QWlWOM;vdOP6u zj&+Z0_^>|Eb2?d>Q&%7L{ZgeXl=uz77>;z~9!guf3R$m=Sc$A9p)r`?TQ%?&?sdIR z`n3XQDWsms_jRijxw*`{&zf#G_^%1uc{#qcK=gB>#lu+2L4IqWOne1K!4! zyN~xJ+*Tad0Lufd5k0uqS#OVlhkt=6rCCv~>%8F(`PM%7u4caed=Rv}^dy0Fw^@lN z>jR_bKG(F{F#S&%)Lpa@&?o7?@Z)F1Q`egJ=JnF^{cA>L!YMQR!z3*l)$aELDK9jq zfElZXV%{}nU%gES^w$*hSpL~oK8yqb=}s_KDGEBf54jX_lfM2XToWamyE997%v+Nj z{M;QjSc45KGeNDxy4Qt2uFg~^SMVLAZ93wSyaUCr#mN*pTsR5Q#dND~Mjyq+a1@ zne|KS0G2BI3s0Qn$V4J@=WFwUc(*~|U7RCvecF>KHYsg%iI zjpIuZP2Ysb-XHhPUokbl{-pZ8*!59?x|NQbWDP>&zx9zPD&;}jA=p@-lV|tw zz6S`Sn1A*$)6_Riz-?+tgkVpCTs~?9l7icZ@_bff^K>9N*fD*rlu)YrcY;ewLQ&Q| zFWFp*Da50#Qm!fWDYC}-ob5&&d`i?xQDn8DJZ!$r=1&6h^6Vxd(nDJHM2)^f4`-x! z95l@5#!5{P4O$ z!14Ve7Hj(di@h(8hq7sYey6r~tN5i!XY*>}m3HDjMiwlHHIjPd*OywCGKclZ0=xtHgDdVX(z z^zn(y<(lidzUTKikK;U#bAa5bEx>gu_A-dZqudG5GB7>QeQ-F_QIE))H8?iF({5{f zTmFT6y{+ll22SeBk2%8%&#AD%B9g}89P=Xd2%jqTHi`Z-F1F7uVlMHXKj}^8_0fi@ zETFY+U!qS=yRN`|CDJm|Q03+$q2cg@^I_W}+PoB`7vq36j&lHV5?IJL2=+|@k_#iU zL_PcBU*mHCy;mwfHfveNcqnm)Y5cQdM+mBV@K%JU4y?8`G#~%|EJ-n-G>>MzTeSNt;jC8c@FvZd@PMZ9YO)_U-F_K z8_}8@KDM8g?6ZJz2j_}*94=d>>IHUZ*+vJMD28GG47k8pVkR=1MrLmfHeyf4aWv*> z{p2o8H-&Nq8<=F{%gIfwdj^bkDL$iik@?ke@|4f5uBoCi{4?jno;q>bQD!f2MJUSI zcm0DuVx9p@4NCAOYp9h0r;Qi@91rULoHrT+M>Ok+4Z711$WNDwoSItvop=LB?^2cn zms*FBdoOrp>!);yb67*tnf7FSobT;Ns{ut(2E!xDg%{+#%K0gI7^*|f21@;|>z{i` zp$igC%gBm7MK^u;g5;)DX~1ONa(tZMe*BF0(PC*)#*{l#oTE88(I(tIW)W7)ah2Tr zN{Wcd=0m-4&N;6-`Qk0=m$l=T9;`!(c|pAj+M2Pd^l*Ez*4nc*)S;PI#)^-N7p~X6 zSBx;5Qs%z0dfc-8`13L2gEFh~DSuL?uptE1AIijw)OB~Ko=KdmTnVw6OVd4RGcA=U9tWulE=C$a651;PczNU&*!VK zV^S?zwqONwnp~Wx_5d!h3%gyr=+ zC3L8yL)_u4!Nr1UW6=#GnSF*A2njEDUWfZatgDI{3ENhOk2(;wYYkMfhIPCeHWC6H z7H0MAJYtM7kqACAOjY-wzhK~dN44nv6In^V#C?#tnceOtZ3<>sM6L&~RtJrke1nsi zz*Ey}wA)-)v2P+aXGZNMuN9E*#V-Xu3EsYo)^ZoNQ!9z-H9IsCc!o=2^ws-mwk#HGg$=9>dzQ-sFQx>$#2(mHkY!HXf(Zl~MB}rNP$ap4*5i zW2JI(*Q+1ob#}j#RoLZ!3Rd^oB!i}7?Wz0O#Toh@qQfq6y2QlWL)@?548fk_$g7e# zy>7-!{l^{c_J>;-T?^|@eNDez>U{-g<)5z4Ea%9?_ndy-K zsr72B-}_-pwiPWC4I=`Gz`Io4@=puWjjY}hgj{5Xtw>?L6n*~DI`OZ8VB7wp^FzOi zk{huH3tN_h?b-;++G|u3B|cuir(5!0{yWcJTE(s$3Uf<0%I|eMX<)uYBkn!zdI2*B z4`nHu37l_)??SkCj>NC12=%2*g|q1pN{VO;cnd!wBq(N^v(qnIyEaX~xYL^@{Mi!K z$|767$R+S_JHyTjE){e-X*d4(F?`*9w~U*ed8!pus%!7n*X}85zx9NiH|8dui?J@1 zMly|e6|n3p0pr(?2D%sRnd9`4Jsx0ukK(;&v+-d7*dzR=`PLz+jXK&NuAK!e^u0mtKww8}ln#OpNX5rGhk{%dE&ADF4i5@^8ckiv zUhcL5_{o=i3RXSyxA^_X@zeix?Q1Kx5I|z6Xj$e~0x|ESJ}BnW-o&wvWOchB*djBC zk}<8+ueOTkzw_fLaAe>r5%UC(TJuQ((wuF^$N#~J_4^%oY3znyt=v}-pkRcv(bPE^ zJE&jAh`ASIf8_J&zVJS|?kD<;Luk&+#MJ82qiv}^zBN@xS2*BrUYX10uyB~HUTk3LS2G(*^$WOLJAUa<$S$iS58JsI3mN`5RAUE-z z)*px2e+k>V1J(&+fj1WBb}g(imm_R*=cB97Bovb!J~*<<|@hlc37K za_*mHzF)KazvQpJI_mHWMZjvKnaY*hrXqKL4N`z7uw}7&%N$ zx(5#${2TuZxS^%trMr@LbcnaLnAoH!p;ELbf8LQpSl#yz3%TAn{c%sDA1i1p3^-Up zWu!0|{U$}1)3~{S49RN9K=|eY8u_t+zLN#?gR}cCRm}~~&SaV0lV=mcna6ID-e^*& zCwwzAY7zS5`1*VK+}yv=wZ0yf?Y3h}NcgvSIQ&`1GLM!)i}zHsye}Jx(+>D2 zQ^L}kA9B&ff^x$E^SdRne#^Cij|!k`Ln~36TyYLLCVc^X)?d)#hapdkBe2lwOxM9$ zdZJ)OY*f=r851c>CaFsOJ51a%#+uv^$%hU~@?$R+lkKl$B#k@?mI;D+oi`&gDsvC2 ziXE7u5f7Esx#+sj11wKtGXm~xX}c14of68$-cqzQ$i30UK&rMI$KxA*n6f=B3t1l1 zUCo41wlpf%3CrRZXPQ5p>GO@TIdJno< z%=wr>9-9a1Y~PUh&5>DY1;vJv@e82LKI7IyPbybV{9#Ii;3|Aw$h-+#PmA-De#vTj z%@O(B(`_S*d5FXvUPv!!ha)7RDi;dpAynaUGQNGy5*mxlmPvc2h>Fd z5ah{~$)G^74@{CvBq`zjGlyGP5xwU+7!DS6&Qvcx-CI|G?}>C}aU;Mj}3reRTp)}J_G$g{?ehU|g|xCuqiipADK z&es~eyTcJQlU*dU=Hchepw9V0;oQ?=TbkED)tkEd$alBE*~$CHjoGq8YCO65wVeu8 zqX(oa{bl##rc?H|+K-1iP)w9DFBnd-4ku=(Sr&-cIPVN-jD;5$K4ik-`8*}cX!10t ztcRaRq*>{UI6D6n_dZs;_~ZTZmLW)%e2qcPK-sF8QmRLclVV<3e6^Vi>&~}joaThA zxSQr1IpA@8@lXM7Z ze~F6cR@TE)EX`f=Zgv9-cJ|rIgZDWC`CNhHs74zRi9<1sA?ZKiMM9^wI>KxAzi}+Q zzc_tanji=@c2GZ~>KafrxXiD_0j|KNj2y!JiCeU0X-f9VajTwK>ioGovub^XEhI`N zif4ql+La|g1(zp>9=hGiPdue1nPTIubWm2eVuDJ*|80u9;T!zOXvj>l8pcs&Zy~u6 zOkSPEaz-lScpQCW>i8x3Yi5>(Ji$Be2}-n2X}JzYAU`D#Hw>TQD@W2O>l74mB7vYS zBsar`UoBQq29qVpWj4JW06)ye$!WIV6*V3Fs1gkE3Uy~CIP_MK_jjdqX4?JqWF)d- z>E6BTLUVL#JIA{}cB`+-fgMb?5PYP2Wh=sRqs8W6nx^D%%aYAH(7&QD*&%(eFyJgHj>qQ<9} z(Tb1pL zIpLaJ9DGiD#cwHDYo)_yvXt`XY1`H9w#S^Fh*Epj{{Wv(r6EC4O*%>-5|;c2uvi&v zM<%DL6Q>Kt@!gviWi`YCRq+``^=dsBr-Clb?`2JGri)}v?rF6XJoPE{jM1&N{)qU=s82)Jxs(N?DavPv-LHv^H5xcL^~XsS7TN1N_`_WJf^U_(IdfNE&<=L2 zvcp$Zi^x(Ke2BaTUMBjCm8i?zUskujB8d$iy3^z#ScjS(){m&sg1A20N(IE9am=@H zd=};_F6xi-n?Hd4R>ap0U~e+ZNf=%8(0fZ%mWo1wSg*uLOumloSJ%g9l~I4p8X3rUXo`GH5f=VT{Y+BV_0cM1;DBOLUPNtC z|FisulyPZi;1b(ZZ|?P8nf|t-bx1IozN6TBXud7Cpuo`kcFYZDfw#)0E|9tU ztk1y^f;B>{xI4x#R@u15Qzyx<`tzTT`$_wRU>&!(So;mUBXDiRz|q-~dw$8n!_*izl=zcP`aP%F(f#ays^00Ihf^+N7M^)1@peKcDgZ_IMF}<@Q?;XNz}V;~>yT9CXN1QP*>ZNp7QQ)Uc3_5F z%2tQFlh^T51evqYxIa)>mZqH3tJ>iwm+iM3h891~6tfc|OkR1gULZ(4h<6N&(SYp{ zaAv<)#VyYuK2TB=3>j5m9O{Rc4{$|jyt;93T1-pcYoETNMxBZmgv)EDtGIM#Xf)VX z0hvh*Z#48h9epX~&8t|}tG$grEGpqlS&rQW+c(}+Zmu+v91(?+^3^7o2OVYA2nJ3n zrs9X-fdWr`kOMr>Q;Uim8o8~vt+fOy6=UvMwv1N5E6 zve+38D+8Ys`70&F#Ez6lLE2(BKHVAaDxvun6WdB{vjv6p*Ob~H#ggA{VF7tg1#Fp? zIQd_j$0qR_LGmL3aC1d>9r6UsY5Q*bm44&<@t>`xgD3T~iW9eQ>Fdhee!uYseZ81| zETF9|Am}m;arB=T5R&Al0P^U~1*G^xy7(^S4CZR0U>&Q{5rRmF#EUOjZwbwo9JR6P zhic3E{EG0-h~~Fe09xOL&FYZ(ksr?^$9jQ@>mRLX{ZSUV1&sl)h}3{Z)C4mp0FBtR zSy)VstXKc9|BwCOQ!4(h4t!@{(TYTm;XQ^`iNb{ZXK8$zb{T^~Q>QfE+?zXOt8U9> z-I@`E*AEwV*TtH!SnBrFY_o_%-3a>-Bo;QCb;cE$0ZV|~>G;9a|F3Q8|L6hxqgo1J z>Ve-$#5#lnM6z{=^hSF#5%8t30ZG6xz98HE{{iZ#;6cSIP``N%Nw*A_=ege9Q&f9n z!*f(Xo{D+iG^M)u?6ix$6Ru88er)9hD)N}mFSz5#_88swP-R*3$gnq?!b&T3(N|65I$cgpB2^g2 z!HMoxj3K}N{p^@A>T@0elCz1)1iN!(gsvzoAYNCmb=+QM$?J&~mEU`KFg(d~>A9QJ z=WM^xK&3;(=GtzZ(%6aV*((eztC26@q8B(1Q}Rcgf~+i6De(lpZKloeE@dLho zF9j$+yAJSJ$_I%gLMmVr=Ug{ME^icJ0jIr6gZkX@|A(H(JcpiQA4c+?Z6|4OM`(I4 z8b(tS(%vyHT#g_@&`&cG2;KfMHKnv2xenBiBvh05iPw#F$flHEv>@)P#*?eEM3a^d zCn#eJ_qjfg*pE@9n_{0lSSiU=)>Wk~(iqRC5jd)>ah5P=+bkM!xw}_s2fhuVSrKm= z4w<_WDDer&l-Y=SAF1e-1nf$_lRCJsuj5;6%#U8ypDml;8vTCn@f*vtg}6*WZ7k2f zH4J$pCrQ^ga!V5CKb7gn6&n)Zl>@YozF-n2HQYkiVtW)CAWLJcgad_8qQwlz23nCtv_@6*9LCwiG%V%*|m?_6MEH~-NDOC1cDlq{ghdXYP4;n=e|tGY7F?j)Rf zH?n0OZMB7j<+l_k{h7yptBr47;9HGL%3OKB_f`C_-}}*~v?VrUquOp;SvR6_lNP$M z3BZ}jIkli3+-~hZvCl8ZT{W#VVmQH2H|)5Exw&MeZ|l8dRm*z!zbc(dg5D>?5NAw+ z5~UKmbvmv`h$@bB5c_#a@94y9HS8Of&=%_ue|vnu+IC>1x(;DohwRIuTm)=cbU+$% z!|H{D*#oZ34GMp0aYMkaY?n1^i3}r4o_@4whtD{M-J^d=99EeJV~Wk2QRb=XO# z(CkiVJV3imd7`SsHG(AJ>sDmjf9Ek@^ID5E-%w|~6luN&2*2_FKeYUw=CFOo17sU<^_8Q zoHonMVatbI12$bNK4j~+FU)FJz!)u!SqXgw`%3Eb3kqMmU4IzxF3X6`RxzMn&7;Oc z2-p=4`nHgoTw!F|1M3iM8fDT`ur`TjNB|ieHx2;J)NDv%dN?xBcu&6fXC$*ljn1{e z8M8YiW(|Hkh!W(o;ngx`cgrpbP4@lapC@(SoVWaVm+fd@JcaVU+kcQ3`&SoN{-DqQOeFVbo(s_D+mY`FK}w)U{#%**m(3d{g^`Ds_ zK)DF!Ush8lF&#zWap_Xg{8wkoh3>w1vmiJp{9&qozV1{7fDj-DdhvE0a-?i6ztrog zmlE>l--{sLFj;w2Hv4ED(rZ0N1LjGTa=Y`ixl`X>@imfq_9F7EF!%m_)u~yX4Hb&J z8c4D&=5AOnMy)qeOPE@ewG8CJXyFM{=y4xMg9{^Y4iY7ZKJWl!{>%OS>Eme^{WzPT zdL|CCSBT;B6rTpU^EkM)v9#hJjSc%NkMimsEY3JE>PU#RPgM2THM|aCA=--U8Mkl1 z?m5CGbU;t9B$WObt&g|~bGHXRuG|KPWKPAH@oT(3V3_}99nv2`-md-$ng{Xl{eueb z*NX0+{{J5aK0oJ#zvD(vAOS$h=o(O}Z#TKwpbA^l4}t&ZhKc{KQcTX6l)K~oydiC( zz#fB94{NJOY=2BKq5UdLhIEsNpo0(Z2i4W`5oBp)ZdHQGaLNmUh;}}HI$@-D()8we zode;f+7BQwr(;u&3o^j;5TRL#arPxC+N6wsIDD~#{W=Y$)*;zD3;gqJTss`qar#xU z*Kc%J2sosC?7QV^zglEMG%AC_$F6L+qHq2cZ&epy|1UfkfSUgaOgFp_2n>#7GOJxx z%L*HE_7q4i!dii;urUZ)y9}497@)>j3gAnhwqRSKy3~{JbYPjVn4)n2lFV%SKfMZBU`ILCZ!J}*+%^Gny!Mr zy%p*Z(`Xy0ZZP4hOw@U?qO}WNriz5%-tWt1@2lG4AARoM32hAe+@)6{Sdbt?iQ42e zcCSOY$-p$?RR&VvkC#jQ(Q=F5`unnF9Ejl6O4q$znYqJh2E9^t`}KPd)FK8_qbeqL zjyJ0b%2%#Kn2UL_B~b=enl=c)|Ih8!K&iy|lXhjjvoL$(bMjEwwP4l5V^#a_h0?1( zb2ldlRyZJFHA^fsFK^yhTX-DsxK{i0$_p^ydcaGmZyjO*hg!Ym>E?;kc%$I*xN#R;M-0B#8%(yZP}R@wIJzKzqUhCI zpNW3*msckN<`AfXfYjpTgtIm_LGYBAbkBj;tOtHV2seXihV-81UAf(h*X7k_SRroS zsph~9v5R^N*oz*7g6Wp8&LdZ(3|2h9jh(Pi4ulz*euj0>0(NuNyvMz4aM{>+>O;m_ z-(#~oa*Jq_pM$SA(6F*u{k*|d=XFRMVsdf?vJCoA+2hF1XGqUMl?Mhs{17=|yAFA0 zw+^vECxO`ufeJRm5ya!gCLT#1Ia8?XvtZ)lh>Kf+8hXjWKJ4QprUd!VCm2$a0>Rd} z8#y(&4$;7&(PNY#0{#=`+hCc#hKL6Et}sXZByJk@q#}*u$!9*XQ@pd)FOHf<>hnM0 zEz{ZKp=lyc#OdaID6^Ew6FO#hs10Lnc`(M(CjEf%Eed`-^Cax&L3X2^cn5Mz^$iF4B65xT2XlwYs+i+S-&qE!#7e#J2ed)@P(RpWn^PZlL#!f27@p0pxuN}I* zpv3ds0PZ}^$@0`i5?0{DItb7-*j%`*1r(dfk6@L<0KQSdL01IjMn|MUJxVf}nK~3c z24=9|8zgy__<$A&u$q3H<^SY;A)te$q~k>b%|6(|*Z+o$LoYRw+&7#*kFS);NwHNudkKoaoQTj~1}@ z%w5iII3U{WGL!G6P*fzc2f#$M;s%HO&Rbx*=I~3P}VHuT&mpHG=cHyiSbrKT=0Pj#GngN8~w4&NNq&iN+zQWo_ z_PxDyPWCFUcUuY2_$fa_G4kY7+^BVkY1%qu{$}3%ntR2nIr&+sH(P;^Y&>gp)7k6BX{Sv0&QF>gAYB4!NGcY(0yh30z0&eF_@~_R*^J2Kq3LKOnEa*_a)4f)|xH~dH$am`gNE_vW0;+)U{oL=HILFm0!ng&Yr zf$!YoY+3e&sjiLjecz5pb#3!*jZhlZl*?A_{R0OJgYtHXe-@|rbd2TaRVpd!2K-WG z5+PuXIl*y*Yl`T8b#_y{`q|Y*JsVq!mm8VjKSxTmDVWqaU&1|GRT*W;x-WOhImq^^&9T$Vq1CX*BSG~oXxqHc zAO}oS0JmAy{PG6Q!YzDZ(xHy{`U+DlT@^n5TDGJkih^QC8!lxLh)@Ca9V+|iX(uq+r6Qv%Nz3L!7*_-K7_Oqw4F{cWwoUg8L ze4lZI%a!4ixglW&Zf~m#M5Ue_a<9D#ZFZ>sWZqq1@DcQYA%DCpKi(DaF8_E}Hbn9N z^Kr!yN$1JgG$bE(1B)|?(LF}FVd3%mTIk`!Srps0~vCCq#uhN}diJB)+;?=Qid{C(G zw8#eDDq10LfIc!)Ts3@!wQTmzAehc`7OZhiENpG};IceGIdy`QD~BmRL5b%~fcwlC z7~Ip)izLgcla7c4BIiYrpQg&zA)J8!NPo z04)v@pk%)pQbGYh3HOkd(Zm2?9k4_`1K=j%-IgzFbkJ{bhh&0ybv|Y^C^@|IXv*CF zyh|+VF4X;3&qqQO7MfMT++;*uogk8F41XM}MR~bn>WlyW;sT%HPF_V8Wr`v0fI;r( zUJBOO7Seq+ybS(Z@QO7C9&*_S<~a+5_|)tXqlZT%msc+FhQL!-#`PF0>)bDW^d z@`83tF@TJ+&Ll3q9M>y@+Dg2Ac*RC-`eazH?2c}09}_NR44e5N6UA5&cb*wmh@oA- z{n?z5aE$v+wVhv7xB_{6rv;A{>sDd>(L)hEnNHR>)!k90em740X$V6&xoZq>twUT( zsE12SJg`WJrE6_UeYf+0t~^#~UVj_d)?d-~ey;0%r3&u2zLoaF9gBc&^4Bwq;c zwi0@}meQ1at?|=zf2bK+QS>5gyMp#hBDzw)JHCI;s87}X1Mh2rVfS2_W#wrti*fj@ z$`nX+K;F{pIjYojB~1bY8Zp}g6BLqq8bY^RA5$9pA+rz6pgUj-6HiZ6y4y-sa5wJ}m>jQHb)h6jN?-c$ zrKx7mV{D<0EjPP59l5u4XMu_h$hd1)D&JKs!ayTIt%mIe`{I!PBUq4p@o{us@)F%_;| zXjwRef;}FL&dSLqdRtmMr5fQ2^3y5Ucvv`;so6(PQm-t6f;K*l(!G<>wG)Kmy{}(C z_NO{gLFILbcs6rF=>XC3`Gjj`Lt*j#=g=b;&e;o6ltXTBdwXg*xm-zr&_C+?(p`GU z=7{69YtqhhJ1{S2#dn+=wL7M$G0!&o9A8%>!aU|F<6BuqFZ5Ku@cPN=W9iH{h7AR0 zU&qv{i4ylJbK_K*dbqBJr8qD%v0QbdP^NoY3_C@rt40UOg+Gz@{nVaYBKDyYjbYZL zcgz(P=($QQ1EHB8+y+vFYZ9xuga`T38d5ybgSOofl&H8tMdhSD58YLgt6r+7Xry1t zddA%yrCMY)9<^eM4y8Oo>DxiCZfZf*q=u6SmF2#? zxx?|FD+Plds!@GbesBNL!Ma`o&FOxM7?U=^h)~j!*K8N!6j*z5KAT$pI74PEkE0hP z35s}HsQte`nXIgTxtW9PI)n@CHwpxiHJ_}qd0l-tUX!dr2Sf@x*vi?1x4Gp(lJ9~w zS>0iT9DAPiGl&G4-?A7(__lvz+V|T73d(ONK=&%VdZT;QZe&wg-%VC2<^T6kx8KdZ z6ew8;%TK*xZ{|YNWMj{ve$?i}z0kdSfOH8`xHQzHUP%^xRMo=RA{x0{@51L{w&9)J zsm~Qvm=|DGuz>v1ZLLz$Gm@Hl`o;Gml}(kf>yX!Co~it;bkVC-MS97ZmMb;=Pc=MJ zXI3Rr)2G?sCU(c$^8U$K^V_Ynl%MT5EzGM?xwqq$i(}vSEo&|Y`d<>YvGH&ti8v>z)C=kazcOx5kTT5e*W6NONaB4u=ups_U>eF2; z{6!&q-`>{WE*3LYsAgS5k~JedsxpwpKYy6gt9mnQ;KCIt zlw7!3_RPi(c;2Z!IhyH=WLJ>PAfhXFrCQ{ptlqLly)&EJtE9um7|Y|#L?H!`3?KI( zxn^*!Aj@PC0MRH`UGPSO>rb03XrDJa2EF@g%t58#tP$&%=Ufx^A~6NFUvOylWqK?zgUlDrp~6wK8hP3)3909G7;*D`S&r@_XG&g80d4_3vl zS-&Kn-KL3;Iqrj~sympIkxc7$EB-iJWa=78A z^_T~JN_pJRcHg`~NGK_dNH*ZA9AMf0iq958W%kB9G@I>_uV8&rkVc?y^3`stS)CJMp5Ssqt%iC~_er zXQ5wDLGT&L5KkRoll-b($VpI^^6mJXTwJbA-4j~zxBf#>4MW^#BK+Mcuw_gLvIsk3 z@vux_0Q@XcdcGXp)U)aVmeu)UC3lOuM^CsH7e3 z>on>`XJ`mAlJvH9NT2o^L+7-s-&$$h^L{H< zyVsbmL-BgGx6*g<&oY04Gm`dT{Rv$OIFnt+;`H?0@Y$YA-A;)b@3xFW z#MGAV=#l_6c9!-Q-TIaXi!~W-muYBaIU)#*bzFxjqd$U(O{nrMwT7iza#BhOQ9fHnN&210>1U200mw+Dv4|mupNX2lf!UeM|XGz@qOGwW2qP zGgsrmxzLVH7M_u&^?_cwlqf)SB8b!$*)4YEzYtD&ee@3I{JGi?+7k>-(Y@;X5e@u> zVB~N{xfL3ZtFkI*mb>tVXNmh2!uDP5OXgq@vKc z7tv<;&!{xZGH7mzWXJKe-tj2q^RMI&UM#OwQ>oy_7Gq%aB%AjAQu<14*&Wv@t&B!w zl~S$!)^%0gW`9EPtb}_gau??e!>XDLt61d{ z%do7WR+zW}e=%t6Zd;pb`{@1vN-h^%$Jl=xE`NJp_&vw|n3A!T{t!?-#7gp2(Ap?f z1{;;?-}oZf@(x5CxiS;9+WZNQD_cgL8_N6lgFKfTapoZl%cVTWhZA<6E%i1FmN6LU z(q)ggbg96o+|1yqz@am*tFI~`x+WwO-KGx6UcMe}{cyMM^oeWcV_NW9liAx+gNw}V z$ayyFaZqpLOJ=VF`WQp&I%I}4NV19`n#0(Tt(rjZD+9GPg!C0pz&%kJY1f(Ta@V#q zb3!^ir9GYhd?Y15QOv$_lud?LX1iH7`ekm|DK0a$?2f6~{l(t=%C27T*eI8~o8h?; zq;QPMnr{F()OATa06D8gk_K|;7#wsna7q5of)}O*YqyLN?iEZe?#q%xZv!VIvPD|p zSi#UO>sm>Z03@@5p&eZq7QS~p?3ud2+Xl8qB{|OR=c-*91pCI4XbhC@+dLWTGB8Ry zFwn;AuduLRP0XXo2q>X{X2Ja*IoFR_qyMnJ|CLA#N|gZ;h@7+wQD9OrX zzKx`6Q$kks*-FIuvyS<0Wb+dkWV4ZV#?e?OX9IH6m08*{y0FmuSwej!%62 z<;$&YegS-_GkZK#H=RIWV&pdXth^<1kq)*ZI@zzZkI zy*XfRw-F=AmFSJ_dD9=H2b6LuCxT4#|Mp5$(8Ot1Hknkfq!{mSH4Ar%-Qkbz-fy%R zef)(2qi?AFlA5^j1+LVZjGbLwJB!Vk&W5E&h9m?lQVodCqhF#`tg-!%VeK-}vIer9Q1YNTf&_jpt32-uO&g>9hO-|F$kc zYee;}N*|dIt^(d()WhdKe4o}ozXJ>Wa2eQ=fW6VBp0h;qDh8YBXG=K51`@cek{nNYHeuGH(@BG_8{8+yo;6N4Nm(_I$*q#DGZyQ(u+qz%r zN03{}W<{(w7<%Ly#a}%7>K6qaV4zXNpVu{5dWpba181>7GD^zCU3}xA)NMQ>t3+B&WtUngu z))wG0Sb(2F(aGkX#Im_3xd9sA4pgsfg2vr{vH#@PoeCRQR1KfKt;7f}=sap=Pv`Hq z+khRJ;0AEzZcHtV@j#gL<(`K#Jh+4g89r!wEG*1*sl0_@kNNO>y_|Z5wIfYAvGCcJ znf)uKpgh8>TM}P$-2Dy#^J+WGo9*0ynPDDv4NrE%=Tw(U;HT1G`H-m!0v_9_@g$FS zs7)(9_Y$(U8j#2#`r#gWTU!|HHe~4cPW)--`~gXJ;89bOI;{9b&aOgHc;!>nj=YlU zHhQsRFJCjR(0s~6?Ey-?$g0TYo}!zD4|xjT;83EYp+l?QdB`EjyrD8U?fo!KUqmu%r1*KU)EvTDRiB> zp&(Y0>Hrl7?cLEkLoV|RX#H05c7&O7_t89St>YK1Lu6wux-LBxI4N6EZHgl#(b{1s zt1-`G?4FKGNx%NsYm)2#8GGP3vr)3Gy3BGAZyB?22wqo1uYDnbwRT_V$0&Yr!HhbY z?Tle>hHzY3uF}CI-eg3}DLLD0>IYsk;U{cfpXir9GDQkQK!e=nvD)64u&j(H)?rF< zj#4d0h_vPAH-kRBX?ZU@*$!Aqba4CvXec9dS;fabs1feqvpmc`yY%MVpw6zkQo(l$ zyYW@038VhW-NFi2UTD5Nd9j)IfvERugsDRhPPV?;x<7dp7Nj7#t0Yf7x5FkXr64Zy zLI+ps9GZ@Ru9%G2VR)ICQBl;{l$N6_SU$^qTcz79Jb5j4s;`y05P>}9!C18@cR9}P z&f=?=YFHKj7_Jun+~s<8zDoQ1Crow< zlg_@Ver;??c+ZEC3c^UfOO!L`;k19eCK!g7^3HNM$SouQMfSoARr5uEe?cK7lOy^N z#yTE4h*n5!sejy>JPA)4P z*?qC}3m%c$7|r@67*oa-=YdT9z9 zrd`Pw%yB?!yXt-&pH*@P%qcN+77C5%KXa*kWg8Ty%n^&Cl;qfbqAZ7y_yihwEI^z} z>8yQoJL!aZ=;JkcJ-c%G{xi55PyP_yl6b5|=kb}y*8v0Nk#ck%f<2NVOct(ZLe{cf zUi2RH))qa;K*d1UE4mak&x}RoOuYk>5o?^!`tN<8abTKrsJ{A!mz3ur;v<7@caKz( z9DZT$Bm3f`-h0D^M#SxoUcUAfK-{FPO@wqPv5em$W>gvTb#PnopLNK2wSVar2c?g= z*LmM=G;<7Lw@Ujo+=3;Kz_cWVF@Apa09J~ooXzLR4sSNPHww3#oCopEN>Gqc#})#; za~M!7=PAZL*}Gz@joI6s?rQnW_8&a5)~3#jmu1elt$A<5Pa_xxl$s z!YX{T%9Lf#taPY-h-H_$m3T?Ic9a&Fdqtl2uU#~1WKp^5J9?vb7bGF+b_+iS&Po#Q zEZ#G$cw84h7$OPHQQyTHTd>D#>}10^8EU1TrG-p<*RVtx23EaT(9GhNS{!_U#A+s1 z50^Y=-LCYLRjeGcO3PUA?E`G|G$ZZKy3bJ!oD_pAIy*$XY zwKj0nTDBMiMb6)YH^xIIkQTc$Lf`?B8rJql8Muz^FX9)l&nDJqHbNOh0+e=J7Of48 zhm}Uy?Wc9P=D5#q`)W4lYB$HK>ZGeH!l3cQ>aqM}p|-xA`qA0xulfca9u=Kzd1g!y zy1NqbrO}$X=&hV3izD;d$LTwJ%6N8>4pTnB$9YSd1h1S=JVMY8oWZe*y~_3J^GajF z@o+=xc_7rMN8>*d&t^@^@rN%&1YkNzapP9(Q->lyg{5#hHnU&kyoO_LbpMH{I}GI_ zYS~RY37wT>OR>m$;$u^}c!rAT!)1M@r1M%~(4}pcHQA4` z(K|3x%uFhCBCFp{)+;F?qy{(&aXu=Rr4o^TH~ZemQu@}oj)oZF@HYziZoZpL?PId;){0CiaYa>o*lD+tcD&R?-pY1aeBE1FR>K$Wcj3JA zB#|~Af`sVvx;C2S=};cG!6~hkSNd=w!4v!rb%aOGkE4zN^|COH3% zX#Cq$voB!FJx~fl0YqzD9_fVr3xm)0jJ`>@s};pXW&b{e}lJcC_wt05saw z#m@w-L-IU7tfgOr-&j>@5xnh9Uc|j+t5eZ0K8_!(Bp>N|(wS$Y(B#Y@K*jLdCtU;L zb;~s|uM-R{11=)PVDsD+aaSZ4mbVr&Wo$A-DigQuVJ1(jITmoawHLDwml01W9K7?xf-qqHcum^pEn@?Ehu{#^Ij?p|a3K#pUQm2eZEKjc(S;*L~ z<+cvBG!DK~6+5##KTvp};&^i1(Zh6=0lbvF5!L#SoA4$zu3>N5olFAb3gq;L3|Q2! zmfxvTQhL~|D2Pc*qU92)SW$`^h7wh$rW!3XcUK)KY&^xuRTDVv;OfbS7S1E?P9Jx6 zrXN?5EzNFAm5n|eb^qg?J$aScb?%Bn~L2Fo^ci17D1}GUC}%8hpXS%*ptYjuj{L46QzGU|6_!Ap6Y~TJvYR zYY8@q*twd0(-wL@;?CSFwDn+3Y$LQ%lHL!43%{ zMlJke9Lu8=jJmZYYk!!cTeLRcYN7=tw9W*Yi{u_}pi1KANE2UlODiQ^TVNY2AG!vs zVy--nN7mlKhwSt`II(y9-L~}f`kf}q3eD8{_WDk%Z||)=BMDg*Fl4?Rw{4<#*c;xP zcyi1Y`mV&vORQM(Daaw@hkJ$iryTm%9@0DJ2zroAU%!sj=S>^s>vkI!q4|gqSB_9?zxE~?BW=FtRt4&)g9`-O zR&bcI=JOg#NH|x+tdOvM&RVzmb>j=|T`?n&k!^9*(P|g76=)W0cAqrjs4}<{+7{xM zlN{+Hp74>Y} zLw(px9d?qe+rUGE`3}QEgj4TYo9uPscy#fxbmXIE66ai8-RC`^U-KW1$$vJD@>iVi z2UW8*s%B%4&Hu+9yGhl!0IEh8l#^{zHM{;lqH4Znqrwfx!j|P=zmvWEpZ}2Hwl2Z7kb+&i z*ZfRC=l!!yocIL*+5_P^e}k?cEdMX9Ax(6DU+R>|u=RyE_iY9>?^yUohTiF$RX^V9 zd13Mbj=Ue}-B0kSP3PZM+l-IDsb>H8)MoxWf9FSC=vLBYAXx+1F6#KD|JrljY>a>d zwI|4B#=MACng(Dgg!_w)HPJ8vl#~FyyW!TtPxf`%s6hMq=J2meM`=0GJ1wOX#j*+K z8;*1yP5S9r5{b5kDgp|KC-GwH0ZQFXz1M5AJ6lNm#aV_s&yWA-?ei~ zL2Ik+>EGLNKbrDYzdbw-sNKzjvxh*7!yY#gSnIaYCYtmLrFg-YG><^YR9*^Z zrwI`?B0qzhgT{L>IYuudsQNL;%$lml-Y1BF zQv~7oc>f4D=KdV}La2ZT8~6FueUSF2#dcU|Fc{i~Grv0@!gUHEvHHY%wD$?ewfSp;M%1-ZlAZC9H8-*L8#q%!Zgn{6w-QIB|OYyGf$VyFnY>o%hndhIG z9;Td}y(Jc&#M}TGawb{#xbMQI5Kz8foRUH~U;D(=mN2Nwm@2-FxO$i}QmX9d!yX-Cvt#>@Fa4Z8D?Fwy~nRGfVVBG}04-ON#f zhM<_tIm)y>ODFqw@^%AiXk0+BB5S@SL`k^K*8PlAe(TUI#!X9%-B?Q0JCg20N5l-b z2PCUt2o<^@!b&_dWYxC36f3^0<^U1fuX8?Met~_)TX`SjM52sIaO+3a)ED@NlpZbj z#Gv=I2bw*})TjCh+M)9zRJ|%ng@~?kdFGItXGQ|q;kP+GB}erWk96~{rP$ow-jTXb z^Kie#&X33sjYuWyn6)RlO;-nX^E-s|Oyo}$#@RfJg|5h6><}fLISlar&a2k{-h z=R}+Ou|1=?c*ii3eO&IYe*9?gIwW^~F%J{__9Ujxk{YE9(%=skoA}`08SkOL*lR9I z8`HPhDO@+hzKGUMzocyHclUwvU6~OeqQA5n_>bnp|J$|sf3~#!!Iy89FYhLwNH8Q< zzDADy9WR+IGf0~J*?6gra3;<<6Jc_;{ibkfqqF_4lc{}bwO<@$VSW&iz+#bP1p17W zTvMq2S;tqes%kwh1PFNEpIh|4;(~9)b>@4g8aLRRIUi6Pb1gncC+%H3!Y{kn!hWDA zLyHlY80pi@b)#}|Z)KxI(0=aK1WH()Bzg$$x6l6})|p@zk|)vOl;5BlanS6l=$mXx zeme*RLV88f1ZwUEh9Wt)lok^Tt#8j0V%y;YZi9TYF6n}jEAx+v!v@*#$ znwW`yw^SQ5zLFo}#+8M>Dus@(xzK%Q*dKD=)IptVRbK`;u8^Fu#K*+J>Zm5pz3?8P z1MWxSENNQulpzda%6y@%JAgh>MLxl$POgjqX!PkzlYp{W2X3Kth{NPMME(D<_uX+# zrCYS22nqrgP(g}PM4CvK8W3rs6pdM4HqfU63LzO0UvO zfIxsGybovQ-qGcy1Ot$wH7Q6h8y9_$YBw{!_ zTFLKk

OD_y&-(m^k-#Lm4Q|jNe1{7ZGewu!WSNF2tabywS^!xf|Ec$|VR;n%-hF zyXk198B{7^TE?;u5FrTCvGQT6MLGBMgll2>lG%=r(S_!Pt2BnU;%MYMfav*NfFl{Q zfKf-0VZc(=#d+q>udji#Cnqu+5u9HWP`zE&jO_~P^6LW#8LYw}m2QQERgz_T?}am~ zTI}~NjUGdY80WO(r~!$Opw`fWf3*oo%+i6I!PbZ;STJaQ!X2LEBCJD36S{n$HpA|+ z@VgF@-HuDhj>qekJ>UztCRyM>w;9JtU{F0^wWoijA12YSMEdN+p$wC!yIcYv(-L(} zRx=5<-VMH_jwjWQTnh5hhgKUge2|3hof6&8ppc*AP3Q0Z@HtQ3qwXCLVsY_FRUWAl(+Svteoh3}eet*utV#iM8DY0yjvVo1 z`iY+E-;jRcKE#2&mH4f4X7;o2yIl07jCWQ4joSWqt@vV}le|hPIy}fPSw$`FD$$*N zdoB_08VR2#`#P-jW|aFk3Vl=o5#_=t})>Vp@OoXaB&= zzQrX0%*+m8W;9v=Gs}!!=jQ%tHmikjicj??8~+ZPIrV6rBz(XSpMlmsG+2Gmrc(Mu zW_tFVjeEdF93NT*?Q@Sh*?!t$r(vn_+2?Cclr5-8BpUF$kiO6vCNYAiMF2kbeb|s& zW=4_MO!d*ZTL@Yr&Ica+XD~fw$EZ) zOQpNf2W>tQdEPT{FI=Z#Q0iqU17y-o!H#O)6JwEKrlWVH{Ya=d?Wo{mUs4acrIsp| z`-RN33*of^@e}P+>tlUOjf~sS1tqv=yA6k&!!EPIS8tZN=Qa3YwIEu6Jd0D`L znDm{Gam5~V(HmkHdT4e`6~=ETbf9Y4MBl!Q3)pVBekCl(1#&F}U838LQy~i)iKU_9 zX*skb#2CuYN#s`~4m(HXYUu=@<+3Dkf}bkpmT8SpXDD>fd;(3+avAXIzu?A^pmL(R z$%I6OfDW)*Zzaoerp`y3!phO^^FC5EL+>!+GZ#Xg$4v@vJ{2W9522ywYca=6t*ute z(4Y&e5C_efc^2GY*3U~EL$7N~Ri-z+Oh`FLG||{BE$Qh3yX~elLUnm@9aUp|(CBEz z{kN*jv!S7Cvf^z1w?u}K`$S@Q8M?r<fxVz`jx~YGtP@s_P&IR3?XsvVTCkqjN%ct5!6iF2? z^O6mg6oj*541?tkp{Fmf}^rkXR$~3gr6*_b4lm_R@JEj0qP@Q7hG%ud-e{QV#lQi0#N_ zhDDIaa^8gZ0%<%32&RS6$qz>1B&LZvber?hTkS_;>|~AUi%MPb3*^4ZXrDVXB0UeB zX2*&INocQxAG~s93f`BA@ZZwUnO`3VI^w4h3-SaN<_DNqP=~yhJ>Y_>iT2p2SuJN= zfC34_@tkWaAn7w1&!$=PiK zYfIa|;T~FrQe?4 ziIM;2vopU6sQ;sB4!=jG_(6mCce?kVn(-hFoB=deRiE&0wgmI5|CpARXvI20eXkoebZIC#*8`dZ%P&MUk=%C z?>rtWM8ei{Vm5Mlz^9*9hH~NRqX6~{zG~ORm{S|cisC0$n8@2arQ+c<*pwlbAbtk~ z&GmK*iERYOo>xUrXrp_SA>rJ1-meZwEt*4VEj+}qg_h(#%g7yllM;3f%)`U;Hab1+ zyyqcA<0S0CfPZzA1T+lp-Q=gn@f9!N@03;!oQdS9XjC3M5K5kQSxtJnru?9}ha?sr z*K%UPtN&R-uVLX6h;Q4lI5DCnAaRzBB*Y(~IWVbzlw>?hv9SiufYVBT&__0YYGF2{ zw{}qQ5otj69a%$jQV^)zY!_s{0#x_!FDM1@<7_S9 zbmKnO%UZOj%Ww4Lu!#=d+LMq>{sBzx(YubY+|w^eOH>)0F4MM(z9n7!l3G{tf|2%J zF7g!3*?L6ML>$eNQhD;AEH{UE>?TBKp{K}-f!&w!anVybZx$SdAC??>JQitv{us1+tA)~fJfR?PS+fYyML0 z#C^XV=PD3P#DJFbwFOK^RTWCY_`&sk3b}fHA4@S~i*gg&{kYFgC?s4ea62@>@w^84?_QBiFc{;pABBPYMi8Vsm%ZO+B`xg~ZBDxI zzILG4#@0WydS_V3?A&<`qR(5X-*XJ>lg`u^ES>oWFCqTliYq^AtZ%HbY=N+%E7(g9 z`lfVruH@#6H1F#ODj-_my{-V|cTEj)#M^J^$!dP5*ZdaS`}^`LGAh(P;iwjRJ#6}w zhf|L<4jjI8WY=M8aWK?5%}|)}`s?)JzZ~KE9P#=aAB+Fc$^CtWE%{bdFrgW`sLl`4 zEgnvSpI@1C`Q*ATcoMd#Hjn)2IZtViZ?Vkp^_u@Gs4Ml^46^ICuR?;JQjWujYJOTL z)7oMkpCKTK8(V1bOgr*Wb$y#4waNiTipEZy1v;!!$1-(VVqX;By(`Y4tUeS%116*H z$~BtUhROZ4vD;s>akD1n3JFv|3`JE$dT|<9pP@3zSNd7^^rMr;Ftd4&-Wv2Yyx3)Q z`q2&3>yYu7mvPivpB9i+>O2(QdC9YgcGZapgg6drK=IVZDy@Pui5rp;~-Pv47GS8Lu|ni zla_%^6QNREoV|SAEyq?;c`x4)+fZ9TQJjR+z(=2AtqHFRfOrJ2ZrCS{vR3kPUCEIL z$X$Bab5kQ#`zptEbkw<7Z@-{~^zM?i4OF(Je7!7oF#; zsgAbEe9t+t%{ef7n1u-kLm>dzCt?zJT{Iw1<-p;3_Sx5r5r?jL%Cfv68OY$NUVKe{ z5A)>mt-w<`(IKsaX}fbsnQKc8&ZFF17w+mmcem4(In-(uWg_JnabWt6vu+(5`(0ca z&@0vG$5LvS6a=e~t{b)~jwJA$Qig_r!JZY54h)smUVQ`m{mFto$i{e|&SHg~>=oU(HKLu)k$vc8nD!*U4 zeIe2Ny+rWuIR1k>{Y^APHhk^6ZivUv{-5h(OaKX2XJIQUgwPBJQ2{>l@mWu4m+$5A zzvX@YyHVk*ou7<49MF2=lplg_Y*PUfkL7w5*WQ^*ajvPEJTAmg|j4_K;NcC#MUHsT)LMgpPXdYMhWVs{89Sw=r<9%@R! zD;OhO&8VMUvb}1P6!+3MKA5CaZx>wNcz22$H#TC3C#sQ#(kZZwNpN!U%C+uy6Fbh` z%P=X9>AOzM@8?*6f->U{@7&8CYM^>8jQi<~iS0|0`8z;-Fcbd_IT{IAz?T4Mx<7M! zIjMO8Gf%6Cg^_;u^bBgZk-3nO@~}$c1;-q@JhUg|^=t7aN3g3|iYe7;q>?LfGEBe5 z=2)mDinpOe_M)>kr&M4EQOd$9L8S$VrOzC+1~iTuXXwoACgd<;Kak!4`*8SN#k}&0 zmZ3W)8MGT~m)^gxAq}ACQ04Yit|w=7(cS%fyavsj zbggr@iEIkUE3NBMmlh&fV=3}TLtJ{o?)L7#43#%D8mNTRIvG?u>LaVltooY|W=`x+l)`3sVw)iW6uO$_t+-KI4(#- zV;56ZKj=Y1Tto@K*UyDEz0U`xEF(wv>#V#Jc;CG&HJY}jeV%b=NT|CjDs%c4KYXZ9 zNxk6R3qyTXQrDLn7byJKsjrhXe74~w-J>n6vYhcP-)m;qzLL}y$&(ienfa{ROeQlnVTx3A=N1eU&yE?97Y= zYPEx$>xJtKi|X%N2r7N^crTy?`_@)BtC5hl3F#9Vjm?e4g=MsCPynYGKO9E`G=KlT z!IIBw@PF>GfL3QFJYU##7TWA2(`n7lZ0`ShoeSl<j>z1UkLI{zYj$TCp&x}m6+0I@W_)%-0{&W}1e7sPk?Y4KAee@4F#Yca z9ijiL%W=N&>31E5sE#bv>d0_E;gCprNVZ*1<=%GrKwac|ZPwKPjAr?t(L4W7zxzW6 z{TqB0F#r}v`qfv}7ZIHJXz1wEKKMH|7#QGjsRj!E^ZXk=Yg_mPa#76+Im5|McmQ_m z-TVeKCH`Ko`G*xszt=4PCm;XO=>J``Xm>BXE)O}!GG?r~0mOG+ix+`7^l&Q? zm?(S|E~5c%YQ4s}AG)$90(9wtJz0G(Tltpv`LEj3@vmwf7=N{YdLHt&A-Jg zzSnF1tL7#32fIDrE$jU|Gwg5ttRD^2Z&<(KJx`Dt|JiIliXj|X0xm`oh9{Q5sJ37i zV+lotiwItPG?=xDY5_ngJ$!ll1WDKRZ+P$jl$QU|=l^1D^RGh(Bwwc-{MOpPsVVWl zrg`wK-{*%m!tZlzh_;R6oO4; z7YttK)q=liJ!;X2G!0ri0=B!pI|;*Yj63C03uf?ia{^utl=i)75^H$#D07!eDYLytJBwNuA?Na|H*}Y{``M6MftO{lEgss z33oReJ`H&JY{r*1qyTIo5=N*5{$+y8I5rhtFFH+liCh%1y-uUIujj)3rZ?zHj$lP^MGE>B1^PK#Ee-Oo zt7L(~{qy;K$k9~G>{$eU+@~jd;E&`6J0Xs_f8eiQyDEzO35RCc%ACavFZS6CRR`(L& z+joN||D^T-u**D96G;X^H5*f7TZZ|`jF>o$RYZ7r90jCq7S0cH+nmgq4pLmMU@cEU zgnLM1VHL(b27K_OxMVU%a&4qqNt9aI+2~MEztLX1K@V^>n8Pu@h{= z8hk=nT`@{UT{7fV_={a0EJE-K1&I6MA%2>gA!`b=V%`(btM_=G6U3xSJa|>I(*L=H zlHcp^{2-isgJp70!4@e&r#c(()#DgN=)4onvQK zfuzV=`nnyI%@&8BU)s4knSwQ002WYINQdf=j;K_2gUw2buaJr7nak_Emc5Of8 zf474Hf+Z_QV#S8wSVa%!_iH(`rN~XlnOL@vmsAh1$NHsJ5c@VEUKa-z%HTuvi|Q+; z(JReQ-6Q5rHX&x?UFE4j@96+Z94#d8Z#=8tinx5Q*Zn|lz5%(>)dEXR&yM45;CND0 zq(@8a;+YmaTow1IjOYB;k@=sG5dM9{Q(EU60{rhe%U>U-W*NrgKRj6|yioeQB&KZ& z%66qy*~H-u`mQ&l{~e_fLm97ymRg1ZpS&gm!oHXynw~7yqQzXH8~wKn#+&JDZJ$!i z$cTa*{xAwYJFp2k&6a()B6d+-N@NQ&;OsbjHpD2u3K1J_2X^+b>sXvHy_REA9~-k;JDQHx);tO7xEn7`D-4!v12O8Yj%Z>u}@8( z+Bcp3apQ0K%75s^{o(0;ifun!)Sq%urqjrsxJ&U)u3DYQA+zZ)ckecXSRSEc`1=M{ zZX^C9^sf*$+3-3z$TxmzRdf&qE!-h2X<-c#jA4B(jt-o$!SKVO2LtPaU{rMnLKGhh9eo0H zrIwMeE!rGlYb5=f5L9XGPlw>6PvBceu9I|r4w3)a$AFg*fLv5|7}#(GgsA(|abPo5 zSPK>W9XAT(v3#KQa}YinJdIqL2YjgAD7iuPHO?3YK)%yP5d8EZ6My#e|0%!E6Z;uc z%%1KuYMCcs#(cspYvpMn?0~;M+=`q9IsIvNgzVoU=K2F6`UZrEYieMTtQ^R?IYR;b zYrP=@3Ohj-%m)9W{&Ii;K^8TDndcx&aWK-E*F*yp8*Jw{ddMPoHrr01E6y@ zj|Xtoc;&oG>0E#?%ySI)8@n1hyjH1Q+yN0$0A1`Bk&$gKo$d)V8OCT3UEe**_L72 zZ9w6#1=QE^06w`IN2B!(f%T8HZAn@PUqf{ATX2?JWRPqSmyv{rN=D?jp`$TQ7KB$i z$aSvkByr#Q>->?g{uE}A~ED5fs^Dt~IUD}#>=?5A z&EN0~WRTDM=zk&6>(8c4NkH|jo<`#r8*|tIkmQz;;iA@c<1L_z#2wJ-ZlJ1f#wT@XFW7B#2A+cb4p6hNBC?)d7*u3M9drO!myATbE}Dv9V1bkR+0_;Um_ddT}K5 zfhKLd#Dvri(5zXif~`>G1J;SEEYqLmJ^f>T|MND(9grA&a-(aIe|o38Yy+9aC*kn- zmrwm6^2V28{~zNP-!h7^jmd%_s~?1S1q1J(J=yXeY{^i$2gj_5EhFT+041Cx6$Hmj zBq;u*uteaVujB{Dx#kDKF#L}H`(_XJ^ZNi!(pOE)@ayTRr^Fxqz<=!PKTkJ}wtZ%k zanzqglVu7}S$|p%{w<;gKZrr!&^M%D7{Gaou&iUYs6Zm)NCuWWcd zG7Y|Eeneoo4GA#6a+dXtkLLHi@7^Eg_pScyhxvU5DgHNgONhV0#J={Lop?ur0$!C+ zb`_9ts6e7p!Lr5ymRX^C2j1w$Ss)kr&Ec3yFxz3#LV$qW^w?cO`3J&fPwA6?@`FGB zQZ!I(53p|^Bd}5kjz1b$OhXKREwzBc-^qe6pF=^vo}R{0-}&nI;o3p5gB>CfCQGSl z*e7R7qHZUsu*W~QsDiE*8#`qR8UsROkMjktSoIaXiq;Y!!L%%AXH(v{ zdUhc$oISu>J9v~~?R23q%|7#f#>XPzuNHHcPT~?gm@?v>6s#SBGaO~Y<6na2iE^7m zU$Eix7o}1+f^nJd0k=Hw2vW;sbDh+@75ZoX!mpndt&V&HCf3eu>ueKmfxaNl3CY&* z72>f=)BSvJagxl|*>EQpd6qZh3kWeZwC2Hstd(wt1FREPjBa%klDbjqQYaze_J&2} zI^O1+XK!g$7G}P=s?8UaB+O4|#?0#D?p4p<&$(vg5no_2Ch7WQmh-~wm{XEgxcv#% zGX5%-eb??@EhzRrKXGiATh)|gTbh@Jp84+dnDOUtwT1C&joSGJ4-VcR_bqwWGOjZn zl3{h@OF-WbaE3=#aJ|4b%aE}4ZB6q>XMWgJ^> z*9Hnfjz$;&uiG)u>?tktRaf@M`LutXYfVaTS<_ zo1&j=LLOR(7SV>fwZD$a>c)^+EVx|GrRbTRdvrUG57?3OD?f8$TD zmq8LN_lm9`YclMelsHq^al0&!(Ta1`Po``W!Z3I}^(mW(+2J&ioaGO`!_xfK`zkGE zCa*wi=!YVvcy!9ObIXd_=X8eKoDV;7kzi!Nud^aoipbq}Y0-7Ed7*4Li5&a$7r_OdkQ#AVIU=67^s z<($|&wGAU;DQfQza^M~bLT7QcJsc=LlbBF65jpNFjHEH3cc2WH#|Z>pD8EyB7YNqu z#YmgDm&X=2w4%eIrI+ShJ?LtT`oSg0uj!o^ZtCnE?VuY_r8;1axwa>@zcP2<J7=tU>zECFK4{KZ1jHw6_=hKYM;WlFVBt0EO1_85 zng@B`t;L1;z^clubsIEV_(~gj1*L9sHN0TlZ5%Wd`(VFL6kmRx*skThacih4NfbC$ z`W4+Gt}#st=h{VFsjhWe-8g0vBZP0SUk~iORm4!>{|hp?cCt-MhJnu3!AU}ae!{)Q96hkxsS$*`lv%CEvrCJw=qrN|S@v30 z4K6bvke^zH@5Sll<_uUS94&roa}8?i5tBdKot>L%p{qEeQRMt|`Sp;;-F%JslF7CJ zX@3qezDmENSt?tFhp=$8wPx#_S8eRgg+d#eh5^m~GK(rG-3`QwfdvKMF&bjcRdD;9 zM(gu%KQz(0nIOg#lo9_(JmCVXgZ-u8hMQC}~X+94{`VQ8{hjhJFpu7Ev3dZN)@5FCe+$s9U zZsXq$mzg@tA`% zYMvqfb$;L}bjFO%y(Q`7VlKMXg+kWctb6Q(ul+IMjWo8$a2s68^fXLMx85Wbf#C5u zP;hP81-yl!;;S!Qo^QcRe|Zg=Ov$i2PgU0k)s1kSy2d?=v&&$9@d)-wx`o2*jLM5V zsY~k3RlO@Xp75^htHoVbqmhasxGs8=m=CF*>Zg|{FO5(cXMPA*T$WY2tim4F5qbJ( zcG&Sx&*;gM{$YL)eWOaj0|jm32c|WI@ee&jcCbGU3sa_vBhZ>h=p*hWyQanHc+bWw z+J(}x?h?WrDM?=nx-+nNqz?>EgP^)>=rDm9?b1nYe(81;+#L1)Y8C!yrG456{M=1$ z?e8KS-h_lbTZqNqNd)PT)&FoStqCl~d!UcNJA>XeNVW+?0IlFF0Iu>0a+!NxN&##p zYtGn&7~8=o$X=>@#xAN_!lyr2<0(9)MSpEj-{I%KBH^YH$EIAhOl3OOec_buPLAU_ z6Uy&Ha2d7vOfploup`$W3rOvLTOK|G+b8^5gs$xk^|tDhI;WZ`X~It6c^c+0jh(ss zA~fe7iLG6zdvijHVa3Mp*hA+$urA>Qcaf`ZV_IViUsFJjjqHhr=YtE-%B-FNd0Q{e z#<12Hn$~vqj@!}0CH@k7QH*|mmI7DRMVqP+t9JES(&3oI`LPagVg=&m$Rl}J4^Gai zN3cH9#PM~wQ<+uS(p7#C9d*#^&@tD28xVi`Xndxe0^4!TQ1ktdW5|hv_vHDNxvZb3 zViS9wtobn!X^P!)!SP^3?4t)hy=GkrT@;^)#cV>z3En(Yrw41@l=~lTrLrR;zlQ>! zK>7yj_c^(A_fM(ws@TViEn5ZzO&cFfpVZa2pbOujwQA%{WjuhES0}VB^p#Q`QrXse zeF=Ap&35ft=}k+0LT$4F<;%vAgKo_kcc=Ty+!!7t7UeCSun5GTo1H)K+wW*ai3wQgn*@g;eyiTdI_g3X1bA6W8$8n@@Km1q% za*2WTvR`fBhYvNlLh|cV0eBv@{<(u8i%f_bNXPea^5o5t{r3y^i-{Fm zIOjY$=w_1s^eO7F-OeV)8qF8weLRSc+4($a9oI$`%AV3pM{2q8XW5S0UDsk%t}1tu zIqo8NS>MKB>AKE*gAtLOT6gzcC^Xc@NC`g7GNP~fn(=ip=T4~R`RYe5_p9%|j_R-% z(YH{I+P_5m^p-Ym{q`TxM6i$a;?7ZW5 zG#wK6)FHKBqm=sW$!3GJL{h|YMt&L0j>~oIA3{H_(;;(3zOr813;NpA_@BZ-=Qwm5 za`Y_FOF)2`!?(7m1l=$g)_=;U61-ui-@k6EA4vCS3C}+o>2BE#ug}SVqn-||6J_ro zo(MJW~Uhv8`pxURX*1M89Y!lwSiOFO{VI`gc|W`zv_9OZ*RP z`b||zbNPw~n)nK{saqBhBFL^;>&&-vmP$QrmzIdCb}}p@{Z$bNzR~Lu59BRv#dILMFRlMplPnoG6XX1P2D0nO*szgV)R(6g+dkMaRT~ho@KATE8SKakiZ&l> zPvz;#Hj+1S6i|+B45o>Yo(kj_uA9$Y(lT;5hDl2&T%oLgBz=;{YY|~WUa%leca2i- zV&kcy&LXN=4eQ(vMMh_#^%ap;tVjD91A5;4eT(CQj{+|`y&}PJUrFL{yjHRad7Q*S zXLo!=>b&}KM(VTn5yo`Ia^-EOUivQ|8bt963TwSS=r5RaPI|k@!}u;R`oQdubbcr8@|eN<8C!Rw{fOSvD%YBFvQ;Gg-pu7cti`-y7lK-;;7v+q+@*UJg^B`1KZ- zmk`R^nr2M_dY~&xVT7|l1X7_v3B=3!MSMiv`v_!8Rr=4jqiEQ0h0*R z?Skgy4-21XQR%9dv=_K1OF#34+zqx6d|{SFZ7t!xNI zgg6$=(X(Df;o(TaZ6>f61&qAjZ%+MD+59VLf2ocA&#r9VpgD1V>qxva7*C927%C?Q zl?^Y*+!4N@vYC?t%q7+W1gLDZwkjLACvx%h49Inct?@fs$&~lflt-pP( z#RT^1)j8GN_YuW&?2-0%h$xdo(kpxGvtyN8y_4drSdEBW#CEXS&DpMBF1m2Tink@& zYOFnbjF8`E)_pC(A@NnZLG+uodjBEs9V~50i7`33IBCtjE@rHz3|*l+ouBMaVG$(I zHS21z_&V^MzivvbCa_WSDqBA{ym*`B^wuDO$}e37qQb4n#U%W%SVreoGC2Rq6Mra_ z!TlKSP8QQWd`0rCgpxI8N)s#`r8-tgXhP&o-ZNo)ZK{93GMl;P;t7k57m{=Znl)EFO z9gmFZSQ9&HBtdtnN!8WF!BQ-W{x!tZRP*fFlRV)dV$n~QQipfU(t}J3Nv1RX5^>eq zDOpvFqHjb5IW$Xa&Tzw*uO-4IbXV$l`k)D>$?nBun}Y8w`_kT2ZfClZ_({}y@?%ua zaWmH7XBsxW5WDp@GtPO%md0zR21*i+l@;VC2)w&`3L_t$?PBW888XUVBEYMjZIR>y z>$D#kQ=`0H`(8J;xv25UnbmU*ij+l{$%~pydF(0NP@SyYN8{r_(Cl=R?#OpPrtP;# zvMb;{2~n}O)~k_cB1?5RerpzZs~TYL69G;z1!TS7FCpGY)irJsWfy!cri+X5wtwzX zPK=0)Ck4;gz+Rjgx;$#Q>2f7X^ETfsBH5cmH0LRI5#&ms6Wt_;Gg6k>PVHwqle^~F zqGyu>Ot|(W6OqQ;jR6{H_FtN_30OTisd^TUv0N4=TAHot49s_s&cg2MMSm1S#l5hWA=WVUu8R4rJJX4R za%M}KV^o}1W=F3Ii$z+@ojmOlD3|6{lN=}79M9!?VqT-p(cyCc`GyP7;!Yt0{ee4O zZ`i_$88iuc@In4%Q4hGYXmpCOetz*;u6-{=nH8jn=8dFVeE2QZV=wVPjh0um>1;H! z>1R@7lI}5Y=#jIM%??TEvZt3hHw@KW;!4YNIf9fIFM=svor=(1;zt!!0|Gs7aN&hPYVO}A!8!>B5?mWZR zYjh$(>u6Al%8CplbNTzwN?l;&(?Wx6^@nx`I4ePKf9Nv&b9EWq(CEEGb-uzZ#OC_Z zn~-X0U7dWNhJmXm!mbpB)0zrC9}IS34xb^A{`r9uW+ik6-Xnl8Gtk7-n20P6Y?+xB zayG~=41yJv381PXXR3FlLzYgykfSrjc3@LN%pZiPqMaH}G^qkXAF4v8zF| zu!K+20k*I+ozpV=I=oWV!eKXq*PW6P9}S+s?544Wxp@D%EQ@1;vV(9h+wi?e9c~q# z_gjJI?#1|@usIUzSI9+)>#>-P{affb_nPb0;|b)}<9j?)3`^+PJz_~fHI%O2{FAom z7oxgzORf6az8moA1^5J&u^@l8pU2Q-I$C~V$Smx4x3}j!nYgL z=7kSqjGi1j`uuIbSx}J**OWL};c`^;scxo|P0Yg7nhiGvAyPrrI;0HuRt8EQS_c*9 z+q)boJQ7VQ5=nAnPPMiw;2r(=V%GTt>s$J_W=w@=^P8M->a>$%Q#vZgpuS4btecL! zK@LK-C7RKGh`1Iy6doblNoI9PG?zKaiLTx-)51Zp+U~$D3U@`PB+1nx+R(G__51+z z+w=>pu>&;PgYIua`spxDZUNSt5EcP=DKy&M&hSLD_t(&MVV25(JzbzO>9SULDG!0Z4k%acY2Ad(;|6f% z<+1UeT$B4k&DQz69%g27l$F{F7~f0xw4JRn1gXYaV>Jxork&vwrQMN*p`yQ3QX1*h zrINIhuJ<|l?vSS}`pd9;P?~%g#B4p0A*<$LizBzhLEUSRqld=htovAa;5g=u=_%pU$=Jasq^Ko_HIfRADXTZ^m( zgQYqH`YUUZf&RlU$}0I(IMlh(B#70V`gkKrw&oXME|>}_zr@=+FG_?=495gWXIT4U zF-%k<22b?Uk+B+M1+{@u4i2`CbWcvU3sf@uQ#tHgCnThZg*dWA-n|nV2$%SfDZ&vt z$3^OTto4JKW(VrR{`<4pt@0WAJS+VPkv)+ghuu(G3=cL671hd8FTmK3bffBn_rPv9 zUDoL@+xN(qaooh>M!>$IJ97Ky<~ez(8({f_2#ynwu^@Jz23a5S zF3TbZkjDQh=kxuM^AUebKlQgeCjLn|Un#(4Sd+JMJ|U3lla(5*fX0FDFgy!E=tP*I zp*TO67Q9mn5E1Aug>4U^R;I){HfuAG(2|9n{-FM}h18U~&4U zx&H1GX*@!Ru&Q;3QOjeo9?xu}1MU1q%m>-++1KUS!7w2AlQKKt>W3x2No zx<*h%bQfgw?m9)bULiZ?ow)%?@`9=lI@wPNM*v@S2&Rc@GzN)BNTVVs?ngntr^a2? z@I=|$GQP*(7BcHnLhIp{Drx-&WtW-b@lbj1A5(v40h_lr^9+1%)`aj6jLiJkvASi z+^W3Je6D(x90EW6LB=pTu^Qj8EaD0ZjUX+;8sTI*RhrS>t&@{4aV#v2Mo7JEcg}dD zM`yiZ&1m1*eeySM7BMZD+bD}OnP=cv*1KMqEwk>n?d!Fq)jW6oV@hz(i=LA>az59Z zNty1#)8bI|1+k?xs|r}2&FXb>Q{wVsL@)fua?0tueMVs4OQ4Zr1F7^0Dd?Lkmv+m0!?) z$PX&#Z*G_01c4O_^k=(g|A(oZSFi~L1wKSL1j|FGz_dfEP%IA$# zUS4%%?5L9TILQ&Tk&^ms`;DS^ZZ%YO)I6uF2vP|+c=R=wNY<`^wwsE}Ivb9knD0$d{h zq$4r?ZNWn%xp3}1)MWDtBa>QM%-S%!koC3~z0`MK$jDApV+GGVom z0??)PyPFVZIt-o7FoE4#6;xe3!T*AWe*y76|9=VYk6b*a2S*JM>N|(QxyTTijirZ= z8f_uSNPI{}_1ISR)Lyv>+4&&@eE`LR#yZ!r#j0%+Uk?(uaaGRiiQm?~?BTq|q+rwz9qHJFl*5D5*%;SuXqF_{i@2WoIUIy*_A=LWr-KI2 z!71}j?J&Fr{_J(-?xX21&Rn4GCo&}DMLloa)^Xf$QH`rp8;+t`pMZz!Ka;#oJFwV- zN1svNp;;2Ry>3K@D~L5teDwCyb7oc*IoBYiJ|tMw8VW_Xcr0dYeuO;Fk9-n<;><$N z@{c`Jpw- z@IQqIW)L1g6fEU=A5m$W)evI6_>u@!yh=MZKU&zykZ}LmfnLoGG4qgO0>;KwqT}!b z>!3oVd)$Dv7Fp-x*c0DKw#-_j< zL0BJxnolKsg(3{hh8H6)eojAm!69IvKOKt&$DFZYgkpnD2-i`P{?tD~n)uuiZ9QK4x^5XTys!0hzpN^_ z$O7{JwFJ=H5Qj&i!dh09y2@Z69C%E!V47hc^2he#-(nEDG&uH~9oQ$nAQxf*BPZqX zXaT*nhEOV*5rWLsO^6%t7fnzHQmPuoTvBweHhY?(F&kE;* zWU#2|_i9`kQEVgd2`d03fWbG|bp5u9f_xk}1tG5LAS590!6?78zbX{0p(B%oyK5Al zd#L4JrOp5R=#I}z-_pU6J=yBlm@Jh3FVE^k;K4;LHE+_1 z!n?)JvyAJ93gv4lzi%aKO%}p-Hk>?(xVY^|wm=3$8fO}(@cM(MMlTZVb2#^-d!#pW zj2=ttk!ZW5;E0gX`r)cK>aLL`=D!GaK6k*x?5Y*r`72LjPe1a!6`2UDRi5S^Sw>_} z*SK*kgf2R>mUZ>^C?*Q_AO@hY(<|vi)y;xtO3%uKyM{n~;QFPb{W+uHn;xGBTXCni zg2MTHx)z+vS;7dMdlw6zF}PItx{tNOzkQwzIe zi7HP{OqF{l#OEB%zoF4zt+7j}J}bTI1&EyWg#04?s>>G`Hz8A~Y3->u;Qt8xRF)2F z>EM&&WNHmJtC|f^BBEq=An8|=csiV1#RwsRTs+rqVmWbOHARa!O2A5GrXzia6}jy> zc9#Ctk!d5Cw;NTaQ>d(^$x)O=cOD& zkKMT9sU+wlZ`5CPyyTw)VRW_Oxm8;H;#}1ANxv6c2=2BY(c(YeE&u3fq2vdUQT2bv z1SE(FPS~KIVgk>*QeEIZZh;E;;u;_@8|VgkG_fqwpjq$^ch{sLt~LJz5Q)*cnuZMN zcjMWkF4@9@XQlbp#e*YkJxNZY3CaEM`VOi_nX$dp={DhMwwNkL@OdSo1bQ1S_qMR~ zh6L|}JN^V!ybQM5}=oO5$c(>J^@&_wrUkNlCvTg8Kz^l%L7eko^yQ+!8NS~pL@ z@fQf{Sb5`v!}IqT%Dp}qnLX*?^?@G657XEVOz|k|_!qYNkOhflHEuzCLwnNsxhfh{~sebq}NV*%Le)(=@K2M_ZcJU&GvqL*Vz53LTrCVNJW z{2Y~H@Y17A$d!@FbG3pH$7i$N1P3$i=sW3;Oe!E(A}8Gh2G5#QXD4^l9BGG&R3}i# z#N^K!AHcdMIGO2a?kX;2GJR-Djt}Y1iN4<6e#`LUsI!k(Ii0ynb7QossHo}FN$G@t={mfAyH;DLTaq zRin@Up0xZt8v2BiIqJD0O`*urTzBdNU%dzXp0SO^$^m-TlI({m7N-g3KK6x?kf`H=OQ~>tn0E)m>7W5MT#`s!Sbx0@Y zEqnHuwAilfPNzW*G6GreZfr;Wbp_I<<={;S_X{$$ZI)QqN{K?>W1?m&DUB^HLOWRr zBC|64u$L0AA3k=hqP&c8&UTS&g=OS|TAtZ_k>=fyF~UBqMW)le#HGWzFQnF9Q0u{8 zNTTwvcCU(|-j|3QdWEsTw~w`8J?L5?kctDuenV--TFYYov?oDFN%c=N6_hk48u|`g zY%ABe363o;d4R$sfYwAqlLRFD5=IQeoL>( zaSG_I@!nfFo~ly$A!(DU{fbPd-;U6)T_rg>Ag_Ni@oEk=AN?a3dX1&4kcYaF0A*Y6 z8nbVI*DTKoVl8qWemYAh1;kT_kJ%k}h7RkFzT4b12kkWBEI7*b^KG>)oJ^u4BA2@w z44!lybc(rPxR5TWl4_wDTq&^Uu70tiPwI5x?Du{|Lg#Z3x5bC97eWZkkcdG=UFf;6k8Of9_sKcwoNB@;-qe7jqckSFP{~ep%+( z?_3>z|B32f>Q{35;TG>m7C-F|0+f;?ZNt0$V^@4!7MipaqN<9YuAfb^Nn|zyF z4-KKih}e|#VlB*u4;DF4T6hB+DgexdT08{Y5A&OTHq|rxzhz@ z3;XZeN4={q;OBL%M)Wb1ow5%O zjVaR~0`f@;6s4i71wv_*Sv0Za`gg`36d?z!$BmM67N)Bu!=n3u~zJl8eN&P`VxR zriw5=WXl_^kY_r^G7qX6FP9@hag)VVsfF!X=jiHq$=?iU_2*^yC3Bx^lQ^gtX%J#y zAZJP}XGx*3XD2u7AjKm;o`=5+QGN+2_}X*aWf-sIB=Z$i0lJ0KpEl$scMUDku#6a1 z(=19eaxl+A3=a1t}VOc1PS{}c0)@A^Wlz^eJ5)MuZA64qDoI- zvgnlJTWC>kC04hIUs<**ms?C*q02wL;PYhnUDL--2179j#KV}ev*97*Z-U+4NPD>| z`IH_`bTWq&h*mTOMLLf4pl1y#E?%^ce`&7!aP?jPaZM1Mb9le>XaAsOe%`kKN4Mw& zN&&dWx;621@&B;*=7CV|?f>{7O=T;QY%>Zek|adTDA|%|(a2gULX0(I9ZT7laBxDF zkjj>jecz)9k$sGPpRtT##`Jr4p6~PdoTqc1b9&a#d7fYYq`B{VX70J~`+8s3^}1fy z>!O4MKyDnHuTAwSR|ZrRK*QSr!J9HCRDK7Sw9gC?`REMz9{fHVofle9ka* z_dfuX`?crzKYiPGJNz%RLla)^z}P7WeObUUs&c~w4d$S7_<@k=KU%IW^Qa#Mg=_SG-u+G;3?Qle$#-TYRLdyAX zW(PbtaWVZ)0N+C=VFOqnH>@6|^4G)@Dn!MFfni-S(_$#`1?g3#a{D1W50c zalu-it+n@KXopI;cwRqNm{H!|eC+%(gvO@>RsP*57lUqu?gac9e>AwhH=E<+1P%G zIuFxJT{g!3iyR?MqNZYxcf%^{JD?^i?rWo`;lak@gf<>h{Me~Tx$Cucw)L`a8{AaY zF6mdX0g2F;5&nW5g%L*Dj~}hAbau82rtJIfvCls)>cD&r2?3 zgzbf(->hucSPq~Rw~y4!8BIXG4s*Y2M*bIRM((dX4w6z4uui^m`Y9>J_GQwPT-UeU z8vvUqZY_L2SRo^Ffwqy%pO#3qpO#2*XIiD{pO(nbKk^a@m&1f#ef^=zlCxWy_d!v? zgT~v@n}=np*}8)|$VEhy3X#iiqz=a8;>5j_7NQ|!cU*T@jZco!(fXIG`x~uJ*T?CG z6;X7KJ$Ye}-H@2Yo_T7(&8uyV*K=`ElCY$+>tFKk?WrM8QAQas(9h)D$6isON~%O|?~i zn-{~M8FEXDgjx%fIKoArhIZZD?I|X|?93G<&&Q%o}wBHyQEwmSokYi{q`qcLs;!Q0?)vq*- zhWFbHT z&KTou_xEJz%DQ*#LR?|3cme8SiNL|ogPq}@tEsXu*qPW{ z*YAb0!X(}`3=3-8o!jv!dqao*9YOmtV!yQ+n#0l>?^GnUa^w@7x4fcnXrWBerQtr7 zk$T4k{~5AfvODd>$fvd&k)|PUiy8GOINau#;@sm-1U;F{pi^A>G{1pCWdE#iqzu>Y zRl$tSoYa(c&S@Bw?3PBRXKHrLjGu{Ih#pzE80wSk9s07efDZrC0FQ0RHdiA`1mrl* zX|x}wdFI5O$ktve6JbJu$;Cssve3=L+CJa~OFQEFwgewmx%T9& zDt;<=2i3yiP=@vCnf@%_$v9yazBFg2we@XolO^zn2tlIELn7r$zlP`6h{n*>Ga!^#;VtzHn? zYg*aZtorX4rjcVW#q2j|yPI~$F@aMMbj&8K=Oi_bu&qkKuBXYz$s1O)MLF%HS6>cJOO&f>Va(Q8rS^_^NrT$Yz5Z(&`x-szERKqGG|ShR9LH+m}j3o0ux z7L7D)6_OU_3L%=%7ll$av*viV`@bACsB;pqYVC~(EDF6|d?k^|X8QO=dGEN2F>R(< za6b!;;QF2J$_|b;LTITrMuro(M|C3huN2wR9h35lsD9-pG%9kFqtKib4Va>byg^O< z1fX3X^inja9~SnnOJuDeZVNCHC}MvK$D38FAg>9a0x`1HnL!2sV#~t42 zjj1Gi$(hc|AUK8)OP($lm8Vn-a zaLg-)>y52<(Vmi+Ydv^Lp}i_-z5v$~@qXJpRV!l86Sy9%YCb6LDqz|n5;7sY>?@z7 z(&nZ0135+l}uJq!;L-bu|4Zt=ga*=6%r14ux+CWR=6zHc`yNIw7VewiI7#1 zjQ%#+?FR8aZA(WVUft0oD8Tq$$~z%}XrfBar=1&epw$2d%JiFDsV&gSH?9JRYx6=Q zo1*d%bcnP@B@(A8{i%EQCuAXrl7*m*G_df2LqxS0Urvx`YqWksVn?n61YV*(Vw(qrD_OpKAdDCDy*RBh_u%>mQ zhxVavV6gsmV_nKMZZOUr%dBuc_imzMIoTO+Ra6lvUfby`{3E9kxn(?4q)n>15fMw1 zbwAThm5D*XJ8ol~#MV||cN>jNg`QtsHItiq-o%fYv>rEdCS|(J^SaCAoO>-I;BueB zizFapo|4D2)7*6l!)3N%Buzf?gxh;L8Xl|Rs>g=CKx>(ogOQ$!a}GK63a!^^u#syp zZ|{J3poG zGfZyx_=$)|DlR4qY&R80>WYsYMBOf^3cIev$0#}%MScYjg*Xhu$r{heU3IpYC}ZNa zw9>*&k7pxRcMh_=jZ|!8<+7n`KvVY+%`#u8WRk56pDao^2GXa!ANcXv`Dl?BHO3iv zq#Se&oRP{2*4G^tn{O@Sn&M9?845Wx1WfSB|E7+>_qu-IDou9uG zu19EQcL%hF_(l|RB_3vIEq8dKjc*Ut26ZHpc(o4P01u-_P@5@BR8(l&MLNY-h{|Bx z$isa!CgP6nET@ID_<@$Y@3LzcHBQytzF`-Bam}g;%XAVBi*T@Qw|Jpiwr`P{U1^$M zn;DApm}{j9KF6evCN=O%Y@6lrC;2_T|27~#STu87?3v4jJ#Sa%=I=73Y$Ypby|j?o zJQii@CLUTyQMa)|((R7`HIyllv7>a)4sE&;6Man%7KtuFh-k>*}6a%^3t5SjmZ|7_{L;;tEIse8*w zUTxd&H$UMGq1GiJ}Vd2*#jK5EEN>vX7! zmxW;-Lu+`LRe29uHJvb~Q?{+TC*^pxc{Q445sY^}G$smDsH1gKS8ikGaP6_lh;-yQ;*gJSYwfaFOp(vIYdg0Ji zHI)V9@3)%mHWrru!m_k6&&l+uLCZt~+GgVZMjIb-$R`PCoO2l?CIqR1h$;*U>7#-wqXs(chXE=IkAlrA8XL zZ8cY|+M}EeMH{Z6h{irPeD$+Ur26aR4!Rac~MA_L(~E`d~cwY%W7W?7%m z4+>F7+`R?qYQ~aRb4?2iit^rs^>^;bUKr!OeE5;*^Hqo~&_vP>;E4@__@FIpvu3qC%YK z5z^UT$kph~;k2_v`R$^5dy1W4t8^>b>BD>j{ zj@{3(^)$P;w;jSjPLv=ZiU={a5_>VIdepMP8PV)_6*bLtVRQoSS!uW%At51d8Wj1r zaW)lr5B8;XXL7*Cz2hUL<|w$gQ_lKWSPJ?MA@ho#`6c0j24+!gV0nY-hG-u3FoOm{ zpkKB&m#ozWZKX*3coRy~U+l|A#FN`Iq{k6#gxo-cWZ`fHUVFFknP-8_ml}3f+shr` zT6`I@R?|Y&vZ{G%+(J20en5ETItwlwzT9SrroIXN*y-86S@C45F0JdS6Gb;RY@;?T+V{AbvlKIxlZ|0wH#dT#>!r)D0ozw zdOCkeN%P45J0i1oO1#5IItnFx-wH6pJ_T+a^h=-+c<8HH~LLrK3`q z9J8cCzKPAYV7}%Yc7VCKJx1sbZ)LHKBGE9HDi!VvT&i#mc7%*2e{{u?#JverSN^tJ zll2DKnL^}4=XP-=CJ+ZIAQ+^#=YESs1wyy$L{o_L5vM>V?s|vq$7QscSq&7-<+$Iu zJqymz03bMXzr~WB_-f4kQRcSeozeWRiP}Bo%8gaf90g8m2Yl}KIh4%PcNcOQnsq-w zjl$%^Nd`p@#_@!VhS)?&f^cQ}3r5OqdxgE~sdpdcAnrMGZ!_mJN1lfevD+*xh`xp& z4-vJ4&<856v7qlTr`Tr*&9g)lV)ipsYUDZ+Bz(esPEyJaWE##&YRNw4E{2~pg;>uz zbSY+r5_7AwC5+1AuMxKTItxWikJYrXZaFj^)>u5D_%gQ!&TZ8nlah%sF?q81tP6ES zA+)5|>fUWL+A%vHm$}|ojACcKB9NAz+J+H2oq)@}E74a-e&SLsz?^S}xZrN4e_dQ4*% zv6x#&`z^O#q@ta1HesEn(e4MrAzn0S4Qy~h`e4T=6*OrRO|%C;&^h3Xg4TvioAx2u z+@&2!kr#A~Eyjsnr6>1?MJH`mOr1{nWk0USOOo|hs2R4^ z?@zs#D%~-%Jp%Z#Xj6cqUqct(Y=-$*k0k=gb@AUt~hEiv&J+X z^}fi_Px?l?*ojB;U-q5zGAM+knH z^X+FT<^zcLw-MOriN(Q*%EgP42~IgdL6EmZE`P@nj_Ifw`7D@q)~ENZ+B*~1L=$z&1y z^Xx{Tv%$o(b+VB6oMe`l^=jYb4K|zTnH7xYg$lxL7+3eB;p-|^{3R00`S)}fZado0 zWrH_2e0eiJtHwb>dw6cvy)^qq`Yag=HH!i6zH5=X6W3a`!(e!;%mk>p=kgnCEYdYs z3o;ySs>*ii)`5m=6Wr=!4$S;zcvo&tXu8}>z2L6=6!UUPwc_rzeR|Qa zY8=9Jy_ZI$lTxVr<7w;$9@vIq!Qx(82PM6djE7RUkLXb#+yQ z7|cbWOh7^PdG-WVI>%O-?Ux;DZr9fLuO0?*?*pmYYpz@ah(40TffZ`5#QiDpb$h#z zv24eyI#33}+-L?>K(@4@iBvT38B%bbzshX6ydu4QJsyc2NY^AvjP_EGYVtCn7fj~C zsBHC48wxVM%9dgOVDXI6I}SR-GaUN1MG2|35%(Sy^PcziO&~S6F!ok!L#?`=XsY%c zJCOglXj@^#p#V3*`|!(FpK`Lt$_wK94L2NAKU8-4`W&~#@+@m$&Cx6yS=LUWw}wS1 z9|CoVGf%48Ptr|o4tpo-Q?CIc)v}~j|$!%+aVAS#``CKjYv&5H7rR6DDqD8&;slpAV9PgOF4NC z(N40anYA!&EsSx$m*@FnBD-7*!%5FVJu)^DQT^HDzHI|#HWOkA&W%=&$!HvsjOyon z|9C5%^cjK}A9P7ni!Kt)46#8kuC=?)KTl@P-Ggz~B3=wcM7^w;?0*-68;$lq@B;}D zjuKGOb%ZPRaks@Yrbl)!Eu72{S*PQ>2hE``hK}ri=cdI$*N9;leG`FW((r6vHZmE6 zh?MnxZt)_cC8nxo6BX*H+9j>`DcPE+$z4}ZZk#}dE7xRi0s|5E+<{5D z5b|jzX^(`pBVKzEe|<0a+;Y~BwClNA9SVY0X_!M2%tmkfqhE!oPrCGOzcJs|$Nd>{ zcm0cT6JtTnH{JhW9mYzGUW{UX7iFVttH>nA@@XN0+TReJ103$ZVEWS8*ncm8=4L zh*!oNIqN4G^r&92f14)CE}5utR0(GfS)DPCSAp1iBFN58_^q=|SGvlT9a2sDR6jHc ziC#P-H_NR)GiO+j+*ZhMa<#C87#qB}7Vl=dY;e;{@zh|OEvsC0%UZ7KQ?J9b`j#fs z&T;z3y^o0}4qZnbU^rsv-E?6!2R&Nl^%*j1au+3UPKq6%Qlji8U0~M+U*<@bRF(BA z&b@x>bqdbrev(#0j>sfeQ4HE?9qi$+x8l*U6(1{UV+qzxXL<@f1u(L-y}8W%ybGTp zH``hyo{Vo)IV+gjuC3)f3xc!~hC>0guw)t_v)z^(nkt8J2^F=5pLGq|jO^cowqk~i zs|I;(Z5Q(+goB8W34_>jV!E3nfsXods@!%`YBm`{w_bx-8~NM4nVzu}VT39?%wE+| z*-O$cd>MYabw$S730)SER2Eh-cRvU+rNA_*O+H&LVwK`iw2-KL+GQ9OTjsEjQgW)% zuBTKU=)i0r8_h@}n!)+8R@Td2@LLoa2TCe@WN0a=%xH~62O5WI#cgE-NRQ(Ad9M|B ztR*bE;I{eSGriAne zijRi%Sw{-#ac;YHJoT=j>HgIQ4Cw^{KEK%=m+?SB!yzSq^&tcQ<5|v+-c0ijYq@ux zdMQw%vMoz@=ZZDYh^9jp~ZXsYLDEpVUmK6lIE@e8uzGK#8@4Fca(Pa?Z+(X?^rOXgoM$;C=Txe=ywuGkMee z)N1a<59MOdrePc=Y!+{c{YSN6c>8%h{x)rUjrPwF{i4IkY+U}h_TZDnp)ujaeVZfN zf_EQe=G4qHj4e0Z2i03Xiz>Exb4{1=vdWU0;?juwF-=OTWK^VURJ-wy^V7e1SM*JG zQ#rvJy5twDM(d!k4m?PDx9mPrx}#B6V!gsD>#3KNyJ8v$o5&7YoeG~J$oXp@1~$rM zmim*;xlg^9zuhbH6qr{N`SA;K6UE>?#rBa$)cnAGw5aD7>ssZvsstuj#ZQ*19hSB- zLh2W5dYq%THa-$9GLWbQJso?hxrQqCe$C`TB@5Xl#bmDkS&HKPhqL(iypb=nB`@!0 zvBdH=Y!D(nUi2#!ugk(Nt8KTs#x!AMRaIR{2i9)=<@rd zj_2z5P}`T#8=?*0^MhZDfB%zb8}9d%0{H>gm;9gv)I+t5uT!J}&{mRz?d``-bIy7K z1|8UqBr*SKx4}=li?Tue6px+epj+tv8wtE$vmU;F?eg36(ZN}SRxPVLjZmCFLplTI zk_jTzsCE|YCsELXry>sdqPqJR%nHs4V`1{h&Gfm>yK(zMopMdiH3;onC{$)mvCT5z zn__^i&0fwdIF|ROSM+RA%DmMi+IRSHmnWQ@q1UN!Y0vA@Vy6c~>LtsMneOP3F00GU zRJZeav8fYs8?g&Zocd(vON55SXg7O3k;jK{$J0xgU#bl$hPtLJ`E;(`C_Tei8=lDR z$HyG@(xYZpES|U??zjY*4tB$YD%Q7M?(bt|&f0!a`wri2O1M5`b~<-YDLJ3Wpi&ro z9Y79zJVGHQcg(KuhA**SR4h6?6|}a%c0HfS8fczrv1r`0C@FM1WX~*Hu^ebRme0F@~16bp2}k`yQ8fYTAg`DBQ&GCC~(wb;zk>PoOnb1980%t+U4| z*uId9uhkwL<~E3Qc#qTW^ioU=N_lGe%Bqwzv)5 zZV;895_&{8maQ_14%*BlS4!+6cbzp7yj%=1XyHE1A7*=812+coalv~j8V8L5?Ckwm zbtR{faet@S3!aGs%M-V@@0VbO z35j(jko8?h9_5!y76cc*^{MN?Cn@o9g&-s_rZsUPrPecL#Z`e<3-%GFY(^tuZb8q= z??oYCm`#ZQdF!=RdAB|_YrFB6TU?rQ4iLjsq;*R3NqmT=P<)RPvJ~YTrm9_>%sO^~ zJae>kA}l(Kd|Ib-qWtlJ(8RMYX1iFqxx%bISwnOIav!AuBAo@T|UBoKrjW` z4S)NboZq1D?CXw{$$OFSBKhbA4|92rt0-lSN_Ud3NCo3Yy-t46kD>})BJ&(bW?YfY zm-I*)e(zAkx7Zi9CDvUI$Ctr*Fg7e^)hC1=q(vZP&yBrI@PrQTzRTbmM6}i%E1MoZ z)_c!N~odrh=b>rW8;_*pfQ= zVJcZ!y$e@fTG4NA!51zhNm|uu^T<FY#Vb3ZrmJHe@vggQn1GF zbQi-ds2TWdOVm7fO>giK?@_KuMB&7>5x%vB`?Wm-b)NfVH(SLNJ71PHKnBl-9lxjq z74nh_bA!00bH9Y!>UT>8}c_;)Y;&NV{N!XD5q&<=~aG8ZvT6Y9uSKsp`R^=vd)G!ioueK~n^3|3h< zYpZWyppc73b7)B|MW-sI4}8q)ro<7F`Y&2R!_O4xqCUztS08TO5w@MpecucadS3cL zmxWy{;3SJYZ`)S7_^s>tTZClUxwxvoPD=a+E)t(exaW~++O|kBxtlSRE02|h)3(ub zB+$PbP#E*^$PCAZugkloS{ZNe&Ju7~ZthHonCFp(uH=|D%KK9fU<)pjeGT*7LJ?;p z>uVgAUCt`DChk;)hNJC_x1L${%LWyAJKS`zey-Pc+|Cca_&U-5Q%J}5A8dpJg}CW{ zINn|%|Fk%f_w;oek%(J99LE-$=s1`oJ-l-H;amy^c&oB!1c?VKq$l?%`jgWURaFf3 zULLP6N=Cdt%+NQT108Lue@?X{weq&^>ne=jON+3q$A#b`5>?r@!5##q5clG-oai%BjU2q#Hjt`(;FxgzC$Cjh+ct8a(>XJ! ztpKnb`V)3hMoo3v{NZqn#TvIj*zhcOKX;0%5bHxw02c_a=Yh;m=OW0lL>}z6_OwqH zZ}yDLx`sTw483){1md`;=Y{YqpgN2VNI=N0VhiVy@+52|ubu(yw%6@t*Tah*O@MN0 zq=E_UZAc2>5zo?pUxLHnJ_y}u6+6)k5${`?M@>ZoZF^FYU7(=Q zjc^^0nq~yv>*V3w9r;_yi&GSjT&iI0*cJ9HnSDKN?}+<9F;$OG5n<7wf7XyF zQ1W zo}Ms1XI;o?lsBRTv76rScyr!m^!3D^mbQCF`izpWc~24a8OlNvd!B|vC&*mF!%fdV)tG7Ucd64u-Ke_kC;iZ$Gg-f z(~<;l_(lski5oujZ-jHrkRtl5OxOxF)`X0nsYj~Ur#TP28Y2jKh;^28WzETxd`Sxq zYSxElp52Ja*>Q7!Ld?NN666 zhr$)y!WGpUOoK}9PuqcJH^eRK`XM9c6 z@A4?X;Djx|2y5E2PR0N}Q#*zj#P~Q3jBQ}64OMG8KFc$-^q|0suJ#2Flxl_PWM4gZ-_<7#=fhN4@4tdFPkxeT~yO41X^d^9ykzG@5m8L}XqoeLvR)w?>8(vxRs^dF@) zEySsXRYmL^r@Imu1AKk#dW$SqlKlEVLqZU+8Lk*OA;$^G`V?qSp_k2q(Qmg)PRO|G zKsyVr%0wwPBR$hdcU~z{hNd6Z8j4gC}An zQdUWFoSnUa8br571gv{;0c6;WMurAy?_U`zb41u^H9@2A*hqG9&^1`@^Ashs;H_M; z(Vl&&MkNPa_PpPIBB!brx5ozwWNw!FMJ`)yN!4C9SfucGYTBkHY!@~iNV45ebAkqF z2Up6>@i-;GV8tJRyvX1>mjJO0@e%NgUdOs(?6Y1aeMZVvRBE&sl4 z`Dw4h<>h4E(_{xt!-9KCw%G9Iq2(ga>YkE=t2dv_lUm0c9Cq7HGUDjZzH*>-D9{Q) zU`Ujyz`MB(WLwp}o{mU;UKnOQFr+RgVhbDc*M}@8F?`Y=rgeG>x&3(g+8ehQqk~d; z^+73>#iqtU4(B@y5Liq~EWY`Cyj!<;?k+PDSdo4a3I9Mg?|2s3Ftu*tb=M9C8C+p|or z!?OjHdtm&{f4o`1LKXHuvzYhLgv@z|YE8IUtY8Hz86A?Rja$lO^S6IWcG>mHD1`U! zDdq>PuDIS;?(&2VLXx-RJMk#%=J0nVYbz#`H#{E?tkQ$8R<9)9B@f}P+F2~F#f9ZW z4IS|#R_|#PW_qNVAz*D>u@hgUfKCBf$lAl~XPKkoGQw|uXyNT2JeM7LTvgv}jLLYA zJXeDdCNzcnhr4s#Lo@q3eT<`T(*t3cz(jPhf;}5j({xNW=vXR6F{#e_>d01S?j*;R zCCV$Tg2!EkWb&g}Nh(V>J5k0+JcRyE^C{bw;@h?^@6Hf-K-Mzq6y0|GskZacTm>)t ziQ9L$5!G}+&nz1m@Mbe6kKlhP@aUna%f*F(aXmX)i#?jJgE`J&SW$G%JH)ys`DS{* z<-VwX!_*7TD)A42WPuM=j}C}IFmy8PE|Sl#KG@j-)B6l@$!or} zxe6Q10OiqH+09SHrm!bD(t&!K=(}UV*u@zBfcdJrk;Fu|EH)l&?O-WUvE8>&OV|3? zpnhY|6&Hiu(IGFGI&ACswH4zU&TW|&(;6m|F_Eqx{Ie`iY~}|=9|UtaZ;To7Oh8@`qD01 zP33Z6PS;}@3)Rcdj1FDaaBLCY6Jt}EB5WMO$|m?3!hb4o>l2TWE3D9?TU@oy_hk<0##jX>JQQ+l-3?)oYZ4j@ zKM$EN_rRk7$Ynn8#J*D|c%g@|5yZAx{`TeAp!vb#Tz{UV1~Im6*PBMM%obd(S5AiW z*LNOv;dNm)SZ{3>|b(4G6v_+<^ZH119bK@|VXoosL0G%O*L;jrl8yBQ-6@ z9Nnr|2CZC=JZk0W@eX>S_|fM{@>ui@xhgrh**M2B2~qYQ|PZccV?FIvg)8yY@EL|mL?U|>je(}5PG zU257@8}YP0^OeAUoeIhD^8^Y1EKfKU-QzBN;E3t-DAEmATjmJtSbFT7i4rQ%-I)-U z?c$M&iLL8#j^%m4GOj^7{CM1upIFIZ?ptg=sKbZ^te}PlG8p!gBfE66V#mEA#jDZ6 zrcEDN+EG8V(bq~f2hNN;&KXA#OlmKa4XPvJ z2b_H`RXrNi<_ps8a8wakKEB=7b)Ph93|g1j&eeZ?wp5U@2RyRH=_HpI2{>3AZ$#G% zQZlQvanhJ}K!Mm0Y`v!c_?xnT#!0zH$Dze-v>{;`HxaDR>5l%m@%2yvuuwzdb{%EV3eON-Vdp%Z%|z5CWp=L)bLH1adW9!I#7uwrpTsA5x|6wE5dalX1fbPA!Az^6A+x?FD7yv^gV)>%{ign?l z7hK1&PK{^Q+n+yQbl13p9|^16*>jI~{Yqa$$<7uq)~$rC7&$gvv3|Yt=-L8o>3sJ= z_WmJN(+H1nyXiAE&^*o+P$JZJ5i-3yO3*;*A%l{~i=9`!S+Bu0m*iR>bK8%XAi~oEd zXi$EW51L9c0pD*I$T)bh%@6TN3L}8Wr=R6YZpCBAul6A4$k=ImFU6d%+q3_{*Z+#i z@$~=TIsGBu56U_GfWE9il-Tmgsam~)Is;^U!a##bA|{qrgEHAX@|+=wEDX4t{%1c! zio&-JM^@06e}>>>s)} zOgPReQ(9aj5ccvk3M7zqr^rZ}kN(uwXNW(jE}~vtQi34?lX?sX1PyjtC9TuoG-w5{NJCD@3!#!Ysku>OhWvbJaTIa z86j4oKC@2JvO5}j&?Uv|$I**zrt4PSunfW?pkO7q!-2GlxC0F=Sb!q&oXAFf!Uo1p z%yWv+e4UJIzOPw7>!tZ_d*5yE_igXH|Ni^pp72pLOCMRSE+&ruK~#&CRh8-c>XO0v zDIth_-h#oOGs=JQU%s#Mgv_tfgi^M@Oe~NDO_~PlLqLv#Z(b6Hxc(vo{%hy^pPd=s zz>LvA77>V?=lUrxDIfSL;6{S>xSs;`1f1YAPKvU1tZ@W4~peO+{`VO)-t>!XzT2^mrjYs0 z)Ij~*tq0n);PViY<+om^KlpJ9zdyd6^zXr4{@stjpTF-r-t!H35JM7i1n{qaW-z_; zK#$;{D$xJGc?b~7?RuY*SAGtRl`oOd0P*3y0+i9Bu+z_{8T$Tw#sAa)e#6q2`RVW< z4YK=Zp3g5H2LSSIs~UeaZ9)PUC!HO2W~^#MBoW}buXKB9L6%dro!FKG9aRe;6hAMN z%%=1WkK9k^oBO-o+)L5tH==?6%jkvGttn|)Tj76QNK!tQ8W5MX^S2k$$xz!$UpOD{;NNW;rD^=+#CNDvFBfV+E}q) zf{Fiq656R?A&pc{!45Rhe$X(2{zW0=>jcM3@vnXCFSh?ji+;~5Tia(Ve13UL|Dx;Z?B(6Sk1r4PNo%{L4k0kzKXP~^h*!hUsmRRpqu{I zU-pYkCG*oSs&P8de`rGL&uIw%Q&w0 z|JlzC%5YEni;;azG>r*RcYAmnT(?U)(A*Dy$iV@-8#|r=1}E3V!Z(lhvQPj{*&<&KY%(or^O)$O=K*Ys98qM; z*9N(IJqJ#51QD_W@b@?6u&aiSbaOxalj$Pg8Y}#>kNbu)gYNv_Kbidx>ZpJ2-09lM zzeS-EOa2OK`U+dnfhK?JkNlcX_3I(1L~MOL5{ik{=#(hf7*;&@+@R2wU7@n;k;|As zQnq~(y|`I-RK4f%a+B3i{SBX~*FJinsWY6+s6kPlPO>2=zRBj9G?E$LhgKio(55L) zbU@QF;YM!=?f&Ss_(E}(G}W^&Z4Fmi5&;&*=zClLzt{~ioq z7C$B+&Ll?}P>R^ODTtgU$07o8$4zkryni3Jum7T~qyk-2_a=&o5NObl#IArE3p{hTyw3Wr-hGkq`hx;M zk^Y*R_d)Hwv=4~_mEEZKR@+M=8Ka)03+YAqPG=AVbfB7SO=WIY&##yeo@J&DT==9h z&gh4%@Oi1eX->?&o>%dvJL^Wp>n9>_o@|Ra5A>#150)$iY-;7?5mtG=PigoNmijaX8L~0eMNj*Xa%SUA=YOThdE4}F*xrgty%~4s^fy(Ko;T!|p zA@2-JPs!dwu~w%z55?H|fks{KzU5DAycosTK>qVMw&idy=%|RqPS+j%)hR_ihHs!{iQ_B~kYHhdWJ8RE>4WAJJ&Ov``?cTWR~Q=O((IY{k|tL%s5*V~2UMhWy2T}L^!~xy zvz>Xf^KMbJFFLcxCWKjN?Cr)@Y*xg6+q7PLR1g%&1 z7d|QMZsBk~)`V_Tg8slzl?~@}=gh_g^`7rJc`9yl?1K5oJ>O$BL66>^euRqXp6bse zE02AQUg&m31s=cCmGMg3=6>yM{>xV8Z!MThT=)B|E&2-0y z07OZL=SUAaQag`k6m%{w7V>5{&8v?|LK;*jYsSX}Aegx@sc!chV6oQ*b;@22ke>wV znb0|kpXeH0%gN(RGu5eZIoG@GPQZ2Uv%a5hurfmQ3CPA!-bS9+Gkg2qrO!$viOT88 zH>(LN-?%~S@HR3DDL2U=3!agGlge5DY9p9EZ5NZ&ZuMEEd8>xX)A|jm?#hJ8k{dp! z`(7=Y;>sH53M-3GwZF=}M>Gj=lwh_HyxcnxC=jM}i#fh8Z6VKfac*_eOQ8oFo`3zN z;V=fSThuk=#dShX$MdlgNYuSiFQ<7X!UG*oYnf<0YO%17e-z;(SF`QDfDc<|xkDp% z??^);{m0~21e=1*dxVIs6a|A@N;?iLpExA|D|8-xmWqg{C|II8qyn>VR$zkGu0bBD zb*w1y!j6zfQtryv&G0xy_vB-F_m3Oc>@l=OISVBvOPki3aHbRC0Sl{|OH7vXT5Z#d zp|%%CqBRtCwbADR%4*KS{$7*C#3V*KlG48KLz%s)QA#{4zKD5CZ1^xkG_fjt6-_ro z9JI7q)|NT9?^Q_4G5JIc#QepURd7Ce#R8U&M&sF7EcL|;YqmbN_1z3#+I^MJL&{Jj zY%Q~$d>1X}Xcc;I#!nKRq4uObz;_&yg)x)pSj}b&fQb@O;R+`l13kIs?&k<=+$xrA zJrELB3{#)o$?w(y{jlrpbWRAj-tFD?JJNGZMyo5e{ayK7qbnLJ#;?^LIaqu)@>u89 z+b}f<>R+Ev@b%V%Q~)Lf;*fiQ21=JBLIU;kNC)uG%_AW4a2jw|2CwZv-F$mHV1V-A zfym#62p62h9t(OB!@Lx2{-E&lFVMl?InVEf^FLDIT|E8SD(0nB)d58EoLQ18 z0C0#6H_5$EdlTK22#Cxv2@bMFeN5fCu4gNY5rh7Z7Pom7*Nocrl0zYmF#3+FQC`Ba z@N|xWqcXEyY@2sSqyTZb&2lQp@eYygp|sWkuXjWFjSoonh%00nwo(069q8MtZL4J! z&Ujd2(Zl}Uvwj+pvV5uyVbESTKuz@ORiT`{Za7Ul+^L{;Yk@gd=C!^;VE67F)VIxQ zqbBD-H=t}Sob5BDR*}%F`LOHWx+bZm!dauaz1S#~kSj}6jL9Tq;@gI$-@J-jIor+s zt|gqTP#1gIjlcSMJJdU|aFvCF4p%8j?y|tQvuTfx?6h!=ZcKTRQiQmzWUV^twM#Ej zRqcjULT?K72*I7kL{Qkrm0)71-is35ae&{8*Iphc8lGeI+OMC_F(k*TZ%ZtivAXPOrDzXhglPEK+Jce5}EHTbS}~TaQzV9#;4#p~LBfc7uYv zH*u4b>W_qur#PEV9>K9{(?6#k?6kS(xz8fbGyA5{UNNmngQ0V}UMEVpufZPhKvqsA zu8)~CoMyQ^T#Fy5bG>BPa+u+O-WxXSHrwMnt0xPA+-8N8P7FWvT38>Lz#|)B!$Ggh z#3AtJ^zJlss?}gL=k9}tbr~RZUWWya1Jw`EJqp{KMZF3FQt-4CA-D=7iC8ExoY3AnK6d%&$aX0Vcm6m zkB~;fg32x_vd$wF-m)fk!IIY3&l$t3vLWstN8efLkvZ{hTM~v{86_WbMvfd+a;Mnu zQ*XgVvWcoID$0SDA!e#sr@IM0Q*p0)rxSZV^u*vGqe`&lKE5OO#zXAY^j8A~HB2R& zgQi_GnSHf>`c6q9jje8-)YW7*c5?1$sKZ%eMQptZ>wsd{nS$YSsdh%bHtKxQw>K`z zn<|n!{7J!te7&qFi_(`yW0T&3s=O5pHry~Ww7%O!x^U8+0*}=%h zVr4>zz}#;|QH0(kS)Rr3^jv0+XrD-GEWG=T?3{mqbt0D=fL*?*0LWxF`hapYgCS~u z3Bc~dSjvu8-~iQ(GU~NA4$;1NOpv1kk10h0Ic+3bcI&|%!1`hLQZ)M=PxH5WrvIzs zzk9?#l1KcbvPLF74+11h_6u~C0iJ&ryQ=(lYLo?b{(uC{C%VsF<%@BE@DZx@C43yK zGEdq#^cwr=tjQMRVTO;>Uv;1TQGey{ofmL!zeflB&WZWXiD5Ox%TlAz>|kOrdJEEf zfTDVu`M!0_?GV^up}*ecs?MVsR9P`1~#$-n)$7;-yi(A??p?U z??ubsEwk}e)nhjf4GV%W%Y9^XXO1Sx9XJv*?A)fwcwrA!4ZN@deeTeAz^lZK1Hflw zF^i@XWn^3>WAdqIu{nSNJ2c5PIG=4)70&9)&-#P$x3 zwCO;ffA~%r`)_fp-zj6iwt)V88((gH?#7`+5PjwD#6~+HA=vTQy3Y_32dt!#Y<<=L z+r^EHb})fdjmMq=yRKm6A*-AJ0PyzBt#|*1@A%#p_Ps6a>F>9nFhr9d1CP-TJjR&F zat;~?r$!E-H#dRW&e@!Byg4-rxd+=DlJvG@#4Bl8(;7QBiQd}fr6}_qA^z6``Tvgf zZX6P`S;6`@dqN{4L-6ow@Rzxf1ldYRTiChe2L)9ONY~MXOm@Z|t&0LDl9Ak``*&OHK!7 zQ&*2%n;vPqrd{2>A=HoBa{S^>Dj=@n-`&Y~ck=(6cKc@cE5MzY;NhFA=t(g#9GV(w zB(k*uxRDM$xedq-;V-ghxysarJMOIDK#eqP6_OU_3Ryk>Zvbll9U#wlF!(zd{4HQ` zIQa!|8dYDM2KL9Sc5k2%ufB?12du~s?fO_wKy32KRQfy>;)tu!6D4;TA^-t0mZ{W zv$QLx%cG)Oi#d2U>p1oaL{{@-!4_H@s4ZP6_gn1IE<37K?4cN*H(8!rk6d;{>;+t_ zt4W(IA2GLiiUJY)F_oL0wVRr5!PonbT*-2PV2)<^>=<{R3c0qJPP{++8G_#)KE~NO zsyT$H$&O<6gcm(GdM_Fw%du*w@#e&k#<|1dBo$ zSQKW#qOjzHTvjj8&TgWGM;0K&ur0@kY+J5LTRU{wzU*;Cei@_jVfk<9@%}@TJ@sHG zD?b)g?Haap6>Soph}=qTiEuvJTwa<8x_*UB6TA)Ay(^EA-5PyPqopWFT^p8!d4vOa zJ_k}gP9eGhX6#S9=wFK^?TP|qjnZHCuLJEcPw@VlTbkhg%KsmG?;Y1vyQT|=CQ4NV z5d@+F(m}e^fJ&FHbcljT6ObmIAP6W@1O%i8q=OKZUPBK>q!;PEmxLN131|84nSJ)2 zGvArBfAhUF?>oQ!2fC8AR`O)6_1xvUuIIiVS&*et$ph;ZHLrcbO9AJv(i@Nm?UXy} zH&^(imwiQf`yg$1t;~xkB)PMYQ$UL-juHHSYgLHf6Jz{N1<6?Nf7lxe?;V^rd~(iV>$ zm$!eGvaI9^Zz!-$ol1K_x1{TVrDA!}!s9|ei{ApD7HSs#q6&>RKsl4Zl%CM(;c=(LyiAXQu?<7hySW=7SA^4W6p+O5i%725t!wFZI;u&`)DT| z2oW@XEwHtiIlWy7B?TG+{i-1eR2Eg~RA9yb4@fi5cQL7dA-xnt>w_(t2Zx$Avq=xR zx9U}FaHZJ8*5X{NPTWrl5amU!NLvIYzVz-l4F9EN(<{4z=UFY?cH{P)AhYg7vz2Uo zqa}zfq2=L{R(T6=mx{6n?xh-abqS_FVj$gFnbRs{|(<9cc!`<>p; z@jm`Wh5}HvfFBKvr)(j<^TrFUl|755vC{X$mf6=QVU64*6AX9rAAvx(L0cOCF|_t) zCE)(6$TxP8PYIR`DjabGfmiC7_vWT5;FzUyhcE9})sf?EGq3mDBH>WcWmJx5Iu{n6 zyLN4Bf74k*9`txDqkz*!adH#Zx0%+(-wKNR*4$v&EVKnqv%6U z$O*M(e(T=a9_niMw~OS(4^ng;X?P^!M84O6?(9J@YutP8v^#`vhLFQ+U(Y~-r9cIk zQnkZt$m0Vz0iV1pkAxn>&iTOfvv4ypeaAKAM`s{o9@#09(=*WL_+EVY4#6J~CLo_h ze>Fb?^?sf?+U!5Tsq7-E!-iJRK$45@Y)-H6XP^=FfBzgXiJP1Sylk=Q7_C{+=B~{oA&Ys#?t#Z^|V~Th48D*6T<3dbKM6}D0KaZ0@f{n56 zcSDNX-w&@c(T<`y)%AsN+_n$A*aeJ!vxY=hAm>DPmnRP88VrBJ zA-1O}gdN=8!bXoLc)a2aG+BKHYP5qi2MXyvgq0T~tATJXZ~ztp9v_+cf8V$D*YkMS zM;@$4@+p6RLHhNZ4dH}1Q1E|<68B%7$$x=yHXd)N&a-`)yP#98`oWzP*R#*C2w30( z|NNbb0s3*%blE||*RcI!yZ(61w;zT)<=9&TBnD)qn2eX{CKr~1L=$lSd?-JSg)M07 zgZ?Dy^2{|V6(+Wode;Hf-Hrj(&21Ft$Hinv&%u^<=ItNe4_yIVuIBtNaE_HFn*JS! zQ^&e1lh%2}U371o242#-;&blyn9DNlK{%w?dy3)m>u1Fr;{`V>30xKPi>EAf*y1D z{KcvpV}ZHDQ3P1^CKzDVf#&JR#h6v-Hn7AgD?5Yc@J~k0QJun%sf}ROLWFL^J>w%A z0+$KwG!C{|?kfhW&hEKoC#pc1{9xc&oET_x;8{xYgFcc{wPYvYZJM52E}FNJH9>0i z$U2y`w(zfiBa-h9fzCj`ZU6NhTjIZLYKxzN8fMazHN}!ai9N8L$`wr;!4Qs?tQU`t z&t=_upA1S-e-@>-C1P?Xs7@(L)u6W<%K!=GQ~y>rn5Mo%sA|J47MuKsjL{$EA^LwA z$L|S6h0iZjsxmvg)#Lu~v^15-!e$=!?sikd=v_trH0qY<5B}snIhcfH>xRMN)~K7d zN#P_-Qrmppd^MqLg1maS3v~0InhuG}Oc|_|IG1pw8*H!S4Gt9Q#q$m68QNFgyZ9-b zC-Cp0m=juk-LkHFDRD{}h(1=21c`+fnYAsLvdZ(lmG6NhEvvvk%u75$4N)9*9e2-p z;yyU?5hh@(d5A2BA8j}o`=`WaAii@={v!lm1lDa=2#`SM8ufKgwA_0KhtsC0F zZ*>Ev{)<$q;cE+Lpv(0#glfR9eH%{F(YUPK8rZ_3L()=2RY2+!oUH5tt!uQ2($o!i z(3BSvC}Av7vwF0>BR28n6+*{RE)wMMET!JlaZ=Kw-MjFf!50vbCz$QP$b9V+deXmK zl5HZN#3T5Dqmb3r)_h1HU&GkfC;gj4XG+KMFZ2>xs90p+Xk zdsq3Hvo;oMt=OwHYyduc~NuRG^uk>-<($x!Y;%P)sI*N7W)t{juK+WUz~BR>qft*+E?@ zFYJaB=SGAwO)nGra38~m5E<`k9-y;7|4OKY|EqA8-}8HZ&#Jj>Rv6{x8NDI?$0BdG zTlxD}vZ(eE*O6ap%2PTLZqdTE$~RyPWu4B>o6pK&`){6bT))F%laCOsA9ZFP7TCVX zl*K{*q<6t$gh6RAT0RV}+dVFCu4SRN{k$ag@fO#1eot2x-msi$$5bDdBl4T<;@nFxC35l8 z8uV}iC@?MOe+??-2ZTMb?{e^f6ue&>syv$Z4m+AzFZ3F4WlQ4!6)f(5^c{Di-vMZx zx(C3?&BITG?IFRCT{SWA5wK{szgO?K5{>Yt3j z>KdXE|Emplw8xH{_=QnTK0JwOFV6*jq!jYdq#Xm)c!KP}B5e$`E`CV7Z>KJMz|J_F zMKzUZ*+nEb)ehhd6!$!DE^xwjUJ)4RBJ>-{^AYj!#y!&yPV)Qn$gJtrx!xZ^PiJ8v z+|{lV0l=0%D36Tk2e)3L%bk1QIQHO7$FQ9k^*B+QnWK)}S6US0iVst^@4fb=5s7C3 z_*1{|7p}^{d5e0ECRPTB4PQP7b%w~0i=3d`V^5>ZES)#vlM6Uu$k6#RTKl3gNp}QdSqllT%K- zrFDTM6mcJ$1M|q;@bSY4u2OM6)$iN0@CMj)D>D@9;{Enk9Qt&$HVDb#4W0HLM!qdP zjj~sJzLWhl-mk!|h33F`t7QLZ@1~{xY4;$e=(6#IMQwT6D-QAx;^u-*`$xM=ditlu zgXrQs+%N>qd6H1qFmGK`-f&Iuy$Y8NgZ+8Bsk;TFFfY%-TuMy^r;ykBF!!SpgvQqa zXR!_e>%`5Y{*n==?{-=u2_?O~)grNPUwh?SICUS7;CDiui({sUdoGf0>yK9#w&hl` zSTosw%#L-neoyy^83S21(Y2V*yy2fTvH@wF#sph((B#~6*Geh@|#go85CX&YD^W9)X2PU zJtB27z^uJ_{`5(dCxzR&iQOnWEb;qJI{ey2f1^jXmAAHnf?uVVy_Ued+8wx7+x>Pw*?qK>P2$fXI^?gC(dzF3 zv1ZBHCk?CE!Z168a%B(Yt@F5bi<^nA@yu!O#{OdZKEL&N8>@Ha&av*O`~lO5lQvrE z7>p_KN`4m8^3cZfH}iJ}s^=nv4i_5{t~gG!R9H=RXyXrue|TlhAH9|pXyEZ6^7pFq zPwu9>Egw=^4!aDuKS_oW%RkF&|FQi|e23mP;A+LGa(ThT&rGIAF%6zTOrr9ACJ>S8 zr;9ml9shw9^IGWN(!Q33Uu=anD3i|;9U+8bO}DIcsrlokEhg{i{AgF>uzYb&n`8LB zzxAAR_=N@dV_pHC@bj``1I^pMggMEeJ(?W~AnDC~B6;09Z`(BOa;v)LE_bQ#7G;Xj# z#s{Mvl3=$$`^JI`({}PULQ;9W#q*}{x?J;fFq&O~zWfvsA-(hZz;|L*Z0efYDOImr zDfb>5>+5f`+mG8^rN-UXjFGgxZN6Y4yU}T8X~)Cp(?PO`OuweL(3fsysb8#)qM^<8 zMgP|xE0?@3&u6o74K4P0q{(Z7TW_y;e6A*bK$TX#$l_LZxOn-8+?6`tg^I{SIfy2v zB$ZUUB;Mny-NVPV9>U?d!v#CtFD0` zOx|PFzgt^oegDPbwrdIPx%x@w`a5giy@#I{);pf0Zv99-9d}TZj5w4He{K?kit}YV zSc{dSk8ASG`!R5X({0!r9I0AqkhtL3^=4^vzQSU&80>NAD{|tH-zl~fiLRGGws@%7p*P)O5EG;vyX$oPk`O@v&d( zH1-6(9K46IoBI50F9k&C)eu6bkp;Um7I3#E!N<&~OE?}MNXG;%&ejL<{kzh*ur^OE z%Y4lFCo$R`7UjDgCkd!m>IHk-J@-d1sT`%+^ttIsWF<^62MDMQMR;23e5g9Qw<%9u~Ca#Fh2dZ^?}vpEpEd-HZ}bf3Of##nPZ`OtDcIxG1-)reEw z7*KQ9%0ote3wucXzK72v{tP7jnEmDSKJKAZ=Qhf=_`Q8<2+BuS-5+SsN(rW>a=^i+@8 znhCXooMk^5@+w#XHvsfe^u~h;wJh82d-^(PN!+dJ^JV3P$szYm?_kGT!n$@Ea)j#| zQrLoVK02sD2t9rLsf;-%d2hwodW_cnv;|G-J391B+!kX$y1_+=updX%{0iUa*ZrPj z$>L+ezwkEB2HjRHV)kztx0e`1dCH}z+=Q2zu6%z7ud8B@IRKlSLRx2N{et!suaGQ1W3>wrm3#Mj)pxZSqF$ z7sm63J|6CRJ_|ckHbMS;{ANB^`8|FoF@ zkSD374Pzq%91d*mG)?GKWoy|Xw_4Pj%|PT$txI+3kz49fNpm>pKS?}hV348Y5p_5G zGw_H_xIIwN5VqubDm)?dsCP?sb(L-8Qi&ai_YOSi}4SyfML3 z@Xm`iIy4mCj*}ZtgwiPPQ3< z;^`0et^HQer3Cd`Df@{4jwbHUWbsmH!6J0O0U(U|{umOExd0JZ=dDrXMk?$?oaYQg z4!~v~Y{3U@fHu!+P#)OH`D-Wix1Hx9iyv11WuFeY^PjA|9yz2W)j=A13ZQfBOYv`lpseg(bE8yj<2m};p?Ci@HIzZvej5+K~Bn!-jV_mC#&Jq=I)Vw94o1j>=r zl~;i$@A?Z-J?{CWl*bos?kuO1U7tc;ozEdoO0i-9t@2EWNYNKmt+3M{k_wUBN_*qV zd?oHN*PZ1Y9aX068fNoXtWzu_Qd+QpLLnA_Pz#gvq||G$VVH0b-3K9;kRsK z0>1{^w=x*^Z-cz@0fQlLLjdATjpeQp7!-2=AV1PpWh>t?G$gu&_+QI?oO2#FxEk_k8Y@3@VNAbM$Y0F2AOud)dB& z{AN|c#8PEKE6u8d#UMcr=xgw@Ebmql!}aXXKXO+TzNU|ByrW#=E;O}bUgVwQAPG`o z&?<}5e#)&>Jk=b|UBLfKm%~Fzwr(%=l&5hc$W13!!xi=h#+%TL@e-WYrLE+pbRyv>l>T z%=Z0`;KNsa3y6_wj9Ub^X6L?*79Cxh7kANmcyl&!CLI@IP5ZJ6erKTJ`l&&ZbohIU z62Y5x}A0fSn^Td z7lQX0-VDgFlzx@Z9;(iE)F$UCsZHULC+3R-5XEW{yb zayoA>%gqP=Bie2el8;=*(X1BV4xg6<>nks;+a9LhGZNYfAUD4KYX!JFOjMJ&+6=C* ztX+^b?tSG8d|J-88M-TA3PTD3NXrD`98Ke6VQ{4Bw_)e~tE1h`0%%uIFQE#)VqqhR#ybqk1CLxslO13}f%V7B zZZO)BdV6q-3J2kqW$sQH>u37Kg90r{zs5GM{8f9F{G`etWo`9T8|SMD5Uu%GqBtAp zoy!70Q9dJO08*ujocf%A`Y>&Zs~v0eRX73=`>)`hTZDjqIsXG6DeL)GxV4<01Pbaa06z0 zIq2nwKN(-D==nVH<<{k+H8Z7|S2yh5-7UR)q4T0J1&k)BcAZC^SAD>cSLB2F{HAv= z?z#=Z7w{l6P`o{4js&tdeo6pr+=gM6uJI`aw5wqrUk_b|`N}R3E{X!(;1IAK1fg7S z7zY*zO%ng^X6~$6H9hIt=<74;pYIjgtz9GWqf~=8(CB{BM0(Pq0bBjNsYcHgFZRkB z1IsYMK%!96qrTh4q>g%S`Dga_V8Dl+fewlWxc}~8!CevuU%6`j2YMoOQ(81q`St`r9nO9Zpah_L z0P*oG->`X@z)^2}R0azJ-)9*061_WE?j>Y#B0|tL&z3SHrt$ITiz%6Dg6xuvLN?%p zuHoTCWsz1+aSz1e53?@4D5feIIf&R}2WiNm;Tz33;wLueM0_(LivbKd^Nw^V&CyK( z0n4YMk8_$_-4fUf4vSU@yv|hPQ_;z1Agve!sDYLgt{O)z6bV~&7(h;Gh+{+Cgkrj(-$-a9DL-TnOVi*m)4?P%t*3S*x9K>V$UCy z@Vj#)$7gjNy8KKVqlmHBt*cI0H_a^5Rq|fR@SLH3J5dmUrcwc>dC6{-P$+T_i2W;| zr*fXKY0TQuFjv$S^e#Je^1Bd2Pi9ILI(*|0LtW@ zJ=1c4P{%d_k39nd53qd{d&WM9zdEibYJ{&1xiQA?m$az);TzC(yK)jvyDLgEiL~o7 zBRzRI9~HvZ_a(qofqMbDM^kx@a7_?(5s&R_*ESM4~k1UJ( z-umKQa^kXR9#)+K3rr2t$A9%wcC+BnQd>s!26JTNsRhtt^x?nrZ*N?kPK5yrkU5R9K&rw~+5#A-P1ydeLL@V=%5Y<5 z8+aots|ZE`Sxmsf2=&NAD+Sg+V-pdE5~g@9L0P1K#t3!Wa95WChOQk_yLf{TpHpC@ z;c<=Mm4OSNdImDttJi2`t{+~|V9Q?5tNb|z;2rx!)tS%a~PUsQ-df+8>gclqy*1fxsD5hoN70*T}qh*5#T zL#M@{c7{7X*}&@h)tR`|S)b*Ckpo1axR)e(-QKHJ=BY$@$pQ}0@S#H-yc;X;?bI1| zSnjT)3nwkd{iDA%EgN|cgnx*&b8t&9iEc?g$c@WgH#oY$loiNVo98_2z8q0=l(ub`ds#bt+gWO_$FK;{&liL62WK9^Y zjJ0KxD_S}*=vc`wrv2*}>0!;3C4e~pJaT8XAfDf>@YGJ<@x=DdNKY)KsZhYYQyqP0M}#uB&gkoC8NQ^8%Y*}%W}W;$`l5aN*4{YYfp5(OaM%cZjFBq1NEme(lvQfyC#w zG6!7M@(1<}NOHx#OlD``$))joDN)mE82_mB3=|4jt{SaB0$A~nr2GH&*x$42E&!s0 z%)a$gqq7)8o;>AMN$|OI24Y5U)JyMD_S*@R6}J)s#CN1n~uOS8{pIO*NS?Iq?d-6rvT(2 z+UN{4Rg&zR)bR!GixyW#s2rVvNN|Ob_Ntvm+0R?@3m0Q?@OeVK z7AX>|+D(J2SX?TKQP;Y!tM;dj&GJUXnQMm-*f!YMt=->kj`**9cx*N1iEu#o2JSmk zwIir*`hIqE;qJuA@iC1?%1K(c)2MvJn$AgN7HrlGcb1mRzBvOeWzpu@9|~M+IeGTV zUSNOY;ait(EB+aM6L&*u^?~$dOwb!Z+Y|!?Qh$gLiku9S2UPN^C2T@Yq@}CTzzgRv zG_l?%>}Bw!g>r?-wlpHd+-E?hG2nHYBCHHeFac{S`?RT;j&=SD+!e z;Tx!3?CR*CU8!_k!gs4@7GVqrn%+m@4UFR(hWgO4$rRsetrf$it7jnf^_G#PmvMbv z*+O6Ebzz^u_i%+FFB1buF;`N&zINtyUgrq$YOsbxddOeF%4>BaE~>p6GHLe)nvyeb z#@i}cEUQzmdbfwo)f=`--ei>?UR+sryXvC2!Abct_<=-dWfB~A$E``Gv284Uk>-#r^3 z4(nte4R{YE;8>sl8bD5R64(GFM-kVfgB=FK<#*;Mia>n4P)r+w1nMjU+C&QnTo1@& z=#Px?Jn}sqbDxr4CoIO0m1&E;rg~p^LDrw_v6lvF8jPtNSAOTdD_0@gu^5pB^0`Q> z+Xiyq*ApeZi^^aU@5O4g*GNVd3ksGQTgUQwle#job1SmTt&(K^f6@R|U?9}Ew21X| z%qkvBD~kE`bslXI?>8w)cRj7bssX5x!goW?c>7Kpz}d=V^)>{bal>+WIK8mF9&E zT1k5ja@CGaZ15SfKRZG7=Evq`)h^aXlD^a18Pw~1z1#AfYUFKWvefv591aj3(YqM1 z+Ib{#YE|q*_AfQUjMBL=PVZ*%Vo(Repz~u}k)X+4$xrzkxG*2k#HYGgp>eLJZ`w=c zB?x94HY?LiD1t_N%_1w;UShl<(V@Ut%GWeM!S-W;-lI^(rbQjH!au1W`1gDlkWaG$ zG>|Mt2p?2}+}AgRpfw0;G$CgoXYh$Gr#qVrAUgJ*hn|v|!&Y;FMlXrDSXN5HG-Ur1 zPyzGb`@}!ndw0N*Kyk0~nQZWhymTKH3RnYF3-VwYIZ{dx2QJM`z;3`+u^TFLzxosa znwdgq@NF?*``Mz^p5y(Ylpi)J!6AZYaofemd%HhP*b^*c_5%sT2>&sqg(-D=` zIj*W+k9J8PK0y54;c+-_bX#JC{CVi=XcndTey)^o9Q`YOhv-aPyNVC!P=-~n*R;=a zXp@s}e6nd-d7+9OnaO&7)1d?Q40(K|9C?D%#uI-Hpu?sgH8tEEk_UUeK32^3sNnZf z`+B5E*oe5a6TM@A+MFgpL&`^+WSlUe#St(F{V}NeJE{tz2Jg&42i11()%i! zt51&|BK&63Kri8JvJU=Pb{Jb9AA5R~PKuj0>Xt3Bk-HzDU)Ux386>Cq@{tc+A(Xd3w=-f<_`Jl zXG3%bWeptVC}tl!=H+~ENc`yCj@omI%mcm9e>M53NbXT{wSkma(nrodLcfMlA$GPZ0KO^$1LFr_`&oqtSYbU8gBzS$mDb|sG3 zq$DoOeBG1IG<)!qD#K%bB5os+Skd5ia)g-%<^&3vb1Z98KTiBoL96@0&0MJ2pG2!) zp3RQV*tAg7*MYUCwdy$AQ)d#cm`XIJL!KTmAmUx468dza()at=epimq(HmJ!02*(h zEn1J=4B~El2=*~#f2}(SCmkD<=b3k`dTqeDBY7FuE#$*K}z);c@-}woCS2euDYkUfz15p|R@HdW`6JBW$3ZP!%z5zhQ3)mW; z;TJDNTa9+GnBmZYceqkoOna~p-@=WDKA3&h4b%>B=H_RhJQesveHf0J(MflH+rTy$ znL2uSh;HK@%XNFv%R^NZ=>0gno^~Km`Ze|pM5RsEo;fFAFsPmu7X!z1B;?{k+J#DUS}6x4ntir$1iD(fsHKIksu#VzYR=-?G}OG>!qVB z6K7xS)ug_sp}KtDSsRhnY#$MyiYVIxF81SWveRUYD)uh2_IHInkC8lzuZ6KsGJ`Zs z!4%k(yCKD&%WCn0_gfX8C`ZsL2=B(MH8DgkKp4kxf@ngrzDlyAjOR})BNpy3%6OKO zKo~Ek*r7RH49nx=QB0}pP|Q+AcN+|>BzqQ}_vOiPkc&m0n~O6hcy{W01=`zFzqQSh zOWypu&2^GhRhAQRkY}=To^!G7p-)cMv*~U6WQf`AAGGzZqi{O@S!|5+Y>~r-Z3G8} zbrK6{7_ZUFY850B5Im5|I+9hk^q36leLx)29HV`X`6(a!yl)DdCzG5tNO37~kOm${ zan7E$#U3h-!_GaSn-!xlD#e8WB}8wZo)$a2GJQv88&^>dv(51`iUZpHx?a*&Q1*$(s1m@<3+B_%c9Qj4CYFa~aBHC9ujpTFanIn}^mxI0XE_#BkwtUbSBD9_=*#)l2Dc>>1qIXY`smByzDl8UOU$8?>6 z3e8-JHpT6Vg=W4#CPAg9^AKFDo=!kX_lg? z8_F5J0ZHnN>?h+f>x!ay3fd>TE3Cb9ffxKlF{3Rt-x*g7LN%Fv?|d(2>8-m%PRawK z`+425-FrcHfeC3_OJ5VYh;GkFej5F{#t!t1X)&bz!r_$Jbk^LKq!~Vc+r9hLg)^?B zRPExd45f7(+_D93;m{W#W2%6p5M+3wWW*KR^+<=~o}r#%8ctvD>PbXGgx3|nZ`Id{M?4<^ES ze#XY>j)-l!=_VXLe9INs8BrMY=!Et#wo+aKTlo-u7?W*eYp7?so*X%SDXV=x{I)`G zk+cqZLI0!88_6@;6+!dQ(!J&%#u{v}gNUpSWQYI4lJR^p`q{--;4-YZs9)6hD-G}L z@T$ajBk-)OAm63`Yb8&^B)_NFSFqXoTkX9n6ylm{Nmd{7op}tIuCr0nHBw7h(uRLI37FHb^U7U5CS-|c}@0WLBL>qe-v zB0MA-{sR|TBzn6bbbP4}z*!zo!146MO@v@jzJT$Bs-(B|i<~jBnU`O%y%UM;;!~&-`@c z@Pzj5W@prV*FvWV_u}QfOdN$I{=gJYyiZ%|Clh;uk?i^0%Q3*X-j8pj9i@WJK3e~v zL*60JhOS?lX`_UZ{MbIclKeSV!bRFvpV1);Ve~E*(JYYwt;Es(3OY}s0s6YaHC%UHH`GQxs8uC?fj!mW4d&w>ohpPg+)QwkCJ%+3n@_kUXjIXDMtUB%fvcJF!cru_6iem$m4 zD=TvLrF_=cl{{??fNMuzbc|vxvDV!O1smOl7u7MZ5@2Xj__K}+2 zN>JC-=Udd0sjOS-sUf0YflV|wpx=aO@PmW&vn-@1;~K>BuU&R5-p(Sg zs4U2lpITg_e0L|JhneFH1h27yX_;l4U@1fg7i`%8}`KwD22gdZbagiV|$!K{Iy8r=n zD}e<)l9F#eW3QkT_S!G&dMY!%Y~DLI~{kH31)=YB;(W+z*Ew+n3CR2ha(T z^wFA}I5$nRMK?EG8Wc0aq}VFNgke;@-LPN9mwcQ;ioLoQhQ<%SX3v#aOWTOU_gz*_1=W4Crx8Oi669j69eAe~fSS|m>3}Es z4s3Co3uvlv26`Uy63PZyjEBNTy&(I-7V%0T7vyhZ4d36y8s_nTKIY{=&!Uir5&FmV z&p_6;$v7rs0YT<-9_y0ST`=OXy#o2$*sN2-xhr9Z7S^Nb=XOV{DC?q|4k@+?ytXqF zaBA_|N9F@4qN?aocN-X3Cp$oVZzXWYN%qNnG%@z2wSaS$27k5O`DtsZ!{bM`Bw6Ug zWF!FZ*=n$I(oC2Zxd(!GoZrBt3@Q(h*v}9Q!)iL(p`N!T*{5Lp%W?n1jHKEB;0h6?Gg+mF8X>Zkh{GGR#cFk(=)@!pn1QU8kjQG8yNY2g+u zS0)AnHJ^vy~#YL4;oZiooJprm?Pkir5j#S+R0T3Zx79e^ zLLSlOsv$Bqmh5oskVv4P$f#O*t%F1Kep1%T1$O>|*RVq^7%|b70;_fTbUCIJMYiQ6 zxW9AWE;^`RDl+%5V|@#d(nq`XX8UQ52;w-=-8!}Xd-Uk+7k3xn+@icrD^=vd8{xA7 zRM4d%?pSu65-?`rA?*x@%XNb7Cs1^Ho^wOxUcrWK6JZG4g^N;R8_ zhSnZ*42ay3RARksUiNaX3+>DwYFa}g=B0wKUgHWS7f2UPjaaUqX*tnfO$UD+SgiG} zEDy1iq@xo1xa|%3Q774sBrlO=aCDnToZB#z_|)?NLDKQ4pWHl3I{MJG(#;|bo!6Gw zAdD7h$jT2ned=0Y_lysGDwr;d3vHN2F?pUA)Ugk%{ z??M=`;5CeS*dZ8C!r~-v&qAvos6A(qh_NJ4L*_#3_P#|+Og07rH9o7yLAX7v4ygqh zhI1#X;cD@E*Y)A2p{6z;smM}^Ng~|Wi%}0oZw3%cWI7ExSO^c79Nc2sD~&fe5$1?) z4PZ!`HMiergq99Z*S+7aN?x1($w_X<-+dLLSO)j=9gaTdMX3YmsKPQZLg6^`&zgsQ zJ*@S`jggn`UFs!$DGL%-pu>@lmfJa%?@+HN#+nPJc0IAX`c*qA{=?-TlYt6by~X!1 zp6w?Gqwk7k$>+QmE34p^W*SW6+XIgaLs<$#uE)yxlS?5%H$x{-^1PKnnwa(!E6bn! zpY(`&115*HFZY5&A$GNn`ud#w%LzPJjLC^iE?wC?;Un1xD6+nQ-VT8TyE^kiYMuNU zS(vA1Us@}yvB+v39^ov{ilx>1`lVXLe)N1rzq#~cV7O#wFgzvCZ1?AuMaFq+s_cv2 zq@IX#_?+jre4(`VMCQfw-kGk_e7El+@7}FkQlOM{adW1z%2SEIrZRxT|;5p+(X@T8C2*y0HW zD7mx}>F%IW&qZCZ*4A}bYewoClMbwoB1m+{0b@})eY~FLwha+H*cx!S!jswgxwu*5al)3{%KdxiA_V6ZS-lv# zaGGd~ka=fH=#;%q8}(F9X*V^R$U>x0mGsC$ihMMJ&Eb2)MQROW?7am6R;p-lJpapZ zspIKeUD1{=F@*ku3{vz+bP?`F{jhkkr%Bzn{*084iqi7OAc!vurjajJkt2PqX!=s! zFo6YU4kekkk6UM!J2_V&%JP9zO;2<%o>yV^pv@J0A%N8jimH&rtS6pCFz0;EIF{IL z&uCR>dGbDWdy-Z@y?p%y=tfOt%e<6OY?tpq?P4R+!`v;Vq(*+{P6SJU7*Q{we{}Sv z2`sFqcnEzZ!GHQ;h+DHqn??ps2*dSu$Nda+#jCPaw9V&M_&OiyT>}mSs*6is>SzL8 zCgYOx{tg7*et8DMY%!h&16f9h*ImM16O>I{n`*|q_ZB>V`&20H#^Y-IqEYAmB-~`9 z;f_}hPHL7%p!k{XPPr6WozgX`g5j|Mun_LNq4=r7+U1|}VGDAQ#hR~aPdrItf*VBl zsjk8rL#!h8AQ2FzGDQmE{{8N!)iLL1FOpP247tO8+LDx34GT6f_+2Vb_l)(oy#GjF zhOw=+`Z>#?Zo14C@OS}6@`5;+*~^KR&S9E&DA53^T{f}xJX^6^Po=7|Qqjrd^Z`+Q!{xE66jlNSa```}W(4qWro1el^ohBoyq7=RK zea10{J$zbV#WFFBtYUwb?x&n^@&WKAe0?~w0<#%AL!XENvwMp*Q>qbF9a7hHz31z; ze+%z>{}SFSuqFl)7$&jWQ0c?WdA>+(o#d%cA64`0_#Z9`TWSfQSsO5!0HkyT&jG=$oaR}T8AnTu@MlL4*O6`ATSV|xdvHym#(*N^2 z|KV0+|M?bP5~>46<5Kh@cX+#eF8TtnmpoR=c@{!9=|EyAIWcQ%`DVg9z~~vdOl4{+ zl{>4Zk3T(82Tlpi54|WWb0y&jY3WsR#U0+ccU$4HvLsDrz5UE9KX%$6SsuqvN-l>9 zk!g#7a${LkSZX3J>bCS@Nfb~3!q0lPo(sh+boeqqN-dF>6-kx{wR1(*4LQeUfD1q zJ}V~q;657iWrr&Zb9r#gGcYU6!ErYbYisggE!sp$xs0o}o+UffUn*c@90%e5oHJ#v;o} zY|3z+bL&TZ)s>K#u`jG&dZTmQqSX(ysj;5g=#pT^thJbL6AE_(U4Nirzx&2In1q9l z!%@u!1X1kLe2;`f_2y#awxGcFV~CjE=X{3xlk__$;ae%Qp~asqOrsgR0J8bfmrB2E zs^SE&fm>O6Qj`3_lr12~-S@xTUK1T&Y*X)Win$)o5%a*x_neG^j!*u&rbii3SP@o3 zZAA74S14z2*PsbQsMwp?R>o@-D^=Io{G|{AgAFA1356%_4N?u}7vOF`C-{^($@AI0 zbyRnP_YqD2&Jm}H6%Y!Ob)hMrLtO06Gdg!qATy#?nm8wm-*njk6_B(OzTxMM-c!XK zrm--#&DI7(Y--2oFV^6vx&j6WSISM_8cgk1K6@i|9_84};VwtM zq`>sv?=t40!Gad>I79{{n?!!PI-($P@_g?$d8PXPaUEx_1y+QXa$VJ#|5JI?Wa6=L z1vk+*X-Uw?0dV>IfRky$IAXkPP@^LfdQw(I2G{%v21L;fWS#+}XzQ8R0y_ayh$#n2 z!zcvD5oq!8rB?}7Gb1(Y+#rM;HWEFs&^~$+kY!b(mNPw;#s54eE;V)K#p8zrkJX1d zG_he-YD(YHwWDd`w0C{fp;Kp;s2|>?+$D7Tek!OQ4|mbDFRV~f*`#}O#}o) zg;1oIh;$JU0jVJh0#YL&ARr*U69MT>kS1MPkREy`p#}(XKF^$)XU;vpd*3sa6FhVSlY!;m90;Pqp7!ze9jssJVQ(ZhB zv4(N#HJf1Xj8A;$UL5GJS>ZiZw;NNQX)Yf=$1hRVY%Oi!8ACzle%g`sNCKu*>khln>spFj0$itha2LrqY47fS;Q-!1TXtGu)4Q}0obhS7-s@vYTum8^>!AjAGr)=GGn>sT86HgBi&uyvK zN&{#k0lC9q_~#AvzwevIP`ZY>M1m8k5qJPgr(VfC{Vzn=`e&n8{d=w#c%1(?$Y{Xf z^8$cJ_Yq*EMs4KMZQV7dS<*FtlaPu_Irhr{axKRkijhAelKx%q717T?Bd&bY3%1B^ zZy1fnmcH`I=+%~oDG(ASaj4)zqHby%;_^ke13@n8W8-}yy_P~0np$fB_Q(k^O=SaI zNdB%?nVd*`1;Fg9`9Xxo5v2_h^9{(;cSZzx?lUf_K}u-+^pIathFH zhmbA*T{r)qERp;F>DvEJBd>omdz-(FsPv1BM+pl(q5?!rpC!bd%^Y176$Nwsu z_&;EkQUGNlBw(7VZ^dCK#k1%A*tfxl2-#>OuYlw7xa?kpCvU_grQn)3JszDvG@S89 zGudmK^}Y{yQI1=DJ3xfF0R8{6BHKcn^F&G};uB)i-SVG`O-*)KRwh-*g68DOmZ2OS^IB>5=H+IsBE~2i* z08K%#pS8x3K8cSlBWDa>Oc>zLrOX@x#qAW#lvre>W^dZJCrq|30*AmdR_ObWAF3X? z7FuWwudDYYdNOt0Z(&KDXK~Zxc!W~&zr*g-I(5J;vv~|Gae%Mtjs<*0>1qKA>*G$K zAXgUfzpigB4kr(4RFE$l&^z3;f)r{*wv>~nvvTNGo@<|_?s;));#jD4;(48xJdo=%YTYW_S zyOq$t`|$k7-!X}6hF$`c8xq)DHs+Sy;U^Vsy7E&vP%a~b_EUxzvF+O_A@xEsMOm&7 zZn{|6PSx{yL}?u}XJa5ki;Nf5xIacKYi;X3?I;+Z1O9-RFxr`*HM5;(w|KWM?sjbc z2Sb#o^71Zx9;IZ)p)X!|e^zx!pzNH=n0!qTTxo^An3Z}}EfJevEx}mr1}RF*u^la| zuSy?X(aRRmc>QW4yE^6h3n)#4HiFA8{}T}NSpF5C^S@~krr!l;0uMv35H+Js`vezC zP`dCh{-AN2-c5$r*!pLZ^!qIcSu-bmO%ou`jr4(3Rt3sq5)6+_Ea~o(AGP#mO(l*4 zCm3Nob8Dy*tB>H&7j^BfQG3x_5M#x2N|HY#BV<#*@=${#YG=}F39FvjgxmM}d?Xy^ zU=HS#m!>pQ-@4slQK^kc-<0%CT5aR==9k{Ra^!u*pev5~Vu9Axm9922U+Y(ppyQM; zvNgTyaMqQQZfEzGQ&xd?Bc&x_%+HdmuHAvIMVieTnz8j262-8EP?hq=h<+y2bC$^o z!{QZ#Xu@#eKWI+;qvqDh@BGo4`};wxKl;^w`QBe;-uWYcl~-s|fBuiGra$_Qeh-p? zsOScO_OSVS2L~}H`qajQM7%VS4DEo=Dscs!eahTEX3cQ$Jo#vBo*LJKq&bL6kSpjY zjLNE04B^d_d-dmphGK|B>+z<|#pM%#fZTf3>y+FD*&>0daT058TUnvGo$2;^Uhk!+ zoOpeJ4mF^+?L_;Px7HCeXBLC~Rk?yxI2u_`XC^P0>L;Xtmmnd$8D4*+<@n#er|xRV z5IHC+{Eh#-pFAx9DG*R?$!q}Q2cz=?=kpZ4bq=Tx=nm)Ub2OfRPbXnSx#df)-@}f)&@REfe?p);Y;#fqmnO z@`0^iWV4y}i!ysiKYit$B)tJSWf?&9fw&1ex_-mms=v(MU0wt5mz19dVg^Mg(JtL% zYqeQlUxs67ba`H9>0Ot}MKM$m4iyqVW(oR+4WCLq`+YNNHW4giet2 zBEwG>zka>vPYe+EWI5DGSPV-pC<16Knl8t6xyWhYPlh8Hu;Es63fLMBwR;n{EV?L4 zBUpSNW4bPVyI$LSK1chml$Ct`knef>nj3X}nZE3ZG%|z=1$`$qPW)%=soHe?SeVO* zZCXh5_@D5aITDx0>sop2&3Qi^j)}$#cRHRwN-luB;zT%`T6qYy)VUu$XgQbNeBcjL z58^l<-icHou-VSH>h3*CG_IqwAzh*ecy@i>+5m$h4scGW%3Spo9Y*)JX$+OAZGh4Q z2I%C~c*wjH7Vd#_TD;(mno#WZLAuk*$>e}KxlpVnwOc#QKYDPT1X}jxAINNgT(3;l zbwjmpJNk?>)V!3IS~qQBe3YW3k&w;#s0mRm zV)VcESpk%Dy71qbm3q2|@1Qt((G!!l?#%wMGXY?~O&jGapA&<8j_tb=Tk3gKEfMZI zdOCht^NrNo=6=yTh<$EjoCF@DDJ z3^^*TLWwYaq;CZm@<4r)i3`}JFL}xkc3b4(#|-~L(DHB%J-N?=l>4y)V#e`VpQQ{I zEn@u#cmQu2q5e=Rb{FkEx0saOsts{%e*Uy-VJq`@3hi96k;}VPvg%>=i>v z%1V$6L-4@%Bcq36pjcr*z)D>vL_CYc=YyL$MZjR4;d_7|Xo+4$lLKT-6aVS9)-Auu ziG^?jOKPe+QhVb_w!_njmgb)a3hCS>QSoEVB%un^t*4_ULirtpdyk(h zdxM(b%$+%khg8JO`Pa9{WZ9$q{W6Z6QBIk(r!QT(`QYc^_%0jJoo?kX4q_(<>wgF^ z@n7`d>@$y-2qh{Fz3^fI)Kd*=CNVFejAC52&A~Q`L6JqBSVbrNG0c7Z> ztC>gU`pe!%GnkFGmN1MeYeIF~=IkT;-0WQCUv}z*$`;O_zeBpd^bJpmYOULB_F^V- zL+Jt%1XtgbN7%}{=lPKxhM`*PU9uu>S>3*$_t9$w;ou~vulK`55jzt@kxsXwqL9nq zOS)*^FzS0*#s?k0v-h)ChjkvAL!|PeQd{rO$x~2w1GY(jC0WymRN z6B5%<&F`RW2Mu=ziU3JrzPG-$ofD6ccoIITXDU|ROSKi@Zh6^@5!My7e(Cj+D5I|@ z^i>v-rSEfG!rJ#vWcpWFHdQal4=RJ8=Vv$~KC}~vT}<|9WmzX$KTIlT_4|NVUJn_S}kKZR{SihQPRG7on%o#IWk1W6zXhTDu)-USyq+o;QL~OB(?I z56%F`GBlH5nlMB?CUF5e2!70Sv_7_sSyo`;Q~Btbu@=!0axkDrh?_wejtCr-zD)*y zxgr8=B}Y?*HbqG@6L&p3?Fvo&d*kOjzLLb(p3Aigj50hsbPPE_0r`L#DAIj=Wmq8j zvR))P@ru8xdk{(Fsa^@U(fB#<)6cELC?gR#h|bRJs&Yu6g?f#;!QKjAL}Y=1eYKa_ zGUl8dZL2C3ER{nce=uTaKwm+bse_Q-c6d9GLitvTT|GspfrbIH)=Ox6?~~5Q$5p`P z=7r$Sp+n7hQna?Q$^7<@zaGbPsj&1acAfpcJBvfx5Bex%EgM}O+SoW{B+8$<0ta8^ z-B6ch(QeVn_jZ-wMW4n~w;)_}lT!N|X5h~>7_Uz7BaADs2MLSTrFg_v{6aZ~N2m7# z{`c(~GUe;`ude<{WD zVL+o#ACi(*|2P)JX!}^5@97?y>&uHOPzWCpbkWDu;Y_eZJWWL4y>yp@l4#{%!}og8 zp>Rz5p&~GPAadI<{Sr5Ms9{yc&v4@oal!sau+e_?x>x}6t4o4Rjs1;{hkhit7P*7n34+ZtKEaX?9~kxQEJgK zy4gTNWE1em-m`RtUhABwiv|jP_rA(y_{!0@AKMlOJX4o}UK-qI<6pRjm?=bzWE|EV z^m2)Dzw)JceSZ%k(BJlPcyuVo%%?l*6DVAkJ!Mq>5eF)p7oGG489aHTKk05PO3q!r6tIX_Y&*}mmuen?Xv({U`7uVoxx}~Gz|e* z58!aju?LhC7Ew1GXcvd1SOCrk;L7iKo}e|Ndb)b{MA2;*_E#=f_RLA1OxuE)+2eLH zF)ab3&yuNmXP3vlIm>nfJla~rIm`#2xxK}JY<13OAU!>GcCP|CAbA$*w#cVUdfN(> zlqR9e5`*;E)$g~}PHmQ1%e~)73DR0mxxah=KwoKy`-eoQzcsrA)$grSjB3;1D|m6f%HfMxS1NG&&K&1sK`q1AZ1- zK=G#VOc-DZGdNC=0PZiT?bxG%6d1Fyiv^u-m~*KFVhg~@>FckX{C&gJ|NVv!ev!>g zBp4AE)Gk=83#;;5U}Aoe{j>*v)fcj#E#$l_CGZ>!Rq04s)F<(-XstkGY8Zqq%atqI zcg>8qxKT~0K(>boDdFqfG!G>`Da^#kGkKC8{ZHlcb&n(M%Qohq19t*pTgxgCgt*YlouY&K+SYtoMdY)MRL4A%6*(Cca)uxp=q+Jw6#SPELUPBg;IQqUKRa) z26SPS%`|N!sr;Skgz6Q!48(DPO8^I*Ofr4v9gqq^Rbxneiq5#L2;Pav>pWVmGA*GD z|E^2%D!-5Xb*po?qdpYiT}}aI*d=>_vVgMyrW=zgk;fN!O+^Dmw#G0hQJ2h>@#J$S z<8m?&PS(dsJUEGyC;P(5etD7~oa9X>dE!Z4{QuT@;ct0HMzW=z@a z<_83|rDhv~<42mml2KSjd9;`lui|Y2cX|V=C6WM=k$C=t9O5yq*PGq3Dp^r9W6tsd zl8tOPgg*r^THkC0`ZG!_AqgTt>w}PV(xnD!;FyX@{Y5t23L^2nlA$^I*#FT10f3^v zq*5<|a7hFxDXd-ZnOTvjZJo&qu3JAkm_!cesk_TA>ROXR8dM#^ubu+%x`5+nR|{>)|}jKGa`Qg`Q@@^mAiX zwbZ^Y8B%lz`^s|K4V_baGn3;|)9B0O&S0)Ltkg#~3r>Rx4|haI%<|(bp8T#CfBjv= zM90fLbD2fhkm{!nFavJe%;9uC_Jv5yT*@Is7D`Nv0A4|W0kV2KXB(+*7I@0aJ_mSa zMN%U^pkF1?p$OT3t?drW78KxbJgN8ZN6yh(06-S-z?Qfw(V+>fs|_Q7W2h)(%)4R7)1}W5dXsns6llNHalm+yPN9)$ zXUcE4aRx;56E0?rNHzhGv>zVO6nJZ(dF;@S0pF}+N!5GlcHED1K9!0rk!@-efoGCy zjw7+G0_i>`@RV!LgFN_@P85L&+@CL@LFbe@L%KS+h}RDlbWGE8ul}fd%6%*;2YSm_ zV#o1yMoX%#q>-fo*Q9WPD*m<}G?*I`O62Zq_(IflJF6pXw_0nH`VQ9+$Ae?qeDt$1 zZoomNWFke(W67dTruW#V%neS3oU;Lp2hm~xrc(G~Ql14rTnxCmQ6JE(K+YlCe;|R; zQ3cuo86oJ{pqofZ?*K-v{)|J407CSv3T`-|Tu<(04+uuL1L!h~RHpyX7QN zr^hdAWw)_m`StsE?kzQF{0%gr#>fHL!i<@(Q0Zq18kN8z6;g5~mQtv8?rv?Ceb;23 z*t1nz|9T6l3rf;!3y_vG1IJ}Axsfj6=wh&;9^=h%G7+NtqJsAzZ|NsMg>LV)w7oUo zJg$0D@U}?cQ_bt?szB*f=Rh?^(Z~&24@@&Wh`~-6Qks@}ImE@;0RtGq*&%=Ka?~fu z=f#U_jVp1Be)7oBjwldiVeq3p?Y5-y<=H;O8kZ8{BzH4EbD~O!!raDR?q0);EQnNx z;4w!y<&G$L+g**CbfpWE^*^>>5Mo&7QNlBk5?<(?4M~;1&-sXSS3l2(WjJB>Vcm~R zpG50RiuC(eHt9df0VV(3((bQ%@A+gzN%1SXO&%!h%jlx<$(3`?ZcN9mCX&;dl2Yw$ z8uIovKrwSLlO|+9AEI0oD4XQe>TaYG7`}i=7NI;N%EahqeMtU&=Tz19Ag0*o@{h=8 zo4dJ;tQ@3n2Ko17-3L((F**<=t@=>zcLe9671jyR9$qekaWN(AgKDcIU(D^d?W=w! zx|j+8Q~~8PD(7PP+82lx*mFWmcG@a+k%Upna9>J3~Aw z_X%v50#}q4=!WH0NfIAkQ_*$hU12qN>?e8#eU%W0I~?urQFm8)bN8)_i=+QM8Nk_y zn4Hb_6}BJY%d8A7%d0QZ_N2AZD=7^QxcnYCe0+!ETB^C>PB4yXPPtlRCC?V`G zmlfSLV};mz>rD%{zie7$mpumR{1f_z1R6enBFwkTdD^C{T}I=Pm$X(SYxQ`A#$ixd1Z z${e~T=)NAPdO>|jD}v59h#H5^T(TzcZ4^SiU75D^n?VO4H{>C(WtD^+>eW9b00(C9 z1{&a*H0)_9iu|djs^LG~MvS-(-<<|TL83DIMmV%TXiI7WiTQoHZeN|SwT^ekLCk-b zt^6usq2+)Juv~_VlQX+N{yN}qys?TRq}?HS=28rP0L()gaur3I^MAypL)_3=L;^id z!SuUr+cd)tKxN3o73dCAM#RV8#fcvT{XBzZPQK(>&FYZROSVg#Dqn9I(m|EXT7Y9< zFW_0eBx?n?wCjUrQbNl`jMN;k$J^tTe1fJl8NSn;YDAIU zA{!DTCK$Om3lyXqdVi5M>s|pwu)ci7;Yki$)w-*Do(j;&MreQlEuVf%UP2Y__CA;k9#L8;kzp}b5+eb(mUPqG{hsV<~D zNIl~a6GBLzwh8rtnAx-F8RH8z6p+EHM*%41#rpC{peGf zmOy6;YOfa%lD@zgm-C(XT2A#?ICFb*vmVe5lWlN_jgwE-C*gVgSZdez*r}&!fKh1KdMZUsUcPcf`5$5@m2?x$^YKopuwiKI$}p?;l0* z{6(g?Ptq7i;ya*NH6LJqN%aO~M~+a!ylXPUpqu)GZ?RkUu@^^HihUP0cuXj`aEAA$ z^W7*dUBd9j@5vW|f=Wg`C$g^&Q214Vh%ZnCZ8$x2$qbMHj-g1>X0L&P`9(&&DHN2l zBZQn=Z~H~Yo=#$n`__BN+_3c$NUb9DKWW|J1QGb4!&=uJ> z!*8&qd8X(qM>RhH;SzLIQ$A=d2FXnLs-l3#qpE>k$i}-pb-8NCW13J&5Y`0UOu0q$ zETA_mJ;;F$*9iIDOq{TW-J1c$JK>7-XP!PI1m#6EHzTE@;&iHkbErk=_AfHAsI(B7 z0V$0NZ0jqxP3AFb_2FTF$ZeCwNKD5+^JPV++oK3BSQl&qZEB@@1;!)RC*l#3#+PFE zMh#6Yf{q4Nl3BC{aNSi6e&4!&7pI6cKC27>p-TYhLZv@MF&ax>F>CPT>cN0YN<7;WD)z|AE>v=sBLnD*CqH-qh5Jr5xhFqP59#iUesHTGg zK>YTj534zAJhL7|4R6Iz0)0JpbqId+v!?rvh3;#kWy7KTq>W{t5d{Fs4w0rYe z;Ba}OHU?QP2N$SyO7te5yCHL#Bz6Fl7|5B{5-`E*YjRA24HSW@6&H;cYorcT7gT%$ zYr&0P{Diocz48GHZsV?f?(O$x8ot>S_>71(get^HbH!W#Tll%7bB%lXTn<%$lIUKd zyn^{xA~Pt$zA!zL5vBN%kt?klBu*e&w0X51yOaE?;eJzF;!{XWBrt6KRSj>!obtoa z)e2Gve+EGS0KIhgk3DY#gjQ2}y11jM?&Mj+`fHW;Fxsbayw86a-Csd8UF5%CTC3oQ zNxm{9@XjjpOAMp;`>T(V*ET!Wd>d3z2%lC^JI%ppeK5gTrW@;1~=A7C425|p0z z5#}wZgV!?@<(&fo)~p>wP@~ji1_>L@~fa-9P%AHb9Dg)c$Bp77a%c>B*j0|o_FZ|p!j*fvLAD= zAzD8l^WKL6;DZa;kgAo533XWgu*-`$v%UglWB4Ag?Ks4+hNMGl?|=0&zDC)? z8W5DKt-94nw9i4O)N^>uc^GXxkX`T2G7+}s6Mf>F_-aEW3}wFTzbUCdoDiO8^3z1w znd0~-`0>}io`UNhgIRHp400SdEJ{j~SFFbvMZQEJ$XF$~qqw!cM9HM_c8al+$#;9~ zH>2O`VTMtuO9N-0xP&IYKPsTc-Q18@2OI6qs?a^$sVYjZ!TJOT2YG5|{nBpmyfKLSqftE6=}ND5g=-@7n1A8G701 z#GX+sqSzDEnJSMTVX4Q^qPAN zW7ifMcnn|2ijpbiKVn)XSqC-dT!yiX-}`o~8y#~Pt3V7!zeJxUNXWFzvd&tN%Dabr zQ{Vj}yJ|m?V`VkfcmNcv?rqRueFeUj5H8w)KiGt~VPt<}wD|#q3YnH^HO*w}%5Z9o zY{Q-rLm&@9bO(Z@^p9*r7r3tayJwD?hg>0NhzK#)Y=^*gdVw-WE*tQvy91Pfk$E12sTkm|i@X9+^lTelhsOL-6q&N2Cyf!as(vuiRfE^{`)9%dqWf4ck}!3qTPA#FYhC|2C^Z}@>snKgi% z!I}1kx(O;NXDh0X^0bj;8rDGr=q%sh($b30rGfp!*F(-c;^NvN(;WFMW#+zqy@aT? z1ngF0|IJArnQxMpWmFnI;WWp?(U%z zU`F;%9(C*u0QsLnsvfx-YCD{RMM<6ycVN}yX#Js|%lY1y(aIxQMIHIAqS|6sOy%Vz zqM)?S!E3l+=cPum7QXk9EcrKA&P<0KPS#S))CU|#&$QMCi#X*7eH7`tC*^zqRLj2p zmL=wz@ue-f?3-P$sm3M61&yb04ZFXW^uva0$p@^&tFf%ge!<)p_Oaf}UbjCr#aqS(_ZZvlUx|rv7 zb9Vy@GQuVDcTQZ=4Oy8;KG$g{Dk}Ua;uETiLAMI4A$0cb!tx&<@;FaV-FPVwpFU`n#H4CLL^-MkC+23brOGIJed&rc592+xVU8=i?%T8m=22rK{({> z6EE_6d-PF%kb?Pac_kcP!pi1*oF3#qrERE0uT+^br~E%Mvz$N7%$zu6zRxC^NBYx$ z`P{zC&Styq0>z?r`s&XBD>Jqf*s!=r>Sky&7)}|pR^8!v#lP8kJrHe>n#3YBt$Mh^ z)JD9AaXkh%Z7X?F>Wi8BvtQmi?}w~K#1#QH4wC6P#>+--EyrVPew2cXBVOK^Zcbi2 z6DD}Ks_fjyyp&j$(nvNser#E}skGCBC4}rI?Y0g;YkH8QK<0LGFn4P+Nm(AeG+%FAbVCA2|NX z1KfJ;JEI323mFRME=uZ&t<1R|x&iIVC=!e(Y;-)~T~)Q!(L5Bl)?kmJG2$p8P%uiU zB%*M`!Gm7e!M*je2ktMAC5IpSdU-`V2wUui9qAuRSU zjX5?22-u}e<{DVtTPlt2!R?N7DXouGN zO(yOs=1^p(<#14cMr+`hoAx6N%A77Vk(-E1s`tu=0_;ac_i~v=dL_H%x zVFw{y7CN#wX~xR{rn3k|FN}jeaqcb+0u(RCT$T5XTugsKeL!dYKBDz9^+vdPUVFa0 zxIRwLN=9+$ug)%=g2UwY`2Yl#$evkJEgZl-?(cgYOkpD`tWpVeRt@i!DRJVQ_Rn+F zw9oE<1c3%01PKT6S1PHtK=z*1NfH+=SymkxJ$~?B7q)IHOG2(AH@5)m5$mUB;<$yF zOY%MFrv~)9&Qg)RM5-5d96iqW*Ao(AQ=KyHDsJz?%FlUpRvo_NT+H+-^mg6HV3=S0 zA}dJNi>WVo0Yq|=48h-dChPLcoW0Q}lG9Fud!2fPdRT44tF$hd9hl7 z=YkPxq@N6<@KN+g*#&k zN0s`q@}?*~$lM?&UrloFw{3}nz?%zE{I2hjonqV}i4C+rbLEZ1j{_J}6phggOMCA0 zP7*{e%cG@?V%UE1M_&oA;f>M52}c1zuH?%{C3ms!NnDlc-kQ-()qZxPZsV~pEi^oz zL`X4w)8No`FV=qEQpFts(=z6ZTCTs|Z?)CW>SmBEyJT?`_(&bwR{2fP+fSZ~oUJ!L zdtjGT8v!KYpO*e-_HtrB{pz23ESm7J0HDze({k1@{^k96;c?!Bbf*=0RpfAm?WcHU zwHKWhO*;r1ZGtX(+|DTD)J)Ig>s(x-UIA`A7?!Jt+IFKV*?8mT(x(++<<&lSRa&I^ z(%*%ycz)->;POFisbez&GCqhS^_1vh0O!fiB}mZ6O&AW?F-ZNXCgH{4_+@APp>(=OP;On zd>qdoc~M~W>dlA`n8C9`DW$9PVT(&l#nO?_?+PBk{wNZQyhXF4@7vXgEXs4pQ_*KgodtoV48%^jnf(-o*oY0i<)@&+%rdCa9o6ShZTh5+cV<* z6vC*>^5SX{^UUd#nWKYs>cGtSYYd-(ZubTyFGL3Y2P7u66x?p+Mb0};2KB7tYFf{? zB&utxeZR2)LE=}U#<1w{f{lwaqwU|sZEkYAJLliryOzIn|K|7%sXKE5z*fP*ING?- zg1H7D=}0y-HQ~2F{Ygf#*OE-)wgT8e{6FtCG%sfs0!SS%h$H~-$AFw!m6FVw0~ zDMYXOw`oZ|GL^i#apH+`)Gz{ac7ktqoefblrXI!|vL(Uo1xk!G`;L`zvqzr)*-|7{ zo>4=b*$XKhfqc47mdgdM|+@~%7X+^?9 zuEg>xje7ryIoWMA{P1ZkPe_rqD?uNIc}8;-W+sX?D?xTmg|W-YI(`PYW*Xd z_q1AYtj@vB8};wgLKEal;#7Ly-*UUMeq#-0;U-bA;=A+g(J?)`4^!bIPY43IjjW`u z!Mz(<`S0CW48N|nS#zd@yFgip@=kUxc7fNv~e1^n9AWCX#8o zZ*yis(^K$pV#HBWB=#YRX=k2s;|0=2;Qa(D*%cX3e`AQ_;c&kB+12ZvkAIO(fm3>! zR(=0OcGtb6Lfd`7VjoWp{;7&XVJ54qd8>Y?Qs0lR9{LSuuF8FDt%jl`Xnj|YF5AD( zY=4`V>spMezWv;Qv#RI{iOB}YvX)p<2$w)We1_lE(pY4ANGlVy5N-3ARc9d$+%)Y_>0)b19#lN(;Ewt^5hp$+T(hs4apV0RQ#1m=Rx*fz<4c4dO zScSpE=)n{heclW=?Qa8_xT|?yMM-ppStbTOhT2xG^D7zbft=H?RQrpx6!O*b#Ud&P z78B;a7c4Q$gkye=4vrg+^M=^=nxt2+^Z55oN6?@8A|B4QG}(yoO};B)f_|P2`=`$ZkFfSgVtGPWfdjm59|#r0?)Kw zlp5sA**g{A%`U4uVZ71M9t*IWU4iiM4{?6`oi(xZw6wnSxO^vm)DcX-vv@dD%hsdo zw|CaWN?ZAf{OfLn!i)eL^n!=)_Jel;+XFOh1cMjZxU2eIUa?8;87r3zCYu2NurWi} zHs`0qA#h7q>UAwx>h24f_K>BSOj;184muE|2 zp2RU=IV`7~J))8kpRGBvR@DjmdJ0LSuA?(3_n=UI{yQLIVxhXL?L5MVf<4`H3Y?@s z(kr^Dt5NU0|6T@Oox8RIos>V8@fv#O(ILeAcHsf6Z-?)9EmHB%N$07P*H0&7!26FM zgU>z>u(aS7Nvsf-%aSExMlAReenURoNlQUA%(C#htA}qc&c>kPrGwSFuJsvKNY@k3gZ~iCyQjJv~HllHBP%{2>pyK!J r8VkSj@3WVJg7QCKi*E9hYt}p&gOhoCG6zoPz{wo=+jD^W*U$e36F{y) From 36160ccf4100c009fc3b323f98cc08e54f0bda87 Mon Sep 17 00:00:00 2001 From: joschrew Date: Mon, 9 Jan 2023 11:36:23 +0100 Subject: [PATCH 025/226] Change functionsignature docker/ssh clientcreation --- ocrd/ocrd/network/deployer.py | 24 ++++++++++++++++-------- ocrd/ocrd/network/deployment_utils.py | 8 ++++---- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index a6662a8f1..374ef4c2a 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -79,10 +79,12 @@ def _deploy_processing_worker(self, processor, host: HostData, deploy_type: Depl if deploy_type == DeployType.native: if not host.ssh_client: - host.ssh_client = create_ssh_client(host) + host.ssh_client = create_ssh_client(host.address, host.username, host.password, + host.keypath) else: if not host.docker_client: - host.docker_client = create_docker_client(host) + host.docker_client = create_docker_client(host.address, host.username, + host.password, host.keypath) for _ in range(processor.count): if deploy_type == DeployType.native: assert host.ssh_client # to satisfy mypy @@ -113,7 +115,8 @@ def _deploy_queue(self, image: str = 'rabbitmq', detach: bool = True, remove: bo # and receiving messages from queues is part of the RabbitMQ Library # Which is part of the OCR-D WebAPI implementation. - client = create_docker_client(self.mq_data) + client = create_docker_client(self.mq_data.address, self.mq_data.username, + self.mq_data.password, self.mq_data.keypath) if not ports: # 5672, 5671 - used by AMQP 0-9-1 and AMQP 1.0 clients without and with TLS # 15672, 15671: HTTP API clients, management UI and rabbitmqadmin, without and with TLS @@ -146,7 +149,8 @@ def _deploy_mongodb(self, image: str = 'mongo', detach: bool = True, remove: boo if not self.mongo_data or not self.mongo_data.address: self.log.debug('canceled mongo-deploy: no mongo_db in config') return "" - client = create_docker_client(self.mongo_data) + client = create_docker_client(self.mongo_data.address, self.mongo_data.username, + self.mongo_data.password, self.mongo_data.keypath) if not ports: ports = { 27017: self.mongo_data.port @@ -176,7 +180,8 @@ def _kill_queue(self) -> None: else: self.log.debug(f'trying to kill queue with id: {self.mq_data.pid} now') - client = create_docker_client(self.mq_data) + client = create_docker_client(self.mq_data.address, self.mq_data.username, + self.mq_data.password, self.mq_data.keypath) client.containers.get(self.mq_data.pid).stop() self.mq_data.pid = None client.close() @@ -189,7 +194,8 @@ def _kill_mongodb(self) -> None: else: self.log.debug(f'trying to kill mongdb with id: {self.mongo_data.pid} now') - client = create_docker_client(self.mongo_data) + client = create_docker_client(self.mongo_data.address, self.mongo_data.username, + self.mongo_data.password, self.mongo_data.keypath) client.containers.get(self.mongo_data.pid).stop() self.mongo_data.pid = None client.close() @@ -198,9 +204,11 @@ def _kill_mongodb(self) -> None: def _kill_processing_workers(self) -> None: for host in self.hosts: if host.ssh_client: - host.ssh_client = create_ssh_client(host) + host.ssh_client = create_ssh_client(host.address, host.username, host.password, + host.keypath) if host.docker_client: - host.docker_client = create_docker_client(host) + host.ssh_client = create_docker_client(host.address, host.username, host.password, + host.keypath) for p in host.processors_native: for pid in p.pids: host.ssh_client.exec_command(f'kill {pid}') diff --git a/ocrd/ocrd/network/deployment_utils.py b/ocrd/ocrd/network/deployment_utils.py index f73d81333..57d8ecfbb 100644 --- a/ocrd/ocrd/network/deployment_utils.py +++ b/ocrd/ocrd/network/deployment_utils.py @@ -47,8 +47,8 @@ def get_processor(parameter: dict, processor_class: type) -> Union[type, None]: return None -def create_ssh_client(obj: Any) -> paramiko.SSHClient: - address, username, password, keypath = obj.address, obj.username, obj.password, obj.keypath +def create_ssh_client(address: str, username: str, password: Union[str, None], + keypath: Union[str, None]) -> paramiko.SSHClient: assert address and username, 'address and username are mandatory' assert bool(password) is not bool(keypath), 'expecting either password or keypath, not both' @@ -63,8 +63,8 @@ def create_ssh_client(obj: Any) -> paramiko.SSHClient: return client -def create_docker_client(obj: Any) -> CustomDockerClient: - address, username, password, keypath = obj.address, obj.username, obj.password, obj.keypath +def create_docker_client(address: str, username: str, password: Union[str, None], + keypath: Union[str, None]) -> CustomDockerClient: assert address and username, 'address and username are mandatory' assert bool(password) is not bool(keypath), 'expecting either password or keypath ' \ 'provided, not both' From 23b0db91e59417edb14b7ef69f24f2fb3abf78c7 Mon Sep 17 00:00:00 2001 From: joschrew Date: Mon, 9 Jan 2023 12:25:18 +0100 Subject: [PATCH 026/226] Remove (not working) close_clients function --- ocrd/ocrd/network/deployer.py | 6 ++++-- ocrd/ocrd/network/deployment_utils.py | 6 ------ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index 374ef4c2a..a915ac3f6 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -4,7 +4,6 @@ getLogger ) from ocrd.network.deployment_utils import ( - close_clients, create_docker_client, create_ssh_client ) @@ -66,7 +65,10 @@ def _deploy_processing_workers(self, hosts: List[HostData], rabbitmq_address: st for p in host.processors_docker: # Ideally, pass the rabbitmq server and mongodb addresses here self._deploy_processing_worker(p, host, DeployType.docker, rabbitmq_address, mongodb_address) - close_clients(host) + if host.ssh_client: + host.ssh_client.close() + if host.docker_client: + host.docker_client.close() def _deploy_processing_worker(self, processor, host: HostData, deploy_type: DeployType, rabbitmq_server: str = '', mongodb: str = '') -> None: diff --git a/ocrd/ocrd/network/deployment_utils.py b/ocrd/ocrd/network/deployment_utils.py index 57d8ecfbb..6a880e7d3 100644 --- a/ocrd/ocrd/network/deployment_utils.py +++ b/ocrd/ocrd/network/deployment_utils.py @@ -71,12 +71,6 @@ def create_docker_client(address: str, username: str, password: Union[str, None] return CustomDockerClient(username, address, password=password, keypath=keypath) -def close_clients(*args) -> None: - for client in args: - if hasattr(client, 'close') and callable(client.close): - client.close() - - class CustomDockerClient(docker.DockerClient): """Wrapper for docker.DockerClient to use an own SshHttpAdapter. From f218669366f98ed13c8a99dfd3ea27f5327c668e Mon Sep 17 00:00:00 2001 From: joschrew Date: Mon, 9 Jan 2023 13:36:39 +0100 Subject: [PATCH 027/226] Move, rename and modify config-classes --- ocrd/ocrd/network/deployer.py | 106 +++---------------------- ocrd/ocrd/network/deployment_config.py | 91 +++++++++++++++++++++ ocrd/ocrd/network/deployment_utils.py | 12 +++ 3 files changed, 112 insertions(+), 97 deletions(-) create mode 100644 ocrd/ocrd/network/deployment_config.py diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index a915ac3f6..9eda74dea 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -5,8 +5,10 @@ ) from ocrd.network.deployment_utils import ( create_docker_client, - create_ssh_client + create_ssh_client, + DeployType, ) +from ocrd.network.deployment_config import * from ocrd.network.processing_worker import ProcessingWorker from typing import List, Dict, Union @@ -38,9 +40,9 @@ def __init__(self, config: Dict) -> None: """ self.log = getLogger(__name__) self.log.debug('Deployer-init()') - self.mongo_data = MongoData(config['mongo_db']) - self.mq_data = QueueData(config['message_queue']) - self.hosts = HostData.from_config(config) + self.mongo_data = MongoConfig(config['mongo_db']) + self.mq_data = QueueConfig(config['message_queue']) + self.hosts = HostConfig.from_config(config) def deploy_all(self) -> None: """ Deploy the message queue and all processors defined in the config-file @@ -56,7 +58,7 @@ def kill_all(self) -> None: self._kill_mongodb() self._kill_processing_workers() - def _deploy_processing_workers(self, hosts: List[HostData], rabbitmq_address: str, + def _deploy_processing_workers(self, hosts: List[HostConfig], rabbitmq_address: str, mongodb_address: str) -> None: for host in hosts: for p in host.processors_native: @@ -70,7 +72,7 @@ def _deploy_processing_workers(self, hosts: List[HostData], rabbitmq_address: st if host.docker_client: host.docker_client.close() - def _deploy_processing_worker(self, processor, host: HostData, deploy_type: DeployType, + def _deploy_processing_worker(self, processor, host: HostConfig, deploy_type: DeployType, rabbitmq_server: str = '', mongodb: str = '') -> None: self.log.debug(f'deploy "{deploy_type}" processor: "{processor}" on "{host.address}"') assert not processor.pids, 'processors already deployed. Pids are present. Host: ' \ @@ -210,7 +212,7 @@ def _kill_processing_workers(self) -> None: host.keypath) if host.docker_client: host.ssh_client = create_docker_client(host.address, host.username, host.password, - host.keypath) + host.keypath) for p in host.processors_native: for pid in p.pids: host.ssh_client.exec_command(f'kill {pid}') @@ -228,93 +230,3 @@ def _kill_processing_workers(self) -> None: # Then _kill_processing_workers should just call this method in a loop def _kill_processing_worker(self) -> None: pass - - -# TODO: These should be separated from the Deployer logic -# TODO: Moreover, some of these classes must be just @dataclasses -class HostData: - """Class to wrap information for all processing-server-hosts. - - Config information and runtime information is stored here. This class - should not do much but hold config information and runtime information. I - hope to make the code better understandable this way. Deployer should still - be the class who does things and this class here should be mostly passive - """ - - def __init__(self, config: dict) -> None: - self.address = config['address'] - self.username = config['username'] - self.password = config.get('password', None) - self.keypath = config.get('path_to_privkey', None) - assert self.password or self.keypath, 'Host in configfile with neither password nor keyfile' - self.processors_native = [] - self.processors_docker = [] - for x in config['deploy_processors']: - if x['deploy_type'] == 'native': - self.processors_native.append( - self.Processor(x['name'], x['number_of_instance'], DeployType.native) - ) - elif x['deploy_type'] == 'docker': - self.processors_docker.append( - self.Processor(x['name'], x['number_of_instance'], DeployType.docker) - ) - else: - assert False, f'unknown deploy_type: "{x.deploy_type}"' - self.ssh_client = None - self.docker_client = None - - @classmethod - def from_config(cls, config: Dict) -> List: - res = [] - for x in config['hosts']: - res.append(cls(x)) - return res - - class Processor: - def __init__(self, name: str, count: int, deploy_type: DeployType) -> None: - self.name = name - self.count = count - self.deploy_type = deploy_type - self.pids: List = [] - - def add_started_pid(self, pid) -> None: - self.pids.append(pid) - - -class MongoData: - """ Class to hold information for Mongodb-Docker container - """ - - def __init__(self, config: Dict) -> None: - self.address = config['address'] - self.port = int(config['port']) - self.username = config['ssh']['username'] - self.keypath = config['ssh'].get('path_to_privkey', None) - self.password = config['ssh'].get('password', None) - self.credentials = (config['credentials']['username'], config['credentials']['password']) - self.pid = None - - -class QueueData: - """ Class to hold information for RabbitMQ-Docker container - """ - - def __init__(self, config: Dict) -> None: - self.address = config['address'] - self.port = int(config['port']) - self.username = config['ssh']['username'] - self.keypath = config['ssh'].get('path_to_privkey', None) - self.password = config['ssh'].get('password', None) - self.credentials = (config['credentials']['username'], config['credentials']['password']) - self.pid = None - - -class DeployType(Enum): - """ Deploy-Type of the processing server. - """ - docker = 1 - native = 2 - - @staticmethod - def from_str(label: str) -> DeployType: - return DeployType[label.lower()] diff --git a/ocrd/ocrd/network/deployment_config.py b/ocrd/ocrd/network/deployment_config.py new file mode 100644 index 000000000..2e93601ff --- /dev/null +++ b/ocrd/ocrd/network/deployment_config.py @@ -0,0 +1,91 @@ +# TODO: this probably breaks python 3.6. Think about whether we really want to use this +from __future__ import annotations +from ocrd.network.deployment_utils import DeployType +from typing import List, Dict + +__all__ = [ + 'HostConfig', + 'ProcessorConfig', + 'MongoConfig', + 'QueueConfig', +] + + +class HostConfig: + """Class to wrap information for all processing-server-hosts. + + Config information and runtime information is stored here. This class + should not do much but hold config information and runtime information. I + hope to make the code better understandable this way. Deployer should still + be the class who does things and this class here should be mostly passive + """ + + def __init__(self, config: dict) -> None: + self.address = config['address'] + self.username = config['username'] + self.password = config.get('password', None) + self.keypath = config.get('path_to_privkey', None) + assert self.password or self.keypath, 'Host in configfile with neither password nor keyfile' + self.processors_native = [] + self.processors_docker = [] + for x in config['deploy_processors']: + if x['deploy_type'] == 'native': + self.processors_native.append( + ProcessorConfig(x['name'], x['number_of_instance'], DeployType.native) + ) + elif x['deploy_type'] == 'docker': + self.processors_docker.append( + ProcessorConfig(x['name'], x['number_of_instance'], DeployType.docker) + ) + else: + assert False, f'unknown deploy_type: "{x.deploy_type}"' + self.ssh_client = None + self.docker_client = None + + @staticmethod + def from_config(config: Dict) -> List: + res = [] + for x in config['hosts']: + res.append(HostConfig(x)) + return res + + +class ProcessorConfig: + """ Class wrapping information from config file for a Processing-Server/Worker + """ + def __init__(self, name: str, count: int, deploy_type: DeployType) -> None: + self.name = name + self.count = count + self.deploy_type = deploy_type + self.pids: List = [] + + def add_started_pid(self, pid) -> None: + self.pids.append(pid) + + +class MongoConfig: + """ Class to hold information for Mongodb-Docker container + """ + + def __init__(self, config: Dict) -> None: + self.address = config['address'] + self.port = int(config['port']) + self.username = config['ssh']['username'] + self.keypath = config['ssh'].get('path_to_privkey', None) + self.password = config['ssh'].get('password', None) + self.credentials = (config['credentials']['username'], config['credentials']['password']) + self.pid = None + + +class QueueConfig: + """ Class to hold information for RabbitMQ-Docker container + """ + + def __init__(self, config: Dict) -> None: + self.address = config['address'] + self.port = int(config['port']) + self.username = config['ssh']['username'] + self.keypath = config['ssh'].get('path_to_privkey', None) + self.password = config['ssh'].get('password', None) + self.credentials = (config['credentials']['username'], config['credentials']['password']) + self.pid = None diff --git a/ocrd/ocrd/network/deployment_utils.py b/ocrd/ocrd/network/deployment_utils.py index 6a880e7d3..60cd431e3 100644 --- a/ocrd/ocrd/network/deployment_utils.py +++ b/ocrd/ocrd/network/deployment_utils.py @@ -9,6 +9,7 @@ getLogger ) from typing import Callable, Union, Any +from enum import Enum # Method adopted from Triet's implementation @@ -117,3 +118,14 @@ def _create_paramiko_client(self, base_url: str) -> None: elif self.keypath: self.ssh_params['key_filename'] = self.keypath self.ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy) + + +class DeployType(Enum): + """ Deploy-Type of the processing server. + """ + docker = 1 + native = 2 + + @staticmethod + def from_str(label: str) -> DeployType: + return DeployType[label.lower()] From 078fa3190196197bdbfc83baa8aa1501675c2269 Mon Sep 17 00:00:00 2001 From: joschrew Date: Tue, 10 Jan 2023 09:52:23 +0100 Subject: [PATCH 028/226] Change enum value comparision for deploy_type --- ocrd/ocrd/network/deployment_config.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ocrd/ocrd/network/deployment_config.py b/ocrd/ocrd/network/deployment_config.py index 2e93601ff..ea87ce687 100644 --- a/ocrd/ocrd/network/deployment_config.py +++ b/ocrd/ocrd/network/deployment_config.py @@ -29,16 +29,15 @@ def __init__(self, config: dict) -> None: self.processors_native = [] self.processors_docker = [] for x in config['deploy_processors']: - if x['deploy_type'] == 'native': + if x['deploy_type'] == DeployType.native.name: self.processors_native.append( ProcessorConfig(x['name'], x['number_of_instance'], DeployType.native) ) - elif x['deploy_type'] == 'docker': + else: + assert x['deploy_type'] == DeployType.docker.name self.processors_docker.append( ProcessorConfig(x['name'], x['number_of_instance'], DeployType.docker) ) - else: - assert False, f'unknown deploy_type: "{x.deploy_type}"' self.ssh_client = None self.docker_client = None From 4f8d70049327d16496096836fb3033dfb5f288ca Mon Sep 17 00:00:00 2001 From: joschrew Date: Tue, 10 Jan 2023 11:19:38 +0100 Subject: [PATCH 029/226] Improve for loops over processors --- ocrd/ocrd/network/deployer.py | 39 ++++++++++++-------------- ocrd/ocrd/network/deployment_config.py | 22 ++++++--------- ocrd/ocrd/network/deployment_utils.py | 6 ++++ 3 files changed, 32 insertions(+), 35 deletions(-) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index 9eda74dea..1a25ee93f 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -61,27 +61,24 @@ def kill_all(self) -> None: def _deploy_processing_workers(self, hosts: List[HostConfig], rabbitmq_address: str, mongodb_address: str) -> None: for host in hosts: - for p in host.processors_native: - # Ideally, pass the rabbitmq server and mongodb addresses here - self._deploy_processing_worker(p, host, DeployType.native, rabbitmq_address, mongodb_address) - for p in host.processors_docker: - # Ideally, pass the rabbitmq server and mongodb addresses here - self._deploy_processing_worker(p, host, DeployType.docker, rabbitmq_address, mongodb_address) + for processor in host.processors: + self._deploy_processing_worker(processor, host, rabbitmq_address, mongodb_address) if host.ssh_client: host.ssh_client.close() if host.docker_client: host.docker_client.close() - def _deploy_processing_worker(self, processor, host: HostConfig, deploy_type: DeployType, + def _deploy_processing_worker(self, processor: ProcessorConfig, host: HostConfig, rabbitmq_server: str = '', mongodb: str = '') -> None: - self.log.debug(f'deploy "{deploy_type}" processor: "{processor}" on "{host.address}"') + self.log.debug(f'deploy "{processor.deploy_type}" processor: "{processor}" on' + f'"{host.address}"') assert not processor.pids, 'processors already deployed. Pids are present. Host: ' \ '{host.__dict__}. Processor: {processor.__dict__}' # Create the specific RabbitMQ queue here based on the OCR-D processor name (processor.name) # self.rmq_publisher.create_queue(queue_name=processor.name) - if deploy_type == DeployType.native: + if processor.deploy_type == DeployType.native: if not host.ssh_client: host.ssh_client = create_ssh_client(host.address, host.username, host.password, host.keypath) @@ -90,7 +87,7 @@ def _deploy_processing_worker(self, processor, host: HostConfig, deploy_type: De host.docker_client = create_docker_client(host.address, host.username, host.password, host.keypath) for _ in range(processor.count): - if deploy_type == DeployType.native: + if processor.deploy_type == DeployType.native: assert host.ssh_client # to satisfy mypy # This method should be rather part of the ProcessingWorker # The Processing Worker can just invoke a static method of ProcessingWorker @@ -213,17 +210,17 @@ def _kill_processing_workers(self) -> None: if host.docker_client: host.ssh_client = create_docker_client(host.address, host.username, host.password, host.keypath) - for p in host.processors_native: - for pid in p.pids: - host.ssh_client.exec_command(f'kill {pid}') - p.pids = [] - for p in host.processors_docker: - for pid in p.pids: - self.log.debug(f'trying to kill docker container: {pid}') - # TODO: think about timeout. - # think about using threads to kill parallelized to reduce waiting time - host.docker_client.containers.get(pid).stop() - p.pids = [] + for processor in host.processors: + if processor.deploy_type.is_native(): + for pid in processor.pids: + host.ssh_client.exec_command(f'kill {pid}') + else: + for pid in processor.pids: + self.log.debug(f'trying to kill docker container: {pid}') + # TODO: think about timeout. + # think about using threads to kill parallelized to reduce waiting time + host.docker_client.containers.get(pid).stop() + processor.pids = [] # May be good to have more flexibility here # TODO: Support that functionality as well. diff --git a/ocrd/ocrd/network/deployment_config.py b/ocrd/ocrd/network/deployment_config.py index ea87ce687..a125ad798 100644 --- a/ocrd/ocrd/network/deployment_config.py +++ b/ocrd/ocrd/network/deployment_config.py @@ -26,26 +26,20 @@ def __init__(self, config: dict) -> None: self.password = config.get('password', None) self.keypath = config.get('path_to_privkey', None) assert self.password or self.keypath, 'Host in configfile with neither password nor keyfile' - self.processors_native = [] - self.processors_docker = [] - for x in config['deploy_processors']: - if x['deploy_type'] == DeployType.native.name: - self.processors_native.append( - ProcessorConfig(x['name'], x['number_of_instance'], DeployType.native) - ) - else: - assert x['deploy_type'] == DeployType.docker.name - self.processors_docker.append( - ProcessorConfig(x['name'], x['number_of_instance'], DeployType.docker) - ) + self.processors = [] + for processor in config['deploy_processors']: + deploy_type = DeployType.from_str(processor['deploy_type']) + self.processors.append( + ProcessorConfig(processor['name'], processor['number_of_instance'], deploy_type) + ) self.ssh_client = None self.docker_client = None @staticmethod def from_config(config: Dict) -> List: res = [] - for x in config['hosts']: - res.append(HostConfig(x)) + for host in config['hosts']: + res.append(HostConfig(host)) return res diff --git a/ocrd/ocrd/network/deployment_utils.py b/ocrd/ocrd/network/deployment_utils.py index 60cd431e3..24d5af64a 100644 --- a/ocrd/ocrd/network/deployment_utils.py +++ b/ocrd/ocrd/network/deployment_utils.py @@ -129,3 +129,9 @@ class DeployType(Enum): @staticmethod def from_str(label: str) -> DeployType: return DeployType[label.lower()] + + def is_native(self) -> bool: + return self == DeployType.native + + def is_docker(self) -> bool: + return self == DeployType.docker From 6b549f1d37bef8bcaee1a9668504d171721cb772 Mon Sep 17 00:00:00 2001 From: joschrew Date: Tue, 10 Jan 2023 13:52:08 +0100 Subject: [PATCH 030/226] Remove and change some asserts While developing I added multiple asserts to discover errors earlier. Some of them are hard to understand, not needed any more or can be (and are) converted to exceptions. --- ocrd/ocrd/network/deployment_config.py | 1 - ocrd/ocrd/network/deployment_utils.py | 12 ++++-------- ocrd/ocrd/network/processing_broker.py | 1 - ocrd/ocrd/network/processing_worker.py | 2 +- 4 files changed, 5 insertions(+), 11 deletions(-) diff --git a/ocrd/ocrd/network/deployment_config.py b/ocrd/ocrd/network/deployment_config.py index a125ad798..58ac736dd 100644 --- a/ocrd/ocrd/network/deployment_config.py +++ b/ocrd/ocrd/network/deployment_config.py @@ -25,7 +25,6 @@ def __init__(self, config: dict) -> None: self.username = config['username'] self.password = config.get('password', None) self.keypath = config.get('path_to_privkey', None) - assert self.password or self.keypath, 'Host in configfile with neither password nor keyfile' self.processors = [] for processor in config['deploy_processors']: deploy_type = DeployType.from_str(processor['deploy_type']) diff --git a/ocrd/ocrd/network/deployment_utils.py b/ocrd/ocrd/network/deployment_utils.py index 24d5af64a..17c0ca9ed 100644 --- a/ocrd/ocrd/network/deployment_utils.py +++ b/ocrd/ocrd/network/deployment_utils.py @@ -50,9 +50,6 @@ def get_processor(parameter: dict, processor_class: type) -> Union[type, None]: def create_ssh_client(address: str, username: str, password: Union[str, None], keypath: Union[str, None]) -> paramiko.SSHClient: - assert address and username, 'address and username are mandatory' - assert bool(password) is not bool(keypath), 'expecting either password or keypath, not both' - client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy) log = getLogger(__name__) @@ -66,9 +63,6 @@ def create_ssh_client(address: str, username: str, password: Union[str, None], def create_docker_client(address: str, username: str, password: Union[str, None], keypath: Union[str, None]) -> CustomDockerClient: - assert address and username, 'address and username are mandatory' - assert bool(password) is not bool(keypath), 'expecting either password or keypath ' \ - 'provided, not both' return CustomDockerClient(username, address, password=password, keypath=keypath) @@ -86,8 +80,10 @@ class CustomDockerClient(docker.DockerClient): """ def __init__(self, user: str, host: str, **kwargs) -> None: - assert user and host, 'user and host must be set' - assert 'password' in kwargs or 'keypath' in kwargs, 'one of password and keyfile is needed' + if not user or not host: + raise ValueError("Missing argument: user and host must both be provided") + if not 'password' in kwargs and not 'keypath' in kwargs: + raise ValueError("Missing argument: one of password and keyfile is needed") self.api = docker.APIClient(f'ssh://{host}', use_ssh_client=True, version='1.41') ssh_adapter = self.CustomSshHttpAdapter(f'ssh://{user}@{host}:22', **kwargs) self.api.mount('http+docker://ssh', ssh_adapter) diff --git a/ocrd/ocrd/network/processing_broker.py b/ocrd/ocrd/network/processing_broker.py index 994e2784b..9f4e57837 100644 --- a/ocrd/ocrd/network/processing_broker.py +++ b/ocrd/ocrd/network/processing_broker.py @@ -52,7 +52,6 @@ def start(self) -> None: """ start processing broker with uvicorn """ - assert self.config, 'config was not parsed correctly' self.log.debug(f'starting uvicorn. Host: {self.host}. Port: {self.port}') uvicorn.run(self, host=self.hostname, port=self.port) diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index 3b51b4c23..bbaa078c4 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -94,5 +94,5 @@ def start_docker_processor(client: CustomDockerClient, name: str, _queue_address log.debug(f'start docker processor: {name}') # TODO: add real command here to start processing server here res = client.containers.run('debian', 'sleep 31', detach=True, remove=True) - assert res and res.id, 'run docker container failed' + assert res and res.id, 'run processor in docker-container failed' return res.id From f5ad740f854bf405c25475b16482b5962b30a281 Mon Sep 17 00:00:00 2001 From: joschrew Date: Tue, 10 Jan 2023 14:01:12 +0100 Subject: [PATCH 031/226] Add a few explanatory comments --- ocrd/ocrd/network/processing_worker.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index bbaa078c4..92277a7b6 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -67,6 +67,9 @@ def start_consuming(self) -> None: # self.rmq_consumer.start_consuming() pass + # TODO: queue_address and _database_address are prefixed with underscore because they are not + # needed yet (otherwise flak8 complains). But they will be needed once the real + # processing_worker is called here. Then they should be renamed @staticmethod def start_native_processor(client: SSHClient, name: str, _queue_address: str, _database_address: str) -> str: @@ -76,6 +79,11 @@ def start_native_processor(client: SSHClient, name: str, _queue_address: str, stdin, stdout = channel.makefile('wb'), channel.makefile('rb') # TODO: add real command here to start processing server here cmd = 'sleep 23s' + # the only way to make it work to start a process in the background and return early is + # this construction. The pid of the last started background process is printed with + # `echo $!` but it is printed inbetween other output. Because of that I added `xyz` before + # and after the code to easily be able to filter out the pid via regex when returning from + # the function stdin.write(f'{cmd} & \n echo xyz$!xyz \n exit \n') output = stdout.read().decode('utf-8') stdout.close() @@ -87,6 +95,9 @@ def start_native_processor(client: SSHClient, name: str, _queue_address: str, # error if try to call) return re.search(r'xyz([0-9]+)xyz', output).group(1) + # TODO: queue_address and _database_address are prefixed with underscore because they are not + # needed yet (otherwise flak8 complains). But they will be needed once the real + # processing_worker is called here. Then they should be renamed @staticmethod def start_docker_processor(client: CustomDockerClient, name: str, _queue_address: str, _database_address: str) -> str: From 02e3c4c51fcddde5c130af56ccbce2950941226f Mon Sep 17 00:00:00 2001 From: joschrew Date: Tue, 10 Jan 2023 16:22:18 +0100 Subject: [PATCH 032/226] Change and and comments Comments had wrong format and added one comment with a task to be done --- ocrd/ocrd/network/processing_broker.py | 43 +++++++++++++------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/ocrd/ocrd/network/processing_broker.py b/ocrd/ocrd/network/processing_broker.py index 9f4e57837..37e2f848a 100644 --- a/ocrd/ocrd/network/processing_broker.py +++ b/ocrd/ocrd/network/processing_broker.py @@ -38,15 +38,13 @@ def __init__(self, config_path: str, host: str, port: int) -> None: # summary='TODO: summary for apidesc', # TODO: add response model? add a response body at all? ) - - """ - Publish messages based on the API calls - Here is a call example to be adopted later - - # The message type is bytes - # Call this method to publish a message - self.rmq_publisher.publish_to_queue(queue_name='queue_name', message='message') - """ + # TODO: + # Publish messages based on the API calls + # Here is a call example to be adopted later + # + # # The message type is bytes + # # Call this method to publish a message + # self.rmq_publisher.publish_to_queue(queue_name='queue_name', message='message') def start(self) -> None: """ @@ -74,6 +72,10 @@ async def on_shutdown(self) -> None: - ensure queue is empty or processor is not currently running - connect to hosts and kill pids """ + # TODO: remove the try/except before beta. This is only needed for development. All + # exceptions this function (on_shutdown) throws are ignored / not printed, when it is used + # as shutdown-hook as it is now. So this try/except and logging is neccessary to make them + # visible when testing try: await self.stop_deployed_agents() except: @@ -87,16 +89,15 @@ async def stop_deployed_agents(self) -> None: @staticmethod def configure_publisher(config_file: str) -> 'RMQPublisher': rmq_publisher = 'RMQPublisher Object' - """ - Here is a template implementation to be adopted later - - rmq_publisher = RMQPublisher(host='localhost', port=5672, vhost='/') - # The credentials are configured inside definitions.json - # when building the RabbitMQ docker image - rmq_publisher.authenticate_and_connect( - username='default-publisher', - password='default-publisher' - ) - rmq_publisher.enable_delivery_confirmations() - """ + # TODO: + # Here is a template implementation to be adopted later + # + # rmq_publisher = RMQPublisher(host='localhost', port=5672, vhost='/') + # # The credentials are configured inside definitions.json + # # when building the RabbitMQ docker image + # rmq_publisher.authenticate_and_connect( + # username='default-publisher', + # password='default-publisher' + # ) + # rmq_publisher.enable_delivery_confirmations() return rmq_publisher From fde789a803b10bd374a0e9f9d14cc63a50685940 Mon Sep 17 00:00:00 2001 From: joschrew Date: Tue, 10 Jan 2023 16:43:06 +0100 Subject: [PATCH 033/226] Change config file validation and handling --- ocrd/ocrd/cli/processing_broker.py | 5 ---- ocrd/ocrd/network/deployer.py | 8 +++---- ocrd/ocrd/network/deployment_config.py | 17 +++++++------ ocrd/ocrd/network/processing_broker.py | 20 +++++++--------- ocrd_validators/ocrd_validators/__init__.py | 2 ++ .../ocrd_validators}/config.schema.yml | 0 .../processing_broker_validator.py | 24 +++++++++++++++++++ 7 files changed, 49 insertions(+), 27 deletions(-) rename {ocrd/ocrd/network => ocrd_validators/ocrd_validators}/config.schema.yml (100%) create mode 100644 ocrd_validators/ocrd_validators/processing_broker_validator.py diff --git a/ocrd/ocrd/cli/processing_broker.py b/ocrd/ocrd/cli/processing_broker.py index fb8dba547..84a92ecfc 100644 --- a/ocrd/ocrd/cli/processing_broker.py +++ b/ocrd/ocrd/cli/processing_broker.py @@ -8,7 +8,6 @@ import click from ocrd_utils import initLogging from ocrd.network import ProcessingBroker -import sys @click.command('processing-broker') @@ -19,10 +18,6 @@ def processing_broker_cli(path_to_config, address: str): Start and manage processing servers (workers) with the processing broker """ initLogging() - res = ProcessingBroker.validate_config(path_to_config) - if res: - print(f"config is invalid: {res}") - sys.exit(1) try: host, port = address.split(":") port_int = int(port) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index 1a25ee93f..391ce72a4 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -33,16 +33,16 @@ class Deployer: for managing information, not for actually doing things. """ - def __init__(self, config: Dict) -> None: + def __init__(self, config: ProcessingBrokerConfig) -> None: """ Args: config (Config): values from config file wrapped into class `Config` """ self.log = getLogger(__name__) self.log.debug('Deployer-init()') - self.mongo_data = MongoConfig(config['mongo_db']) - self.mq_data = QueueConfig(config['message_queue']) - self.hosts = HostConfig.from_config(config) + self.mongo_data = config.mongo_config + self.mq_data = config.queue_config + self.hosts = config.hosts_config def deploy_all(self) -> None: """ Deploy the message queue and all processors defined in the config-file diff --git a/ocrd/ocrd/network/deployment_config.py b/ocrd/ocrd/network/deployment_config.py index 58ac736dd..7c07676dc 100644 --- a/ocrd/ocrd/network/deployment_config.py +++ b/ocrd/ocrd/network/deployment_config.py @@ -4,6 +4,7 @@ from typing import List, Dict __all__ = [ + 'ProcessingBrokerConfig', 'HostConfig', 'ProcessorConfig', 'MongoConfig', @@ -11,6 +12,15 @@ ] +class ProcessingBrokerConfig: + def __init__(self, config: dict): + self.mongo_config = MongoConfig(config['mongo_db']) + self.queue_config = QueueConfig(config['message_queue']) + self.hosts_config = [] + for host in config['hosts']: + self.hosts_config.append(HostConfig(host)) + + class HostConfig: """Class to wrap information for all processing-server-hosts. @@ -34,13 +44,6 @@ def __init__(self, config: dict) -> None: self.ssh_client = None self.docker_client = None - @staticmethod - def from_config(config: Dict) -> List: - res = [] - for host in config['hosts']: - res.append(HostConfig(host)) - return res - class ProcessorConfig: """ Class wrapping information from config file for a Processing-Server/Worker diff --git a/ocrd/ocrd/network/processing_broker.py b/ocrd/ocrd/network/processing_broker.py index 37e2f848a..3cb91f489 100644 --- a/ocrd/ocrd/network/processing_broker.py +++ b/ocrd/ocrd/network/processing_broker.py @@ -6,6 +6,8 @@ from jsonschema import validate, ValidationError from ocrd_utils.package_resources import resource_string from typing import Union +from ocrd_validators import ProcessingBrokerValidator +from ocrd.network.deployment_config import ProcessingBrokerConfig class ProcessingBroker(FastAPI): @@ -18,8 +20,7 @@ def __init__(self, config_path: str, host: str, port: int) -> None: super().__init__(on_shutdown=[self.on_shutdown]) self.hostname = host self.port = port - with open(config_path) as fin: - self.config = yaml.safe_load(fin) + self.config = ProcessingBroker.parse_config(config_path) self.deployer = Deployer(self.config) # Deploy everything specified in the configuration self.deployer.deploy_all() @@ -54,16 +55,13 @@ def start(self) -> None: uvicorn.run(self, host=self.hostname, port=self.port) @staticmethod - def validate_config(config_path: str) -> Union[str, None]: + def parse_config(config_path: str) -> ProcessingBrokerConfig: with open(config_path) as fin: obj = yaml.safe_load(fin) - # TODO: move schema to another place?! - schema = yaml.safe_load(resource_string(__name__, 'config.schema.yml')) - try: - validate(obj, schema) - except ValidationError as e: - return f'{e.message}. At {e.json_path}' - return None + report = ProcessingBrokerValidator.validate(obj) + if not report.is_valid: + raise Exception(f"Processing-Broker configuration file is invalid:\n{report.errors}") + return ProcessingBrokerConfig(obj) async def on_shutdown(self) -> None: # TODO: shutdown docker containers @@ -87,7 +85,7 @@ async def stop_deployed_agents(self) -> None: # TODO: add correct typehint if RMQPublisher is available here in core @staticmethod - def configure_publisher(config_file: str) -> 'RMQPublisher': + def configure_publisher(config_file: ProcessingBrokerConfig) -> 'RMQPublisher': rmq_publisher = 'RMQPublisher Object' # TODO: # Here is a template implementation to be adopted later diff --git a/ocrd_validators/ocrd_validators/__init__.py b/ocrd_validators/ocrd_validators/__init__.py index 4819017dd..39acb9482 100644 --- a/ocrd_validators/ocrd_validators/__init__.py +++ b/ocrd_validators/ocrd_validators/__init__.py @@ -11,6 +11,7 @@ 'XsdValidator', 'XsdMetsValidator', 'XsdPageValidator', + 'ProcessingBrokerValidator', ] from .parameter_validator import ParameterValidator @@ -22,3 +23,4 @@ from .xsd_validator import XsdValidator from .xsd_mets_validator import XsdMetsValidator from .xsd_page_validator import XsdPageValidator +from .processing_broker_validator import ProcessingBrokerValidator diff --git a/ocrd/ocrd/network/config.schema.yml b/ocrd_validators/ocrd_validators/config.schema.yml similarity index 100% rename from ocrd/ocrd/network/config.schema.yml rename to ocrd_validators/ocrd_validators/config.schema.yml diff --git a/ocrd_validators/ocrd_validators/processing_broker_validator.py b/ocrd_validators/ocrd_validators/processing_broker_validator.py new file mode 100644 index 000000000..0e0c26262 --- /dev/null +++ b/ocrd_validators/ocrd_validators/processing_broker_validator.py @@ -0,0 +1,24 @@ +""" +Validating configuration file for the Processing-Broker +""" +from .json_validator import JsonValidator +import yaml +from ocrd_utils.package_resources import resource_string + + +# TODO: provide a link somewhere in this file as it is done in ocrd_tool.schema.yml but best with a +# working link. Currently it is here: +# https://github.com/OCR-D/spec/pull/222/files#diff-a71bf71cbc7d9ce94fded977f7544aba4df9e7bdb8fc0cf1014e14eb67a9b273 +# But that is a PR not merged yet +class ProcessingBrokerValidator(JsonValidator): + """ + JsonValidator validating against the schema for the Processing Broker + """ + + @staticmethod + def validate(obj): + """ + Validate against schema for Processing-Broker + """ + schema = yaml.safe_load(resource_string(__name__, 'config.schema.yml')) + return JsonValidator.validate(obj, schema) From ab74eec074dde20b5f83d72ff8bd06b51e582037 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Wed, 11 Jan 2023 14:51:27 +0100 Subject: [PATCH 034/226] refactor: port -> ports_mapping --- ocrd/ocrd/network/__init__.py | 1 - ocrd/ocrd/network/deployer.py | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/ocrd/ocrd/network/__init__.py b/ocrd/ocrd/network/__init__.py index d1b3e7432..2725b4b3a 100644 --- a/ocrd/ocrd/network/__init__.py +++ b/ocrd/ocrd/network/__init__.py @@ -1,6 +1,5 @@ # This network package is supposed to contain all the packages and modules to realize the network architecture: # https://user-images.githubusercontent.com/7795705/203554094-62ce135a-b367-49ba-9960-ffe1b7d39b2c.jpg -# The architecture is also available as an image: ocrd_network_arch.jpg # For reference, currently: # 1. The WebAPI is available here: diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index 391ce72a4..c78f34f01 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -110,7 +110,7 @@ def _deploy_processing_worker(self, processor: ProcessorConfig, host: HostConfig processor.add_started_pid(pid) def _deploy_queue(self, image: str = 'rabbitmq', detach: bool = True, remove: bool = True, - ports: Union[Dict, None] = None) -> str: + ports_mapping: Union[Dict, None] = None) -> str: # This method deploys the RabbitMQ Server. # Handling of creation of queues, submitting messages to queues, # and receiving messages from queues is part of the RabbitMQ Library @@ -118,12 +118,12 @@ def _deploy_queue(self, image: str = 'rabbitmq', detach: bool = True, remove: bo client = create_docker_client(self.mq_data.address, self.mq_data.username, self.mq_data.password, self.mq_data.keypath) - if not ports: + if not ports_mapping: # 5672, 5671 - used by AMQP 0-9-1 and AMQP 1.0 clients without and with TLS # 15672, 15671: HTTP API clients, management UI and rabbitmqadmin, without and with TLS # 25672: used for internode and CLI tools communication and is allocated from # a dynamic range (limited to a single port by default, computed as AMQP port + 20000) - ports = { + ports_mapping = { 5672: self.mq_data.port, 15672: 15672, 25672: 25672 @@ -133,7 +133,7 @@ def _deploy_queue(self, image: str = 'rabbitmq', detach: bool = True, remove: bo image=image, detach=detach, remove=remove, - ports=ports + ports=ports_mapping ) assert res and res.id, 'starting message queue failed' self.mq_data.pid = res.id From edaf55d5fb99e790be25af2c88719d5ec4f2ad61 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Wed, 11 Jan 2023 15:09:35 +0100 Subject: [PATCH 035/226] refactor: _queue -> _rabbitmq --- ocrd/ocrd/network/deployer.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index c78f34f01..1604a9ef6 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -48,13 +48,13 @@ def deploy_all(self) -> None: """ Deploy the message queue and all processors defined in the config-file """ # Ideally, this should return the address of the RabbitMQ Server - rabbitmq_address = self._deploy_queue() + rabbitmq_address = self._deploy_rabbitmq() # Ideally, this should return the address of the MongoDB mongodb_address = self._deploy_mongodb() self._deploy_processing_workers(self.hosts, rabbitmq_address, mongodb_address) def kill_all(self) -> None: - self._kill_queue() + self._kill_rabbitmq() self._kill_mongodb() self._kill_processing_workers() @@ -109,7 +109,7 @@ def _deploy_processing_worker(self, processor: ProcessorConfig, host: HostConfig _database_address=mongodb) processor.add_started_pid(pid) - def _deploy_queue(self, image: str = 'rabbitmq', detach: bool = True, remove: bool = True, + def _deploy_rabbitmq(self, image: str = 'rabbitmq', detach: bool = True, remove: bool = True, ports_mapping: Union[Dict, None] = None) -> str: # This method deploys the RabbitMQ Server. # Handling of creation of queues, submitting messages to queues, @@ -135,25 +135,25 @@ def _deploy_queue(self, image: str = 'rabbitmq', detach: bool = True, remove: bo remove=remove, ports=ports_mapping ) - assert res and res.id, 'starting message queue failed' + assert res and res.id, 'starting rabbitmq failed' self.mq_data.pid = res.id client.close() - self.log.debug('deployed queue') + self.log.debug('deployed rabbitmq') - # Not implemented yet - # Note: The queue address is not just the IP address + # TODO: Not implemented yet + # Note: The queue address is not just the IP address queue_address = 'RabbitMQ Server address' return queue_address def _deploy_mongodb(self, image: str = 'mongo', detach: bool = True, remove: bool = True, - ports: Union[Dict, None] = None) -> str: + ports_mapping: Union[Dict, None] = None) -> str: if not self.mongo_data or not self.mongo_data.address: self.log.debug('canceled mongo-deploy: no mongo_db in config') return "" client = create_docker_client(self.mongo_data.address, self.mongo_data.username, self.mongo_data.password, self.mongo_data.keypath) - if not ports: - ports = { + if not ports_mapping: + ports_mapping = { 27017: self.mongo_data.port } # TODO: use rm here or not? Should the mongodb be reused? @@ -162,31 +162,31 @@ def _deploy_mongodb(self, image: str = 'mongo', detach: bool = True, remove: boo image=image, detach=detach, remove=remove, - ports=ports + ports=ports_mapping ) assert res and res.id, 'starting mongodb failed' self.mongo_data.pid = res.id client.close() self.log.debug('deployed mongodb') - # Not implemented yet - # Note: The mongodb address is not just the IP address + # TODO: Not implemented yet + # Note: The mongodb address is not just the IP address mongodb_address = 'MongoDB Address' return mongodb_address - def _kill_queue(self) -> None: + def _kill_rabbitmq(self) -> None: if not self.mq_data.pid: - self.log.debug('kill_queue: queue not running') + self.log.debug('kill_rabbitmq: rabbitmq server is not running') return else: - self.log.debug(f'trying to kill queue with id: {self.mq_data.pid} now') + self.log.debug(f'trying to kill rabbitmq with id: {self.mq_data.pid} now') client = create_docker_client(self.mq_data.address, self.mq_data.username, self.mq_data.password, self.mq_data.keypath) client.containers.get(self.mq_data.pid).stop() self.mq_data.pid = None client.close() - self.log.debug('stopped queue') + self.log.debug('stopped rabbitmq') def _kill_mongodb(self) -> None: if not self.mongo_data or not self.mongo_data.pid: From 87a83a8dc357cd5eaa683d5058891105153094ce Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Wed, 11 Jan 2023 16:14:35 +0100 Subject: [PATCH 036/226] Fix typo in kill_processing_workers --- ocrd/ocrd/network/deployer.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index 1604a9ef6..7e1696fd6 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -110,7 +110,7 @@ def _deploy_processing_worker(self, processor: ProcessorConfig, host: HostConfig processor.add_started_pid(pid) def _deploy_rabbitmq(self, image: str = 'rabbitmq', detach: bool = True, remove: bool = True, - ports_mapping: Union[Dict, None] = None) -> str: + ports_mapping: Union[Dict, None] = None) -> str: # This method deploys the RabbitMQ Server. # Handling of creation of queues, submitting messages to queues, # and receiving messages from queues is part of the RabbitMQ Library @@ -205,11 +205,9 @@ def _kill_mongodb(self) -> None: def _kill_processing_workers(self) -> None: for host in self.hosts: if host.ssh_client: - host.ssh_client = create_ssh_client(host.address, host.username, host.password, - host.keypath) + host.ssh_client = create_ssh_client(host.address, host.username, host.password, host.keypath) if host.docker_client: - host.ssh_client = create_docker_client(host.address, host.username, host.password, - host.keypath) + host.docker_client = create_docker_client(host.address, host.username, host.password, host.keypath) for processor in host.processors: if processor.deploy_type.is_native(): for pid in processor.pids: From e82de103e52d8f96d7f538e4e22899502838b0cc Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Wed, 11 Jan 2023 16:36:54 +0100 Subject: [PATCH 037/226] Implement kill_processing_worker separately --- ocrd/ocrd/network/deployer.py | 38 +++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index 7e1696fd6..6c5604895 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -203,25 +203,29 @@ def _kill_mongodb(self) -> None: self.log.debug('stopped mongodb') def _kill_processing_workers(self) -> None: + # Kill processing worker hosts for host in self.hosts: if host.ssh_client: host.ssh_client = create_ssh_client(host.address, host.username, host.password, host.keypath) if host.docker_client: host.docker_client = create_docker_client(host.address, host.username, host.password, host.keypath) - for processor in host.processors: - if processor.deploy_type.is_native(): - for pid in processor.pids: - host.ssh_client.exec_command(f'kill {pid}') - else: - for pid in processor.pids: - self.log.debug(f'trying to kill docker container: {pid}') - # TODO: think about timeout. - # think about using threads to kill parallelized to reduce waiting time - host.docker_client.containers.get(pid).stop() - processor.pids = [] - - # May be good to have more flexibility here - # TODO: Support that functionality as well. - # Then _kill_processing_workers should just call this method in a loop - def _kill_processing_worker(self) -> None: - pass + # Kill deployed OCR-D processor instances on this Processing worker host + self._kill_processing_worker(host) + + def _kill_processing_worker(self, host: HostConfig) -> None: + for processor in host.processors: + if processor.deploy_type.is_native(): + for pid in processor.pids: + host.ssh_client.exec_command(f'kill {pid}') + elif processor.deploy_type.is_docker(): + for pid in processor.pids: + self.log.debug(f'trying to kill docker container: {pid}') + # TODO: think about timeout. + # think about using threads to kill parallelized to reduce waiting time + host.docker_client.containers.get(pid).stop() + else: + # Error case, should never enter here + # Handle error cases here (if needed) + self.log.error(f"Deploy type of {processor.name} is neither of the allowed types") + pass + processor.pids = [] From 612dc0bb46f61b27cfdf278fe54107906f6135d3 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Wed, 11 Jan 2023 18:38:37 +0100 Subject: [PATCH 038/226] Improve comments and checks --- ocrd/ocrd/network/deployer.py | 32 ++++++++++++++------------ ocrd/ocrd/network/deployment_config.py | 5 ++-- ocrd/ocrd/network/processing_worker.py | 7 +++--- 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index 6c5604895..0fd2e5ee5 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -70,6 +70,7 @@ def _deploy_processing_workers(self, hosts: List[HostConfig], rabbitmq_address: def _deploy_processing_worker(self, processor: ProcessorConfig, host: HostConfig, rabbitmq_server: str = '', mongodb: str = '') -> None: + self.log.debug(f'deploy "{processor.deploy_type}" processor: "{processor}" on' f'"{host.address}"') assert not processor.pids, 'processors already deployed. Pids are present. Host: ' \ @@ -80,34 +81,36 @@ def _deploy_processing_worker(self, processor: ProcessorConfig, host: HostConfig if processor.deploy_type == DeployType.native: if not host.ssh_client: - host.ssh_client = create_ssh_client(host.address, host.username, host.password, - host.keypath) - else: + host.ssh_client = create_ssh_client(host.address, host.username, host.password, host.keypath) + elif processor.deploy_type == DeployType.docker: if not host.docker_client: - host.docker_client = create_docker_client(host.address, host.username, - host.password, host.keypath) + host.docker_client = create_docker_client(host.address, host.username, host.password, host.keypath) + else: + # Error case, should never enter here. Handle error cases here (if needed) + self.log.error(f"Deploy type of {processor.name} is neither of the allowed types") + pass + for _ in range(processor.count): if processor.deploy_type == DeployType.native: assert host.ssh_client # to satisfy mypy - # This method should be rather part of the ProcessingWorker - # The Processing Worker can just invoke a static method of ProcessingWorker - # that creates an instance of the ProcessingWorker (Native instance) pid = ProcessingWorker.start_native_processor( client=host.ssh_client, name=processor.name, _queue_address=rabbitmq_server, _database_address=mongodb) - else: + processor.add_started_pid(pid) + elif processor.deploy_type == DeployType.docker: assert host.docker_client # to satisfy mypy - # This method should be rather part of the ProcessingWorker - # The Processing Worker can just invoke a static method of ProcessingWorker - # that creates an instance of the ProcessingWorker (Docker instance) pid = ProcessingWorker.start_docker_processor( client=host.docker_client, name=processor.name, _queue_address=rabbitmq_server, _database_address=mongodb) - processor.add_started_pid(pid) + processor.add_started_pid(pid) + else: + # Error case, should never enter here. Handle error cases here (if needed) + self.log.error(f"Deploy type of {processor.name} is neither of the allowed types") + pass def _deploy_rabbitmq(self, image: str = 'rabbitmq', detach: bool = True, remove: bool = True, ports_mapping: Union[Dict, None] = None) -> str: @@ -224,8 +227,7 @@ def _kill_processing_worker(self, host: HostConfig) -> None: # think about using threads to kill parallelized to reduce waiting time host.docker_client.containers.get(pid).stop() else: - # Error case, should never enter here - # Handle error cases here (if needed) + # Error case, should never enter here. Handle error cases here (if needed) self.log.error(f"Deploy type of {processor.name} is neither of the allowed types") pass processor.pids = [] diff --git a/ocrd/ocrd/network/deployment_config.py b/ocrd/ocrd/network/deployment_config.py index 7c07676dc..6ad73a0be 100644 --- a/ocrd/ocrd/network/deployment_config.py +++ b/ocrd/ocrd/network/deployment_config.py @@ -22,7 +22,7 @@ def __init__(self, config: dict): class HostConfig: - """Class to wrap information for all processing-server-hosts. + """Class to wrap information for all processing-worker-hosts. Config information and runtime information is stored here. This class should not do much but hold config information and runtime information. I @@ -46,7 +46,8 @@ def __init__(self, config: dict) -> None: class ProcessorConfig: - """ Class wrapping information from config file for a Processing-Server/Worker + """ + Class wrapping information from config file for an OCR-D processor """ def __init__(self, name: str, count: int, deploy_type: DeployType) -> None: self.name = name diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index 92277a7b6..98bc18c26 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -16,10 +16,11 @@ class ProcessingWorker: - def __init__(self, processor_arguments: dict, queue_address: str, - database_address: str) -> None: - self.log = getLogger(__name__) + def __init__(self, processor_name: str, processor_arguments: dict, + queue_address: str, database_address: str) -> None: + self.log = getLogger(__name__) + self.processor_name = processor_name # Required arguments to run the OCR-D Processor self.processor_arguments = processor_arguments # processor.name is # RabbitMQ Address - This contains at least the From 76825fcaeee952aa60e9948290191c54e6c061b4 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Thu, 12 Jan 2023 14:28:13 +0100 Subject: [PATCH 039/226] Add processing worker cli template --- ocrd/ocrd/cli/__init__.py | 2 ++ ocrd/ocrd/cli/processing_worker.py | 54 ++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 ocrd/ocrd/cli/processing_worker.py diff --git a/ocrd/ocrd/cli/__init__.py b/ocrd/ocrd/cli/__init__.py index 8f1fc2472..a429ad7dc 100644 --- a/ocrd/ocrd/cli/__init__.py +++ b/ocrd/ocrd/cli/__init__.py @@ -32,6 +32,7 @@ def get_help(self, ctx): from .zip import zip_cli from .log import log_cli from .processing_broker import processing_broker_cli +from .processing_worker import processing_worker_cli @click.group() @@ -51,3 +52,4 @@ def cli(**kwargs): # pylint: disable=unused-argument cli.add_command(log_cli) cli.add_command(resmgr_cli) cli.add_command(processing_broker_cli) +cli.add_command(processing_worker_cli) diff --git a/ocrd/ocrd/cli/processing_worker.py b/ocrd/ocrd/cli/processing_worker.py new file mode 100644 index 000000000..b42496680 --- /dev/null +++ b/ocrd/ocrd/cli/processing_worker.py @@ -0,0 +1,54 @@ +""" +OCR-D CLI: start the processing worker + +.. click:: ocrd.cli.processing_worker:zip_cli + :prog: ocrd processing-worker + :nested: full +""" +import click +from ocrd_utils import initLogging +from ocrd.network import ProcessingWorker + + +@click.command('processing-worker') +@click.argument('processor_name', required=True, type=click.STRING) +@click.option('-q', '--queue', + default="localhost:5672/", + help='The host, port, and virtual host of the RabbitMQ Server') +@click.option('-d', '--database', + default="localhost:27018", + help='The host and port of the MongoDB') +def processing_worker_cli(processor_name: str, queue: str, database: str): + """ + Start a processing worker (a specific ocr-d processor) + """ + initLogging() + try: + # TODO: Check here if the provided `processor_name` exists + processor_name = "ocrd-dummy" + + # TODO: Parse the actual RabbitMQ Server address - `queue` + rmq_host = "localhost" + rmq_port = 5672 + rmq_vhost = "/" + rmq_url = f"{rmq_host}:{rmq_port}{rmq_vhost}" + + # TODO: Parse the actual MongoDB address - `database` + db_prefix = "mongodb://" + db_host = "localhost" + db_port = 27018 + db_url = f"{db_prefix}{db_host}:{db_port}" + except ValueError: + raise click.UsageError('Wrong/Bad arguments format provided. Check the help sections') + + processing_worker = ProcessingWorker( + processor_name=processor_name, + processor_arguments={}, + rmq_host=rmq_host, + rmq_port=rmq_port, + rmq_vhost=rmq_vhost, + db_url=db_url + ) + + # TODO: Start the processing worker in the background + # and make it listen to the specific RabbitMQ queue From 8c60172edf56c3193d12c7c1b08790371d83f4f4 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Thu, 12 Jan 2023 14:30:57 +0100 Subject: [PATCH 040/226] Copy RabbitMQ utility library from the WebAPI impl repo --- .../rabbitmq_utils/Dockerfile-RabbitMQ | 10 + ocrd/ocrd/network/rabbitmq_utils/__init__.py | 15 +- ocrd/ocrd/network/rabbitmq_utils/config.toml | 9 + ocrd/ocrd/network/rabbitmq_utils/connector.py | 261 ++++++++++++++++++ ocrd/ocrd/network/rabbitmq_utils/constants.py | 45 +++ ocrd/ocrd/network/rabbitmq_utils/consumer.py | 156 +++++++++++ .../network/rabbitmq_utils/definitions.json | 85 ++++++ .../network/rabbitmq_utils/ocrd_messages.py | 57 ++++ ocrd/ocrd/network/rabbitmq_utils/publisher.py | 195 +++++++++++++ .../ocrd/network/rabbitmq_utils/rabbitmq.conf | 14 + 10 files changed, 845 insertions(+), 2 deletions(-) create mode 100644 ocrd/ocrd/network/rabbitmq_utils/Dockerfile-RabbitMQ create mode 100644 ocrd/ocrd/network/rabbitmq_utils/config.toml create mode 100644 ocrd/ocrd/network/rabbitmq_utils/connector.py create mode 100644 ocrd/ocrd/network/rabbitmq_utils/constants.py create mode 100644 ocrd/ocrd/network/rabbitmq_utils/consumer.py create mode 100755 ocrd/ocrd/network/rabbitmq_utils/definitions.json create mode 100644 ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py create mode 100644 ocrd/ocrd/network/rabbitmq_utils/publisher.py create mode 100755 ocrd/ocrd/network/rabbitmq_utils/rabbitmq.conf diff --git a/ocrd/ocrd/network/rabbitmq_utils/Dockerfile-RabbitMQ b/ocrd/ocrd/network/rabbitmq_utils/Dockerfile-RabbitMQ new file mode 100644 index 000000000..bb0c223e0 --- /dev/null +++ b/ocrd/ocrd/network/rabbitmq_utils/Dockerfile-RabbitMQ @@ -0,0 +1,10 @@ +FROM rabbitmq:3.8.27-management-alpine + +ADD ./rabbitmq.conf /etc/rabbitmq/ + +RUN chown rabbitmq:rabbitmq /etc/rabbitmq/rabbitmq.conf + +ADD --chown=rabbitmq ./definitions.json /etc/rabbitmq/ +ENV RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS="-rabbitmq_management load_definitions \"/etc/rabbitmq/definitions.json\"" + +EXPOSE 5672 15672 diff --git a/ocrd/ocrd/network/rabbitmq_utils/__init__.py b/ocrd/ocrd/network/rabbitmq_utils/__init__.py index 506e33044..fe570dc20 100644 --- a/ocrd/ocrd/network/rabbitmq_utils/__init__.py +++ b/ocrd/ocrd/network/rabbitmq_utils/__init__.py @@ -1,2 +1,13 @@ -# This rabbitmq_utils package is supposed to contain the code that is currently available under: -# https://github.com/OCR-D/ocrd-webapi-implementation/tree/main/ocrd_webapi/rabbitmq +__all__ = [ + "RMQConsumer", + "RMQConnector", + "RMQPublisher", +] + +from .consumer import RMQConsumer +from .connector import RMQConnector +from .publisher import RMQPublisher +from .ocrd_messages import ( + OcrdProcessingMessage, + OcrdResultMessage +) diff --git a/ocrd/ocrd/network/rabbitmq_utils/config.toml b/ocrd/ocrd/network/rabbitmq_utils/config.toml new file mode 100644 index 000000000..86b431b5c --- /dev/null +++ b/ocrd/ocrd/network/rabbitmq_utils/config.toml @@ -0,0 +1,9 @@ +# "rabbit-mq-host" when Dockerized +rabbit_mq_host = "localhost" +rabbit_mq_port = 5672 +rabbit_mq_vhost = "/" + +default_exchange_name = "ocrd-webapi-default" +default_exchange_type = "direct" +default_queue = "ocrd-webapi-default" +default_router = "ocrd-webapi-default" diff --git a/ocrd/ocrd/network/rabbitmq_utils/connector.py b/ocrd/ocrd/network/rabbitmq_utils/connector.py new file mode 100644 index 000000000..c0f658571 --- /dev/null +++ b/ocrd/ocrd/network/rabbitmq_utils/connector.py @@ -0,0 +1,261 @@ +""" +The source code in this file is adapted by reusing +some part of the source code from the official +RabbitMQ documentation. +""" +from typing import Any, Optional + +from pika import ( + BasicProperties, + BlockingConnection, + ConnectionParameters, + PlainCredentials +) + +from ocrd_webapi.rabbitmq.constants import ( + DEFAULT_EXCHANGER_NAME, + DEFAULT_EXCHANGER_TYPE, + DEFAULT_QUEUE, + DEFAULT_ROUTER, + RABBIT_MQ_HOST as HOST, + RABBIT_MQ_PORT as PORT, + RABBIT_MQ_VHOST as VHOST, + PREFETCH_COUNT +) + + +class RMQConnector: + def __init__(self, logger, host: str = HOST, port: int = PORT, vhost: str = VHOST): + self._logger = logger + self._host = host + self._port = port + self._vhost = vhost + + # According to the documentation, Pika blocking + # connections are not thread-safe! + self._connection = None + self._channel = None + + # Should try reconnecting again + self._try_reconnecting = False + # If the module has been stopped with a + # keyboard interruption, i.e., CTRL + C + self._gracefully_stopped = False + + @staticmethod + def declare_and_bind_defaults(connection, channel): + if connection and connection.is_open: + if channel and channel.is_open: + # Declare the default exchange agent + RMQConnector.exchange_declare( + channel=channel, + exchange_name=DEFAULT_EXCHANGER_NAME, + exchange_type=DEFAULT_EXCHANGER_TYPE, + ) + # Declare the default queue + RMQConnector.queue_declare( + channel, + queue_name=DEFAULT_QUEUE + ) + # Bind the default queue to the default exchange + RMQConnector.queue_bind( + channel, + queue_name=DEFAULT_QUEUE, + exchange_name=DEFAULT_EXCHANGER_NAME, + routing_key=DEFAULT_ROUTER + ) + + # Connection related methods + @staticmethod + def open_blocking_connection( + credentials: PlainCredentials, + host: str = HOST, + port: int = PORT, + vhost: str = VHOST + ) -> BlockingConnection: + blocking_connection = BlockingConnection( + parameters=ConnectionParameters( + host=host, + port=port, + virtual_host=vhost, + credentials=credentials, + # TODO: The heartbeat should not be disabled (0)! + heartbeat=0 + ), + ) + return blocking_connection + + @staticmethod + def open_blocking_channel(blocking_connection): + if blocking_connection and blocking_connection.is_open: + channel = blocking_connection.channel() + return channel + + @staticmethod + def exchange_bind( + channel, + destination_exchange: str, + source_exchange: str, + routing_key: str, + parameters: Optional[Any] = None + ): + if parameters is None: + parameters = {} + if channel and channel.is_open: + channel.exchange_bind( + destination=destination_exchange, + source=source_exchange, + routing_key=routing_key, + parameters=parameters + ) + + @staticmethod + def exchange_declare( + channel, + exchange_name: str, + exchange_type: str, + passive: bool = False, + durable: bool = False, + auto_delete: bool = False, + internal: bool = False, + arguments: Optional[Any] = None + ): + if arguments is None: + arguments = {} + if channel and channel.is_open: + exchange = channel.exchange_declare( + exchange=exchange_name, + exchange_type=exchange_type, + # Only check to see if the exchange exists + passive=passive, + # Survive a reboot of RabbitMQ + durable=durable, + # Remove when no more queues are bound to it + auto_delete=auto_delete, + # Can only be published to by other exchanges + internal=internal, + # Custom key/value pair arguments for the exchange + arguments=arguments + ) + return exchange + + @staticmethod + def exchange_delete(channel, exchange_name: str, if_unused: bool = False): + # Deletes queue only if unused + if channel and channel.is_open: + channel.exchange_delete(exchange=exchange_name, if_unused=if_unused) + + @staticmethod + def exchange_unbind( + channel, + destination_exchange: str, + source_exchange: str, + routing_key: str, + arguments: Optional[Any] = None + ): + if arguments is None: + arguments = {} + if channel and channel.is_open: + channel.exchange_unbind( + destination=destination_exchange, + source=source_exchange, + routing_key=routing_key, + arguments=arguments + ) + + @staticmethod + def queue_bind(channel, queue_name: str, exchange_name: str, routing_key: str, arguments: Optional[Any] = None): + if arguments is None: + arguments = {} + if channel and channel.is_open: + channel.queue_bind(queue=queue_name, exchange=exchange_name, routing_key=routing_key, arguments=arguments) + + @staticmethod + def queue_declare( + channel, + queue_name: str, + passive: bool = False, + durable: bool = False, + exclusive: bool = False, + auto_delete: bool = False, + arguments: Optional[Any] = None + ): + if arguments is None: + arguments = {} + if channel and channel.is_open: + queue = channel.queue_declare( + queue=queue_name, + # Only check to see if the queue exists and + # raise ChannelClosed exception if it does not + passive=passive, + # Survive reboots of the broker + durable=durable, + # Only allow access by the current connection + exclusive=exclusive, + # Delete after consumer cancels or disconnects + auto_delete=auto_delete, + # Custom key/value pair arguments for the queue + arguments=arguments + ) + return queue + + @staticmethod + def queue_delete(channel, queue_name: str, if_unused: bool = False, if_empty: bool = False): + if channel and channel.is_open: + channel.queue_delete( + queue=queue_name, + # Only delete if the queue is unused + if_unused=if_unused, + # Only delete if the queue is empty + if_empty=if_empty + ) + + @staticmethod + def queue_purge(channel, queue_name: str): + if channel and channel.is_open: + channel.queue_purge(queue=queue_name) + + @staticmethod + def queue_unbind(channel, queue_name: str, exchange_name: str, routing_key: str, arguments: Optional[Any] = None): + if arguments is None: + arguments = {} + if channel and channel.is_open: + channel.queue_unbind( + queue=queue_name, + exchange=exchange_name, + routing_key=routing_key, + arguments=arguments + ) + + @staticmethod + def set_qos(channel, prefetch_size: int = 0, prefetch_count: int = PREFETCH_COUNT, global_qos: bool = False): + if channel and channel.is_open: + channel.basic_qos( + # No specific limit if set to 0 + prefetch_size=prefetch_size, + prefetch_count=prefetch_count, + # Should the qos apply to all channels of the connection + global_qos=global_qos + ) + + @staticmethod + def confirm_delivery(channel): + if channel and channel.is_open: + channel.confirm_delivery() + + @staticmethod + def basic_publish(channel, exchange_name: str, routing_key: str, message_body: bytes, properties: BasicProperties): + if channel and channel.is_open: + channel.basic_publish( + exchange=exchange_name, + routing_key=routing_key, + body=message_body, + properties=properties + ) + + """ + @staticmethod + def basic_consume(channel): + # TODO: provide a general consume method here as well + pass + """ diff --git a/ocrd/ocrd/network/rabbitmq_utils/constants.py b/ocrd/ocrd/network/rabbitmq_utils/constants.py new file mode 100644 index 000000000..d3ec58ad2 --- /dev/null +++ b/ocrd/ocrd/network/rabbitmq_utils/constants.py @@ -0,0 +1,45 @@ +import logging +from pkg_resources import resource_filename +import tomli + +__all__ = [ + "DEFAULT_EXCHANGER_NAME", + "DEFAULT_EXCHANGER_TYPE", + "DEFAULT_QUEUE", + "DEFAULT_ROUTER", + "RABBIT_MQ_HOST", + "RABBIT_MQ_PORT", + "RABBIT_MQ_VHOST", + "RECONNECT_WAIT", + "RECONNECT_TRIES", + "PREFETCH_COUNT", + "LOG_FORMAT", + "LOG_LEVEL" +] + +TOML_FILENAME: str = resource_filename(__name__, 'config.toml') +TOML_FD = open(TOML_FILENAME, mode='rb') +TOML_CONFIG = tomli.load(TOML_FD) +TOML_FD.close() + +DEFAULT_EXCHANGER_NAME: str = TOML_CONFIG["default_exchange_name"] +DEFAULT_EXCHANGER_TYPE: str = TOML_CONFIG["default_exchange_type"] +DEFAULT_QUEUE: str = TOML_CONFIG["default_queue"] +DEFAULT_ROUTER: str = TOML_CONFIG["default_router"] + +# "rabbit-mq-host" when Dockerized +RABBIT_MQ_HOST: str = TOML_CONFIG["rabbit_mq_host"] +RABBIT_MQ_PORT: int = TOML_CONFIG["rabbit_mq_port"] +RABBIT_MQ_VHOST: str = TOML_CONFIG["rabbit_mq_vhost"] + +# Wait seconds before next reconnect try +RECONNECT_WAIT: int = 5 +# Reconnect tries before timeout +RECONNECT_TRIES: int = 3 +# QOS, i.e., how many messages to consume in a single go +# Check here: https://www.rabbitmq.com/consumer-prefetch.html +PREFETCH_COUNT: int = 1 + +# TODO: Integrate the OCR-D Logger once the logging in OCR-D is improved/optimized +LOG_FORMAT: str = '%(levelname) -10s %(asctime)s %(name) -30s %(funcName) -35s %(lineno) -5d: %(message)s' +LOG_LEVEL: int = logging.WARNING diff --git a/ocrd/ocrd/network/rabbitmq_utils/consumer.py b/ocrd/ocrd/network/rabbitmq_utils/consumer.py new file mode 100644 index 000000000..d734561e4 --- /dev/null +++ b/ocrd/ocrd/network/rabbitmq_utils/consumer.py @@ -0,0 +1,156 @@ +""" +The source code in this file is adapted by reusing +some part of the source code from the official +RabbitMQ documentation. +""" + +import logging +from time import sleep +from typing import Any, Union + +from pika import ( + PlainCredentials +) + +from ocrd_webapi.rabbitmq.constants import ( + DEFAULT_QUEUE, + LOG_FORMAT, + LOG_LEVEL, + RABBIT_MQ_HOST as HOST, + RABBIT_MQ_PORT as PORT, + RABBIT_MQ_VHOST as VHOST +) +from ocrd_webapi.rabbitmq.connector import RMQConnector + + +class RMQConsumer(RMQConnector): + def __init__(self, host: str = HOST, port: int = PORT, vhost: str = VHOST, logger_name: str = None): + if logger_name is None: + logger_name = __name__ + logger = logging.getLogger(logger_name) + logging.getLogger(logger_name).setLevel(LOG_LEVEL) + # This may mess up the global logger + logging.basicConfig(level=logging.WARNING) + super().__init__(logger=logger, host=host, port=port, vhost=vhost) + + self.consumer_tag = None + self.consuming = False + self.was_consuming = False + self.closing = False + + self.reconnect_delay = 0 + + def authenticate_and_connect(self, username: str, password: str): + credentials = PlainCredentials( + username=username, + password=password, + erase_on_connect=False # Delete credentials once connected + ) + self._connection = RMQConnector.open_blocking_connection( + host=self._host, + port=self._port, + vhost=self._vhost, + credentials=credentials, + ) + self._channel = RMQConnector.open_blocking_channel(self._connection) + + def setup_defaults(self): + RMQConnector.declare_and_bind_defaults(self._connection, self._channel) + + def example_run(self): + self.configure_consuming() + try: + self.start_consuming() + except KeyboardInterrupt: + self._logger.info("Keyboard interruption detected. Closing down peacefully.") + exit(0) + # TODO: Clean leftovers here and inform the RabbitMQ + # server about the disconnection of the consumer + # TODO: Implement the reconnect mechanism + except Exception: + reconnect_delay = 10 + self._logger.info(f'Reconnecting after {reconnect_delay} seconds') + sleep(reconnect_delay) + + def get_one_message( + self, + queue_name: str, + auto_ack: bool = False + ) -> Union[Any, None]: + message = None + if self._channel and self._channel.is_open: + message = self._channel.basic_get( + queue=queue_name, + auto_ack=auto_ack + ) + return message + + def configure_consuming( + self, + queue_name: str = None, + callback_method: Any = None + ): + self._logger.debug('Issuing consumer related RPC commands') + self._logger.debug('Adding consumer cancellation callback') + self._channel.add_on_cancel_callback(self.__on_consumer_cancelled) + if queue_name is None: + queue_name = DEFAULT_QUEUE + if callback_method is None: + callback_method = self.__on_message_received + self.consumer_tag = self._channel.basic_consume( + queue_name, + callback_method + ) + self.was_consuming = True + self.consuming = True + + def start_consuming(self): + if self._channel and self._channel.is_open: + self._channel.start_consuming() + + def get_waiting_message_count(self): + if self._channel and self._channel.is_open: + return self._channel.get_waiting_message_count() + + def __on_consumer_cancelled(self, frame: Any): + self._logger.warning(f'The consumer was cancelled remotely in frame: {frame}') + if self._channel: + self._channel.close() + + def __on_message_received( + self, + channel, + basic_deliver, + properties, + body + ): + tag = basic_deliver.delivery_tag + app_id = properties.app_id + message = body.decode() + self._logger.debug(f'Received message #{tag} from {app_id}: {message}') + self._logger.debug(f'Received message on channel: {channel}') + self.__ack_message(tag) + + def __ack_message(self, delivery_tag): + self._logger.debug(f'Acknowledging message {delivery_tag}') + self._channel.basic_ack(delivery_tag) + + +def main(): + # Connect to localhost:5672 by + # using the virtual host "/" (%2F) + consumer = RMQConsumer(host="localhost", port=5672, vhost="/") + # Configured with definitions.json when building the RabbitMQ image + # Check Dockerfile-RabbitMQ + consumer.authenticate_and_connect( + username="default-consumer", + password="default-consumer" + ) + consumer.setup_defaults() + consumer.example_run() + + +if __name__ == '__main__': + # RabbitMQ Server must be running before starting the example + # I.e., make start-rabbitmq + main() diff --git a/ocrd/ocrd/network/rabbitmq_utils/definitions.json b/ocrd/ocrd/network/rabbitmq_utils/definitions.json new file mode 100755 index 000000000..3f4df0c86 --- /dev/null +++ b/ocrd/ocrd/network/rabbitmq_utils/definitions.json @@ -0,0 +1,85 @@ +{ + "users": [ + { + "name": "guest", + "password": "guest", + "hashing_algorithm": "rabbit_password_hashing_sha256", + "tags": "administrator" + }, + { + "name": "admin", + "password": "admin", + "hashing_algorithm": "rabbit_password_hashing_sha256", + "tags": "administrator" + }, + { + "name": "default-consumer", + "password": "default-consumer", + "hashing_algorithm": "rabbit_password_hashing_sha256", + "tags": "administrator" + }, + { + "name": "default-publisher", + "password": "default-publisher", + "hashing_algorithm": "rabbit_password_hashing_sha256", + "tags": "administrator" + }, + { + "name": "test-session", + "password": "test-session", + "hashing_algorithm": "rabbit_password_hashing_sha256", + "tags": "administrator" + } + ], + "vhosts": [ + { + "name": "/" + }, + { + "name": "test" + } + + ], + "permissions": [ + { + "user": "guest", + "vhost": "/", + "configure": ".*", + "write": ".*", + "read": ".*" + }, + { + "user": "admin", + "vhost": "/", + "configure": ".*", + "write": ".*", + "read": ".*" + }, + { + "user": "default-consumer", + "vhost": "/", + "configure": ".*", + "write": ".*", + "read": ".*" + }, + { + "user": "default-publisher", + "vhost": "/", + "configure": ".*", + "write": ".*", + "read": ".*" + }, + { + "user": "test-session", + "vhost": "test", + "configure": ".*", + "write": ".*", + "read": ".*" + } + ], + "parameters": [], + "policies": [], + "queues": [], + "exchanges": [], + "bindings": [] +} diff --git a/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py b/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py new file mode 100644 index 000000000..4a83310e4 --- /dev/null +++ b/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py @@ -0,0 +1,57 @@ +# Check here for more details: Message structure #139 +from datetime import datetime +from typing import Any, Dict, List + + +class OcrdProcessingMessage: + def __init__( + self, + job_id: str = None, + processor_name: str = None, + created_time: int = None, + path_to_mets: str = None, + workspace_id: str = None, + input_file_grps: List[str] = None, + output_file_grps: List[str] = None, + page_id: str = None, + parameters: Dict[str, Any] = None, + result_queue_name: str = None, + ): + if not job_id: + raise ValueError(f"job_id must be set") + if not processor_name: + raise ValueError(f"processor_name must be set") + if not created_time: + # We should not raise a ValueError but just calculate it + created_time = int(datetime.utcnow().timestamp()) + if not input_file_grps or len(input_file_grps) == 0: + raise ValueError(f"input_file_grps must be set and contain at least 1 element") + if not (workspace_id or path_to_mets): + raise ValueError(f"Either `workspace_id` or `path_to_mets` must be set") + + self.job_id = job_id # uuid + self.processor_name = processor_name # "ocrd-.*" + # Either of these two below + self.workspace_id = workspace_id # uuid + self.path_to_mets = path_to_mets # absolute path + self.input_file_grps = input_file_grps + self.output_file_grps = output_file_grps + # e.g., "PHYS_0005..PHYS_0010" will process only pages between 5-10 + self.page_id = page_id + # e.g., "ocrd-cis-ocropy-binarize-result" + self.result_queue = result_queue_name + # processor parameters + self.parameters = parameters + self.created_time = created_time + + # TODO: Implement the validator checks, e.g., + # if the processor name matches the expected regex + + +class OcrdResultMessage: + def __init__(self, job_id: str, status: str, workspace_id: str, path_to_mets: str): + self.job_id = job_id + self.status = status + # Either of these two below + self.workspace_id = workspace_id + self.path_to_mets = path_to_mets diff --git a/ocrd/ocrd/network/rabbitmq_utils/publisher.py b/ocrd/ocrd/network/rabbitmq_utils/publisher.py new file mode 100644 index 000000000..204498785 --- /dev/null +++ b/ocrd/ocrd/network/rabbitmq_utils/publisher.py @@ -0,0 +1,195 @@ +""" +The source code in this file is adapted by reusing +some part of the source code from the official +RabbitMQ documentation. +""" + +import logging +from time import sleep +from typing import Any, Optional + +from pika import ( + BasicProperties, + PlainCredentials +) + +from ocrd_webapi.rabbitmq.constants import ( + DEFAULT_EXCHANGER_NAME, + DEFAULT_ROUTER, + LOG_FORMAT, + LOG_LEVEL, + RABBIT_MQ_HOST as HOST, + RABBIT_MQ_PORT as PORT, + RABBIT_MQ_VHOST as VHOST +) +from ocrd_webapi.rabbitmq.connector import RMQConnector + + +class RMQPublisher(RMQConnector): + def __init__(self, host: str = HOST, port: int = PORT, vhost: str = VHOST, logger_name: str = None): + if logger_name is None: + logger_name = __name__ + logger = logging.getLogger(logger_name) + logging.getLogger(logger_name).setLevel(LOG_LEVEL) + # This may mess up the global logger + logging.basicConfig(level=logging.WARNING) + super().__init__(logger=logger, host=host, port=port, vhost=vhost) + + self.message_counter = 0 + self.deliveries = {} + self.acked_counter = 0 + self.nacked_counter = 0 + self.running = True + + def authenticate_and_connect(self, username: str, password: str): + credentials = PlainCredentials( + username=username, + password=password, + erase_on_connect=False # Delete credentials once connected + ) + self._connection = RMQConnector.open_blocking_connection( + host=self._host, + port=self._port, + vhost=self._vhost, + credentials=credentials, + ) + self._channel = RMQConnector.open_blocking_channel(self._connection) + + def setup_defaults(self): + RMQConnector.declare_and_bind_defaults(self._connection, self._channel) + + def example_run(self): + while True: + try: + messages = 1 + message = f"#{messages}" + self.publish_to_queue(queue_name=DEFAULT_ROUTER, message=message) + messages += 1 + sleep(2) + except KeyboardInterrupt: + self._logger.info("Keyboard interruption detected. Closing down peacefully.") + exit(0) + # TODO: Clean leftovers here and inform the RabbitMQ + # server about the disconnection of the publisher + # TODO: Implement the reconnect mechanism + except Exception: + reconnect_delay = 10 + self._logger.info(f'Reconnecting after {reconnect_delay} seconds') + sleep(reconnect_delay) + + def create_queue( + self, + queue_name: str, + exchange_name: Optional[str] = None, + exchange_type: Optional[str] = None + ): + if exchange_name is None: + exchange_name = DEFAULT_EXCHANGER_NAME + if exchange_type is None: + exchange_type = "direct" + + RMQConnector.exchange_declare( + channel=self._channel, + exchange_name=exchange_name, + exchange_type=exchange_type + ) + RMQConnector.queue_declare( + channel=self._channel, + queue_name=queue_name + ) + RMQConnector.queue_bind( + channel=self._channel, + queue_name=queue_name, + exchange_name=exchange_name, + # the routing key matches the queue name + routing_key=queue_name + ) + + def publish_to_queue( + self, + queue_name: str, + message: Any, + exchange_name: Optional[str] = None, + properties: Optional[Any] = None): + if exchange_name is None: + exchange_name = DEFAULT_EXCHANGER_NAME + if properties is None: + headers = {'OCR-D WebApi Header': 'OCR-D WebApi Value'} + properties = BasicProperties( + app_id='webapi-processing-broker', + content_type='application/json', + headers=headers + ) + + # Note: There is no way to publish to a queue directly. + # Publishing happens through an exchange agent with + # a routing key - specified when binding the queue to the exchange + RMQConnector.basic_publish( + self._channel, + exchange_name=exchange_name, + # The routing key and the queue name must match! + routing_key=queue_name, + message_body=message, + properties=properties + ) + + self.message_counter += 1 + self.deliveries[self.message_counter] = True + self._logger.info(f'Published message #{self.message_counter}') + + def enable_delivery_confirmations(self): + self._logger.debug('Enabling delivery confirmations (Confirm.Select RPC)') + RMQConnector.confirm_delivery(channel=self._channel) + + # TODO: Find a way to use this callback method, + # seems not possible with Blocking Connections + def __on_delivery_confirmation(self, frame): + confirmation_type = frame.method.NAME.split('.')[1].lower() + delivery_tag: int = frame.method.delivery_tag + ack_multiple = frame.method.multiple + + self._logger.debug(f'Received: {confirmation_type} ' + f'for tag: {delivery_tag} ' + f'(multiple: {ack_multiple})') + + if confirmation_type == 'ack': + self.acked_counter += 1 + elif confirmation_type == 'nack': + self.nacked_counter += 1 + + del self.deliveries[delivery_tag] + + if ack_multiple: + for tmp_tag in list(self.deliveries.keys()): + if tmp_tag <= delivery_tag: + self.acked_counter += 1 + del self.deliveries[tmp_tag] + + # TODO: Check here for stale entries inside the _deliveries + # and attempt to re-delivery with max amount of tries (not defined yet) + + self._logger.debug( + 'Published %i messages, %i have yet to be confirmed, ' + '%i were acked and %i were nacked', self.message_counter, + len(self.deliveries), self.acked_counter, self.nacked_counter) + + +def main(): + # Connect to localhost:5672 by + # using the virtual host "/" (%2F) + publisher = RMQPublisher(host="localhost", port=5672, vhost="/") + # Configured with definitions.json when building the RabbitMQ image + # Check Dockerfile-RabbitMQ + publisher.authenticate_and_connect( + username="default-publisher", + password="default-publisher" + ) + publisher.setup_defaults() + publisher.enable_delivery_confirmations() + publisher.example_run() + + +if __name__ == '__main__': + # RabbitMQ Server must be running before starting the example + # I.e., make start-rabbitmq + main() diff --git a/ocrd/ocrd/network/rabbitmq_utils/rabbitmq.conf b/ocrd/ocrd/network/rabbitmq_utils/rabbitmq.conf new file mode 100755 index 000000000..2cb3a539b --- /dev/null +++ b/ocrd/ocrd/network/rabbitmq_utils/rabbitmq.conf @@ -0,0 +1,14 @@ +default_user = guest +default_pass = guest + +loopback_users.guest = true + +listeners.tcp.default = 5672 +management.tcp.port = 15672 +management.load_definitions = /etc/rabbitmq/definitions.json + +ssl_options.verify = verify_peer +ssl_options.fail_if_no_peer_cert = false + +# 60 minutes in milliseconds +consumer_timeout = 3600000 From 736a6063cfbdd9e424cfc76815e83166191b5547 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Thu, 12 Jan 2023 14:32:16 +0100 Subject: [PATCH 041/226] Add the reference note to the RabbitMQ utils --- ocrd/ocrd/network/rabbitmq_utils/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ocrd/ocrd/network/rabbitmq_utils/__init__.py b/ocrd/ocrd/network/rabbitmq_utils/__init__.py index fe570dc20..a250ce88c 100644 --- a/ocrd/ocrd/network/rabbitmq_utils/__init__.py +++ b/ocrd/ocrd/network/rabbitmq_utils/__init__.py @@ -1,3 +1,6 @@ +# The RabbitMQ utils are directly copied from the OCR-D WebAPI implementation repo +# https://github.com/OCR-D/ocrd-webapi-implementation/tree/main/ocrd_webapi/rabbitmq + __all__ = [ "RMQConsumer", "RMQConnector", From 1ec937145c29b547444e58b686832109b7206890 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Thu, 12 Jan 2023 15:21:18 +0100 Subject: [PATCH 042/226] Use actual RMQConsumer inside the processing_worker --- ocrd/ocrd/cli/processing_worker.py | 6 +- ocrd/ocrd/network/__init__.py | 1 + ocrd/ocrd/network/deployment_utils.py | 46 +--------- ocrd/ocrd/network/processing_worker.py | 113 ++++++++++++++++--------- 4 files changed, 82 insertions(+), 84 deletions(-) diff --git a/ocrd/ocrd/cli/processing_worker.py b/ocrd/ocrd/cli/processing_worker.py index b42496680..5bd0fbb07 100644 --- a/ocrd/ocrd/cli/processing_worker.py +++ b/ocrd/ocrd/cli/processing_worker.py @@ -50,5 +50,7 @@ def processing_worker_cli(processor_name: str, queue: str, database: str): db_url=db_url ) - # TODO: Start the processing worker in the background - # and make it listen to the specific RabbitMQ queue + # TODO: Load the OCR-D processor in the memory cache + + # Start consuming with the configuration settings above + processing_worker.start_consuming() diff --git a/ocrd/ocrd/network/__init__.py b/ocrd/ocrd/network/__init__.py index 2725b4b3a..aba941597 100644 --- a/ocrd/ocrd/network/__init__.py +++ b/ocrd/ocrd/network/__init__.py @@ -18,3 +18,4 @@ # It could also be a separate package on its own under `core` with the name `ocrd_network`. # TODO: Correctly identify all current and potential future dependencies. from .processing_broker import ProcessingBroker +from .processing_worker import ProcessingWorker diff --git a/ocrd/ocrd/network/deployment_utils.py b/ocrd/ocrd/network/deployment_utils.py index 17c0ca9ed..73c11704e 100644 --- a/ocrd/ocrd/network/deployment_utils.py +++ b/ocrd/ocrd/network/deployment_utils.py @@ -1,51 +1,13 @@ from __future__ import annotations +from enum import Enum +from typing import Union + import docker from docker.transport import SSHHTTPAdapter -from frozendict import frozendict -from functools import lru_cache, wraps import paramiko import urllib.parse -from ocrd_utils import ( - getLogger -) -from typing import Callable, Union, Any -from enum import Enum - -# Method adopted from Triet's implementation -# https://github.com/OCR-D/core/pull/884/files#diff-8b69cb85b5ffcfb93a053791dec62a2f909a0669ae33d8a2412f246c3b01f1a3R260 -def freeze_args(func: Callable) -> Callable: - """ - Transform mutable dictionary into immutable. Useful to be compatible with cache - Code taken from `this post `_ - """ - - @wraps(func) - def wrapped(*args, **kwargs) -> Callable: - args = tuple([frozendict(arg) if isinstance(arg, dict) else arg for arg in args]) - kwargs = {k: frozendict(v) if isinstance(v, dict) else v for k, v in kwargs.items()} - return func(*args, **kwargs) - return wrapped - - -# Method adopted from Triet's implementation -# https://github.com/OCR-D/core/pull/884/files#diff-8b69cb85b5ffcfb93a053791dec62a2f909a0669ae33d8a2412f246c3b01f1a3R260 -@freeze_args -@lru_cache(maxsize=32) -def get_processor(parameter: dict, processor_class: type) -> Union[type, None]: - """ - Call this function to get back an instance of a processor. The results are cached based on the parameters. - Args: - parameter (dict): a dictionary of parameters. - processor_class: the concrete `:py:class:~ocrd.Processor` class. - Returns: - When the concrete class of the processor is unknown, `None` is returned. Otherwise, an instance of the - `:py:class:~ocrd.Processor` is returned. - """ - if processor_class: - dict_params = dict(parameter) if parameter else None - return processor_class(workspace=None, parameter=dict_params) - return None +from ocrd_utils import getLogger def create_ssh_client(address: str, username: str, password: Union[str, None], diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index 98bc18c26..a9e5aa0f2 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -7,66 +7,99 @@ # is a single OCR-D Processor instance. import re -from ocrd_utils import ( - getLogger -) -from typing import Callable -from ocrd.network.deployment_utils import CustomDockerClient +from typing import Callable, Union +from frozendict import frozendict +from functools import lru_cache, wraps from paramiko import SSHClient +from ocrd_utils import getLogger +from ocrd.network.deployment_utils import CustomDockerClient + +from ocrd.network.rabbitmq_utils import RMQConsumer + class ProcessingWorker: def __init__(self, processor_name: str, processor_arguments: dict, - queue_address: str, database_address: str) -> None: + rmq_host: str, rmq_port: int, rmq_vhost: str, db_url: str) -> None: self.log = getLogger(__name__) + # ocr-d processor instance to be started self.processor_name = processor_name - # Required arguments to run the OCR-D Processor - self.processor_arguments = processor_arguments # processor.name is - # RabbitMQ Address - This contains at least the - # host name, port, and the virtual host - self.rmq_address = queue_address - self.mongodb_address = database_address - - # RMQConsumer object must be created here, reference: RabbitMQ Library (WebAPI Implementation) - # Based on the API calls the ProcessingWorker will receive messages from the running instance - # of the RabbitMQ Server (deployed by the Processing Broker) through the RMQConsumer object. - self.rmq_consumer = self.configure_consumer( - config_file="", - callback_method=self.on_consumed_message - ) - - # TODO: change typehint for return if class is finally part of core(ocrd_network) - @staticmethod - def configure_consumer(config_file: str, callback_method: Callable) -> 'RMQConsumer': - rmq_consumer = 'RMQConsumer Object' + # other potential parameters to be used + self.processor_arguments = processor_arguments + + self.db_url = db_url + + self.rmq_host = rmq_host + self.rmq_port = rmq_port + self.rmq_vhost = rmq_vhost + + # These could also be made configurable, + # not relevant for the current state + self.rmq_username = "default-consumer" + self.rmq_password = "default-consumer" + + self.rmq_consumer = self.connect_consumer() + + # Method adopted from Triet's implementation + # https://github.com/OCR-D/core/pull/884/files#diff-8b69cb85b5ffcfb93a053791dec62a2f909a0669ae33d8a2412f246c3b01f1a3R260 + def freeze_args(func: Callable) -> Callable: + """ + Transform mutable dictionary into immutable. Useful to be compatible with cache + Code taken from `this post `_ """ - Here is a template implementation to be adopted later - rmq_consumer = RMQConsumer(host='localhost', port=5672, vhost='/') - # The credentials are configured inside definitions.json - # when building the RabbitMQ docker image - rmq_consumer.authenticate_and_connect( - username='default-consumer', - password='default-consumer' - ) + @wraps(func) + def wrapped(*args, **kwargs) -> Callable: + args = tuple([frozendict(arg) if isinstance(arg, dict) else arg for arg in args]) + kwargs = {k: frozendict(v) if isinstance(v, dict) else v for k, v in kwargs.items()} + return func(*args, **kwargs) - #Note: The queue name here is the processor.name by definition - rmq_consumer.configure_consuming(queue_name='queue_name', callback_method=funcPtr) + return wrapped + # Method adopted from Triet's implementation + # https://github.com/OCR-D/core/pull/884/files#diff-8b69cb85b5ffcfb93a053791dec62a2f909a0669ae33d8a2412f246c3b01f1a3R260 + @freeze_args + @lru_cache(maxsize=32) + def get_processor(parameter: dict, processor_class: type) -> Union[type, None]: + """ + Call this function to get back an instance of a processor. The results are cached based on the parameters. + Args: + parameter (dict): a dictionary of parameters. + processor_class: the concrete `:py:class:~ocrd.Processor` class. + Returns: + When the concrete class of the processor is unknown, `None` is returned. Otherwise, an instance of the + `:py:class:~ocrd.Processor` is returned. """ + if processor_class: + dict_params = dict(parameter) if parameter else None + return processor_class(workspace=None, parameter=dict_params) + return None + + def connect_consumer(self) -> RMQConsumer: + rmq_consumer = RMQConsumer(host=self.rmq_host, port=self.rmq_port, vhost=self.rmq_vhost) + rmq_consumer.authenticate_and_connect(username=self.rmq_username, password=self.rmq_password) return rmq_consumer # Define what happens every time a message is consumed from the queue def on_consumed_message(self) -> None: + # TODO: Get the OCR-D processor instance back from the memory cache + # self.get_processor(...) pass - # A separate thread must be created here to listen - # to the queue since this is a blocking action def start_consuming(self) -> None: - # Blocks here and listens for messages coming from the specified queue - # self.rmq_consumer.start_consuming() - pass + if self.rmq_consumer: + self.rmq_consumer.configure_consuming( + queue_name=self.processor_name, + callback_method=self.on_consumed_message + ) + # TODO: A separate thread must be created here to listen + # to the queue since this is a blocking action + self.rmq_consumer.start_consuming() + else: + raise Exception("The RMQ Consumer is not connected/configured properly") + + # TODO: queue_address and _database_address are prefixed with underscore because they are not # needed yet (otherwise flak8 complains). But they will be needed once the real From 51690ee88d47ab50040b5490c9a41cb3756d0e7b Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Thu, 12 Jan 2023 15:47:57 +0100 Subject: [PATCH 043/226] Use actual RMQPublisher inside the processing_broker --- ocrd/ocrd/network/processing_broker.py | 53 ++++++++++++-------------- 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/ocrd/ocrd/network/processing_broker.py b/ocrd/ocrd/network/processing_broker.py index 3cb91f489..1f59c6263 100644 --- a/ocrd/ocrd/network/processing_broker.py +++ b/ocrd/ocrd/network/processing_broker.py @@ -1,13 +1,16 @@ import uvicorn from fastapi import FastAPI -from ocrd.network.deployer import Deployer -from ocrd_utils import getLogger import yaml from jsonschema import validate, ValidationError from ocrd_utils.package_resources import resource_string from typing import Union + +from ocrd_utils import getLogger from ocrd_validators import ProcessingBrokerValidator + +from ocrd.network.deployer import Deployer from ocrd.network.deployment_config import ProcessingBrokerConfig +from ocrd.network.rabbitmq_utils import RMQPublisher class ProcessingBroker(FastAPI): @@ -26,10 +29,18 @@ def __init__(self, config_path: str, host: str, port: int) -> None: self.deployer.deploy_all() self.log = getLogger(__name__) - # RMQPublisher object must be created here, reference: RabbitMQ Library (WebAPI Implementation) - # Based on the API calls the ProcessingBroker will send messages to the running instance - # of the RabbitMQ Server (deployed by the Deployer object) through the RMQPublisher object. - self.rmq_publisher = self.configure_publisher(self.config) + # RabbitMQ related fields, hard coded initially + self.rmq_host = "localhost" + self.rmq_port = 5672 + self.rmq_vhost = "/" + + # These could also be made configurable, + # not relevant for the current state + self.rmq_username = "default-publisher" + self.rmq_password = "default-publisher" + + self.rmq_publisher = self.connect_publisher() + self.rmq_publisher.enable_delivery_confirmations() # Enable acks self.router.add_api_route( path='/stop', @@ -39,13 +50,11 @@ def __init__(self, config_path: str, host: str, port: int) -> None: # summary='TODO: summary for apidesc', # TODO: add response model? add a response body at all? ) - # TODO: - # Publish messages based on the API calls - # Here is a call example to be adopted later - # - # # The message type is bytes - # # Call this method to publish a message - # self.rmq_publisher.publish_to_queue(queue_name='queue_name', message='message') + + # TODO: Call this after the rest of the API is implemented + # Example of publishing a message inside a specific queue + # The message type is bytes + # self.rmq_publisher.publish_to_queue(queue_name="queue_name", message="message") def start(self) -> None: """ @@ -83,19 +92,7 @@ async def on_shutdown(self) -> None: async def stop_deployed_agents(self) -> None: self.deployer.kill_all() - # TODO: add correct typehint if RMQPublisher is available here in core - @staticmethod - def configure_publisher(config_file: ProcessingBrokerConfig) -> 'RMQPublisher': - rmq_publisher = 'RMQPublisher Object' - # TODO: - # Here is a template implementation to be adopted later - # - # rmq_publisher = RMQPublisher(host='localhost', port=5672, vhost='/') - # # The credentials are configured inside definitions.json - # # when building the RabbitMQ docker image - # rmq_publisher.authenticate_and_connect( - # username='default-publisher', - # password='default-publisher' - # ) - # rmq_publisher.enable_delivery_confirmations() + def connect_publisher(self) -> RMQPublisher: + rmq_publisher = RMQPublisher(host=self.rmq_host, port=self.rmq_port, vhost=self.rmq_vhost) + rmq_publisher.authenticate_and_connect(username=self.rmq_username, password=self.rmq_password) return rmq_publisher From af5c0b4b4ddb6ba20b597e6cc2e1e1655c7e6a44 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Thu, 12 Jan 2023 16:12:49 +0100 Subject: [PATCH 044/226] Refactor and optimize imports --- ocrd/ocrd/network/deployer.py | 11 +++++------ ocrd/ocrd/network/deployment_config.py | 3 ++- ocrd/ocrd/network/deployment_utils.py | 20 +++++++++++--------- ocrd/ocrd/network/processing_broker.py | 9 +++------ ocrd/ocrd/network/processing_worker.py | 8 ++++---- 5 files changed, 25 insertions(+), 26 deletions(-) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index 0fd2e5ee5..64b58f972 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -1,16 +1,15 @@ from __future__ import annotations -from enum import Enum -from ocrd_utils import ( - getLogger -) +from typing import List, Dict, Union + +from ocrd_utils import getLogger + +from ocrd.network.deployment_config import * from ocrd.network.deployment_utils import ( create_docker_client, create_ssh_client, DeployType, ) -from ocrd.network.deployment_config import * from ocrd.network.processing_worker import ProcessingWorker -from typing import List, Dict, Union # Abstraction of the Deployment functionality diff --git a/ocrd/ocrd/network/deployment_config.py b/ocrd/ocrd/network/deployment_config.py index 6ad73a0be..488b8607f 100644 --- a/ocrd/ocrd/network/deployment_config.py +++ b/ocrd/ocrd/network/deployment_config.py @@ -1,8 +1,9 @@ # TODO: this probably breaks python 3.6. Think about whether we really want to use this from __future__ import annotations -from ocrd.network.deployment_utils import DeployType from typing import List, Dict +from ocrd.network.deployment_utils import DeployType + __all__ = [ 'ProcessingBrokerConfig', 'HostConfig', diff --git a/ocrd/ocrd/network/deployment_utils.py b/ocrd/ocrd/network/deployment_utils.py index 73c11704e..0922353fb 100644 --- a/ocrd/ocrd/network/deployment_utils.py +++ b/ocrd/ocrd/network/deployment_utils.py @@ -2,18 +2,18 @@ from enum import Enum from typing import Union -import docker +from docker import APIClient, DockerClient from docker.transport import SSHHTTPAdapter -import paramiko +from paramiko import AutoAddPolicy, SSHClient import urllib.parse from ocrd_utils import getLogger def create_ssh_client(address: str, username: str, password: Union[str, None], - keypath: Union[str, None]) -> paramiko.SSHClient: - client = paramiko.SSHClient() - client.set_missing_host_key_policy(paramiko.AutoAddPolicy) + keypath: Union[str, None]) -> SSHClient: + client = SSHClient() + client.set_missing_host_key_policy(AutoAddPolicy) log = getLogger(__name__) log.debug(f'creating ssh-client with username: "{username}", keypath: "{keypath}". ' f'host: {address}') @@ -28,7 +28,7 @@ def create_docker_client(address: str, username: str, password: Union[str, None] return CustomDockerClient(username, address, password=password, keypath=keypath) -class CustomDockerClient(docker.DockerClient): +class CustomDockerClient(DockerClient): """Wrapper for docker.DockerClient to use an own SshHttpAdapter. This makes it possible to use provided password/keyfile for connecting with @@ -42,11 +42,13 @@ class CustomDockerClient(docker.DockerClient): """ def __init__(self, user: str, host: str, **kwargs) -> None: + # TODO: Call to the super class __init__ is missing here, + # may this potentially become an issue? if not user or not host: raise ValueError("Missing argument: user and host must both be provided") if not 'password' in kwargs and not 'keypath' in kwargs: raise ValueError("Missing argument: one of password and keyfile is needed") - self.api = docker.APIClient(f'ssh://{host}', use_ssh_client=True, version='1.41') + self.api = APIClient(f'ssh://{host}', use_ssh_client=True, version='1.41') ssh_adapter = self.CustomSshHttpAdapter(f'ssh://{user}@{host}:22', **kwargs) self.api.mount('http+docker://ssh', ssh_adapter) @@ -64,7 +66,7 @@ def _create_paramiko_client(self, base_url: str) -> None: this method is called in the superclass constructor. Overwriting allows to set password/keypath for internal paramiko-client """ - self.ssh_client = paramiko.SSHClient() + self.ssh_client = SSHClient() parsed_base_url = urllib.parse.urlparse(base_url) self.ssh_params = { 'hostname': parsed_base_url.hostname, @@ -75,7 +77,7 @@ def _create_paramiko_client(self, base_url: str) -> None: self.ssh_params['password'] = self.password elif self.keypath: self.ssh_params['key_filename'] = self.keypath - self.ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy) + self.ssh_client.set_missing_host_key_policy(AutoAddPolicy) class DeployType(Enum): diff --git a/ocrd/ocrd/network/processing_broker.py b/ocrd/ocrd/network/processing_broker.py index 1f59c6263..4b8fa6752 100644 --- a/ocrd/ocrd/network/processing_broker.py +++ b/ocrd/ocrd/network/processing_broker.py @@ -1,9 +1,6 @@ -import uvicorn from fastapi import FastAPI -import yaml -from jsonschema import validate, ValidationError -from ocrd_utils.package_resources import resource_string -from typing import Union +import uvicorn +from yaml import safe_load from ocrd_utils import getLogger from ocrd_validators import ProcessingBrokerValidator @@ -66,7 +63,7 @@ def start(self) -> None: @staticmethod def parse_config(config_path: str) -> ProcessingBrokerConfig: with open(config_path) as fin: - obj = yaml.safe_load(fin) + obj = safe_load(fin) report = ProcessingBrokerValidator.validate(obj) if not report.is_valid: raise Exception(f"Processing-Broker configuration file is invalid:\n{report.errors}") diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index a9e5aa0f2..a2101575f 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -6,15 +6,15 @@ # According to the current requirements, each ProcessingWorker # is a single OCR-D Processor instance. -import re -from typing import Callable, Union from frozendict import frozendict from functools import lru_cache, wraps from paramiko import SSHClient +from re import search as re_search +from typing import Callable, Union from ocrd_utils import getLogger -from ocrd.network.deployment_utils import CustomDockerClient +from ocrd.network.deployment_utils import CustomDockerClient from ocrd.network.rabbitmq_utils import RMQConsumer @@ -127,7 +127,7 @@ def start_native_processor(client: SSHClient, name: str, _queue_address: str, # Since the docker version returns PID, this should also return PID for consistency # TODO: mypy error: ignore or fix. Problem: re.search returns Optional (can be None, causes # error if try to call) - return re.search(r'xyz([0-9]+)xyz', output).group(1) + return re_search(r'xyz([0-9]+)xyz', output).group(1) # TODO: queue_address and _database_address are prefixed with underscore because they are not # needed yet (otherwise flak8 complains). But they will be needed once the real From fb213e8be33ef02cec813dbed7c86af360f5d3d9 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Thu, 12 Jan 2023 16:30:42 +0100 Subject: [PATCH 045/226] Fix imports in rabbitmq_utils, add pika as a requirement --- ocrd/ocrd/network/rabbitmq_utils/connector.py | 2 +- ocrd/ocrd/network/rabbitmq_utils/consumer.py | 4 ++-- ocrd/ocrd/network/rabbitmq_utils/publisher.py | 4 ++-- ocrd/requirements.txt | 6 ++++-- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/ocrd/ocrd/network/rabbitmq_utils/connector.py b/ocrd/ocrd/network/rabbitmq_utils/connector.py index c0f658571..6527519bb 100644 --- a/ocrd/ocrd/network/rabbitmq_utils/connector.py +++ b/ocrd/ocrd/network/rabbitmq_utils/connector.py @@ -12,7 +12,7 @@ PlainCredentials ) -from ocrd_webapi.rabbitmq.constants import ( +from ocrd.network.rabbitmq_utils.constants import ( DEFAULT_EXCHANGER_NAME, DEFAULT_EXCHANGER_TYPE, DEFAULT_QUEUE, diff --git a/ocrd/ocrd/network/rabbitmq_utils/consumer.py b/ocrd/ocrd/network/rabbitmq_utils/consumer.py index d734561e4..0453f71cd 100644 --- a/ocrd/ocrd/network/rabbitmq_utils/consumer.py +++ b/ocrd/ocrd/network/rabbitmq_utils/consumer.py @@ -12,7 +12,7 @@ PlainCredentials ) -from ocrd_webapi.rabbitmq.constants import ( +from ocrd.network.rabbitmq_utils.constants import ( DEFAULT_QUEUE, LOG_FORMAT, LOG_LEVEL, @@ -20,7 +20,7 @@ RABBIT_MQ_PORT as PORT, RABBIT_MQ_VHOST as VHOST ) -from ocrd_webapi.rabbitmq.connector import RMQConnector +from ocrd.network.rabbitmq_utils.connector import RMQConnector class RMQConsumer(RMQConnector): diff --git a/ocrd/ocrd/network/rabbitmq_utils/publisher.py b/ocrd/ocrd/network/rabbitmq_utils/publisher.py index 204498785..a69ee92fa 100644 --- a/ocrd/ocrd/network/rabbitmq_utils/publisher.py +++ b/ocrd/ocrd/network/rabbitmq_utils/publisher.py @@ -13,7 +13,7 @@ PlainCredentials ) -from ocrd_webapi.rabbitmq.constants import ( +from ocrd.network.rabbitmq_utils.constants import ( DEFAULT_EXCHANGER_NAME, DEFAULT_ROUTER, LOG_FORMAT, @@ -22,7 +22,7 @@ RABBIT_MQ_PORT as PORT, RABBIT_MQ_VHOST as VHOST ) -from ocrd_webapi.rabbitmq.connector import RMQConnector +from ocrd.network.rabbitmq_utils.connector import RMQConnector class RMQPublisher(RMQConnector): diff --git a/ocrd/requirements.txt b/ocrd/requirements.txt index f438876b5..6ea8f44c4 100644 --- a/ocrd/requirements.txt +++ b/ocrd/requirements.txt @@ -11,8 +11,10 @@ Deprecated == 1.2.0 memory-profiler >= 0.58.0 sparklines >= 0.4.2 python-magic -uvicorn -fastapi +uvicorn>=0.17.6 +fastapi>=0.78.0 docker paramiko frozendict~=2.3.4 +pika>=1.2.0 +tomli>=2.0.0 From 41defe3f4faa18f7145c0334a842345a1d336eeb Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Thu, 12 Jan 2023 16:38:52 +0100 Subject: [PATCH 046/226] Remove config file and tomli as a requirement --- ocrd/ocrd/network/rabbitmq_utils/config.toml | 9 -------- ocrd/ocrd/network/rabbitmq_utils/constants.py | 21 +++++++------------ ocrd/requirements.txt | 1 - 3 files changed, 7 insertions(+), 24 deletions(-) delete mode 100644 ocrd/ocrd/network/rabbitmq_utils/config.toml diff --git a/ocrd/ocrd/network/rabbitmq_utils/config.toml b/ocrd/ocrd/network/rabbitmq_utils/config.toml deleted file mode 100644 index 86b431b5c..000000000 --- a/ocrd/ocrd/network/rabbitmq_utils/config.toml +++ /dev/null @@ -1,9 +0,0 @@ -# "rabbit-mq-host" when Dockerized -rabbit_mq_host = "localhost" -rabbit_mq_port = 5672 -rabbit_mq_vhost = "/" - -default_exchange_name = "ocrd-webapi-default" -default_exchange_type = "direct" -default_queue = "ocrd-webapi-default" -default_router = "ocrd-webapi-default" diff --git a/ocrd/ocrd/network/rabbitmq_utils/constants.py b/ocrd/ocrd/network/rabbitmq_utils/constants.py index d3ec58ad2..05f6b7932 100644 --- a/ocrd/ocrd/network/rabbitmq_utils/constants.py +++ b/ocrd/ocrd/network/rabbitmq_utils/constants.py @@ -1,6 +1,4 @@ import logging -from pkg_resources import resource_filename -import tomli __all__ = [ "DEFAULT_EXCHANGER_NAME", @@ -17,20 +15,15 @@ "LOG_LEVEL" ] -TOML_FILENAME: str = resource_filename(__name__, 'config.toml') -TOML_FD = open(TOML_FILENAME, mode='rb') -TOML_CONFIG = tomli.load(TOML_FD) -TOML_FD.close() - -DEFAULT_EXCHANGER_NAME: str = TOML_CONFIG["default_exchange_name"] -DEFAULT_EXCHANGER_TYPE: str = TOML_CONFIG["default_exchange_type"] -DEFAULT_QUEUE: str = TOML_CONFIG["default_queue"] -DEFAULT_ROUTER: str = TOML_CONFIG["default_router"] +DEFAULT_EXCHANGER_NAME: str = "ocrd-webapi-default" +DEFAULT_EXCHANGER_TYPE: str = "direct" +DEFAULT_QUEUE: str = "ocrd-webapi-default" +DEFAULT_ROUTER: str = "ocrd-webapi-default" # "rabbit-mq-host" when Dockerized -RABBIT_MQ_HOST: str = TOML_CONFIG["rabbit_mq_host"] -RABBIT_MQ_PORT: int = TOML_CONFIG["rabbit_mq_port"] -RABBIT_MQ_VHOST: str = TOML_CONFIG["rabbit_mq_vhost"] +RABBIT_MQ_HOST: str = "localhost" +RABBIT_MQ_PORT: int = 5672 +RABBIT_MQ_VHOST: str = "/" # Wait seconds before next reconnect try RECONNECT_WAIT: int = 5 diff --git a/ocrd/requirements.txt b/ocrd/requirements.txt index 6ea8f44c4..ed2d22e64 100644 --- a/ocrd/requirements.txt +++ b/ocrd/requirements.txt @@ -17,4 +17,3 @@ docker paramiko frozendict~=2.3.4 pika>=1.2.0 -tomli>=2.0.0 From 03c0946ed9e6bb105868bc0522c392a02973ada3 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Thu, 12 Jan 2023 16:46:06 +0100 Subject: [PATCH 047/226] Refactor defaults in rabbitmq_utils webapi->network --- ocrd/ocrd/network/rabbitmq_utils/constants.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ocrd/ocrd/network/rabbitmq_utils/constants.py b/ocrd/ocrd/network/rabbitmq_utils/constants.py index 05f6b7932..0ba9334bf 100644 --- a/ocrd/ocrd/network/rabbitmq_utils/constants.py +++ b/ocrd/ocrd/network/rabbitmq_utils/constants.py @@ -15,10 +15,10 @@ "LOG_LEVEL" ] -DEFAULT_EXCHANGER_NAME: str = "ocrd-webapi-default" +DEFAULT_EXCHANGER_NAME: str = "ocrd-network-default" DEFAULT_EXCHANGER_TYPE: str = "direct" -DEFAULT_QUEUE: str = "ocrd-webapi-default" -DEFAULT_ROUTER: str = "ocrd-webapi-default" +DEFAULT_QUEUE: str = "ocrd-network-default" +DEFAULT_ROUTER: str = "ocrd-network-default" # "rabbit-mq-host" when Dockerized RABBIT_MQ_HOST: str = "localhost" From ca1b1b92efa06d922405b530d19400acef260571 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Thu, 12 Jan 2023 17:08:11 +0100 Subject: [PATCH 048/226] Comment out RabbitMQ publisher/consumer --- ocrd/ocrd/cli/processing_worker.py | 2 +- ocrd/ocrd/network/processing_broker.py | 4 ++-- ocrd/ocrd/network/processing_worker.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ocrd/ocrd/cli/processing_worker.py b/ocrd/ocrd/cli/processing_worker.py index 5bd0fbb07..44610abaf 100644 --- a/ocrd/ocrd/cli/processing_worker.py +++ b/ocrd/ocrd/cli/processing_worker.py @@ -53,4 +53,4 @@ def processing_worker_cli(processor_name: str, queue: str, database: str): # TODO: Load the OCR-D processor in the memory cache # Start consuming with the configuration settings above - processing_worker.start_consuming() + # processing_worker.start_consuming() diff --git a/ocrd/ocrd/network/processing_broker.py b/ocrd/ocrd/network/processing_broker.py index 4b8fa6752..2f6aabf65 100644 --- a/ocrd/ocrd/network/processing_broker.py +++ b/ocrd/ocrd/network/processing_broker.py @@ -36,8 +36,8 @@ def __init__(self, config_path: str, host: str, port: int) -> None: self.rmq_username = "default-publisher" self.rmq_password = "default-publisher" - self.rmq_publisher = self.connect_publisher() - self.rmq_publisher.enable_delivery_confirmations() # Enable acks + # self.rmq_publisher = self.connect_publisher() + # self.rmq_publisher.enable_delivery_confirmations() # Enable acks self.router.add_api_route( path='/stop', diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index a2101575f..7e8c20a31 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -39,7 +39,7 @@ def __init__(self, processor_name: str, processor_arguments: dict, self.rmq_username = "default-consumer" self.rmq_password = "default-consumer" - self.rmq_consumer = self.connect_consumer() + # self.rmq_consumer = self.connect_consumer() # Method adopted from Triet's implementation # https://github.com/OCR-D/core/pull/884/files#diff-8b69cb85b5ffcfb93a053791dec62a2f909a0669ae33d8a2412f246c3b01f1a3R260 From 49089c7c1baad2a3b7df4fd3db8d6f81fbdd6ce0 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Thu, 12 Jan 2023 17:28:38 +0100 Subject: [PATCH 049/226] Improve deploy/kill order --- ocrd/ocrd/network/deployer.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index 64b58f972..72e9efe1f 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -46,6 +46,8 @@ def __init__(self, config: ProcessingBrokerConfig) -> None: def deploy_all(self) -> None: """ Deploy the message queue and all processors defined in the config-file """ + # The order of deploying may be important to recover from previous state + # Ideally, this should return the address of the RabbitMQ Server rabbitmq_address = self._deploy_rabbitmq() # Ideally, this should return the address of the MongoDB @@ -53,10 +55,21 @@ def deploy_all(self) -> None: self._deploy_processing_workers(self.hosts, rabbitmq_address, mongodb_address) def kill_all(self) -> None: - self._kill_rabbitmq() - self._kill_mongodb() + # The order of killing is important to optimize graceful shutdown in the future + # If RabbitMQ server is killed before killing Processing Workers, that may have + # bad outcome and leave Processing Workers in an unpredictable state + + # First kill the active Processing Workers + # They may still want to update something in the db before closing + # They may still want to nack the currently processed messages back to the RabbitMQ Server self._kill_processing_workers() + # Second kill the MongoDB + self._kill_mongodb() + + # Third kill the RabbitMQ Server + self._kill_rabbitmq() + def _deploy_processing_workers(self, hosts: List[HostConfig], rabbitmq_address: str, mongodb_address: str) -> None: for host in hosts: From 346bb2c04dccc241f0c99ddef4cf2265ff4ff08f Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Thu, 12 Jan 2023 18:07:52 +0100 Subject: [PATCH 050/226] Split configurations for better readability --- ocrd/ocrd/network/deployer.py | 21 +++++++++++---------- ocrd/ocrd/network/processing_broker.py | 13 +++++++++---- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index 72e9efe1f..364380f34 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -11,12 +11,11 @@ ) from ocrd.network.processing_worker import ProcessingWorker - # Abstraction of the Deployment functionality -# The Deployer agent is in the middle between -# the ProcessingBroker agent and the ProcessingWorker agents -# ProcessingBroker provides the configuration file to the Deployer agent -# The Deployer agent creates the ProcessingWorker agents. +# The ProcessingServer (currently still called Broker) provides the configuration parameters to the Deployer agent. +# The Deployer agent deploys the RabbitMQ Server, MongoDB and the Processing Hosts. +# Each Processing Host may have several Processing Workers. +# Each Processing Worker is an instance of an OCR-D processor. # TODO: # Ideally, the interaction among the agents should happen through @@ -32,16 +31,18 @@ class Deployer: for managing information, not for actually doing things. """ - def __init__(self, config: ProcessingBrokerConfig) -> None: + def __init__(self, queue_config: QueueConfig, mongo_config: MongoConfig, hosts_config: List[HostConfig]) -> None: """ Args: - config (Config): values from config file wrapped into class `Config` + queue_config: RabbitMQ related configuration + mongo_config: MongoDB related configuration + hosts_config: Processing Hosts related configurations """ self.log = getLogger(__name__) self.log.debug('Deployer-init()') - self.mongo_data = config.mongo_config - self.mq_data = config.queue_config - self.hosts = config.hosts_config + self.mongo_data = mongo_config + self.mq_data = queue_config + self.hosts = hosts_config def deploy_all(self) -> None: """ Deploy the message queue and all processors defined in the config-file diff --git a/ocrd/ocrd/network/processing_broker.py b/ocrd/ocrd/network/processing_broker.py index 2f6aabf65..47f224fac 100644 --- a/ocrd/ocrd/network/processing_broker.py +++ b/ocrd/ocrd/network/processing_broker.py @@ -18,13 +18,16 @@ class ProcessingBroker(FastAPI): def __init__(self, config_path: str, host: str, port: int) -> None: # TODO: set other args: title, description, version, openapi_tags super().__init__(on_shutdown=[self.on_shutdown]) + self.log = getLogger(__name__) + self.hostname = host self.port = port self.config = ProcessingBroker.parse_config(config_path) - self.deployer = Deployer(self.config) - # Deploy everything specified in the configuration - self.deployer.deploy_all() - self.log = getLogger(__name__) + self.deployer = Deployer( + queue_config=self.config.queue_config, + mongo_config=self.config.mongo_config, + hosts_config=self.config.hosts_config + ) # RabbitMQ related fields, hard coded initially self.rmq_host = "localhost" @@ -57,6 +60,8 @@ def start(self) -> None: """ start processing broker with uvicorn """ + # Deploy everything specified in the configuration + self.deployer.deploy_all() self.log.debug(f'starting uvicorn. Host: {self.host}. Port: {self.port}') uvicorn.run(self, host=self.hostname, port=self.port) From 1a290a2b7d50bba2a55277223281005ab31ca2ba Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Fri, 13 Jan 2023 10:40:54 +0100 Subject: [PATCH 051/226] Remove typing which was auto added by the IDE --- ocrd/ocrd/network/processing_worker.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index 7e8c20a31..ede76dea6 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -43,7 +43,7 @@ def __init__(self, processor_name: str, processor_arguments: dict, # Method adopted from Triet's implementation # https://github.com/OCR-D/core/pull/884/files#diff-8b69cb85b5ffcfb93a053791dec62a2f909a0669ae33d8a2412f246c3b01f1a3R260 - def freeze_args(func: Callable) -> Callable: + def freeze_args(func): """ Transform mutable dictionary into immutable. Useful to be compatible with cache Code taken from `this post `_ @@ -61,7 +61,7 @@ def wrapped(*args, **kwargs) -> Callable: # https://github.com/OCR-D/core/pull/884/files#diff-8b69cb85b5ffcfb93a053791dec62a2f909a0669ae33d8a2412f246c3b01f1a3R260 @freeze_args @lru_cache(maxsize=32) - def get_processor(parameter: dict, processor_class: type) -> Union[type, None]: + def get_processor(parameter: dict, processor_class=None): """ Call this function to get back an instance of a processor. The results are cached based on the parameters. Args: @@ -83,7 +83,7 @@ def connect_consumer(self) -> RMQConsumer: # Define what happens every time a message is consumed from the queue def on_consumed_message(self) -> None: - # TODO: Get the OCR-D processor instance back from the memory cache + # TODO: Start the OCR-D processor or get from the cache # self.get_processor(...) pass From 1453b7422f48681705ebeb4e096305be7a6768a6 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Fri, 13 Jan 2023 14:21:15 +0100 Subject: [PATCH 052/226] change scope of get_processor --- ocrd/ocrd/network/processing_worker.py | 76 +++++++++++++------------- 1 file changed, 37 insertions(+), 39 deletions(-) diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index ede76dea6..b20fc8300 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -10,7 +10,6 @@ from functools import lru_cache, wraps from paramiko import SSHClient from re import search as re_search -from typing import Callable, Union from ocrd_utils import getLogger @@ -41,41 +40,6 @@ def __init__(self, processor_name: str, processor_arguments: dict, # self.rmq_consumer = self.connect_consumer() - # Method adopted from Triet's implementation - # https://github.com/OCR-D/core/pull/884/files#diff-8b69cb85b5ffcfb93a053791dec62a2f909a0669ae33d8a2412f246c3b01f1a3R260 - def freeze_args(func): - """ - Transform mutable dictionary into immutable. Useful to be compatible with cache - Code taken from `this post `_ - """ - - @wraps(func) - def wrapped(*args, **kwargs) -> Callable: - args = tuple([frozendict(arg) if isinstance(arg, dict) else arg for arg in args]) - kwargs = {k: frozendict(v) if isinstance(v, dict) else v for k, v in kwargs.items()} - return func(*args, **kwargs) - - return wrapped - - # Method adopted from Triet's implementation - # https://github.com/OCR-D/core/pull/884/files#diff-8b69cb85b5ffcfb93a053791dec62a2f909a0669ae33d8a2412f246c3b01f1a3R260 - @freeze_args - @lru_cache(maxsize=32) - def get_processor(parameter: dict, processor_class=None): - """ - Call this function to get back an instance of a processor. The results are cached based on the parameters. - Args: - parameter (dict): a dictionary of parameters. - processor_class: the concrete `:py:class:~ocrd.Processor` class. - Returns: - When the concrete class of the processor is unknown, `None` is returned. Otherwise, an instance of the - `:py:class:~ocrd.Processor` is returned. - """ - if processor_class: - dict_params = dict(parameter) if parameter else None - return processor_class(workspace=None, parameter=dict_params) - return None - def connect_consumer(self) -> RMQConsumer: rmq_consumer = RMQConsumer(host=self.rmq_host, port=self.rmq_port, vhost=self.rmq_vhost) rmq_consumer.authenticate_and_connect(username=self.rmq_username, password=self.rmq_password) @@ -84,7 +48,7 @@ def connect_consumer(self) -> RMQConsumer: # Define what happens every time a message is consumed from the queue def on_consumed_message(self) -> None: # TODO: Start the OCR-D processor or get from the cache - # self.get_processor(...) + # get_processor(...) pass def start_consuming(self) -> None: @@ -99,8 +63,6 @@ def start_consuming(self) -> None: else: raise Exception("The RMQ Consumer is not connected/configured properly") - - # TODO: queue_address and _database_address are prefixed with underscore because they are not # needed yet (otherwise flak8 complains). But they will be needed once the real # processing_worker is called here. Then they should be renamed @@ -141,3 +103,39 @@ def start_docker_processor(client: CustomDockerClient, name: str, _queue_address res = client.containers.run('debian', 'sleep 31', detach=True, remove=True) assert res and res.id, 'run processor in docker-container failed' return res.id + + +# Method adopted from Triet's implementation +# https://github.com/OCR-D/core/pull/884/files#diff-8b69cb85b5ffcfb93a053791dec62a2f909a0669ae33d8a2412f246c3b01f1a3R260 +def freeze_args(func): + """ + Transform mutable dictionary into immutable. Useful to be compatible with cache + Code taken from `this post `_ + """ + + @wraps(func) + def wrapped(*args, **kwargs): + args = tuple([frozendict(arg) if isinstance(arg, dict) else arg for arg in args]) + kwargs = {k: frozendict(v) if isinstance(v, dict) else v for k, v in kwargs.items()} + return func(*args, **kwargs) + return wrapped + + +# Method adopted from Triet's implementation +# https://github.com/OCR-D/core/pull/884/files#diff-8b69cb85b5ffcfb93a053791dec62a2f909a0669ae33d8a2412f246c3b01f1a3R260 +@freeze_args +@lru_cache(maxsize=32) +def get_processor(parameter: dict, processor_class=None): + """ + Call this function to get back an instance of a processor. The results are cached based on the parameters. + Args: + parameter (dict): a dictionary of parameters. + processor_class: the concrete `:py:class:~ocrd.Processor` class. + Returns: + When the concrete class of the processor is unknown, `None` is returned. Otherwise, an instance of the + `:py:class:~ocrd.Processor` is returned. + """ + if processor_class: + dict_params = dict(parameter) if parameter else None + return processor_class(workspace=None, parameter=dict_params) + return None From 2933a06e809ba29fe9e0f5d6bd62d34b53aa1e88 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Fri, 13 Jan 2023 14:59:21 +0100 Subject: [PATCH 053/226] Move methods from Processing Worker to Deployer --- ocrd/ocrd/cli/processing_worker.py | 3 +- ocrd/ocrd/network/deployer.py | 45 ++++++++++++++++++++++++++ ocrd/ocrd/network/processing_worker.py | 44 ------------------------- 3 files changed, 47 insertions(+), 45 deletions(-) diff --git a/ocrd/ocrd/cli/processing_worker.py b/ocrd/ocrd/cli/processing_worker.py index 44610abaf..060a316fd 100644 --- a/ocrd/ocrd/cli/processing_worker.py +++ b/ocrd/ocrd/cli/processing_worker.py @@ -50,7 +50,8 @@ def processing_worker_cli(processor_name: str, queue: str, database: str): db_url=db_url ) - # TODO: Load the OCR-D processor in the memory cache + # TODO: Remove. It's just to test starting the OCR-D processor + processing_worker.on_consumed_message() # Start consuming with the configuration settings above # processing_worker.start_consuming() diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index 364380f34..fb43270e7 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -1,5 +1,7 @@ from __future__ import annotations from typing import List, Dict, Union +from paramiko import SSHClient +from re import search as re_search from ocrd_utils import getLogger @@ -7,6 +9,7 @@ from ocrd.network.deployment_utils import ( create_docker_client, create_ssh_client, + CustomDockerClient, DeployType, ) from ocrd.network.processing_worker import ProcessingWorker @@ -244,3 +247,45 @@ def _kill_processing_worker(self, host: HostConfig) -> None: self.log.error(f"Deploy type of {processor.name} is neither of the allowed types") pass processor.pids = [] + + + # TODO: queue_address and _database_address are prefixed with underscore because they are not + # needed yet (otherwise flak8 complains). But they will be needed once the real + # processing_worker is called here. Then they should be renamed + @staticmethod + def start_native_processor(client: SSHClient, name: str, _queue_address: str, + _database_address: str) -> str: + log = getLogger(__name__) + log.debug(f'start native processor: {name}') + channel = client.invoke_shell() + stdin, stdout = channel.makefile('wb'), channel.makefile('rb') + # TODO: add real command here to start processing server here + cmd = 'sleep 23s' + # the only way to make it work to start a process in the background and return early is + # this construction. The pid of the last started background process is printed with + # `echo $!` but it is printed inbetween other output. Because of that I added `xyz` before + # and after the code to easily be able to filter out the pid via regex when returning from + # the function + stdin.write(f'{cmd} & \n echo xyz$!xyz \n exit \n') + output = stdout.read().decode('utf-8') + stdout.close() + stdin.close() + # What does this return and is supposed to return? + # Putting some comments when using patterns is always appreciated + # Since the docker version returns PID, this should also return PID for consistency + # TODO: mypy error: ignore or fix. Problem: re.search returns Optional (can be None, causes + # error if try to call) + return re_search(r'xyz([0-9]+)xyz', output).group(1) + + # TODO: queue_address and _database_address are prefixed with underscore because they are not + # needed yet (otherwise flak8 complains). But they will be needed once the real + # processing_worker is called here. Then they should be renamed + @staticmethod + def start_docker_processor(client: CustomDockerClient, name: str, _queue_address: str, + _database_address: str) -> str: + log = getLogger(__name__) + log.debug(f'start docker processor: {name}') + # TODO: add real command here to start processing server here + res = client.containers.run('debian', 'sleep 31', detach=True, remove=True) + assert res and res.id, 'run processor in docker-container failed' + return res.id diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index b20fc8300..bf94bdf31 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -8,12 +8,9 @@ from frozendict import frozendict from functools import lru_cache, wraps -from paramiko import SSHClient -from re import search as re_search from ocrd_utils import getLogger -from ocrd.network.deployment_utils import CustomDockerClient from ocrd.network.rabbitmq_utils import RMQConsumer @@ -63,47 +60,6 @@ def start_consuming(self) -> None: else: raise Exception("The RMQ Consumer is not connected/configured properly") - # TODO: queue_address and _database_address are prefixed with underscore because they are not - # needed yet (otherwise flak8 complains). But they will be needed once the real - # processing_worker is called here. Then they should be renamed - @staticmethod - def start_native_processor(client: SSHClient, name: str, _queue_address: str, - _database_address: str) -> str: - log = getLogger(__name__) - log.debug(f'start native processor: {name}') - channel = client.invoke_shell() - stdin, stdout = channel.makefile('wb'), channel.makefile('rb') - # TODO: add real command here to start processing server here - cmd = 'sleep 23s' - # the only way to make it work to start a process in the background and return early is - # this construction. The pid of the last started background process is printed with - # `echo $!` but it is printed inbetween other output. Because of that I added `xyz` before - # and after the code to easily be able to filter out the pid via regex when returning from - # the function - stdin.write(f'{cmd} & \n echo xyz$!xyz \n exit \n') - output = stdout.read().decode('utf-8') - stdout.close() - stdin.close() - # What does this return and is supposed to return? - # Putting some comments when using patterns is always appreciated - # Since the docker version returns PID, this should also return PID for consistency - # TODO: mypy error: ignore or fix. Problem: re.search returns Optional (can be None, causes - # error if try to call) - return re_search(r'xyz([0-9]+)xyz', output).group(1) - - # TODO: queue_address and _database_address are prefixed with underscore because they are not - # needed yet (otherwise flak8 complains). But they will be needed once the real - # processing_worker is called here. Then they should be renamed - @staticmethod - def start_docker_processor(client: CustomDockerClient, name: str, _queue_address: str, - _database_address: str) -> str: - log = getLogger(__name__) - log.debug(f'start docker processor: {name}') - # TODO: add real command here to start processing server here - res = client.containers.run('debian', 'sleep 31', detach=True, remove=True) - assert res and res.id, 'run processor in docker-container failed' - return res.id - # Method adopted from Triet's implementation # https://github.com/OCR-D/core/pull/884/files#diff-8b69cb85b5ffcfb93a053791dec62a2f909a0669ae33d8a2412f246c3b01f1a3R260 From 33003916080afc3cdf930078a9d1ed2c0b5d1472 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Fri, 13 Jan 2023 15:00:08 +0100 Subject: [PATCH 054/226] Refactor name -> processor_name --- ocrd/ocrd/network/deployer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index fb43270e7..38d57dcda 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -253,10 +253,10 @@ def _kill_processing_worker(self, host: HostConfig) -> None: # needed yet (otherwise flak8 complains). But they will be needed once the real # processing_worker is called here. Then they should be renamed @staticmethod - def start_native_processor(client: SSHClient, name: str, _queue_address: str, + def start_native_processor(client: SSHClient, processor_name: str, _queue_address: str, _database_address: str) -> str: log = getLogger(__name__) - log.debug(f'start native processor: {name}') + log.debug(f'start native processor: {processor_name}') channel = client.invoke_shell() stdin, stdout = channel.makefile('wb'), channel.makefile('rb') # TODO: add real command here to start processing server here @@ -281,10 +281,10 @@ def start_native_processor(client: SSHClient, name: str, _queue_address: str, # needed yet (otherwise flak8 complains). But they will be needed once the real # processing_worker is called here. Then they should be renamed @staticmethod - def start_docker_processor(client: CustomDockerClient, name: str, _queue_address: str, + def start_docker_processor(client: CustomDockerClient, processor_name: str, _queue_address: str, _database_address: str) -> str: log = getLogger(__name__) - log.debug(f'start docker processor: {name}') + log.debug(f'start docker processor: {processor_name}') # TODO: add real command here to start processing server here res = client.containers.run('debian', 'sleep 31', detach=True, remove=True) assert res and res.id, 'run processor in docker-container failed' From 9a7641bb18db2b564c09463b67f9a82c305ba8d5 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Fri, 13 Jan 2023 15:02:39 +0100 Subject: [PATCH 055/226] Finish transfering of methods from Worker to Deployer --- ocrd/ocrd/network/deployer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index 38d57dcda..3a4798aac 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -109,17 +109,17 @@ def _deploy_processing_worker(self, processor: ProcessorConfig, host: HostConfig for _ in range(processor.count): if processor.deploy_type == DeployType.native: assert host.ssh_client # to satisfy mypy - pid = ProcessingWorker.start_native_processor( + pid = self.start_native_processor( client=host.ssh_client, - name=processor.name, + processor_name=processor.name, _queue_address=rabbitmq_server, _database_address=mongodb) processor.add_started_pid(pid) elif processor.deploy_type == DeployType.docker: assert host.docker_client # to satisfy mypy - pid = ProcessingWorker.start_docker_processor( + pid = self.start_docker_processor( client=host.docker_client, - name=processor.name, + processor_name=processor.name, _queue_address=rabbitmq_server, _database_address=mongodb) processor.add_started_pid(pid) From a059f1c17c350ffdf9f10389d076ec9114c6b7c3 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Fri, 13 Jan 2023 18:12:18 +0100 Subject: [PATCH 056/226] Improve the template of processing worker --- ocrd/ocrd/cli/processing_worker.py | 16 ++++++++++++---- ocrd/ocrd/network/processing_worker.py | 26 +++++++++++++++++++------- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/ocrd/ocrd/cli/processing_worker.py b/ocrd/ocrd/cli/processing_worker.py index 060a316fd..b4212ec4a 100644 --- a/ocrd/ocrd/cli/processing_worker.py +++ b/ocrd/ocrd/cli/processing_worker.py @@ -6,7 +6,11 @@ :nested: full """ import click -from ocrd_utils import initLogging +from subprocess import run, PIPE +from ocrd_utils import ( + initLogging, + parse_json_string_with_comments +) from ocrd.network import ProcessingWorker @@ -24,9 +28,6 @@ def processing_worker_cli(processor_name: str, queue: str, database: str): """ initLogging() try: - # TODO: Check here if the provided `processor_name` exists - processor_name = "ocrd-dummy" - # TODO: Parse the actual RabbitMQ Server address - `queue` rmq_host = "localhost" rmq_port = 5672 @@ -41,9 +42,16 @@ def processing_worker_cli(processor_name: str, queue: str, database: str): except ValueError: raise click.UsageError('Wrong/Bad arguments format provided. Check the help sections') + ocrd_tool = parse_json_string_with_comments( + run([processor_name, '--dump-json'], stdout=PIPE, check=True, universal_newlines=True).stdout + ) + processing_worker = ProcessingWorker( processor_name=processor_name, processor_arguments={}, + # TODO: Send the proper processor_class. How? + processor_class="", + ocrd_tool=ocrd_tool, rmq_host=rmq_host, rmq_port=rmq_port, rmq_vhost=rmq_vhost, diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index bf94bdf31..a55379f98 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -10,22 +10,27 @@ from functools import lru_cache, wraps from ocrd_utils import getLogger - from ocrd.network.rabbitmq_utils import RMQConsumer class ProcessingWorker: - def __init__(self, processor_name: str, processor_arguments: dict, + def __init__(self, processor_name: str, processor_arguments: dict, processor_class, ocrd_tool: dict, rmq_host: str, rmq_port: int, rmq_vhost: str, db_url: str) -> None: - self.log = getLogger(__name__) # ocr-d processor instance to be started self.processor_name = processor_name # other potential parameters to be used self.processor_arguments = processor_arguments + # ocr-d processor object to be instantiated + self.processor_class = processor_class - self.db_url = db_url + # Instantiation of the self.processor_class + # Instantiated inside `on_consumed_message` + self.processor_instance = None + + self.ocrd_tool = ocrd_tool + self.db_url = db_url self.rmq_host = rmq_host self.rmq_port = rmq_port self.rmq_vhost = rmq_vhost @@ -44,9 +49,16 @@ def connect_consumer(self) -> RMQConsumer: # Define what happens every time a message is consumed from the queue def on_consumed_message(self) -> None: - # TODO: Start the OCR-D processor or get from the cache - # get_processor(...) - pass + # 1. Load the OCR-D processor in the memory cache on first message consumed + # 2. Load the OCR-D processor from the memory cache on every other message consumed + self.processor_instance = get_processor(self.processor_arguments, self.processor_class) + if self.processor_instance: + self.log.debug(f"Loading processor instance of `{self.processor_name}` succeeded.") + else: + self.log.debug(f"Loading processor instance of `{self.processor_name}` failed.") + + # TODO: Do the processing of the current message + # self.processor_instance.X(...) def start_consuming(self) -> None: if self.rmq_consumer: From 29c30e5b62e8cd4830bdf713de9398e034a8d6b0 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Mon, 16 Jan 2023 15:55:13 +0100 Subject: [PATCH 057/226] Import the job and tool models from prev implementation --- ocrd/ocrd/network/models/__init__.py | 0 ocrd/ocrd/network/models/job.py | 57 +++++++++++++++++++++++++++ ocrd/ocrd/network/models/ocrd_tool.py | 16 ++++++++ 3 files changed, 73 insertions(+) create mode 100644 ocrd/ocrd/network/models/__init__.py create mode 100644 ocrd/ocrd/network/models/job.py create mode 100644 ocrd/ocrd/network/models/ocrd_tool.py diff --git a/ocrd/ocrd/network/models/__init__.py b/ocrd/ocrd/network/models/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ocrd/ocrd/network/models/job.py b/ocrd/ocrd/network/models/job.py new file mode 100644 index 000000000..f34e217f7 --- /dev/null +++ b/ocrd/ocrd/network/models/job.py @@ -0,0 +1,57 @@ +# These are the models directly taken from the Triet's implementation: +# REST API wrapper for the processor #884 + +# TODO: In the OCR-D WebAPI implementation we did a clear separation between +# the business response models and the low level database models. In order to achieve +# better modularity, we should use the same approach in the network package as well. + + +from datetime import datetime +from enum import Enum +from typing import List, Optional + +from beanie import Document +from pydantic import BaseModel + + +class StateEnum(str, Enum): + queued = 'QUEUED' + running = 'RUNNING' + success = 'SUCCESS' # TODO: SUCCEEDED for consistency + failed = 'FAILED' + + +class JobInput(BaseModel): + path: str + description: Optional[str] = None + input_file_grps: List[str] + output_file_grps: Optional[List[str]] + page_id: Optional[str] = None + parameters: dict = None # Always set to an empty dict when it's None, otherwise it won't pass the ocrd validation + + class Config: + schema_extra = { + "example": { + "path": "/path/to/mets.xml", + "description": "The description of this execution", + "input_file_grps": ["INPUT_FILE_GROUP"], + "output_file_grps": ["OUTPUT_FILE_GROUP"], + "page_id": "PAGE_ID", + "parameters": {} + } + } + + +class Job(Document): + path: str + description: Optional[str] + state: StateEnum + input_file_grps: List[str] + output_file_grps: Optional[List[str]] + page_id: Optional[str] + parameters: Optional[dict] + start_time: Optional[datetime] + end_time: Optional[datetime] + + class Settings: + use_enum_values = True diff --git a/ocrd/ocrd/network/models/ocrd_tool.py b/ocrd/ocrd/network/models/ocrd_tool.py new file mode 100644 index 000000000..a60b547ab --- /dev/null +++ b/ocrd/ocrd/network/models/ocrd_tool.py @@ -0,0 +1,16 @@ +# This model is directly taken from the Triet's implementation: +# REST API wrapper for the processor #884 + +from typing import List, Optional + +from pydantic import BaseModel + + +class OcrdTool(BaseModel): + executable: str + categories: List[str] + description: str + input_file_grp: List[str] + output_file_grp: Optional[List[str]] + steps: List[str] + parameters: Optional[dict] = None From 908b118e5aef9b9cca593859169d247486c40400 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Tue, 17 Jan 2023 14:45:08 +0100 Subject: [PATCH 058/226] Add processing worker wrappers for run_cli and run_processor --- ocrd/ocrd/cli/processing_worker.py | 3 -- ocrd/ocrd/network/processing_worker.py | 70 ++++++++++++++++++++++++-- 2 files changed, 67 insertions(+), 6 deletions(-) diff --git a/ocrd/ocrd/cli/processing_worker.py b/ocrd/ocrd/cli/processing_worker.py index b4212ec4a..77b70f68e 100644 --- a/ocrd/ocrd/cli/processing_worker.py +++ b/ocrd/ocrd/cli/processing_worker.py @@ -48,9 +48,6 @@ def processing_worker_cli(processor_name: str, queue: str, database: str): processing_worker = ProcessingWorker( processor_name=processor_name, - processor_arguments={}, - # TODO: Send the proper processor_class. How? - processor_class="", ocrd_tool=ocrd_tool, rmq_host=rmq_host, rmq_port=rmq_port, diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index a55379f98..2aeddf1b5 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -8,21 +8,24 @@ from frozendict import frozendict from functools import lru_cache, wraps +import json +from typing import List from ocrd_utils import getLogger +from ocrd.processor.helpers import run_cli, run_processor from ocrd.network.rabbitmq_utils import RMQConsumer +from ocrd.network.ocrd_messages import OcrdProcessingMessage, OcrdResultMessage class ProcessingWorker: - def __init__(self, processor_name: str, processor_arguments: dict, processor_class, ocrd_tool: dict, + def __init__(self, processor_name: str, processor_arguments: dict, ocrd_tool: dict, rmq_host: str, rmq_port: int, rmq_vhost: str, db_url: str) -> None: self.log = getLogger(__name__) # ocr-d processor instance to be started self.processor_name = processor_name + # other potential parameters to be used self.processor_arguments = processor_arguments - # ocr-d processor object to be instantiated - self.processor_class = processor_class # Instantiation of the self.processor_class # Instantiated inside `on_consumed_message` @@ -72,6 +75,67 @@ def start_consuming(self) -> None: else: raise Exception("The RMQ Consumer is not connected/configured properly") + def process_message(self, ocrd_message: OcrdProcessingMessage): + pass + + def run_cli_from_worker( + self, + executable: str, + workspace, + page_id: str, + input_file_grps: List[str], + output_file_grps: List[str], + parameter: dict + ): + input_file_grps_str = ','.join(input_file_grps) + output_file_grps_str = ','.join(output_file_grps) + + return_code = run_cli( + executable=executable, + workspace=workspace, + page_id=page_id, + input_file_grp=input_file_grps_str, + output_file_grp=output_file_grps_str, + parameter=json.dumps(parameter), + mets_url=workspace.mets_target + ) + + if return_code != 0: + self.log.error(f'{executable} exited with non-zero return value {return_code}.') + else: + self.log.debug(f'{executable} exited with success.') + + def run_processor_from_worker( + self, + processor_class, + workspace, + page_id: str, + parameter: dict, + input_file_grps: List[str], + output_file_grps: List[str] + ): + input_file_grps_str = ','.join(input_file_grps) + output_file_grps_str = ','.join(output_file_grps) + + success = True + try: + run_processor( + processorClass=processor_class, + workspace=workspace, + page_id=page_id, + parameter=parameter, + input_file_grp=input_file_grps_str, + output_file_grp=output_file_grps_str + ) + except Exception as e: + success = False + self.log.exception(e) + + if not success: + self.log.error(f'{processor_class} failed with an exception.') + else: + self.log.debug(f'{processor_class} exited with success.') + # Method adopted from Triet's implementation # https://github.com/OCR-D/core/pull/884/files#diff-8b69cb85b5ffcfb93a053791dec62a2f909a0669ae33d8a2412f246c3b01f1a3R260 From 680b706da2a45f69fd37644ec3a9fc2b977e8000 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Tue, 17 Jan 2023 14:46:57 +0100 Subject: [PATCH 059/226] Fix import of ocrd_messages --- ocrd/ocrd/network/processing_worker.py | 2 +- ocrd/ocrd/network/rabbitmq_utils/__init__.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index 2aeddf1b5..225006669 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -14,7 +14,7 @@ from ocrd_utils import getLogger from ocrd.processor.helpers import run_cli, run_processor from ocrd.network.rabbitmq_utils import RMQConsumer -from ocrd.network.ocrd_messages import OcrdProcessingMessage, OcrdResultMessage +from ocrd.network.rabbitmq_utils import OcrdProcessingMessage, OcrdResultMessage class ProcessingWorker: diff --git a/ocrd/ocrd/network/rabbitmq_utils/__init__.py b/ocrd/ocrd/network/rabbitmq_utils/__init__.py index a250ce88c..b6dd062e4 100644 --- a/ocrd/ocrd/network/rabbitmq_utils/__init__.py +++ b/ocrd/ocrd/network/rabbitmq_utils/__init__.py @@ -5,6 +5,8 @@ "RMQConsumer", "RMQConnector", "RMQPublisher", + "OcrdProcessingMessage", + "OcrdResultMessage" ] from .consumer import RMQConsumer From 9de2175268b382e3e94c59748f0cebc8be3ea3c2 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Tue, 17 Jan 2023 19:27:23 +0100 Subject: [PATCH 060/226] RabbitMQ and MongoDB addr handling helpers --- ocrd/ocrd/network/helpers.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 ocrd/ocrd/network/helpers.py diff --git a/ocrd/ocrd/network/helpers.py b/ocrd/ocrd/network/helpers.py new file mode 100644 index 000000000..e69de29bb From a2481bd737993be230120496b95cf94155d70244 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Tue, 17 Jan 2023 20:59:10 +0100 Subject: [PATCH 061/226] Add helpers for addr parsing --- ocrd/ocrd/network/helpers.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/ocrd/ocrd/network/helpers.py b/ocrd/ocrd/network/helpers.py index e69de29bb..7dda00a21 100644 --- a/ocrd/ocrd/network/helpers.py +++ b/ocrd/ocrd/network/helpers.py @@ -0,0 +1,23 @@ +from typing import Tuple +from re import split + + +def verify_and_build_database_url(mongodb_address: str, database_prefix: str = "mongodb://") -> str: + elements = mongodb_address.split(':', 1) + if len(elements) != 2: + raise ValueError("The database address is in wrong format") + db_host = elements[0] + db_port = int(elements[1]) + mongodb_url = f"{database_prefix}{db_host}:{db_port}" + return mongodb_url + + +def verify_and_parse_rabbitmq_addr(rabbitmq_address: str) -> Tuple[str, int, str]: + elements = split(pattern=r':|/', string=rabbitmq_address) + if len(elements) != 3: + raise ValueError("The RabbitMQ address is in wrong format") + rmq_host = elements[0] + rmq_port = int(elements[1]) + # Handle the case with default virtual host + rmq_vhost = elements[2] if elements[2] else '/' + return rmq_host, rmq_port, rmq_vhost From c83fec923cd340540c1c6b9ca55ff291607c5837 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Tue, 17 Jan 2023 21:01:42 +0100 Subject: [PATCH 062/226] Extend decorators to accept --queue and --database --- ocrd/ocrd/decorators/__init__.py | 37 +++++++++++++++++++++++- ocrd/ocrd/decorators/ocrd_cli_options.py | 6 +++- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/ocrd/ocrd/decorators/__init__.py b/ocrd/ocrd/decorators/__init__.py index ff908c3e7..b3a31a716 100644 --- a/ocrd/ocrd/decorators/__init__.py +++ b/ocrd/ocrd/decorators/__init__.py @@ -1,6 +1,8 @@ from os.path import isfile from os import environ import sys +from contextlib import redirect_stdout +from io import StringIO import click @@ -10,9 +12,11 @@ set_json_key_value_overrides, ) -from ocrd_utils import getLogger, initLogging +from ocrd_utils import getLogger, initLogging, parse_json_string_with_comments from ocrd_validators import WorkspaceValidator +from ocrd.network import ProcessingWorker + from ..resolver import Resolver from ..processor.base import run_processor @@ -35,6 +39,8 @@ def ocrd_cli_wrap_processor( overwrite=False, show_resource=None, list_resources=False, + queue=None, + database=None, **kwargs ): if not sys.argv[1:]: @@ -51,6 +57,35 @@ def ocrd_cli_wrap_processor( list_resources=list_resources ) sys.exit() + # If either of these two is provided but not both + if bool(queue) != bool(database): + raise Exception("Both queue and database addresses must be provided - not just either of them.") + # If both of these are provided - start the processing worker instead of the processor - processorClass + if queue and database: + initLogging() + + # Get the ocrd_tool dictionary + f_out = StringIO() + with redirect_stdout(f_out): + processorClass(workspace=None, dump_json=True) + # TODO: Verify this. There is `ocrd_tool` parameter passed as an argument. + # The following line overwrites the passed parameter. + ocrd_tool = parse_json_string_with_comments(f_out.getvalue()) + + try: + processing_worker = ProcessingWorker( + rabbitmq_addr=queue, + mongodb_addr=database, + processor_name=ocrd_tool['executable'], + ocrd_tool=ocrd_tool, + processor_class=processorClass, + ) + # The RMQConsumer is initialized and a connection to the RabbitMQ is performed + processing_worker.connect_consumer() + # Start consuming from the queue with name `processor_name` + processing_worker.start_consuming() + except Exception as e: + raise Exception(f"Processing worker has failed with error: {e}") else: initLogging() LOG = getLogger('ocrd_cli_wrap_processor') diff --git a/ocrd/ocrd/decorators/ocrd_cli_options.py b/ocrd/ocrd/decorators/ocrd_cli_options.py index 3a1e07e50..1d14a5f20 100644 --- a/ocrd/ocrd/decorators/ocrd_cli_options.py +++ b/ocrd/ocrd/decorators/ocrd_cli_options.py @@ -1,4 +1,4 @@ -from click import option +from click import option, STRING from .parameter_option import parameter_option, parameter_override_option from .loglevel_option import loglevel_option @@ -26,6 +26,10 @@ def cli(mets_url): option('-O', '--output-file-grp', help='File group(s) used as output.', default='OUTPUT'), option('-g', '--page-id', help="ID(s) of the pages to process"), option('--overwrite', help="Overwrite the output file group or a page range (--page-id)", is_flag=True, default=False), + option('--queue', help="The RabbitMQ server address in format: {host}:{port}/{vhost}", + is_flag=True, default="localhost:5672/", type=STRING), + option('--database', help="The MongoDB address in format: {host}:{port}. `mongodb://` prefix is auto appended.", + is_flag=True, default="localhost:27018", type=STRING), option('-C', '--show-resource', help='Dump the content of processor resource RESNAME', metavar='RESNAME'), option('-L', '--list-resources', is_flag=True, default=False, help='List names of processor resources'), parameter_option, From 1d8c1b05a8b12c254b63012bc0f3b312563305a1 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Tue, 17 Jan 2023 21:04:07 +0100 Subject: [PATCH 063/226] Extend processing worker --- ocrd/ocrd/cli/processing_worker.py | 45 +++---- ocrd/ocrd/network/deployer.py | 2 +- ocrd/ocrd/network/processing_worker.py | 162 +++++++++++++++---------- 3 files changed, 115 insertions(+), 94 deletions(-) diff --git a/ocrd/ocrd/cli/processing_worker.py b/ocrd/ocrd/cli/processing_worker.py index 77b70f68e..8915a4c33 100644 --- a/ocrd/ocrd/cli/processing_worker.py +++ b/ocrd/ocrd/cli/processing_worker.py @@ -11,7 +11,7 @@ initLogging, parse_json_string_with_comments ) -from ocrd.network import ProcessingWorker +from ocrd.network.processing_worker import ProcessingWorker @click.command('processing-worker') @@ -27,36 +27,23 @@ def processing_worker_cli(processor_name: str, queue: str, database: str): Start a processing worker (a specific ocr-d processor) """ initLogging() - try: - # TODO: Parse the actual RabbitMQ Server address - `queue` - rmq_host = "localhost" - rmq_port = 5672 - rmq_vhost = "/" - rmq_url = f"{rmq_host}:{rmq_port}{rmq_vhost}" - - # TODO: Parse the actual MongoDB address - `database` - db_prefix = "mongodb://" - db_host = "localhost" - db_port = 27018 - db_url = f"{db_prefix}{db_host}:{db_port}" - except ValueError: - raise click.UsageError('Wrong/Bad arguments format provided. Check the help sections') + # Get the ocrd_tool dictionary ocrd_tool = parse_json_string_with_comments( run([processor_name, '--dump-json'], stdout=PIPE, check=True, universal_newlines=True).stdout ) - processing_worker = ProcessingWorker( - processor_name=processor_name, - ocrd_tool=ocrd_tool, - rmq_host=rmq_host, - rmq_port=rmq_port, - rmq_vhost=rmq_vhost, - db_url=db_url - ) - - # TODO: Remove. It's just to test starting the OCR-D processor - processing_worker.on_consumed_message() - - # Start consuming with the configuration settings above - # processing_worker.start_consuming() + try: + processing_worker = ProcessingWorker( + rabbitmq_addr=queue, + mongodb_addr=database, + processor_name=ocrd_tool['executable'], + ocrd_tool=ocrd_tool, + processor_class=None, # For readability purposes assigned here + ) + # The RMQConsumer is initialized and a connection to the RabbitMQ is performed + processing_worker.connect_consumer() + # Start consuming from the queue with name `processor_name` + processing_worker.start_consuming() + except Exception as e: + raise Exception(f"Processing worker has failed with error: {e}") diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index 3a4798aac..c28a555b2 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -92,7 +92,7 @@ def _deploy_processing_worker(self, processor: ProcessorConfig, host: HostConfig assert not processor.pids, 'processors already deployed. Pids are present. Host: ' \ '{host.__dict__}. Processor: {processor.__dict__}' - # Create the specific RabbitMQ queue here based on the OCR-D processor name (processor.name) + # TODO: Create the specific RabbitMQ queue here based on the OCR-D processor name (processor.name) # self.rmq_publisher.create_queue(queue_name=processor.name) if processor.deploy_type == DeployType.native: diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index 225006669..39de3d0e5 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -11,114 +11,121 @@ import json from typing import List +from ocrd import Resolver from ocrd_utils import getLogger from ocrd.processor.helpers import run_cli, run_processor -from ocrd.network.rabbitmq_utils import RMQConsumer -from ocrd.network.rabbitmq_utils import OcrdProcessingMessage, OcrdResultMessage +from ocrd.network.helpers import ( + verify_and_build_database_url, + verify_and_parse_rabbitmq_addr +) +from ocrd.network.rabbitmq_utils import ( + OcrdProcessingMessage, + OcrdResultMessage, + RMQConsumer +) class ProcessingWorker: - def __init__(self, processor_name: str, processor_arguments: dict, ocrd_tool: dict, - rmq_host: str, rmq_port: int, rmq_vhost: str, db_url: str) -> None: + def __init__(self, rabbitmq_addr, mongodb_addr, processor_name, ocrd_tool: dict, processor_class=None) -> None: self.log = getLogger(__name__) - # ocr-d processor instance to be started - self.processor_name = processor_name - - # other potential parameters to be used - self.processor_arguments = processor_arguments - # Instantiation of the self.processor_class - # Instantiated inside `on_consumed_message` - self.processor_instance = None + try: + self.db_url = verify_and_build_database_url(mongodb_addr, database_prefix="mongodb://") + self.log.debug(f"MongoDB URL: {self.db_url}") + self.rmq_host, self.rmq_port, self.rmq_vhost = verify_and_parse_rabbitmq_addr(rabbitmq_addr) + self.log.debug(f"RabbitMQ Server URL: {self.rmq_host}:{self.rmq_port}/{self.rmq_vhost}") + except ValueError as e: + raise ValueError(e) self.ocrd_tool = ocrd_tool - self.db_url = db_url - self.rmq_host = rmq_host - self.rmq_port = rmq_port - self.rmq_vhost = rmq_vhost + # The str name of the OCR-D processor instance to be started + self.processor_name = processor_name - # These could also be made configurable, - # not relevant for the current state - self.rmq_username = "default-consumer" - self.rmq_password = "default-consumer" + # The processor class to be used to instantiate the processor + # Think of this as a func pointer to the constructor of the respective OCR-D processor + self.processor_class = processor_class - # self.rmq_consumer = self.connect_consumer() + # Gets assigned when `connect_consumer` is called on the working object + self.rmq_consumer = None - def connect_consumer(self) -> RMQConsumer: - rmq_consumer = RMQConsumer(host=self.rmq_host, port=self.rmq_port, vhost=self.rmq_vhost) - rmq_consumer.authenticate_and_connect(username=self.rmq_username, password=self.rmq_password) - return rmq_consumer + def connect_consumer(self, username="default-consumer", password="default-consumer"): + self.log.debug(f"Connecting to RabbitMQ server: {self.rmq_host}:{self.rmq_port}{self.rmq_vhost}") + self.rmq_consumer = RMQConsumer(host=self.rmq_host, port=self.rmq_port, vhost=self.rmq_vhost) + self.rmq_consumer.authenticate_and_connect(username=username, password=password) + self.log.debug(f"Successfully connected.") # Define what happens every time a message is consumed from the queue def on_consumed_message(self) -> None: - # 1. Load the OCR-D processor in the memory cache on first message consumed - # 2. Load the OCR-D processor from the memory cache on every other message consumed - self.processor_instance = get_processor(self.processor_arguments, self.processor_class) - if self.processor_instance: - self.log.debug(f"Loading processor instance of `{self.processor_name}` succeeded.") - else: - self.log.debug(f"Loading processor instance of `{self.processor_name}` failed.") - - # TODO: Do the processing of the current message - # self.processor_instance.X(...) + # TODO: Receive the actual consumed message and pass it as a parameter to `process_message` + self.process_message() def start_consuming(self) -> None: if self.rmq_consumer: + self.log.debug(f"Configuring consuming from queue: {self.processor_name}") self.rmq_consumer.configure_consuming( queue_name=self.processor_name, callback_method=self.on_consumed_message ) - # TODO: A separate thread must be created here to listen - # to the queue since this is a blocking action + self.log.debug(f"Starting consuming from queue: {self.processor_name}") + # Starting consuming is a blocking action self.rmq_consumer.start_consuming() else: raise Exception("The RMQ Consumer is not connected/configured properly") - def process_message(self, ocrd_message: OcrdProcessingMessage): - pass + def process_message(self, ocrd_message: OcrdProcessingMessage = None): + # TODO: Extract the required data fields from the received OcrdProcessingMessage - def run_cli_from_worker( - self, - executable: str, - workspace, - page_id: str, - input_file_grps: List[str], - output_file_grps: List[str], - parameter: dict - ): - input_file_grps_str = ','.join(input_file_grps) - output_file_grps_str = ','.join(output_file_grps) + # This can be path if invoking `run_processor` + # but must be ocrd.Workspace if invoking `run_cli`. + workspace_path = "/home/mm/Desktop/ws_example/mets.xml" - return_code = run_cli( - executable=executable, - workspace=workspace, - page_id=page_id, - input_file_grp=input_file_grps_str, - output_file_grp=output_file_grps_str, - parameter=json.dumps(parameter), - mets_url=workspace.mets_target - ) + page_id = "PHYS_0001" + input_file_grps = ["DEFAULT"] + output_file_grps = ["OCR-D-BIN"] + parameter = {} - if return_code != 0: - self.log.error(f'{executable} exited with non-zero return value {return_code}.') + # Build the workspace from the workspace_path + workspace = Resolver().workspace_from_url(workspace_path) + + # TODO: Currently, no caching is performed. + if self.processor_class: + self.log.debug(f"Invoking the pythonic processor: {self.processor_name}") + self.log.debug(f"Invoking the processor_class: {self.processor_class}") + self.run_processor_from_worker( + processor_class=self.processor_class, + workspace=workspace, + page_id=page_id, + input_file_grps=input_file_grps, + output_file_grps=output_file_grps, + parameter=parameter + ) else: - self.log.debug(f'{executable} exited with success.') + self.log.debug(f"Invoking the cli: {self.processor_name}") + self.run_cli_from_worker( + executable=self.processor_name, + workspace=workspace, + page_id=page_id, + input_file_grps=input_file_grps, + output_file_grps=output_file_grps, + parameter=parameter + ) def run_processor_from_worker( self, processor_class, workspace, page_id: str, - parameter: dict, input_file_grps: List[str], - output_file_grps: List[str] + output_file_grps: List[str], + parameter: dict, ): input_file_grps_str = ','.join(input_file_grps) output_file_grps_str = ','.join(output_file_grps) success = True try: + # TODO: Use the cached_processor flag here once #972 is merged to core run_processor( processorClass=processor_class, workspace=workspace, @@ -136,6 +143,33 @@ def run_processor_from_worker( else: self.log.debug(f'{processor_class} exited with success.') + def run_cli_from_worker( + self, + executable: str, + workspace, + page_id: str, + input_file_grps: List[str], + output_file_grps: List[str], + parameter: dict + ): + input_file_grps_str = ','.join(input_file_grps) + output_file_grps_str = ','.join(output_file_grps) + + return_code = run_cli( + executable=executable, + workspace=workspace, + page_id=page_id, + input_file_grp=input_file_grps_str, + output_file_grp=output_file_grps_str, + parameter=json.dumps(parameter), + mets_url=workspace.mets_target + ) + + if return_code != 0: + self.log.error(f'{executable} exited with non-zero return value {return_code}.') + else: + self.log.debug(f'{executable} exited with success.') + # Method adopted from Triet's implementation # https://github.com/OCR-D/core/pull/884/files#diff-8b69cb85b5ffcfb93a053791dec62a2f909a0669ae33d8a2412f246c3b01f1a3R260 From 62ae01b83e99e66261b4d69c36f1eb288dc4e555 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Tue, 17 Jan 2023 21:27:38 +0100 Subject: [PATCH 064/226] Remove default values for --queue and --database --- ocrd/ocrd/decorators/ocrd_cli_options.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ocrd/ocrd/decorators/ocrd_cli_options.py b/ocrd/ocrd/decorators/ocrd_cli_options.py index 1d14a5f20..60a177396 100644 --- a/ocrd/ocrd/decorators/ocrd_cli_options.py +++ b/ocrd/ocrd/decorators/ocrd_cli_options.py @@ -27,9 +27,9 @@ def cli(mets_url): option('-g', '--page-id', help="ID(s) of the pages to process"), option('--overwrite', help="Overwrite the output file group or a page range (--page-id)", is_flag=True, default=False), option('--queue', help="The RabbitMQ server address in format: {host}:{port}/{vhost}", - is_flag=True, default="localhost:5672/", type=STRING), + is_flag=True, type=STRING), option('--database', help="The MongoDB address in format: {host}:{port}. `mongodb://` prefix is auto appended.", - is_flag=True, default="localhost:27018", type=STRING), + is_flag=True, type=STRING), option('-C', '--show-resource', help='Dump the content of processor resource RESNAME', metavar='RESNAME'), option('-L', '--list-resources', is_flag=True, default=False, help='List names of processor resources'), parameter_option, From eeb797b6bdb358fedb902dc54aec9488c72aeded Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Tue, 17 Jan 2023 21:44:22 +0100 Subject: [PATCH 065/226] Increase sleep command of the dummy docker container --- ocrd/ocrd/network/deployer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index c28a555b2..b1f2c8731 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -286,6 +286,6 @@ def start_docker_processor(client: CustomDockerClient, processor_name: str, _que log = getLogger(__name__) log.debug(f'start docker processor: {processor_name}') # TODO: add real command here to start processing server here - res = client.containers.run('debian', 'sleep 31', detach=True, remove=True) + res = client.containers.run('debian', 'sleep 500', detach=True, remove=True) assert res and res.id, 'run processor in docker-container failed' return res.id From fd24c4ffcf3a66e2c27666c21558ff39bfdafb6a Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Tue, 17 Jan 2023 21:48:16 +0100 Subject: [PATCH 066/226] Fix rabbitmq path in logging of processing_worker --- ocrd/ocrd/network/processing_worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index 39de3d0e5..6cac2afaa 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -33,7 +33,7 @@ def __init__(self, rabbitmq_addr, mongodb_addr, processor_name, ocrd_tool: dict, self.db_url = verify_and_build_database_url(mongodb_addr, database_prefix="mongodb://") self.log.debug(f"MongoDB URL: {self.db_url}") self.rmq_host, self.rmq_port, self.rmq_vhost = verify_and_parse_rabbitmq_addr(rabbitmq_addr) - self.log.debug(f"RabbitMQ Server URL: {self.rmq_host}:{self.rmq_port}/{self.rmq_vhost}") + self.log.debug(f"RabbitMQ Server URL: {self.rmq_host}:{self.rmq_port}{self.rmq_vhost}") except ValueError as e: raise ValueError(e) From 6359bdf79d0dea5521480c6d08b52934b04885c0 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Wed, 18 Jan 2023 00:08:21 +0100 Subject: [PATCH 067/226] Refactor processing broker + activate queue creation --- ocrd/ocrd/cli/processing_broker.py | 5 +- ocrd/ocrd/network/deployer.py | 86 ++++++++++++++------------ ocrd/ocrd/network/processing_broker.py | 83 +++++++++++++++++++------ 3 files changed, 112 insertions(+), 62 deletions(-) diff --git a/ocrd/ocrd/cli/processing_broker.py b/ocrd/ocrd/cli/processing_broker.py index 84a92ecfc..ea935abf1 100644 --- a/ocrd/ocrd/cli/processing_broker.py +++ b/ocrd/ocrd/cli/processing_broker.py @@ -23,5 +23,6 @@ def processing_broker_cli(path_to_config, address: str): port_int = int(port) except ValueError: raise click.UsageError('The --adddress option must have the format IP:PORT') - app = ProcessingBroker(path_to_config, host, port_int) - app.start() + processing_broker = ProcessingBroker(path_to_config, host, port_int) + # Start the Processing Broker aka the Processing Server (the new name) + processing_broker.start() diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index b1f2c8731..de3da4332 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -47,54 +47,53 @@ def __init__(self, queue_config: QueueConfig, mongo_config: MongoConfig, hosts_c self.mq_data = queue_config self.hosts = hosts_config + # Avoid using this method + # TODO: Should be removed def deploy_all(self) -> None: """ Deploy the message queue and all processors defined in the config-file """ # The order of deploying may be important to recover from previous state - - # Ideally, this should return the address of the RabbitMQ Server - rabbitmq_address = self._deploy_rabbitmq() - # Ideally, this should return the address of the MongoDB - mongodb_address = self._deploy_mongodb() - self._deploy_processing_workers(self.hosts, rabbitmq_address, mongodb_address) + rabbitmq_url = self.deploy_rabbitmq() + mongodb_url = self.deploy_mongodb() + self.deploy_hosts(self.hosts, rabbitmq_url, mongodb_url) def kill_all(self) -> None: + self.log.debug("Killing all deployed agents") # The order of killing is important to optimize graceful shutdown in the future # If RabbitMQ server is killed before killing Processing Workers, that may have # bad outcome and leave Processing Workers in an unpredictable state - # First kill the active Processing Workers + # First kill the active Processing Workers on Processing Hosts # They may still want to update something in the db before closing # They may still want to nack the currently processed messages back to the RabbitMQ Server - self._kill_processing_workers() + self.kill_hosts() + self.log.debug("Killed deployed agents") # Second kill the MongoDB - self._kill_mongodb() + self.kill_mongodb() # Third kill the RabbitMQ Server - self._kill_rabbitmq() + self.kill_rabbitmq() - def _deploy_processing_workers(self, hosts: List[HostConfig], rabbitmq_address: str, - mongodb_address: str) -> None: + def deploy_hosts(self, hosts: List[HostConfig], rabbitmq_url: str, mongodb_url: str) -> None: + self.log.debug("Deploying hosts") for host in hosts: for processor in host.processors: - self._deploy_processing_worker(processor, host, rabbitmq_address, mongodb_address) + self._deploy_processing_worker(processor, host, rabbitmq_url, mongodb_url) if host.ssh_client: host.ssh_client.close() if host.docker_client: host.docker_client.close() + self.log.debug("Hosts deployed") def _deploy_processing_worker(self, processor: ProcessorConfig, host: HostConfig, - rabbitmq_server: str = '', mongodb: str = '') -> None: + rabbitmq_url: str, mongodb_url: str) -> None: self.log.debug(f'deploy "{processor.deploy_type}" processor: "{processor}" on' f'"{host.address}"') assert not processor.pids, 'processors already deployed. Pids are present. Host: ' \ '{host.__dict__}. Processor: {processor.__dict__}' - # TODO: Create the specific RabbitMQ queue here based on the OCR-D processor name (processor.name) - # self.rmq_publisher.create_queue(queue_name=processor.name) - if processor.deploy_type == DeployType.native: if not host.ssh_client: host.ssh_client = create_ssh_client(host.address, host.username, host.password, host.keypath) @@ -112,23 +111,23 @@ def _deploy_processing_worker(self, processor: ProcessorConfig, host: HostConfig pid = self.start_native_processor( client=host.ssh_client, processor_name=processor.name, - _queue_address=rabbitmq_server, - _database_address=mongodb) + _queue_url=rabbitmq_url, + _database_url=mongodb_url) processor.add_started_pid(pid) elif processor.deploy_type == DeployType.docker: assert host.docker_client # to satisfy mypy pid = self.start_docker_processor( client=host.docker_client, processor_name=processor.name, - _queue_address=rabbitmq_server, - _database_address=mongodb) + _queue_url=rabbitmq_url, + _database_url=mongodb_url) processor.add_started_pid(pid) else: # Error case, should never enter here. Handle error cases here (if needed) self.log.error(f"Deploy type of {processor.name} is neither of the allowed types") pass - def _deploy_rabbitmq(self, image: str = 'rabbitmq', detach: bool = True, remove: bool = True, + def deploy_rabbitmq(self, image: str = 'rabbitmq', detach: bool = True, remove: bool = True, ports_mapping: Union[Dict, None] = None) -> str: # This method deploys the RabbitMQ Server. # Handling of creation of queues, submitting messages to queues, @@ -159,12 +158,15 @@ def _deploy_rabbitmq(self, image: str = 'rabbitmq', detach: bool = True, remove: client.close() self.log.debug('deployed rabbitmq') - # TODO: Not implemented yet - # Note: The queue address is not just the IP address - queue_address = 'RabbitMQ Server address' - return queue_address + # Build the RabbitMQ Server URL to return + rmq_host = self.mq_data.address + rmq_port = self.mq_data.port + rmq_vhost = "/" # the default virtual host + + rabbitmq_url = f"{rmq_host}:{rmq_port}{rmq_vhost}" + return rabbitmq_url - def _deploy_mongodb(self, image: str = 'mongo', detach: bool = True, remove: bool = True, + def deploy_mongodb(self, image: str = 'mongo', detach: bool = True, remove: bool = True, ports_mapping: Union[Dict, None] = None) -> str: if not self.mongo_data or not self.mongo_data.address: self.log.debug('canceled mongo-deploy: no mongo_db in config') @@ -188,12 +190,14 @@ def _deploy_mongodb(self, image: str = 'mongo', detach: bool = True, remove: boo client.close() self.log.debug('deployed mongodb') - # TODO: Not implemented yet - # Note: The mongodb address is not just the IP address - mongodb_address = 'MongoDB Address' - return mongodb_address + # Build the MongoDB URL to return + mongodb_prefix = "mongodb://" + mongodb_host = self.mongo_data.address + mongodb_port = self.mongo_data.port + mongodb_url = f"{mongodb_prefix}{mongodb_host}:{mongodb_port}" + return mongodb_url - def _kill_rabbitmq(self) -> None: + def kill_rabbitmq(self) -> None: if not self.mq_data.pid: self.log.debug('kill_rabbitmq: rabbitmq server is not running') return @@ -207,7 +211,7 @@ def _kill_rabbitmq(self) -> None: client.close() self.log.debug('stopped rabbitmq') - def _kill_mongodb(self) -> None: + def kill_mongodb(self) -> None: if not self.mongo_data or not self.mongo_data.pid: self.log.debug('kill_mongdb: mongodb not running') return @@ -221,17 +225,17 @@ def _kill_mongodb(self) -> None: client.close() self.log.debug('stopped mongodb') - def _kill_processing_workers(self) -> None: - # Kill processing worker hosts + def kill_hosts(self) -> None: + # Kill processing hosts for host in self.hosts: if host.ssh_client: host.ssh_client = create_ssh_client(host.address, host.username, host.password, host.keypath) if host.docker_client: host.docker_client = create_docker_client(host.address, host.username, host.password, host.keypath) # Kill deployed OCR-D processor instances on this Processing worker host - self._kill_processing_worker(host) + self.kill_processing_worker(host) - def _kill_processing_worker(self, host: HostConfig) -> None: + def kill_processing_worker(self, host: HostConfig) -> None: for processor in host.processors: if processor.deploy_type.is_native(): for pid in processor.pids: @@ -253,8 +257,8 @@ def _kill_processing_worker(self, host: HostConfig) -> None: # needed yet (otherwise flak8 complains). But they will be needed once the real # processing_worker is called here. Then they should be renamed @staticmethod - def start_native_processor(client: SSHClient, processor_name: str, _queue_address: str, - _database_address: str) -> str: + def start_native_processor(client: SSHClient, processor_name: str, _queue_url: str, + _database_url: str) -> str: log = getLogger(__name__) log.debug(f'start native processor: {processor_name}') channel = client.invoke_shell() @@ -281,8 +285,8 @@ def start_native_processor(client: SSHClient, processor_name: str, _queue_addres # needed yet (otherwise flak8 complains). But they will be needed once the real # processing_worker is called here. Then they should be renamed @staticmethod - def start_docker_processor(client: CustomDockerClient, processor_name: str, _queue_address: str, - _database_address: str) -> str: + def start_docker_processor(client: CustomDockerClient, processor_name: str, _queue_url: str, + _database_url: str) -> str: log = getLogger(__name__) log.debug(f'start docker processor: {processor_name}') # TODO: add real command here to start processing server here diff --git a/ocrd/ocrd/network/processing_broker.py b/ocrd/ocrd/network/processing_broker.py index 47f224fac..45c69de1d 100644 --- a/ocrd/ocrd/network/processing_broker.py +++ b/ocrd/ocrd/network/processing_broker.py @@ -1,6 +1,8 @@ from fastapi import FastAPI import uvicorn +from time import sleep from yaml import safe_load +from typing import List from ocrd_utils import getLogger from ocrd_validators import ProcessingBrokerValidator @@ -22,32 +24,36 @@ def __init__(self, config_path: str, host: str, port: int) -> None: self.hostname = host self.port = port - self.config = ProcessingBroker.parse_config(config_path) + # TODO: Ideally the parse_config should return a Tuple with the 3 configs assigned below + # to prevent passing the entire parsed config around to methods. + parsed_config = ProcessingBroker.parse_config(config_path) + self.queue_config = parsed_config.queue_config + self.mongo_config = parsed_config.mongo_config + self.hosts_config = parsed_config.hosts_config self.deployer = Deployer( - queue_config=self.config.queue_config, - mongo_config=self.config.mongo_config, - hosts_config=self.config.hosts_config + queue_config=self.queue_config, + mongo_config=self.mongo_config, + hosts_config=self.hosts_config ) + # TODO: Parse the RabbitMQ related data from the `queue_config` + # above instead of using the hard coded ones below + # RabbitMQ related fields, hard coded initially self.rmq_host = "localhost" self.rmq_port = 5672 self.rmq_vhost = "/" - # These could also be made configurable, - # not relevant for the current state - self.rmq_username = "default-publisher" - self.rmq_password = "default-publisher" - - # self.rmq_publisher = self.connect_publisher() - # self.rmq_publisher.enable_delivery_confirmations() # Enable acks + # Gets assigned when `connect_publisher` is called on the working object + # Note for peer: Check under self.start() + self.rmq_publisher = None self.router.add_api_route( path='/stop', endpoint=self.stop_deployed_agents, methods=['POST'], # tags=['TODO: add a tag'], - # summary='TODO: summary for apidesc', + # summary='TODO: summary for api desc', # TODO: add response model? add a response body at all? ) @@ -58,11 +64,44 @@ def __init__(self, config_path: str, host: str, port: int) -> None: def start(self) -> None: """ - start processing broker with uvicorn + deploy things and start the processing broker (aka server) with uvicorn + """ + """ + Note for a peer: + Deploying everything together at once is a bad approach. First the RabbitMQ Server and the MongoDB + should be deployed. Then the RMQPublisher of the Processing Broker (aka Processing Server) should + connect to the running RabbitMQ server. After that point the Processing Workers should be deployed. + The RMQPublisher should be connected before deploying Processing Workers because the message queues to + which the Processing Workers listen to are created based on the deployed processor. """ # Deploy everything specified in the configuration - self.deployer.deploy_all() - self.log.debug(f'starting uvicorn. Host: {self.host}. Port: {self.port}') + # self.deployer.deploy_all() + + # Deploy the RabbitMQ Server, get the URL of the deployed agent + rabbitmq_url = self.deployer.deploy_rabbitmq() + + # Deploy the MongoDB, get the URL of the deployed agent + mongodb_url = self.deployer.deploy_mongodb() + + # Give enough time for the RabbitMQ server to get deployed and get fully configured + # Needed to prevent connection of the publisher before the RabbitMQ is deployed + sleep(3) # TODO: Sleeping here is bad and better check should be performed + + # The RMQPublisher is initialized and a connection to the RabbitMQ is performed + self.connect_publisher() + + # Create the message queues based on the occurrence of `processor.name` in the config file + for host in self.hosts_config: + for processor in host.processors: + # The existence/validity of the processor.name is not tested. + # Even if an ocr-d processor does not exist, the queue is created + self.log.debug(f"Creating a message queue with id: {processor.name}") + self.rmq_publisher.create_queue(queue_name=processor.name) + + # Deploy processing hosts where processing workers are running on + self.deployer.deploy_hosts(self.hosts_config, rabbitmq_url, mongodb_url) + + self.log.debug(f'Starting uvicorn: {self.host}:{self.port}') uvicorn.run(self, host=self.hostname, port=self.port) @staticmethod @@ -94,7 +133,13 @@ async def on_shutdown(self) -> None: async def stop_deployed_agents(self) -> None: self.deployer.kill_all() - def connect_publisher(self) -> RMQPublisher: - rmq_publisher = RMQPublisher(host=self.rmq_host, port=self.rmq_port, vhost=self.rmq_vhost) - rmq_publisher.authenticate_and_connect(username=self.rmq_username, password=self.rmq_password) - return rmq_publisher + def connect_publisher(self, username="default-publisher", password="default-publisher", enable_acks=True): + self.log.debug(f"Connecting to RabbitMQ server: {self.rmq_host}:{self.rmq_port}{self.rmq_vhost}") + self.rmq_publisher = RMQPublisher(host=self.rmq_host, port=self.rmq_port, vhost=self.rmq_vhost) + self.rmq_publisher.authenticate_and_connect(username=username, password=password) + self.log.debug(f"Successfully connected.") + if enable_acks: + self.rmq_publisher.enable_delivery_confirmations() + self.log.debug(f"Delivery confirmations are enabled") + else: + self.log.debug(f"Delivery confirmations are disabled") From ef34e677b3066c0663b9af6981df8e86eab86fe5 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Wed, 18 Jan 2023 00:28:32 +0100 Subject: [PATCH 068/226] Log messages received from the Management UI on port 15672 --- ocrd/ocrd/network/processing_worker.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index 6cac2afaa..38eae9f43 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -56,9 +56,15 @@ def connect_consumer(self, username="default-consumer", password="default-consum self.log.debug(f"Successfully connected.") # Define what happens every time a message is consumed from the queue - def on_consumed_message(self) -> None: - # TODO: Receive the actual consumed message and pass it as a parameter to `process_message` - self.process_message() + def on_consumed_message(self, channel, method, properties, body) -> None: + self.log.debug(f"Received from ch: {channel}, method: {method}, properties: {properties}, body: {body}") + + # TODO: + # 1. Parse here the received message body to OcrdProcessingMessage + # processing_message = OcrdProcessingMessage(...) + # 2. Send the ocrd_message as a parameter to self.process_message + # self.process_message(ocrd_message=processing_message) + pass def start_consuming(self) -> None: if self.rmq_consumer: @@ -73,7 +79,7 @@ def start_consuming(self) -> None: else: raise Exception("The RMQ Consumer is not connected/configured properly") - def process_message(self, ocrd_message: OcrdProcessingMessage = None): + def process_message(self, processing_message: OcrdProcessingMessage = None): # TODO: Extract the required data fields from the received OcrdProcessingMessage # This can be path if invoking `run_processor` From cfcc036254593a448589f07e54528c61deffcbb1 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Wed, 18 Jan 2023 14:57:57 +0100 Subject: [PATCH 069/226] Improve logging + add some TODOs and remarks --- ocrd/ocrd/network/deployer.py | 95 +++++++++++++++++--------- ocrd/ocrd/network/processing_broker.py | 28 +++++--- ocrd/ocrd/network/processing_worker.py | 12 ++-- 3 files changed, 88 insertions(+), 47 deletions(-) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index de3da4332..83ba9c3cb 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -42,11 +42,20 @@ def __init__(self, queue_config: QueueConfig, mongo_config: MongoConfig, hosts_c hosts_config: Processing Hosts related configurations """ self.log = getLogger(__name__) - self.log.debug('Deployer-init()') + self.log.debug('The Deployer of the ProcessingServer was invoked') self.mongo_data = mongo_config self.mq_data = queue_config self.hosts = hosts_config + # TODO: We should have a data structure here to manage the connections and PIDs: + # - RabbitMQ - (host address, pid on that host) + # - MongoDB - (host address, pid on that host) + # - Processing Hosts - (host address) + # - Processing Workers - (pid on that host address) + # The PIDs are stored for future usage - i.e. for killing them forcefully/gracefully. + # Currently, the connections (ssh_client, docker_client) and + # the PIDs are stored inside the config data classes + # Avoid using this method # TODO: Should be removed def deploy_all(self) -> None: @@ -58,7 +67,6 @@ def deploy_all(self) -> None: self.deploy_hosts(self.hosts, rabbitmq_url, mongodb_url) def kill_all(self) -> None: - self.log.debug("Killing all deployed agents") # The order of killing is important to optimize graceful shutdown in the future # If RabbitMQ server is killed before killing Processing Workers, that may have # bad outcome and leave Processing Workers in an unpredictable state @@ -67,7 +75,6 @@ def kill_all(self) -> None: # They may still want to update something in the db before closing # They may still want to nack the currently processed messages back to the RabbitMQ Server self.kill_hosts() - self.log.debug("Killed deployed agents") # Second kill the MongoDB self.kill_mongodb() @@ -76,16 +83,20 @@ def kill_all(self) -> None: self.kill_rabbitmq() def deploy_hosts(self, hosts: List[HostConfig], rabbitmq_url: str, mongodb_url: str) -> None: - self.log.debug("Deploying hosts") + self.log.debug("Starting to deploy hosts") for host in hosts: + self.log.debug(f"Deploying processing workers on host: {host.address}") for processor in host.processors: self._deploy_processing_worker(processor, host, rabbitmq_url, mongodb_url) + # TODO: These connections, just like the PIDs, should not be kept in the config data classes + # The connections are correctly closed on host level, but created on processing worker level? if host.ssh_client: host.ssh_client.close() if host.docker_client: host.docker_client.close() - self.log.debug("Hosts deployed") + # TODO: Creating connections if missing should probably occur when deploying hosts not when + # deploying processing workers. The deploy_type checks and opening connections creates duplicate code. def _deploy_processing_worker(self, processor: ProcessorConfig, host: HostConfig, rabbitmq_url: str, mongodb_url: str) -> None: @@ -94,6 +105,7 @@ def _deploy_processing_worker(self, processor: ProcessorConfig, host: HostConfig assert not processor.pids, 'processors already deployed. Pids are present. Host: ' \ '{host.__dict__}. Processor: {processor.__dict__}' + # TODO: The check for available ssh or docker connections should probably happen inside `deploy_hosts` if processor.deploy_type == DeployType.native: if not host.ssh_client: host.ssh_client = create_ssh_client(host.address, host.username, host.password, host.keypath) @@ -102,7 +114,7 @@ def _deploy_processing_worker(self, processor: ProcessorConfig, host: HostConfig host.docker_client = create_docker_client(host.address, host.username, host.password, host.keypath) else: # Error case, should never enter here. Handle error cases here (if needed) - self.log.error(f"Deploy type of {processor.name} is neither of the allowed types") + self.log.error(f"Failed to deploy: {processor.name}. The deploy type is unknown.") pass for _ in range(processor.count): @@ -123,16 +135,24 @@ def _deploy_processing_worker(self, processor: ProcessorConfig, host: HostConfig _database_url=mongodb_url) processor.add_started_pid(pid) else: + # TODO: Weirdly there is a duplication of code inside this method # Error case, should never enter here. Handle error cases here (if needed) - self.log.error(f"Deploy type of {processor.name} is neither of the allowed types") + self.log.error(f"Failed to deploy: {processor.name}. The deploy type is unknown.") pass def deploy_rabbitmq(self, image: str = 'rabbitmq', detach: bool = True, remove: bool = True, ports_mapping: Union[Dict, None] = None) -> str: + # Note for a peer # This method deploys the RabbitMQ Server. # Handling of creation of queues, submitting messages to queues, # and receiving messages from queues is part of the RabbitMQ Library # Which is part of the OCR-D WebAPI implementation. + self.log.debug(f"Trying to deploy image[{image}], with modes: detach[{detach}], remove[{remove}]") + + if not self.mongo_data or not self.mongo_data.address: + self.log.error(f"Deploying RabbitMQ has failed - missing configuration.") + # TODO: Raise an error instead of silently ignoring it + return "" client = create_docker_client(self.mq_data.address, self.mq_data.username, self.mq_data.password, self.mq_data.keypath) @@ -146,6 +166,7 @@ def deploy_rabbitmq(self, image: str = 'rabbitmq', detach: bool = True, remove: 15672: 15672, 25672: 25672 } + self.log.debug(f"Ports mapping: {ports_mapping}") # TODO: use rm here or not? Should queues be reused? res = client.containers.run( image=image, @@ -153,10 +174,10 @@ def deploy_rabbitmq(self, image: str = 'rabbitmq', detach: bool = True, remove: remove=remove, ports=ports_mapping ) - assert res and res.id, 'starting rabbitmq failed' + assert res and res.id, \ + f'Failed to start RabbitMQ docker container on host: {self.mq_data.address}' self.mq_data.pid = res.id client.close() - self.log.debug('deployed rabbitmq') # Build the RabbitMQ Server URL to return rmq_host = self.mq_data.address @@ -164,19 +185,25 @@ def deploy_rabbitmq(self, image: str = 'rabbitmq', detach: bool = True, remove: rmq_vhost = "/" # the default virtual host rabbitmq_url = f"{rmq_host}:{rmq_port}{rmq_vhost}" + self.log.debug(f"The RabbitMQ server was deployed on url: {rabbitmq_url}") return rabbitmq_url def deploy_mongodb(self, image: str = 'mongo', detach: bool = True, remove: bool = True, ports_mapping: Union[Dict, None] = None) -> str: + self.log.debug(f"Trying to deploy image[{image}], with modes: detach[{detach}], remove[{remove}]") + if not self.mongo_data or not self.mongo_data.address: - self.log.debug('canceled mongo-deploy: no mongo_db in config') + self.log.error(f"Deploying MongoDB has failed - missing configuration.") + # TODO: Raise an error instead of silently ignoring it return "" + client = create_docker_client(self.mongo_data.address, self.mongo_data.username, self.mongo_data.password, self.mongo_data.keypath) if not ports_mapping: ports_mapping = { 27017: self.mongo_data.port } + self.log.debug(f"Ports mapping: {ports_mapping}") # TODO: use rm here or not? Should the mongodb be reused? # TODO: what about the data-dir? Must data be preserved? res = client.containers.run( @@ -185,49 +212,54 @@ def deploy_mongodb(self, image: str = 'mongo', detach: bool = True, remove: bool remove=remove, ports=ports_mapping ) - assert res and res.id, 'starting mongodb failed' + assert res and res.id, \ + f'Failed to start MongoDB docker container on host: {self.mongo_data.address}' self.mongo_data.pid = res.id client.close() - self.log.debug('deployed mongodb') # Build the MongoDB URL to return mongodb_prefix = "mongodb://" mongodb_host = self.mongo_data.address mongodb_port = self.mongo_data.port mongodb_url = f"{mongodb_prefix}{mongodb_host}:{mongodb_port}" + self.log.debug(f"The MongoDB was deployed on url: {mongodb_url}") return mongodb_url def kill_rabbitmq(self) -> None: - if not self.mq_data.pid: - self.log.debug('kill_rabbitmq: rabbitmq server is not running') + # TODO: The PID must not be stored in the configuration `mq_data`. + if not self.mq_data or not self.mq_data.pid: + self.log.warning(f"No running RabbitMQ instance found") + # TODO: Ignoring this silently is problematic in the future return - else: - self.log.debug(f'trying to kill rabbitmq with id: {self.mq_data.pid} now') + self.log.debug(f"Trying to stop the deployed RabbitMQ with PID: {self.mq_data.pid}") client = create_docker_client(self.mq_data.address, self.mq_data.username, self.mq_data.password, self.mq_data.keypath) client.containers.get(self.mq_data.pid).stop() self.mq_data.pid = None client.close() - self.log.debug('stopped rabbitmq') + self.log.debug('The RabbitMQ is stopped') def kill_mongodb(self) -> None: + # TODO: The PID must not be stored in the configuration `mongo_data`. if not self.mongo_data or not self.mongo_data.pid: - self.log.debug('kill_mongdb: mongodb not running') + self.log.warning(f"No running MongoDB instance found") + # TODO: Ignoring this silently is problematic in the future return - else: - self.log.debug(f'trying to kill mongdb with id: {self.mongo_data.pid} now') + self.log.debug(f"Trying to stop the deployed MongoDB with PID: {self.mongo_data.pid}") client = create_docker_client(self.mongo_data.address, self.mongo_data.username, self.mongo_data.password, self.mongo_data.keypath) client.containers.get(self.mongo_data.pid).stop() self.mongo_data.pid = None client.close() - self.log.debug('stopped mongodb') + self.log.debug('The MongoDB is stopped') def kill_hosts(self) -> None: + self.log.debug("Starting to kill/stop hosts") # Kill processing hosts for host in self.hosts: + self.log.debug(f"Killing/Stopping processing workers on host: {host.address}") if host.ssh_client: host.ssh_client = create_ssh_client(host.address, host.username, host.password, host.keypath) if host.docker_client: @@ -239,28 +271,27 @@ def kill_processing_worker(self, host: HostConfig) -> None: for processor in host.processors: if processor.deploy_type.is_native(): for pid in processor.pids: + self.log.debug(f'Trying to kill/stop native processor: {processor.name}, with PID: {pid}') + # TODO: For graceful shutdown we may want to send additional parameters to kill host.ssh_client.exec_command(f'kill {pid}') elif processor.deploy_type.is_docker(): for pid in processor.pids: - self.log.debug(f'trying to kill docker container: {pid}') + self.log.debug(f'Trying to kill/stop docker container processor: {processor.name}, with PID: {pid}') # TODO: think about timeout. # think about using threads to kill parallelized to reduce waiting time host.docker_client.containers.get(pid).stop() else: # Error case, should never enter here. Handle error cases here (if needed) - self.log.error(f"Deploy type of {processor.name} is neither of the allowed types") + self.log.error(f"Failed to kill: {processor.name}. The deploy type is unknown.") pass processor.pids = [] - - # TODO: queue_address and _database_address are prefixed with underscore because they are not - # needed yet (otherwise flak8 complains). But they will be needed once the real - # processing_worker is called here. Then they should be renamed + # TODO: This method may not fit anymore. Should be further investigated. @staticmethod def start_native_processor(client: SSHClient, processor_name: str, _queue_url: str, _database_url: str) -> str: log = getLogger(__name__) - log.debug(f'start native processor: {processor_name}') + log.debug(f'Starting native processor: {processor_name}') channel = client.invoke_shell() stdin, stdout = channel.makefile('wb'), channel.makefile('rb') # TODO: add real command here to start processing server here @@ -281,15 +312,13 @@ def start_native_processor(client: SSHClient, processor_name: str, _queue_url: s # error if try to call) return re_search(r'xyz([0-9]+)xyz', output).group(1) - # TODO: queue_address and _database_address are prefixed with underscore because they are not - # needed yet (otherwise flak8 complains). But they will be needed once the real - # processing_worker is called here. Then they should be renamed + # TODO: This method may not fit anymore. Should be further investigated. @staticmethod def start_docker_processor(client: CustomDockerClient, processor_name: str, _queue_url: str, _database_url: str) -> str: log = getLogger(__name__) - log.debug(f'start docker processor: {processor_name}') + log.debug(f'Starting docker container processor: {processor_name}') # TODO: add real command here to start processing server here - res = client.containers.run('debian', 'sleep 500', detach=True, remove=True) + res = client.containers.run('debian', 'sleep 500s', detach=True, remove=True) assert res and res.id, 'run processor in docker-container failed' return res.id diff --git a/ocrd/ocrd/network/processing_broker.py b/ocrd/ocrd/network/processing_broker.py index 45c69de1d..633be6564 100644 --- a/ocrd/ocrd/network/processing_broker.py +++ b/ocrd/ocrd/network/processing_broker.py @@ -90,15 +90,11 @@ def start(self) -> None: # The RMQPublisher is initialized and a connection to the RabbitMQ is performed self.connect_publisher() - # Create the message queues based on the occurrence of `processor.name` in the config file - for host in self.hosts_config: - for processor in host.processors: - # The existence/validity of the processor.name is not tested. - # Even if an ocr-d processor does not exist, the queue is created - self.log.debug(f"Creating a message queue with id: {processor.name}") - self.rmq_publisher.create_queue(queue_name=processor.name) + self.log.debug(f"Starting to create message queues on RabbitMQ instance url: {rabbitmq_url}") + self.create_message_queues() # Deploy processing hosts where processing workers are running on + # Note: A deployed processing worker starts listening to a message queue with id processor.name self.deployer.deploy_hosts(self.hosts_config, rabbitmq_url, mongodb_url) self.log.debug(f'Starting uvicorn: {self.host}:{self.port}') @@ -126,6 +122,8 @@ async def on_shutdown(self) -> None: # visible when testing try: await self.stop_deployed_agents() + # TODO: This except block is trapping the user if nothing is following after the keyword. + # Is that the expected behaviour here? except: self.log.debug('error stopping processing servers: ', exc_info=True) raise @@ -134,12 +132,24 @@ async def stop_deployed_agents(self) -> None: self.deployer.kill_all() def connect_publisher(self, username="default-publisher", password="default-publisher", enable_acks=True): - self.log.debug(f"Connecting to RabbitMQ server: {self.rmq_host}:{self.rmq_port}{self.rmq_vhost}") + self.log.debug(f"Connecting RMQPublisher to RabbitMQ server: {self.rmq_host}:{self.rmq_port}{self.rmq_vhost}") self.rmq_publisher = RMQPublisher(host=self.rmq_host, port=self.rmq_port, vhost=self.rmq_vhost) + # TODO: Remove this information before the release + self.log.debug(f"RMQPublisher authenticates with username: {username}, password: {password}") self.rmq_publisher.authenticate_and_connect(username=username, password=password) - self.log.debug(f"Successfully connected.") if enable_acks: self.rmq_publisher.enable_delivery_confirmations() self.log.debug(f"Delivery confirmations are enabled") else: self.log.debug(f"Delivery confirmations are disabled") + self.log.debug(f"Successfully connected RMQPublisher.") + + def create_message_queues(self): + # Create the message queues based on the occurrence of `processor.name` in the config file + for host in self.hosts_config: + for processor in host.processors: + # The existence/validity of the processor.name is not tested. + # Even if an ocr-d processor does not exist, the queue is created + self.log.debug(f"Creating a message queue with id: {processor.name}") + # TODO: We may want to track here if there are already queues with the same name + self.rmq_publisher.create_queue(queue_name=processor.name) diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index 38eae9f43..77dc07ec5 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -31,9 +31,9 @@ def __init__(self, rabbitmq_addr, mongodb_addr, processor_name, ocrd_tool: dict, try: self.db_url = verify_and_build_database_url(mongodb_addr, database_prefix="mongodb://") - self.log.debug(f"MongoDB URL: {self.db_url}") + self.log.debug(f"Verified MongoDB URL: {self.db_url}") self.rmq_host, self.rmq_port, self.rmq_vhost = verify_and_parse_rabbitmq_addr(rabbitmq_addr) - self.log.debug(f"RabbitMQ Server URL: {self.rmq_host}:{self.rmq_port}{self.rmq_vhost}") + self.log.debug(f"Verified RabbitMQ Server URL: {self.rmq_host}:{self.rmq_port}{self.rmq_vhost}") except ValueError as e: raise ValueError(e) @@ -50,10 +50,12 @@ def __init__(self, rabbitmq_addr, mongodb_addr, processor_name, ocrd_tool: dict, self.rmq_consumer = None def connect_consumer(self, username="default-consumer", password="default-consumer"): - self.log.debug(f"Connecting to RabbitMQ server: {self.rmq_host}:{self.rmq_port}{self.rmq_vhost}") + self.log.debug(f"Connecting RMQConsumer to RabbitMQ server: {self.rmq_host}:{self.rmq_port}{self.rmq_vhost}") self.rmq_consumer = RMQConsumer(host=self.rmq_host, port=self.rmq_port, vhost=self.rmq_vhost) + # TODO: Remove this information before the release + self.log.debug(f"RMQConsumer authenticates with username: {username}, password: {password}") self.rmq_consumer.authenticate_and_connect(username=username, password=password) - self.log.debug(f"Successfully connected.") + self.log.debug(f"Successfully connected RMQConsumer.") # Define what happens every time a message is consumed from the queue def on_consumed_message(self, channel, method, properties, body) -> None: @@ -77,7 +79,7 @@ def start_consuming(self) -> None: # Starting consuming is a blocking action self.rmq_consumer.start_consuming() else: - raise Exception("The RMQ Consumer is not connected/configured properly") + raise Exception("The RMQConsumer is not connected/configured properly") def process_message(self, processing_message: OcrdProcessingMessage = None): # TODO: Extract the required data fields from the received OcrdProcessingMessage From 3bd6f70aa825c57df27a4c206aad1a1f3903a737 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Wed, 18 Jan 2023 17:16:14 +0100 Subject: [PATCH 070/226] Retrieve ocrd_tool by using util method --- ocrd/ocrd/cli/processing_worker.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/ocrd/ocrd/cli/processing_worker.py b/ocrd/ocrd/cli/processing_worker.py index 8915a4c33..ee6598396 100644 --- a/ocrd/ocrd/cli/processing_worker.py +++ b/ocrd/ocrd/cli/processing_worker.py @@ -9,6 +9,7 @@ from subprocess import run, PIPE from ocrd_utils import ( initLogging, + get_ocrd_tool_json, parse_json_string_with_comments ) from ocrd.network.processing_worker import ProcessingWorker @@ -29,9 +30,13 @@ def processing_worker_cli(processor_name: str, queue: str, database: str): initLogging() # Get the ocrd_tool dictionary - ocrd_tool = parse_json_string_with_comments( - run([processor_name, '--dump-json'], stdout=PIPE, check=True, universal_newlines=True).stdout - ) + # ocrd_tool = parse_json_string_with_comments( + # run([processor_name, '--dump-json'], stdout=PIPE, check=True, universal_newlines=True).stdout + # ) + + ocrd_tool = get_ocrd_tool_json(processor_name) + if not ocrd_tool: + raise Exception(f"The ocrd_tool is empty or missing") try: processing_worker = ProcessingWorker( From 7138979caf7404bc32c83431413330b915df9cb6 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Wed, 18 Jan 2023 17:43:34 +0100 Subject: [PATCH 071/226] Force logging settings - paramiko=INFO, ocrd.network=DEBUG --- ocrd/ocrd/cli/processing_broker.py | 5 +++++ ocrd/ocrd/cli/processing_worker.py | 3 +++ ocrd/ocrd/decorators/__init__.py | 3 +++ 3 files changed, 11 insertions(+) diff --git a/ocrd/ocrd/cli/processing_broker.py b/ocrd/ocrd/cli/processing_broker.py index ea935abf1..51a95a5b7 100644 --- a/ocrd/ocrd/cli/processing_broker.py +++ b/ocrd/ocrd/cli/processing_broker.py @@ -8,6 +8,7 @@ import click from ocrd_utils import initLogging from ocrd.network import ProcessingBroker +import logging @click.command('processing-broker') @@ -18,6 +19,10 @@ def processing_broker_cli(path_to_config, address: str): Start and manage processing servers (workers) with the processing broker """ initLogging() + # TODO: Remove before the release + logging.getLogger('paramiko.transport').setLevel(logging.INFO) + logging.getLogger('ocrd.network').setLevel(logging.DEBUG) + try: host, port = address.split(":") port_int = int(port) diff --git a/ocrd/ocrd/cli/processing_worker.py b/ocrd/ocrd/cli/processing_worker.py index ee6598396..8643bc098 100644 --- a/ocrd/ocrd/cli/processing_worker.py +++ b/ocrd/ocrd/cli/processing_worker.py @@ -6,6 +6,7 @@ :nested: full """ import click +import logging from subprocess import run, PIPE from ocrd_utils import ( initLogging, @@ -28,6 +29,8 @@ def processing_worker_cli(processor_name: str, queue: str, database: str): Start a processing worker (a specific ocr-d processor) """ initLogging() + # TODO: Remove before the release + logging.getLogger('ocrd.network').setLevel(logging.DEBUG) # Get the ocrd_tool dictionary # ocrd_tool = parse_json_string_with_comments( diff --git a/ocrd/ocrd/decorators/__init__.py b/ocrd/ocrd/decorators/__init__.py index b3a31a716..b54ec59cf 100644 --- a/ocrd/ocrd/decorators/__init__.py +++ b/ocrd/ocrd/decorators/__init__.py @@ -63,6 +63,9 @@ def ocrd_cli_wrap_processor( # If both of these are provided - start the processing worker instead of the processor - processorClass if queue and database: initLogging() + # TODO: Remove before the release + import logging + logging.getLogger('ocrd.network').setLevel(logging.DEBUG) # Get the ocrd_tool dictionary f_out = StringIO() From 0f826fda62498f861968981dcbe0c5787fe83b8b Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Wed, 18 Jan 2023 21:53:43 +0100 Subject: [PATCH 072/226] Implement on_consumed_message method --- ocrd/ocrd/network/processing_worker.py | 77 +++++++++++++++---- .../network/rabbitmq_utils/ocrd_messages.py | 40 ++++++++++ 2 files changed, 100 insertions(+), 17 deletions(-) diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index 77dc07ec5..099c269ee 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -49,6 +49,10 @@ def __init__(self, rabbitmq_addr, mongodb_addr, processor_name, ocrd_tool: dict, # Gets assigned when `connect_consumer` is called on the working object self.rmq_consumer = None + # TODO: In case the processing worker should publish OcrdResultMessage type message to + # the message queue with name {processor_name}-result, the RMQPublisher should be instantiated + # self.rmq_publisher = None + def connect_consumer(self, username="default-consumer", password="default-consumer"): self.log.debug(f"Connecting RMQConsumer to RabbitMQ server: {self.rmq_host}:{self.rmq_port}{self.rmq_vhost}") self.rmq_consumer = RMQConsumer(host=self.rmq_host, port=self.rmq_port, vhost=self.rmq_vhost) @@ -57,16 +61,43 @@ def connect_consumer(self, username="default-consumer", password="default-consum self.rmq_consumer.authenticate_and_connect(username=username, password=password) self.log.debug(f"Successfully connected RMQConsumer.") - # Define what happens every time a message is consumed from the queue - def on_consumed_message(self, channel, method, properties, body) -> None: - self.log.debug(f"Received from ch: {channel}, method: {method}, properties: {properties}, body: {body}") + # Define what happens every time a message is consumed + # from the queue with name self.processor_name + def on_consumed_message(self, channel, delivery, properties, body) -> None: + self.log.debug(f"Received from: \nchannel: {channel},\ndelivery: {delivery}, \nproperties: {properties}, \nbody: {body}") + consumer_tag = delivery.consumer_tag + delivery_tag: int = delivery.delivery_tag + is_redelivered: bool = delivery.redelivered + message_headers: dict = properties.headers + + self.log.debug(f"Consumer tag: {consumer_tag}") + self.log.debug(f"Message delivery tag: {delivery_tag}") + self.log.debug(f"Is redelivered message: {is_redelivered}") + self.log.debug(f"Message headers: {message_headers}") + + try: + self.log.debug(f"Trying to decode processing message with tag: {delivery_tag}") + processing_message: OcrdProcessingMessage = OcrdProcessingMessage.decode(body, encoding="utf-8") + except Exception as e: + self.log.error(f"Failed to decode processing message body: {body}") + self.log.error(f"Nacking processing message with tag: {delivery_tag}") + channel.basic_nack(delivery_tag=delivery_tag, multiple=False, requeue=False) + raise Exception(f"Failed to decode processing message with tag: {delivery_tag}, reason: {e}") + + try: + # TODO: Note to peer: ideally we should avoid doing database related actions + # in this method, and handle database related interactions inside `self.process_message()` + self.log.debug(f"Starting to process the received message: {processing_message}") + self.process_message(processing_message=processing_message) + except Exception as e: + self.log.error(f"Failed to process processing message with tag: {delivery_tag}") + self.log.error(f"Nacking processing message with tag: {delivery_tag}") + channel.basic_nack(delivery_tag=delivery_tag, multiple=False, requeue=False) + raise Exception(f"Failed to process processing message with tag: {delivery_tag}, reason: {e}") - # TODO: - # 1. Parse here the received message body to OcrdProcessingMessage - # processing_message = OcrdProcessingMessage(...) - # 2. Send the ocrd_message as a parameter to self.process_message - # self.process_message(ocrd_message=processing_message) - pass + self.log.debug(f"Successfully processed message ") + self.log.debug(f"Acking message with tag: {delivery_tag}") + channel.basic_ack(delivery_tag=delivery_tag, multiple=False) def start_consuming(self) -> None: if self.rmq_consumer: @@ -81,21 +112,33 @@ def start_consuming(self) -> None: else: raise Exception("The RMQConsumer is not connected/configured properly") - def process_message(self, processing_message: OcrdProcessingMessage = None): - # TODO: Extract the required data fields from the received OcrdProcessingMessage + # TODO: Better error handling required to catch exceptions + def process_message(self, processing_message: OcrdProcessingMessage): + # Verify that the processor name in the processing message + # matches the processor name of the current processing worker + if self.processor_name != processing_message.processor_name: + raise ValueError(f"Processor name is not matching. " + f"Expected: {self.processor_name}, Got: {processing_message.processor_name}") # This can be path if invoking `run_processor` # but must be ocrd.Workspace if invoking `run_cli`. - workspace_path = "/home/mm/Desktop/ws_example/mets.xml" - - page_id = "PHYS_0001" - input_file_grps = ["DEFAULT"] - output_file_grps = ["OCR-D-BIN"] - parameter = {} + workspace_path = processing_message.path_to_mets # Build the workspace from the workspace_path workspace = Resolver().workspace_from_url(workspace_path) + page_id = processing_message.page_id + input_file_grps = processing_message.input_file_grps + output_file_grps = processing_message.output_file_grps + parameter = processing_message.parameters + + # TODO: Use job_id - will be relevant for the MongoDB to assign job status + # Note to peer: We should encapsulate database related actions to keep this method simple + job_id = processing_message.job_id + + if processing_message.result_queue: + self.log.warning(f"Publishing results to a message queue from the Processing Worker is not supported yet") + # TODO: Currently, no caching is performed. if self.processor_class: self.log.debug(f"Invoking the pythonic processor: {self.processor_name}") diff --git a/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py b/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py index 4a83310e4..295ab3bbc 100644 --- a/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py +++ b/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py @@ -1,8 +1,14 @@ # Check here for more details: Message structure #139 +from __future__ import annotations from datetime import datetime +from pickle import dumps, loads from typing import Any, Dict, List +# TODO: Maybe there is a more compact way to achieve the serialization/deserialization? +# Using ProtocolBuffers should decrease the size of the messages in bytes. +# It should be considered once we have a basic running prototype. + class OcrdProcessingMessage: def __init__( self, @@ -47,6 +53,26 @@ def __init__( # TODO: Implement the validator checks, e.g., # if the processor name matches the expected regex + @staticmethod + def encode(ocrd_processing_message: OcrdProcessingMessage) -> bytes: + return dumps(ocrd_processing_message) + + @staticmethod + def decode(ocrd_processing_message: bytes, encoding="utf-8") -> OcrdProcessingMessage: + data = loads(ocrd_processing_message, encoding=encoding) + return OcrdProcessingMessage( + job_id=data.job_id, + processor_name=data.processor_name, + created_time=data.created_time, + path_to_mets=data.path_to_mets, + workspace_id=data.workspace_id, + input_file_grps=data.input_file_grps, + output_file_grps=data.output_file_grps, + page_id=data.page_id, + parameters=data.parameters, + result_queue_name=data.result_queue_name + ) + class OcrdResultMessage: def __init__(self, job_id: str, status: str, workspace_id: str, path_to_mets: str): @@ -55,3 +81,17 @@ def __init__(self, job_id: str, status: str, workspace_id: str, path_to_mets: st # Either of these two below self.workspace_id = workspace_id self.path_to_mets = path_to_mets + + @staticmethod + def encode(ocrd_result_message: OcrdResultMessage) -> bytes: + return dumps(ocrd_result_message) + + @staticmethod + def decode(ocrd_result_message: bytes, encoding="utf-8") -> OcrdResultMessage: + data = loads(ocrd_result_message, encoding=encoding) + return OcrdResultMessage( + job_id=data.job_id, + status=data.status, + workspace_id=data.workspace_id, + path_to_mets=data.path_to_mets + ) From 2a5d442baad7f9014f24f3bd1811c50f6e34b817 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Wed, 18 Jan 2023 22:28:31 +0100 Subject: [PATCH 073/226] default message constructing methods --- ocrd/ocrd/network/helpers.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/ocrd/ocrd/network/helpers.py b/ocrd/ocrd/network/helpers.py index 7dda00a21..4d8223651 100644 --- a/ocrd/ocrd/network/helpers.py +++ b/ocrd/ocrd/network/helpers.py @@ -1,6 +1,11 @@ from typing import Tuple from re import split +from ocrd.network.rabbitmq_utils import ( + OcrdProcessingMessage, + OcrdResultMessage +) + def verify_and_build_database_url(mongodb_address: str, database_prefix: str = "mongodb://") -> str: elements = mongodb_address.split(':', 1) @@ -21,3 +26,27 @@ def verify_and_parse_rabbitmq_addr(rabbitmq_address: str) -> Tuple[str, int, str # Handle the case with default virtual host rmq_vhost = elements[2] if elements[2] else '/' return rmq_host, rmq_port, rmq_vhost + + +def construct_dummy_processing_message() -> OcrdProcessingMessage: + return OcrdProcessingMessage( + job_id="dummy-job-id", + processor_name="ocrd-dummy", + created_time=None, # Auto generated if None + path_to_mets="/home/mm/Desktop/ws_example/mets.xml", + workspace_id=None, # Not required, workspace is not uploaded through the Workspace Server + input_file_grps=["DEFAULT"], + output_file_grps=["DUMMY-OUTPUT"], + page_id="PHYS0001..PHYS0003", # Process only the first 3 pages + parameters={}, + result_queue_name=None # Not implemented yet, do not set + ) + + +def construct_dummy_result_message() -> OcrdResultMessage: + return OcrdResultMessage( + job_id="dummy-job_id", + status="RUNNING", + workspace_id="dummy-workspace_id", + path_to_mets="/home/mm/Desktop/ws_example/mets.xml" + ) From 7912ae3e8e3ef66fa9992fd8c1a080503e12ab3e Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Wed, 18 Jan 2023 22:50:09 +0100 Subject: [PATCH 074/226] Append default global vhost, when not provided --- ocrd/ocrd/network/helpers.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/ocrd/ocrd/network/helpers.py b/ocrd/ocrd/network/helpers.py index 4d8223651..f29228918 100644 --- a/ocrd/ocrd/network/helpers.py +++ b/ocrd/ocrd/network/helpers.py @@ -19,13 +19,19 @@ def verify_and_build_database_url(mongodb_address: str, database_prefix: str = " def verify_and_parse_rabbitmq_addr(rabbitmq_address: str) -> Tuple[str, int, str]: elements = split(pattern=r':|/', string=rabbitmq_address) - if len(elements) != 3: - raise ValueError("The RabbitMQ address is in wrong format") - rmq_host = elements[0] - rmq_port = int(elements[1]) - # Handle the case with default virtual host - rmq_vhost = elements[2] if elements[2] else '/' - return rmq_host, rmq_port, rmq_vhost + if len(elements) == 3: + rmq_host = elements[0] + rmq_port = int(elements[1]) + rmq_vhost = f"/{elements[2]}" + return rmq_host, rmq_port, rmq_vhost + + if len(elements) == 2: + rmq_host = elements[0] + rmq_port = int(elements[1]) + rmq_vhost = "/" # The default global vhost + return rmq_host, rmq_port, rmq_vhost + + raise ValueError("The RabbitMQ address is in wrong format. Expected format: {host}:{port}/{vhost}") def construct_dummy_processing_message() -> OcrdProcessingMessage: From fb119531b9c376c74e65b306330e1d92630f7139 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Wed, 18 Jan 2023 23:23:31 +0100 Subject: [PATCH 075/226] database prefix should be passed as an argument --- ocrd/ocrd/cli/processing_worker.py | 2 +- ocrd/ocrd/network/helpers.py | 11 +++++++++-- ocrd/ocrd/network/processing_worker.py | 4 ++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/ocrd/ocrd/cli/processing_worker.py b/ocrd/ocrd/cli/processing_worker.py index 8643bc098..3f54713a9 100644 --- a/ocrd/ocrd/cli/processing_worker.py +++ b/ocrd/ocrd/cli/processing_worker.py @@ -22,7 +22,7 @@ default="localhost:5672/", help='The host, port, and virtual host of the RabbitMQ Server') @click.option('-d', '--database', - default="localhost:27018", + default="mongodb://localhost:27018", help='The host and port of the MongoDB') def processing_worker_cli(processor_name: str, queue: str, database: str): """ diff --git a/ocrd/ocrd/network/helpers.py b/ocrd/ocrd/network/helpers.py index f29228918..41e1cb438 100644 --- a/ocrd/ocrd/network/helpers.py +++ b/ocrd/ocrd/network/helpers.py @@ -7,8 +7,15 @@ ) -def verify_and_build_database_url(mongodb_address: str, database_prefix: str = "mongodb://") -> str: - elements = mongodb_address.split(':', 1) +def verify_database_url(mongodb_address: str) -> str: + database_prefix = "mongodb://" + if not mongodb_address.startswith(database_prefix): + error_msg = f"The database address must start with a prefix: {database_prefix}" + raise ValueError(error_msg) + + address_without_prefix = mongodb_address[len(database_prefix):] + print(f"Address without prefix: {address_without_prefix}") + elements = address_without_prefix.split(':', 1) if len(elements) != 2: raise ValueError("The database address is in wrong format") db_host = elements[0] diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index 099c269ee..8023ed511 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -15,7 +15,7 @@ from ocrd_utils import getLogger from ocrd.processor.helpers import run_cli, run_processor from ocrd.network.helpers import ( - verify_and_build_database_url, + verify_database_url, verify_and_parse_rabbitmq_addr ) from ocrd.network.rabbitmq_utils import ( @@ -30,7 +30,7 @@ def __init__(self, rabbitmq_addr, mongodb_addr, processor_name, ocrd_tool: dict, self.log = getLogger(__name__) try: - self.db_url = verify_and_build_database_url(mongodb_addr, database_prefix="mongodb://") + self.db_url = verify_database_url(mongodb_addr) self.log.debug(f"Verified MongoDB URL: {self.db_url}") self.rmq_host, self.rmq_port, self.rmq_vhost = verify_and_parse_rabbitmq_addr(rabbitmq_addr) self.log.debug(f"Verified RabbitMQ Server URL: {self.rmq_host}:{self.rmq_port}{self.rmq_vhost}") From e93d7022749baa9f4b74464ee24b7d6f9d06d0e8 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Wed, 18 Jan 2023 23:54:41 +0100 Subject: [PATCH 076/226] Fix --queue and --database, not flags --- ocrd/ocrd/decorators/ocrd_cli_options.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ocrd/ocrd/decorators/ocrd_cli_options.py b/ocrd/ocrd/decorators/ocrd_cli_options.py index 60a177396..2b512d64b 100644 --- a/ocrd/ocrd/decorators/ocrd_cli_options.py +++ b/ocrd/ocrd/decorators/ocrd_cli_options.py @@ -26,10 +26,8 @@ def cli(mets_url): option('-O', '--output-file-grp', help='File group(s) used as output.', default='OUTPUT'), option('-g', '--page-id', help="ID(s) of the pages to process"), option('--overwrite', help="Overwrite the output file group or a page range (--page-id)", is_flag=True, default=False), - option('--queue', help="The RabbitMQ server address in format: {host}:{port}/{vhost}", - is_flag=True, type=STRING), - option('--database', help="The MongoDB address in format: {host}:{port}. `mongodb://` prefix is auto appended.", - is_flag=True, type=STRING), + option('--queue', help="The RabbitMQ server address in format: {host}:{port}/{vhost}", type=STRING), + option('--database', help="The MongoDB address in format: mongodb://{host}:{port}", type=STRING), option('-C', '--show-resource', help='Dump the content of processor resource RESNAME', metavar='RESNAME'), option('-L', '--list-resources', is_flag=True, default=False, help='List names of processor resources'), parameter_option, From 952e6e9e767744dcc56e97c73f2f8a8c0f229913 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Wed, 18 Jan 2023 23:56:36 +0100 Subject: [PATCH 077/226] Add remarks about start_*_processor methods --- ocrd/ocrd/network/deployer.py | 44 +++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index 83ba9c3cb..331ea3b34 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -123,16 +123,18 @@ def _deploy_processing_worker(self, processor: ProcessorConfig, host: HostConfig pid = self.start_native_processor( client=host.ssh_client, processor_name=processor.name, - _queue_url=rabbitmq_url, - _database_url=mongodb_url) + queue_url=rabbitmq_url, + database_url=mongodb_url + ) processor.add_started_pid(pid) elif processor.deploy_type == DeployType.docker: assert host.docker_client # to satisfy mypy pid = self.start_docker_processor( client=host.docker_client, processor_name=processor.name, - _queue_url=rabbitmq_url, - _database_url=mongodb_url) + queue_url=rabbitmq_url, + database_url=mongodb_url + ) processor.add_started_pid(pid) else: # TODO: Weirdly there is a duplication of code inside this method @@ -286,12 +288,21 @@ def kill_processing_worker(self, host: HostConfig) -> None: pass processor.pids = [] - # TODO: This method may not fit anymore. Should be further investigated. - @staticmethod - def start_native_processor(client: SSHClient, processor_name: str, _queue_url: str, - _database_url: str) -> str: - log = getLogger(__name__) - log.debug(f'Starting native processor: {processor_name}') + # Note: Invoking a pythonic processor is slightly different from the description in the spec. + # In order to achieve the exact spec call all ocr-d processors should be refactored... + # TODO: To deploy a processing worker (i.e. an ocr-d processor): + # 1. Invoke pythonic processor: + # ` --queue= --database= + # Omit the `processing-worker` argument. + # 2. Invoke non-pythonic processor: + # `ocrd processing-worker --queue= --database=` + # E.g., olena-binarize + + def start_native_processor(self, client: SSHClient, + processor_name: str, queue_url: str, database_url: str) -> str: + self.log.debug(f'Starting native processor: {processor_name}') + # TODO: queue_url and database_url are ready to be used + self.log.debug(f"The processor connects to queue: {queue_url} and mongodb: {database_url}") channel = client.invoke_shell() stdin, stdout = channel.makefile('wb'), channel.makefile('rb') # TODO: add real command here to start processing server here @@ -312,13 +323,12 @@ def start_native_processor(client: SSHClient, processor_name: str, _queue_url: s # error if try to call) return re_search(r'xyz([0-9]+)xyz', output).group(1) - # TODO: This method may not fit anymore. Should be further investigated. - @staticmethod - def start_docker_processor(client: CustomDockerClient, processor_name: str, _queue_url: str, - _database_url: str) -> str: - log = getLogger(__name__) - log.debug(f'Starting docker container processor: {processor_name}') + def start_docker_processor(self, client: CustomDockerClient, + processor_name: str, queue_url: str, database_url: str) -> str: + self.log.debug(f'Starting docker container processor: {processor_name}') + # TODO: queue_url and database_url are ready to be used + self.log.debug(f"The processor connects to queue: {queue_url} and mongodb: {database_url}") # TODO: add real command here to start processing server here res = client.containers.run('debian', 'sleep 500s', detach=True, remove=True) - assert res and res.id, 'run processor in docker-container failed' + assert res and res.id, f'Running processor: {processor_name} in docker-container failed' return res.id From 43050ab8db40a758028a1d1f34e105fa53efb249 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Thu, 19 Jan 2023 01:07:14 +0100 Subject: [PATCH 078/226] Provide a basic testing endpoint to processing broker (aka server) --- ocrd/ocrd/network/processing_broker.py | 28 ++++++++++++++----- .../network/rabbitmq_utils/ocrd_messages.py | 2 +- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/ocrd/ocrd/network/processing_broker.py b/ocrd/ocrd/network/processing_broker.py index 633be6564..2327af715 100644 --- a/ocrd/ocrd/network/processing_broker.py +++ b/ocrd/ocrd/network/processing_broker.py @@ -1,4 +1,4 @@ -from fastapi import FastAPI +from fastapi import FastAPI, status import uvicorn from time import sleep from yaml import safe_load @@ -9,7 +9,8 @@ from ocrd.network.deployer import Deployer from ocrd.network.deployment_config import ProcessingBrokerConfig -from ocrd.network.rabbitmq_utils import RMQPublisher +from ocrd.network.rabbitmq_utils import RMQPublisher, OcrdProcessingMessage +from ocrd.network.helpers import construct_dummy_processing_message class ProcessingBroker(FastAPI): @@ -57,10 +58,12 @@ def __init__(self, config_path: str, host: str, port: int) -> None: # TODO: add response model? add a response body at all? ) - # TODO: Call this after the rest of the API is implemented - # Example of publishing a message inside a specific queue - # The message type is bytes - # self.rmq_publisher.publish_to_queue(queue_name="queue_name", message="message") + self.router.add_api_route( + path='/test-dummy', + endpoint=self.publish_default_processing_message, + methods=['POST'], + status_code=status.HTTP_202_ACCEPTED + ) def start(self) -> None: """ @@ -97,7 +100,7 @@ def start(self) -> None: # Note: A deployed processing worker starts listening to a message queue with id processor.name self.deployer.deploy_hosts(self.hosts_config, rabbitmq_url, mongodb_url) - self.log.debug(f'Starting uvicorn: {self.host}:{self.port}') + self.log.debug(f'Starting uvicorn: {self.hostname}:{self.port}') uvicorn.run(self, host=self.hostname, port=self.port) @staticmethod @@ -153,3 +156,14 @@ def create_message_queues(self): self.log.debug(f"Creating a message queue with id: {processor.name}") # TODO: We may want to track here if there are already queues with the same name self.rmq_publisher.create_queue(queue_name=processor.name) + + def publish_default_processing_message(self): + processing_message = construct_dummy_processing_message() + queue_name = processing_message.processor_name + encoded_processing_message = OcrdProcessingMessage.encode(processing_message) + if self.rmq_publisher: + self.log.debug("Publishing the default processing message") + self.rmq_publisher.publish_to_queue(queue_name=queue_name, message=encoded_processing_message) + else: + self.log.error("RMQPublisher is not connected") + raise Exception("RMQPublisher is not connected") diff --git a/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py b/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py index 295ab3bbc..caaf0aef2 100644 --- a/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py +++ b/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py @@ -70,7 +70,7 @@ def decode(ocrd_processing_message: bytes, encoding="utf-8") -> OcrdProcessingMe output_file_grps=data.output_file_grps, page_id=data.page_id, parameters=data.parameters, - result_queue_name=data.result_queue_name + result_queue_name=data.result_queue ) From ae18c080f0b249dd6023699dac676ba5bed9fb87 Mon Sep 17 00:00:00 2001 From: joschrew Date: Fri, 20 Jan 2023 10:04:18 +0100 Subject: [PATCH 079/226] Make address param for processing broker required --- ocrd/ocrd/cli/processing_broker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ocrd/ocrd/cli/processing_broker.py b/ocrd/ocrd/cli/processing_broker.py index 51a95a5b7..4fa70edb9 100644 --- a/ocrd/ocrd/cli/processing_broker.py +++ b/ocrd/ocrd/cli/processing_broker.py @@ -13,7 +13,7 @@ @click.command('processing-broker') @click.argument('path_to_config', required=True, type=click.STRING) -@click.option('-a', '--address', help='Host name/IP, port to bind the Processing-Broker to') +@click.option('-a', '--address', help='Host (name/IP) and port to bind the Processing-Broker to. Example: localhost:8080', required=True) def processing_broker_cli(path_to_config, address: str): """ Start and manage processing servers (workers) with the processing broker From fdfaf8d8832f4171b9388153d13dee81a7971096 Mon Sep 17 00:00:00 2001 From: joschrew Date: Fri, 20 Jan 2023 15:26:01 +0100 Subject: [PATCH 080/226] Use credentials from config for rabbitmq messaging The used rabbitmq image does not have preconfigured credentials. This would only be possible when building the image ourselfs. To avoid that for now the default credentials are used for messaging (currently only to publish but consume will follow). --- ocrd/ocrd/network/deployer.py | 21 +++++++-------------- ocrd/ocrd/network/processing_broker.py | 9 +++++---- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index 331ea3b34..05ccfad87 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -56,16 +56,6 @@ def __init__(self, queue_config: QueueConfig, mongo_config: MongoConfig, hosts_c # Currently, the connections (ssh_client, docker_client) and # the PIDs are stored inside the config data classes - # Avoid using this method - # TODO: Should be removed - def deploy_all(self) -> None: - """ Deploy the message queue and all processors defined in the config-file - """ - # The order of deploying may be important to recover from previous state - rabbitmq_url = self.deploy_rabbitmq() - mongodb_url = self.deploy_mongodb() - self.deploy_hosts(self.hosts, rabbitmq_url, mongodb_url) - def kill_all(self) -> None: # The order of killing is important to optimize graceful shutdown in the future # If RabbitMQ server is killed before killing Processing Workers, that may have @@ -142,8 +132,8 @@ def _deploy_processing_worker(self, processor: ProcessorConfig, host: HostConfig self.log.error(f"Failed to deploy: {processor.name}. The deploy type is unknown.") pass - def deploy_rabbitmq(self, image: str = 'rabbitmq', detach: bool = True, remove: bool = True, - ports_mapping: Union[Dict, None] = None) -> str: + def deploy_rabbitmq(self, image: str = 'rabbitmq:3-management', detach: bool = True, + remove: bool = True, ports_mapping: Union[Dict, None] = None) -> str: # Note for a peer # This method deploys the RabbitMQ Server. # Handling of creation of queues, submitting messages to queues, @@ -169,12 +159,15 @@ def deploy_rabbitmq(self, image: str = 'rabbitmq', detach: bool = True, remove: 25672: 25672 } self.log.debug(f"Ports mapping: {ports_mapping}") - # TODO: use rm here or not? Should queues be reused? res = client.containers.run( image=image, detach=detach, remove=remove, - ports=ports_mapping + ports=ports_mapping, + environment=[ + f'RABBITMQ_DEFAULT_USER={self.mq_data.credentials[0]}', + f'RABBITMQ_DEFAULT_PASS={self.mq_data.credentials[1]}' + ] ) assert res and res.id, \ f'Failed to start RabbitMQ docker container on host: {self.mq_data.address}' diff --git a/ocrd/ocrd/network/processing_broker.py b/ocrd/ocrd/network/processing_broker.py index 2327af715..c2e4664cd 100644 --- a/ocrd/ocrd/network/processing_broker.py +++ b/ocrd/ocrd/network/processing_broker.py @@ -71,10 +71,10 @@ def start(self) -> None: """ """ Note for a peer: - Deploying everything together at once is a bad approach. First the RabbitMQ Server and the MongoDB - should be deployed. Then the RMQPublisher of the Processing Broker (aka Processing Server) should + Deploying everything together at once is a bad approach. First the RabbitMQ Server and the MongoDB + should be deployed. Then the RMQPublisher of the Processing Broker (aka Processing Server) should connect to the running RabbitMQ server. After that point the Processing Workers should be deployed. - The RMQPublisher should be connected before deploying Processing Workers because the message queues to + The RMQPublisher should be connected before deploying Processing Workers because the message queues to which the Processing Workers listen to are created based on the deployed processor. """ # Deploy everything specified in the configuration @@ -91,7 +91,8 @@ def start(self) -> None: sleep(3) # TODO: Sleeping here is bad and better check should be performed # The RMQPublisher is initialized and a connection to the RabbitMQ is performed - self.connect_publisher() + self.connect_publisher(username=self.queue_config.credentials[0], + password=self.queue_config.credentials[1]) self.log.debug(f"Starting to create message queues on RabbitMQ instance url: {rabbitmq_url}") self.create_message_queues() From f387f92d839391110456dd0833378affe03b3e98 Mon Sep 17 00:00:00 2001 From: joschrew Date: Fri, 20 Jan 2023 15:55:31 +0100 Subject: [PATCH 081/226] Add new parameters to processor help output --- ocrd/ocrd/processor/helpers.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ocrd/ocrd/processor/helpers.py b/ocrd/ocrd/processor/helpers.py index e07333eaa..ae54c9843 100644 --- a/ocrd/ocrd/processor/helpers.py +++ b/ocrd/ocrd/processor/helpers.py @@ -93,9 +93,9 @@ def run_processor( mem_usage = memory_usage(proc=processor.process, # only run process once max_iterations=1, - interval=.1, timeout=None, timestamps=True, + interval=.1, timeout=None, timestamps=True, # include sub-processes - multiprocess=True, include_children=True, + multiprocess=True, include_children=True, # get proportional set size instead of RSS backend=backend) mem_usage_values = [mem for mem, _ in mem_usage] @@ -184,7 +184,7 @@ def run_cli( def generate_processor_help(ocrd_tool, processor_instance=None): """Generate a string describing the full CLI of this processor including params. - + Args: ocrd_tool (dict): this processor's ``tools`` section of the module's ``ocrd-tool.json`` processor_instance (object, optional): the processor implementation @@ -233,6 +233,8 @@ def wrap(s): -g, --page-id ID Physical page ID(s) to process --overwrite Remove existing output pages/images (with --page-id, remove only those) + --queue The RabbitMQ server address in format: {host}:{port}/{vhost}" + --database The MongoDB address in format: mongodb://{host}:{port}" --profile Enable profiling --profile-file Write cProfile stats to this file. Implies --profile -p, --parameter JSON-PATH Parameters, either verbatim JSON string From d35d0067eb8d8379ab40aea29a3fa2546d59f184 Mon Sep 17 00:00:00 2001 From: joschrew Date: Fri, 20 Jan 2023 16:20:38 +0100 Subject: [PATCH 082/226] Improve error message for processing worker --- ocrd/ocrd/cli/processing_worker.py | 2 +- ocrd/ocrd/decorators/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ocrd/ocrd/cli/processing_worker.py b/ocrd/ocrd/cli/processing_worker.py index 3f54713a9..f3beacc8e 100644 --- a/ocrd/ocrd/cli/processing_worker.py +++ b/ocrd/ocrd/cli/processing_worker.py @@ -54,4 +54,4 @@ def processing_worker_cli(processor_name: str, queue: str, database: str): # Start consuming from the queue with name `processor_name` processing_worker.start_consuming() except Exception as e: - raise Exception(f"Processing worker has failed with error: {e}") + raise Exception("Processing worker has failed with error") from e diff --git a/ocrd/ocrd/decorators/__init__.py b/ocrd/ocrd/decorators/__init__.py index b54ec59cf..376f33bf6 100644 --- a/ocrd/ocrd/decorators/__init__.py +++ b/ocrd/ocrd/decorators/__init__.py @@ -88,7 +88,7 @@ def ocrd_cli_wrap_processor( # Start consuming from the queue with name `processor_name` processing_worker.start_consuming() except Exception as e: - raise Exception(f"Processing worker has failed with error: {e}") + raise Exception("Processing worker has failed with error") from e else: initLogging() LOG = getLogger('ocrd_cli_wrap_processor') From beca781ae483839d88bc1c9fcfc1aacc35d1b3b2 Mon Sep 17 00:00:00 2001 From: joschrew Date: Mon, 23 Jan 2023 09:13:06 +0100 Subject: [PATCH 083/226] Insert credentials on rabbitmq startup Credentials are needed to publish and consume messages. We currently use default-credenials for publisher and consumer which must be added to rabbitmq on startup --- ocrd/ocrd/network/deployer.py | 15 +++++++++++++++ ocrd/ocrd/network/processing_broker.py | 7 +------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index 05ccfad87..d629eeefc 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -13,6 +13,7 @@ DeployType, ) from ocrd.network.processing_worker import ProcessingWorker +import time # Abstraction of the Deployment functionality # The ProcessingServer (currently still called Broker) provides the configuration parameters to the Deployer agent. @@ -172,6 +173,7 @@ def deploy_rabbitmq(self, image: str = 'rabbitmq:3-management', detach: bool = T assert res and res.id, \ f'Failed to start RabbitMQ docker container on host: {self.mq_data.address}' self.mq_data.pid = res.id + self.init_rabbitmq(client, res.id) client.close() # Build the RabbitMQ Server URL to return @@ -183,6 +185,19 @@ def deploy_rabbitmq(self, image: str = 'rabbitmq:3-management', detach: bool = T self.log.debug(f"The RabbitMQ server was deployed on url: {rabbitmq_url}") return rabbitmq_url + def init_rabbitmq(self, client: CustomDockerClient, rabbitmq_id: str): + """ Add users, wait for startup + """ + # TODO: rabbitmq was started, but needs some time to be ready to use. sleep is working for + # now but the init-process should rather be made in a loop and be redone on error + # until the container is responsive or a timeout is expired (and raise in this case) + time.sleep(3) + container = client.containers.get(rabbitmq_id) + container.exec_run("rabbitmqctl add_user default-publisher default-publisher") + container.exec_run("rabbitmqctl add_user default-consumer default-consumer") + + + def deploy_mongodb(self, image: str = 'mongo', detach: bool = True, remove: bool = True, ports_mapping: Union[Dict, None] = None) -> str: self.log.debug(f"Trying to deploy image[{image}], with modes: detach[{detach}], remove[{remove}]") diff --git a/ocrd/ocrd/network/processing_broker.py b/ocrd/ocrd/network/processing_broker.py index c2e4664cd..0c7a046cb 100644 --- a/ocrd/ocrd/network/processing_broker.py +++ b/ocrd/ocrd/network/processing_broker.py @@ -86,13 +86,8 @@ def start(self) -> None: # Deploy the MongoDB, get the URL of the deployed agent mongodb_url = self.deployer.deploy_mongodb() - # Give enough time for the RabbitMQ server to get deployed and get fully configured - # Needed to prevent connection of the publisher before the RabbitMQ is deployed - sleep(3) # TODO: Sleeping here is bad and better check should be performed - # The RMQPublisher is initialized and a connection to the RabbitMQ is performed - self.connect_publisher(username=self.queue_config.credentials[0], - password=self.queue_config.credentials[1]) + self.connect_publisher() self.log.debug(f"Starting to create message queues on RabbitMQ instance url: {rabbitmq_url}") self.create_message_queues() From f2c3e6edaccec09daa2bbfa4e56e2e6a7bae149e Mon Sep 17 00:00:00 2001 From: joschrew Date: Mon, 23 Jan 2023 17:35:50 +0100 Subject: [PATCH 084/226] Change message exchange from pickle to str for now Pickle may be/is probably the right choice to exchange the messages but for testing string is better. Because that way the processing messages can be written easily to the queue. --- ocrd/ocrd/network/processing_broker.py | 3 ++- ocrd/ocrd/network/processing_worker.py | 3 ++- .../network/rabbitmq_utils/ocrd_messages.py | 26 +++++++++++++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/ocrd/ocrd/network/processing_broker.py b/ocrd/ocrd/network/processing_broker.py index 0c7a046cb..46fc340c9 100644 --- a/ocrd/ocrd/network/processing_broker.py +++ b/ocrd/ocrd/network/processing_broker.py @@ -156,7 +156,8 @@ def create_message_queues(self): def publish_default_processing_message(self): processing_message = construct_dummy_processing_message() queue_name = processing_message.processor_name - encoded_processing_message = OcrdProcessingMessage.encode(processing_message) + # TODO: switch back to pickle?! + encoded_processing_message = OcrdProcessingMessage.encode_yml(processing_message) if self.rmq_publisher: self.log.debug("Publishing the default processing message") self.rmq_publisher.publish_to_queue(queue_name=queue_name, message=encoded_processing_message) diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index 8023ed511..1394aea3d 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -77,7 +77,8 @@ def on_consumed_message(self, channel, delivery, properties, body) -> None: try: self.log.debug(f"Trying to decode processing message with tag: {delivery_tag}") - processing_message: OcrdProcessingMessage = OcrdProcessingMessage.decode(body, encoding="utf-8") + # TODO: switch back to pickle?! + processing_message: OcrdProcessingMessage = OcrdProcessingMessage.decode_yml(body) except Exception as e: self.log.error(f"Failed to decode processing message body: {body}") self.log.error(f"Nacking processing message with tag: {delivery_tag}") diff --git a/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py b/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py index caaf0aef2..7d61d4a84 100644 --- a/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py +++ b/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py @@ -3,6 +3,7 @@ from datetime import datetime from pickle import dumps, loads from typing import Any, Dict, List +import yaml # TODO: Maybe there is a more compact way to achieve the serialization/deserialization? @@ -73,6 +74,31 @@ def decode(ocrd_processing_message: bytes, encoding="utf-8") -> OcrdProcessingMe result_queue_name=data.result_queue ) + @staticmethod + def encode_yml(ocrd_processing_message: OcrdProcessingMessage) -> bytes: + """convert OcrdProcessingMessage to yml + """ + return yaml.dump(ocrd_processing_message.__dict__, indent=2).encode('utf-8') + + @staticmethod + def decode_yml(ocrd_processing_message: bytes) -> OcrdProcessingMessage: + """Parse OcrdProcessingMessage from yml + """ + msg = ocrd_processing_message.decode('utf-8') + data = yaml.load(msg, Loader=yaml.Loader) + return OcrdProcessingMessage( + job_id=data.get("job_id", None), + processor_name=data.get("processor_name", None), + created_time=data.get("created_time", None), + path_to_mets=data.get("path_to_mets", None), + workspace_id=data.get("workspace_id", None), + input_file_grps=data.get("input_file_grps", None), + output_file_grps=data.get("output_file_grps", None), + page_id=data.get("page_id", None), + parameters=data.get("parameters", None), + result_queue_name=data.get("result_queue", None), + ) + class OcrdResultMessage: def __init__(self, job_id: str, status: str, workspace_id: str, path_to_mets: str): From 2f045b3c418e3e973d11a6a9b05a939ee73fda06 Mon Sep 17 00:00:00 2001 From: joschrew Date: Tue, 24 Jan 2023 10:47:33 +0100 Subject: [PATCH 085/226] Change rabbitmq default credentials Previously there were 'default-publisher' and 'default-consumer'. I changed that to 'default'. Reason: For the rabbitmq no custom Dockerfile is used. So users have to be added through rabbitmqctl via docker-exec. This raises error if too many invocations happen. Maybe misuse of the exec commands or bug in paramiko/docker-sdk is the cause. But for now the easiest way to circumvent is to just reduce the invocation count. --- ocrd/ocrd/network/deployer.py | 7 +++---- ocrd/ocrd/network/processing_broker.py | 2 +- ocrd/ocrd/network/processing_worker.py | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index d629eeefc..5668d8fe7 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -193,10 +193,9 @@ def init_rabbitmq(self, client: CustomDockerClient, rabbitmq_id: str): # until the container is responsive or a timeout is expired (and raise in this case) time.sleep(3) container = client.containers.get(rabbitmq_id) - container.exec_run("rabbitmqctl add_user default-publisher default-publisher") - container.exec_run("rabbitmqctl add_user default-consumer default-consumer") - - + container.exec_run('rabbitmqctl add_user default default') + container.exec_run('rabbitmqctl set_user_tags default administrator') + container.exec_run('rabbitmqctl set_permissions -p / default ".*" ".*" ".*"') def deploy_mongodb(self, image: str = 'mongo', detach: bool = True, remove: bool = True, ports_mapping: Union[Dict, None] = None) -> str: diff --git a/ocrd/ocrd/network/processing_broker.py b/ocrd/ocrd/network/processing_broker.py index 46fc340c9..4c2b9c3ef 100644 --- a/ocrd/ocrd/network/processing_broker.py +++ b/ocrd/ocrd/network/processing_broker.py @@ -130,7 +130,7 @@ async def on_shutdown(self) -> None: async def stop_deployed_agents(self) -> None: self.deployer.kill_all() - def connect_publisher(self, username="default-publisher", password="default-publisher", enable_acks=True): + def connect_publisher(self, username="default", password="default", enable_acks=True): self.log.debug(f"Connecting RMQPublisher to RabbitMQ server: {self.rmq_host}:{self.rmq_port}{self.rmq_vhost}") self.rmq_publisher = RMQPublisher(host=self.rmq_host, port=self.rmq_port, vhost=self.rmq_vhost) # TODO: Remove this information before the release diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index 1394aea3d..f29d04be9 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -53,7 +53,7 @@ def __init__(self, rabbitmq_addr, mongodb_addr, processor_name, ocrd_tool: dict, # the message queue with name {processor_name}-result, the RMQPublisher should be instantiated # self.rmq_publisher = None - def connect_consumer(self, username="default-consumer", password="default-consumer"): + def connect_consumer(self, username="default", password="default"): self.log.debug(f"Connecting RMQConsumer to RabbitMQ server: {self.rmq_host}:{self.rmq_port}{self.rmq_vhost}") self.rmq_consumer = RMQConsumer(host=self.rmq_host, port=self.rmq_port, vhost=self.rmq_vhost) # TODO: Remove this information before the release From d1c1f50ab0f21444b58fb2de7fa18f2751e87012 Mon Sep 17 00:00:00 2001 From: joschrew Date: Tue, 24 Jan 2023 10:56:58 +0100 Subject: [PATCH 086/226] Add optional path_to_bin_dir to config and use it It is asumed that on the targed machines all required ocrd-processors are available in PATH to be called. At least for testing (and I think for 'real' usage later too) it is much easier to create a venv with ocrd to be used. Now with the path to this venv-bin-dir (dir where all executables of venv can be found) it is possible to use it for the processing-workers --- ocrd/ocrd/network/deployer.py | 50 ++++++++++++------- ocrd/ocrd/network/deployment_config.py | 1 + .../ocrd_validators/config.schema.yml | 3 ++ 3 files changed, 36 insertions(+), 18 deletions(-) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index 5668d8fe7..aa2db675f 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -14,6 +14,7 @@ ) from ocrd.network.processing_worker import ProcessingWorker import time +from pathlib import Path # Abstraction of the Deployment functionality # The ProcessingServer (currently still called Broker) provides the configuration parameters to the Deployer agent. @@ -115,7 +116,8 @@ def _deploy_processing_worker(self, processor: ProcessorConfig, host: HostConfig client=host.ssh_client, processor_name=processor.name, queue_url=rabbitmq_url, - database_url=mongodb_url + database_url=mongodb_url, + bin_dir=host.binpath, ) processor.add_started_pid(pid) elif processor.deploy_type == DeployType.docker: @@ -305,30 +307,42 @@ def kill_processing_worker(self, host: HostConfig) -> None: # `ocrd processing-worker --queue= --database=` # E.g., olena-binarize - def start_native_processor(self, client: SSHClient, - processor_name: str, queue_url: str, database_url: str) -> str: + def start_native_processor(self, client: SSHClient, processor_name: str, queue_url: str, + database_url: str, bin_dir: str = None) -> str: + """ start a processor natively on a host via ssh + + Args: + client: paramiko SSHClient to execute commands on a host + processor_name: name of processor to run + queue_url: url to rabbitmq + database_url: url to database + bin_dir (optional): path to where processor executables can be found + + Returns: + str: pid of running process + """ + # TODO: some processors are bashlib. They have to be started differently. Open Question: + # how to find out if a processor is bashlib self.log.debug(f'Starting native processor: {processor_name}') - # TODO: queue_url and database_url are ready to be used - self.log.debug(f"The processor connects to queue: {queue_url} and mongodb: {database_url}") + self.log.debug(f'The processor connects to queue: {queue_url} and mongodb: {database_url}') channel = client.invoke_shell() stdin, stdout = channel.makefile('wb'), channel.makefile('rb') - # TODO: add real command here to start processing server here - cmd = 'sleep 23s' - # the only way to make it work to start a process in the background and return early is - # this construction. The pid of the last started background process is printed with - # `echo $!` but it is printed inbetween other output. Because of that I added `xyz` before - # and after the code to easily be able to filter out the pid via regex when returning from - # the function + if bin_dir: + path = Path(bin_dir) / processor_name + else: + path = processor_name + cmd = f"{path} --database {database_url} --queue {queue_url}" + # the only way (I could find) to make it work to start a process in the background and + # return early is this construction. The pid of the last started background process is + # printed with `echo $!` but it is printed inbetween other output. Because of that I added + # `xyz` before and after the code to easily be able to filter out the pid via regex when + # returning from the function stdin.write(f'{cmd} & \n echo xyz$!xyz \n exit \n') output = stdout.read().decode('utf-8') + self.log.debug(f"Output for processor {processor_name}: {output}") stdout.close() stdin.close() - # What does this return and is supposed to return? - # Putting some comments when using patterns is always appreciated - # Since the docker version returns PID, this should also return PID for consistency - # TODO: mypy error: ignore or fix. Problem: re.search returns Optional (can be None, causes - # error if try to call) - return re_search(r'xyz([0-9]+)xyz', output).group(1) + return re_search(r'xyz([0-9]+)xyz', output).group(1) # type: ignore def start_docker_processor(self, client: CustomDockerClient, processor_name: str, queue_url: str, database_url: str) -> str: diff --git a/ocrd/ocrd/network/deployment_config.py b/ocrd/ocrd/network/deployment_config.py index 488b8607f..f08994435 100644 --- a/ocrd/ocrd/network/deployment_config.py +++ b/ocrd/ocrd/network/deployment_config.py @@ -36,6 +36,7 @@ def __init__(self, config: dict) -> None: self.username = config['username'] self.password = config.get('password', None) self.keypath = config.get('path_to_privkey', None) + self.binpath = config.get('path_to_bin_dir', None) self.processors = [] for processor in config['deploy_processors']: deploy_type = DeployType.from_str(processor['deploy_type']) diff --git a/ocrd_validators/ocrd_validators/config.schema.yml b/ocrd_validators/ocrd_validators/config.schema.yml index f9d3294ba..3dac8bfd0 100644 --- a/ocrd_validators/ocrd_validators/config.schema.yml +++ b/ocrd_validators/ocrd_validators/config.schema.yml @@ -74,6 +74,9 @@ properties: path_to_privkey: description: Path to private key file type: string + path_to_bin_dir: + description: Path to where processor executables can be found on the target machine + type: string deploy_processors: description: List of processors which will be deployed type: array From 344e5936a58b87c2791c3886087df78510b9e49c Mon Sep 17 00:00:00 2001 From: joschrew Date: Wed, 25 Jan 2023 11:46:34 +0100 Subject: [PATCH 087/226] Add route to p-broker to start a p-worker --- ocrd/ocrd/network/helpers.py | 20 ++++++++++++ ocrd/ocrd/network/models/processor.py | 17 ++++++++++ ocrd/ocrd/network/processing_broker.py | 31 ++++++++++++++++++- .../network/rabbitmq_utils/ocrd_messages.py | 24 ++++++++++++-- 4 files changed, 88 insertions(+), 4 deletions(-) create mode 100644 ocrd/ocrd/network/models/processor.py diff --git a/ocrd/ocrd/network/helpers.py b/ocrd/ocrd/network/helpers.py index 41e1cb438..cefb21987 100644 --- a/ocrd/ocrd/network/helpers.py +++ b/ocrd/ocrd/network/helpers.py @@ -1,5 +1,8 @@ from typing import Tuple from re import split +from pathlib import Path +from os import environ +from os.path import join, exists from ocrd.network.rabbitmq_utils import ( OcrdProcessingMessage, @@ -41,6 +44,23 @@ def verify_and_parse_rabbitmq_addr(rabbitmq_address: str) -> Tuple[str, int, str raise ValueError("The RabbitMQ address is in wrong format. Expected format: {host}:{port}/{vhost}") +def get_workspaces_dir() -> str: + """get the path to the workspaces folder + + The processing-workers must have access to the workspaces. First idea is that they are provided + via nfs and allways available under $XDG_DATA_HOME/ocrd-workspaces. This function provides the + absolute path to the folder and raises a ValueError if it is not available + """ + if 'XDG_DATA_HOME' in environ: + xdg_data_home = environ['XDG_DATA_HOME'] + else: + xdg_data_home = join(environ['HOME'], '.local', 'share') + res = join(xdg_data_home, 'ocrd-workspaces') + if not exists(res): + raise ValueError('Ocrd-Workspaces directory not found. Expected \'{res}\'') + return res + + def construct_dummy_processing_message() -> OcrdProcessingMessage: return OcrdProcessingMessage( job_id="dummy-job-id", diff --git a/ocrd/ocrd/network/models/processor.py b/ocrd/ocrd/network/models/processor.py new file mode 100644 index 000000000..a6272de93 --- /dev/null +++ b/ocrd/ocrd/network/models/processor.py @@ -0,0 +1,17 @@ +from pydantic import BaseModel +from typing import Dict, Any, Optional + + +class ProcessorArgs(BaseModel): + workspace_id: str = '' + input_file_grps: str = '' + output_file_grps: str = '' + page_id: str = '' + parameters: Optional[Dict[str, Any]] = {} + + +# TODO: this does not conform to the openapi.yml. It should?! +class ProcessorJob(BaseModel): + job_id: str + workspace_id: str + processor_name: str diff --git a/ocrd/ocrd/network/processing_broker.py b/ocrd/ocrd/network/processing_broker.py index 4c2b9c3ef..bde74dde9 100644 --- a/ocrd/ocrd/network/processing_broker.py +++ b/ocrd/ocrd/network/processing_broker.py @@ -10,7 +10,9 @@ from ocrd.network.deployer import Deployer from ocrd.network.deployment_config import ProcessingBrokerConfig from ocrd.network.rabbitmq_utils import RMQPublisher, OcrdProcessingMessage -from ocrd.network.helpers import construct_dummy_processing_message +from ocrd.network.helpers import construct_dummy_processing_message, get_workspaces_dir +from ocrd.network.models.processor import ProcessorArgs, ProcessorJob +from pathlib import Path class ProcessingBroker(FastAPI): @@ -65,6 +67,15 @@ def __init__(self, config_path: str, host: str, port: int) -> None: status_code=status.HTTP_202_ACCEPTED ) + self.router.add_api_route( + path='/processor/{processor_name}', + endpoint=self.run_processor, + methods=['POST'], + tags=['processing'], + status_code=status.HTTP_200_OK, + operation_id='runProcessor', + ) + def start(self) -> None: """ deploy things and start the processing broker (aka server) with uvicorn @@ -164,3 +175,21 @@ def publish_default_processing_message(self): else: self.log.error("RMQPublisher is not connected") raise Exception("RMQPublisher is not connected") + + # TODO: how do we want to do the whole model-stuff? Webapi (openapi.yml) uses ProcessorJob + def run_processor(self, processor_name: str, p_args: ProcessorArgs) -> ProcessorJob: + # TODO: save job in mongodb and get a job-id this way. Look into how this was done in pr-884 + job_id = 'dummy-id-1' + # TODO: how do we want that stuff with the workspaces to work? + workspaces_path = get_workspaces_dir() + processing_message = OcrdProcessingMessage.from_params( + processor_name, job_id, workspaces_path, p_args + ) + encoded_processing_message = OcrdProcessingMessage.encode_yml(processing_message) + if self.rmq_publisher: + self.rmq_publisher.publish_to_queue(processor_name, encoded_processing_message) + else: + raise Exception('RMQPublisher is not connected') + + return ProcessorJob(job_id=job_id, workspace_id=p_args.workspace_id, + processor_name=processor_name) diff --git a/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py b/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py index 7d61d4a84..1e52d9260 100644 --- a/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py +++ b/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py @@ -2,8 +2,10 @@ from __future__ import annotations from datetime import datetime from pickle import dumps, loads -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional import yaml +from ocrd.network.models.processor import ProcessorArgs +from pathlib import Path # TODO: Maybe there is a more compact way to achieve the serialization/deserialization? @@ -44,11 +46,11 @@ def __init__( self.input_file_grps = input_file_grps self.output_file_grps = output_file_grps # e.g., "PHYS_0005..PHYS_0010" will process only pages between 5-10 - self.page_id = page_id + self.page_id = page_id if page_id else None # e.g., "ocrd-cis-ocropy-binarize-result" self.result_queue = result_queue_name # processor parameters - self.parameters = parameters + self.parameters = parameters if parameters else None self.created_time = created_time # TODO: Implement the validator checks, e.g., @@ -99,6 +101,22 @@ def decode_yml(ocrd_processing_message: bytes) -> OcrdProcessingMessage: result_queue_name=data.get("result_queue", None), ) + @staticmethod + def from_params(processor_name: str, job_id: str, workspaces_dir: str, p_args: ProcessorArgs + ) -> OcrdProcessingMessage: + input_file_grps = [x.strip() for x in filter(None, p_args.input_file_grps.split(','))] + output_file_grps = [x.strip() for x in filter(None, p_args.output_file_grps.split(','))] + return OcrdProcessingMessage( + job_id=job_id, + processor_name=processor_name, + path_to_mets=str(Path(workspaces_dir) / p_args.workspace_id / 'mets.xml'), + workspace_id=p_args.workspace_id, + input_file_grps=input_file_grps, + output_file_grps=output_file_grps, + page_id=p_args.page_id, + parameters=p_args.parameters, + ) + class OcrdResultMessage: def __init__(self, job_id: str, status: str, workspace_id: str, path_to_mets: str): From 77a316d8f3af3f9469193a653847a9727a67ff93 Mon Sep 17 00:00:00 2001 From: joschrew Date: Wed, 25 Jan 2023 11:48:30 +0100 Subject: [PATCH 088/226] Add typehints for a method in processing_worker --- ocrd/ocrd/network/processing_worker.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index f29d04be9..67f619726 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -11,6 +11,9 @@ import json from typing import List +import pika.spec +import pika.adapters.blocking_connection + from ocrd import Resolver from ocrd_utils import getLogger from ocrd.processor.helpers import run_cli, run_processor @@ -63,8 +66,12 @@ def connect_consumer(self, username="default", password="default"): # Define what happens every time a message is consumed # from the queue with name self.processor_name - def on_consumed_message(self, channel, delivery, properties, body) -> None: - self.log.debug(f"Received from: \nchannel: {channel},\ndelivery: {delivery}, \nproperties: {properties}, \nbody: {body}") + def on_consumed_message( + self, + channel: pika.adapters.blocking_connection.BlockingChannel, + delivery: pika.spec.Basic.Deliver, + properties: pika.spec.BasicProperties, + body: bytes) -> None: consumer_tag = delivery.consumer_tag delivery_tag: int = delivery.delivery_tag is_redelivered: bool = delivery.redelivered From e16e3d42df04129864f1b3acf33868c70d9a9cb0 Mon Sep 17 00:00:00 2001 From: joschrew Date: Wed, 25 Jan 2023 12:24:53 +0100 Subject: [PATCH 089/226] Replace double quoted strings with single quoted --- ocrd/ocrd/network/deployer.py | 60 +++++++++---------- ocrd/ocrd/network/deployment_utils.py | 4 +- ocrd/ocrd/network/helpers.py | 36 +++++------ ocrd/ocrd/network/models/job.py | 14 ++--- ocrd/ocrd/network/processing_broker.py | 28 ++++----- ocrd/ocrd/network/processing_worker.py | 58 +++++++++--------- ocrd/ocrd/network/rabbitmq_utils/__init__.py | 10 ++-- ocrd/ocrd/network/rabbitmq_utils/constants.py | 38 ++++++------ ocrd/ocrd/network/rabbitmq_utils/consumer.py | 8 +-- .../network/rabbitmq_utils/ocrd_messages.py | 32 +++++----- ocrd/ocrd/network/rabbitmq_utils/publisher.py | 12 ++-- 11 files changed, 150 insertions(+), 150 deletions(-) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index aa2db675f..f10299d18 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -75,9 +75,9 @@ def kill_all(self) -> None: self.kill_rabbitmq() def deploy_hosts(self, hosts: List[HostConfig], rabbitmq_url: str, mongodb_url: str) -> None: - self.log.debug("Starting to deploy hosts") + self.log.debug('Starting to deploy hosts') for host in hosts: - self.log.debug(f"Deploying processing workers on host: {host.address}") + self.log.debug(f'Deploying processing workers on host: {host.address}') for processor in host.processors: self._deploy_processing_worker(processor, host, rabbitmq_url, mongodb_url) # TODO: These connections, just like the PIDs, should not be kept in the config data classes @@ -92,8 +92,8 @@ def deploy_hosts(self, hosts: List[HostConfig], rabbitmq_url: str, mongodb_url: def _deploy_processing_worker(self, processor: ProcessorConfig, host: HostConfig, rabbitmq_url: str, mongodb_url: str) -> None: - self.log.debug(f'deploy "{processor.deploy_type}" processor: "{processor}" on' - f'"{host.address}"') + self.log.debug(f'deploy \'{processor.deploy_type}\' processor: \'{processor}\' on' + f'\'{host.address}\'') assert not processor.pids, 'processors already deployed. Pids are present. Host: ' \ '{host.__dict__}. Processor: {processor.__dict__}' @@ -106,7 +106,7 @@ def _deploy_processing_worker(self, processor: ProcessorConfig, host: HostConfig host.docker_client = create_docker_client(host.address, host.username, host.password, host.keypath) else: # Error case, should never enter here. Handle error cases here (if needed) - self.log.error(f"Failed to deploy: {processor.name}. The deploy type is unknown.") + self.log.error(f'Failed to deploy: {processor.name}. The deploy type is unknown.') pass for _ in range(processor.count): @@ -132,7 +132,7 @@ def _deploy_processing_worker(self, processor: ProcessorConfig, host: HostConfig else: # TODO: Weirdly there is a duplication of code inside this method # Error case, should never enter here. Handle error cases here (if needed) - self.log.error(f"Failed to deploy: {processor.name}. The deploy type is unknown.") + self.log.error(f'Failed to deploy: {processor.name}. The deploy type is unknown.') pass def deploy_rabbitmq(self, image: str = 'rabbitmq:3-management', detach: bool = True, @@ -142,12 +142,12 @@ def deploy_rabbitmq(self, image: str = 'rabbitmq:3-management', detach: bool = T # Handling of creation of queues, submitting messages to queues, # and receiving messages from queues is part of the RabbitMQ Library # Which is part of the OCR-D WebAPI implementation. - self.log.debug(f"Trying to deploy image[{image}], with modes: detach[{detach}], remove[{remove}]") + self.log.debug(f'Trying to deploy image[{image}], with modes: detach[{detach}], remove[{remove}]') if not self.mongo_data or not self.mongo_data.address: - self.log.error(f"Deploying RabbitMQ has failed - missing configuration.") + self.log.error(f'Deploying RabbitMQ has failed - missing configuration.') # TODO: Raise an error instead of silently ignoring it - return "" + return '' client = create_docker_client(self.mq_data.address, self.mq_data.username, self.mq_data.password, self.mq_data.keypath) @@ -161,7 +161,7 @@ def deploy_rabbitmq(self, image: str = 'rabbitmq:3-management', detach: bool = T 15672: 15672, 25672: 25672 } - self.log.debug(f"Ports mapping: {ports_mapping}") + self.log.debug(f'Ports mapping: {ports_mapping}') res = client.containers.run( image=image, detach=detach, @@ -181,10 +181,10 @@ def deploy_rabbitmq(self, image: str = 'rabbitmq:3-management', detach: bool = T # Build the RabbitMQ Server URL to return rmq_host = self.mq_data.address rmq_port = self.mq_data.port - rmq_vhost = "/" # the default virtual host + rmq_vhost = '/' # the default virtual host - rabbitmq_url = f"{rmq_host}:{rmq_port}{rmq_vhost}" - self.log.debug(f"The RabbitMQ server was deployed on url: {rabbitmq_url}") + rabbitmq_url = f'{rmq_host}:{rmq_port}{rmq_vhost}' + self.log.debug(f'The RabbitMQ server was deployed on url: {rabbitmq_url}') return rabbitmq_url def init_rabbitmq(self, client: CustomDockerClient, rabbitmq_id: str): @@ -201,12 +201,12 @@ def init_rabbitmq(self, client: CustomDockerClient, rabbitmq_id: str): def deploy_mongodb(self, image: str = 'mongo', detach: bool = True, remove: bool = True, ports_mapping: Union[Dict, None] = None) -> str: - self.log.debug(f"Trying to deploy image[{image}], with modes: detach[{detach}], remove[{remove}]") + self.log.debug(f'Trying to deploy image[{image}], with modes: detach[{detach}], remove[{remove}]') if not self.mongo_data or not self.mongo_data.address: - self.log.error(f"Deploying MongoDB has failed - missing configuration.") + self.log.error(f'Deploying MongoDB has failed - missing configuration.') # TODO: Raise an error instead of silently ignoring it - return "" + return '' client = create_docker_client(self.mongo_data.address, self.mongo_data.username, self.mongo_data.password, self.mongo_data.keypath) @@ -214,7 +214,7 @@ def deploy_mongodb(self, image: str = 'mongo', detach: bool = True, remove: bool ports_mapping = { 27017: self.mongo_data.port } - self.log.debug(f"Ports mapping: {ports_mapping}") + self.log.debug(f'Ports mapping: {ports_mapping}') # TODO: use rm here or not? Should the mongodb be reused? # TODO: what about the data-dir? Must data be preserved? res = client.containers.run( @@ -229,20 +229,20 @@ def deploy_mongodb(self, image: str = 'mongo', detach: bool = True, remove: bool client.close() # Build the MongoDB URL to return - mongodb_prefix = "mongodb://" + mongodb_prefix = 'mongodb://' mongodb_host = self.mongo_data.address mongodb_port = self.mongo_data.port - mongodb_url = f"{mongodb_prefix}{mongodb_host}:{mongodb_port}" - self.log.debug(f"The MongoDB was deployed on url: {mongodb_url}") + mongodb_url = f'{mongodb_prefix}{mongodb_host}:{mongodb_port}' + self.log.debug(f'The MongoDB was deployed on url: {mongodb_url}') return mongodb_url def kill_rabbitmq(self) -> None: # TODO: The PID must not be stored in the configuration `mq_data`. if not self.mq_data or not self.mq_data.pid: - self.log.warning(f"No running RabbitMQ instance found") + self.log.warning(f'No running RabbitMQ instance found') # TODO: Ignoring this silently is problematic in the future return - self.log.debug(f"Trying to stop the deployed RabbitMQ with PID: {self.mq_data.pid}") + self.log.debug(f'Trying to stop the deployed RabbitMQ with PID: {self.mq_data.pid}') client = create_docker_client(self.mq_data.address, self.mq_data.username, self.mq_data.password, self.mq_data.keypath) @@ -254,10 +254,10 @@ def kill_rabbitmq(self) -> None: def kill_mongodb(self) -> None: # TODO: The PID must not be stored in the configuration `mongo_data`. if not self.mongo_data or not self.mongo_data.pid: - self.log.warning(f"No running MongoDB instance found") + self.log.warning(f'No running MongoDB instance found') # TODO: Ignoring this silently is problematic in the future return - self.log.debug(f"Trying to stop the deployed MongoDB with PID: {self.mongo_data.pid}") + self.log.debug(f'Trying to stop the deployed MongoDB with PID: {self.mongo_data.pid}') client = create_docker_client(self.mongo_data.address, self.mongo_data.username, self.mongo_data.password, self.mongo_data.keypath) @@ -267,10 +267,10 @@ def kill_mongodb(self) -> None: self.log.debug('The MongoDB is stopped') def kill_hosts(self) -> None: - self.log.debug("Starting to kill/stop hosts") + self.log.debug('Starting to kill/stop hosts') # Kill processing hosts for host in self.hosts: - self.log.debug(f"Killing/Stopping processing workers on host: {host.address}") + self.log.debug(f'Killing/Stopping processing workers on host: {host.address}') if host.ssh_client: host.ssh_client = create_ssh_client(host.address, host.username, host.password, host.keypath) if host.docker_client: @@ -293,7 +293,7 @@ def kill_processing_worker(self, host: HostConfig) -> None: host.docker_client.containers.get(pid).stop() else: # Error case, should never enter here. Handle error cases here (if needed) - self.log.error(f"Failed to kill: {processor.name}. The deploy type is unknown.") + self.log.error(f'Failed to kill: {processor.name}. The deploy type is unknown.') pass processor.pids = [] @@ -331,7 +331,7 @@ def start_native_processor(self, client: SSHClient, processor_name: str, queue_u path = Path(bin_dir) / processor_name else: path = processor_name - cmd = f"{path} --database {database_url} --queue {queue_url}" + cmd = f'{path} --database {database_url} --queue {queue_url}' # the only way (I could find) to make it work to start a process in the background and # return early is this construction. The pid of the last started background process is # printed with `echo $!` but it is printed inbetween other output. Because of that I added @@ -339,7 +339,7 @@ def start_native_processor(self, client: SSHClient, processor_name: str, queue_u # returning from the function stdin.write(f'{cmd} & \n echo xyz$!xyz \n exit \n') output = stdout.read().decode('utf-8') - self.log.debug(f"Output for processor {processor_name}: {output}") + self.log.debug(f'Output for processor {processor_name}: {output}') stdout.close() stdin.close() return re_search(r'xyz([0-9]+)xyz', output).group(1) # type: ignore @@ -348,7 +348,7 @@ def start_docker_processor(self, client: CustomDockerClient, processor_name: str, queue_url: str, database_url: str) -> str: self.log.debug(f'Starting docker container processor: {processor_name}') # TODO: queue_url and database_url are ready to be used - self.log.debug(f"The processor connects to queue: {queue_url} and mongodb: {database_url}") + self.log.debug(f'The processor connects to queue: {queue_url} and mongodb: {database_url}') # TODO: add real command here to start processing server here res = client.containers.run('debian', 'sleep 500s', detach=True, remove=True) assert res and res.id, f'Running processor: {processor_name} in docker-container failed' diff --git a/ocrd/ocrd/network/deployment_utils.py b/ocrd/ocrd/network/deployment_utils.py index 0922353fb..80152f53b 100644 --- a/ocrd/ocrd/network/deployment_utils.py +++ b/ocrd/ocrd/network/deployment_utils.py @@ -45,9 +45,9 @@ def __init__(self, user: str, host: str, **kwargs) -> None: # TODO: Call to the super class __init__ is missing here, # may this potentially become an issue? if not user or not host: - raise ValueError("Missing argument: user and host must both be provided") + raise ValueError('Missing argument: user and host must both be provided') if not 'password' in kwargs and not 'keypath' in kwargs: - raise ValueError("Missing argument: one of password and keyfile is needed") + raise ValueError('Missing argument: one of password and keyfile is needed') self.api = APIClient(f'ssh://{host}', use_ssh_client=True, version='1.41') ssh_adapter = self.CustomSshHttpAdapter(f'ssh://{user}@{host}:22', **kwargs) self.api.mount('http+docker://ssh', ssh_adapter) diff --git a/ocrd/ocrd/network/helpers.py b/ocrd/ocrd/network/helpers.py index cefb21987..882c8822c 100644 --- a/ocrd/ocrd/network/helpers.py +++ b/ocrd/ocrd/network/helpers.py @@ -11,19 +11,19 @@ def verify_database_url(mongodb_address: str) -> str: - database_prefix = "mongodb://" + database_prefix = 'mongodb://' if not mongodb_address.startswith(database_prefix): - error_msg = f"The database address must start with a prefix: {database_prefix}" + error_msg = f'The database address must start with a prefix: {database_prefix}' raise ValueError(error_msg) address_without_prefix = mongodb_address[len(database_prefix):] - print(f"Address without prefix: {address_without_prefix}") + print(f'Address without prefix: {address_without_prefix}') elements = address_without_prefix.split(':', 1) if len(elements) != 2: - raise ValueError("The database address is in wrong format") + raise ValueError('The database address is in wrong format') db_host = elements[0] db_port = int(elements[1]) - mongodb_url = f"{database_prefix}{db_host}:{db_port}" + mongodb_url = f'{database_prefix}{db_host}:{db_port}' return mongodb_url @@ -32,16 +32,16 @@ def verify_and_parse_rabbitmq_addr(rabbitmq_address: str) -> Tuple[str, int, str if len(elements) == 3: rmq_host = elements[0] rmq_port = int(elements[1]) - rmq_vhost = f"/{elements[2]}" + rmq_vhost = f'/{elements[2]}' return rmq_host, rmq_port, rmq_vhost if len(elements) == 2: rmq_host = elements[0] rmq_port = int(elements[1]) - rmq_vhost = "/" # The default global vhost + rmq_vhost = '/' # The default global vhost return rmq_host, rmq_port, rmq_vhost - raise ValueError("The RabbitMQ address is in wrong format. Expected format: {host}:{port}/{vhost}") + raise ValueError('The RabbitMQ address is in wrong format. Expected format: {host}:{port}/{vhost}') def get_workspaces_dir() -> str: @@ -63,14 +63,14 @@ def get_workspaces_dir() -> str: def construct_dummy_processing_message() -> OcrdProcessingMessage: return OcrdProcessingMessage( - job_id="dummy-job-id", - processor_name="ocrd-dummy", + job_id='dummy-job-id', + processor_name='ocrd-dummy', created_time=None, # Auto generated if None - path_to_mets="/home/mm/Desktop/ws_example/mets.xml", + path_to_mets='/home/mm/Desktop/ws_example/mets.xml', workspace_id=None, # Not required, workspace is not uploaded through the Workspace Server - input_file_grps=["DEFAULT"], - output_file_grps=["DUMMY-OUTPUT"], - page_id="PHYS0001..PHYS0003", # Process only the first 3 pages + input_file_grps=['DEFAULT'], + output_file_grps=['DUMMY-OUTPUT'], + page_id='PHYS0001..PHYS0003', # Process only the first 3 pages parameters={}, result_queue_name=None # Not implemented yet, do not set ) @@ -78,8 +78,8 @@ def construct_dummy_processing_message() -> OcrdProcessingMessage: def construct_dummy_result_message() -> OcrdResultMessage: return OcrdResultMessage( - job_id="dummy-job_id", - status="RUNNING", - workspace_id="dummy-workspace_id", - path_to_mets="/home/mm/Desktop/ws_example/mets.xml" + job_id='dummy-job_id', + status='RUNNING', + workspace_id='dummy-workspace_id', + path_to_mets='/home/mm/Desktop/ws_example/mets.xml' ) diff --git a/ocrd/ocrd/network/models/job.py b/ocrd/ocrd/network/models/job.py index f34e217f7..4d48fbc95 100644 --- a/ocrd/ocrd/network/models/job.py +++ b/ocrd/ocrd/network/models/job.py @@ -31,13 +31,13 @@ class JobInput(BaseModel): class Config: schema_extra = { - "example": { - "path": "/path/to/mets.xml", - "description": "The description of this execution", - "input_file_grps": ["INPUT_FILE_GROUP"], - "output_file_grps": ["OUTPUT_FILE_GROUP"], - "page_id": "PAGE_ID", - "parameters": {} + 'example': { + 'path': '/path/to/mets.xml', + 'description': 'The description of this execution', + 'input_file_grps': ['INPUT_FILE_GROUP'], + 'output_file_grps': ['OUTPUT_FILE_GROUP'], + 'page_id': 'PAGE_ID', + 'parameters': {} } } diff --git a/ocrd/ocrd/network/processing_broker.py b/ocrd/ocrd/network/processing_broker.py index bde74dde9..00e106f2d 100644 --- a/ocrd/ocrd/network/processing_broker.py +++ b/ocrd/ocrd/network/processing_broker.py @@ -43,9 +43,9 @@ def __init__(self, config_path: str, host: str, port: int) -> None: # above instead of using the hard coded ones below # RabbitMQ related fields, hard coded initially - self.rmq_host = "localhost" + self.rmq_host = 'localhost' self.rmq_port = 5672 - self.rmq_vhost = "/" + self.rmq_vhost = '/' # Gets assigned when `connect_publisher` is called on the working object # Note for peer: Check under self.start() @@ -100,7 +100,7 @@ def start(self) -> None: # The RMQPublisher is initialized and a connection to the RabbitMQ is performed self.connect_publisher() - self.log.debug(f"Starting to create message queues on RabbitMQ instance url: {rabbitmq_url}") + self.log.debug(f'Starting to create message queues on RabbitMQ instance url: {rabbitmq_url}') self.create_message_queues() # Deploy processing hosts where processing workers are running on @@ -116,7 +116,7 @@ def parse_config(config_path: str) -> ProcessingBrokerConfig: obj = safe_load(fin) report = ProcessingBrokerValidator.validate(obj) if not report.is_valid: - raise Exception(f"Processing-Broker configuration file is invalid:\n{report.errors}") + raise Exception(f'Processing-Broker configuration file is invalid:\n{report.errors}') return ProcessingBrokerConfig(obj) async def on_shutdown(self) -> None: @@ -141,18 +141,18 @@ async def on_shutdown(self) -> None: async def stop_deployed_agents(self) -> None: self.deployer.kill_all() - def connect_publisher(self, username="default", password="default", enable_acks=True): - self.log.debug(f"Connecting RMQPublisher to RabbitMQ server: {self.rmq_host}:{self.rmq_port}{self.rmq_vhost}") + def connect_publisher(self, username='default', password='default', enable_acks=True): + self.log.debug(f'Connecting RMQPublisher to RabbitMQ server: {self.rmq_host}:{self.rmq_port}{self.rmq_vhost}') self.rmq_publisher = RMQPublisher(host=self.rmq_host, port=self.rmq_port, vhost=self.rmq_vhost) # TODO: Remove this information before the release - self.log.debug(f"RMQPublisher authenticates with username: {username}, password: {password}") + self.log.debug(f'RMQPublisher authenticates with username: {username}, password: {password}') self.rmq_publisher.authenticate_and_connect(username=username, password=password) if enable_acks: self.rmq_publisher.enable_delivery_confirmations() - self.log.debug(f"Delivery confirmations are enabled") + self.log.debug(f'Delivery confirmations are enabled') else: - self.log.debug(f"Delivery confirmations are disabled") - self.log.debug(f"Successfully connected RMQPublisher.") + self.log.debug(f'Delivery confirmations are disabled') + self.log.debug(f'Successfully connected RMQPublisher.') def create_message_queues(self): # Create the message queues based on the occurrence of `processor.name` in the config file @@ -160,7 +160,7 @@ def create_message_queues(self): for processor in host.processors: # The existence/validity of the processor.name is not tested. # Even if an ocr-d processor does not exist, the queue is created - self.log.debug(f"Creating a message queue with id: {processor.name}") + self.log.debug(f'Creating a message queue with id: {processor.name}') # TODO: We may want to track here if there are already queues with the same name self.rmq_publisher.create_queue(queue_name=processor.name) @@ -170,11 +170,11 @@ def publish_default_processing_message(self): # TODO: switch back to pickle?! encoded_processing_message = OcrdProcessingMessage.encode_yml(processing_message) if self.rmq_publisher: - self.log.debug("Publishing the default processing message") + self.log.debug('Publishing the default processing message') self.rmq_publisher.publish_to_queue(queue_name=queue_name, message=encoded_processing_message) else: - self.log.error("RMQPublisher is not connected") - raise Exception("RMQPublisher is not connected") + self.log.error('RMQPublisher is not connected') + raise Exception('RMQPublisher is not connected') # TODO: how do we want to do the whole model-stuff? Webapi (openapi.yml) uses ProcessorJob def run_processor(self, processor_name: str, p_args: ProcessorArgs) -> ProcessorJob: diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index 67f619726..6e67e3874 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -34,9 +34,9 @@ def __init__(self, rabbitmq_addr, mongodb_addr, processor_name, ocrd_tool: dict, try: self.db_url = verify_database_url(mongodb_addr) - self.log.debug(f"Verified MongoDB URL: {self.db_url}") + self.log.debug(f'Verified MongoDB URL: {self.db_url}') self.rmq_host, self.rmq_port, self.rmq_vhost = verify_and_parse_rabbitmq_addr(rabbitmq_addr) - self.log.debug(f"Verified RabbitMQ Server URL: {self.rmq_host}:{self.rmq_port}{self.rmq_vhost}") + self.log.debug(f'Verified RabbitMQ Server URL: {self.rmq_host}:{self.rmq_port}{self.rmq_vhost}') except ValueError as e: raise ValueError(e) @@ -56,13 +56,13 @@ def __init__(self, rabbitmq_addr, mongodb_addr, processor_name, ocrd_tool: dict, # the message queue with name {processor_name}-result, the RMQPublisher should be instantiated # self.rmq_publisher = None - def connect_consumer(self, username="default", password="default"): - self.log.debug(f"Connecting RMQConsumer to RabbitMQ server: {self.rmq_host}:{self.rmq_port}{self.rmq_vhost}") + def connect_consumer(self, username='default', password='default'): + self.log.debug(f'Connecting RMQConsumer to RabbitMQ server: {self.rmq_host}:{self.rmq_port}{self.rmq_vhost}') self.rmq_consumer = RMQConsumer(host=self.rmq_host, port=self.rmq_port, vhost=self.rmq_vhost) # TODO: Remove this information before the release - self.log.debug(f"RMQConsumer authenticates with username: {username}, password: {password}") + self.log.debug(f'RMQConsumer authenticates with username: {username}, password: {password}') self.rmq_consumer.authenticate_and_connect(username=username, password=password) - self.log.debug(f"Successfully connected RMQConsumer.") + self.log.debug(f'Successfully connected RMQConsumer.') # Define what happens every time a message is consumed # from the queue with name self.processor_name @@ -77,56 +77,56 @@ def on_consumed_message( is_redelivered: bool = delivery.redelivered message_headers: dict = properties.headers - self.log.debug(f"Consumer tag: {consumer_tag}") - self.log.debug(f"Message delivery tag: {delivery_tag}") - self.log.debug(f"Is redelivered message: {is_redelivered}") - self.log.debug(f"Message headers: {message_headers}") + self.log.debug(f'Consumer tag: {consumer_tag}') + self.log.debug(f'Message delivery tag: {delivery_tag}') + self.log.debug(f'Is redelivered message: {is_redelivered}') + self.log.debug(f'Message headers: {message_headers}') try: - self.log.debug(f"Trying to decode processing message with tag: {delivery_tag}") + self.log.debug(f'Trying to decode processing message with tag: {delivery_tag}') # TODO: switch back to pickle?! processing_message: OcrdProcessingMessage = OcrdProcessingMessage.decode_yml(body) except Exception as e: - self.log.error(f"Failed to decode processing message body: {body}") - self.log.error(f"Nacking processing message with tag: {delivery_tag}") + self.log.error(f'Failed to decode processing message body: {body}') + self.log.error(f'Nacking processing message with tag: {delivery_tag}') channel.basic_nack(delivery_tag=delivery_tag, multiple=False, requeue=False) - raise Exception(f"Failed to decode processing message with tag: {delivery_tag}, reason: {e}") + raise Exception(f'Failed to decode processing message with tag: {delivery_tag}, reason: {e}') try: # TODO: Note to peer: ideally we should avoid doing database related actions # in this method, and handle database related interactions inside `self.process_message()` - self.log.debug(f"Starting to process the received message: {processing_message}") + self.log.debug(f'Starting to process the received message: {processing_message}') self.process_message(processing_message=processing_message) except Exception as e: - self.log.error(f"Failed to process processing message with tag: {delivery_tag}") - self.log.error(f"Nacking processing message with tag: {delivery_tag}") + self.log.error(f'Failed to process processing message with tag: {delivery_tag}') + self.log.error(f'Nacking processing message with tag: {delivery_tag}') channel.basic_nack(delivery_tag=delivery_tag, multiple=False, requeue=False) - raise Exception(f"Failed to process processing message with tag: {delivery_tag}, reason: {e}") + raise Exception(f'Failed to process processing message with tag: {delivery_tag}, reason: {e}') - self.log.debug(f"Successfully processed message ") - self.log.debug(f"Acking message with tag: {delivery_tag}") + self.log.debug(f'Successfully processed message ') + self.log.debug(f'Acking message with tag: {delivery_tag}') channel.basic_ack(delivery_tag=delivery_tag, multiple=False) def start_consuming(self) -> None: if self.rmq_consumer: - self.log.debug(f"Configuring consuming from queue: {self.processor_name}") + self.log.debug(f'Configuring consuming from queue: {self.processor_name}') self.rmq_consumer.configure_consuming( queue_name=self.processor_name, callback_method=self.on_consumed_message ) - self.log.debug(f"Starting consuming from queue: {self.processor_name}") + self.log.debug(f'Starting consuming from queue: {self.processor_name}') # Starting consuming is a blocking action self.rmq_consumer.start_consuming() else: - raise Exception("The RMQConsumer is not connected/configured properly") + raise Exception('The RMQConsumer is not connected/configured properly') # TODO: Better error handling required to catch exceptions def process_message(self, processing_message: OcrdProcessingMessage): # Verify that the processor name in the processing message # matches the processor name of the current processing worker if self.processor_name != processing_message.processor_name: - raise ValueError(f"Processor name is not matching. " - f"Expected: {self.processor_name}, Got: {processing_message.processor_name}") + raise ValueError(f'Processor name is not matching. ' + f'Expected: {self.processor_name}, Got: {processing_message.processor_name}') # This can be path if invoking `run_processor` # but must be ocrd.Workspace if invoking `run_cli`. @@ -145,12 +145,12 @@ def process_message(self, processing_message: OcrdProcessingMessage): job_id = processing_message.job_id if processing_message.result_queue: - self.log.warning(f"Publishing results to a message queue from the Processing Worker is not supported yet") + self.log.warning(f'Publishing results to a message queue from the Processing Worker is not supported yet') # TODO: Currently, no caching is performed. if self.processor_class: - self.log.debug(f"Invoking the pythonic processor: {self.processor_name}") - self.log.debug(f"Invoking the processor_class: {self.processor_class}") + self.log.debug(f'Invoking the pythonic processor: {self.processor_name}') + self.log.debug(f'Invoking the processor_class: {self.processor_class}') self.run_processor_from_worker( processor_class=self.processor_class, workspace=workspace, @@ -160,7 +160,7 @@ def process_message(self, processing_message: OcrdProcessingMessage): parameter=parameter ) else: - self.log.debug(f"Invoking the cli: {self.processor_name}") + self.log.debug(f'Invoking the cli: {self.processor_name}') self.run_cli_from_worker( executable=self.processor_name, workspace=workspace, diff --git a/ocrd/ocrd/network/rabbitmq_utils/__init__.py b/ocrd/ocrd/network/rabbitmq_utils/__init__.py index b6dd062e4..2ef092fa8 100644 --- a/ocrd/ocrd/network/rabbitmq_utils/__init__.py +++ b/ocrd/ocrd/network/rabbitmq_utils/__init__.py @@ -2,11 +2,11 @@ # https://github.com/OCR-D/ocrd-webapi-implementation/tree/main/ocrd_webapi/rabbitmq __all__ = [ - "RMQConsumer", - "RMQConnector", - "RMQPublisher", - "OcrdProcessingMessage", - "OcrdResultMessage" + 'RMQConsumer', + 'RMQConnector', + 'RMQPublisher', + 'OcrdProcessingMessage', + 'OcrdResultMessage' ] from .consumer import RMQConsumer diff --git a/ocrd/ocrd/network/rabbitmq_utils/constants.py b/ocrd/ocrd/network/rabbitmq_utils/constants.py index 0ba9334bf..a53fada89 100644 --- a/ocrd/ocrd/network/rabbitmq_utils/constants.py +++ b/ocrd/ocrd/network/rabbitmq_utils/constants.py @@ -1,29 +1,29 @@ import logging __all__ = [ - "DEFAULT_EXCHANGER_NAME", - "DEFAULT_EXCHANGER_TYPE", - "DEFAULT_QUEUE", - "DEFAULT_ROUTER", - "RABBIT_MQ_HOST", - "RABBIT_MQ_PORT", - "RABBIT_MQ_VHOST", - "RECONNECT_WAIT", - "RECONNECT_TRIES", - "PREFETCH_COUNT", - "LOG_FORMAT", - "LOG_LEVEL" + 'DEFAULT_EXCHANGER_NAME', + 'DEFAULT_EXCHANGER_TYPE', + 'DEFAULT_QUEUE', + 'DEFAULT_ROUTER', + 'RABBIT_MQ_HOST', + 'RABBIT_MQ_PORT', + 'RABBIT_MQ_VHOST', + 'RECONNECT_WAIT', + 'RECONNECT_TRIES', + 'PREFETCH_COUNT', + 'LOG_FORMAT', + 'LOG_LEVEL' ] -DEFAULT_EXCHANGER_NAME: str = "ocrd-network-default" -DEFAULT_EXCHANGER_TYPE: str = "direct" -DEFAULT_QUEUE: str = "ocrd-network-default" -DEFAULT_ROUTER: str = "ocrd-network-default" +DEFAULT_EXCHANGER_NAME: str = 'ocrd-network-default' +DEFAULT_EXCHANGER_TYPE: str = 'direct' +DEFAULT_QUEUE: str = 'ocrd-network-default' +DEFAULT_ROUTER: str = 'ocrd-network-default' -# "rabbit-mq-host" when Dockerized -RABBIT_MQ_HOST: str = "localhost" +# 'rabbit-mq-host' when Dockerized +RABBIT_MQ_HOST: str = 'localhost' RABBIT_MQ_PORT: int = 5672 -RABBIT_MQ_VHOST: str = "/" +RABBIT_MQ_VHOST: str = '/' # Wait seconds before next reconnect try RECONNECT_WAIT: int = 5 diff --git a/ocrd/ocrd/network/rabbitmq_utils/consumer.py b/ocrd/ocrd/network/rabbitmq_utils/consumer.py index 0453f71cd..eb1f03707 100644 --- a/ocrd/ocrd/network/rabbitmq_utils/consumer.py +++ b/ocrd/ocrd/network/rabbitmq_utils/consumer.py @@ -62,7 +62,7 @@ def example_run(self): try: self.start_consuming() except KeyboardInterrupt: - self._logger.info("Keyboard interruption detected. Closing down peacefully.") + self._logger.info('Keyboard interruption detected. Closing down peacefully.') exit(0) # TODO: Clean leftovers here and inform the RabbitMQ # server about the disconnection of the consumer @@ -139,12 +139,12 @@ def __ack_message(self, delivery_tag): def main(): # Connect to localhost:5672 by # using the virtual host "/" (%2F) - consumer = RMQConsumer(host="localhost", port=5672, vhost="/") + consumer = RMQConsumer(host='localhost', port=5672, vhost='/') # Configured with definitions.json when building the RabbitMQ image # Check Dockerfile-RabbitMQ consumer.authenticate_and_connect( - username="default-consumer", - password="default-consumer" + username='default-consumer', + password='default-consumer' ) consumer.setup_defaults() consumer.example_run() diff --git a/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py b/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py index 1e52d9260..475e9274a 100644 --- a/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py +++ b/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py @@ -27,16 +27,16 @@ def __init__( result_queue_name: str = None, ): if not job_id: - raise ValueError(f"job_id must be set") + raise ValueError('job_id must be set') if not processor_name: - raise ValueError(f"processor_name must be set") + raise ValueError('processor_name must be set') if not created_time: # We should not raise a ValueError but just calculate it created_time = int(datetime.utcnow().timestamp()) if not input_file_grps or len(input_file_grps) == 0: - raise ValueError(f"input_file_grps must be set and contain at least 1 element") + raise ValueError('input_file_grps must be set and contain at least 1 element') if not (workspace_id or path_to_mets): - raise ValueError(f"Either `workspace_id` or `path_to_mets` must be set") + raise ValueError('Either `workspace_id` or `path_to_mets` must be set') self.job_id = job_id # uuid self.processor_name = processor_name # "ocrd-.*" @@ -61,7 +61,7 @@ def encode(ocrd_processing_message: OcrdProcessingMessage) -> bytes: return dumps(ocrd_processing_message) @staticmethod - def decode(ocrd_processing_message: bytes, encoding="utf-8") -> OcrdProcessingMessage: + def decode(ocrd_processing_message: bytes, encoding='utf-8') -> OcrdProcessingMessage: data = loads(ocrd_processing_message, encoding=encoding) return OcrdProcessingMessage( job_id=data.job_id, @@ -89,16 +89,16 @@ def decode_yml(ocrd_processing_message: bytes) -> OcrdProcessingMessage: msg = ocrd_processing_message.decode('utf-8') data = yaml.load(msg, Loader=yaml.Loader) return OcrdProcessingMessage( - job_id=data.get("job_id", None), - processor_name=data.get("processor_name", None), - created_time=data.get("created_time", None), - path_to_mets=data.get("path_to_mets", None), - workspace_id=data.get("workspace_id", None), - input_file_grps=data.get("input_file_grps", None), - output_file_grps=data.get("output_file_grps", None), - page_id=data.get("page_id", None), - parameters=data.get("parameters", None), - result_queue_name=data.get("result_queue", None), + job_id=data.get('job_id', None), + processor_name=data.get('processor_name', None), + created_time=data.get('created_time', None), + path_to_mets=data.get('path_to_mets', None), + workspace_id=data.get('workspace_id', None), + input_file_grps=data.get('input_file_grps', None), + output_file_grps=data.get('output_file_grps', None), + page_id=data.get('page_id', None), + parameters=data.get('parameters', None), + result_queue_name=data.get('result_queue', None), ) @staticmethod @@ -131,7 +131,7 @@ def encode(ocrd_result_message: OcrdResultMessage) -> bytes: return dumps(ocrd_result_message) @staticmethod - def decode(ocrd_result_message: bytes, encoding="utf-8") -> OcrdResultMessage: + def decode(ocrd_result_message: bytes, encoding='utf-8') -> OcrdResultMessage: data = loads(ocrd_result_message, encoding=encoding) return OcrdResultMessage( job_id=data.job_id, diff --git a/ocrd/ocrd/network/rabbitmq_utils/publisher.py b/ocrd/ocrd/network/rabbitmq_utils/publisher.py index a69ee92fa..9b7615483 100644 --- a/ocrd/ocrd/network/rabbitmq_utils/publisher.py +++ b/ocrd/ocrd/network/rabbitmq_utils/publisher.py @@ -62,12 +62,12 @@ def example_run(self): while True: try: messages = 1 - message = f"#{messages}" + message = f'#{messages}' self.publish_to_queue(queue_name=DEFAULT_ROUTER, message=message) messages += 1 sleep(2) except KeyboardInterrupt: - self._logger.info("Keyboard interruption detected. Closing down peacefully.") + self._logger.info('Keyboard interruption detected. Closing down peacefully.') exit(0) # TODO: Clean leftovers here and inform the RabbitMQ # server about the disconnection of the publisher @@ -86,7 +86,7 @@ def create_queue( if exchange_name is None: exchange_name = DEFAULT_EXCHANGER_NAME if exchange_type is None: - exchange_type = "direct" + exchange_type = 'direct' RMQConnector.exchange_declare( channel=self._channel, @@ -177,12 +177,12 @@ def __on_delivery_confirmation(self, frame): def main(): # Connect to localhost:5672 by # using the virtual host "/" (%2F) - publisher = RMQPublisher(host="localhost", port=5672, vhost="/") + publisher = RMQPublisher(host='localhost', port=5672, vhost='/') # Configured with definitions.json when building the RabbitMQ image # Check Dockerfile-RabbitMQ publisher.authenticate_and_connect( - username="default-publisher", - password="default-publisher" + username='default-publisher', + password='default-publisher' ) publisher.setup_defaults() publisher.enable_delivery_confirmations() From 512ea7751855a4164687fb2f9dddf2983a67285a Mon Sep 17 00:00:00 2001 From: joschrew Date: Wed, 25 Jan 2023 16:02:26 +0100 Subject: [PATCH 090/226] Add missing typehints for network modules --- ocrd/ocrd/network/deployer.py | 2 +- ocrd/ocrd/network/deployment_config.py | 2 +- ocrd/ocrd/network/processing_broker.py | 13 +++-- ocrd/ocrd/network/processing_worker.py | 17 +++--- ocrd/ocrd/network/rabbitmq_utils/connector.py | 56 +++++++++++-------- ocrd/ocrd/network/rabbitmq_utils/consumer.py | 45 ++++++++------- .../network/rabbitmq_utils/ocrd_messages.py | 6 +- ocrd/ocrd/network/rabbitmq_utils/publisher.py | 26 +++++---- 8 files changed, 93 insertions(+), 74 deletions(-) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index f10299d18..d0f27896b 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -187,7 +187,7 @@ def deploy_rabbitmq(self, image: str = 'rabbitmq:3-management', detach: bool = T self.log.debug(f'The RabbitMQ server was deployed on url: {rabbitmq_url}') return rabbitmq_url - def init_rabbitmq(self, client: CustomDockerClient, rabbitmq_id: str): + def init_rabbitmq(self, client: CustomDockerClient, rabbitmq_id: str) -> None: """ Add users, wait for startup """ # TODO: rabbitmq was started, but needs some time to be ready to use. sleep is working for diff --git a/ocrd/ocrd/network/deployment_config.py b/ocrd/ocrd/network/deployment_config.py index f08994435..3d6f93db0 100644 --- a/ocrd/ocrd/network/deployment_config.py +++ b/ocrd/ocrd/network/deployment_config.py @@ -14,7 +14,7 @@ class ProcessingBrokerConfig: - def __init__(self, config: dict): + def __init__(self, config: dict) -> None: self.mongo_config = MongoConfig(config['mongo_db']) self.queue_config = QueueConfig(config['message_queue']) self.hosts_config = [] diff --git a/ocrd/ocrd/network/processing_broker.py b/ocrd/ocrd/network/processing_broker.py index 00e106f2d..1a00f65e3 100644 --- a/ocrd/ocrd/network/processing_broker.py +++ b/ocrd/ocrd/network/processing_broker.py @@ -141,7 +141,8 @@ async def on_shutdown(self) -> None: async def stop_deployed_agents(self) -> None: self.deployer.kill_all() - def connect_publisher(self, username='default', password='default', enable_acks=True): + def connect_publisher(self, username: str = 'default', password: str = 'default', + enable_acks: bool =True) -> None: self.log.debug(f'Connecting RMQPublisher to RabbitMQ server: {self.rmq_host}:{self.rmq_port}{self.rmq_vhost}') self.rmq_publisher = RMQPublisher(host=self.rmq_host, port=self.rmq_port, vhost=self.rmq_vhost) # TODO: Remove this information before the release @@ -149,12 +150,12 @@ def connect_publisher(self, username='default', password='default', enable_acks= self.rmq_publisher.authenticate_and_connect(username=username, password=password) if enable_acks: self.rmq_publisher.enable_delivery_confirmations() - self.log.debug(f'Delivery confirmations are enabled') + self.log.debug('Delivery confirmations are enabled') else: - self.log.debug(f'Delivery confirmations are disabled') - self.log.debug(f'Successfully connected RMQPublisher.') + self.log.debug('Delivery confirmations are disabled') + self.log.debug('Successfully connected RMQPublisher.') - def create_message_queues(self): + def create_message_queues(self) -> None: # Create the message queues based on the occurrence of `processor.name` in the config file for host in self.hosts_config: for processor in host.processors: @@ -164,7 +165,7 @@ def create_message_queues(self): # TODO: We may want to track here if there are already queues with the same name self.rmq_publisher.create_queue(queue_name=processor.name) - def publish_default_processing_message(self): + def publish_default_processing_message(self) -> None: processing_message = construct_dummy_processing_message() queue_name = processing_message.processor_name # TODO: switch back to pickle?! diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index 6e67e3874..93c8c0d37 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -9,7 +9,7 @@ from frozendict import frozendict from functools import lru_cache, wraps import json -from typing import List +from typing import List, Callable, Type, Union import pika.spec import pika.adapters.blocking_connection @@ -17,6 +17,7 @@ from ocrd import Resolver from ocrd_utils import getLogger from ocrd.processor.helpers import run_cli, run_processor +from ocrd.processor.base import Processor from ocrd.network.helpers import ( verify_database_url, verify_and_parse_rabbitmq_addr @@ -56,7 +57,7 @@ def __init__(self, rabbitmq_addr, mongodb_addr, processor_name, ocrd_tool: dict, # the message queue with name {processor_name}-result, the RMQPublisher should be instantiated # self.rmq_publisher = None - def connect_consumer(self, username='default', password='default'): + def connect_consumer(self, username: str = 'default', password: str = 'default') -> None: self.log.debug(f'Connecting RMQConsumer to RabbitMQ server: {self.rmq_host}:{self.rmq_port}{self.rmq_vhost}') self.rmq_consumer = RMQConsumer(host=self.rmq_host, port=self.rmq_port, vhost=self.rmq_vhost) # TODO: Remove this information before the release @@ -121,12 +122,12 @@ def start_consuming(self) -> None: raise Exception('The RMQConsumer is not connected/configured properly') # TODO: Better error handling required to catch exceptions - def process_message(self, processing_message: OcrdProcessingMessage): + def process_message(self, processing_message: OcrdProcessingMessage) -> None: # Verify that the processor name in the processing message # matches the processor name of the current processing worker if self.processor_name != processing_message.processor_name: - raise ValueError(f'Processor name is not matching. ' - f'Expected: {self.processor_name}, Got: {processing_message.processor_name}') + raise ValueError(f'Processor name is not matching. Expected: {self.processor_name},' + f'Got: {processing_message.processor_name}') # This can be path if invoking `run_processor` # but must be ocrd.Workspace if invoking `run_cli`. @@ -232,14 +233,14 @@ def run_cli_from_worker( # Method adopted from Triet's implementation # https://github.com/OCR-D/core/pull/884/files#diff-8b69cb85b5ffcfb93a053791dec62a2f909a0669ae33d8a2412f246c3b01f1a3R260 -def freeze_args(func): +def freeze_args(func: Callable) -> Callable: """ Transform mutable dictionary into immutable. Useful to be compatible with cache Code taken from `this post `_ """ @wraps(func) - def wrapped(*args, **kwargs): + def wrapped(*args, **kwargs) -> Callable: args = tuple([frozendict(arg) if isinstance(arg, dict) else arg for arg in args]) kwargs = {k: frozendict(v) if isinstance(v, dict) else v for k, v in kwargs.items()} return func(*args, **kwargs) @@ -250,7 +251,7 @@ def wrapped(*args, **kwargs): # https://github.com/OCR-D/core/pull/884/files#diff-8b69cb85b5ffcfb93a053791dec62a2f909a0669ae33d8a2412f246c3b01f1a3R260 @freeze_args @lru_cache(maxsize=32) -def get_processor(parameter: dict, processor_class=None): +def get_processor(parameter: dict, processor_class=None) -> Union[Type[Processor], None]: """ Call this function to get back an instance of a processor. The results are cached based on the parameters. Args: diff --git a/ocrd/ocrd/network/rabbitmq_utils/connector.py b/ocrd/ocrd/network/rabbitmq_utils/connector.py index 6527519bb..d64a466cd 100644 --- a/ocrd/ocrd/network/rabbitmq_utils/connector.py +++ b/ocrd/ocrd/network/rabbitmq_utils/connector.py @@ -3,7 +3,7 @@ some part of the source code from the official RabbitMQ documentation. """ -from typing import Any, Optional +from typing import Any, Optional, Union from pika import ( BasicProperties, @@ -11,6 +11,7 @@ ConnectionParameters, PlainCredentials ) +from pika.adapters.blocking_connection import BlockingChannel from ocrd.network.rabbitmq_utils.constants import ( DEFAULT_EXCHANGER_NAME, @@ -25,7 +26,7 @@ class RMQConnector: - def __init__(self, logger, host: str = HOST, port: int = PORT, vhost: str = VHOST): + def __init__(self, logger, host: str = HOST, port: int = PORT, vhost: str = VHOST) -> None: self._logger = logger self._host = host self._port = port @@ -43,7 +44,7 @@ def __init__(self, logger, host: str = HOST, port: int = PORT, vhost: str = VHOS self._gracefully_stopped = False @staticmethod - def declare_and_bind_defaults(connection, channel): + def declare_and_bind_defaults(connection: BlockingConnection, channel: BlockingChannel) -> None: if connection and connection.is_open: if channel and channel.is_open: # Declare the default exchange agent @@ -86,19 +87,20 @@ def open_blocking_connection( return blocking_connection @staticmethod - def open_blocking_channel(blocking_connection): - if blocking_connection and blocking_connection.is_open: - channel = blocking_connection.channel() + def open_blocking_channel(connection: BlockingConnection) -> Union[BlockingChannel, None]: + if connection and connection.is_open: + channel = connection.channel() return channel + return None @staticmethod def exchange_bind( - channel, + channel: BlockingChannel, destination_exchange: str, source_exchange: str, routing_key: str, parameters: Optional[Any] = None - ): + ) -> None: if parameters is None: parameters = {} if channel and channel.is_open: @@ -111,7 +113,7 @@ def exchange_bind( @staticmethod def exchange_declare( - channel, + channel: BlockingChannel, exchange_name: str, exchange_type: str, passive: bool = False, @@ -119,7 +121,7 @@ def exchange_declare( auto_delete: bool = False, internal: bool = False, arguments: Optional[Any] = None - ): + ) -> None: if arguments is None: arguments = {} if channel and channel.is_open: @@ -140,19 +142,20 @@ def exchange_declare( return exchange @staticmethod - def exchange_delete(channel, exchange_name: str, if_unused: bool = False): + def exchange_delete(channel: BlockingChannel, exchange_name: str, + if_unused: bool = False) -> None: # Deletes queue only if unused if channel and channel.is_open: channel.exchange_delete(exchange=exchange_name, if_unused=if_unused) @staticmethod def exchange_unbind( - channel, + channel: BlockingChannel, destination_exchange: str, source_exchange: str, routing_key: str, arguments: Optional[Any] = None - ): + ) -> None: if arguments is None: arguments = {} if channel and channel.is_open: @@ -164,7 +167,8 @@ def exchange_unbind( ) @staticmethod - def queue_bind(channel, queue_name: str, exchange_name: str, routing_key: str, arguments: Optional[Any] = None): + def queue_bind(channel: BlockingChannel, queue_name: str, exchange_name: str, routing_key: str, + arguments: Optional[Any] = None) -> None: if arguments is None: arguments = {} if channel and channel.is_open: @@ -172,14 +176,14 @@ def queue_bind(channel, queue_name: str, exchange_name: str, routing_key: str, a @staticmethod def queue_declare( - channel, + channel: BlockingChannel, queue_name: str, passive: bool = False, durable: bool = False, exclusive: bool = False, auto_delete: bool = False, arguments: Optional[Any] = None - ): + ) -> None: if arguments is None: arguments = {} if channel and channel.is_open: @@ -200,7 +204,8 @@ def queue_declare( return queue @staticmethod - def queue_delete(channel, queue_name: str, if_unused: bool = False, if_empty: bool = False): + def queue_delete(channel: BlockingChannel, queue_name: str, if_unused: bool = False, + if_empty: bool = False) -> None: if channel and channel.is_open: channel.queue_delete( queue=queue_name, @@ -211,12 +216,13 @@ def queue_delete(channel, queue_name: str, if_unused: bool = False, if_empty: bo ) @staticmethod - def queue_purge(channel, queue_name: str): + def queue_purge(channel: BlockingChannel, queue_name: str) -> None: if channel and channel.is_open: channel.queue_purge(queue=queue_name) @staticmethod - def queue_unbind(channel, queue_name: str, exchange_name: str, routing_key: str, arguments: Optional[Any] = None): + def queue_unbind(channel: BlockingChannel, queue_name: str, exchange_name: str, + routing_key: str, arguments: Optional[Any] = None) -> None: if arguments is None: arguments = {} if channel and channel.is_open: @@ -228,7 +234,8 @@ def queue_unbind(channel, queue_name: str, exchange_name: str, routing_key: str, ) @staticmethod - def set_qos(channel, prefetch_size: int = 0, prefetch_count: int = PREFETCH_COUNT, global_qos: bool = False): + def set_qos(channel: BlockingChannel, prefetch_size: int = 0, + prefetch_count: int = PREFETCH_COUNT, global_qos: bool = False) -> None: if channel and channel.is_open: channel.basic_qos( # No specific limit if set to 0 @@ -239,12 +246,13 @@ def set_qos(channel, prefetch_size: int = 0, prefetch_count: int = PREFETCH_COUN ) @staticmethod - def confirm_delivery(channel): + def confirm_delivery(channel: BlockingChannel) -> None: if channel and channel.is_open: channel.confirm_delivery() @staticmethod - def basic_publish(channel, exchange_name: str, routing_key: str, message_body: bytes, properties: BasicProperties): + def basic_publish(channel: BlockingChannel, exchange_name: str, routing_key: str, + message_body: bytes, properties: BasicProperties) -> None: if channel and channel.is_open: channel.basic_publish( exchange=exchange_name, @@ -253,9 +261,9 @@ def basic_publish(channel, exchange_name: str, routing_key: str, message_body: b properties=properties ) - """ + """ @staticmethod - def basic_consume(channel): + def basic_consume(channel: BlockingChannel) -> None: # TODO: provide a general consume method here as well pass """ diff --git a/ocrd/ocrd/network/rabbitmq_utils/consumer.py b/ocrd/ocrd/network/rabbitmq_utils/consumer.py index eb1f03707..918fc3a79 100644 --- a/ocrd/ocrd/network/rabbitmq_utils/consumer.py +++ b/ocrd/ocrd/network/rabbitmq_utils/consumer.py @@ -11,10 +11,15 @@ from pika import ( PlainCredentials ) +from pika.spec import ( + BasicProperties, + Basic, +) +from pika.adapters.blocking_connection import BlockingChannel + from ocrd.network.rabbitmq_utils.constants import ( DEFAULT_QUEUE, - LOG_FORMAT, LOG_LEVEL, RABBIT_MQ_HOST as HOST, RABBIT_MQ_PORT as PORT, @@ -24,8 +29,9 @@ class RMQConsumer(RMQConnector): - def __init__(self, host: str = HOST, port: int = PORT, vhost: str = VHOST, logger_name: str = None): - if logger_name is None: + def __init__(self, host: str = HOST, port: int = PORT, vhost: str = VHOST, + logger_name: str = '') -> None: + if not logger_name: logger_name = __name__ logger = logging.getLogger(logger_name) logging.getLogger(logger_name).setLevel(LOG_LEVEL) @@ -40,7 +46,7 @@ def __init__(self, host: str = HOST, port: int = PORT, vhost: str = VHOST, logge self.reconnect_delay = 0 - def authenticate_and_connect(self, username: str, password: str): + def authenticate_and_connect(self, username: str, password: str) -> None: credentials = PlainCredentials( username=username, password=password, @@ -54,10 +60,10 @@ def authenticate_and_connect(self, username: str, password: str): ) self._channel = RMQConnector.open_blocking_channel(self._connection) - def setup_defaults(self): + def setup_defaults(self) -> None: RMQConnector.declare_and_bind_defaults(self._connection, self._channel) - def example_run(self): + def example_run(self) -> None: self.configure_consuming() try: self.start_consuming() @@ -87,13 +93,13 @@ def get_one_message( def configure_consuming( self, - queue_name: str = None, + queue_name: str = '', callback_method: Any = None - ): + ) -> None: self._logger.debug('Issuing consumer related RPC commands') self._logger.debug('Adding consumer cancellation callback') self._channel.add_on_cancel_callback(self.__on_consumer_cancelled) - if queue_name is None: + if not queue_name: queue_name = DEFAULT_QUEUE if callback_method is None: callback_method = self.__on_message_received @@ -104,26 +110,27 @@ def configure_consuming( self.was_consuming = True self.consuming = True - def start_consuming(self): + def start_consuming(self) -> None: if self._channel and self._channel.is_open: self._channel.start_consuming() - def get_waiting_message_count(self): + def get_waiting_message_count(self) -> Union[int, None]: if self._channel and self._channel.is_open: return self._channel.get_waiting_message_count() + return None - def __on_consumer_cancelled(self, frame: Any): + def __on_consumer_cancelled(self, frame: Any) -> None: self._logger.warning(f'The consumer was cancelled remotely in frame: {frame}') if self._channel: self._channel.close() def __on_message_received( self, - channel, - basic_deliver, - properties, - body - ): + channel: BlockingChannel, + basic_deliver: Basic.Deliver, + properties: BasicProperties, + body: bytes + ) -> None: tag = basic_deliver.delivery_tag app_id = properties.app_id message = body.decode() @@ -131,12 +138,12 @@ def __on_message_received( self._logger.debug(f'Received message on channel: {channel}') self.__ack_message(tag) - def __ack_message(self, delivery_tag): + def __ack_message(self, delivery_tag: int) -> None: self._logger.debug(f'Acknowledging message {delivery_tag}') self._channel.basic_ack(delivery_tag) -def main(): +def main() -> None: # Connect to localhost:5672 by # using the virtual host "/" (%2F) consumer = RMQConsumer(host='localhost', port=5672, vhost='/') diff --git a/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py b/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py index 475e9274a..5d6d86fe8 100644 --- a/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py +++ b/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py @@ -25,7 +25,7 @@ def __init__( page_id: str = None, parameters: Dict[str, Any] = None, result_queue_name: str = None, - ): + ) -> None: if not job_id: raise ValueError('job_id must be set') if not processor_name: @@ -119,7 +119,7 @@ def from_params(processor_name: str, job_id: str, workspaces_dir: str, p_args: P class OcrdResultMessage: - def __init__(self, job_id: str, status: str, workspace_id: str, path_to_mets: str): + def __init__(self, job_id: str, status: str, workspace_id: str, path_to_mets: str) -> None: self.job_id = job_id self.status = status # Either of these two below @@ -131,7 +131,7 @@ def encode(ocrd_result_message: OcrdResultMessage) -> bytes: return dumps(ocrd_result_message) @staticmethod - def decode(ocrd_result_message: bytes, encoding='utf-8') -> OcrdResultMessage: + def decode(ocrd_result_message: bytes, encoding: str = 'utf-8') -> OcrdResultMessage: data = loads(ocrd_result_message, encoding=encoding) return OcrdResultMessage( job_id=data.job_id, diff --git a/ocrd/ocrd/network/rabbitmq_utils/publisher.py b/ocrd/ocrd/network/rabbitmq_utils/publisher.py index 9b7615483..850e8b805 100644 --- a/ocrd/ocrd/network/rabbitmq_utils/publisher.py +++ b/ocrd/ocrd/network/rabbitmq_utils/publisher.py @@ -26,7 +26,8 @@ class RMQPublisher(RMQConnector): - def __init__(self, host: str = HOST, port: int = PORT, vhost: str = VHOST, logger_name: str = None): + def __init__(self, host: str = HOST, port: int = PORT, vhost: str = VHOST, + logger_name: str = None) -> None: if logger_name is None: logger_name = __name__ logger = logging.getLogger(logger_name) @@ -41,7 +42,7 @@ def __init__(self, host: str = HOST, port: int = PORT, vhost: str = VHOST, logge self.nacked_counter = 0 self.running = True - def authenticate_and_connect(self, username: str, password: str): + def authenticate_and_connect(self, username: str, password: str) -> None: credentials = PlainCredentials( username=username, password=password, @@ -55,10 +56,10 @@ def authenticate_and_connect(self, username: str, password: str): ) self._channel = RMQConnector.open_blocking_channel(self._connection) - def setup_defaults(self): + def setup_defaults(self) -> None: RMQConnector.declare_and_bind_defaults(self._connection, self._channel) - def example_run(self): + def example_run(self) -> None: while True: try: messages = 1 @@ -82,7 +83,7 @@ def create_queue( queue_name: str, exchange_name: Optional[str] = None, exchange_type: Optional[str] = None - ): + ) -> None: if exchange_name is None: exchange_name = DEFAULT_EXCHANGER_NAME if exchange_type is None: @@ -108,9 +109,10 @@ def create_queue( def publish_to_queue( self, queue_name: str, - message: Any, + message: bytes, exchange_name: Optional[str] = None, - properties: Optional[Any] = None): + properties: Optional[BasicProperties] = None + ) -> None: if exchange_name is None: exchange_name = DEFAULT_EXCHANGER_NAME if properties is None: @@ -137,20 +139,20 @@ def publish_to_queue( self.deliveries[self.message_counter] = True self._logger.info(f'Published message #{self.message_counter}') - def enable_delivery_confirmations(self): + def enable_delivery_confirmations(self) -> None: self._logger.debug('Enabling delivery confirmations (Confirm.Select RPC)') RMQConnector.confirm_delivery(channel=self._channel) # TODO: Find a way to use this callback method, # seems not possible with Blocking Connections - def __on_delivery_confirmation(self, frame): + def __on_delivery_confirmation(self, frame) -> None: confirmation_type = frame.method.NAME.split('.')[1].lower() delivery_tag: int = frame.method.delivery_tag ack_multiple = frame.method.multiple self._logger.debug(f'Received: {confirmation_type} ' - f'for tag: {delivery_tag} ' - f'(multiple: {ack_multiple})') + f'for tag: {delivery_tag} ' + f'(multiple: {ack_multiple})') if confirmation_type == 'ack': self.acked_counter += 1 @@ -174,7 +176,7 @@ def __on_delivery_confirmation(self, frame): len(self.deliveries), self.acked_counter, self.nacked_counter) -def main(): +def main() -> None: # Connect to localhost:5672 by # using the virtual host "/" (%2F) publisher = RMQPublisher(host='localhost', port=5672, vhost='/') From e876cf9c5a1921a1c0b88a290bba92d8e525db00 Mon Sep 17 00:00:00 2001 From: joschrew Date: Wed, 25 Jan 2023 16:36:11 +0100 Subject: [PATCH 091/226] Review (mainly comments for) CustomDockerClient Changes to _create_paramiko_client: It was decided to use the info from ~/.ssh/config as default/fallback and extend this information with provided password and/or path to keyfile (values from processing- server-config wins). --- ocrd/ocrd/network/deployment_utils.py | 28 ++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/ocrd/ocrd/network/deployment_utils.py b/ocrd/ocrd/network/deployment_utils.py index 80152f53b..402ae7349 100644 --- a/ocrd/ocrd/network/deployment_utils.py +++ b/ocrd/ocrd/network/deployment_utils.py @@ -5,7 +5,6 @@ from docker import APIClient, DockerClient from docker.transport import SSHHTTPAdapter from paramiko import AutoAddPolicy, SSHClient -import urllib.parse from ocrd_utils import getLogger @@ -38,15 +37,24 @@ class CustomDockerClient(DockerClient): XXX: inspired by https://github.com/docker/docker-py/issues/2416 . Should be replaced when docker-sdk provides its own way to make it possible to use custom SSH Credentials. Possible Problems: APIClient must be given the API-version because it cannot connect prior to read it. I - could imagine this could cause Problems + could imagine this could cause Problems. This is not a rushed implementation and was the only + workaround I could find that allows password/keyfile to be used (by default only keyfile from + ~/.ssh/config can be used to authentificate via ssh) + + XXX 2: Reasons to extend DockerClient: The code-changes regarding the connection should be in + one place so I decided to create `CustomSshHttpAdapter` as an inner class. The super + constructor *must not* be called to make this workaround work. Otherwise the APIClient + constructor would be invoced without `version` and that would cause a connection-attempt before + this workaround can be applied. """ def __init__(self, user: str, host: str, **kwargs) -> None: - # TODO: Call to the super class __init__ is missing here, - # may this potentially become an issue? + # the super-constructor is not called on purpose: it solely instantiates the APIClient. The + # missing `version` in that call would raise an error. APIClient is provided here as a + # replacement for what the super-constructor does if not user or not host: raise ValueError('Missing argument: user and host must both be provided') - if not 'password' in kwargs and not 'keypath' in kwargs: + if 'password' not in kwargs and 'keypath' not in kwargs: raise ValueError('Missing argument: one of password and keyfile is needed') self.api = APIClient(f'ssh://{host}', use_ssh_client=True, version='1.41') ssh_adapter = self.CustomSshHttpAdapter(f'ssh://{user}@{host}:22', **kwargs) @@ -64,15 +72,9 @@ def __init__(self, base_url, password: Union[str, None] = None, def _create_paramiko_client(self, base_url: str) -> None: """ this method is called in the superclass constructor. Overwriting allows to set - password/keypath for internal paramiko-client + password/keypath for the internal paramiko-client """ - self.ssh_client = SSHClient() - parsed_base_url = urllib.parse.urlparse(base_url) - self.ssh_params = { - 'hostname': parsed_base_url.hostname, - 'port': parsed_base_url.port, - 'username': parsed_base_url.username, - } + super()._create_paramiko_client(base_url) if self.password: self.ssh_params['password'] = self.password elif self.keypath: From 59e841830383b5e2d96c55500f06cac7eb5f20fc Mon Sep 17 00:00:00 2001 From: joschrew Date: Thu, 26 Jan 2023 08:03:25 +0100 Subject: [PATCH 092/226] Remind myself to rename the broker --- ocrd/ocrd/cli/processing_broker.py | 1 + ocrd/ocrd/network/processing_broker.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ocrd/ocrd/cli/processing_broker.py b/ocrd/ocrd/cli/processing_broker.py index 4fa70edb9..ce9dba30d 100644 --- a/ocrd/ocrd/cli/processing_broker.py +++ b/ocrd/ocrd/cli/processing_broker.py @@ -11,6 +11,7 @@ import logging +# TODO: rename to processing-server @click.command('processing-broker') @click.argument('path_to_config', required=True, type=click.STRING) @click.option('-a', '--address', help='Host (name/IP) and port to bind the Processing-Broker to. Example: localhost:8080', required=True) diff --git a/ocrd/ocrd/network/processing_broker.py b/ocrd/ocrd/network/processing_broker.py index 1a00f65e3..f1aa0c94a 100644 --- a/ocrd/ocrd/network/processing_broker.py +++ b/ocrd/ocrd/network/processing_broker.py @@ -14,7 +14,7 @@ from ocrd.network.models.processor import ProcessorArgs, ProcessorJob from pathlib import Path - +# TODO: rename to ProcessingServer (module-file too) class ProcessingBroker(FastAPI): """ TODO: doc for ProcessingBroker and its methods From 5900c739ed4d5078f3507aa36d254297f8813df4 Mon Sep 17 00:00:00 2001 From: joschrew Date: Thu, 26 Jan 2023 14:08:06 +0100 Subject: [PATCH 093/226] Reuse parts of older impl for running a processor It is not clear yet if the processing-broker should implement parts of the webapi directly or if it should be usable on its own. In the latter case the webapi would delegate to the processing-broker. Anyway I changed the processing part to reuse code of the previous processing-server PR (884) and it also uses the same processing endpoint for now. This way it is currently easier for me to proceed but maybe it has to be changed later again. --- ocrd/ocrd/network/database.py | 9 +++ ocrd/ocrd/network/models/job.py | 5 +- ocrd/ocrd/network/models/processor.py | 17 ---- ocrd/ocrd/network/processing_broker.py | 79 ++++++++++++------- .../network/rabbitmq_utils/ocrd_messages.py | 24 +++--- ocrd/requirements.txt | 1 + 6 files changed, 73 insertions(+), 62 deletions(-) create mode 100644 ocrd/ocrd/network/database.py delete mode 100644 ocrd/ocrd/network/models/processor.py diff --git a/ocrd/ocrd/network/database.py b/ocrd/ocrd/network/database.py new file mode 100644 index 000000000..b61d7f130 --- /dev/null +++ b/ocrd/ocrd/network/database.py @@ -0,0 +1,9 @@ +from beanie import init_beanie +from motor.motor_asyncio import AsyncIOMotorClient + +from ocrd.network.models.job import Job + + +async def initiate_database(db_url: str): + client = AsyncIOMotorClient(db_url) + await init_beanie(database=client.get_default_database(default='ocrd'), document_models=[Job]) diff --git a/ocrd/ocrd/network/models/job.py b/ocrd/ocrd/network/models/job.py index 4d48fbc95..44dda7e0d 100644 --- a/ocrd/ocrd/network/models/job.py +++ b/ocrd/ocrd/network/models/job.py @@ -22,16 +22,18 @@ class StateEnum(str, Enum): class JobInput(BaseModel): + processor_name: str path: str description: Optional[str] = None input_file_grps: List[str] output_file_grps: Optional[List[str]] page_id: Optional[str] = None - parameters: dict = None # Always set to an empty dict when it's None, otherwise it won't pass the ocrd validation + parameters: dict = {} # Always set to an empty dict when it's None, otherwise it won't pass the ocrd validation class Config: schema_extra = { 'example': { + 'processor_name': 'ocrd-dummy', 'path': '/path/to/mets.xml', 'description': 'The description of this execution', 'input_file_grps': ['INPUT_FILE_GROUP'], @@ -43,6 +45,7 @@ class Config: class Job(Document): + processor_name: str path: str description: Optional[str] state: StateEnum diff --git a/ocrd/ocrd/network/models/processor.py b/ocrd/ocrd/network/models/processor.py deleted file mode 100644 index a6272de93..000000000 --- a/ocrd/ocrd/network/models/processor.py +++ /dev/null @@ -1,17 +0,0 @@ -from pydantic import BaseModel -from typing import Dict, Any, Optional - - -class ProcessorArgs(BaseModel): - workspace_id: str = '' - input_file_grps: str = '' - output_file_grps: str = '' - page_id: str = '' - parameters: Optional[Dict[str, Any]] = {} - - -# TODO: this does not conform to the openapi.yml. It should?! -class ProcessorJob(BaseModel): - job_id: str - workspace_id: str - processor_name: str diff --git a/ocrd/ocrd/network/processing_broker.py b/ocrd/ocrd/network/processing_broker.py index f1aa0c94a..9949df6c8 100644 --- a/ocrd/ocrd/network/processing_broker.py +++ b/ocrd/ocrd/network/processing_broker.py @@ -1,18 +1,24 @@ -from fastapi import FastAPI, status +from fastapi import FastAPI, status, HTTPException import uvicorn from time import sleep from yaml import safe_load from typing import List -from ocrd_utils import getLogger +from ocrd_utils import ( + getLogger, + get_ocrd_tool_json, +) from ocrd_validators import ProcessingBrokerValidator from ocrd.network.deployer import Deployer from ocrd.network.deployment_config import ProcessingBrokerConfig from ocrd.network.rabbitmq_utils import RMQPublisher, OcrdProcessingMessage from ocrd.network.helpers import construct_dummy_processing_message, get_workspaces_dir -from ocrd.network.models.processor import ProcessorArgs, ProcessorJob +from ocrd.network.models.job import Job, JobInput, StateEnum from pathlib import Path +from ocrd_validators import ParameterValidator +from ocrd.network.database import initiate_database + # TODO: rename to ProcessingServer (module-file too) class ProcessingBroker(FastAPI): @@ -22,7 +28,7 @@ class ProcessingBroker(FastAPI): def __init__(self, config_path: str, host: str, port: int) -> None: # TODO: set other args: title, description, version, openapi_tags - super().__init__(on_shutdown=[self.on_shutdown]) + super().__init__(on_startup=[self.on_startup], on_shutdown=[self.on_shutdown]) self.log = getLogger(__name__) self.hostname = host @@ -68,12 +74,15 @@ def __init__(self, config_path: str, host: str, port: int) -> None: ) self.router.add_api_route( - path='/processor/{processor_name}', + path='/process', endpoint=self.run_processor, methods=['POST'], tags=['processing'], status_code=status.HTTP_200_OK, - operation_id='runProcessor', + summary='Submit a job to this processor', + response_model=Job, + response_model_exclude_unset=True, + response_model_exclude_none=True ) def start(self) -> None: @@ -95,7 +104,7 @@ def start(self) -> None: rabbitmq_url = self.deployer.deploy_rabbitmq() # Deploy the MongoDB, get the URL of the deployed agent - mongodb_url = self.deployer.deploy_mongodb() + self.mongodb_url = self.deployer.deploy_mongodb() # The RMQPublisher is initialized and a connection to the RabbitMQ is performed self.connect_publisher() @@ -105,7 +114,7 @@ def start(self) -> None: # Deploy processing hosts where processing workers are running on # Note: A deployed processing worker starts listening to a message queue with id processor.name - self.deployer.deploy_hosts(self.hosts_config, rabbitmq_url, mongodb_url) + self.deployer.deploy_hosts(self.hosts_config, rabbitmq_url, self.mongodb_url) self.log.debug(f'Starting uvicorn: {self.hostname}:{self.port}') uvicorn.run(self, host=self.hostname, port=self.port) @@ -119,6 +128,11 @@ def parse_config(config_path: str) -> ProcessingBrokerConfig: raise Exception(f'Processing-Broker configuration file is invalid:\n{report.errors}') return ProcessingBrokerConfig(obj) + async def on_startup(self): + self.log.debug('jetzt kommt das mit der Datenbank') + await initiate_database(db_url=self.mongodb_url) + self.log.debug('das mit der Datenbank ist durch') + async def on_shutdown(self) -> None: # TODO: shutdown docker containers """ @@ -126,17 +140,9 @@ async def on_shutdown(self) -> None: - ensure queue is empty or processor is not currently running - connect to hosts and kill pids """ - # TODO: remove the try/except before beta. This is only needed for development. All - # exceptions this function (on_shutdown) throws are ignored / not printed, when it is used - # as shutdown-hook as it is now. So this try/except and logging is neccessary to make them - # visible when testing - try: - await self.stop_deployed_agents() - # TODO: This except block is trapping the user if nothing is following after the keyword. - # Is that the expected behaviour here? - except: - self.log.debug('error stopping processing servers: ', exc_info=True) - raise + # Errors are logged if the logger is configured in a specific way. Seems to conflict with + # somehow ocrd-logging-configuration + await self.stop_deployed_agents() async def stop_deployed_agents(self) -> None: self.deployer.kill_all() @@ -178,19 +184,32 @@ def publish_default_processing_message(self) -> None: raise Exception('RMQPublisher is not connected') # TODO: how do we want to do the whole model-stuff? Webapi (openapi.yml) uses ProcessorJob - def run_processor(self, processor_name: str, p_args: ProcessorArgs) -> ProcessorJob: + async def run_processor(self, data: JobInput) -> Job: # TODO: save job in mongodb and get a job-id this way. Look into how this was done in pr-884 - job_id = 'dummy-id-1' - # TODO: how do we want that stuff with the workspaces to work? - workspaces_path = get_workspaces_dir() - processing_message = OcrdProcessingMessage.from_params( - processor_name, job_id, workspaces_path, p_args - ) + job = Job(**data.dict(exclude_unset=True, exclude_none=True), state=StateEnum.queued) + await job.insert() + + if data.parameters: + # this validation with ocrd-tool also ensures that the processor is available + ocrd_tool = get_ocrd_tool_json(job.processor_name) + if not ocrd_tool: + # is available but it's ocr-d-tool is not? + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=(f'Processor \'{job.processor_name}\' not available. It\'s ocrd_tool is ' + 'empty or missing') + ) + validator = ParameterValidator(ocrd_tool) + report = validator.validate(data.parameters) + if not report.is_valid: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=report.errors, + ) + processing_message = OcrdProcessingMessage.from_job(job) encoded_processing_message = OcrdProcessingMessage.encode_yml(processing_message) if self.rmq_publisher: - self.rmq_publisher.publish_to_queue(processor_name, encoded_processing_message) + self.rmq_publisher.publish_to_queue(job.processor_name, encoded_processing_message) else: raise Exception('RMQPublisher is not connected') - - return ProcessorJob(job_id=job_id, workspace_id=p_args.workspace_id, - processor_name=processor_name) + return job diff --git a/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py b/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py index 5d6d86fe8..2a4d5b10b 100644 --- a/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py +++ b/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py @@ -4,7 +4,7 @@ from pickle import dumps, loads from typing import Any, Dict, List, Optional import yaml -from ocrd.network.models.processor import ProcessorArgs +from ocrd.network.models.job import Job from pathlib import Path @@ -21,7 +21,7 @@ def __init__( path_to_mets: str = None, workspace_id: str = None, input_file_grps: List[str] = None, - output_file_grps: List[str] = None, + output_file_grps: Optional[List[str]] = None, page_id: str = None, parameters: Dict[str, Any] = None, result_queue_name: str = None, @@ -102,19 +102,15 @@ def decode_yml(ocrd_processing_message: bytes) -> OcrdProcessingMessage: ) @staticmethod - def from_params(processor_name: str, job_id: str, workspaces_dir: str, p_args: ProcessorArgs - ) -> OcrdProcessingMessage: - input_file_grps = [x.strip() for x in filter(None, p_args.input_file_grps.split(','))] - output_file_grps = [x.strip() for x in filter(None, p_args.output_file_grps.split(','))] + def from_job(job: Job) -> OcrdProcessingMessage: return OcrdProcessingMessage( - job_id=job_id, - processor_name=processor_name, - path_to_mets=str(Path(workspaces_dir) / p_args.workspace_id / 'mets.xml'), - workspace_id=p_args.workspace_id, - input_file_grps=input_file_grps, - output_file_grps=output_file_grps, - page_id=p_args.page_id, - parameters=p_args.parameters, + job_id=job.id, + processor_name=job.processor_name, + path_to_mets=job.path, + input_file_grps=job.input_file_grps, + output_file_grps=job.output_file_grps, + page_id=job.page_id, + parameters=job.parameters, ) diff --git a/ocrd/requirements.txt b/ocrd/requirements.txt index ed2d22e64..9cefbe8fe 100644 --- a/ocrd/requirements.txt +++ b/ocrd/requirements.txt @@ -17,3 +17,4 @@ docker paramiko frozendict~=2.3.4 pika>=1.2.0 +beanie~=1.7 From 2323facdd08db7310f6a9d96563a0419cb9b7d00 Mon Sep 17 00:00:00 2001 From: joschrew Date: Thu, 26 Jan 2023 15:16:34 +0100 Subject: [PATCH 094/226] Add processor enpoints from previous impl (pr 884) --- ocrd/ocrd/network/models/job.py | 2 - ocrd/ocrd/network/processing_broker.py | 55 +++++++++++++++++++++----- 2 files changed, 45 insertions(+), 12 deletions(-) diff --git a/ocrd/ocrd/network/models/job.py b/ocrd/ocrd/network/models/job.py index 44dda7e0d..6ee9fc3a4 100644 --- a/ocrd/ocrd/network/models/job.py +++ b/ocrd/ocrd/network/models/job.py @@ -22,7 +22,6 @@ class StateEnum(str, Enum): class JobInput(BaseModel): - processor_name: str path: str description: Optional[str] = None input_file_grps: List[str] @@ -33,7 +32,6 @@ class JobInput(BaseModel): class Config: schema_extra = { 'example': { - 'processor_name': 'ocrd-dummy', 'path': '/path/to/mets.xml', 'description': 'The description of this execution', 'input_file_grps': ['INPUT_FILE_GROUP'], diff --git a/ocrd/ocrd/network/processing_broker.py b/ocrd/ocrd/network/processing_broker.py index 9949df6c8..67fbf82a9 100644 --- a/ocrd/ocrd/network/processing_broker.py +++ b/ocrd/ocrd/network/processing_broker.py @@ -1,8 +1,7 @@ from fastapi import FastAPI, status, HTTPException import uvicorn -from time import sleep from yaml import safe_load -from typing import List +from typing import Dict from ocrd_utils import ( getLogger, @@ -13,11 +12,11 @@ from ocrd.network.deployer import Deployer from ocrd.network.deployment_config import ProcessingBrokerConfig from ocrd.network.rabbitmq_utils import RMQPublisher, OcrdProcessingMessage -from ocrd.network.helpers import construct_dummy_processing_message, get_workspaces_dir +from ocrd.network.helpers import construct_dummy_processing_message from ocrd.network.models.job import Job, JobInput, StateEnum -from pathlib import Path from ocrd_validators import ParameterValidator from ocrd.network.database import initiate_database +from beanie import PydanticObjectId # TODO: rename to ProcessingServer (module-file too) @@ -57,6 +56,7 @@ def __init__(self, config_path: str, host: str, port: int) -> None: # Note for peer: Check under self.start() self.rmq_publisher = None + # Create routes self.router.add_api_route( path='/stop', endpoint=self.stop_deployed_agents, @@ -74,7 +74,7 @@ def __init__(self, config_path: str, host: str, port: int) -> None: ) self.router.add_api_route( - path='/process', + path='/process/{processor_name}', endpoint=self.run_processor, methods=['POST'], tags=['processing'], @@ -85,6 +85,27 @@ def __init__(self, config_path: str, host: str, port: int) -> None: response_model_exclude_none=True ) + self.router.add_api_route( + path='/process/{processor_name}/{job_id}', + endpoint=self.get_job, + methods=['GET'], + tags=['Processing'], + status_code=status.HTTP_200_OK, + summary='Get information about a job based on its ID', + response_model=Job, + response_model_exclude_unset=True, + response_model_exclude_none=True + ) + + self.router.add_api_route( + path='/process/{processor_name}', + endpoint=self.get_processor_info, + methods=['GET'], + tags=['Processing'], + status_code=status.HTTP_200_OK, + summary='Get information about this processor.', + ) + def start(self) -> None: """ deploy things and start the processing broker (aka server) with uvicorn @@ -184,19 +205,20 @@ def publish_default_processing_message(self) -> None: raise Exception('RMQPublisher is not connected') # TODO: how do we want to do the whole model-stuff? Webapi (openapi.yml) uses ProcessorJob - async def run_processor(self, data: JobInput) -> Job: + async def run_processor(self, processor_name: str, data: JobInput) -> Job: # TODO: save job in mongodb and get a job-id this way. Look into how this was done in pr-884 - job = Job(**data.dict(exclude_unset=True, exclude_none=True), state=StateEnum.queued) + job = Job(**data.dict(exclude_unset=True, exclude_none=True), processor_name=processor_name, + state=StateEnum.queued) await job.insert() if data.parameters: # this validation with ocrd-tool also ensures that the processor is available - ocrd_tool = get_ocrd_tool_json(job.processor_name) + ocrd_tool = get_ocrd_tool_json(processor_name) if not ocrd_tool: # is available but it's ocr-d-tool is not? raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=(f'Processor \'{job.processor_name}\' not available. It\'s ocrd_tool is ' + detail=(f'Processor \'{processor_name}\' not available. It\'s ocrd_tool is ' 'empty or missing') ) validator = ParameterValidator(ocrd_tool) @@ -209,7 +231,20 @@ async def run_processor(self, data: JobInput) -> Job: processing_message = OcrdProcessingMessage.from_job(job) encoded_processing_message = OcrdProcessingMessage.encode_yml(processing_message) if self.rmq_publisher: - self.rmq_publisher.publish_to_queue(job.processor_name, encoded_processing_message) + self.rmq_publisher.publish_to_queue(processor_name, encoded_processing_message) else: raise Exception('RMQPublisher is not connected') return job + + async def get_processor_info(self, processor_name) -> Dict: + return get_ocrd_tool_json(processor_name) + + async def get_job(self, processor_name: str, job_id: PydanticObjectId) -> Job: + # TODO: the state has to be set after processing is finished somewhere/somehow/sometime + job = await Job.get(job_id) + if job: + return job + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail='Job not found.' + ) From c57e75c312ea8310834ad470874b863893996d7b Mon Sep 17 00:00:00 2001 From: joschrew Date: Thu, 26 Jan 2023 15:42:12 +0100 Subject: [PATCH 095/226] Change comments and format and do some cleanup --- ocrd/ocrd/network/deployer.py | 74 +++++++++++--------------- ocrd/ocrd/network/processing_broker.py | 14 +++-- 2 files changed, 38 insertions(+), 50 deletions(-) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index d0f27896b..e9ef81f1c 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -1,5 +1,5 @@ from __future__ import annotations -from typing import List, Dict, Union +from typing import List, Dict, Union, Optional from paramiko import SSHClient from re import search as re_search @@ -12,12 +12,12 @@ CustomDockerClient, DeployType, ) -from ocrd.network.processing_worker import ProcessingWorker import time from pathlib import Path # Abstraction of the Deployment functionality -# The ProcessingServer (currently still called Broker) provides the configuration parameters to the Deployer agent. +# The ProcessingServer (currently still called Broker) provides the configuration parameters to the +# Deployer agent. # The Deployer agent deploys the RabbitMQ Server, MongoDB and the Processing Hosts. # Each Processing Host may have several Processing Workers. # Each Processing Worker is an instance of an OCR-D processor. @@ -100,14 +100,13 @@ def _deploy_processing_worker(self, processor: ProcessorConfig, host: HostConfig # TODO: The check for available ssh or docker connections should probably happen inside `deploy_hosts` if processor.deploy_type == DeployType.native: if not host.ssh_client: - host.ssh_client = create_ssh_client(host.address, host.username, host.password, host.keypath) - elif processor.deploy_type == DeployType.docker: - if not host.docker_client: - host.docker_client = create_docker_client(host.address, host.username, host.password, host.keypath) + host.ssh_client = create_ssh_client(host.address, host.username, host.password, + host.keypath) else: - # Error case, should never enter here. Handle error cases here (if needed) - self.log.error(f'Failed to deploy: {processor.name}. The deploy type is unknown.') - pass + assert processor.deploy_type == DeployType.docker + if not host.docker_client: + host.docker_client = create_docker_client(host.address, host.username, + host.password, host.keypath) for _ in range(processor.count): if processor.deploy_type == DeployType.native: @@ -120,7 +119,8 @@ def _deploy_processing_worker(self, processor: ProcessorConfig, host: HostConfig bin_dir=host.binpath, ) processor.add_started_pid(pid) - elif processor.deploy_type == DeployType.docker: + else: + assert processor.deploy_type == DeployType.docker assert host.docker_client # to satisfy mypy pid = self.start_docker_processor( client=host.docker_client, @@ -129,11 +129,6 @@ def _deploy_processing_worker(self, processor: ProcessorConfig, host: HostConfig database_url=mongodb_url ) processor.add_started_pid(pid) - else: - # TODO: Weirdly there is a duplication of code inside this method - # Error case, should never enter here. Handle error cases here (if needed) - self.log.error(f'Failed to deploy: {processor.name}. The deploy type is unknown.') - pass def deploy_rabbitmq(self, image: str = 'rabbitmq:3-management', detach: bool = True, remove: bool = True, ports_mapping: Union[Dict, None] = None) -> str: @@ -144,10 +139,8 @@ def deploy_rabbitmq(self, image: str = 'rabbitmq:3-management', detach: bool = T # Which is part of the OCR-D WebAPI implementation. self.log.debug(f'Trying to deploy image[{image}], with modes: detach[{detach}], remove[{remove}]') - if not self.mongo_data or not self.mongo_data.address: - self.log.error(f'Deploying RabbitMQ has failed - missing configuration.') - # TODO: Raise an error instead of silently ignoring it - return '' + if not self.mq_data or not self.mq_data.address: + raise ValueError('Deploying RabbitMQ has failed - missing configuration.') client = create_docker_client(self.mq_data.address, self.mq_data.username, self.mq_data.password, self.mq_data.keypath) @@ -204,9 +197,7 @@ def deploy_mongodb(self, image: str = 'mongo', detach: bool = True, remove: bool self.log.debug(f'Trying to deploy image[{image}], with modes: detach[{detach}], remove[{remove}]') if not self.mongo_data or not self.mongo_data.address: - self.log.error(f'Deploying MongoDB has failed - missing configuration.') - # TODO: Raise an error instead of silently ignoring it - return '' + raise ValueError('Deploying MongoDB has failed - missing configuration.') client = create_docker_client(self.mongo_data.address, self.mongo_data.username, self.mongo_data.password, self.mongo_data.keypath) @@ -215,16 +206,16 @@ def deploy_mongodb(self, image: str = 'mongo', detach: bool = True, remove: bool 27017: self.mongo_data.port } self.log.debug(f'Ports mapping: {ports_mapping}') - # TODO: use rm here or not? Should the mongodb be reused? - # TODO: what about the data-dir? Must data be preserved? + # TODO: what about the data-dir? Must data be preserved between runs? res = client.containers.run( image=image, detach=detach, remove=remove, ports=ports_mapping ) - assert res and res.id, \ - f'Failed to start MongoDB docker container on host: {self.mongo_data.address}' + if not res or not res.id: + raise RuntimeError('Failed to start MongoDB docker container on host: ' + f'{self.mongo_data.address}') self.mongo_data.pid = res.id client.close() @@ -237,10 +228,10 @@ def deploy_mongodb(self, image: str = 'mongo', detach: bool = True, remove: bool return mongodb_url def kill_rabbitmq(self) -> None: - # TODO: The PID must not be stored in the configuration `mq_data`. + # TODO: The PID must not be stored in the configuration `mq_data`. Why not? if not self.mq_data or not self.mq_data.pid: self.log.warning(f'No running RabbitMQ instance found') - # TODO: Ignoring this silently is problematic in the future + # TODO: Ignoring this silently is problematic in the future. Why? return self.log.debug(f'Trying to stop the deployed RabbitMQ with PID: {self.mq_data.pid}') @@ -252,10 +243,10 @@ def kill_rabbitmq(self) -> None: self.log.debug('The RabbitMQ is stopped') def kill_mongodb(self) -> None: - # TODO: The PID must not be stored in the configuration `mongo_data`. + # TODO: The PID must not be stored in the configuration `mongo_data`. Why not? if not self.mongo_data or not self.mongo_data.pid: self.log.warning(f'No running MongoDB instance found') - # TODO: Ignoring this silently is problematic in the future + # TODO: Ignoring this silently is problematic in the future. Why? return self.log.debug(f'Trying to stop the deployed MongoDB with PID: {self.mongo_data.pid}') @@ -272,9 +263,11 @@ def kill_hosts(self) -> None: for host in self.hosts: self.log.debug(f'Killing/Stopping processing workers on host: {host.address}') if host.ssh_client: - host.ssh_client = create_ssh_client(host.address, host.username, host.password, host.keypath) + host.ssh_client = create_ssh_client(host.address, host.username, host.password, + host.keypath) if host.docker_client: - host.docker_client = create_docker_client(host.address, host.username, host.password, host.keypath) + host.docker_client = create_docker_client(host.address, host.username, + host.password, host.keypath) # Kill deployed OCR-D processor instances on this Processing worker host self.kill_processing_worker(host) @@ -285,16 +278,13 @@ def kill_processing_worker(self, host: HostConfig) -> None: self.log.debug(f'Trying to kill/stop native processor: {processor.name}, with PID: {pid}') # TODO: For graceful shutdown we may want to send additional parameters to kill host.ssh_client.exec_command(f'kill {pid}') - elif processor.deploy_type.is_docker(): + else: + assert processor.deploy_type.is_docker() for pid in processor.pids: self.log.debug(f'Trying to kill/stop docker container processor: {processor.name}, with PID: {pid}') # TODO: think about timeout. # think about using threads to kill parallelized to reduce waiting time host.docker_client.containers.get(pid).stop() - else: - # Error case, should never enter here. Handle error cases here (if needed) - self.log.error(f'Failed to kill: {processor.name}. The deploy type is unknown.') - pass processor.pids = [] # Note: Invoking a pythonic processor is slightly different from the description in the spec. @@ -308,7 +298,7 @@ def kill_processing_worker(self, host: HostConfig) -> None: # E.g., olena-binarize def start_native_processor(self, client: SSHClient, processor_name: str, queue_url: str, - database_url: str, bin_dir: str = None) -> str: + database_url: str, bin_dir: Optional[str] = None) -> str: """ start a processor natively on a host via ssh Args: @@ -334,9 +324,9 @@ def start_native_processor(self, client: SSHClient, processor_name: str, queue_u cmd = f'{path} --database {database_url} --queue {queue_url}' # the only way (I could find) to make it work to start a process in the background and # return early is this construction. The pid of the last started background process is - # printed with `echo $!` but it is printed inbetween other output. Because of that I added - # `xyz` before and after the code to easily be able to filter out the pid via regex when - # returning from the function + # printed with `echo $!` but it is printed inbetween other output. Because of that I added + # `xyz` before and after the code to easily be able to filter out the pid via regex when + # returning from the function stdin.write(f'{cmd} & \n echo xyz$!xyz \n exit \n') output = stdout.read().decode('utf-8') self.log.debug(f'Output for processor {processor_name}: {output}') diff --git a/ocrd/ocrd/network/processing_broker.py b/ocrd/ocrd/network/processing_broker.py index 67fbf82a9..331f536e8 100644 --- a/ocrd/ocrd/network/processing_broker.py +++ b/ocrd/ocrd/network/processing_broker.py @@ -155,14 +155,11 @@ async def on_startup(self): self.log.debug('das mit der Datenbank ist durch') async def on_shutdown(self) -> None: - # TODO: shutdown docker containers """ - hosts and pids should be stored somewhere - ensure queue is empty or processor is not currently running - connect to hosts and kill pids """ - # Errors are logged if the logger is configured in a specific way. Seems to conflict with - # somehow ocrd-logging-configuration await self.stop_deployed_agents() async def stop_deployed_agents(self) -> None: @@ -183,7 +180,8 @@ def connect_publisher(self, username: str = 'default', password: str = 'default' self.log.debug('Successfully connected RMQPublisher.') def create_message_queues(self) -> None: - # Create the message queues based on the occurrence of `processor.name` in the config file + """Create the message queues based on the occurrence of `processor.name` in the config file + """ for host in self.hosts_config: for processor in host.processors: # The existence/validity of the processor.name is not tested. @@ -199,14 +197,14 @@ def publish_default_processing_message(self) -> None: encoded_processing_message = OcrdProcessingMessage.encode_yml(processing_message) if self.rmq_publisher: self.log.debug('Publishing the default processing message') - self.rmq_publisher.publish_to_queue(queue_name=queue_name, message=encoded_processing_message) + self.rmq_publisher.publish_to_queue(queue_name=queue_name, + message=encoded_processing_message) else: self.log.error('RMQPublisher is not connected') raise Exception('RMQPublisher is not connected') # TODO: how do we want to do the whole model-stuff? Webapi (openapi.yml) uses ProcessorJob async def run_processor(self, processor_name: str, data: JobInput) -> Job: - # TODO: save job in mongodb and get a job-id this way. Look into how this was done in pr-884 job = Job(**data.dict(exclude_unset=True, exclude_none=True), processor_name=processor_name, state=StateEnum.queued) await job.insert() @@ -215,7 +213,6 @@ async def run_processor(self, processor_name: str, data: JobInput) -> Job: # this validation with ocrd-tool also ensures that the processor is available ocrd_tool = get_ocrd_tool_json(processor_name) if not ocrd_tool: - # is available but it's ocr-d-tool is not? raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=(f'Processor \'{processor_name}\' not available. It\'s ocrd_tool is ' @@ -240,7 +237,8 @@ async def get_processor_info(self, processor_name) -> Dict: return get_ocrd_tool_json(processor_name) async def get_job(self, processor_name: str, job_id: PydanticObjectId) -> Job: - # TODO: the state has to be set after processing is finished somewhere/somehow/sometime + # TODO: the state has to be set after processing is finished somewhere in the + # processing-worker or the result_queue must be processd somewhere job = await Job.get(job_id) if job: return job From 868d58ded87f3c0a3876e16775520fa998b0b06e Mon Sep 17 00:00:00 2001 From: joschrew Date: Mon, 30 Jan 2023 08:26:19 +0100 Subject: [PATCH 096/226] Adapt to latest config changes --- ocrd/ocrd/network/deployer.py | 2 +- ocrd/ocrd/network/deployment_config.py | 15 ++++++++------- .../ocrd_validators/config.schema.yml | 16 ++++++++-------- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index e9ef81f1c..7e4381051 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -89,7 +89,7 @@ def deploy_hosts(self, hosts: List[HostConfig], rabbitmq_url: str, mongodb_url: # TODO: Creating connections if missing should probably occur when deploying hosts not when # deploying processing workers. The deploy_type checks and opening connections creates duplicate code. - def _deploy_processing_worker(self, processor: ProcessorConfig, host: HostConfig, + def _deploy_processing_worker(self, processor: WorkerConfig, host: HostConfig, rabbitmq_url: str, mongodb_url: str) -> None: self.log.debug(f'deploy \'{processor.deploy_type}\' processor: \'{processor}\' on' diff --git a/ocrd/ocrd/network/deployment_config.py b/ocrd/ocrd/network/deployment_config.py index 3d6f93db0..e5bdde58c 100644 --- a/ocrd/ocrd/network/deployment_config.py +++ b/ocrd/ocrd/network/deployment_config.py @@ -7,7 +7,7 @@ __all__ = [ 'ProcessingBrokerConfig', 'HostConfig', - 'ProcessorConfig', + 'WorkerConfig', 'MongoConfig', 'QueueConfig', ] @@ -15,8 +15,8 @@ class ProcessingBrokerConfig: def __init__(self, config: dict) -> None: - self.mongo_config = MongoConfig(config['mongo_db']) - self.queue_config = QueueConfig(config['message_queue']) + self.mongo_config = MongoConfig(config['database']) + self.queue_config = QueueConfig(config['process_queue']) self.hosts_config = [] for host in config['hosts']: self.hosts_config.append(HostConfig(host)) @@ -36,18 +36,19 @@ def __init__(self, config: dict) -> None: self.username = config['username'] self.password = config.get('password', None) self.keypath = config.get('path_to_privkey', None) + # TODO: this is only for testing. Remove here and from config.schema.yml after test/development-phase self.binpath = config.get('path_to_bin_dir', None) self.processors = [] - for processor in config['deploy_processors']: - deploy_type = DeployType.from_str(processor['deploy_type']) + for worker in config['workers']: + deploy_type = DeployType.from_str(worker['deploy_type']) self.processors.append( - ProcessorConfig(processor['name'], processor['number_of_instance'], deploy_type) + WorkerConfig(worker['name'], worker['number_of_instance'], deploy_type) ) self.ssh_client = None self.docker_client = None -class ProcessorConfig: +class WorkerConfig: """ Class wrapping information from config file for an OCR-D processor """ diff --git a/ocrd_validators/ocrd_validators/config.schema.yml b/ocrd_validators/ocrd_validators/config.schema.yml index 3dac8bfd0..0db5b85fd 100644 --- a/ocrd_validators/ocrd_validators/config.schema.yml +++ b/ocrd_validators/ocrd_validators/config.schema.yml @@ -4,9 +4,9 @@ description: Schema for the Processing Broker configuration file type: object additionalProperties: false required: - - message_queue + - process_queue properties: - message_queue: + process_queue: description: Information about the Message Queue type: object additionalProperties: false @@ -15,7 +15,7 @@ properties: - port properties: address: - description: The IP address or domain name of the machine where Message Queue is deployed + description: The IP address or domain name of the machine where the Message Queue is deployed $ref: "#/$defs/address" port: description: The port number of the Message Queue @@ -26,7 +26,7 @@ properties: ssh: description: Information required for an SSH connection $ref: "#/$defs/ssh" - mongo_db: + database: description: Information about the MongoDB type: object additionalProperties: false @@ -57,7 +57,7 @@ properties: required: - address - username - - deploy_processors + - workers oneOf: - required: - password @@ -75,10 +75,10 @@ properties: description: Path to private key file type: string path_to_bin_dir: - description: Path to where processor executables can be found on the target machine + description: TODO - remove again after testing/development phase type: string - deploy_processors: - description: List of processors which will be deployed + workers: + description: List of workers which will be deployed type: array minItems: 1 items: From 53e9b475db919b173ca8eceb4d7b3195fe9c72b5 Mon Sep 17 00:00:00 2001 From: joschrew Date: Mon, 30 Jan 2023 13:34:29 +0100 Subject: [PATCH 097/226] Use definitions.json for rabbitmq --- ocrd/ocrd/network/deployer.py | 22 +++++++--------------- ocrd/ocrd/network/processing_broker.py | 9 +++++++-- ocrd/ocrd/network/processing_worker.py | 3 ++- 3 files changed, 16 insertions(+), 18 deletions(-) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index 7e4381051..2038c97c3 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -155,6 +155,8 @@ def deploy_rabbitmq(self, image: str = 'rabbitmq:3-management', detach: bool = T 25672: 25672 } self.log.debug(f'Ports mapping: {ports_mapping}') + local_defs_path = Path(__file__).parent.resolve() / 'rabbitmq_utils' / 'definitions.json' + container_defs_path = "/etc/rabbitmq/definitions.json" res = client.containers.run( image=image, detach=detach, @@ -162,13 +164,15 @@ def deploy_rabbitmq(self, image: str = 'rabbitmq:3-management', detach: bool = T ports=ports_mapping, environment=[ f'RABBITMQ_DEFAULT_USER={self.mq_data.credentials[0]}', - f'RABBITMQ_DEFAULT_PASS={self.mq_data.credentials[1]}' - ] + f'RABBITMQ_DEFAULT_PASS={self.mq_data.credentials[1]}', + ('RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS=' + f'-rabbitmq_management load_definitions "{container_defs_path}"'), + ], + volumes={local_defs_path: {'bind': container_defs_path, 'mode': 'ro'}} ) assert res and res.id, \ f'Failed to start RabbitMQ docker container on host: {self.mq_data.address}' self.mq_data.pid = res.id - self.init_rabbitmq(client, res.id) client.close() # Build the RabbitMQ Server URL to return @@ -180,18 +184,6 @@ def deploy_rabbitmq(self, image: str = 'rabbitmq:3-management', detach: bool = T self.log.debug(f'The RabbitMQ server was deployed on url: {rabbitmq_url}') return rabbitmq_url - def init_rabbitmq(self, client: CustomDockerClient, rabbitmq_id: str) -> None: - """ Add users, wait for startup - """ - # TODO: rabbitmq was started, but needs some time to be ready to use. sleep is working for - # now but the init-process should rather be made in a loop and be redone on error - # until the container is responsive or a timeout is expired (and raise in this case) - time.sleep(3) - container = client.containers.get(rabbitmq_id) - container.exec_run('rabbitmqctl add_user default default') - container.exec_run('rabbitmqctl set_user_tags default administrator') - container.exec_run('rabbitmqctl set_permissions -p / default ".*" ".*" ".*"') - def deploy_mongodb(self, image: str = 'mongo', detach: bool = True, remove: bool = True, ports_mapping: Union[Dict, None] = None) -> str: self.log.debug(f'Trying to deploy image[{image}], with modes: detach[{detach}], remove[{remove}]') diff --git a/ocrd/ocrd/network/processing_broker.py b/ocrd/ocrd/network/processing_broker.py index 331f536e8..13b4c838b 100644 --- a/ocrd/ocrd/network/processing_broker.py +++ b/ocrd/ocrd/network/processing_broker.py @@ -17,6 +17,7 @@ from ocrd_validators import ParameterValidator from ocrd.network.database import initiate_database from beanie import PydanticObjectId +from time import sleep # TODO: rename to ProcessingServer (module-file too) @@ -127,6 +128,10 @@ def start(self) -> None: # Deploy the MongoDB, get the URL of the deployed agent self.mongodb_url = self.deployer.deploy_mongodb() + # Give enough time for the RabbitMQ server to get deployed and get fully configured + # Needed to prevent connection of the publisher before the RabbitMQ is deployed + sleep(3) # TODO: Sleeping here is bad and better check should be performed + # The RMQPublisher is initialized and a connection to the RabbitMQ is performed self.connect_publisher() @@ -165,8 +170,8 @@ async def on_shutdown(self) -> None: async def stop_deployed_agents(self) -> None: self.deployer.kill_all() - def connect_publisher(self, username: str = 'default', password: str = 'default', - enable_acks: bool =True) -> None: + def connect_publisher(self, username: str = 'default-publisher', + password: str = 'default-publisher', enable_acks: bool =True) -> None: self.log.debug(f'Connecting RMQPublisher to RabbitMQ server: {self.rmq_host}:{self.rmq_port}{self.rmq_vhost}') self.rmq_publisher = RMQPublisher(host=self.rmq_host, port=self.rmq_port, vhost=self.rmq_vhost) # TODO: Remove this information before the release diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index 93c8c0d37..bcb539f57 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -57,7 +57,8 @@ def __init__(self, rabbitmq_addr, mongodb_addr, processor_name, ocrd_tool: dict, # the message queue with name {processor_name}-result, the RMQPublisher should be instantiated # self.rmq_publisher = None - def connect_consumer(self, username: str = 'default', password: str = 'default') -> None: + def connect_consumer(self, username: str = 'default-consumer', + password: str = 'default-consumer') -> None: self.log.debug(f'Connecting RMQConsumer to RabbitMQ server: {self.rmq_host}:{self.rmq_port}{self.rmq_vhost}') self.rmq_consumer = RMQConsumer(host=self.rmq_host, port=self.rmq_port, vhost=self.rmq_vhost) # TODO: Remove this information before the release From 72a8e5cf949e47bf1563a6ac22fc848afc0547fc Mon Sep 17 00:00:00 2001 From: joschrew Date: Mon, 30 Jan 2023 14:52:24 +0100 Subject: [PATCH 098/226] Rename processing broker to processing server --- ocrd/ocrd/cli/__init__.py | 4 ++-- ...cessing_broker.py => processing_server.py} | 21 +++++++++--------- ocrd/ocrd/network/__init__.py | 4 ++-- ocrd/ocrd/network/deployer.py | 2 +- ocrd/ocrd/network/deployment_config.py | 4 ++-- ...cessing_broker.py => processing_server.py} | 22 +++++++++---------- ocrd/ocrd/network/rabbitmq_utils/connector.py | 2 +- ocrd/ocrd/network/rabbitmq_utils/publisher.py | 2 +- ocrd_validators/ocrd_validators/__init__.py | 4 ++-- ...ator.py => processing_server_validator.py} | 8 +++---- 10 files changed, 36 insertions(+), 37 deletions(-) rename ocrd/ocrd/cli/{processing_broker.py => processing_server.py} (59%) rename ocrd/ocrd/network/{processing_broker.py => processing_server.py} (94%) rename ocrd_validators/ocrd_validators/{processing_broker_validator.py => processing_server_validator.py} (80%) diff --git a/ocrd/ocrd/cli/__init__.py b/ocrd/ocrd/cli/__init__.py index a429ad7dc..d645daddf 100644 --- a/ocrd/ocrd/cli/__init__.py +++ b/ocrd/ocrd/cli/__init__.py @@ -31,7 +31,7 @@ def get_help(self, ctx): from ocrd.decorators import ocrd_loglevel from .zip import zip_cli from .log import log_cli -from .processing_broker import processing_broker_cli +from .processing_server import processing_server_cli from .processing_worker import processing_worker_cli @@ -51,5 +51,5 @@ def cli(**kwargs): # pylint: disable=unused-argument cli.add_command(validate_cli) cli.add_command(log_cli) cli.add_command(resmgr_cli) -cli.add_command(processing_broker_cli) +cli.add_command(processing_server_cli) cli.add_command(processing_worker_cli) diff --git a/ocrd/ocrd/cli/processing_broker.py b/ocrd/ocrd/cli/processing_server.py similarity index 59% rename from ocrd/ocrd/cli/processing_broker.py rename to ocrd/ocrd/cli/processing_server.py index ce9dba30d..27b1a91c1 100644 --- a/ocrd/ocrd/cli/processing_broker.py +++ b/ocrd/ocrd/cli/processing_server.py @@ -1,23 +1,23 @@ """ -OCR-D CLI: start the processing broker +OCR-D CLI: start the processing server -.. click:: ocrd.cli.processing_broker:zip_cli - :prog: ocrd processing-broker +.. click:: ocrd.cli.processing_server:zip_cli + :prog: ocrd processing-server :nested: full """ import click from ocrd_utils import initLogging -from ocrd.network import ProcessingBroker +from ocrd.network import ProcessingServer import logging # TODO: rename to processing-server -@click.command('processing-broker') +@click.command('processing-server') @click.argument('path_to_config', required=True, type=click.STRING) -@click.option('-a', '--address', help='Host (name/IP) and port to bind the Processing-Broker to. Example: localhost:8080', required=True) -def processing_broker_cli(path_to_config, address: str): +@click.option('-a', '--address', help='Host (name/IP) and port to bind the Processing-Server to. Example: localhost:8080', required=True) +def processing_server_cli(path_to_config, address: str): """ - Start and manage processing servers (workers) with the processing broker + Start and manage processing servers (workers) with the processing server """ initLogging() # TODO: Remove before the release @@ -29,6 +29,5 @@ def processing_broker_cli(path_to_config, address: str): port_int = int(port) except ValueError: raise click.UsageError('The --adddress option must have the format IP:PORT') - processing_broker = ProcessingBroker(path_to_config, host, port_int) - # Start the Processing Broker aka the Processing Server (the new name) - processing_broker.start() + processing_server = ProcessingServer(path_to_config, host, port_int) + processing_server.start() diff --git a/ocrd/ocrd/network/__init__.py b/ocrd/ocrd/network/__init__.py index aba941597..62868cdf5 100644 --- a/ocrd/ocrd/network/__init__.py +++ b/ocrd/ocrd/network/__init__.py @@ -6,7 +6,7 @@ # https://github.com/OCR-D/ocrd-webapi-implementation # 2. The RabbitMQ Library (i.e., utils) is available here: # https://github.com/OCR-D/ocrd-webapi-implementation/tree/main/ocrd_webapi/rabbitmq -# 3. Some potentially more useful code to be adopted for the Processing Broker/Worker is available here: +# 3. Some potentially more useful code to be adopted for the Processing Server/Worker is available here: # https://github.com/OCR-D/core/pull/884 # 4. The Mets Server discussion/implementation is available here: # https://github.com/OCR-D/core/pull/966 @@ -17,5 +17,5 @@ # This package, currently, is under the `core/ocrd` package. # It could also be a separate package on its own under `core` with the name `ocrd_network`. # TODO: Correctly identify all current and potential future dependencies. -from .processing_broker import ProcessingBroker +from .processing_server import ProcessingServer from .processing_worker import ProcessingWorker diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index 2038c97c3..faaf5a87a 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -16,7 +16,7 @@ from pathlib import Path # Abstraction of the Deployment functionality -# The ProcessingServer (currently still called Broker) provides the configuration parameters to the +# The ProcessingServer (currently still called Server) provides the configuration parameters to the # Deployer agent. # The Deployer agent deploys the RabbitMQ Server, MongoDB and the Processing Hosts. # Each Processing Host may have several Processing Workers. diff --git a/ocrd/ocrd/network/deployment_config.py b/ocrd/ocrd/network/deployment_config.py index e5bdde58c..e859ed5ee 100644 --- a/ocrd/ocrd/network/deployment_config.py +++ b/ocrd/ocrd/network/deployment_config.py @@ -5,7 +5,7 @@ from ocrd.network.deployment_utils import DeployType __all__ = [ - 'ProcessingBrokerConfig', + 'ProcessingServerConfig', 'HostConfig', 'WorkerConfig', 'MongoConfig', @@ -13,7 +13,7 @@ ] -class ProcessingBrokerConfig: +class ProcessingServerConfig: def __init__(self, config: dict) -> None: self.mongo_config = MongoConfig(config['database']) self.queue_config = QueueConfig(config['process_queue']) diff --git a/ocrd/ocrd/network/processing_broker.py b/ocrd/ocrd/network/processing_server.py similarity index 94% rename from ocrd/ocrd/network/processing_broker.py rename to ocrd/ocrd/network/processing_server.py index 13b4c838b..36494221f 100644 --- a/ocrd/ocrd/network/processing_broker.py +++ b/ocrd/ocrd/network/processing_server.py @@ -7,10 +7,10 @@ getLogger, get_ocrd_tool_json, ) -from ocrd_validators import ProcessingBrokerValidator +from ocrd_validators import ProcessingServerValidator from ocrd.network.deployer import Deployer -from ocrd.network.deployment_config import ProcessingBrokerConfig +from ocrd.network.deployment_config import ProcessingServerConfig from ocrd.network.rabbitmq_utils import RMQPublisher, OcrdProcessingMessage from ocrd.network.helpers import construct_dummy_processing_message from ocrd.network.models.job import Job, JobInput, StateEnum @@ -21,9 +21,9 @@ # TODO: rename to ProcessingServer (module-file too) -class ProcessingBroker(FastAPI): +class ProcessingServer(FastAPI): """ - TODO: doc for ProcessingBroker and its methods + TODO: doc for ProcessingServer and its methods """ def __init__(self, config_path: str, host: str, port: int) -> None: @@ -35,7 +35,7 @@ def __init__(self, config_path: str, host: str, port: int) -> None: self.port = port # TODO: Ideally the parse_config should return a Tuple with the 3 configs assigned below # to prevent passing the entire parsed config around to methods. - parsed_config = ProcessingBroker.parse_config(config_path) + parsed_config = ProcessingServer.parse_config(config_path) self.queue_config = parsed_config.queue_config self.mongo_config = parsed_config.mongo_config self.hosts_config = parsed_config.hosts_config @@ -109,12 +109,12 @@ def __init__(self, config_path: str, host: str, port: int) -> None: def start(self) -> None: """ - deploy things and start the processing broker (aka server) with uvicorn + deploy things and start the processing server with uvicorn """ """ Note for a peer: Deploying everything together at once is a bad approach. First the RabbitMQ Server and the MongoDB - should be deployed. Then the RMQPublisher of the Processing Broker (aka Processing Server) should + should be deployed. Then the RMQPublisher of the Processing Server (aka Processing Server) should connect to the running RabbitMQ server. After that point the Processing Workers should be deployed. The RMQPublisher should be connected before deploying Processing Workers because the message queues to which the Processing Workers listen to are created based on the deployed processor. @@ -146,13 +146,13 @@ def start(self) -> None: uvicorn.run(self, host=self.hostname, port=self.port) @staticmethod - def parse_config(config_path: str) -> ProcessingBrokerConfig: + def parse_config(config_path: str) -> ProcessingServerConfig: with open(config_path) as fin: obj = safe_load(fin) - report = ProcessingBrokerValidator.validate(obj) + report = ProcessingServerValidator.validate(obj) if not report.is_valid: - raise Exception(f'Processing-Broker configuration file is invalid:\n{report.errors}') - return ProcessingBrokerConfig(obj) + raise Exception(f'Processing-Server configuration file is invalid:\n{report.errors}') + return ProcessingServerConfig(obj) async def on_startup(self): self.log.debug('jetzt kommt das mit der Datenbank') diff --git a/ocrd/ocrd/network/rabbitmq_utils/connector.py b/ocrd/ocrd/network/rabbitmq_utils/connector.py index d64a466cd..46abbc1e8 100644 --- a/ocrd/ocrd/network/rabbitmq_utils/connector.py +++ b/ocrd/ocrd/network/rabbitmq_utils/connector.py @@ -192,7 +192,7 @@ def queue_declare( # Only check to see if the queue exists and # raise ChannelClosed exception if it does not passive=passive, - # Survive reboots of the broker + # Survive reboots of the server durable=durable, # Only allow access by the current connection exclusive=exclusive, diff --git a/ocrd/ocrd/network/rabbitmq_utils/publisher.py b/ocrd/ocrd/network/rabbitmq_utils/publisher.py index 850e8b805..971a578d0 100644 --- a/ocrd/ocrd/network/rabbitmq_utils/publisher.py +++ b/ocrd/ocrd/network/rabbitmq_utils/publisher.py @@ -118,7 +118,7 @@ def publish_to_queue( if properties is None: headers = {'OCR-D WebApi Header': 'OCR-D WebApi Value'} properties = BasicProperties( - app_id='webapi-processing-broker', + app_id='webapi-processing-server', content_type='application/json', headers=headers ) diff --git a/ocrd_validators/ocrd_validators/__init__.py b/ocrd_validators/ocrd_validators/__init__.py index 39acb9482..b461c897c 100644 --- a/ocrd_validators/ocrd_validators/__init__.py +++ b/ocrd_validators/ocrd_validators/__init__.py @@ -11,7 +11,7 @@ 'XsdValidator', 'XsdMetsValidator', 'XsdPageValidator', - 'ProcessingBrokerValidator', + 'ProcessingServerValidator', ] from .parameter_validator import ParameterValidator @@ -23,4 +23,4 @@ from .xsd_validator import XsdValidator from .xsd_mets_validator import XsdMetsValidator from .xsd_page_validator import XsdPageValidator -from .processing_broker_validator import ProcessingBrokerValidator +from .processing_server_validator import ProcessingServerValidator diff --git a/ocrd_validators/ocrd_validators/processing_broker_validator.py b/ocrd_validators/ocrd_validators/processing_server_validator.py similarity index 80% rename from ocrd_validators/ocrd_validators/processing_broker_validator.py rename to ocrd_validators/ocrd_validators/processing_server_validator.py index 0e0c26262..e3b5c9d8b 100644 --- a/ocrd_validators/ocrd_validators/processing_broker_validator.py +++ b/ocrd_validators/ocrd_validators/processing_server_validator.py @@ -1,5 +1,5 @@ """ -Validating configuration file for the Processing-Broker +Validating configuration file for the Processing-Server """ from .json_validator import JsonValidator import yaml @@ -10,15 +10,15 @@ # working link. Currently it is here: # https://github.com/OCR-D/spec/pull/222/files#diff-a71bf71cbc7d9ce94fded977f7544aba4df9e7bdb8fc0cf1014e14eb67a9b273 # But that is a PR not merged yet -class ProcessingBrokerValidator(JsonValidator): +class ProcessingServerValidator(JsonValidator): """ - JsonValidator validating against the schema for the Processing Broker + JsonValidator validating against the schema for the Processing Server """ @staticmethod def validate(obj): """ - Validate against schema for Processing-Broker + Validate against schema for Processing-Server """ schema = yaml.safe_load(resource_string(__name__, 'config.schema.yml')) return JsonValidator.validate(obj, schema) From 0f399eaac6929b3b580ccc27cd4584942dd5883e Mon Sep 17 00:00:00 2001 From: joschrew Date: Mon, 30 Jan 2023 16:40:53 +0100 Subject: [PATCH 099/226] Set job state after worker finishes --- ocrd/ocrd/network/processing_worker.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index bcb539f57..75116de38 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -9,7 +9,7 @@ from frozendict import frozendict from functools import lru_cache, wraps import json -from typing import List, Callable, Type, Union +from typing import List, Callable, Type, Union, Any import pika.spec import pika.adapters.blocking_connection @@ -22,11 +22,13 @@ verify_database_url, verify_and_parse_rabbitmq_addr ) +from ocrd.network.models.job import StateEnum from ocrd.network.rabbitmq_utils import ( OcrdProcessingMessage, OcrdResultMessage, RMQConsumer ) +import pymongo class ProcessingWorker: @@ -153,7 +155,7 @@ def process_message(self, processing_message: OcrdProcessingMessage) -> None: if self.processor_class: self.log.debug(f'Invoking the pythonic processor: {self.processor_name}') self.log.debug(f'Invoking the processor_class: {self.processor_class}') - self.run_processor_from_worker( + success = self.run_processor_from_worker( processor_class=self.processor_class, workspace=workspace, page_id=page_id, @@ -163,7 +165,7 @@ def process_message(self, processing_message: OcrdProcessingMessage) -> None: ) else: self.log.debug(f'Invoking the cli: {self.processor_name}') - self.run_cli_from_worker( + success = self.run_cli_from_worker( executable=self.processor_name, workspace=workspace, page_id=page_id, @@ -171,6 +173,7 @@ def process_message(self, processing_message: OcrdProcessingMessage) -> None: output_file_grps=output_file_grps, parameter=parameter ) + self.set_job_state(job_id, success) def run_processor_from_worker( self, @@ -180,7 +183,7 @@ def run_processor_from_worker( input_file_grps: List[str], output_file_grps: List[str], parameter: dict, - ): + ) -> bool: input_file_grps_str = ','.join(input_file_grps) output_file_grps_str = ','.join(output_file_grps) @@ -203,6 +206,7 @@ def run_processor_from_worker( self.log.error(f'{processor_class} failed with an exception.') else: self.log.debug(f'{processor_class} exited with success.') + return success def run_cli_from_worker( self, @@ -212,7 +216,7 @@ def run_cli_from_worker( input_file_grps: List[str], output_file_grps: List[str], parameter: dict - ): + ) -> bool: input_file_grps_str = ','.join(input_file_grps) output_file_grps_str = ','.join(output_file_grps) @@ -230,6 +234,17 @@ def run_cli_from_worker( self.log.error(f'{executable} exited with non-zero return value {return_code}.') else: self.log.debug(f'{executable} exited with success.') + return return_code == 0 + + def set_job_state(self, job_id: Any, success: bool): + """Set the job status in mongodb to either success or failed + """ + # TODO: the way to interact with mongodb needs to be thought about. This is to make it work + # for now for better testing. Beanie seems not suitable as the worker is not async + state = StateEnum.success if success else StateEnum.failed + with pymongo.MongoClient(self.db_url) as client: + db = client['ocrd'] + db.Job.update_one({'_id': job_id}, {'$set': {'state': state}}, upsert=False) # Method adopted from Triet's implementation From 8143614bd16d02552af340d613fbf31a71732e7b Mon Sep 17 00:00:00 2001 From: joschrew Date: Mon, 30 Jan 2023 17:40:18 +0100 Subject: [PATCH 100/226] Add enpoint to list processors --- ocrd/ocrd/network/processing_server.py | 31 ++++++++++++++++++++------ 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/ocrd/ocrd/network/processing_server.py b/ocrd/ocrd/network/processing_server.py index 36494221f..ce2f94650 100644 --- a/ocrd/ocrd/network/processing_server.py +++ b/ocrd/ocrd/network/processing_server.py @@ -1,7 +1,7 @@ from fastapi import FastAPI, status, HTTPException import uvicorn from yaml import safe_load -from typing import Dict +from typing import Dict, Set from ocrd_utils import ( getLogger, @@ -18,6 +18,7 @@ from ocrd.network.database import initiate_database from beanie import PydanticObjectId from time import sleep +import json # TODO: rename to ProcessingServer (module-file too) @@ -75,7 +76,7 @@ def __init__(self, config_path: str, host: str, port: int) -> None: ) self.router.add_api_route( - path='/process/{processor_name}', + path='/processor/{processor_name}', endpoint=self.run_processor, methods=['POST'], tags=['processing'], @@ -87,10 +88,10 @@ def __init__(self, config_path: str, host: str, port: int) -> None: ) self.router.add_api_route( - path='/process/{processor_name}/{job_id}', + path='/processor/{processor_name}/{job_id}', endpoint=self.get_job, methods=['GET'], - tags=['Processing'], + tags=['processing'], status_code=status.HTTP_200_OK, summary='Get information about a job based on its ID', response_model=Job, @@ -99,12 +100,21 @@ def __init__(self, config_path: str, host: str, port: int) -> None: ) self.router.add_api_route( - path='/process/{processor_name}', + path='/processor/{processor_name}', endpoint=self.get_processor_info, methods=['GET'], - tags=['Processing'], + tags=['processing', 'discovery'], status_code=status.HTTP_200_OK, - summary='Get information about this processor.', + summary='Get information about this processor', + ) + + self.router.add_api_route( + path='/processor', + endpoint=self.list_processors, + methods=['GET'], + tags=['processing', 'discovery'], + status_code=status.HTTP_200_OK, + summary='Get a list of all available processors', ) def start(self) -> None: @@ -251,3 +261,10 @@ async def get_job(self, processor_name: str, job_id: PydanticObjectId) -> Job: status_code=status.HTTP_404_NOT_FOUND, detail='Job not found.' ) + + async def list_processors(self) -> str: + res = set([]) + for host in self.hosts_config: + for processor in host.processors: + res.add(processor.name) + return json.dumps(list(res)) From 871323bbe5dbed628a73d3a07cc8dd16e57d4d2b Mon Sep 17 00:00:00 2001 From: joschrew Date: Thu, 2 Feb 2023 08:31:27 +0100 Subject: [PATCH 101/226] Remove redundant log output --- ocrd/ocrd/network/processing_server.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ocrd/ocrd/network/processing_server.py b/ocrd/ocrd/network/processing_server.py index ce2f94650..edaaf0c49 100644 --- a/ocrd/ocrd/network/processing_server.py +++ b/ocrd/ocrd/network/processing_server.py @@ -165,9 +165,7 @@ def parse_config(config_path: str) -> ProcessingServerConfig: return ProcessingServerConfig(obj) async def on_startup(self): - self.log.debug('jetzt kommt das mit der Datenbank') await initiate_database(db_url=self.mongodb_url) - self.log.debug('das mit der Datenbank ist durch') async def on_shutdown(self) -> None: """ From 7f4f1d704954a38afcd74bbb23a215890c9c36fd Mon Sep 17 00:00:00 2001 From: joschrew Date: Thu, 2 Feb 2023 10:00:03 +0100 Subject: [PATCH 102/226] Ensure processor availability before processing --- ocrd/ocrd/network/processing_server.py | 49 +++++++++++++++++--------- 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/ocrd/ocrd/network/processing_server.py b/ocrd/ocrd/network/processing_server.py index edaaf0c49..1cc0c2abb 100644 --- a/ocrd/ocrd/network/processing_server.py +++ b/ocrd/ocrd/network/processing_server.py @@ -58,6 +58,8 @@ def __init__(self, config_path: str, host: str, port: int) -> None: # Note for peer: Check under self.start() self.rmq_publisher = None + self._processor_list = None + # Create routes self.router.add_api_route( path='/stop', @@ -216,14 +218,26 @@ def publish_default_processing_message(self) -> None: self.log.error('RMQPublisher is not connected') raise Exception('RMQPublisher is not connected') + @property + def processor_list(self): + if self._processor_list: + return self._processor_list + res = set([]) + for host in self.hosts_config: + for processor in host.processors: + res.add(processor.name) + self._processor_list = list(res) + return self._processor_list + # TODO: how do we want to do the whole model-stuff? Webapi (openapi.yml) uses ProcessorJob async def run_processor(self, processor_name: str, data: JobInput) -> Job: - job = Job(**data.dict(exclude_unset=True, exclude_none=True), processor_name=processor_name, - state=StateEnum.queued) - await job.insert() - + self.log.debug('processing_server.run_processor() called') + if processor_name not in self.processor_list: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail='Processor not available' + ) if data.parameters: - # this validation with ocrd-tool also ensures that the processor is available ocrd_tool = get_ocrd_tool_json(processor_name) if not ocrd_tool: raise HTTPException( @@ -234,10 +248,11 @@ async def run_processor(self, processor_name: str, data: JobInput) -> Job: validator = ParameterValidator(ocrd_tool) report = validator.validate(data.parameters) if not report.is_valid: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=report.errors, - ) + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=report.errors) + + job = Job(**data.dict(exclude_unset=True, exclude_none=True), processor_name=processor_name, + state=StateEnum.queued) + await job.insert() processing_message = OcrdProcessingMessage.from_job(job) encoded_processing_message = OcrdProcessingMessage.encode_yml(processing_message) if self.rmq_publisher: @@ -247,11 +262,16 @@ async def run_processor(self, processor_name: str, data: JobInput) -> Job: return job async def get_processor_info(self, processor_name) -> Dict: + self.log.debug('processing_server.get_processor_info() called') + if processor_name not in self.processor_list: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail='Processor not available' + ) return get_ocrd_tool_json(processor_name) async def get_job(self, processor_name: str, job_id: PydanticObjectId) -> Job: - # TODO: the state has to be set after processing is finished somewhere in the - # processing-worker or the result_queue must be processd somewhere + self.log.debug('processing_server.get_job() called') job = await Job.get(job_id) if job: return job @@ -261,8 +281,5 @@ async def get_job(self, processor_name: str, job_id: PydanticObjectId) -> Job: ) async def list_processors(self) -> str: - res = set([]) - for host in self.hosts_config: - for processor in host.processors: - res.add(processor.name) - return json.dumps(list(res)) + self.log.debug('processing_server.list_processors() called') + return json.dumps(self.processor_list) From 1caffd6658c1107d412ab897e74723d33492102b Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Thu, 2 Feb 2023 16:49:19 +0100 Subject: [PATCH 103/226] Implement the result-queue logic - worker uses RMQPublisher --- ocrd/ocrd/network/processing_worker.py | 52 ++++++++++++++++--- .../network/rabbitmq_utils/ocrd_messages.py | 19 +++++++ 2 files changed, 63 insertions(+), 8 deletions(-) diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index 75116de38..7bd8b5378 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -26,7 +26,8 @@ from ocrd.network.rabbitmq_utils import ( OcrdProcessingMessage, OcrdResultMessage, - RMQConsumer + RMQConsumer, + RMQPublisher ) import pymongo @@ -52,12 +53,14 @@ def __init__(self, rabbitmq_addr, mongodb_addr, processor_name, ocrd_tool: dict, # Think of this as a func pointer to the constructor of the respective OCR-D processor self.processor_class = processor_class - # Gets assigned when `connect_consumer` is called on the working object + # Gets assigned when `connect_consumer` is called on the worker object + # Used to consume OcrdProcessingMessage from the queue with name {processor_name} self.rmq_consumer = None - # TODO: In case the processing worker should publish OcrdResultMessage type message to - # the message queue with name {processor_name}-result, the RMQPublisher should be instantiated - # self.rmq_publisher = None + # Gets assigned when the `connect_publisher` is called on the worker object + # The publisher is connected when the `result_queue` field of the OcrdProcessingMessage is set for first time + # Used to publish OcrdResultMessage type message to the queue with name {processor_name}-result + self.rmq_publisher = None def connect_consumer(self, username: str = 'default-consumer', password: str = 'default-consumer') -> None: @@ -68,6 +71,20 @@ def connect_consumer(self, username: str = 'default-consumer', self.rmq_consumer.authenticate_and_connect(username=username, password=password) self.log.debug(f'Successfully connected RMQConsumer.') + def connect_publisher(self, username: str = 'default-publisher', + password: str = 'default-publisher', enable_acks: bool = True) -> None: + self.log.debug(f'Connecting RMQPublisher to RabbitMQ server: {self.rmq_host}:{self.rmq_port}{self.rmq_vhost}') + self.rmq_publisher = RMQPublisher(host=self.rmq_host, port=self.rmq_port, vhost=self.rmq_vhost) + # TODO: Remove this information before the release + self.log.debug(f'RMQPublisher authenticates with username: {username}, password: {password}') + self.rmq_publisher.authenticate_and_connect(username=username, password=password) + if enable_acks: + self.rmq_publisher.enable_delivery_confirmations() + self.log.debug('Delivery confirmations are enabled') + else: + self.log.debug('Delivery confirmations are disabled') + self.log.debug('Successfully connected RMQPublisher.') + # Define what happens every time a message is consumed # from the queue with name self.processor_name def on_consumed_message( @@ -155,7 +172,7 @@ def process_message(self, processing_message: OcrdProcessingMessage) -> None: if self.processor_class: self.log.debug(f'Invoking the pythonic processor: {self.processor_name}') self.log.debug(f'Invoking the processor_class: {self.processor_class}') - success = self.run_processor_from_worker( + job_status = self.run_processor_from_worker( processor_class=self.processor_class, workspace=workspace, page_id=page_id, @@ -165,7 +182,7 @@ def process_message(self, processing_message: OcrdProcessingMessage) -> None: ) else: self.log.debug(f'Invoking the cli: {self.processor_name}') - success = self.run_cli_from_worker( + job_status = self.run_cli_from_worker( executable=self.processor_name, workspace=workspace, page_id=page_id, @@ -173,7 +190,26 @@ def process_message(self, processing_message: OcrdProcessingMessage) -> None: output_file_grps=output_file_grps, parameter=parameter ) - self.set_job_state(job_id, success) + self.set_job_state(job_id, job_status) + + # If the result_queue field is set, send the job status to a result queue + if processing_message.result_queue: + if self.rmq_publisher is None: + self.connect_publisher() + + # create_queue method is idempotent - nothing happens if + # a queue with the specified name already exists + self.rmq_publisher.create_queue(queue_name=processing_message.result_queue) + self.rmq_publisher.publish_to_queue( + queue_name=processing_message.result_queue, + message=OcrdResultMessage( + job_id=job_id, + status=job_status, + # Either path_to_mets or workspace_id must be set (mutually exclusive) + path_to_mets=processing_message.path_to_mets, + workspace_id=None + ) + ) def run_processor_from_worker( self, diff --git a/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py b/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py index 2a4d5b10b..e9479a015 100644 --- a/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py +++ b/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py @@ -135,3 +135,22 @@ def decode(ocrd_result_message: bytes, encoding: str = 'utf-8') -> OcrdResultMes workspace_id=data.workspace_id, path_to_mets=data.path_to_mets ) + + @staticmethod + def encode_yml(ocrd_result_message: OcrdResultMessage) -> bytes: + """convert OcrdResultMessage to yml + """ + return yaml.dump(ocrd_result_message.__dict__, indent=2).encode('utf-8') + + @staticmethod + def decode_yml(ocrd_result_message: bytes) -> OcrdResultMessage: + """Parse OcrdResultMessage from yml + """ + msg = ocrd_result_message.decode('utf-8') + data = yaml.load(msg, Loader=yaml.Loader) + return OcrdResultMessage( + job_id=data.get('job_id', None), + status=data.get('status', None), + path_to_mets=data.get('path_to_mets', None), + workspace_id=data.get('workspace_id', None), + ) From a16aa02fadc16cce55df6870272b1606293d99ad Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Thu, 2 Feb 2023 17:02:20 +0100 Subject: [PATCH 104/226] Send encoded result message --- ocrd/ocrd/network/processing_worker.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index 7bd8b5378..c847f48d3 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -200,15 +200,18 @@ def process_message(self, processing_message: OcrdProcessingMessage) -> None: # create_queue method is idempotent - nothing happens if # a queue with the specified name already exists self.rmq_publisher.create_queue(queue_name=processing_message.result_queue) + + result_message = OcrdResultMessage( + job_id=job_id, + status=job_status, + # Either path_to_mets or workspace_id must be set (mutually exclusive) + path_to_mets=processing_message.path_to_mets, + workspace_id=None + ) + encoded_result_message = OcrdResultMessage.encode_yml(result_message) self.rmq_publisher.publish_to_queue( queue_name=processing_message.result_queue, - message=OcrdResultMessage( - job_id=job_id, - status=job_status, - # Either path_to_mets or workspace_id must be set (mutually exclusive) - path_to_mets=processing_message.path_to_mets, - workspace_id=None - ) + message=encoded_result_message ) def run_processor_from_worker( From ab326f7b686fd78e4ad1b0a5a158f6c546544b66 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Wed, 8 Feb 2023 16:44:19 +0100 Subject: [PATCH 105/226] Add exception handler to debug validation --- ocrd/ocrd/network/processing_server.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/ocrd/ocrd/network/processing_server.py b/ocrd/ocrd/network/processing_server.py index 1cc0c2abb..1e0def1a4 100644 --- a/ocrd/ocrd/network/processing_server.py +++ b/ocrd/ocrd/network/processing_server.py @@ -1,4 +1,6 @@ -from fastapi import FastAPI, status, HTTPException +from fastapi import FastAPI, status, Request, HTTPException +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse import uvicorn from yaml import safe_load from typing import Dict, Set @@ -119,6 +121,13 @@ def __init__(self, config_path: str, host: str, port: int) -> None: summary='Get a list of all available processors', ) + @self.exception_handler(RequestValidationError) + async def validation_exception_handler(request: Request, exc: RequestValidationError): + exc_str = f'{exc}'.replace('\n', ' ').replace(' ', ' ') + self.log.error(f"{request}: {exc_str}") + content = {'status_code': 10422, 'message': exc_str, 'data': None} + return JSONResponse(content=content, status_code=status.HTTP_422_UNPROCESSABLE_ENTITY) + def start(self) -> None: """ deploy things and start the processing server with uvicorn From 4368b539ce09b3591387bf6de16476cbfddb9b1b Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Thu, 9 Feb 2023 17:57:37 +0100 Subject: [PATCH 106/226] Workers push finished job status to result queue --- ocrd/ocrd/network/processing_worker.py | 15 ++++++--------- ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py | 2 +- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index c847f48d3..0c98352d4 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -165,14 +165,11 @@ def process_message(self, processing_message: OcrdProcessingMessage) -> None: # Note to peer: We should encapsulate database related actions to keep this method simple job_id = processing_message.job_id - if processing_message.result_queue: - self.log.warning(f'Publishing results to a message queue from the Processing Worker is not supported yet') - # TODO: Currently, no caching is performed. if self.processor_class: self.log.debug(f'Invoking the pythonic processor: {self.processor_name}') self.log.debug(f'Invoking the processor_class: {self.processor_class}') - job_status = self.run_processor_from_worker( + return_status = self.run_processor_from_worker( processor_class=self.processor_class, workspace=workspace, page_id=page_id, @@ -182,7 +179,7 @@ def process_message(self, processing_message: OcrdProcessingMessage) -> None: ) else: self.log.debug(f'Invoking the cli: {self.processor_name}') - job_status = self.run_cli_from_worker( + return_status = self.run_cli_from_worker( executable=self.processor_name, workspace=workspace, page_id=page_id, @@ -190,7 +187,8 @@ def process_message(self, processing_message: OcrdProcessingMessage) -> None: output_file_grps=output_file_grps, parameter=parameter ) - self.set_job_state(job_id, job_status) + job_status = StateEnum.success if return_status else StateEnum.failed + self.set_job_state(job_id, return_status) # If the result_queue field is set, send the job status to a result queue if processing_message.result_queue: @@ -200,10 +198,9 @@ def process_message(self, processing_message: OcrdProcessingMessage) -> None: # create_queue method is idempotent - nothing happens if # a queue with the specified name already exists self.rmq_publisher.create_queue(queue_name=processing_message.result_queue) - result_message = OcrdResultMessage( - job_id=job_id, - status=job_status, + job_id=str(job_id), + status=job_status.value, # Either path_to_mets or workspace_id must be set (mutually exclusive) path_to_mets=processing_message.path_to_mets, workspace_id=None diff --git a/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py b/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py index e9479a015..1a235c278 100644 --- a/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py +++ b/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py @@ -48,7 +48,7 @@ def __init__( # e.g., "PHYS_0005..PHYS_0010" will process only pages between 5-10 self.page_id = page_id if page_id else None # e.g., "ocrd-cis-ocropy-binarize-result" - self.result_queue = result_queue_name + self.result_queue = result_queue_name if result_queue_name else (self.processor_name + "-result") # processor parameters self.parameters = parameters if parameters else None self.created_time = created_time From e2452e473ed57731f38b770fd7071e32a3474cf5 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Thu, 9 Feb 2023 18:07:21 +0100 Subject: [PATCH 107/226] Example NF script (still broken) executed by the WF Server to submit jobs to Processing Server --- ocrd/ocrd/network/assets/wf_server_example.nf | 202 ++++++++++++++++++ 1 file changed, 202 insertions(+) create mode 100644 ocrd/ocrd/network/assets/wf_server_example.nf diff --git a/ocrd/ocrd/network/assets/wf_server_example.nf b/ocrd/ocrd/network/assets/wf_server_example.nf new file mode 100644 index 000000000..220c7d141 --- /dev/null +++ b/ocrd/ocrd/network/assets/wf_server_example.nf @@ -0,0 +1,202 @@ +@Grab(group='org.springframework.boot', module='spring-boot-starter-amqp', version='2.2.2.RELEASE') +import org.springframework.amqp.core.* +import org.springframework.amqp.rabbit.connection.CachingConnectionFactory +import org.springframework.amqp.rabbit.core.RabbitAdmin +import org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer +import org.springframework.amqp.rabbit.listener.api.* +import java.io.BufferedWriter +import java.io.OutputStreamWriter +import java.nio.charset.Charset +nextflow.enable.dsl=2 + +// These parameters can also be overwritten with values passed from the CLI +// when executing this script, i.e., --processing_server_address address +params.processing_server_address = "localhost:8080" +params.mets = "/home/mm/Desktop/example_ws/data/mets.xml" +// This is the entry point for the first ocr-d processor call in the Workflow +params.input_file_grp = "OCR-D-IMG" + +params.rmq_address = "localhost:5672" +params.rmq_username = "default-consumer" +params.rmq_password = "default-consumer" +params.rmq_exchange = "ocrd-network-default" + + +log.info """\ + O C R - D - W O R K F L O W - W E B A P I - 1 + ====================================================== + processing_server_address : ${params.processing_server_address} + mets : ${params.mets} + input_file_grp : ${params.input_file_grp} + """ + .stripIndent() + + +// This global variable is used to track the status of +// the previous process job in the when block of the current process job +job_status_flag = "NONE" + + +// RabbitMQ related globals +rmq_uri = "amqp://${params.rmq_username}:${params.rmq_password}@${params.rmq_address}" +println(rmq_uri) + +def produce_job_input_json(input_grp, output_grp, page_id, ocrd_params){ + // TODO: Using string builder should be more computationally efficient + def json_body = """{"path": "${params.mets}",""" + if (input_grp != null) + json_body = json_body + """ "input_file_grps": ["${input_grp}"]""" + if (output_grp != null) + json_body = json_body + """, "output_file_grps": ["${output_grp}"]""" + if (page_id != null) + json_body = json_body + """, "page_id": ${page_id}""" + if (ocrd_params != null) + json_body = json_body + """, "parameters": ${ocrd_params}""" + else + json_body = json_body + """, "parameters": {}""" + + json_body = json_body + """}""" + return json_body +} + +def post_processing_job(ocrd_processor, input_grp, output_grp, page_id, ocrd_params){ + def post_connection = new URL("http://${params.processing_server_address}/processor/${ocrd_processor}").openConnection() + post_connection.setDoOutput(true) + post_connection.setRequestMethod("POST") + post_connection.setRequestProperty("accept", "application/json") + post_connection.setRequestProperty("Content-Type", "application/json") + + def json_body = produce_job_input_json(input_grp, output_grp, page_id, ocrd_params) + println(json_body) + + def httpRequestBodyWriter = new BufferedWriter(new OutputStreamWriter(post_connection.getOutputStream())) + httpRequestBodyWriter.write(json_body) + httpRequestBodyWriter.close() + + def response_code = post_connection.getResponseCode() + println("Response code: " + response_code) + if (response_code.equals(200)){ + def json = post_connection.getInputStream().getText() + println("ResponseJSON: " + json) + } +} + +def configure_queue_listener(result_queue_name){ + cf = new CachingConnectionFactory(new URI(rmq_uri)) + def rmq_admin = new RabbitAdmin(cf) + def rmq_exchange = new DirectExchange(params.rmq_exchange, false, false) + rmq_admin.declareExchange(rmq_exchange) + + def rmq_queue = new Queue(result_queue_name, false) + rmq_admin.declareQueue(rmq_queue) + rmq_admin.declareBinding(BindingBuilder.bind(rmq_queue).to(rmq_exchange).withQueueName()) + + def listener = new SimpleMessageListenerContainer() + listener.setConnectionFactory(cf) + listener.setQueues(rmq_queue) + listener.setMessageListener(new ChannelAwareMessageListener() { + @Override + void onMessage(Message message, com.rabbitmq.client.Channel channel) { + println "Message received ${new Date()}" + // println (message) + def delivery_tag = message.getMessageProperties().getDeliveryTag() + def consumer_tag = message.getMessageProperties().getConsumerTag() + println "Consumer tag: ${consumer_tag}" + println "Delivery tag: ${delivery_tag}" + job_status = find_job_status(parse_body(message.getBody())) + println "JobStatus: ${job_status}" + // Overwrites the global status flag + job_status_flag = job_status + channel.basicAck(delivery_tag, false) + // channel.basicCancel(consumer_tag) + println "Trying to stop listener" + // channel.close(0, "Closing the channel after successful consumption.") + listener.stop() + println "After stop listener" + } + String parse_body(byte[] bytes) { + if (bytes) { + new String(bytes, Charset.forName('UTF-8')) + } + } + String find_job_status(String message_body){ + // TODO: Use Regex + if (message_body.contains("SUCCESS")){ + return "SUCCESS" + } + else if (message_body.contains("FAILED")){ + return "FAILED" + } + else if (message_body.contains("RUNNING")){ + return "RUNNING" + } + else if (message_body.contains("QUEUED")){ + return "QUEUED" + } + else { + return "NONE" + } + } + }) + + return listener +} + +def exec_block_logic(ocrd_processor_str, input_dir, output_dir, page_id, ocrd_params){ + String result_queue = "${ocrd_processor_str}-result" + post_processing_job(ocrd_processor_str, input_dir, output_dir, null, null) + job_status_flag = "INITIALIZED" + def listener = configure_queue_listener(result_queue) + // The job_status_flag is with value "INITIALIZED" here + println "Starting listening, flag: ${job_status_flag}" + listener.start() + // The job_status_flag must be with value "SUCCESS" or "FAILED" here + println "Ended listening, flag: ${job_status_flag}" + + // The job_status_flag gets overwritten inside the onMessage + // method when a message is consumed from the result queue + return job_status_flag +} + +process binarize { + maxForks 1 + + input: + val input_dir + val output_dir + + output: + val output_dir + val job_status + + exec: + job_status = exec_block_logic("ocrd-cis-ocropy-binarize", input_dir, output_dir, null, null) + println "binarize returning flag: ${job_status}" +} + +process crop { + maxForks 1 + + input: + val input_dir + val output_dir + val prev_job_status + + when: + prev_job_status == "SUCCESS" + + output: + val output_dir + val job_status + + exec: + job_status = exec_block_logic("ocrd-anybaseocr-crop", input_dir, output_dir, null, null) + println "crop returning flag: returning job status: ${job_status}" +} + +workflow { + + main: + binarize(params.input_file_grp, "OCR-D-BIN") + crop(binarize.out[0], "OCR-D-CROP", binarize.out[1]) +} From de3b41a9cf6f82ea2e368f71664da7db7a2d64f7 Mon Sep 17 00:00:00 2001 From: joschrew Date: Thu, 9 Feb 2023 17:53:59 +0100 Subject: [PATCH 108/226] Add workspace_id parameter to processor-call --- ocrd/ocrd/network/models/job.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ocrd/ocrd/network/models/job.py b/ocrd/ocrd/network/models/job.py index 6ee9fc3a4..15aaea13f 100644 --- a/ocrd/ocrd/network/models/job.py +++ b/ocrd/ocrd/network/models/job.py @@ -22,7 +22,8 @@ class StateEnum(str, Enum): class JobInput(BaseModel): - path: str + path: Optional[str] = None + workspace_id: Optional[str] = None description: Optional[str] = None input_file_grps: List[str] output_file_grps: Optional[List[str]] From 8b583d9fd261fc047e69a55204b7f0025d59026e Mon Sep 17 00:00:00 2001 From: joschrew Date: Thu, 9 Feb 2023 17:55:14 +0100 Subject: [PATCH 109/226] Add workspace model basically that from webapi --- ocrd/ocrd/network/database.py | 6 +++++- ocrd/ocrd/network/models/workspace.py | 26 ++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 ocrd/ocrd/network/models/workspace.py diff --git a/ocrd/ocrd/network/database.py b/ocrd/ocrd/network/database.py index b61d7f130..33ed8ef2c 100644 --- a/ocrd/ocrd/network/database.py +++ b/ocrd/ocrd/network/database.py @@ -2,8 +2,12 @@ from motor.motor_asyncio import AsyncIOMotorClient from ocrd.network.models.job import Job +from ocrd.network.models.workspace import Workspace async def initiate_database(db_url: str): client = AsyncIOMotorClient(db_url) - await init_beanie(database=client.get_default_database(default='ocrd'), document_models=[Job]) + await init_beanie( + database=client.get_default_database(default='ocrd'), + document_models=[Job, Workspace] + ) diff --git a/ocrd/ocrd/network/models/workspace.py b/ocrd/ocrd/network/models/workspace.py new file mode 100644 index 000000000..300b4ef4f --- /dev/null +++ b/ocrd/ocrd/network/models/workspace.py @@ -0,0 +1,26 @@ +from beanie import Document +from typing import Optional + + +class Workspace(Document): + """ + Model to store a workspace in the mongo-database. + + Information to handle workspaces and from bag-info.txt are stored here. + + Attributes: + ocrd_identifier Ocrd-Identifier (mandatory) + bagit_profile_identifier BagIt-Profile-Identifier (mandatory) + ocrd_base_version_checksum Ocrd-Base-Version-Checksum (mandatory) + ocrd_mets Ocrd-Mets (optional) + bag_info_adds bag-info.txt can also (optionally) contain additional + key-value-pairs which are saved here + """ + id: str + workspace_mets_path: str + ocrd_identifier: str + bagit_profile_identifier: str + ocrd_base_version_checksum: Optional[str] + ocrd_mets: Optional[str] + bag_info_adds: Optional[dict] + deleted: bool = False From 5d218a15f2879b750992dd1c7e262a6dee58e498 Mon Sep 17 00:00:00 2001 From: joschrew Date: Thu, 9 Feb 2023 17:57:48 +0100 Subject: [PATCH 110/226] Add workspace_id/path param-check to run_processor --- ocrd/ocrd/network/processing_server.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ocrd/ocrd/network/processing_server.py b/ocrd/ocrd/network/processing_server.py index 1e0def1a4..2588f5f7b 100644 --- a/ocrd/ocrd/network/processing_server.py +++ b/ocrd/ocrd/network/processing_server.py @@ -246,6 +246,12 @@ async def run_processor(self, processor_name: str, data: JobInput) -> Job: status_code=status.HTTP_404_NOT_FOUND, detail='Processor not available' ) + if bool(data.path) == bool(data.workspace_id): + print(f"DATA: {data.__dict__}") + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail='Either \'path\' or \'workspace_id\' must be set' + ) if data.parameters: ocrd_tool = get_ocrd_tool_json(processor_name) if not ocrd_tool: From 1142449067c0bea1049c1ed3d8c41f13d8508f7c Mon Sep 17 00:00:00 2001 From: joschrew Date: Fri, 10 Feb 2023 13:49:05 +0100 Subject: [PATCH 111/226] Resolve mets path with workspace id --- ocrd/ocrd/network/processing_server.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/ocrd/ocrd/network/processing_server.py b/ocrd/ocrd/network/processing_server.py index 2588f5f7b..53a9d51bf 100644 --- a/ocrd/ocrd/network/processing_server.py +++ b/ocrd/ocrd/network/processing_server.py @@ -16,6 +16,7 @@ from ocrd.network.rabbitmq_utils import RMQPublisher, OcrdProcessingMessage from ocrd.network.helpers import construct_dummy_processing_message from ocrd.network.models.job import Job, JobInput, StateEnum +from ocrd.network.models.workspace import Workspace from ocrd_validators import ParameterValidator from ocrd.network.database import initiate_database from beanie import PydanticObjectId @@ -246,12 +247,8 @@ async def run_processor(self, processor_name: str, data: JobInput) -> Job: status_code=status.HTTP_404_NOT_FOUND, detail='Processor not available' ) - if bool(data.path) == bool(data.workspace_id): - print(f"DATA: {data.__dict__}") - raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail='Either \'path\' or \'workspace_id\' must be set' - ) + + # validate additional parameters if data.parameters: ocrd_tool = get_ocrd_tool_json(processor_name) if not ocrd_tool: @@ -265,6 +262,21 @@ async def run_processor(self, processor_name: str, data: JobInput) -> Job: if not report.is_valid: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=report.errors) + # determine path to mets if workspace_id is provided + if bool(data.path) == bool(data.workspace_id): + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail='Either \'path\' or \'workspace_id\' must be set' + ) + elif data.workspace_id: + workspace = await Workspace.get(data.workspace_id) + if not workspace: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=f'Workspace for id \'{data.workspace_id}\' not existing' + ) + data.path = workspace.workspace_mets_path + job = Job(**data.dict(exclude_unset=True, exclude_none=True), processor_name=processor_name, state=StateEnum.queued) await job.insert() From be0fe51672ac70c368182baa534e1bb98f2b3f3e Mon Sep 17 00:00:00 2001 From: joschrew Date: Fri, 10 Feb 2023 14:29:59 +0100 Subject: [PATCH 112/226] Check off some trivial todos --- ocrd/ocrd/network/deployer.py | 4 ++ ocrd/ocrd/network/processing_server.py | 59 ++++++++++++-------------- 2 files changed, 32 insertions(+), 31 deletions(-) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index faaf5a87a..a1d0d2872 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -132,6 +132,8 @@ def _deploy_processing_worker(self, processor: WorkerConfig, host: HostConfig, def deploy_rabbitmq(self, image: str = 'rabbitmq:3-management', detach: bool = True, remove: bool = True, ports_mapping: Union[Dict, None] = None) -> str: + """Start docker-container with rabbitmq + """ # Note for a peer # This method deploys the RabbitMQ Server. # Handling of creation of queues, submitting messages to queues, @@ -186,6 +188,8 @@ def deploy_rabbitmq(self, image: str = 'rabbitmq:3-management', detach: bool = T def deploy_mongodb(self, image: str = 'mongo', detach: bool = True, remove: bool = True, ports_mapping: Union[Dict, None] = None) -> str: + """ Start mongodb in docker + """ self.log.debug(f'Trying to deploy image[{image}], with modes: detach[{detach}], remove[{remove}]') if not self.mongo_data or not self.mongo_data.address: diff --git a/ocrd/ocrd/network/processing_server.py b/ocrd/ocrd/network/processing_server.py index 53a9d51bf..8ad691086 100644 --- a/ocrd/ocrd/network/processing_server.py +++ b/ocrd/ocrd/network/processing_server.py @@ -24,15 +24,21 @@ import json -# TODO: rename to ProcessingServer (module-file too) class ProcessingServer(FastAPI): - """ - TODO: doc for ProcessingServer and its methods + """FastAPI app to make ocr-d processor calls + + The Processing-Server receives calls conforming to the ocr-d webapi regarding the processoing part. + It can run ocrd-processors and provides enpoints to discover processors and watch the job + status. + The Processing-Server does not execute the processors itself but starts up a queue and a + database to delegate the calls to processing workers. They are started by the Processing-Server + and the communication goes through the queue. """ def __init__(self, config_path: str, host: str, port: int) -> None: - # TODO: set other args: title, description, version, openapi_tags - super().__init__(on_startup=[self.on_startup], on_shutdown=[self.on_shutdown]) + super().__init__(on_startup=[self.on_startup], on_shutdown=[self.on_shutdown], + title="OCR-D Processing Server", + description="OCR-D processing and processors") self.log = getLogger(__name__) self.hostname = host @@ -68,16 +74,17 @@ def __init__(self, config_path: str, host: str, port: int) -> None: path='/stop', endpoint=self.stop_deployed_agents, methods=['POST'], - # tags=['TODO: add a tag'], - # summary='TODO: summary for api desc', - # TODO: add response model? add a response body at all? + tags=['tools'], + summary='Stop database, queue and processing-workers', ) + # TODO: do we still need this? Remove it?! self.router.add_api_route( path='/test-dummy', endpoint=self.publish_default_processing_message, methods=['POST'], - status_code=status.HTTP_202_ACCEPTED + status_code=status.HTTP_202_ACCEPTED, + summary='Was this just for testing or do we need this' ) self.router.add_api_route( @@ -130,24 +137,10 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE return JSONResponse(content=content, status_code=status.HTTP_422_UNPROCESSABLE_ENTITY) def start(self) -> None: + """ deploy agents (db, queue, workers) and start the processing server with uvicorn """ - deploy things and start the processing server with uvicorn - """ - """ - Note for a peer: - Deploying everything together at once is a bad approach. First the RabbitMQ Server and the MongoDB - should be deployed. Then the RMQPublisher of the Processing Server (aka Processing Server) should - connect to the running RabbitMQ server. After that point the Processing Workers should be deployed. - The RMQPublisher should be connected before deploying Processing Workers because the message queues to - which the Processing Workers listen to are created based on the deployed processor. - """ - # Deploy everything specified in the configuration - # self.deployer.deploy_all() - - # Deploy the RabbitMQ Server, get the URL of the deployed agent rabbitmq_url = self.deployer.deploy_rabbitmq() - # Deploy the MongoDB, get the URL of the deployed agent self.mongodb_url = self.deployer.deploy_mongodb() # Give enough time for the RabbitMQ server to get deployed and get fully configured @@ -218,7 +211,7 @@ def create_message_queues(self) -> None: def publish_default_processing_message(self) -> None: processing_message = construct_dummy_processing_message() queue_name = processing_message.processor_name - # TODO: switch back to pickle?! + # TODO: switch back to pickle?! imo we should go with pickle or json and remove one of them encoded_processing_message = OcrdProcessingMessage.encode_yml(processing_message) if self.rmq_publisher: self.log.debug('Publishing the default processing message') @@ -241,14 +234,15 @@ def processor_list(self): # TODO: how do we want to do the whole model-stuff? Webapi (openapi.yml) uses ProcessorJob async def run_processor(self, processor_name: str, data: JobInput) -> Job: - self.log.debug('processing_server.run_processor() called') + """ Queue a processor job + """ if processor_name not in self.processor_list: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail='Processor not available' ) - # validate additional parameters + # validate additional parameters if data.parameters: ocrd_tool = get_ocrd_tool_json(processor_name) if not ocrd_tool: @@ -262,7 +256,7 @@ async def run_processor(self, processor_name: str, data: JobInput) -> Job: if not report.is_valid: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=report.errors) - # determine path to mets if workspace_id is provided + # determine path to mets if workspace_id is provided if bool(data.path) == bool(data.workspace_id): raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, @@ -289,7 +283,8 @@ async def run_processor(self, processor_name: str, data: JobInput) -> Job: return job async def get_processor_info(self, processor_name) -> Dict: - self.log.debug('processing_server.get_processor_info() called') + """ Return a processor's ocrd-tool.json + """ if processor_name not in self.processor_list: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, @@ -298,7 +293,8 @@ async def get_processor_info(self, processor_name) -> Dict: return get_ocrd_tool_json(processor_name) async def get_job(self, processor_name: str, job_id: PydanticObjectId) -> Job: - self.log.debug('processing_server.get_job() called') + """ Return job-information from the database + """ job = await Job.get(job_id) if job: return job @@ -308,5 +304,6 @@ async def get_job(self, processor_name: str, job_id: PydanticObjectId) -> Job: ) async def list_processors(self) -> str: - self.log.debug('processing_server.list_processors() called') + """ Return a list of all available processors + """ return json.dumps(self.processor_list) From 67753b10e517b94558124c127a830f6ed2f0e849 Mon Sep 17 00:00:00 2001 From: joschrew Date: Fri, 10 Feb 2023 15:14:33 +0100 Subject: [PATCH 113/226] Stop splitting config in ProcessingServer Reason: Perparing for separating config and runtime information (following commit(s)) --- ocrd/ocrd/network/deployer.py | 8 +++--- ocrd/ocrd/network/processing_server.py | 34 ++++++++++++-------------- 2 files changed, 19 insertions(+), 23 deletions(-) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index a1d0d2872..b8a6b8e56 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -36,7 +36,7 @@ class Deployer: for managing information, not for actually doing things. """ - def __init__(self, queue_config: QueueConfig, mongo_config: MongoConfig, hosts_config: List[HostConfig]) -> None: + def __init__(self, config: ProcessingServerConfig) -> None: """ Args: queue_config: RabbitMQ related configuration @@ -45,9 +45,9 @@ def __init__(self, queue_config: QueueConfig, mongo_config: MongoConfig, hosts_c """ self.log = getLogger(__name__) self.log.debug('The Deployer of the ProcessingServer was invoked') - self.mongo_data = mongo_config - self.mq_data = queue_config - self.hosts = hosts_config + self.mongo_data = config.mongo_config + self.mq_data = config.queue_config + self.hosts = config.hosts_config # TODO: We should have a data structure here to manage the connections and PIDs: # - RabbitMQ - (host address, pid on that host) diff --git a/ocrd/ocrd/network/processing_server.py b/ocrd/ocrd/network/processing_server.py index 8ad691086..1d4824f8e 100644 --- a/ocrd/ocrd/network/processing_server.py +++ b/ocrd/ocrd/network/processing_server.py @@ -27,8 +27,8 @@ class ProcessingServer(FastAPI): """FastAPI app to make ocr-d processor calls - The Processing-Server receives calls conforming to the ocr-d webapi regarding the processoing part. - It can run ocrd-processors and provides enpoints to discover processors and watch the job + The Processing-Server receives calls conforming to the ocr-d webapi regarding the processoing + part. It can run ocrd-processors and provides enpoints to discover processors and watch the job status. The Processing-Server does not execute the processors itself but starts up a queue and a database to delegate the calls to processing workers. They are started by the Processing-Server @@ -45,15 +45,8 @@ def __init__(self, config_path: str, host: str, port: int) -> None: self.port = port # TODO: Ideally the parse_config should return a Tuple with the 3 configs assigned below # to prevent passing the entire parsed config around to methods. - parsed_config = ProcessingServer.parse_config(config_path) - self.queue_config = parsed_config.queue_config - self.mongo_config = parsed_config.mongo_config - self.hosts_config = parsed_config.hosts_config - self.deployer = Deployer( - queue_config=self.queue_config, - mongo_config=self.mongo_config, - hosts_config=self.hosts_config - ) + self.config = ProcessingServer.parse_config(config_path) + self.deployer = Deployer(self.config) # TODO: Parse the RabbitMQ related data from the `queue_config` # above instead of using the hard coded ones below @@ -150,12 +143,13 @@ def start(self) -> None: # The RMQPublisher is initialized and a connection to the RabbitMQ is performed self.connect_publisher() - self.log.debug(f'Starting to create message queues on RabbitMQ instance url: {rabbitmq_url}') + self.log.debug(f'Creating message queues on RabbitMQ instance url: {rabbitmq_url}') self.create_message_queues() # Deploy processing hosts where processing workers are running on - # Note: A deployed processing worker starts listening to a message queue with id processor.name - self.deployer.deploy_hosts(self.hosts_config, rabbitmq_url, self.mongodb_url) + # Note: A deployed processing worker starts listening to a message queue with id + # processor.name + self.deployer.deploy_hosts(self.config.hosts_config, rabbitmq_url, self.mongodb_url) self.log.debug(f'Starting uvicorn: {self.hostname}:{self.port}') uvicorn.run(self, host=self.hostname, port=self.port) @@ -185,10 +179,12 @@ async def stop_deployed_agents(self) -> None: def connect_publisher(self, username: str = 'default-publisher', password: str = 'default-publisher', enable_acks: bool =True) -> None: - self.log.debug(f'Connecting RMQPublisher to RabbitMQ server: {self.rmq_host}:{self.rmq_port}{self.rmq_vhost}') - self.rmq_publisher = RMQPublisher(host=self.rmq_host, port=self.rmq_port, vhost=self.rmq_vhost) + self.log.debug(f'Connecting RMQPublisher to RabbitMQ server: {self.rmq_host}:' + f'{self.rmq_port}{self.rmq_vhost}') + self.rmq_publisher = RMQPublisher(host=self.rmq_host, port=self.rmq_port, + vhost=self.rmq_vhost) # TODO: Remove this information before the release - self.log.debug(f'RMQPublisher authenticates with username: {username}, password: {password}') + self.log.debug(f'RMQPublisher authenticate with username: {username}, password: {password}') self.rmq_publisher.authenticate_and_connect(username=username, password=password) if enable_acks: self.rmq_publisher.enable_delivery_confirmations() @@ -200,7 +196,7 @@ def connect_publisher(self, username: str = 'default-publisher', def create_message_queues(self) -> None: """Create the message queues based on the occurrence of `processor.name` in the config file """ - for host in self.hosts_config: + for host in self.config.hosts_config: for processor in host.processors: # The existence/validity of the processor.name is not tested. # Even if an ocr-d processor does not exist, the queue is created @@ -226,7 +222,7 @@ def processor_list(self): if self._processor_list: return self._processor_list res = set([]) - for host in self.hosts_config: + for host in self.config.hosts_config: for processor in host.processors: res.add(processor.name) self._processor_list = list(res) From 6a526af85aca376eb621d6e5919934225aed0e9c Mon Sep 17 00:00:00 2001 From: joschrew Date: Fri, 10 Feb 2023 15:30:37 +0100 Subject: [PATCH 114/226] Stop storing pid for queue and mongo in config --- ocrd/ocrd/network/deployer.py | 65 +++++++++++++------------- ocrd/ocrd/network/deployment_config.py | 6 +-- 2 files changed, 35 insertions(+), 36 deletions(-) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index b8a6b8e56..53fe98e56 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -45,9 +45,10 @@ def __init__(self, config: ProcessingServerConfig) -> None: """ self.log = getLogger(__name__) self.log.debug('The Deployer of the ProcessingServer was invoked') - self.mongo_data = config.mongo_config - self.mq_data = config.queue_config + self.config = config self.hosts = config.hosts_config + self.mongo_pid = None + self.mq_pid = None # TODO: We should have a data structure here to manage the connections and PIDs: # - RabbitMQ - (host address, pid on that host) @@ -141,18 +142,18 @@ def deploy_rabbitmq(self, image: str = 'rabbitmq:3-management', detach: bool = T # Which is part of the OCR-D WebAPI implementation. self.log.debug(f'Trying to deploy image[{image}], with modes: detach[{detach}], remove[{remove}]') - if not self.mq_data or not self.mq_data.address: + if not self.config or not self.config.queue.address: raise ValueError('Deploying RabbitMQ has failed - missing configuration.') - client = create_docker_client(self.mq_data.address, self.mq_data.username, - self.mq_data.password, self.mq_data.keypath) + client = create_docker_client(self.config.queue.address, self.config.queue.username, + self.config.queue.password, self.config.queue.keypath) if not ports_mapping: # 5672, 5671 - used by AMQP 0-9-1 and AMQP 1.0 clients without and with TLS # 15672, 15671: HTTP API clients, management UI and rabbitmqadmin, without and with TLS # 25672: used for internode and CLI tools communication and is allocated from # a dynamic range (limited to a single port by default, computed as AMQP port + 20000) ports_mapping = { - 5672: self.mq_data.port, + 5672: self.config.queue.port, 15672: 15672, 25672: 25672 } @@ -165,21 +166,21 @@ def deploy_rabbitmq(self, image: str = 'rabbitmq:3-management', detach: bool = T remove=remove, ports=ports_mapping, environment=[ - f'RABBITMQ_DEFAULT_USER={self.mq_data.credentials[0]}', - f'RABBITMQ_DEFAULT_PASS={self.mq_data.credentials[1]}', + f'RABBITMQ_DEFAULT_USER={self.config.queue.credentials[0]}', + f'RABBITMQ_DEFAULT_PASS={self.config.queue.credentials[1]}', ('RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS=' f'-rabbitmq_management load_definitions "{container_defs_path}"'), ], volumes={local_defs_path: {'bind': container_defs_path, 'mode': 'ro'}} ) assert res and res.id, \ - f'Failed to start RabbitMQ docker container on host: {self.mq_data.address}' - self.mq_data.pid = res.id + f'Failed to start RabbitMQ docker container on host: {self.config.mongo.address}' + self.mq_pid = res.id client.close() # Build the RabbitMQ Server URL to return - rmq_host = self.mq_data.address - rmq_port = self.mq_data.port + rmq_host = self.config.queue.address + rmq_port = self.config.queue.port rmq_vhost = '/' # the default virtual host rabbitmq_url = f'{rmq_host}:{rmq_port}{rmq_vhost}' @@ -192,14 +193,14 @@ def deploy_mongodb(self, image: str = 'mongo', detach: bool = True, remove: bool """ self.log.debug(f'Trying to deploy image[{image}], with modes: detach[{detach}], remove[{remove}]') - if not self.mongo_data or not self.mongo_data.address: + if not self.config or not self.config.mongo.address: raise ValueError('Deploying MongoDB has failed - missing configuration.') - client = create_docker_client(self.mongo_data.address, self.mongo_data.username, - self.mongo_data.password, self.mongo_data.keypath) + client = create_docker_client(self.config.mongo.address, self.config.mongo.username, + self.config.mongo.password, self.config.mongo.keypath) if not ports_mapping: ports_mapping = { - 27017: self.mongo_data.port + 27017: self.config.mongo.port } self.log.debug(f'Ports mapping: {ports_mapping}') # TODO: what about the data-dir? Must data be preserved between runs? @@ -211,45 +212,45 @@ def deploy_mongodb(self, image: str = 'mongo', detach: bool = True, remove: bool ) if not res or not res.id: raise RuntimeError('Failed to start MongoDB docker container on host: ' - f'{self.mongo_data.address}') - self.mongo_data.pid = res.id + f'{self.config.mongo.address}') + self.mongo_pid = res.id client.close() # Build the MongoDB URL to return mongodb_prefix = 'mongodb://' - mongodb_host = self.mongo_data.address - mongodb_port = self.mongo_data.port + mongodb_host = self.config.mongo.address + mongodb_port = self.config.mongo.port mongodb_url = f'{mongodb_prefix}{mongodb_host}:{mongodb_port}' self.log.debug(f'The MongoDB was deployed on url: {mongodb_url}') return mongodb_url def kill_rabbitmq(self) -> None: # TODO: The PID must not be stored in the configuration `mq_data`. Why not? - if not self.mq_data or not self.mq_data.pid: + if not self.mq_pid: self.log.warning(f'No running RabbitMQ instance found') # TODO: Ignoring this silently is problematic in the future. Why? return - self.log.debug(f'Trying to stop the deployed RabbitMQ with PID: {self.mq_data.pid}') + self.log.debug(f'Trying to stop the deployed RabbitMQ with PID: {self.mq_pid}') - client = create_docker_client(self.mq_data.address, self.mq_data.username, - self.mq_data.password, self.mq_data.keypath) - client.containers.get(self.mq_data.pid).stop() - self.mq_data.pid = None + client = create_docker_client(self.config.queue.address, self.config.queue.username, + self.config.queue.password, self.config.queue.keypath) + client.containers.get(self.mq_pid).stop() + self.mq_pid = None client.close() self.log.debug('The RabbitMQ is stopped') def kill_mongodb(self) -> None: # TODO: The PID must not be stored in the configuration `mongo_data`. Why not? - if not self.mongo_data or not self.mongo_data.pid: + if not self.mongo_pid: self.log.warning(f'No running MongoDB instance found') # TODO: Ignoring this silently is problematic in the future. Why? return - self.log.debug(f'Trying to stop the deployed MongoDB with PID: {self.mongo_data.pid}') + self.log.debug(f'Trying to stop the deployed MongoDB with PID: {self.mongo_pid}') - client = create_docker_client(self.mongo_data.address, self.mongo_data.username, - self.mongo_data.password, self.mongo_data.keypath) - client.containers.get(self.mongo_data.pid).stop() - self.mongo_data.pid = None + client = create_docker_client(self.config.mongo.address, self.config.mongo.username, + self.config.mongo.password, self.config.mongo.keypath) + client.containers.get(self.mongo_pid).stop() + self.mongo_pid = None client.close() self.log.debug('The MongoDB is stopped') diff --git a/ocrd/ocrd/network/deployment_config.py b/ocrd/ocrd/network/deployment_config.py index e859ed5ee..85e445aad 100644 --- a/ocrd/ocrd/network/deployment_config.py +++ b/ocrd/ocrd/network/deployment_config.py @@ -15,8 +15,8 @@ class ProcessingServerConfig: def __init__(self, config: dict) -> None: - self.mongo_config = MongoConfig(config['database']) - self.queue_config = QueueConfig(config['process_queue']) + self.mongo = MongoConfig(config['database']) + self.queue = QueueConfig(config['process_queue']) self.hosts_config = [] for host in config['hosts']: self.hosts_config.append(HostConfig(host)) @@ -73,7 +73,6 @@ def __init__(self, config: Dict) -> None: self.keypath = config['ssh'].get('path_to_privkey', None) self.password = config['ssh'].get('password', None) self.credentials = (config['credentials']['username'], config['credentials']['password']) - self.pid = None class QueueConfig: @@ -87,4 +86,3 @@ def __init__(self, config: Dict) -> None: self.keypath = config['ssh'].get('path_to_privkey', None) self.password = config['ssh'].get('password', None) self.credentials = (config['credentials']['username'], config['credentials']['password']) - self.pid = None From d9d959eb668a391bac7ee46eb05b76ae60f6a6f9 Mon Sep 17 00:00:00 2001 From: joschrew Date: Fri, 10 Feb 2023 16:08:40 +0100 Subject: [PATCH 115/226] Separating config and data for hosts Previously the config was parsed into classes and the runtime information (pids, connections) were stored inside these classes. Goal is to seperata readonly config-infos from runtime data --- ocrd/ocrd/network/deployer.py | 95 +++++++++++++------------- ocrd/ocrd/network/deployment_config.py | 10 +-- ocrd/ocrd/network/deployment_utils.py | 21 +++++- ocrd/ocrd/network/processing_server.py | 6 +- 4 files changed, 72 insertions(+), 60 deletions(-) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index 53fe98e56..cdb738565 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -11,6 +11,7 @@ create_ssh_client, CustomDockerClient, DeployType, + HostData, ) import time from pathlib import Path @@ -39,14 +40,12 @@ class Deployer: def __init__(self, config: ProcessingServerConfig) -> None: """ Args: - queue_config: RabbitMQ related configuration - mongo_config: MongoDB related configuration - hosts_config: Processing Hosts related configurations + config: Parsed processing-server-configuration """ self.log = getLogger(__name__) self.log.debug('The Deployer of the ProcessingServer was invoked') self.config = config - self.hosts = config.hosts_config + self.hosts = HostData.from_config(config.hosts) self.mongo_pid = None self.mq_pid = None @@ -75,39 +74,39 @@ def kill_all(self) -> None: # Third kill the RabbitMQ Server self.kill_rabbitmq() - def deploy_hosts(self, hosts: List[HostConfig], rabbitmq_url: str, mongodb_url: str) -> None: + def deploy_hosts(self, rabbitmq_url: str, mongodb_url: str) -> None: self.log.debug('Starting to deploy hosts') - for host in hosts: - self.log.debug(f'Deploying processing workers on host: {host.address}') - for processor in host.processors: + for host in self.hosts: + self.log.debug(f'Deploying processing workers on host: {host.config.address}') + for processor in host.config.processors: self._deploy_processing_worker(processor, host, rabbitmq_url, mongodb_url) - # TODO: These connections, just like the PIDs, should not be kept in the config data classes - # The connections are correctly closed on host level, but created on processing worker level? + # TODO: The connections are correctly closed on host level, but created on processing + # worker level? if host.ssh_client: host.ssh_client.close() if host.docker_client: host.docker_client.close() # TODO: Creating connections if missing should probably occur when deploying hosts not when - # deploying processing workers. The deploy_type checks and opening connections creates duplicate code. - def _deploy_processing_worker(self, processor: WorkerConfig, host: HostConfig, + # deploying processing workers. The deploy_type checks and opening connections creates + # duplicate code. + def _deploy_processing_worker(self, processor: WorkerConfig, host: HostData, rabbitmq_url: str, mongodb_url: str) -> None: self.log.debug(f'deploy \'{processor.deploy_type}\' processor: \'{processor}\' on' - f'\'{host.address}\'') - assert not processor.pids, 'processors already deployed. Pids are present. Host: ' \ - '{host.__dict__}. Processor: {processor.__dict__}' + f'\'{host.config.address}\'') - # TODO: The check for available ssh or docker connections should probably happen inside `deploy_hosts` + # TODO: The check for available ssh or docker connections should probably happen inside + # `deploy_hosts` if processor.deploy_type == DeployType.native: if not host.ssh_client: - host.ssh_client = create_ssh_client(host.address, host.username, host.password, - host.keypath) + host.ssh_client = create_ssh_client(host.config.address, host.config.username, + host.config.password, host.config.keypath) else: assert processor.deploy_type == DeployType.docker if not host.docker_client: - host.docker_client = create_docker_client(host.address, host.username, - host.password, host.keypath) + host.docker_client = create_docker_client(host.config.address, host.config.username, + host.config.password, host.config.keypath) for _ in range(processor.count): if processor.deploy_type == DeployType.native: @@ -117,9 +116,9 @@ def _deploy_processing_worker(self, processor: WorkerConfig, host: HostConfig, processor_name=processor.name, queue_url=rabbitmq_url, database_url=mongodb_url, - bin_dir=host.binpath, + bin_dir=host.config.binpath, ) - processor.add_started_pid(pid) + host.pids_native.append(pid) else: assert processor.deploy_type == DeployType.docker assert host.docker_client # to satisfy mypy @@ -129,7 +128,7 @@ def _deploy_processing_worker(self, processor: WorkerConfig, host: HostConfig, queue_url=rabbitmq_url, database_url=mongodb_url ) - processor.add_started_pid(pid) + host.pids_docker.append(pid) def deploy_rabbitmq(self, image: str = 'rabbitmq:3-management', detach: bool = True, remove: bool = True, ports_mapping: Union[Dict, None] = None) -> str: @@ -140,7 +139,8 @@ def deploy_rabbitmq(self, image: str = 'rabbitmq:3-management', detach: bool = T # Handling of creation of queues, submitting messages to queues, # and receiving messages from queues is part of the RabbitMQ Library # Which is part of the OCR-D WebAPI implementation. - self.log.debug(f'Trying to deploy image[{image}], with modes: detach[{detach}], remove[{remove}]') + self.log.debug(f'Trying to deploy image[{image}], with modes: detach[{detach}],' + f'remove[{remove}]') if not self.config or not self.config.queue.address: raise ValueError('Deploying RabbitMQ has failed - missing configuration.') @@ -188,10 +188,11 @@ def deploy_rabbitmq(self, image: str = 'rabbitmq:3-management', detach: bool = T return rabbitmq_url def deploy_mongodb(self, image: str = 'mongo', detach: bool = True, remove: bool = True, - ports_mapping: Union[Dict, None] = None) -> str: + ports_mapping: Union[Dict, None] = None) -> str: """ Start mongodb in docker """ - self.log.debug(f'Trying to deploy image[{image}], with modes: detach[{detach}], remove[{remove}]') + self.log.debug(f'Trying to deploy image[{image}], with modes: detach[{detach}],' + f'remove[{remove}]') if not self.config or not self.config.mongo.address: raise ValueError('Deploying MongoDB has failed - missing configuration.') @@ -227,7 +228,7 @@ def deploy_mongodb(self, image: str = 'mongo', detach: bool = True, remove: bool def kill_rabbitmq(self) -> None: # TODO: The PID must not be stored in the configuration `mq_data`. Why not? if not self.mq_pid: - self.log.warning(f'No running RabbitMQ instance found') + self.log.warning('No running RabbitMQ instance found') # TODO: Ignoring this silently is problematic in the future. Why? return self.log.debug(f'Trying to stop the deployed RabbitMQ with PID: {self.mq_pid}') @@ -242,7 +243,7 @@ def kill_rabbitmq(self) -> None: def kill_mongodb(self) -> None: # TODO: The PID must not be stored in the configuration `mongo_data`. Why not? if not self.mongo_pid: - self.log.warning(f'No running MongoDB instance found') + self.log.warning('No running MongoDB instance found') # TODO: Ignoring this silently is problematic in the future. Why? return self.log.debug(f'Trying to stop the deployed MongoDB with PID: {self.mongo_pid}') @@ -258,31 +259,29 @@ def kill_hosts(self) -> None: self.log.debug('Starting to kill/stop hosts') # Kill processing hosts for host in self.hosts: - self.log.debug(f'Killing/Stopping processing workers on host: {host.address}') + self.log.debug(f'Killing/Stopping processing workers on host: {host.config.address}') if host.ssh_client: - host.ssh_client = create_ssh_client(host.address, host.username, host.password, - host.keypath) + host.ssh_client = create_ssh_client(host.config.address, host.config.username, + host.config.password, host.config.keypath) if host.docker_client: - host.docker_client = create_docker_client(host.address, host.username, - host.password, host.keypath) + host.docker_client = create_docker_client(host.config.address, host.config.username, + host.config.password, host.config.keypath) # Kill deployed OCR-D processor instances on this Processing worker host self.kill_processing_worker(host) - def kill_processing_worker(self, host: HostConfig) -> None: - for processor in host.processors: - if processor.deploy_type.is_native(): - for pid in processor.pids: - self.log.debug(f'Trying to kill/stop native processor: {processor.name}, with PID: {pid}') - # TODO: For graceful shutdown we may want to send additional parameters to kill - host.ssh_client.exec_command(f'kill {pid}') - else: - assert processor.deploy_type.is_docker() - for pid in processor.pids: - self.log.debug(f'Trying to kill/stop docker container processor: {processor.name}, with PID: {pid}') - # TODO: think about timeout. - # think about using threads to kill parallelized to reduce waiting time - host.docker_client.containers.get(pid).stop() - processor.pids = [] + def kill_processing_worker(self, host: HostData) -> None: + for pid in host.pids_native: + self.log.debug(f'Trying to kill/stop native processor: with PID: \'{pid}\'') + # TODO: For graceful shutdown we may want to send additional parameters to kill + host.ssh_client.exec_command(f'kill {pid}') + host.pids_native = [] + + for pid in host.pids_docker: + self.log.debug(f'Trying to kill/stop docker container with PID: {pid}') + # TODO: think about timeout. + # think about using threads to kill parallelized to reduce waiting time + host.docker_client.containers.get(pid).stop() + host.pids_docker = [] # Note: Invoking a pythonic processor is slightly different from the description in the spec. # In order to achieve the exact spec call all ocr-d processors should be refactored... diff --git a/ocrd/ocrd/network/deployment_config.py b/ocrd/ocrd/network/deployment_config.py index 85e445aad..d04596e20 100644 --- a/ocrd/ocrd/network/deployment_config.py +++ b/ocrd/ocrd/network/deployment_config.py @@ -17,9 +17,9 @@ class ProcessingServerConfig: def __init__(self, config: dict) -> None: self.mongo = MongoConfig(config['database']) self.queue = QueueConfig(config['process_queue']) - self.hosts_config = [] + self.hosts = [] for host in config['hosts']: - self.hosts_config.append(HostConfig(host)) + self.hosts.append(HostConfig(host)) class HostConfig: @@ -44,8 +44,6 @@ def __init__(self, config: dict) -> None: self.processors.append( WorkerConfig(worker['name'], worker['number_of_instance'], deploy_type) ) - self.ssh_client = None - self.docker_client = None class WorkerConfig: @@ -56,10 +54,6 @@ def __init__(self, name: str, count: int, deploy_type: DeployType) -> None: self.name = name self.count = count self.deploy_type = deploy_type - self.pids: List = [] - - def add_started_pid(self, pid) -> None: - self.pids.append(pid) class MongoConfig: diff --git a/ocrd/ocrd/network/deployment_utils.py b/ocrd/ocrd/network/deployment_utils.py index 402ae7349..b0e722124 100644 --- a/ocrd/ocrd/network/deployment_utils.py +++ b/ocrd/ocrd/network/deployment_utils.py @@ -1,10 +1,11 @@ from __future__ import annotations from enum import Enum -from typing import Union +from typing import Union, List from docker import APIClient, DockerClient from docker.transport import SSHHTTPAdapter from paramiko import AutoAddPolicy, SSHClient +from ocrd.network.deployment_config import * from ocrd_utils import getLogger @@ -27,6 +28,24 @@ def create_docker_client(address: str, username: str, password: Union[str, None] return CustomDockerClient(username, address, password=password, keypath=keypath) +class HostData: + """class to store runtime information for a host + """ + def __init__(self, config: deployment_config.HostConfig) -> None: + self.config = config + self.ssh_client: Union[SSHClient, None] = None + self.docker_client: Union[CustomDockerClient, None] = None + self.pids_native: List[str] = [] + self.pids_docker: List[str] = [] + + @staticmethod + def from_config(config: List[deployment_config.HostConfig]) -> List[HostData]: + res = [] + for host_config in config: + res.append(HostData(host_config)) + return res + + class CustomDockerClient(DockerClient): """Wrapper for docker.DockerClient to use an own SshHttpAdapter. diff --git a/ocrd/ocrd/network/processing_server.py b/ocrd/ocrd/network/processing_server.py index 1d4824f8e..fd1a38f19 100644 --- a/ocrd/ocrd/network/processing_server.py +++ b/ocrd/ocrd/network/processing_server.py @@ -149,7 +149,7 @@ def start(self) -> None: # Deploy processing hosts where processing workers are running on # Note: A deployed processing worker starts listening to a message queue with id # processor.name - self.deployer.deploy_hosts(self.config.hosts_config, rabbitmq_url, self.mongodb_url) + self.deployer.deploy_hosts(rabbitmq_url, self.mongodb_url) self.log.debug(f'Starting uvicorn: {self.hostname}:{self.port}') uvicorn.run(self, host=self.hostname, port=self.port) @@ -196,7 +196,7 @@ def connect_publisher(self, username: str = 'default-publisher', def create_message_queues(self) -> None: """Create the message queues based on the occurrence of `processor.name` in the config file """ - for host in self.config.hosts_config: + for host in self.config.hosts: for processor in host.processors: # The existence/validity of the processor.name is not tested. # Even if an ocr-d processor does not exist, the queue is created @@ -222,7 +222,7 @@ def processor_list(self): if self._processor_list: return self._processor_list res = set([]) - for host in self.config.hosts_config: + for host in self.config.hosts: for processor in host.processors: res.add(processor.name) self._processor_list = list(res) From f51b3b8132b2df17dbc8a1116986b86c24a6fab9 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Mon, 13 Feb 2023 14:43:37 +0100 Subject: [PATCH 116/226] Removing unnecessary things after meeting with Jonas --- ocrd/ocrd/cli/processing_worker.py | 4 +- ocrd/ocrd/decorators/__init__.py | 1 + ocrd/ocrd/network/__init__.py | 1 - ocrd/ocrd/network/assets/wf_server_example.nf | 202 ------------------ ocrd/ocrd/network/deployer.py | 24 +-- ocrd/ocrd/network/deployment_config.py | 2 +- ocrd/ocrd/network/deployment_utils.py | 8 +- ocrd/ocrd/network/helpers.py | 32 +-- ocrd/ocrd/network/models/job.py | 2 +- ocrd/ocrd/network/processing_server.py | 39 +--- ocrd/ocrd/network/processing_worker.py | 28 ++- .../rabbitmq_utils/Dockerfile-RabbitMQ | 10 - ocrd/ocrd/network/rabbitmq_utils/connector.py | 7 - ocrd/ocrd/network/rabbitmq_utils/consumer.py | 70 +----- .../network/rabbitmq_utils/ocrd_messages.py | 39 ---- ocrd/ocrd/network/rabbitmq_utils/publisher.py | 75 +------ .../processing_server_validator.py | 6 +- 17 files changed, 45 insertions(+), 505 deletions(-) delete mode 100644 ocrd/ocrd/network/assets/wf_server_example.nf delete mode 100644 ocrd/ocrd/network/rabbitmq_utils/Dockerfile-RabbitMQ diff --git a/ocrd/ocrd/cli/processing_worker.py b/ocrd/ocrd/cli/processing_worker.py index f3beacc8e..23dbac2bb 100644 --- a/ocrd/ocrd/cli/processing_worker.py +++ b/ocrd/ocrd/cli/processing_worker.py @@ -7,11 +7,9 @@ """ import click import logging -from subprocess import run, PIPE from ocrd_utils import ( initLogging, - get_ocrd_tool_json, - parse_json_string_with_comments + get_ocrd_tool_json ) from ocrd.network.processing_worker import ProcessingWorker diff --git a/ocrd/ocrd/decorators/__init__.py b/ocrd/ocrd/decorators/__init__.py index 376f33bf6..17cfc77bc 100644 --- a/ocrd/ocrd/decorators/__init__.py +++ b/ocrd/ocrd/decorators/__init__.py @@ -64,6 +64,7 @@ def ocrd_cli_wrap_processor( if queue and database: initLogging() # TODO: Remove before the release + # We are importing the logging here because it's not the ocrd logging but python one import logging logging.getLogger('ocrd.network').setLevel(logging.DEBUG) diff --git a/ocrd/ocrd/network/__init__.py b/ocrd/ocrd/network/__init__.py index 62868cdf5..6b7e368ef 100644 --- a/ocrd/ocrd/network/__init__.py +++ b/ocrd/ocrd/network/__init__.py @@ -16,6 +16,5 @@ # This package, currently, is under the `core/ocrd` package. # It could also be a separate package on its own under `core` with the name `ocrd_network`. -# TODO: Correctly identify all current and potential future dependencies. from .processing_server import ProcessingServer from .processing_worker import ProcessingWorker diff --git a/ocrd/ocrd/network/assets/wf_server_example.nf b/ocrd/ocrd/network/assets/wf_server_example.nf deleted file mode 100644 index 220c7d141..000000000 --- a/ocrd/ocrd/network/assets/wf_server_example.nf +++ /dev/null @@ -1,202 +0,0 @@ -@Grab(group='org.springframework.boot', module='spring-boot-starter-amqp', version='2.2.2.RELEASE') -import org.springframework.amqp.core.* -import org.springframework.amqp.rabbit.connection.CachingConnectionFactory -import org.springframework.amqp.rabbit.core.RabbitAdmin -import org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer -import org.springframework.amqp.rabbit.listener.api.* -import java.io.BufferedWriter -import java.io.OutputStreamWriter -import java.nio.charset.Charset -nextflow.enable.dsl=2 - -// These parameters can also be overwritten with values passed from the CLI -// when executing this script, i.e., --processing_server_address address -params.processing_server_address = "localhost:8080" -params.mets = "/home/mm/Desktop/example_ws/data/mets.xml" -// This is the entry point for the first ocr-d processor call in the Workflow -params.input_file_grp = "OCR-D-IMG" - -params.rmq_address = "localhost:5672" -params.rmq_username = "default-consumer" -params.rmq_password = "default-consumer" -params.rmq_exchange = "ocrd-network-default" - - -log.info """\ - O C R - D - W O R K F L O W - W E B A P I - 1 - ====================================================== - processing_server_address : ${params.processing_server_address} - mets : ${params.mets} - input_file_grp : ${params.input_file_grp} - """ - .stripIndent() - - -// This global variable is used to track the status of -// the previous process job in the when block of the current process job -job_status_flag = "NONE" - - -// RabbitMQ related globals -rmq_uri = "amqp://${params.rmq_username}:${params.rmq_password}@${params.rmq_address}" -println(rmq_uri) - -def produce_job_input_json(input_grp, output_grp, page_id, ocrd_params){ - // TODO: Using string builder should be more computationally efficient - def json_body = """{"path": "${params.mets}",""" - if (input_grp != null) - json_body = json_body + """ "input_file_grps": ["${input_grp}"]""" - if (output_grp != null) - json_body = json_body + """, "output_file_grps": ["${output_grp}"]""" - if (page_id != null) - json_body = json_body + """, "page_id": ${page_id}""" - if (ocrd_params != null) - json_body = json_body + """, "parameters": ${ocrd_params}""" - else - json_body = json_body + """, "parameters": {}""" - - json_body = json_body + """}""" - return json_body -} - -def post_processing_job(ocrd_processor, input_grp, output_grp, page_id, ocrd_params){ - def post_connection = new URL("http://${params.processing_server_address}/processor/${ocrd_processor}").openConnection() - post_connection.setDoOutput(true) - post_connection.setRequestMethod("POST") - post_connection.setRequestProperty("accept", "application/json") - post_connection.setRequestProperty("Content-Type", "application/json") - - def json_body = produce_job_input_json(input_grp, output_grp, page_id, ocrd_params) - println(json_body) - - def httpRequestBodyWriter = new BufferedWriter(new OutputStreamWriter(post_connection.getOutputStream())) - httpRequestBodyWriter.write(json_body) - httpRequestBodyWriter.close() - - def response_code = post_connection.getResponseCode() - println("Response code: " + response_code) - if (response_code.equals(200)){ - def json = post_connection.getInputStream().getText() - println("ResponseJSON: " + json) - } -} - -def configure_queue_listener(result_queue_name){ - cf = new CachingConnectionFactory(new URI(rmq_uri)) - def rmq_admin = new RabbitAdmin(cf) - def rmq_exchange = new DirectExchange(params.rmq_exchange, false, false) - rmq_admin.declareExchange(rmq_exchange) - - def rmq_queue = new Queue(result_queue_name, false) - rmq_admin.declareQueue(rmq_queue) - rmq_admin.declareBinding(BindingBuilder.bind(rmq_queue).to(rmq_exchange).withQueueName()) - - def listener = new SimpleMessageListenerContainer() - listener.setConnectionFactory(cf) - listener.setQueues(rmq_queue) - listener.setMessageListener(new ChannelAwareMessageListener() { - @Override - void onMessage(Message message, com.rabbitmq.client.Channel channel) { - println "Message received ${new Date()}" - // println (message) - def delivery_tag = message.getMessageProperties().getDeliveryTag() - def consumer_tag = message.getMessageProperties().getConsumerTag() - println "Consumer tag: ${consumer_tag}" - println "Delivery tag: ${delivery_tag}" - job_status = find_job_status(parse_body(message.getBody())) - println "JobStatus: ${job_status}" - // Overwrites the global status flag - job_status_flag = job_status - channel.basicAck(delivery_tag, false) - // channel.basicCancel(consumer_tag) - println "Trying to stop listener" - // channel.close(0, "Closing the channel after successful consumption.") - listener.stop() - println "After stop listener" - } - String parse_body(byte[] bytes) { - if (bytes) { - new String(bytes, Charset.forName('UTF-8')) - } - } - String find_job_status(String message_body){ - // TODO: Use Regex - if (message_body.contains("SUCCESS")){ - return "SUCCESS" - } - else if (message_body.contains("FAILED")){ - return "FAILED" - } - else if (message_body.contains("RUNNING")){ - return "RUNNING" - } - else if (message_body.contains("QUEUED")){ - return "QUEUED" - } - else { - return "NONE" - } - } - }) - - return listener -} - -def exec_block_logic(ocrd_processor_str, input_dir, output_dir, page_id, ocrd_params){ - String result_queue = "${ocrd_processor_str}-result" - post_processing_job(ocrd_processor_str, input_dir, output_dir, null, null) - job_status_flag = "INITIALIZED" - def listener = configure_queue_listener(result_queue) - // The job_status_flag is with value "INITIALIZED" here - println "Starting listening, flag: ${job_status_flag}" - listener.start() - // The job_status_flag must be with value "SUCCESS" or "FAILED" here - println "Ended listening, flag: ${job_status_flag}" - - // The job_status_flag gets overwritten inside the onMessage - // method when a message is consumed from the result queue - return job_status_flag -} - -process binarize { - maxForks 1 - - input: - val input_dir - val output_dir - - output: - val output_dir - val job_status - - exec: - job_status = exec_block_logic("ocrd-cis-ocropy-binarize", input_dir, output_dir, null, null) - println "binarize returning flag: ${job_status}" -} - -process crop { - maxForks 1 - - input: - val input_dir - val output_dir - val prev_job_status - - when: - prev_job_status == "SUCCESS" - - output: - val output_dir - val job_status - - exec: - job_status = exec_block_logic("ocrd-anybaseocr-crop", input_dir, output_dir, null, null) - println "crop returning flag: returning job status: ${job_status}" -} - -workflow { - - main: - binarize(params.input_file_grp, "OCR-D-BIN") - crop(binarize.out[0], "OCR-D-CROP", binarize.out[1]) -} diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index cdb738565..db0342191 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -1,5 +1,14 @@ +""" +Abstraction of the Deployment functionality +The ProcessingServer (currently still called Server) provides the configuration parameters to the +Deployer agent. +The Deployer agent deploys the RabbitMQ Server, MongoDB and the Processing Hosts. +Each Processing Host may have several Processing Workers. +Each Processing Worker is an instance of an OCR-D processor. +""" + from __future__ import annotations -from typing import List, Dict, Union, Optional +from typing import Dict, Union, Optional from paramiko import SSHClient from re import search as re_search @@ -13,21 +22,8 @@ DeployType, HostData, ) -import time from pathlib import Path -# Abstraction of the Deployment functionality -# The ProcessingServer (currently still called Server) provides the configuration parameters to the -# Deployer agent. -# The Deployer agent deploys the RabbitMQ Server, MongoDB and the Processing Hosts. -# Each Processing Host may have several Processing Workers. -# Each Processing Worker is an instance of an OCR-D processor. - -# TODO: -# Ideally, the interaction among the agents should happen through -# the defined API calls of the objects to stay loyal to the OOP paradigm -# This would also increase the readability and maintainability of the source code. - # TODO: remove debug log statements before beta, their purpose is development only class Deployer: diff --git a/ocrd/ocrd/network/deployment_config.py b/ocrd/ocrd/network/deployment_config.py index d04596e20..e223b4798 100644 --- a/ocrd/ocrd/network/deployment_config.py +++ b/ocrd/ocrd/network/deployment_config.py @@ -1,6 +1,6 @@ # TODO: this probably breaks python 3.6. Think about whether we really want to use this from __future__ import annotations -from typing import List, Dict +from typing import Dict from ocrd.network.deployment_utils import DeployType diff --git a/ocrd/ocrd/network/deployment_utils.py b/ocrd/ocrd/network/deployment_utils.py index b0e722124..72221a2c7 100644 --- a/ocrd/ocrd/network/deployment_utils.py +++ b/ocrd/ocrd/network/deployment_utils.py @@ -58,12 +58,12 @@ class CustomDockerClient(DockerClient): Problems: APIClient must be given the API-version because it cannot connect prior to read it. I could imagine this could cause Problems. This is not a rushed implementation and was the only workaround I could find that allows password/keyfile to be used (by default only keyfile from - ~/.ssh/config can be used to authentificate via ssh) + ~/.ssh/config can be used to authenticate via ssh) XXX 2: Reasons to extend DockerClient: The code-changes regarding the connection should be in - one place so I decided to create `CustomSshHttpAdapter` as an inner class. The super - constructor *must not* be called to make this workaround work. Otherwise the APIClient - constructor would be invoced without `version` and that would cause a connection-attempt before + one place, so I decided to create `CustomSshHttpAdapter` as an inner class. The super + constructor *must not* be called to make this workaround work. Otherwise, the APIClient + constructor would be invoked without `version` and that would cause a connection-attempt before this workaround can be applied. """ diff --git a/ocrd/ocrd/network/helpers.py b/ocrd/ocrd/network/helpers.py index 882c8822c..e5a53ce2c 100644 --- a/ocrd/ocrd/network/helpers.py +++ b/ocrd/ocrd/network/helpers.py @@ -1,14 +1,8 @@ from typing import Tuple from re import split -from pathlib import Path from os import environ from os.path import join, exists -from ocrd.network.rabbitmq_utils import ( - OcrdProcessingMessage, - OcrdResultMessage -) - def verify_database_url(mongodb_address: str) -> str: database_prefix = 'mongodb://' @@ -48,7 +42,7 @@ def get_workspaces_dir() -> str: """get the path to the workspaces folder The processing-workers must have access to the workspaces. First idea is that they are provided - via nfs and allways available under $XDG_DATA_HOME/ocrd-workspaces. This function provides the + via nfs and always available under $XDG_DATA_HOME/ocrd-workspaces. This function provides the absolute path to the folder and raises a ValueError if it is not available """ if 'XDG_DATA_HOME' in environ: @@ -59,27 +53,3 @@ def get_workspaces_dir() -> str: if not exists(res): raise ValueError('Ocrd-Workspaces directory not found. Expected \'{res}\'') return res - - -def construct_dummy_processing_message() -> OcrdProcessingMessage: - return OcrdProcessingMessage( - job_id='dummy-job-id', - processor_name='ocrd-dummy', - created_time=None, # Auto generated if None - path_to_mets='/home/mm/Desktop/ws_example/mets.xml', - workspace_id=None, # Not required, workspace is not uploaded through the Workspace Server - input_file_grps=['DEFAULT'], - output_file_grps=['DUMMY-OUTPUT'], - page_id='PHYS0001..PHYS0003', # Process only the first 3 pages - parameters={}, - result_queue_name=None # Not implemented yet, do not set - ) - - -def construct_dummy_result_message() -> OcrdResultMessage: - return OcrdResultMessage( - job_id='dummy-job_id', - status='RUNNING', - workspace_id='dummy-workspace_id', - path_to_mets='/home/mm/Desktop/ws_example/mets.xml' - ) diff --git a/ocrd/ocrd/network/models/job.py b/ocrd/ocrd/network/models/job.py index 15aaea13f..c0427e0c2 100644 --- a/ocrd/ocrd/network/models/job.py +++ b/ocrd/ocrd/network/models/job.py @@ -17,7 +17,7 @@ class StateEnum(str, Enum): queued = 'QUEUED' running = 'RUNNING' - success = 'SUCCESS' # TODO: SUCCEEDED for consistency + success = 'SUCCESS' failed = 'FAILED' diff --git a/ocrd/ocrd/network/processing_server.py b/ocrd/ocrd/network/processing_server.py index fd1a38f19..7d738fc0d 100644 --- a/ocrd/ocrd/network/processing_server.py +++ b/ocrd/ocrd/network/processing_server.py @@ -3,7 +3,7 @@ from fastapi.responses import JSONResponse import uvicorn from yaml import safe_load -from typing import Dict, Set +from typing import Dict from ocrd_utils import ( getLogger, @@ -14,7 +14,6 @@ from ocrd.network.deployer import Deployer from ocrd.network.deployment_config import ProcessingServerConfig from ocrd.network.rabbitmq_utils import RMQPublisher, OcrdProcessingMessage -from ocrd.network.helpers import construct_dummy_processing_message from ocrd.network.models.job import Job, JobInput, StateEnum from ocrd.network.models.workspace import Workspace from ocrd_validators import ParameterValidator @@ -27,8 +26,8 @@ class ProcessingServer(FastAPI): """FastAPI app to make ocr-d processor calls - The Processing-Server receives calls conforming to the ocr-d webapi regarding the processoing - part. It can run ocrd-processors and provides enpoints to discover processors and watch the job + The Processing-Server receives calls conforming to the ocr-d webapi regarding the processing + part. It can run ocrd-processors and provides endpoints to discover processors and watch the job status. The Processing-Server does not execute the processors itself but starts up a queue and a database to delegate the calls to processing workers. They are started by the Processing-Server @@ -40,13 +39,11 @@ def __init__(self, config_path: str, host: str, port: int) -> None: title="OCR-D Processing Server", description="OCR-D processing and processors") self.log = getLogger(__name__) - self.hostname = host self.port = port - # TODO: Ideally the parse_config should return a Tuple with the 3 configs assigned below - # to prevent passing the entire parsed config around to methods. self.config = ProcessingServer.parse_config(config_path) self.deployer = Deployer(self.config) + self.mongodb_url = None # TODO: Parse the RabbitMQ related data from the `queue_config` # above instead of using the hard coded ones below @@ -57,9 +54,9 @@ def __init__(self, config_path: str, host: str, port: int) -> None: self.rmq_vhost = '/' # Gets assigned when `connect_publisher` is called on the working object - # Note for peer: Check under self.start() self.rmq_publisher = None + # This list holds all processors mentioned in the config file self._processor_list = None # Create routes @@ -71,15 +68,6 @@ def __init__(self, config_path: str, host: str, port: int) -> None: summary='Stop database, queue and processing-workers', ) - # TODO: do we still need this? Remove it?! - self.router.add_api_route( - path='/test-dummy', - endpoint=self.publish_default_processing_message, - methods=['POST'], - status_code=status.HTTP_202_ACCEPTED, - summary='Was this just for testing or do we need this' - ) - self.router.add_api_route( path='/processor/{processor_name}', endpoint=self.run_processor, @@ -183,7 +171,6 @@ def connect_publisher(self, username: str = 'default-publisher', f'{self.rmq_port}{self.rmq_vhost}') self.rmq_publisher = RMQPublisher(host=self.rmq_host, port=self.rmq_port, vhost=self.rmq_vhost) - # TODO: Remove this information before the release self.log.debug(f'RMQPublisher authenticate with username: {username}, password: {password}') self.rmq_publisher.authenticate_and_connect(username=username, password=password) if enable_acks: @@ -201,22 +188,8 @@ def create_message_queues(self) -> None: # The existence/validity of the processor.name is not tested. # Even if an ocr-d processor does not exist, the queue is created self.log.debug(f'Creating a message queue with id: {processor.name}') - # TODO: We may want to track here if there are already queues with the same name self.rmq_publisher.create_queue(queue_name=processor.name) - def publish_default_processing_message(self) -> None: - processing_message = construct_dummy_processing_message() - queue_name = processing_message.processor_name - # TODO: switch back to pickle?! imo we should go with pickle or json and remove one of them - encoded_processing_message = OcrdProcessingMessage.encode_yml(processing_message) - if self.rmq_publisher: - self.log.debug('Publishing the default processing message') - self.rmq_publisher.publish_to_queue(queue_name=queue_name, - message=encoded_processing_message) - else: - self.log.error('RMQPublisher is not connected') - raise Exception('RMQPublisher is not connected') - @property def processor_list(self): if self._processor_list: @@ -288,7 +261,7 @@ async def get_processor_info(self, processor_name) -> Dict: ) return get_ocrd_tool_json(processor_name) - async def get_job(self, processor_name: str, job_id: PydanticObjectId) -> Job: + async def get_job(self, _processor_name: str, job_id: PydanticObjectId) -> Job: """ Return job-information from the database """ job = await Job.get(job_id) diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index 0c98352d4..cb54edf78 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -1,10 +1,12 @@ -# Abstraction for the Processing Server unit in this arch: -# https://user-images.githubusercontent.com/7795705/203554094-62ce135a-b367-49ba-9960-ffe1b7d39b2c.jpg +""" +Abstraction for the Processing Server unit in this arch: +https://user-images.githubusercontent.com/7795705/203554094-62ce135a-b367-49ba-9960-ffe1b7d39b2c.jpg -# Calls to native OCR-D processor should happen through -# the Processing Worker wrapper to hide low level details. -# According to the current requirements, each ProcessingWorker -# is a single OCR-D Processor instance. +Calls to native OCR-D processor should happen through +the Processing Worker wrapper to hide low level details. +According to the current requirements, each ProcessingWorker +is a single OCR-D Processor instance. +""" from frozendict import frozendict from functools import lru_cache, wraps @@ -66,7 +68,6 @@ def connect_consumer(self, username: str = 'default-consumer', password: str = 'default-consumer') -> None: self.log.debug(f'Connecting RMQConsumer to RabbitMQ server: {self.rmq_host}:{self.rmq_port}{self.rmq_vhost}') self.rmq_consumer = RMQConsumer(host=self.rmq_host, port=self.rmq_port, vhost=self.rmq_vhost) - # TODO: Remove this information before the release self.log.debug(f'RMQConsumer authenticates with username: {username}, password: {password}') self.rmq_consumer.authenticate_and_connect(username=username, password=password) self.log.debug(f'Successfully connected RMQConsumer.') @@ -75,7 +76,6 @@ def connect_publisher(self, username: str = 'default-publisher', password: str = 'default-publisher', enable_acks: bool = True) -> None: self.log.debug(f'Connecting RMQPublisher to RabbitMQ server: {self.rmq_host}:{self.rmq_port}{self.rmq_vhost}') self.rmq_publisher = RMQPublisher(host=self.rmq_host, port=self.rmq_port, vhost=self.rmq_vhost) - # TODO: Remove this information before the release self.log.debug(f'RMQPublisher authenticates with username: {username}, password: {password}') self.rmq_publisher.authenticate_and_connect(username=username, password=password) if enable_acks: @@ -105,7 +105,6 @@ def on_consumed_message( try: self.log.debug(f'Trying to decode processing message with tag: {delivery_tag}') - # TODO: switch back to pickle?! processing_message: OcrdProcessingMessage = OcrdProcessingMessage.decode_yml(body) except Exception as e: self.log.error(f'Failed to decode processing message body: {body}') @@ -114,8 +113,6 @@ def on_consumed_message( raise Exception(f'Failed to decode processing message with tag: {delivery_tag}, reason: {e}') try: - # TODO: Note to peer: ideally we should avoid doing database related actions - # in this method, and handle database related interactions inside `self.process_message()` self.log.debug(f'Starting to process the received message: {processing_message}') self.process_message(processing_message=processing_message) except Exception as e: @@ -160,12 +157,9 @@ def process_message(self, processing_message: OcrdProcessingMessage) -> None: input_file_grps = processing_message.input_file_grps output_file_grps = processing_message.output_file_grps parameter = processing_message.parameters - - # TODO: Use job_id - will be relevant for the MongoDB to assign job status - # Note to peer: We should encapsulate database related actions to keep this method simple job_id = processing_message.job_id - # TODO: Currently, no caching is performed. + # TODO: Currently, no caching is performed - adopt this: https://github.com/OCR-D/core/pull/972 if self.processor_class: self.log.debug(f'Invoking the pythonic processor: {self.processor_name}') self.log.debug(f'Invoking the processor_class: {self.processor_class}') @@ -225,7 +219,7 @@ def run_processor_from_worker( success = True try: - # TODO: Use the cached_processor flag here once #972 is merged to core + # TODO: Currently, no caching is performed - adopt this: https://github.com/OCR-D/core/pull/972 run_processor( processorClass=processor_class, workspace=workspace, @@ -283,6 +277,8 @@ def set_job_state(self, job_id: Any, success: bool): db.Job.update_one({'_id': job_id}, {'$set': {'state': state}}, upsert=False) +# TODO: Currently, no caching is performed - adopt this: https://github.com/OCR-D/core/pull/972 +# These two methods should be placed in their correct modules # Method adopted from Triet's implementation # https://github.com/OCR-D/core/pull/884/files#diff-8b69cb85b5ffcfb93a053791dec62a2f909a0669ae33d8a2412f246c3b01f1a3R260 def freeze_args(func: Callable) -> Callable: diff --git a/ocrd/ocrd/network/rabbitmq_utils/Dockerfile-RabbitMQ b/ocrd/ocrd/network/rabbitmq_utils/Dockerfile-RabbitMQ deleted file mode 100644 index bb0c223e0..000000000 --- a/ocrd/ocrd/network/rabbitmq_utils/Dockerfile-RabbitMQ +++ /dev/null @@ -1,10 +0,0 @@ -FROM rabbitmq:3.8.27-management-alpine - -ADD ./rabbitmq.conf /etc/rabbitmq/ - -RUN chown rabbitmq:rabbitmq /etc/rabbitmq/rabbitmq.conf - -ADD --chown=rabbitmq ./definitions.json /etc/rabbitmq/ -ENV RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS="-rabbitmq_management load_definitions \"/etc/rabbitmq/definitions.json\"" - -EXPOSE 5672 15672 diff --git a/ocrd/ocrd/network/rabbitmq_utils/connector.py b/ocrd/ocrd/network/rabbitmq_utils/connector.py index 46abbc1e8..6f6dbfb47 100644 --- a/ocrd/ocrd/network/rabbitmq_utils/connector.py +++ b/ocrd/ocrd/network/rabbitmq_utils/connector.py @@ -260,10 +260,3 @@ def basic_publish(channel: BlockingChannel, exchange_name: str, routing_key: str body=message_body, properties=properties ) - - """ - @staticmethod - def basic_consume(channel: BlockingChannel) -> None: - # TODO: provide a general consume method here as well - pass - """ diff --git a/ocrd/ocrd/network/rabbitmq_utils/consumer.py b/ocrd/ocrd/network/rabbitmq_utils/consumer.py index 918fc3a79..f9b769ca6 100644 --- a/ocrd/ocrd/network/rabbitmq_utils/consumer.py +++ b/ocrd/ocrd/network/rabbitmq_utils/consumer.py @@ -5,18 +5,9 @@ """ import logging -from time import sleep from typing import Any, Union -from pika import ( - PlainCredentials -) -from pika.spec import ( - BasicProperties, - Basic, -) -from pika.adapters.blocking_connection import BlockingChannel - +from pika import PlainCredentials from ocrd.network.rabbitmq_utils.constants import ( DEFAULT_QUEUE, @@ -63,21 +54,6 @@ def authenticate_and_connect(self, username: str, password: str) -> None: def setup_defaults(self) -> None: RMQConnector.declare_and_bind_defaults(self._connection, self._channel) - def example_run(self) -> None: - self.configure_consuming() - try: - self.start_consuming() - except KeyboardInterrupt: - self._logger.info('Keyboard interruption detected. Closing down peacefully.') - exit(0) - # TODO: Clean leftovers here and inform the RabbitMQ - # server about the disconnection of the consumer - # TODO: Implement the reconnect mechanism - except Exception: - reconnect_delay = 10 - self._logger.info(f'Reconnecting after {reconnect_delay} seconds') - sleep(reconnect_delay) - def get_one_message( self, queue_name: str, @@ -93,16 +69,12 @@ def get_one_message( def configure_consuming( self, - queue_name: str = '', - callback_method: Any = None + queue_name: str, + callback_method: Any ) -> None: self._logger.debug('Issuing consumer related RPC commands') self._logger.debug('Adding consumer cancellation callback') self._channel.add_on_cancel_callback(self.__on_consumer_cancelled) - if not queue_name: - queue_name = DEFAULT_QUEUE - if callback_method is None: - callback_method = self.__on_message_received self.consumer_tag = self._channel.basic_consume( queue_name, callback_method @@ -124,40 +96,6 @@ def __on_consumer_cancelled(self, frame: Any) -> None: if self._channel: self._channel.close() - def __on_message_received( - self, - channel: BlockingChannel, - basic_deliver: Basic.Deliver, - properties: BasicProperties, - body: bytes - ) -> None: - tag = basic_deliver.delivery_tag - app_id = properties.app_id - message = body.decode() - self._logger.debug(f'Received message #{tag} from {app_id}: {message}') - self._logger.debug(f'Received message on channel: {channel}') - self.__ack_message(tag) - - def __ack_message(self, delivery_tag: int) -> None: + def ack_message(self, delivery_tag: int) -> None: self._logger.debug(f'Acknowledging message {delivery_tag}') self._channel.basic_ack(delivery_tag) - - -def main() -> None: - # Connect to localhost:5672 by - # using the virtual host "/" (%2F) - consumer = RMQConsumer(host='localhost', port=5672, vhost='/') - # Configured with definitions.json when building the RabbitMQ image - # Check Dockerfile-RabbitMQ - consumer.authenticate_and_connect( - username='default-consumer', - password='default-consumer' - ) - consumer.setup_defaults() - consumer.example_run() - - -if __name__ == '__main__': - # RabbitMQ Server must be running before starting the example - # I.e., make start-rabbitmq - main() diff --git a/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py b/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py index 1a235c278..3ff252565 100644 --- a/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py +++ b/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py @@ -1,11 +1,9 @@ # Check here for more details: Message structure #139 from __future__ import annotations from datetime import datetime -from pickle import dumps, loads from typing import Any, Dict, List, Optional import yaml from ocrd.network.models.job import Job -from pathlib import Path # TODO: Maybe there is a more compact way to achieve the serialization/deserialization? @@ -53,29 +51,6 @@ def __init__( self.parameters = parameters if parameters else None self.created_time = created_time - # TODO: Implement the validator checks, e.g., - # if the processor name matches the expected regex - - @staticmethod - def encode(ocrd_processing_message: OcrdProcessingMessage) -> bytes: - return dumps(ocrd_processing_message) - - @staticmethod - def decode(ocrd_processing_message: bytes, encoding='utf-8') -> OcrdProcessingMessage: - data = loads(ocrd_processing_message, encoding=encoding) - return OcrdProcessingMessage( - job_id=data.job_id, - processor_name=data.processor_name, - created_time=data.created_time, - path_to_mets=data.path_to_mets, - workspace_id=data.workspace_id, - input_file_grps=data.input_file_grps, - output_file_grps=data.output_file_grps, - page_id=data.page_id, - parameters=data.parameters, - result_queue_name=data.result_queue - ) - @staticmethod def encode_yml(ocrd_processing_message: OcrdProcessingMessage) -> bytes: """convert OcrdProcessingMessage to yml @@ -122,20 +97,6 @@ def __init__(self, job_id: str, status: str, workspace_id: str, path_to_mets: st self.workspace_id = workspace_id self.path_to_mets = path_to_mets - @staticmethod - def encode(ocrd_result_message: OcrdResultMessage) -> bytes: - return dumps(ocrd_result_message) - - @staticmethod - def decode(ocrd_result_message: bytes, encoding: str = 'utf-8') -> OcrdResultMessage: - data = loads(ocrd_result_message, encoding=encoding) - return OcrdResultMessage( - job_id=data.job_id, - status=data.status, - workspace_id=data.workspace_id, - path_to_mets=data.path_to_mets - ) - @staticmethod def encode_yml(ocrd_result_message: OcrdResultMessage) -> bytes: """convert OcrdResultMessage to yml diff --git a/ocrd/ocrd/network/rabbitmq_utils/publisher.py b/ocrd/ocrd/network/rabbitmq_utils/publisher.py index 971a578d0..e30104700 100644 --- a/ocrd/ocrd/network/rabbitmq_utils/publisher.py +++ b/ocrd/ocrd/network/rabbitmq_utils/publisher.py @@ -5,8 +5,7 @@ """ import logging -from time import sleep -from typing import Any, Optional +from typing import Optional from pika import ( BasicProperties, @@ -59,25 +58,6 @@ def authenticate_and_connect(self, username: str, password: str) -> None: def setup_defaults(self) -> None: RMQConnector.declare_and_bind_defaults(self._connection, self._channel) - def example_run(self) -> None: - while True: - try: - messages = 1 - message = f'#{messages}' - self.publish_to_queue(queue_name=DEFAULT_ROUTER, message=message) - messages += 1 - sleep(2) - except KeyboardInterrupt: - self._logger.info('Keyboard interruption detected. Closing down peacefully.') - exit(0) - # TODO: Clean leftovers here and inform the RabbitMQ - # server about the disconnection of the publisher - # TODO: Implement the reconnect mechanism - except Exception: - reconnect_delay = 10 - self._logger.info(f'Reconnecting after {reconnect_delay} seconds') - sleep(reconnect_delay) - def create_queue( self, queue_name: str, @@ -142,56 +122,3 @@ def publish_to_queue( def enable_delivery_confirmations(self) -> None: self._logger.debug('Enabling delivery confirmations (Confirm.Select RPC)') RMQConnector.confirm_delivery(channel=self._channel) - - # TODO: Find a way to use this callback method, - # seems not possible with Blocking Connections - def __on_delivery_confirmation(self, frame) -> None: - confirmation_type = frame.method.NAME.split('.')[1].lower() - delivery_tag: int = frame.method.delivery_tag - ack_multiple = frame.method.multiple - - self._logger.debug(f'Received: {confirmation_type} ' - f'for tag: {delivery_tag} ' - f'(multiple: {ack_multiple})') - - if confirmation_type == 'ack': - self.acked_counter += 1 - elif confirmation_type == 'nack': - self.nacked_counter += 1 - - del self.deliveries[delivery_tag] - - if ack_multiple: - for tmp_tag in list(self.deliveries.keys()): - if tmp_tag <= delivery_tag: - self.acked_counter += 1 - del self.deliveries[tmp_tag] - - # TODO: Check here for stale entries inside the _deliveries - # and attempt to re-delivery with max amount of tries (not defined yet) - - self._logger.debug( - 'Published %i messages, %i have yet to be confirmed, ' - '%i were acked and %i were nacked', self.message_counter, - len(self.deliveries), self.acked_counter, self.nacked_counter) - - -def main() -> None: - # Connect to localhost:5672 by - # using the virtual host "/" (%2F) - publisher = RMQPublisher(host='localhost', port=5672, vhost='/') - # Configured with definitions.json when building the RabbitMQ image - # Check Dockerfile-RabbitMQ - publisher.authenticate_and_connect( - username='default-publisher', - password='default-publisher' - ) - publisher.setup_defaults() - publisher.enable_delivery_confirmations() - publisher.example_run() - - -if __name__ == '__main__': - # RabbitMQ Server must be running before starting the example - # I.e., make start-rabbitmq - main() diff --git a/ocrd_validators/ocrd_validators/processing_server_validator.py b/ocrd_validators/ocrd_validators/processing_server_validator.py index e3b5c9d8b..896782430 100644 --- a/ocrd_validators/ocrd_validators/processing_server_validator.py +++ b/ocrd_validators/ocrd_validators/processing_server_validator.py @@ -7,9 +7,9 @@ # TODO: provide a link somewhere in this file as it is done in ocrd_tool.schema.yml but best with a -# working link. Currently it is here: -# https://github.com/OCR-D/spec/pull/222/files#diff-a71bf71cbc7d9ce94fded977f7544aba4df9e7bdb8fc0cf1014e14eb67a9b273 -# But that is a PR not merged yet +# working link. Currently it is here: +# https://github.com/OCR-D/spec/pull/222/files#diff-a71bf71cbc7d9ce94fded977f7544aba4df9e7bdb8fc0cf1014e14eb67a9b273 +# But that is a PR not merged yet class ProcessingServerValidator(JsonValidator): """ JsonValidator validating against the schema for the Processing Server From 30498a76e81c8070c5907f5984d76a5a9feeb6df Mon Sep 17 00:00:00 2001 From: joschrew Date: Tue, 14 Feb 2023 11:55:15 +0100 Subject: [PATCH 117/226] Add cleanup and exceptionhandling to startup Starting up parts of the service (queue, database, workers) can easily fail (wrong password, keyfile, etc. in the config-file). Logging and cleanup is added to simplified the debugging. --- ocrd/ocrd/network/deployment_utils.py | 11 +++---- ocrd/ocrd/network/processing_server.py | 44 ++++++++++++++------------ 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/ocrd/ocrd/network/deployment_utils.py b/ocrd/ocrd/network/deployment_utils.py index 72221a2c7..b671d96fe 100644 --- a/ocrd/ocrd/network/deployment_utils.py +++ b/ocrd/ocrd/network/deployment_utils.py @@ -14,12 +14,11 @@ def create_ssh_client(address: str, username: str, password: Union[str, None], keypath: Union[str, None]) -> SSHClient: client = SSHClient() client.set_missing_host_key_policy(AutoAddPolicy) - log = getLogger(__name__) - log.debug(f'creating ssh-client with username: "{username}", keypath: "{keypath}". ' - f'host: {address}') - # TODO: connecting could easily fail here: wrong password, wrong path to keyfile etc. Maybe - # would be better to use except and try to give custom error message when failing - client.connect(hostname=address, username=username, password=password, key_filename=keypath) + try: + client.connect(hostname=address, username=username, password=password, key_filename=keypath) + except Exception: + getLogger(__name__).error(f'Error creating SSHClient for host: \'{address}\'') + raise return client diff --git a/ocrd/ocrd/network/processing_server.py b/ocrd/ocrd/network/processing_server.py index 7d738fc0d..a9cfe9f36 100644 --- a/ocrd/ocrd/network/processing_server.py +++ b/ocrd/ocrd/network/processing_server.py @@ -120,26 +120,30 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE def start(self) -> None: """ deploy agents (db, queue, workers) and start the processing server with uvicorn """ - rabbitmq_url = self.deployer.deploy_rabbitmq() - - self.mongodb_url = self.deployer.deploy_mongodb() - - # Give enough time for the RabbitMQ server to get deployed and get fully configured - # Needed to prevent connection of the publisher before the RabbitMQ is deployed - sleep(3) # TODO: Sleeping here is bad and better check should be performed - - # The RMQPublisher is initialized and a connection to the RabbitMQ is performed - self.connect_publisher() - - self.log.debug(f'Creating message queues on RabbitMQ instance url: {rabbitmq_url}') - self.create_message_queues() - - # Deploy processing hosts where processing workers are running on - # Note: A deployed processing worker starts listening to a message queue with id - # processor.name - self.deployer.deploy_hosts(rabbitmq_url, self.mongodb_url) - - self.log.debug(f'Starting uvicorn: {self.hostname}:{self.port}') + try: + rabbitmq_url = self.deployer.deploy_rabbitmq() + + self.mongodb_url = self.deployer.deploy_mongodb() + + # Give enough time for the RabbitMQ server to get deployed and get fully configured + # Needed to prevent connection of the publisher before the RabbitMQ is deployed + sleep(3) # TODO: Sleeping here is bad and better check should be performed + + # The RMQPublisher is initialized and a connection to the RabbitMQ is performed + self.connect_publisher() + + self.log.debug(f'Creating message queues on RabbitMQ instance url: {rabbitmq_url}') + self.create_message_queues() + + # Deploy processing hosts where processing workers are running on + # Note: A deployed processing worker starts listening to a message queue with id + # processor.name + self.deployer.deploy_hosts(rabbitmq_url, self.mongodb_url) + except Exception: + self.log.error('Error during startup of processing server. Trying to kill parts of ' + 'incompletely deployed service') + self.deployer.kill_all() + raise uvicorn.run(self, host=self.hostname, port=self.port) @staticmethod From 6a3836d6880fa63eaa006bf1091788261b0b79a8 Mon Sep 17 00:00:00 2001 From: joschrew Date: Tue, 14 Feb 2023 12:15:34 +0100 Subject: [PATCH 118/226] Remove unnecessary loggging and clean comments --- ocrd/ocrd/network/deployer.py | 69 ++++---------------- ocrd/ocrd/network/processing_server.py | 5 +- ocrd/ocrd/network/rabbitmq_utils/consumer.py | 3 +- 3 files changed, 13 insertions(+), 64 deletions(-) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index db0342191..db6c4b113 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -25,7 +25,6 @@ from pathlib import Path -# TODO: remove debug log statements before beta, their purpose is development only class Deployer: """ Class to wrap the deployment-functionality of the OCR-D Processing-Servers @@ -39,45 +38,27 @@ def __init__(self, config: ProcessingServerConfig) -> None: config: Parsed processing-server-configuration """ self.log = getLogger(__name__) - self.log.debug('The Deployer of the ProcessingServer was invoked') self.config = config self.hosts = HostData.from_config(config.hosts) self.mongo_pid = None self.mq_pid = None - # TODO: We should have a data structure here to manage the connections and PIDs: - # - RabbitMQ - (host address, pid on that host) - # - MongoDB - (host address, pid on that host) - # - Processing Hosts - (host address) - # - Processing Workers - (pid on that host address) - # The PIDs are stored for future usage - i.e. for killing them forcefully/gracefully. - # Currently, the connections (ssh_client, docker_client) and - # the PIDs are stored inside the config data classes - def kill_all(self) -> None: - # The order of killing is important to optimize graceful shutdown in the future - # If RabbitMQ server is killed before killing Processing Workers, that may have - # bad outcome and leave Processing Workers in an unpredictable state + """ kill all started services: workers, database, queue - # First kill the active Processing Workers on Processing Hosts - # They may still want to update something in the db before closing - # They may still want to nack the currently processed messages back to the RabbitMQ Server + The order of killing is important to optimize graceful shutdown in the future. If RabbitMQ + server is killed before killing Processing Workers, that may have bad outcome and leave + Processing Workers in an unpredictable state + """ self.kill_hosts() - - # Second kill the MongoDB self.kill_mongodb() - - # Third kill the RabbitMQ Server self.kill_rabbitmq() def deploy_hosts(self, rabbitmq_url: str, mongodb_url: str) -> None: - self.log.debug('Starting to deploy hosts') for host in self.hosts: self.log.debug(f'Deploying processing workers on host: {host.config.address}') for processor in host.config.processors: self._deploy_processing_worker(processor, host, rabbitmq_url, mongodb_url) - # TODO: The connections are correctly closed on host level, but created on processing - # worker level? if host.ssh_client: host.ssh_client.close() if host.docker_client: @@ -129,12 +110,11 @@ def _deploy_processing_worker(self, processor: WorkerConfig, host: HostData, def deploy_rabbitmq(self, image: str = 'rabbitmq:3-management', detach: bool = True, remove: bool = True, ports_mapping: Union[Dict, None] = None) -> str: """Start docker-container with rabbitmq + + This method deploys the RabbitMQ Server. Handling of creation of queues, submitting messages + to queues, and receiving messages from queues is part of the RabbitMQ Library which is part + of the OCR-D WebAPI implementation. """ - # Note for a peer - # This method deploys the RabbitMQ Server. - # Handling of creation of queues, submitting messages to queues, - # and receiving messages from queues is part of the RabbitMQ Library - # Which is part of the OCR-D WebAPI implementation. self.log.debug(f'Trying to deploy image[{image}], with modes: detach[{detach}],' f'remove[{remove}]') @@ -153,7 +133,6 @@ def deploy_rabbitmq(self, image: str = 'rabbitmq:3-management', detach: bool = T 15672: 15672, 25672: 25672 } - self.log.debug(f'Ports mapping: {ports_mapping}') local_defs_path = Path(__file__).parent.resolve() / 'rabbitmq_utils' / 'definitions.json' container_defs_path = "/etc/rabbitmq/definitions.json" res = client.containers.run( @@ -199,8 +178,6 @@ def deploy_mongodb(self, image: str = 'mongo', detach: bool = True, remove: bool ports_mapping = { 27017: self.config.mongo.port } - self.log.debug(f'Ports mapping: {ports_mapping}') - # TODO: what about the data-dir? Must data be preserved between runs? res = client.containers.run( image=image, detach=detach, @@ -222,13 +199,9 @@ def deploy_mongodb(self, image: str = 'mongo', detach: bool = True, remove: bool return mongodb_url def kill_rabbitmq(self) -> None: - # TODO: The PID must not be stored in the configuration `mq_data`. Why not? if not self.mq_pid: self.log.warning('No running RabbitMQ instance found') - # TODO: Ignoring this silently is problematic in the future. Why? return - self.log.debug(f'Trying to stop the deployed RabbitMQ with PID: {self.mq_pid}') - client = create_docker_client(self.config.queue.address, self.config.queue.username, self.config.queue.password, self.config.queue.keypath) client.containers.get(self.mq_pid).stop() @@ -237,13 +210,9 @@ def kill_rabbitmq(self) -> None: self.log.debug('The RabbitMQ is stopped') def kill_mongodb(self) -> None: - # TODO: The PID must not be stored in the configuration `mongo_data`. Why not? if not self.mongo_pid: self.log.warning('No running MongoDB instance found') - # TODO: Ignoring this silently is problematic in the future. Why? return - self.log.debug(f'Trying to stop the deployed MongoDB with PID: {self.mongo_pid}') - client = create_docker_client(self.config.mongo.address, self.config.mongo.username, self.config.mongo.password, self.config.mongo.keypath) client.containers.get(self.mongo_pid).stop() @@ -268,7 +237,6 @@ def kill_hosts(self) -> None: def kill_processing_worker(self, host: HostData) -> None: for pid in host.pids_native: self.log.debug(f'Trying to kill/stop native processor: with PID: \'{pid}\'') - # TODO: For graceful shutdown we may want to send additional parameters to kill host.ssh_client.exec_command(f'kill {pid}') host.pids_native = [] @@ -279,16 +247,6 @@ def kill_processing_worker(self, host: HostData) -> None: host.docker_client.containers.get(pid).stop() host.pids_docker = [] - # Note: Invoking a pythonic processor is slightly different from the description in the spec. - # In order to achieve the exact spec call all ocr-d processors should be refactored... - # TODO: To deploy a processing worker (i.e. an ocr-d processor): - # 1. Invoke pythonic processor: - # ` --queue= --database= - # Omit the `processing-worker` argument. - # 2. Invoke non-pythonic processor: - # `ocrd processing-worker --queue= --database=` - # E.g., olena-binarize - def start_native_processor(self, client: SSHClient, processor_name: str, queue_url: str, database_url: str, bin_dir: Optional[str] = None) -> str: """ start a processor natively on a host via ssh @@ -303,10 +261,8 @@ def start_native_processor(self, client: SSHClient, processor_name: str, queue_u Returns: str: pid of running process """ - # TODO: some processors are bashlib. They have to be started differently. Open Question: - # how to find out if a processor is bashlib + # TODO: some processors are bashlib. They have to be started differently self.log.debug(f'Starting native processor: {processor_name}') - self.log.debug(f'The processor connects to queue: {queue_url} and mongodb: {database_url}') channel = client.invoke_shell() stdin, stdout = channel.makefile('wb'), channel.makefile('rb') if bin_dir: @@ -321,7 +277,6 @@ def start_native_processor(self, client: SSHClient, processor_name: str, queue_u # returning from the function stdin.write(f'{cmd} & \n echo xyz$!xyz \n exit \n') output = stdout.read().decode('utf-8') - self.log.debug(f'Output for processor {processor_name}: {output}') stdout.close() stdin.close() return re_search(r'xyz([0-9]+)xyz', output).group(1) # type: ignore @@ -329,9 +284,7 @@ def start_native_processor(self, client: SSHClient, processor_name: str, queue_u def start_docker_processor(self, client: CustomDockerClient, processor_name: str, queue_url: str, database_url: str) -> str: self.log.debug(f'Starting docker container processor: {processor_name}') - # TODO: queue_url and database_url are ready to be used - self.log.debug(f'The processor connects to queue: {queue_url} and mongodb: {database_url}') - # TODO: add real command here to start processing server here + # TODO: add real command here to start processing server in docker here res = client.containers.run('debian', 'sleep 500s', detach=True, remove=True) assert res and res.id, f'Running processor: {processor_name} in docker-container failed' return res.id diff --git a/ocrd/ocrd/network/processing_server.py b/ocrd/ocrd/network/processing_server.py index a9cfe9f36..01d7cf75e 100644 --- a/ocrd/ocrd/network/processing_server.py +++ b/ocrd/ocrd/network/processing_server.py @@ -170,18 +170,15 @@ async def stop_deployed_agents(self) -> None: self.deployer.kill_all() def connect_publisher(self, username: str = 'default-publisher', - password: str = 'default-publisher', enable_acks: bool =True) -> None: + password: str = 'default-publisher', enable_acks: bool = True) -> None: self.log.debug(f'Connecting RMQPublisher to RabbitMQ server: {self.rmq_host}:' f'{self.rmq_port}{self.rmq_vhost}') self.rmq_publisher = RMQPublisher(host=self.rmq_host, port=self.rmq_port, vhost=self.rmq_vhost) - self.log.debug(f'RMQPublisher authenticate with username: {username}, password: {password}') self.rmq_publisher.authenticate_and_connect(username=username, password=password) if enable_acks: self.rmq_publisher.enable_delivery_confirmations() self.log.debug('Delivery confirmations are enabled') - else: - self.log.debug('Delivery confirmations are disabled') self.log.debug('Successfully connected RMQPublisher.') def create_message_queues(self) -> None: diff --git a/ocrd/ocrd/network/rabbitmq_utils/consumer.py b/ocrd/ocrd/network/rabbitmq_utils/consumer.py index f9b769ca6..bcc034748 100644 --- a/ocrd/ocrd/network/rabbitmq_utils/consumer.py +++ b/ocrd/ocrd/network/rabbitmq_utils/consumer.py @@ -72,8 +72,7 @@ def configure_consuming( queue_name: str, callback_method: Any ) -> None: - self._logger.debug('Issuing consumer related RPC commands') - self._logger.debug('Adding consumer cancellation callback') + self._logger.debug(f'Configuring consuming with queue: {queue_name}') self._channel.add_on_cancel_callback(self.__on_consumer_cancelled) self.consumer_tag = self._channel.basic_consume( queue_name, From c838ed7f086f894d2a120ceb0034c19f67bd2902 Mon Sep 17 00:00:00 2001 From: joschrew Date: Tue, 14 Feb 2023 15:26:49 +0100 Subject: [PATCH 119/226] Add job response model Previously the job from the database was returned but an own model for the response makes it possible to change both independently --- ocrd/ocrd/network/models/job.py | 26 +++++++++++++++++++++++++- ocrd/ocrd/network/processing_server.py | 16 ++++++++-------- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/ocrd/ocrd/network/models/job.py b/ocrd/ocrd/network/models/job.py index c0427e0c2..6b3b74c0b 100644 --- a/ocrd/ocrd/network/models/job.py +++ b/ocrd/ocrd/network/models/job.py @@ -22,13 +22,15 @@ class StateEnum(str, Enum): class JobInput(BaseModel): + """ Wraps the parameters required to make a run-processor-request + """ path: Optional[str] = None workspace_id: Optional[str] = None description: Optional[str] = None input_file_grps: List[str] output_file_grps: Optional[List[str]] page_id: Optional[str] = None - parameters: dict = {} # Always set to an empty dict when it's None, otherwise it won't pass the ocrd validation + parameters: dict = {} # Always set to empty dict when None, otherwise it fails ocr-d-validation class Config: schema_extra = { @@ -43,9 +45,22 @@ class Config: } +class JobOutput(BaseModel): + """ Wraps output information for a job-response + """ + job_id: str + processor_name: str + state: StateEnum + workspace_path: Optional[str] + workspace_id: Optional[str] + + class Job(Document): + """ Job representation in the database + """ processor_name: str path: str + workspace_id: Optional[str] description: Optional[str] state: StateEnum input_file_grps: List[str] @@ -57,3 +72,12 @@ class Job(Document): class Settings: use_enum_values = True + + def to_job_output(self) -> JobOutput: + return JobOutput( + job_id=str(self.id), + processor_name=self.processor_name, + state=self.state, + workspace_path=self.path if not self.workspace_id else None, + workspace_id=self.workspace_id, + ) diff --git a/ocrd/ocrd/network/processing_server.py b/ocrd/ocrd/network/processing_server.py index 01d7cf75e..9d7a23581 100644 --- a/ocrd/ocrd/network/processing_server.py +++ b/ocrd/ocrd/network/processing_server.py @@ -14,7 +14,7 @@ from ocrd.network.deployer import Deployer from ocrd.network.deployment_config import ProcessingServerConfig from ocrd.network.rabbitmq_utils import RMQPublisher, OcrdProcessingMessage -from ocrd.network.models.job import Job, JobInput, StateEnum +from ocrd.network.models.job import Job, JobInput, JobOutput, StateEnum from ocrd.network.models.workspace import Workspace from ocrd_validators import ParameterValidator from ocrd.network.database import initiate_database @@ -75,19 +75,19 @@ def __init__(self, config_path: str, host: str, port: int) -> None: tags=['processing'], status_code=status.HTTP_200_OK, summary='Submit a job to this processor', - response_model=Job, + response_model=JobOutput, response_model_exclude_unset=True, response_model_exclude_none=True ) self.router.add_api_route( - path='/processor/{processor_name}/{job_id}', + path='/processor/{_processor_name}/{job_id}', endpoint=self.get_job, methods=['GET'], tags=['processing'], status_code=status.HTTP_200_OK, summary='Get information about a job based on its ID', - response_model=Job, + response_model=JobOutput, response_model_exclude_unset=True, response_model_exclude_none=True ) @@ -203,7 +203,7 @@ def processor_list(self): return self._processor_list # TODO: how do we want to do the whole model-stuff? Webapi (openapi.yml) uses ProcessorJob - async def run_processor(self, processor_name: str, data: JobInput) -> Job: + async def run_processor(self, processor_name: str, data: JobInput) -> JobOutput: """ Queue a processor job """ if processor_name not in self.processor_list: @@ -250,7 +250,7 @@ async def run_processor(self, processor_name: str, data: JobInput) -> Job: self.rmq_publisher.publish_to_queue(processor_name, encoded_processing_message) else: raise Exception('RMQPublisher is not connected') - return job + return job.to_job_output() async def get_processor_info(self, processor_name) -> Dict: """ Return a processor's ocrd-tool.json @@ -262,12 +262,12 @@ async def get_processor_info(self, processor_name) -> Dict: ) return get_ocrd_tool_json(processor_name) - async def get_job(self, _processor_name: str, job_id: PydanticObjectId) -> Job: + async def get_job(self, _processor_name: str, job_id: PydanticObjectId) -> JobOutput: """ Return job-information from the database """ job = await Job.get(job_id) if job: - return job + return job.to_job_output() raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail='Job not found.' From d763d4c812ccc9897403b222e4661316d5596540 Mon Sep 17 00:00:00 2001 From: joschrew Date: Tue, 14 Feb 2023 17:14:20 +0100 Subject: [PATCH 120/226] Add mechanism to ensure queue startup --- ocrd/ocrd/network/deployer.py | 20 ++++++++++++++++++++ ocrd/ocrd/network/processing_server.py | 4 ---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index db6c4b113..1d19b4997 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -23,6 +23,8 @@ HostData, ) from pathlib import Path +from ocrd.network.rabbitmq_utils import RMQPublisher +from time import sleep class Deployer: @@ -158,10 +160,28 @@ def deploy_rabbitmq(self, image: str = 'rabbitmq:3-management', detach: bool = T rmq_port = self.config.queue.port rmq_vhost = '/' # the default virtual host + self.wait_for_rabbitmq_availability(rmq_host, rmq_port, rmq_vhost, + self.config.queue.credentials[0], + self.config.queue.credentials[1]) + rabbitmq_url = f'{rmq_host}:{rmq_port}{rmq_vhost}' self.log.debug(f'The RabbitMQ server was deployed on url: {rabbitmq_url}') return rabbitmq_url + def wait_for_rabbitmq_availability(self, host: str, port: str, vhost: str, username: str, + password: str) -> None: + max_waiting_steps = 15 + while max_waiting_steps > 0: + try: + dummy_publisher = RMQPublisher(host=host, port=port, vhost=vhost) + dummy_publisher.authenticate_and_connect(username=username, password=password) + except Exception: + max_waiting_steps -= 1 + sleep(2) + else: + return + raise RuntimeError('Error waiting for queue startup: timeout exceeded') + def deploy_mongodb(self, image: str = 'mongo', detach: bool = True, remove: bool = True, ports_mapping: Union[Dict, None] = None) -> str: """ Start mongodb in docker diff --git a/ocrd/ocrd/network/processing_server.py b/ocrd/ocrd/network/processing_server.py index 9d7a23581..f76e5d082 100644 --- a/ocrd/ocrd/network/processing_server.py +++ b/ocrd/ocrd/network/processing_server.py @@ -125,10 +125,6 @@ def start(self) -> None: self.mongodb_url = self.deployer.deploy_mongodb() - # Give enough time for the RabbitMQ server to get deployed and get fully configured - # Needed to prevent connection of the publisher before the RabbitMQ is deployed - sleep(3) # TODO: Sleeping here is bad and better check should be performed - # The RMQPublisher is initialized and a connection to the RabbitMQ is performed self.connect_publisher() From 5b4e22deef3f6beeb9a0fdeeccedd3e0674bf9da Mon Sep 17 00:00:00 2001 From: joschrew Date: Tue, 14 Feb 2023 17:38:15 +0100 Subject: [PATCH 121/226] Remove a few more unneeded todo-notes --- ocrd/ocrd/network/deployment_config.py | 2 -- ocrd/ocrd/network/models/job.py | 8 -------- ocrd/ocrd/network/processing_server.py | 10 ++-------- ocrd/ocrd/network/processing_worker.py | 4 ++-- ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py | 4 ---- 5 files changed, 4 insertions(+), 24 deletions(-) diff --git a/ocrd/ocrd/network/deployment_config.py b/ocrd/ocrd/network/deployment_config.py index e223b4798..b7206ac89 100644 --- a/ocrd/ocrd/network/deployment_config.py +++ b/ocrd/ocrd/network/deployment_config.py @@ -1,5 +1,3 @@ -# TODO: this probably breaks python 3.6. Think about whether we really want to use this -from __future__ import annotations from typing import Dict from ocrd.network.deployment_utils import DeployType diff --git a/ocrd/ocrd/network/models/job.py b/ocrd/ocrd/network/models/job.py index 6b3b74c0b..116fe9e86 100644 --- a/ocrd/ocrd/network/models/job.py +++ b/ocrd/ocrd/network/models/job.py @@ -1,11 +1,3 @@ -# These are the models directly taken from the Triet's implementation: -# REST API wrapper for the processor #884 - -# TODO: In the OCR-D WebAPI implementation we did a clear separation between -# the business response models and the low level database models. In order to achieve -# better modularity, we should use the same approach in the network package as well. - - from datetime import datetime from enum import Enum from typing import List, Optional diff --git a/ocrd/ocrd/network/processing_server.py b/ocrd/ocrd/network/processing_server.py index f76e5d082..443c99985 100644 --- a/ocrd/ocrd/network/processing_server.py +++ b/ocrd/ocrd/network/processing_server.py @@ -44,13 +44,8 @@ def __init__(self, config_path: str, host: str, port: int) -> None: self.config = ProcessingServer.parse_config(config_path) self.deployer = Deployer(self.config) self.mongodb_url = None - - # TODO: Parse the RabbitMQ related data from the `queue_config` - # above instead of using the hard coded ones below - - # RabbitMQ related fields, hard coded initially - self.rmq_host = 'localhost' - self.rmq_port = 5672 + self.rmq_host = self.config.queue.address + self.rmq_port = self.config.queue.port self.rmq_vhost = '/' # Gets assigned when `connect_publisher` is called on the working object @@ -198,7 +193,6 @@ def processor_list(self): self._processor_list = list(res) return self._processor_list - # TODO: how do we want to do the whole model-stuff? Webapi (openapi.yml) uses ProcessorJob async def run_processor(self, processor_name: str, data: JobInput) -> JobOutput: """ Queue a processor job """ diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index cb54edf78..de040f64e 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -269,8 +269,8 @@ def run_cli_from_worker( def set_job_state(self, job_id: Any, success: bool): """Set the job status in mongodb to either success or failed """ - # TODO: the way to interact with mongodb needs to be thought about. This is to make it work - # for now for better testing. Beanie seems not suitable as the worker is not async + # TODO: the way to interact with mongodb needs to be thought about. Beanie seems not + # suitable as the worker is not async, thus useng pymongo here state = StateEnum.success if success else StateEnum.failed with pymongo.MongoClient(self.db_url) as client: db = client['ocrd'] diff --git a/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py b/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py index 3ff252565..23c87856c 100644 --- a/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py +++ b/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py @@ -6,10 +6,6 @@ from ocrd.network.models.job import Job -# TODO: Maybe there is a more compact way to achieve the serialization/deserialization? -# Using ProtocolBuffers should decrease the size of the messages in bytes. -# It should be considered once we have a basic running prototype. - class OcrdProcessingMessage: def __init__( self, From 84acd77ae5abc0daf23953982a20a14c7b601296 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Thu, 16 Feb 2023 16:10:40 +0100 Subject: [PATCH 122/226] Remove unnecessary parts --- ocrd/ocrd/network/processing_worker.py | 53 +++----------------------- 1 file changed, 6 insertions(+), 47 deletions(-) diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index de040f64e..ff5918a4b 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -8,18 +8,16 @@ is a single OCR-D Processor instance. """ -from frozendict import frozendict -from functools import lru_cache, wraps import json -from typing import List, Callable, Type, Union, Any +from typing import Any, List import pika.spec import pika.adapters.blocking_connection +import pymongo -from ocrd import Resolver from ocrd_utils import getLogger +from ocrd import Resolver from ocrd.processor.helpers import run_cli, run_processor -from ocrd.processor.base import Processor from ocrd.network.helpers import ( verify_database_url, verify_and_parse_rabbitmq_addr @@ -31,7 +29,6 @@ RMQConsumer, RMQPublisher ) -import pymongo class ProcessingWorker: @@ -159,7 +156,6 @@ def process_message(self, processing_message: OcrdProcessingMessage) -> None: parameter = processing_message.parameters job_id = processing_message.job_id - # TODO: Currently, no caching is performed - adopt this: https://github.com/OCR-D/core/pull/972 if self.processor_class: self.log.debug(f'Invoking the pythonic processor: {self.processor_name}') self.log.debug(f'Invoking the processor_class: {self.processor_class}') @@ -219,14 +215,15 @@ def run_processor_from_worker( success = True try: - # TODO: Currently, no caching is performed - adopt this: https://github.com/OCR-D/core/pull/972 run_processor( processorClass=processor_class, workspace=workspace, page_id=page_id, parameter=parameter, input_file_grp=input_file_grps_str, - output_file_grp=output_file_grps_str + output_file_grp=output_file_grps_str, + # TODO: instance caching turned on breaks processors + instance_caching=False ) except Exception as e: success = False @@ -275,41 +272,3 @@ def set_job_state(self, job_id: Any, success: bool): with pymongo.MongoClient(self.db_url) as client: db = client['ocrd'] db.Job.update_one({'_id': job_id}, {'$set': {'state': state}}, upsert=False) - - -# TODO: Currently, no caching is performed - adopt this: https://github.com/OCR-D/core/pull/972 -# These two methods should be placed in their correct modules -# Method adopted from Triet's implementation -# https://github.com/OCR-D/core/pull/884/files#diff-8b69cb85b5ffcfb93a053791dec62a2f909a0669ae33d8a2412f246c3b01f1a3R260 -def freeze_args(func: Callable) -> Callable: - """ - Transform mutable dictionary into immutable. Useful to be compatible with cache - Code taken from `this post `_ - """ - - @wraps(func) - def wrapped(*args, **kwargs) -> Callable: - args = tuple([frozendict(arg) if isinstance(arg, dict) else arg for arg in args]) - kwargs = {k: frozendict(v) if isinstance(v, dict) else v for k, v in kwargs.items()} - return func(*args, **kwargs) - return wrapped - - -# Method adopted from Triet's implementation -# https://github.com/OCR-D/core/pull/884/files#diff-8b69cb85b5ffcfb93a053791dec62a2f909a0669ae33d8a2412f246c3b01f1a3R260 -@freeze_args -@lru_cache(maxsize=32) -def get_processor(parameter: dict, processor_class=None) -> Union[Type[Processor], None]: - """ - Call this function to get back an instance of a processor. The results are cached based on the parameters. - Args: - parameter (dict): a dictionary of parameters. - processor_class: the concrete `:py:class:~ocrd.Processor` class. - Returns: - When the concrete class of the processor is unknown, `None` is returned. Otherwise, an instance of the - `:py:class:~ocrd.Processor` is returned. - """ - if processor_class: - dict_params = dict(parameter) if parameter else None - return processor_class(workspace=None, parameter=dict_params) - return None From b586f70e3b3a5de1799206566c02dde29192cedc Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Thu, 16 Feb 2023 16:51:51 +0100 Subject: [PATCH 123/226] Add logging to file for processing workers --- ocrd/ocrd/network/processing_worker.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index ff5918a4b..2431e8994 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -9,6 +9,8 @@ """ import json +import logging +from os import getpid from typing import Any, List import pika.spec @@ -34,6 +36,12 @@ class ProcessingWorker: def __init__(self, rabbitmq_addr, mongodb_addr, processor_name, ocrd_tool: dict, processor_class=None) -> None: self.log = getLogger(__name__) + # TODO: Provide more flexibility for configuring file logging (i.e. via ENV variables) + file_handler = logging.FileHandler(f'/tmp/worker_{processor_name}_{getpid()}.log', mode='a') + logging_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' + file_handler.setFormatter(logging.Formatter(logging_format)) + file_handler.setLevel(logging.DEBUG) + self.log.addHandler(file_handler) try: self.db_url = verify_database_url(mongodb_addr) @@ -222,7 +230,7 @@ def run_processor_from_worker( parameter=parameter, input_file_grp=input_file_grps_str, output_file_grp=output_file_grps_str, - # TODO: instance caching turned on breaks processors + # TODO: Instance caching turned on breaks processors instance_caching=False ) except Exception as e: From 0bb9bdba52f48b666e91289f93f8742d3cdc5659 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Fri, 17 Feb 2023 13:46:45 +0100 Subject: [PATCH 124/226] Wait 100ms between deployment of ocr-d processors --- ocrd/ocrd/network/deployer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index 1d19b4997..4d43cedf6 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -108,6 +108,7 @@ def _deploy_processing_worker(self, processor: WorkerConfig, host: HostData, database_url=mongodb_url ) host.pids_docker.append(pid) + sleep(0.1) def deploy_rabbitmq(self, image: str = 'rabbitmq:3-management', detach: bool = True, remove: bool = True, ports_mapping: Union[Dict, None] = None) -> str: From 9ab1146350673eb1b8c87c7f471e06114aa2edd9 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Fri, 17 Feb 2023 16:20:26 +0100 Subject: [PATCH 125/226] Dirty fix - disable interactive shell logging for calamari --- ocrd/ocrd/network/processing_worker.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index 2431e8994..6c4ef7e86 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -12,6 +12,8 @@ import logging from os import getpid from typing import Any, List +from tensorflow.keras.utils import disable_interactive_logging + import pika.spec import pika.adapters.blocking_connection @@ -164,6 +166,11 @@ def process_message(self, processing_message: OcrdProcessingMessage) -> None: parameter = processing_message.parameters job_id = processing_message.job_id + # TODO: Find a proper solution for this dirty fix + # Enabled interactive logging throws exception + if self.processor_name == 'ocrd-calamari-recognize': + disable_interactive_logging() + if self.processor_class: self.log.debug(f'Invoking the pythonic processor: {self.processor_name}') self.log.debug(f'Invoking the processor_class: {self.processor_class}') From 6cd2a893a69a0b17d591af82adb56894f5c872c5 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Fri, 17 Feb 2023 16:42:51 +0100 Subject: [PATCH 126/226] Enable instance caching for processors - tested and working --- ocrd/ocrd/network/processing_worker.py | 9 ++++----- ocrd/ocrd/processor/helpers.py | 5 +++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index 6c4ef7e86..e6ed70b30 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -12,12 +12,11 @@ import logging from os import getpid from typing import Any, List -from tensorflow.keras.utils import disable_interactive_logging - import pika.spec import pika.adapters.blocking_connection import pymongo +from tensorflow.keras.utils import disable_interactive_logging from ocrd_utils import getLogger from ocrd import Resolver @@ -167,8 +166,9 @@ def process_message(self, processing_message: OcrdProcessingMessage) -> None: job_id = processing_message.job_id # TODO: Find a proper solution for this dirty fix - # Enabled interactive logging throws exception if self.processor_name == 'ocrd-calamari-recognize': + # Enabled interactive logging throws an exception + # due to a call of sys.stdout.flush() disable_interactive_logging() if self.processor_class: @@ -237,8 +237,7 @@ def run_processor_from_worker( parameter=parameter, input_file_grp=input_file_grps_str, output_file_grp=output_file_grps_str, - # TODO: Instance caching turned on breaks processors - instance_caching=False + instance_caching=True ) except Exception as e: success = False diff --git a/ocrd/ocrd/processor/helpers.py b/ocrd/ocrd/processor/helpers.py index 81f209d92..131eea47d 100644 --- a/ocrd/ocrd/processor/helpers.py +++ b/ocrd/ocrd/processor/helpers.py @@ -14,7 +14,7 @@ from click import wrap_text from ocrd.workspace import Workspace -from ocrd_utils import freeze_args, getLogger +from ocrd_utils import freeze_args, getLogger, pushd_popd __all__ = [ @@ -118,7 +118,8 @@ def run_processor( mem_output += ' max: %.2f MiB min: %.2f MiB' % (max(mem_usage_values), min(mem_usage_values)) logProfile.info(mem_output) else: - processor.process() + with pushd_popd(workspace.directory): + processor.process() t1_wall = perf_counter() - t0_wall t1_cpu = process_time() - t0_cpu logProfile.info("Executing processor '%s' took %fs (wall) %fs (CPU)( [--input-file-grp='%s' --output-file-grp='%s' --parameter='%s' --page-id='%s']" % ( From 144f26296867eb8b8e0618f2e644e99b3283aab5 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Fri, 17 Feb 2023 17:05:08 +0100 Subject: [PATCH 127/226] Add tensorflow to core requirements This is a temporal solution till we find a better way without relying on the TensorFlow library in core. --- ocrd/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/ocrd/requirements.txt b/ocrd/requirements.txt index 9cefbe8fe..232c56d1f 100644 --- a/ocrd/requirements.txt +++ b/ocrd/requirements.txt @@ -18,3 +18,4 @@ paramiko frozendict~=2.3.4 pika>=1.2.0 beanie~=1.7 +tensorflow From 6c3c6d5ae50fbd2217cd5182812ba131c57d484e Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Mon, 20 Feb 2023 15:22:31 +0100 Subject: [PATCH 128/226] Add changes from fix-972 branch --- ocrd/ocrd/processor/helpers.py | 35 ++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/ocrd/ocrd/processor/helpers.py b/ocrd/ocrd/processor/helpers.py index 131eea47d..f1e793bd0 100644 --- a/ocrd/ocrd/processor/helpers.py +++ b/ocrd/ocrd/processor/helpers.py @@ -1,7 +1,7 @@ """ Helper methods for running and documenting processors """ -from os import environ +from os import chdir, environ, getcwd from time import perf_counter, process_time from functools import lru_cache import json @@ -95,6 +95,9 @@ def run_processor( instance_caching=instance_caching ) + old_cwd = getcwd() + chdir(processor.workspace.directory) + ocrd_tool = processor.ocrd_tool name = '%s v%s' % (ocrd_tool['executable'], processor.version) otherrole = ocrd_tool['steps'][0] @@ -104,22 +107,34 @@ def run_processor( t0_cpu = process_time() if any(x in environ.get('OCRD_PROFILE', '') for x in ['RSS', 'PSS']): backend = 'psutil_pss' if 'PSS' in environ['OCRD_PROFILE'] else 'psutil' - mem_usage = memory_usage(proc=processor.process, - # only run process once - max_iterations=1, - interval=.1, timeout=None, timestamps=True, - # include sub-processes - multiprocess=True, include_children=True, - # get proportional set size instead of RSS - backend=backend) + try: + mem_usage = memory_usage(proc=processor.process, + # only run process once + max_iterations=1, + interval=.1, timeout=None, timestamps=True, + # include sub-processes + multiprocess=True, include_children=True, + # get proportional set size instead of RSS + backend=backend) + except Exception as err: + log.exception("Failure in processor '%s'" % ocrd_tool['executable']) + return err + finally: + chdir(old_cwd) mem_usage_values = [mem for mem, _ in mem_usage] mem_output = 'memory consumption: ' mem_output += ''.join(sparklines(mem_usage_values)) mem_output += ' max: %.2f MiB min: %.2f MiB' % (max(mem_usage_values), min(mem_usage_values)) logProfile.info(mem_output) else: - with pushd_popd(workspace.directory): + try: processor.process() + except Exception as err: + log.exception("Failure in processor '%s'" % ocrd_tool['executable']) + return err + finally: + chdir(old_cwd) + t1_wall = perf_counter() - t0_wall t1_cpu = process_time() - t0_cpu logProfile.info("Executing processor '%s' took %fs (wall) %fs (CPU)( [--input-file-grp='%s' --output-file-grp='%s' --parameter='%s' --page-id='%s']" % ( From cb54a6704e9cb5ef3ee3f50eb8887b940f5c00ff Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Mon, 20 Feb 2023 15:59:32 +0100 Subject: [PATCH 129/226] Replace ocrd logging with python logging --- ocrd/ocrd/network/processing_worker.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index e6ed70b30..94150f570 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -10,15 +10,16 @@ import json import logging -from os import getpid +from os import environ, getpid from typing import Any, List import pika.spec import pika.adapters.blocking_connection import pymongo +# This env variable must be set before importing from Keras +environ['TF_CPP_MIN_LOG_LEVEL'] = '3' from tensorflow.keras.utils import disable_interactive_logging -from ocrd_utils import getLogger from ocrd import Resolver from ocrd.processor.helpers import run_cli, run_processor from ocrd.network.helpers import ( @@ -36,7 +37,7 @@ class ProcessingWorker: def __init__(self, rabbitmq_addr, mongodb_addr, processor_name, ocrd_tool: dict, processor_class=None) -> None: - self.log = getLogger(__name__) + self.log = logging.getLogger(__name__) # TODO: Provide more flexibility for configuring file logging (i.e. via ENV variables) file_handler = logging.FileHandler(f'/tmp/worker_{processor_name}_{getpid()}.log', mode='a') logging_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' From 5a328c2ec1098cd631e049f64dcaad05fcd7c81e Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Mon, 20 Feb 2023 16:22:38 +0100 Subject: [PATCH 130/226] Lower the scope of tf keras import --- ocrd/ocrd/network/processing_worker.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index 94150f570..ffce41773 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -16,9 +16,6 @@ import pika.spec import pika.adapters.blocking_connection import pymongo -# This env variable must be set before importing from Keras -environ['TF_CPP_MIN_LOG_LEVEL'] = '3' -from tensorflow.keras.utils import disable_interactive_logging from ocrd import Resolver from ocrd.processor.helpers import run_cli, run_processor @@ -168,6 +165,10 @@ def process_message(self, processing_message: OcrdProcessingMessage) -> None: # TODO: Find a proper solution for this dirty fix if self.processor_name == 'ocrd-calamari-recognize': + # This env variable must be set before importing from Keras + environ['TF_CPP_MIN_LOG_LEVEL'] = '3' + from tensorflow.keras.utils import disable_interactive_logging + # Enabled interactive logging throws an exception # due to a call of sys.stdout.flush() disable_interactive_logging() From c02f0ba53f38ba854c1938d45641f2855c371b32 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Mon, 20 Feb 2023 16:27:56 +0100 Subject: [PATCH 131/226] Revert logging back to ocrd logging in worker --- ocrd/ocrd/network/processing_worker.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index ffce41773..ec6ffb566 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -18,6 +18,7 @@ import pymongo from ocrd import Resolver +from ocrd_utils import getLogger from ocrd.processor.helpers import run_cli, run_processor from ocrd.network.helpers import ( verify_database_url, @@ -34,7 +35,7 @@ class ProcessingWorker: def __init__(self, rabbitmq_addr, mongodb_addr, processor_name, ocrd_tool: dict, processor_class=None) -> None: - self.log = logging.getLogger(__name__) + self.log = getLogger(__name__) # TODO: Provide more flexibility for configuring file logging (i.e. via ENV variables) file_handler = logging.FileHandler(f'/tmp/worker_{processor_name}_{getpid()}.log', mode='a') logging_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' From cef1d0d7fa472b30f584df4efcec647c7ddec5dd Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Tue, 21 Feb 2023 08:31:36 +0100 Subject: [PATCH 132/226] Refactor imports, logging, spacing --- ocrd/ocrd/network/deployer.py | 32 ++++++------- ocrd/ocrd/network/deployment_utils.py | 10 ++-- ocrd/ocrd/network/helpers.py | 4 +- ocrd/ocrd/network/processing_server.py | 63 ++++++++++++++++---------- ocrd/ocrd/network/processing_worker.py | 61 ++++++++++++++++--------- 5 files changed, 103 insertions(+), 67 deletions(-) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index 4d43cedf6..6e4ad4813 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -10,10 +10,12 @@ from __future__ import annotations from typing import Dict, Union, Optional from paramiko import SSHClient +from pathlib import Path from re import search as re_search +from time import sleep -from ocrd_utils import getLogger +from ocrd_utils import getLogger from ocrd.network.deployment_config import * from ocrd.network.deployment_utils import ( create_docker_client, @@ -22,9 +24,7 @@ DeployType, HostData, ) -from pathlib import Path from ocrd.network.rabbitmq_utils import RMQPublisher -from time import sleep class Deployer: @@ -118,8 +118,8 @@ def deploy_rabbitmq(self, image: str = 'rabbitmq:3-management', detach: bool = T to queues, and receiving messages from queues is part of the RabbitMQ Library which is part of the OCR-D WebAPI implementation. """ - self.log.debug(f'Trying to deploy image[{image}], with modes: detach[{detach}],' - f'remove[{remove}]') + self.log.debug(f'Trying to deploy image[{image}], ' + f'with modes: detach[{detach}], remove[{remove}]') if not self.config or not self.config.queue.address: raise ValueError('Deploying RabbitMQ has failed - missing configuration.') @@ -166,7 +166,7 @@ def deploy_rabbitmq(self, image: str = 'rabbitmq:3-management', detach: bool = T self.config.queue.credentials[1]) rabbitmq_url = f'{rmq_host}:{rmq_port}{rmq_vhost}' - self.log.debug(f'The RabbitMQ server was deployed on url: {rabbitmq_url}') + self.log.info(f'The RabbitMQ server was deployed on url: {rabbitmq_url}') return rabbitmq_url def wait_for_rabbitmq_availability(self, host: str, port: str, vhost: str, username: str, @@ -187,8 +187,8 @@ def deploy_mongodb(self, image: str = 'mongo', detach: bool = True, remove: bool ports_mapping: Union[Dict, None] = None) -> str: """ Start mongodb in docker """ - self.log.debug(f'Trying to deploy image[{image}], with modes: detach[{detach}],' - f'remove[{remove}]') + self.log.debug(f'Trying to deploy image[{image}], ' + f'with modes: detach[{detach}], remove[{remove}]') if not self.config or not self.config.mongo.address: raise ValueError('Deploying MongoDB has failed - missing configuration.') @@ -216,7 +216,7 @@ def deploy_mongodb(self, image: str = 'mongo', detach: bool = True, remove: bool mongodb_host = self.config.mongo.address mongodb_port = self.config.mongo.port mongodb_url = f'{mongodb_prefix}{mongodb_host}:{mongodb_port}' - self.log.debug(f'The MongoDB was deployed on url: {mongodb_url}') + self.log.info(f'The MongoDB was deployed on url: {mongodb_url}') return mongodb_url def kill_rabbitmq(self) -> None: @@ -228,7 +228,7 @@ def kill_rabbitmq(self) -> None: client.containers.get(self.mq_pid).stop() self.mq_pid = None client.close() - self.log.debug('The RabbitMQ is stopped') + self.log.info('The RabbitMQ is stopped') def kill_mongodb(self) -> None: if not self.mongo_pid: @@ -239,7 +239,7 @@ def kill_mongodb(self) -> None: client.containers.get(self.mongo_pid).stop() self.mongo_pid = None client.close() - self.log.debug('The MongoDB is stopped') + self.log.info('The MongoDB is stopped') def kill_hosts(self) -> None: self.log.debug('Starting to kill/stop hosts') @@ -263,8 +263,6 @@ def kill_processing_worker(self, host: HostData) -> None: for pid in host.pids_docker: self.log.debug(f'Trying to kill/stop docker container with PID: {pid}') - # TODO: think about timeout. - # think about using threads to kill parallelized to reduce waiting time host.docker_client.containers.get(pid).stop() host.pids_docker = [] @@ -283,7 +281,7 @@ def start_native_processor(self, client: SSHClient, processor_name: str, queue_u str: pid of running process """ # TODO: some processors are bashlib. They have to be started differently - self.log.debug(f'Starting native processor: {processor_name}') + self.log.info(f'Starting native processor: {processor_name}') channel = client.invoke_shell() stdin, stdout = channel.makefile('wb'), channel.makefile('rb') if bin_dir: @@ -302,9 +300,9 @@ def start_native_processor(self, client: SSHClient, processor_name: str, queue_u stdin.close() return re_search(r'xyz([0-9]+)xyz', output).group(1) # type: ignore - def start_docker_processor(self, client: CustomDockerClient, - processor_name: str, queue_url: str, database_url: str) -> str: - self.log.debug(f'Starting docker container processor: {processor_name}') + def start_docker_processor(self, client: CustomDockerClient, processor_name: str, + queue_url: str, database_url: str) -> str: + self.log.info(f'Starting docker container processor: {processor_name}') # TODO: add real command here to start processing server in docker here res = client.containers.run('debian', 'sleep 500s', detach=True, remove=True) assert res and res.id, f'Running processor: {processor_name} in docker-container failed' diff --git a/ocrd/ocrd/network/deployment_utils.py b/ocrd/ocrd/network/deployment_utils.py index b671d96fe..876cdbc16 100644 --- a/ocrd/ocrd/network/deployment_utils.py +++ b/ocrd/ocrd/network/deployment_utils.py @@ -5,9 +5,13 @@ from docker import APIClient, DockerClient from docker.transport import SSHHTTPAdapter from paramiko import AutoAddPolicy, SSHClient -from ocrd.network.deployment_config import * from ocrd_utils import getLogger +from ocrd.network.deployment_config import * + +__all__ = [ + 'DeployType' +] def create_ssh_client(address: str, username: str, password: Union[str, None], @@ -30,7 +34,7 @@ def create_docker_client(address: str, username: str, password: Union[str, None] class HostData: """class to store runtime information for a host """ - def __init__(self, config: deployment_config.HostConfig) -> None: + def __init__(self, config: HostConfig) -> None: self.config = config self.ssh_client: Union[SSHClient, None] = None self.docker_client: Union[CustomDockerClient, None] = None @@ -38,7 +42,7 @@ def __init__(self, config: deployment_config.HostConfig) -> None: self.pids_docker: List[str] = [] @staticmethod - def from_config(config: List[deployment_config.HostConfig]) -> List[HostData]: + def from_config(config: List[HostConfig]) -> List[HostData]: res = [] for host_config in config: res.append(HostData(host_config)) diff --git a/ocrd/ocrd/network/helpers.py b/ocrd/ocrd/network/helpers.py index e5a53ce2c..c7a749654 100644 --- a/ocrd/ocrd/network/helpers.py +++ b/ocrd/ocrd/network/helpers.py @@ -1,7 +1,7 @@ -from typing import Tuple -from re import split from os import environ from os.path import join, exists +from re import split +from typing import Tuple def verify_database_url(mongodb_address: str) -> str: diff --git a/ocrd/ocrd/network/processing_server.py b/ocrd/ocrd/network/processing_server.py index 443c99985..78aa7a7d8 100644 --- a/ocrd/ocrd/network/processing_server.py +++ b/ocrd/ocrd/network/processing_server.py @@ -1,26 +1,35 @@ +import json +from typing import Dict +import uvicorn +from yaml import safe_load + +from beanie import PydanticObjectId from fastapi import FastAPI, status, Request, HTTPException from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse -import uvicorn -from yaml import safe_load -from typing import Dict from ocrd_utils import ( getLogger, get_ocrd_tool_json, ) -from ocrd_validators import ProcessingServerValidator - +from ocrd_validators import ( + ParameterValidator, + ProcessingServerValidator +) +from ocrd.network.database import initiate_database from ocrd.network.deployer import Deployer from ocrd.network.deployment_config import ProcessingServerConfig -from ocrd.network.rabbitmq_utils import RMQPublisher, OcrdProcessingMessage -from ocrd.network.models.job import Job, JobInput, JobOutput, StateEnum +from ocrd.network.rabbitmq_utils import ( + RMQPublisher, + OcrdProcessingMessage +) +from ocrd.network.models.job import ( + Job, + JobInput, + JobOutput, + StateEnum +) from ocrd.network.models.workspace import Workspace -from ocrd_validators import ParameterValidator -from ocrd.network.database import initiate_database -from beanie import PydanticObjectId -from time import sleep -import json class ProcessingServer(FastAPI): @@ -65,7 +74,7 @@ def __init__(self, config_path: str, host: str, port: int) -> None: self.router.add_api_route( path='/processor/{processor_name}', - endpoint=self.run_processor, + endpoint=self.push_processor_job, methods=['POST'], tags=['processing'], status_code=status.HTTP_200_OK, @@ -131,8 +140,8 @@ def start(self) -> None: # processor.name self.deployer.deploy_hosts(rabbitmq_url, self.mongodb_url) except Exception: - self.log.error('Error during startup of processing server. Trying to kill parts of ' - 'incompletely deployed service') + self.log.error('Error during startup of processing server. ' + 'Trying to kill parts of incompletely deployed service') self.deployer.kill_all() raise uvicorn.run(self, host=self.hostname, port=self.port) @@ -162,15 +171,21 @@ async def stop_deployed_agents(self) -> None: def connect_publisher(self, username: str = 'default-publisher', password: str = 'default-publisher', enable_acks: bool = True) -> None: - self.log.debug(f'Connecting RMQPublisher to RabbitMQ server: {self.rmq_host}:' - f'{self.rmq_port}{self.rmq_vhost}') - self.rmq_publisher = RMQPublisher(host=self.rmq_host, port=self.rmq_port, - vhost=self.rmq_vhost) - self.rmq_publisher.authenticate_and_connect(username=username, password=password) + self.log.info(f'Connecting RMQPublisher to RabbitMQ server: ' + f'{self.rmq_host}:{self.rmq_port}{self.rmq_vhost}') + self.rmq_publisher = RMQPublisher( + host=self.rmq_host, + port=self.rmq_port, + vhost=self.rmq_vhost + ) + self.rmq_publisher.authenticate_and_connect( + username=username, + password=password + ) if enable_acks: self.rmq_publisher.enable_delivery_confirmations() - self.log.debug('Delivery confirmations are enabled') - self.log.debug('Successfully connected RMQPublisher.') + self.log.info('Delivery confirmations are enabled') + self.log.info('Successfully connected RMQPublisher.') def create_message_queues(self) -> None: """Create the message queues based on the occurrence of `processor.name` in the config file @@ -179,7 +194,7 @@ def create_message_queues(self) -> None: for processor in host.processors: # The existence/validity of the processor.name is not tested. # Even if an ocr-d processor does not exist, the queue is created - self.log.debug(f'Creating a message queue with id: {processor.name}') + self.log.info(f'Creating a message queue with id: {processor.name}') self.rmq_publisher.create_queue(queue_name=processor.name) @property @@ -193,7 +208,7 @@ def processor_list(self): self._processor_list = list(res) return self._processor_list - async def run_processor(self, processor_name: str, data: JobInput) -> JobOutput: + async def push_processor_job(self, processor_name: str, data: JobInput) -> JobOutput: """ Queue a processor job """ if processor_name not in self.processor_list: diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index ec6ffb566..7fd3ca01f 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -19,7 +19,10 @@ from ocrd import Resolver from ocrd_utils import getLogger -from ocrd.processor.helpers import run_cli, run_processor +from ocrd.processor.helpers import ( + run_cli, + run_processor +) from ocrd.network.helpers import ( verify_database_url, verify_and_parse_rabbitmq_addr @@ -71,24 +74,39 @@ def __init__(self, rabbitmq_addr, mongodb_addr, processor_name, ocrd_tool: dict, def connect_consumer(self, username: str = 'default-consumer', password: str = 'default-consumer') -> None: - self.log.debug(f'Connecting RMQConsumer to RabbitMQ server: {self.rmq_host}:{self.rmq_port}{self.rmq_vhost}') - self.rmq_consumer = RMQConsumer(host=self.rmq_host, port=self.rmq_port, vhost=self.rmq_vhost) - self.log.debug(f'RMQConsumer authenticates with username: {username}, password: {password}') - self.rmq_consumer.authenticate_and_connect(username=username, password=password) - self.log.debug(f'Successfully connected RMQConsumer.') + self.log.info(f'Connecting RMQConsumer to RabbitMQ server: ' + f'{self.rmq_host}:{self.rmq_port}{self.rmq_vhost}') + self.rmq_consumer = RMQConsumer( + host=self.rmq_host, + port=self.rmq_port, + vhost=self.rmq_vhost + ) + self.log.debug(f'RMQConsumer authenticates with username: ' + f'{username}, password: {password}') + self.rmq_consumer.authenticate_and_connect( + username=username, + password=password + ) + self.log.info(f'Successfully connected RMQConsumer.') def connect_publisher(self, username: str = 'default-publisher', password: str = 'default-publisher', enable_acks: bool = True) -> None: - self.log.debug(f'Connecting RMQPublisher to RabbitMQ server: {self.rmq_host}:{self.rmq_port}{self.rmq_vhost}') - self.rmq_publisher = RMQPublisher(host=self.rmq_host, port=self.rmq_port, vhost=self.rmq_vhost) - self.log.debug(f'RMQPublisher authenticates with username: {username}, password: {password}') + self.log.info(f'Connecting RMQPublisher to RabbitMQ server: ' + f'{self.rmq_host}:{self.rmq_port}{self.rmq_vhost}') + self.rmq_publisher = RMQPublisher( + host=self.rmq_host, + port=self.rmq_port, + vhost=self.rmq_vhost + ) + self.log.debug(f'RMQPublisher authenticates with username: ' + f'{username}, password: {password}') self.rmq_publisher.authenticate_and_connect(username=username, password=password) if enable_acks: self.rmq_publisher.enable_delivery_confirmations() - self.log.debug('Delivery confirmations are enabled') + self.log.info('Delivery confirmations are enabled') else: - self.log.debug('Delivery confirmations are disabled') - self.log.debug('Successfully connected RMQPublisher.') + self.log.info('Delivery confirmations are disabled') + self.log.info('Successfully connected RMQPublisher.') # Define what happens every time a message is consumed # from the queue with name self.processor_name @@ -103,9 +121,9 @@ def on_consumed_message( is_redelivered: bool = delivery.redelivered message_headers: dict = properties.headers - self.log.debug(f'Consumer tag: {consumer_tag}') - self.log.debug(f'Message delivery tag: {delivery_tag}') - self.log.debug(f'Is redelivered message: {is_redelivered}') + self.log.debug(f'Consumer tag: {consumer_tag}, ' + f'message delivery tag: {delivery_tag}, ' + f'redelivered: {is_redelivered}') self.log.debug(f'Message headers: {message_headers}') try: @@ -118,7 +136,7 @@ def on_consumed_message( raise Exception(f'Failed to decode processing message with tag: {delivery_tag}, reason: {e}') try: - self.log.debug(f'Starting to process the received message: {processing_message}') + self.log.info(f'Starting to process the received message: {processing_message}') self.process_message(processing_message=processing_message) except Exception as e: self.log.error(f'Failed to process processing message with tag: {delivery_tag}') @@ -126,18 +144,18 @@ def on_consumed_message( channel.basic_nack(delivery_tag=delivery_tag, multiple=False, requeue=False) raise Exception(f'Failed to process processing message with tag: {delivery_tag}, reason: {e}') - self.log.debug(f'Successfully processed message ') + self.log.info(f'Successfully processed message ') self.log.debug(f'Acking message with tag: {delivery_tag}') channel.basic_ack(delivery_tag=delivery_tag, multiple=False) def start_consuming(self) -> None: if self.rmq_consumer: - self.log.debug(f'Configuring consuming from queue: {self.processor_name}') + self.log.info(f'Configuring consuming from queue: {self.processor_name}') self.rmq_consumer.configure_consuming( queue_name=self.processor_name, callback_method=self.on_consumed_message ) - self.log.debug(f'Starting consuming from queue: {self.processor_name}') + self.log.info(f'Starting consuming from queue: {self.processor_name}') # Starting consuming is a blocking action self.rmq_consumer.start_consuming() else: @@ -176,7 +194,6 @@ def process_message(self, processing_message: OcrdProcessingMessage) -> None: if self.processor_class: self.log.debug(f'Invoking the pythonic processor: {self.processor_name}') - self.log.debug(f'Invoking the processor_class: {self.processor_class}') return_status = self.run_processor_from_worker( processor_class=self.processor_class, workspace=workspace, @@ -206,6 +223,7 @@ def process_message(self, processing_message: OcrdProcessingMessage) -> None: # create_queue method is idempotent - nothing happens if # a queue with the specified name already exists self.rmq_publisher.create_queue(queue_name=processing_message.result_queue) + self.log.info(f'Publishing result message to queue: {processing_message.result_queue}') result_message = OcrdResultMessage( job_id=str(job_id), status=job_status.value, @@ -213,6 +231,7 @@ def process_message(self, processing_message: OcrdProcessingMessage) -> None: path_to_mets=processing_message.path_to_mets, workspace_id=None ) + self.log.debug(f'Result message: {result_message}') encoded_result_message = OcrdResultMessage.encode_yml(result_message) self.rmq_publisher.publish_to_queue( queue_name=processing_message.result_queue, @@ -284,7 +303,7 @@ def set_job_state(self, job_id: Any, success: bool): """Set the job status in mongodb to either success or failed """ # TODO: the way to interact with mongodb needs to be thought about. Beanie seems not - # suitable as the worker is not async, thus useng pymongo here + # suitable as the worker is not async, thus using pymongo here state = StateEnum.success if success else StateEnum.failed with pymongo.MongoClient(self.db_url) as client: db = client['ocrd'] From b73c899ff4e0f4c8f93f89a0ff622e90c332dd1e Mon Sep 17 00:00:00 2001 From: joschrew Date: Tue, 21 Feb 2023 10:30:42 +0100 Subject: [PATCH 133/226] Remove config-var path_to_bin_dir again --- ocrd/ocrd/network/deployer.py | 10 ++-------- ocrd/ocrd/network/deployment_config.py | 2 -- ocrd_validators/ocrd_validators/config.schema.yml | 3 --- 3 files changed, 2 insertions(+), 13 deletions(-) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index 6e4ad4813..06362ecbb 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -95,7 +95,6 @@ def _deploy_processing_worker(self, processor: WorkerConfig, host: HostData, processor_name=processor.name, queue_url=rabbitmq_url, database_url=mongodb_url, - bin_dir=host.config.binpath, ) host.pids_native.append(pid) else: @@ -267,7 +266,7 @@ def kill_processing_worker(self, host: HostData) -> None: host.pids_docker = [] def start_native_processor(self, client: SSHClient, processor_name: str, queue_url: str, - database_url: str, bin_dir: Optional[str] = None) -> str: + database_url: str) -> str: """ start a processor natively on a host via ssh Args: @@ -275,7 +274,6 @@ def start_native_processor(self, client: SSHClient, processor_name: str, queue_u processor_name: name of processor to run queue_url: url to rabbitmq database_url: url to database - bin_dir (optional): path to where processor executables can be found Returns: str: pid of running process @@ -284,11 +282,7 @@ def start_native_processor(self, client: SSHClient, processor_name: str, queue_u self.log.info(f'Starting native processor: {processor_name}') channel = client.invoke_shell() stdin, stdout = channel.makefile('wb'), channel.makefile('rb') - if bin_dir: - path = Path(bin_dir) / processor_name - else: - path = processor_name - cmd = f'{path} --database {database_url} --queue {queue_url}' + cmd = f'{processor_name} --database {database_url} --queue {queue_url}' # the only way (I could find) to make it work to start a process in the background and # return early is this construction. The pid of the last started background process is # printed with `echo $!` but it is printed inbetween other output. Because of that I added diff --git a/ocrd/ocrd/network/deployment_config.py b/ocrd/ocrd/network/deployment_config.py index b7206ac89..f6ba4961a 100644 --- a/ocrd/ocrd/network/deployment_config.py +++ b/ocrd/ocrd/network/deployment_config.py @@ -34,8 +34,6 @@ def __init__(self, config: dict) -> None: self.username = config['username'] self.password = config.get('password', None) self.keypath = config.get('path_to_privkey', None) - # TODO: this is only for testing. Remove here and from config.schema.yml after test/development-phase - self.binpath = config.get('path_to_bin_dir', None) self.processors = [] for worker in config['workers']: deploy_type = DeployType.from_str(worker['deploy_type']) diff --git a/ocrd_validators/ocrd_validators/config.schema.yml b/ocrd_validators/ocrd_validators/config.schema.yml index 0db5b85fd..d28b63a3d 100644 --- a/ocrd_validators/ocrd_validators/config.schema.yml +++ b/ocrd_validators/ocrd_validators/config.schema.yml @@ -74,9 +74,6 @@ properties: path_to_privkey: description: Path to private key file type: string - path_to_bin_dir: - description: TODO - remove again after testing/development phase - type: string workers: description: List of workers which will be deployed type: array From 3135a4b57f6c0efe8e2ef4173a93fadae5c08263 Mon Sep 17 00:00:00 2001 From: joschrew Date: Tue, 21 Feb 2023 10:33:24 +0100 Subject: [PATCH 134/226] Rearrange docker and ssh client creation --- ocrd/ocrd/network/deployer.py | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index 06362ecbb..9a51dadc4 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -59,34 +59,38 @@ def kill_all(self) -> None: def deploy_hosts(self, rabbitmq_url: str, mongodb_url: str) -> None: for host in self.hosts: self.log.debug(f'Deploying processing workers on host: {host.config.address}') + + if (any(p.deploy_type == DeployType.native for p in host.config.processors) + and not host.ssh_client): + host.ssh_client = create_ssh_client( + host.config.address, + host.config.username, + host.config.password, + host.config.keypath + ) + if (any(p.deploy_type == DeployType.docker for p in host.config.processors) + and not host.docker_client): + host.ssh_client = create_ssh_client( + host.config.address, + host.config.username, + host.config.password, + host.config.keypath + ) + for processor in host.config.processors: self._deploy_processing_worker(processor, host, rabbitmq_url, mongodb_url) + if host.ssh_client: host.ssh_client.close() if host.docker_client: host.docker_client.close() - # TODO: Creating connections if missing should probably occur when deploying hosts not when - # deploying processing workers. The deploy_type checks and opening connections creates - # duplicate code. def _deploy_processing_worker(self, processor: WorkerConfig, host: HostData, rabbitmq_url: str, mongodb_url: str) -> None: self.log.debug(f'deploy \'{processor.deploy_type}\' processor: \'{processor}\' on' f'\'{host.config.address}\'') - # TODO: The check for available ssh or docker connections should probably happen inside - # `deploy_hosts` - if processor.deploy_type == DeployType.native: - if not host.ssh_client: - host.ssh_client = create_ssh_client(host.config.address, host.config.username, - host.config.password, host.config.keypath) - else: - assert processor.deploy_type == DeployType.docker - if not host.docker_client: - host.docker_client = create_docker_client(host.config.address, host.config.username, - host.config.password, host.config.keypath) - for _ in range(processor.count): if processor.deploy_type == DeployType.native: assert host.ssh_client # to satisfy mypy From c00dc0460d7cb194cefdf39972fe59dc5d5c4604 Mon Sep 17 00:00:00 2001 From: joschrew Date: Wed, 22 Feb 2023 11:01:41 +0100 Subject: [PATCH 135/226] Fix copy paste mistake from last commit --- ocrd/ocrd/network/deployer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index 9a51dadc4..1c37806d3 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -70,7 +70,7 @@ def deploy_hosts(self, rabbitmq_url: str, mongodb_url: str) -> None: ) if (any(p.deploy_type == DeployType.docker for p in host.config.processors) and not host.docker_client): - host.ssh_client = create_ssh_client( + host.docker_client = create_docker_client( host.config.address, host.config.username, host.config.password, From bd3a6c2d29a1ab56066c24c501c491ad21d940fa Mon Sep 17 00:00:00 2001 From: joschrew Date: Wed, 22 Feb 2023 14:15:45 +0100 Subject: [PATCH 136/226] Add workaround for rabbitmq startup on remote host --- ocrd/ocrd/network/deployer.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index 1c37806d3..c7b5384c3 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -139,7 +139,12 @@ def deploy_rabbitmq(self, image: str = 'rabbitmq:3-management', detach: bool = T 15672: 15672, 25672: 25672 } - local_defs_path = Path(__file__).parent.resolve() / 'rabbitmq_utils' / 'definitions.json' + defs_path = self.copy_definitions_to_host( + self.config.queue.address, + self.config.queue.username, + self.config.queue.password, + self.config.queue.keypath + ) container_defs_path = "/etc/rabbitmq/definitions.json" res = client.containers.run( image=image, @@ -152,7 +157,7 @@ def deploy_rabbitmq(self, image: str = 'rabbitmq:3-management', detach: bool = T ('RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS=' f'-rabbitmq_management load_definitions "{container_defs_path}"'), ], - volumes={local_defs_path: {'bind': container_defs_path, 'mode': 'ro'}} + volumes={defs_path: {'bind': container_defs_path, 'mode': 'ro'}} ) assert res and res.id, \ f'Failed to start RabbitMQ docker container on host: {self.config.mongo.address}' @@ -172,6 +177,24 @@ def deploy_rabbitmq(self, image: str = 'rabbitmq:3-management', detach: bool = T self.log.info(f'The RabbitMQ server was deployed on url: {rabbitmq_url}') return rabbitmq_url + def copy_definitions_to_host(self, address: str, username: str, password: str, + keypath: str) -> str: + """ Copy definitions.json to rabbitmq-host + + The rabbitmq is deployed in a container on another host. On this host the definitions.json + must be available to bind-mount it (make it available) to the container. + TODO: to me this looks strange. Maybe there is another possibility to get the definitions + into the rabbitmq container, but for now this is the easiest solution to the problem. + """ + ssh_client = create_ssh_client(address, username, password, keypath) + sftp = ssh_client.open_sftp() + localpath = Path(__file__).parent.resolve() / 'rabbitmq_utils' / 'definitions.json' + remotepath = "/tmp/ocr-d-processing-server-definitions.json" + sftp.put(str(localpath), remotepath) + sftp.close() + ssh_client.close() + return remotepath + def wait_for_rabbitmq_availability(self, host: str, port: str, vhost: str, username: str, password: str) -> None: max_waiting_steps = 15 From a1d5611fda16c8b676054d33c77b713641eb465f Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Thu, 23 Feb 2023 17:03:30 +0100 Subject: [PATCH 137/226] Remove the unnecessary rabbitmq.conf file --- ocrd/ocrd/network/rabbitmq_utils/rabbitmq.conf | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100755 ocrd/ocrd/network/rabbitmq_utils/rabbitmq.conf diff --git a/ocrd/ocrd/network/rabbitmq_utils/rabbitmq.conf b/ocrd/ocrd/network/rabbitmq_utils/rabbitmq.conf deleted file mode 100755 index 2cb3a539b..000000000 --- a/ocrd/ocrd/network/rabbitmq_utils/rabbitmq.conf +++ /dev/null @@ -1,14 +0,0 @@ -default_user = guest -default_pass = guest - -loopback_users.guest = true - -listeners.tcp.default = 5672 -management.tcp.port = 15672 -management.load_definitions = /etc/rabbitmq/definitions.json - -ssl_options.verify = verify_peer -ssl_options.fail_if_no_peer_cert = false - -# 60 minutes in milliseconds -consumer_timeout = 3600000 From de492b8773cd0c05fef2c9f459fbf39cd9a46f90 Mon Sep 17 00:00:00 2001 From: joschrew Date: Fri, 24 Feb 2023 09:50:41 +0100 Subject: [PATCH 138/226] Rename table for workspace in mongo --- ocrd/ocrd/network/models/workspace.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ocrd/ocrd/network/models/workspace.py b/ocrd/ocrd/network/models/workspace.py index 300b4ef4f..ce0e91a29 100644 --- a/ocrd/ocrd/network/models/workspace.py +++ b/ocrd/ocrd/network/models/workspace.py @@ -24,3 +24,6 @@ class Workspace(Document): ocrd_mets: Optional[str] bag_info_adds: Optional[dict] deleted: bool = False + + class Settings: + name = "workspace" From 6c09711a7841ea659024a9e201141d6408ddcaad Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Fri, 24 Feb 2023 13:56:38 +0100 Subject: [PATCH 139/226] Use config credentials for server and workers --- ocrd/ocrd/cli/processing_server.py | 3 +- ocrd/ocrd/cli/processing_worker.py | 5 ++-- ocrd/ocrd/network/deployer.py | 6 ++-- ocrd/ocrd/network/helpers.py | 40 +++++++++++++++----------- ocrd/ocrd/network/processing_server.py | 15 ++++++---- ocrd/ocrd/network/processing_worker.py | 29 +++++++++++-------- 6 files changed, 58 insertions(+), 40 deletions(-) diff --git a/ocrd/ocrd/cli/processing_server.py b/ocrd/ocrd/cli/processing_server.py index 27b1a91c1..fdf49f112 100644 --- a/ocrd/ocrd/cli/processing_server.py +++ b/ocrd/ocrd/cli/processing_server.py @@ -11,7 +11,6 @@ import logging -# TODO: rename to processing-server @click.command('processing-server') @click.argument('path_to_config', required=True, type=click.STRING) @click.option('-a', '--address', help='Host (name/IP) and port to bind the Processing-Server to. Example: localhost:8080', required=True) @@ -28,6 +27,6 @@ def processing_server_cli(path_to_config, address: str): host, port = address.split(":") port_int = int(port) except ValueError: - raise click.UsageError('The --adddress option must have the format IP:PORT') + raise click.UsageError('The --address option must have the format IP:PORT') processing_server = ProcessingServer(path_to_config, host, port_int) processing_server.start() diff --git a/ocrd/ocrd/cli/processing_worker.py b/ocrd/ocrd/cli/processing_worker.py index 23dbac2bb..f08bdfe90 100644 --- a/ocrd/ocrd/cli/processing_worker.py +++ b/ocrd/ocrd/cli/processing_worker.py @@ -17,8 +17,9 @@ @click.command('processing-worker') @click.argument('processor_name', required=True, type=click.STRING) @click.option('-q', '--queue', - default="localhost:5672/", - help='The host, port, and virtual host of the RabbitMQ Server') + default="admin:admin@localhost:5672/", + help='The username, password, host, port, and virtual host of the RabbitMQ Server. ' + 'Format: username:password@host:port/vhost') @click.option('-d', '--database', default="mongodb://localhost:27018", help='The host and port of the MongoDB') diff --git a/ocrd/ocrd/network/deployer.py b/ocrd/ocrd/network/deployer.py index c7b5384c3..5bbb89f7f 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd/ocrd/network/deployer.py @@ -173,9 +173,9 @@ def deploy_rabbitmq(self, image: str = 'rabbitmq:3-management', detach: bool = T self.config.queue.credentials[0], self.config.queue.credentials[1]) - rabbitmq_url = f'{rmq_host}:{rmq_port}{rmq_vhost}' - self.log.info(f'The RabbitMQ server was deployed on url: {rabbitmq_url}') - return rabbitmq_url + rabbitmq_hostinfo = f'{rmq_host}:{rmq_port}{rmq_vhost}' + self.log.info(f'The RabbitMQ server was deployed on host: {rabbitmq_hostinfo}') + return rabbitmq_hostinfo def copy_definitions_to_host(self, address: str, username: str, password: str, keypath: str) -> str: diff --git a/ocrd/ocrd/network/helpers.py b/ocrd/ocrd/network/helpers.py index c7a749654..d2e3ce0fc 100644 --- a/ocrd/ocrd/network/helpers.py +++ b/ocrd/ocrd/network/helpers.py @@ -1,7 +1,6 @@ from os import environ from os.path import join, exists from re import split -from typing import Tuple def verify_database_url(mongodb_address: str) -> str: @@ -21,21 +20,30 @@ def verify_database_url(mongodb_address: str) -> str: return mongodb_url -def verify_and_parse_rabbitmq_addr(rabbitmq_address: str) -> Tuple[str, int, str]: - elements = split(pattern=r':|/', string=rabbitmq_address) - if len(elements) == 3: - rmq_host = elements[0] - rmq_port = int(elements[1]) - rmq_vhost = f'/{elements[2]}' - return rmq_host, rmq_port, rmq_vhost - - if len(elements) == 2: - rmq_host = elements[0] - rmq_port = int(elements[1]) - rmq_vhost = '/' # The default global vhost - return rmq_host, rmq_port, rmq_vhost - - raise ValueError('The RabbitMQ address is in wrong format. Expected format: {host}:{port}/{vhost}') +def verify_and_parse_rabbitmq_addr(rabbitmq_address: str) -> dict: + parsed_data = {} + elements = rabbitmq_address.split('@') + if len(elements) != 2: + raise ValueError('The RabbitMQ address is in wrong format. Expected format: username:password@host:port/vhost') + + credentials = elements[0].split(':') + if len(credentials) != 2: + raise ValueError( + 'The RabbitMQ credentials are in wrong format. Expected format: username:password@host:port/vhost') + + parsed_data['username'] = credentials[0] + parsed_data['password'] = credentials[1] + + host_info = split(pattern=r':|/', string=elements[1]) + if len(host_info) != 3 and len(host_info) != 2: + raise ValueError( + 'The RabbitMQ host info is in wrong format. Expected format: username:password@host:port/vhost') + + parsed_data['host'] = host_info[0] + parsed_data['port'] = int(host_info[1]) + # The default global vhost is / + parsed_data['vhost'] = '/' if len(host_info) == 2 else f'/{host_info[2]}' + return parsed_data def get_workspaces_dir() -> str: diff --git a/ocrd/ocrd/network/processing_server.py b/ocrd/ocrd/network/processing_server.py index 78aa7a7d8..db3fcd727 100644 --- a/ocrd/ocrd/network/processing_server.py +++ b/ocrd/ocrd/network/processing_server.py @@ -56,6 +56,8 @@ def __init__(self, config_path: str, host: str, port: int) -> None: self.rmq_host = self.config.queue.address self.rmq_port = self.config.queue.port self.rmq_vhost = '/' + self.rmq_username = self.config.queue.credentials[0] + self.rmq_password = self.config.queue.credentials[1] # Gets assigned when `connect_publisher` is called on the working object self.rmq_publisher = None @@ -125,7 +127,9 @@ def start(self) -> None: """ deploy agents (db, queue, workers) and start the processing server with uvicorn """ try: - rabbitmq_url = self.deployer.deploy_rabbitmq() + rabbitmq_hostinfo = self.deployer.deploy_rabbitmq() + # Assign the credentials to the rabbitmq url parameter + rabbitmq_url = f'{self.rmq_username}:{self.rmq_password}@{rabbitmq_hostinfo}' self.mongodb_url = self.deployer.deploy_mongodb() @@ -169,8 +173,7 @@ async def on_shutdown(self) -> None: async def stop_deployed_agents(self) -> None: self.deployer.kill_all() - def connect_publisher(self, username: str = 'default-publisher', - password: str = 'default-publisher', enable_acks: bool = True) -> None: + def connect_publisher(self, enable_acks: bool = True) -> None: self.log.info(f'Connecting RMQPublisher to RabbitMQ server: ' f'{self.rmq_host}:{self.rmq_port}{self.rmq_vhost}') self.rmq_publisher = RMQPublisher( @@ -178,9 +181,11 @@ def connect_publisher(self, username: str = 'default-publisher', port=self.rmq_port, vhost=self.rmq_vhost ) + self.log.debug(f'RMQPublisher authenticates with username: ' + f'{self.rmq_username}, password: {self.rmq_password}') self.rmq_publisher.authenticate_and_connect( - username=username, - password=password + username=self.rmq_username, + password=self.rmq_password ) if enable_acks: self.rmq_publisher.enable_delivery_confirmations() diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd/ocrd/network/processing_worker.py index 7fd3ca01f..77fc0b255 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd/ocrd/network/processing_worker.py @@ -49,7 +49,13 @@ def __init__(self, rabbitmq_addr, mongodb_addr, processor_name, ocrd_tool: dict, try: self.db_url = verify_database_url(mongodb_addr) self.log.debug(f'Verified MongoDB URL: {self.db_url}') - self.rmq_host, self.rmq_port, self.rmq_vhost = verify_and_parse_rabbitmq_addr(rabbitmq_addr) + rmq_data = verify_and_parse_rabbitmq_addr(rabbitmq_addr) + self.rmq_username = rmq_data["username"] + self.rmq_password = rmq_data["password"] + self.rmq_host = rmq_data["host"] + self.rmq_port = rmq_data["port"] + self.rmq_vhost = rmq_data["vhost"] + self.log.debug(f'Verified RabbitMQ Credentials: {self.rmq_username}:{self.rmq_password}') self.log.debug(f'Verified RabbitMQ Server URL: {self.rmq_host}:{self.rmq_port}{self.rmq_vhost}') except ValueError as e: raise ValueError(e) @@ -72,8 +78,7 @@ def __init__(self, rabbitmq_addr, mongodb_addr, processor_name, ocrd_tool: dict, # Used to publish OcrdResultMessage type message to the queue with name {processor_name}-result self.rmq_publisher = None - def connect_consumer(self, username: str = 'default-consumer', - password: str = 'default-consumer') -> None: + def connect_consumer(self) -> None: self.log.info(f'Connecting RMQConsumer to RabbitMQ server: ' f'{self.rmq_host}:{self.rmq_port}{self.rmq_vhost}') self.rmq_consumer = RMQConsumer( @@ -82,15 +87,14 @@ def connect_consumer(self, username: str = 'default-consumer', vhost=self.rmq_vhost ) self.log.debug(f'RMQConsumer authenticates with username: ' - f'{username}, password: {password}') + f'{self.rmq_username}, password: {self.rmq_password}') self.rmq_consumer.authenticate_and_connect( - username=username, - password=password + username=self.rmq_username, + password=self.rmq_password ) self.log.info(f'Successfully connected RMQConsumer.') - def connect_publisher(self, username: str = 'default-publisher', - password: str = 'default-publisher', enable_acks: bool = True) -> None: + def connect_publisher(self, enable_acks: bool = True) -> None: self.log.info(f'Connecting RMQPublisher to RabbitMQ server: ' f'{self.rmq_host}:{self.rmq_port}{self.rmq_vhost}') self.rmq_publisher = RMQPublisher( @@ -99,13 +103,14 @@ def connect_publisher(self, username: str = 'default-publisher', vhost=self.rmq_vhost ) self.log.debug(f'RMQPublisher authenticates with username: ' - f'{username}, password: {password}') - self.rmq_publisher.authenticate_and_connect(username=username, password=password) + f'{self.rmq_username}, password: {self.rmq_password}') + self.rmq_publisher.authenticate_and_connect( + username=self.rmq_username, + password=self.rmq_password + ) if enable_acks: self.rmq_publisher.enable_delivery_confirmations() self.log.info('Delivery confirmations are enabled') - else: - self.log.info('Delivery confirmations are disabled') self.log.info('Successfully connected RMQPublisher.') # Define what happens every time a message is consumed From b264652488d60709678fbeb891233b8353d26138 Mon Sep 17 00:00:00 2001 From: joschrew Date: Fri, 24 Feb 2023 14:40:55 +0100 Subject: [PATCH 140/226] Create ocrd_network package --- Dockerfile | 1 + Makefile | 2 +- README.md | 7 +++++ ocrd/ocrd/cli/processing_server.py | 2 +- ocrd/ocrd/cli/processing_worker.py | 2 +- ocrd/ocrd/decorators/__init__.py | 2 +- ocrd/requirements.txt | 7 ----- ocrd/setup.py | 1 + ocrd_network/README.md | 5 ++++ .../ocrd_network}/__init__.py | 3 --- .../ocrd_network}/database.py | 4 +-- .../ocrd_network}/deployer.py | 6 ++--- .../ocrd_network}/deployment_config.py | 2 +- .../ocrd_network}/deployment_utils.py | 2 +- .../ocrd_network}/helpers.py | 0 .../ocrd_network}/models/__init__.py | 0 .../ocrd_network}/models/job.py | 0 .../ocrd_network}/models/ocrd_tool.py | 0 .../ocrd_network}/models/workspace.py | 0 .../ocrd_network}/processing_server.py | 12 ++++----- .../ocrd_network}/processing_worker.py | 6 ++--- .../ocrd_network}/rabbitmq_utils/__init__.py | 0 .../ocrd_network}/rabbitmq_utils/connector.py | 2 +- .../ocrd_network}/rabbitmq_utils/constants.py | 0 .../ocrd_network}/rabbitmq_utils/consumer.py | 4 +-- .../rabbitmq_utils/definitions.json | 0 .../rabbitmq_utils/ocrd_messages.py | 2 +- .../ocrd_network}/rabbitmq_utils/publisher.py | 4 +-- .../ocrd_network}/web_api/__init__.py | 0 ocrd_network/requirements.txt | 6 +++++ ocrd_network/setup.py | 27 +++++++++++++++++++ tox.ini | 1 + 32 files changed, 74 insertions(+), 36 deletions(-) create mode 100644 ocrd_network/README.md rename {ocrd/ocrd/network => ocrd_network/ocrd_network}/__init__.py (86%) rename {ocrd/ocrd/network => ocrd_network/ocrd_network}/database.py (76%) rename {ocrd/ocrd/network => ocrd_network/ocrd_network}/deployer.py (99%) rename {ocrd/ocrd/network => ocrd_network/ocrd_network}/deployment_config.py (98%) rename {ocrd/ocrd/network => ocrd_network/ocrd_network}/deployment_utils.py (99%) rename {ocrd/ocrd/network => ocrd_network/ocrd_network}/helpers.py (100%) rename {ocrd/ocrd/network => ocrd_network/ocrd_network}/models/__init__.py (100%) rename {ocrd/ocrd/network => ocrd_network/ocrd_network}/models/job.py (100%) rename {ocrd/ocrd/network => ocrd_network/ocrd_network}/models/ocrd_tool.py (100%) rename {ocrd/ocrd/network => ocrd_network/ocrd_network}/models/workspace.py (100%) rename {ocrd/ocrd/network => ocrd_network/ocrd_network}/processing_server.py (97%) rename {ocrd/ocrd/network => ocrd_network/ocrd_network}/processing_worker.py (99%) rename {ocrd/ocrd/network => ocrd_network/ocrd_network}/rabbitmq_utils/__init__.py (100%) rename {ocrd/ocrd/network => ocrd_network/ocrd_network}/rabbitmq_utils/connector.py (99%) rename {ocrd/ocrd/network => ocrd_network/ocrd_network}/rabbitmq_utils/constants.py (100%) rename {ocrd/ocrd/network => ocrd_network/ocrd_network}/rabbitmq_utils/consumer.py (96%) rename {ocrd/ocrd/network => ocrd_network/ocrd_network}/rabbitmq_utils/definitions.json (100%) rename {ocrd/ocrd/network => ocrd_network/ocrd_network}/rabbitmq_utils/ocrd_messages.py (99%) rename {ocrd/ocrd/network => ocrd_network/ocrd_network}/rabbitmq_utils/publisher.py (97%) rename {ocrd/ocrd/network => ocrd_network/ocrd_network}/web_api/__init__.py (100%) create mode 100644 ocrd_network/requirements.txt create mode 100644 ocrd_network/setup.py diff --git a/Dockerfile b/Dockerfile index c795242b0..fc81fe6f4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,7 @@ COPY ocrd_models ./ocrd_models COPY ocrd_utils ./ocrd_utils RUN mv ./ocrd_utils/ocrd_logging.conf /etc COPY ocrd_validators/ ./ocrd_validators +COPY ocrd_network/ ./ocrd_network COPY Makefile . COPY README.md . COPY LICENSE . diff --git a/Makefile b/Makefile index 970dc7fa7..cac38d184 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ TESTDIR = tests SPHINX_APIDOC = -BUILD_ORDER = ocrd_utils ocrd_models ocrd_modelfactory ocrd_validators ocrd +BUILD_ORDER = ocrd_utils ocrd_models ocrd_modelfactory ocrd_validators ocrd_network ocrd FIND_VERSION = grep version= ocrd_utils/setup.py|grep -Po "([0-9ab]+\.?)+" diff --git a/README.md b/README.md index a68519bbc..c28dac474 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ * [ocrd_models](#ocrd_models) * [ocrd_modelfactory](#ocrd_modelfactory) * [ocrd_validators](#ocrd_validators) + * [ocrd_network](#ocrd_network) * [ocrd](#ocrd) * [bash library](#bash-library) * [bashlib API](#bashlib-api) @@ -122,6 +123,12 @@ Schemas and routines for validating BagIt, `ocrd-tool.json`, workspaces, METS, p See [README for `ocrd_validators`](./ocrd_validators/README.md) for further information. +### ocrd_network + +Tools for offering (web-)services with OCR-D + +See [README for `ocrd_network`](./ocrd_network/README.md) for further information. + ### ocrd Depends on all of the above, also contains decorators and classes for creating OCR-D processors and CLIs. diff --git a/ocrd/ocrd/cli/processing_server.py b/ocrd/ocrd/cli/processing_server.py index fdf49f112..768dc0d82 100644 --- a/ocrd/ocrd/cli/processing_server.py +++ b/ocrd/ocrd/cli/processing_server.py @@ -7,7 +7,7 @@ """ import click from ocrd_utils import initLogging -from ocrd.network import ProcessingServer +from ocrd_network import ProcessingServer import logging diff --git a/ocrd/ocrd/cli/processing_worker.py b/ocrd/ocrd/cli/processing_worker.py index f08bdfe90..d228eb3b5 100644 --- a/ocrd/ocrd/cli/processing_worker.py +++ b/ocrd/ocrd/cli/processing_worker.py @@ -11,7 +11,7 @@ initLogging, get_ocrd_tool_json ) -from ocrd.network.processing_worker import ProcessingWorker +from ocrd_network.processing_worker import ProcessingWorker @click.command('processing-worker') diff --git a/ocrd/ocrd/decorators/__init__.py b/ocrd/ocrd/decorators/__init__.py index 17cfc77bc..cce6ae93c 100644 --- a/ocrd/ocrd/decorators/__init__.py +++ b/ocrd/ocrd/decorators/__init__.py @@ -15,7 +15,7 @@ from ocrd_utils import getLogger, initLogging, parse_json_string_with_comments from ocrd_validators import WorkspaceValidator -from ocrd.network import ProcessingWorker +from ocrd_network import ProcessingWorker from ..resolver import Resolver from ..processor.base import run_processor diff --git a/ocrd/requirements.txt b/ocrd/requirements.txt index 232c56d1f..e2caacb0e 100644 --- a/ocrd/requirements.txt +++ b/ocrd/requirements.txt @@ -11,11 +11,4 @@ Deprecated == 1.2.0 memory-profiler >= 0.58.0 sparklines >= 0.4.2 python-magic -uvicorn>=0.17.6 -fastapi>=0.78.0 -docker -paramiko -frozendict~=2.3.4 -pika>=1.2.0 -beanie~=1.7 tensorflow diff --git a/ocrd/setup.py b/ocrd/setup.py index 0269893e2..67536620f 100644 --- a/ocrd/setup.py +++ b/ocrd/setup.py @@ -8,6 +8,7 @@ install_requires.append('ocrd_models == %s' % VERSION) install_requires.append('ocrd_modelfactory == %s' % VERSION) install_requires.append('ocrd_validators == %s' % VERSION) +install_requires.append('ocrd_network == %s' % VERSION) setup( name='ocrd', diff --git a/ocrd_network/README.md b/ocrd_network/README.md new file mode 100644 index 000000000..ac2cf41da --- /dev/null +++ b/ocrd_network/README.md @@ -0,0 +1,5 @@ +# ocrd_network + +> OCR-D framework - web API + +See also: https://github.com/OCR-D/core diff --git a/ocrd/ocrd/network/__init__.py b/ocrd_network/ocrd_network/__init__.py similarity index 86% rename from ocrd/ocrd/network/__init__.py rename to ocrd_network/ocrd_network/__init__.py index 6b7e368ef..aef970f88 100644 --- a/ocrd/ocrd/network/__init__.py +++ b/ocrd_network/ocrd_network/__init__.py @@ -13,8 +13,5 @@ # Note: The Mets Server is still not placed on the architecture diagram and probably won't be a part of # the network package. The reason, Mets Server is tightly coupled with the `OcrdWorkspace`. - -# This package, currently, is under the `core/ocrd` package. -# It could also be a separate package on its own under `core` with the name `ocrd_network`. from .processing_server import ProcessingServer from .processing_worker import ProcessingWorker diff --git a/ocrd/ocrd/network/database.py b/ocrd_network/ocrd_network/database.py similarity index 76% rename from ocrd/ocrd/network/database.py rename to ocrd_network/ocrd_network/database.py index 33ed8ef2c..66e4722e7 100644 --- a/ocrd/ocrd/network/database.py +++ b/ocrd_network/ocrd_network/database.py @@ -1,8 +1,8 @@ from beanie import init_beanie from motor.motor_asyncio import AsyncIOMotorClient -from ocrd.network.models.job import Job -from ocrd.network.models.workspace import Workspace +from ocrd_network.models.job import Job +from ocrd_network.models.workspace import Workspace async def initiate_database(db_url: str): diff --git a/ocrd/ocrd/network/deployer.py b/ocrd_network/ocrd_network/deployer.py similarity index 99% rename from ocrd/ocrd/network/deployer.py rename to ocrd_network/ocrd_network/deployer.py index 5bbb89f7f..0bc0ef25f 100644 --- a/ocrd/ocrd/network/deployer.py +++ b/ocrd_network/ocrd_network/deployer.py @@ -16,15 +16,15 @@ from ocrd_utils import getLogger -from ocrd.network.deployment_config import * -from ocrd.network.deployment_utils import ( +from ocrd_network.deployment_config import * +from ocrd_network.deployment_utils import ( create_docker_client, create_ssh_client, CustomDockerClient, DeployType, HostData, ) -from ocrd.network.rabbitmq_utils import RMQPublisher +from ocrd_network.rabbitmq_utils import RMQPublisher class Deployer: diff --git a/ocrd/ocrd/network/deployment_config.py b/ocrd_network/ocrd_network/deployment_config.py similarity index 98% rename from ocrd/ocrd/network/deployment_config.py rename to ocrd_network/ocrd_network/deployment_config.py index f6ba4961a..8cedf4644 100644 --- a/ocrd/ocrd/network/deployment_config.py +++ b/ocrd_network/ocrd_network/deployment_config.py @@ -1,6 +1,6 @@ from typing import Dict -from ocrd.network.deployment_utils import DeployType +from ocrd_network.deployment_utils import DeployType __all__ = [ 'ProcessingServerConfig', diff --git a/ocrd/ocrd/network/deployment_utils.py b/ocrd_network/ocrd_network/deployment_utils.py similarity index 99% rename from ocrd/ocrd/network/deployment_utils.py rename to ocrd_network/ocrd_network/deployment_utils.py index 876cdbc16..7b5c0416e 100644 --- a/ocrd/ocrd/network/deployment_utils.py +++ b/ocrd_network/ocrd_network/deployment_utils.py @@ -7,7 +7,7 @@ from paramiko import AutoAddPolicy, SSHClient from ocrd_utils import getLogger -from ocrd.network.deployment_config import * +from ocrd_network.deployment_config import * __all__ = [ 'DeployType' diff --git a/ocrd/ocrd/network/helpers.py b/ocrd_network/ocrd_network/helpers.py similarity index 100% rename from ocrd/ocrd/network/helpers.py rename to ocrd_network/ocrd_network/helpers.py diff --git a/ocrd/ocrd/network/models/__init__.py b/ocrd_network/ocrd_network/models/__init__.py similarity index 100% rename from ocrd/ocrd/network/models/__init__.py rename to ocrd_network/ocrd_network/models/__init__.py diff --git a/ocrd/ocrd/network/models/job.py b/ocrd_network/ocrd_network/models/job.py similarity index 100% rename from ocrd/ocrd/network/models/job.py rename to ocrd_network/ocrd_network/models/job.py diff --git a/ocrd/ocrd/network/models/ocrd_tool.py b/ocrd_network/ocrd_network/models/ocrd_tool.py similarity index 100% rename from ocrd/ocrd/network/models/ocrd_tool.py rename to ocrd_network/ocrd_network/models/ocrd_tool.py diff --git a/ocrd/ocrd/network/models/workspace.py b/ocrd_network/ocrd_network/models/workspace.py similarity index 100% rename from ocrd/ocrd/network/models/workspace.py rename to ocrd_network/ocrd_network/models/workspace.py diff --git a/ocrd/ocrd/network/processing_server.py b/ocrd_network/ocrd_network/processing_server.py similarity index 97% rename from ocrd/ocrd/network/processing_server.py rename to ocrd_network/ocrd_network/processing_server.py index db3fcd727..52310978f 100644 --- a/ocrd/ocrd/network/processing_server.py +++ b/ocrd_network/ocrd_network/processing_server.py @@ -16,20 +16,20 @@ ParameterValidator, ProcessingServerValidator ) -from ocrd.network.database import initiate_database -from ocrd.network.deployer import Deployer -from ocrd.network.deployment_config import ProcessingServerConfig -from ocrd.network.rabbitmq_utils import ( +from ocrd_network.database import initiate_database +from ocrd_network.deployer import Deployer +from ocrd_network.deployment_config import ProcessingServerConfig +from ocrd_network.rabbitmq_utils import ( RMQPublisher, OcrdProcessingMessage ) -from ocrd.network.models.job import ( +from ocrd_network.models.job import ( Job, JobInput, JobOutput, StateEnum ) -from ocrd.network.models.workspace import Workspace +from ocrd_network.models.workspace import Workspace class ProcessingServer(FastAPI): diff --git a/ocrd/ocrd/network/processing_worker.py b/ocrd_network/ocrd_network/processing_worker.py similarity index 99% rename from ocrd/ocrd/network/processing_worker.py rename to ocrd_network/ocrd_network/processing_worker.py index 77fc0b255..fb922ec66 100644 --- a/ocrd/ocrd/network/processing_worker.py +++ b/ocrd_network/ocrd_network/processing_worker.py @@ -23,12 +23,12 @@ run_cli, run_processor ) -from ocrd.network.helpers import ( +from ocrd_network.helpers import ( verify_database_url, verify_and_parse_rabbitmq_addr ) -from ocrd.network.models.job import StateEnum -from ocrd.network.rabbitmq_utils import ( +from ocrd_network.models.job import StateEnum +from ocrd_network.rabbitmq_utils import ( OcrdProcessingMessage, OcrdResultMessage, RMQConsumer, diff --git a/ocrd/ocrd/network/rabbitmq_utils/__init__.py b/ocrd_network/ocrd_network/rabbitmq_utils/__init__.py similarity index 100% rename from ocrd/ocrd/network/rabbitmq_utils/__init__.py rename to ocrd_network/ocrd_network/rabbitmq_utils/__init__.py diff --git a/ocrd/ocrd/network/rabbitmq_utils/connector.py b/ocrd_network/ocrd_network/rabbitmq_utils/connector.py similarity index 99% rename from ocrd/ocrd/network/rabbitmq_utils/connector.py rename to ocrd_network/ocrd_network/rabbitmq_utils/connector.py index 6f6dbfb47..288f8a203 100644 --- a/ocrd/ocrd/network/rabbitmq_utils/connector.py +++ b/ocrd_network/ocrd_network/rabbitmq_utils/connector.py @@ -13,7 +13,7 @@ ) from pika.adapters.blocking_connection import BlockingChannel -from ocrd.network.rabbitmq_utils.constants import ( +from ocrd_network.rabbitmq_utils.constants import ( DEFAULT_EXCHANGER_NAME, DEFAULT_EXCHANGER_TYPE, DEFAULT_QUEUE, diff --git a/ocrd/ocrd/network/rabbitmq_utils/constants.py b/ocrd_network/ocrd_network/rabbitmq_utils/constants.py similarity index 100% rename from ocrd/ocrd/network/rabbitmq_utils/constants.py rename to ocrd_network/ocrd_network/rabbitmq_utils/constants.py diff --git a/ocrd/ocrd/network/rabbitmq_utils/consumer.py b/ocrd_network/ocrd_network/rabbitmq_utils/consumer.py similarity index 96% rename from ocrd/ocrd/network/rabbitmq_utils/consumer.py rename to ocrd_network/ocrd_network/rabbitmq_utils/consumer.py index bcc034748..a235475b2 100644 --- a/ocrd/ocrd/network/rabbitmq_utils/consumer.py +++ b/ocrd_network/ocrd_network/rabbitmq_utils/consumer.py @@ -9,14 +9,14 @@ from pika import PlainCredentials -from ocrd.network.rabbitmq_utils.constants import ( +from ocrd_network.rabbitmq_utils.constants import ( DEFAULT_QUEUE, LOG_LEVEL, RABBIT_MQ_HOST as HOST, RABBIT_MQ_PORT as PORT, RABBIT_MQ_VHOST as VHOST ) -from ocrd.network.rabbitmq_utils.connector import RMQConnector +from ocrd_network.rabbitmq_utils.connector import RMQConnector class RMQConsumer(RMQConnector): diff --git a/ocrd/ocrd/network/rabbitmq_utils/definitions.json b/ocrd_network/ocrd_network/rabbitmq_utils/definitions.json similarity index 100% rename from ocrd/ocrd/network/rabbitmq_utils/definitions.json rename to ocrd_network/ocrd_network/rabbitmq_utils/definitions.json diff --git a/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py b/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py similarity index 99% rename from ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py rename to ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py index 23c87856c..a270dbecb 100644 --- a/ocrd/ocrd/network/rabbitmq_utils/ocrd_messages.py +++ b/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py @@ -3,7 +3,7 @@ from datetime import datetime from typing import Any, Dict, List, Optional import yaml -from ocrd.network.models.job import Job +from ocrd_network.models.job import Job class OcrdProcessingMessage: diff --git a/ocrd/ocrd/network/rabbitmq_utils/publisher.py b/ocrd_network/ocrd_network/rabbitmq_utils/publisher.py similarity index 97% rename from ocrd/ocrd/network/rabbitmq_utils/publisher.py rename to ocrd_network/ocrd_network/rabbitmq_utils/publisher.py index e30104700..806cdb426 100644 --- a/ocrd/ocrd/network/rabbitmq_utils/publisher.py +++ b/ocrd_network/ocrd_network/rabbitmq_utils/publisher.py @@ -12,7 +12,7 @@ PlainCredentials ) -from ocrd.network.rabbitmq_utils.constants import ( +from ocrd_network.rabbitmq_utils.constants import ( DEFAULT_EXCHANGER_NAME, DEFAULT_ROUTER, LOG_FORMAT, @@ -21,7 +21,7 @@ RABBIT_MQ_PORT as PORT, RABBIT_MQ_VHOST as VHOST ) -from ocrd.network.rabbitmq_utils.connector import RMQConnector +from ocrd_network.rabbitmq_utils.connector import RMQConnector class RMQPublisher(RMQConnector): diff --git a/ocrd/ocrd/network/web_api/__init__.py b/ocrd_network/ocrd_network/web_api/__init__.py similarity index 100% rename from ocrd/ocrd/network/web_api/__init__.py rename to ocrd_network/ocrd_network/web_api/__init__.py diff --git a/ocrd_network/requirements.txt b/ocrd_network/requirements.txt new file mode 100644 index 000000000..d11fb430f --- /dev/null +++ b/ocrd_network/requirements.txt @@ -0,0 +1,6 @@ +uvicorn>=0.17.6 +fastapi>=0.78.0 +docker +paramiko +pika>=1.2.0 +beanie~=1.7 diff --git a/ocrd_network/setup.py b/ocrd_network/setup.py new file mode 100644 index 000000000..eca56aa32 --- /dev/null +++ b/ocrd_network/setup.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +from setuptools import setup, find_packages + +from ocrd_utils import VERSION + +install_requires = open('requirements.txt').read().split('\n') +install_requires.append('ocrd_utils == %s' % VERSION) +install_requires.append('ocrd_validators == %s' % VERSION) + +setup( + name='ocrd_network', + version=VERSION, + description='OCR-D framework - web API', + long_description=open('README.md').read(), + long_description_content_type='text/markdown', + author='Konstantin Baierer', + author_email='unixprog@gmail.com', + url='https://github.com/OCR-D/core', + license='Apache License 2.0', + packages=find_packages(exclude=('tests', 'docs')), + include_package_data=True, + install_requires=install_requires, + package_data={ + '': ['*.yml', '*.xsd'] + }, + keywords=['OCR', 'OCR-D'] +) diff --git a/tox.ini b/tox.ini index da51fa85b..c085f437b 100644 --- a/tox.ini +++ b/tox.ini @@ -13,6 +13,7 @@ deps = -rocrd_models/requirements.txt -rocrd_modelfactory/requirements.txt -rocrd_validators/requirements.txt + -rocrd_network/requirements.txt -rocrd/requirements.txt commands = - make install test From 826d9a0519f1e904b1d6774373ecf63a3e1e488e Mon Sep 17 00:00:00 2001 From: joschrew Date: Fri, 24 Feb 2023 15:12:43 +0100 Subject: [PATCH 141/226] Remove usage of definitions.json with rabbitmq --- ocrd_network/ocrd_network/deployer.py | 32 +------ .../rabbitmq_utils/definitions.json | 85 ------------------- 2 files changed, 2 insertions(+), 115 deletions(-) delete mode 100755 ocrd_network/ocrd_network/rabbitmq_utils/definitions.json diff --git a/ocrd_network/ocrd_network/deployer.py b/ocrd_network/ocrd_network/deployer.py index 0bc0ef25f..1ab728347 100644 --- a/ocrd_network/ocrd_network/deployer.py +++ b/ocrd_network/ocrd_network/deployer.py @@ -139,13 +139,6 @@ def deploy_rabbitmq(self, image: str = 'rabbitmq:3-management', detach: bool = T 15672: 15672, 25672: 25672 } - defs_path = self.copy_definitions_to_host( - self.config.queue.address, - self.config.queue.username, - self.config.queue.password, - self.config.queue.keypath - ) - container_defs_path = "/etc/rabbitmq/definitions.json" res = client.containers.run( image=image, detach=detach, @@ -153,11 +146,8 @@ def deploy_rabbitmq(self, image: str = 'rabbitmq:3-management', detach: bool = T ports=ports_mapping, environment=[ f'RABBITMQ_DEFAULT_USER={self.config.queue.credentials[0]}', - f'RABBITMQ_DEFAULT_PASS={self.config.queue.credentials[1]}', - ('RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS=' - f'-rabbitmq_management load_definitions "{container_defs_path}"'), - ], - volumes={defs_path: {'bind': container_defs_path, 'mode': 'ro'}} + f'RABBITMQ_DEFAULT_PASS={self.config.queue.credentials[1]}' + ] ) assert res and res.id, \ f'Failed to start RabbitMQ docker container on host: {self.config.mongo.address}' @@ -177,24 +167,6 @@ def deploy_rabbitmq(self, image: str = 'rabbitmq:3-management', detach: bool = T self.log.info(f'The RabbitMQ server was deployed on host: {rabbitmq_hostinfo}') return rabbitmq_hostinfo - def copy_definitions_to_host(self, address: str, username: str, password: str, - keypath: str) -> str: - """ Copy definitions.json to rabbitmq-host - - The rabbitmq is deployed in a container on another host. On this host the definitions.json - must be available to bind-mount it (make it available) to the container. - TODO: to me this looks strange. Maybe there is another possibility to get the definitions - into the rabbitmq container, but for now this is the easiest solution to the problem. - """ - ssh_client = create_ssh_client(address, username, password, keypath) - sftp = ssh_client.open_sftp() - localpath = Path(__file__).parent.resolve() / 'rabbitmq_utils' / 'definitions.json' - remotepath = "/tmp/ocr-d-processing-server-definitions.json" - sftp.put(str(localpath), remotepath) - sftp.close() - ssh_client.close() - return remotepath - def wait_for_rabbitmq_availability(self, host: str, port: str, vhost: str, username: str, password: str) -> None: max_waiting_steps = 15 diff --git a/ocrd_network/ocrd_network/rabbitmq_utils/definitions.json b/ocrd_network/ocrd_network/rabbitmq_utils/definitions.json deleted file mode 100755 index 3f4df0c86..000000000 --- a/ocrd_network/ocrd_network/rabbitmq_utils/definitions.json +++ /dev/null @@ -1,85 +0,0 @@ -{ - "users": [ - { - "name": "guest", - "password": "guest", - "hashing_algorithm": "rabbit_password_hashing_sha256", - "tags": "administrator" - }, - { - "name": "admin", - "password": "admin", - "hashing_algorithm": "rabbit_password_hashing_sha256", - "tags": "administrator" - }, - { - "name": "default-consumer", - "password": "default-consumer", - "hashing_algorithm": "rabbit_password_hashing_sha256", - "tags": "administrator" - }, - { - "name": "default-publisher", - "password": "default-publisher", - "hashing_algorithm": "rabbit_password_hashing_sha256", - "tags": "administrator" - }, - { - "name": "test-session", - "password": "test-session", - "hashing_algorithm": "rabbit_password_hashing_sha256", - "tags": "administrator" - } - ], - "vhosts": [ - { - "name": "/" - }, - { - "name": "test" - } - - ], - "permissions": [ - { - "user": "guest", - "vhost": "/", - "configure": ".*", - "write": ".*", - "read": ".*" - }, - { - "user": "admin", - "vhost": "/", - "configure": ".*", - "write": ".*", - "read": ".*" - }, - { - "user": "default-consumer", - "vhost": "/", - "configure": ".*", - "write": ".*", - "read": ".*" - }, - { - "user": "default-publisher", - "vhost": "/", - "configure": ".*", - "write": ".*", - "read": ".*" - }, - { - "user": "test-session", - "vhost": "test", - "configure": ".*", - "write": ".*", - "read": ".*" - } - ], - "parameters": [], - "policies": [], - "queues": [], - "exchanges": [], - "bindings": [] -} From ad98e12e36fddc8f0c4b6171d418ac5bd2fa7f5e Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Fri, 24 Feb 2023 15:23:17 +0100 Subject: [PATCH 142/226] Fix dependency in requirements --- ocrd/requirements.txt | 2 -- ocrd_network/requirements.txt | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/ocrd/requirements.txt b/ocrd/requirements.txt index e2caacb0e..ad30bc1f8 100644 --- a/ocrd/requirements.txt +++ b/ocrd/requirements.txt @@ -10,5 +10,3 @@ pyyaml Deprecated == 1.2.0 memory-profiler >= 0.58.0 sparklines >= 0.4.2 -python-magic -tensorflow diff --git a/ocrd_network/requirements.txt b/ocrd_network/requirements.txt index d11fb430f..3a09ed71a 100644 --- a/ocrd_network/requirements.txt +++ b/ocrd_network/requirements.txt @@ -4,3 +4,4 @@ docker paramiko pika>=1.2.0 beanie~=1.7 +tensorflow From fffb7dc1dc041702739befa8df6b97ceee746bf4 Mon Sep 17 00:00:00 2001 From: joschrew Date: Mon, 27 Feb 2023 09:13:00 +0100 Subject: [PATCH 143/226] Update doc for processing server --- ocrd/ocrd/cli/processing_server.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ocrd/ocrd/cli/processing_server.py b/ocrd/ocrd/cli/processing_server.py index 768dc0d82..ba61453c8 100644 --- a/ocrd/ocrd/cli/processing_server.py +++ b/ocrd/ocrd/cli/processing_server.py @@ -16,7 +16,12 @@ @click.option('-a', '--address', help='Host (name/IP) and port to bind the Processing-Server to. Example: localhost:8080', required=True) def processing_server_cli(path_to_config, address: str): """ - Start and manage processing servers (workers) with the processing server + Start and manage processing workers with the processing server + + PATH_TO_CONFIG is a yaml file to configure the server and the workers. See + https://github.com/OCR-D/spec/pull/222/files#diff-a71bf71cbc7d9ce94fded977f7544aba4df9e7bdb8fc0cf1014e14eb67a9b273 + for further information (TODO: update path when spec is available/merged) + """ initLogging() # TODO: Remove before the release From f5e45b32f6e0849b324acc3425b43563af0ed010 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Thu, 2 Mar 2023 12:42:38 +0100 Subject: [PATCH 144/226] Improve exception message Co-authored-by: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> --- ocrd/ocrd/decorators/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ocrd/ocrd/decorators/__init__.py b/ocrd/ocrd/decorators/__init__.py index cce6ae93c..85f239825 100644 --- a/ocrd/ocrd/decorators/__init__.py +++ b/ocrd/ocrd/decorators/__init__.py @@ -59,7 +59,7 @@ def ocrd_cli_wrap_processor( sys.exit() # If either of these two is provided but not both if bool(queue) != bool(database): - raise Exception("Both queue and database addresses must be provided - not just either of them.") + raise Exception("Options --queue and --database require each other.") # If both of these are provided - start the processing worker instead of the processor - processorClass if queue and database: initLogging() From 9829b18c068aaec2aeb987892aacf7c0953b8e1c Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Thu, 2 Mar 2023 12:43:48 +0100 Subject: [PATCH 145/226] Remove parsing of stdout in ocrd decorators Co-authored-by: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> --- ocrd/ocrd/decorators/__init__.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/ocrd/ocrd/decorators/__init__.py b/ocrd/ocrd/decorators/__init__.py index 85f239825..e3830057a 100644 --- a/ocrd/ocrd/decorators/__init__.py +++ b/ocrd/ocrd/decorators/__init__.py @@ -69,12 +69,8 @@ def ocrd_cli_wrap_processor( logging.getLogger('ocrd.network').setLevel(logging.DEBUG) # Get the ocrd_tool dictionary - f_out = StringIO() - with redirect_stdout(f_out): - processorClass(workspace=None, dump_json=True) - # TODO: Verify this. There is `ocrd_tool` parameter passed as an argument. - # The following line overwrites the passed parameter. - ocrd_tool = parse_json_string_with_comments(f_out.getvalue()) + processor = processorClass(workspace=None, dump_json=True) + ocrd_tool = processor.ocrd_tool try: processing_worker = ProcessingWorker( From b73e99acdf8c1c6a2f560d67df185a84fef5c77a Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Thu, 2 Mar 2023 12:50:50 +0100 Subject: [PATCH 146/226] Update processing worker cli help Co-authored-by: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> --- ocrd/ocrd/cli/processing_worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ocrd/ocrd/cli/processing_worker.py b/ocrd/ocrd/cli/processing_worker.py index d228eb3b5..8068a0ce9 100644 --- a/ocrd/ocrd/cli/processing_worker.py +++ b/ocrd/ocrd/cli/processing_worker.py @@ -22,7 +22,7 @@ 'Format: username:password@host:port/vhost') @click.option('-d', '--database', default="mongodb://localhost:27018", - help='The host and port of the MongoDB') + help='URL of the MongoDB service') def processing_worker_cli(processor_name: str, queue: str, database: str): """ Start a processing worker (a specific ocr-d processor) From cf7321d0a54674a681b06ec946d6e5595ae5700a Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Thu, 2 Mar 2023 13:06:15 +0100 Subject: [PATCH 147/226] Update ocrd-network package info in __init__ --- ocrd_network/ocrd_network/__init__.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/ocrd_network/ocrd_network/__init__.py b/ocrd_network/ocrd_network/__init__.py index aef970f88..97de2b9a2 100644 --- a/ocrd_network/ocrd_network/__init__.py +++ b/ocrd_network/ocrd_network/__init__.py @@ -1,13 +1,22 @@ # This network package is supposed to contain all the packages and modules to realize the network architecture: -# https://user-images.githubusercontent.com/7795705/203554094-62ce135a-b367-49ba-9960-ffe1b7d39b2c.jpg +# https://github.com/OCR-D/spec/pull/222/files#diff-8d0dae8c9277ff1003df93c5359c82a12d3f5c8452281f87781921921204d283 # For reference, currently: -# 1. The WebAPI is available here: -# https://github.com/OCR-D/ocrd-webapi-implementation -# 2. The RabbitMQ Library (i.e., utils) is available here: -# https://github.com/OCR-D/ocrd-webapi-implementation/tree/main/ocrd_webapi/rabbitmq +# 1. The WebAPI is available here: https://github.com/OCR-D/ocrd-webapi-implementation +# The ocrd-webapi-implementation repo implements the Discovery / Workflow / Workspace endpoints of the WebAPI currently. +# This Processing Server PR implements just the Processor endpoint of the WebAPI. +# Once we have this merged to core under ocrd-network, the other endpoints will be adapted to ocrd-network +# and then the ocrd-webapi-implementation repo can be archived for reference. + +# 2. The RabbitMQ Library (i.e., utils) is used as an API to abstract and +# simplify (from the view point of processing server and workers) interactions with the RabbitMQ Server. +# The library was adopted from: https://github.com/OCR-D/ocrd-webapi-implementation/tree/main/ocrd_webapi/rabbitmq + # 3. Some potentially more useful code to be adopted for the Processing Server/Worker is available here: # https://github.com/OCR-D/core/pull/884 +# Update: Should be revisited again for adopting any relevant parts (if necessary). +# Nothing relevant is under the radar for now. + # 4. The Mets Server discussion/implementation is available here: # https://github.com/OCR-D/core/pull/966 From 132b93f2c0160b4a5bcbb3fd216ac0076a7dcc7d Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Thu, 2 Mar 2023 13:09:30 +0100 Subject: [PATCH 148/226] Update ocrd_network/ocrd_network/deployer.py info Co-authored-by: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> --- ocrd_network/ocrd_network/deployer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ocrd_network/ocrd_network/deployer.py b/ocrd_network/ocrd_network/deployer.py index 1ab728347..ce1b53ee9 100644 --- a/ocrd_network/ocrd_network/deployer.py +++ b/ocrd_network/ocrd_network/deployer.py @@ -1,8 +1,8 @@ """ -Abstraction of the Deployment functionality -The ProcessingServer (currently still called Server) provides the configuration parameters to the -Deployer agent. -The Deployer agent deploys the RabbitMQ Server, MongoDB and the Processing Hosts. +Abstraction of the deployment functionality for processors. + +The Processing Server provides the configuration parameters to the Deployer agent. +The Deployer agent runs the RabbitMQ Server, MongoDB and the Processing Hosts. Each Processing Host may have several Processing Workers. Each Processing Worker is an instance of an OCR-D processor. """ From a2a7fe99f2463000f28f5e55e479a456369e83ad Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Thu, 2 Mar 2023 13:25:39 +0100 Subject: [PATCH 149/226] Change getcwd() call location --- ocrd/ocrd/processor/helpers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ocrd/ocrd/processor/helpers.py b/ocrd/ocrd/processor/helpers.py index f1e793bd0..8e15243cc 100644 --- a/ocrd/ocrd/processor/helpers.py +++ b/ocrd/ocrd/processor/helpers.py @@ -84,6 +84,8 @@ def run_processor( log = getLogger('ocrd.processor.helpers.run_processor') log.debug("Running processor %s", processorClass) + old_cwd = getcwd() + processor = get_processor( processor_class=processorClass, parameter=parameter, @@ -95,7 +97,6 @@ def run_processor( instance_caching=instance_caching ) - old_cwd = getcwd() chdir(processor.workspace.directory) ocrd_tool = processor.ocrd_tool From 22311f10166398b339c39a9d466b7730753eb340 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Thu, 2 Mar 2023 13:27:53 +0100 Subject: [PATCH 150/226] Update ocrd_network/ocrd_network/deployer.py help Co-authored-by: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> --- ocrd_network/ocrd_network/deployer.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ocrd_network/ocrd_network/deployer.py b/ocrd_network/ocrd_network/deployer.py index ce1b53ee9..2c699ef8b 100644 --- a/ocrd_network/ocrd_network/deployer.py +++ b/ocrd_network/ocrd_network/deployer.py @@ -28,10 +28,11 @@ class Deployer: - """ Class to wrap the deployment-functionality of the OCR-D Processing-Servers + """Wraps the deployment functionality of the Processing Server - Deployer is the one acting. Config is for representation of the config-file only. DeployHost is - for managing information, not for actually doing things. + Deployer is the one acting. + :py:attr:`config` is for representation of the config file only. + :py:attr:`hosts` is for managing processor information, not for actually processing. """ def __init__(self, config: ProcessingServerConfig) -> None: From f9c541427b1c5d102a2d49d9d885302553241850 Mon Sep 17 00:00:00 2001 From: Mehmed Mustafa Date: Thu, 2 Mar 2023 13:28:36 +0100 Subject: [PATCH 151/226] Update ocrd_network/ocrd_network/deployer.py Co-authored-by: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> --- ocrd_network/ocrd_network/deployer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ocrd_network/ocrd_network/deployer.py b/ocrd_network/ocrd_network/deployer.py index 2c699ef8b..27c8e25ab 100644 --- a/ocrd_network/ocrd_network/deployer.py +++ b/ocrd_network/ocrd_network/deployer.py @@ -38,7 +38,7 @@ class Deployer: def __init__(self, config: ProcessingServerConfig) -> None: """ Args: - config: Parsed processing-server-configuration + config (:py:class:`ProcessingServerConfig`): parsed configuration of the Processing Server """ self.log = getLogger(__name__) self.config = config From 4e66ed33796b94cdfb20de193c99ac8530e4f5ec Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Thu, 2 Mar 2023 14:27:47 +0100 Subject: [PATCH 152/226] Raise the error instead of returning it --- ocrd/ocrd/processor/helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ocrd/ocrd/processor/helpers.py b/ocrd/ocrd/processor/helpers.py index 8e15243cc..3878ac645 100644 --- a/ocrd/ocrd/processor/helpers.py +++ b/ocrd/ocrd/processor/helpers.py @@ -119,7 +119,7 @@ def run_processor( backend=backend) except Exception as err: log.exception("Failure in processor '%s'" % ocrd_tool['executable']) - return err + raise err finally: chdir(old_cwd) mem_usage_values = [mem for mem, _ in mem_usage] @@ -132,7 +132,7 @@ def run_processor( processor.process() except Exception as err: log.exception("Failure in processor '%s'" % ocrd_tool['executable']) - return err + raise err finally: chdir(old_cwd) From d1f287df4300d401473700dff5f2fcbc908e8f2b Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Thu, 2 Mar 2023 14:33:27 +0100 Subject: [PATCH 153/226] Revert getcwd() location - failing tests --- ocrd/ocrd/processor/helpers.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ocrd/ocrd/processor/helpers.py b/ocrd/ocrd/processor/helpers.py index 3878ac645..7fe560836 100644 --- a/ocrd/ocrd/processor/helpers.py +++ b/ocrd/ocrd/processor/helpers.py @@ -84,8 +84,6 @@ def run_processor( log = getLogger('ocrd.processor.helpers.run_processor') log.debug("Running processor %s", processorClass) - old_cwd = getcwd() - processor = get_processor( processor_class=processorClass, parameter=parameter, @@ -97,6 +95,7 @@ def run_processor( instance_caching=instance_caching ) + old_cwd = getcwd() chdir(processor.workspace.directory) ocrd_tool = processor.ocrd_tool From ddc92d74b1c03a1c011fe6cffd8fa1703e88412f Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Thu, 2 Mar 2023 14:59:31 +0100 Subject: [PATCH 154/226] Fix set_job_state --- ocrd_network/ocrd_network/processing_worker.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/ocrd_network/ocrd_network/processing_worker.py b/ocrd_network/ocrd_network/processing_worker.py index fb922ec66..b652aab38 100644 --- a/ocrd_network/ocrd_network/processing_worker.py +++ b/ocrd_network/ocrd_network/processing_worker.py @@ -197,6 +197,7 @@ def process_message(self, processing_message: OcrdProcessingMessage) -> None: # due to a call of sys.stdout.flush() disable_interactive_logging() + self.set_job_state(job_id, StateEnum.running) if self.processor_class: self.log.debug(f'Invoking the pythonic processor: {self.processor_name}') return_status = self.run_processor_from_worker( @@ -217,8 +218,8 @@ def process_message(self, processing_message: OcrdProcessingMessage) -> None: output_file_grps=output_file_grps, parameter=parameter ) - job_status = StateEnum.success if return_status else StateEnum.failed - self.set_job_state(job_id, return_status) + job_state = StateEnum.success if return_status else StateEnum.failed + self.set_job_state(job_id, job_state) # If the result_queue field is set, send the job status to a result queue if processing_message.result_queue: @@ -231,7 +232,7 @@ def process_message(self, processing_message: OcrdProcessingMessage) -> None: self.log.info(f'Publishing result message to queue: {processing_message.result_queue}') result_message = OcrdResultMessage( job_id=str(job_id), - status=job_status.value, + status=job_state.value, # Either path_to_mets or workspace_id must be set (mutually exclusive) path_to_mets=processing_message.path_to_mets, workspace_id=None @@ -300,16 +301,16 @@ def run_cli_from_worker( if return_code != 0: self.log.error(f'{executable} exited with non-zero return value {return_code}.') + return False else: self.log.debug(f'{executable} exited with success.') - return return_code == 0 + return True - def set_job_state(self, job_id: Any, success: bool): - """Set the job status in mongodb to either success or failed + def set_job_state(self, job_id: Any, state: StateEnum): + """Set the job status in mongodb """ # TODO: the way to interact with mongodb needs to be thought about. Beanie seems not # suitable as the worker is not async, thus using pymongo here - state = StateEnum.success if success else StateEnum.failed with pymongo.MongoClient(self.db_url) as client: db = client['ocrd'] db.Job.update_one({'_id': job_id}, {'$set': {'state': state}}, upsert=False) From 75b5d4e683f4dba4e2eb3fdc88b0513df62a92b4 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Fri, 3 Mar 2023 13:57:15 +0100 Subject: [PATCH 155/226] Make conditional TF import --- .../ocrd_network/processing_worker.py | 21 ++++++++++--------- ocrd_network/requirements.txt | 1 - 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/ocrd_network/ocrd_network/processing_worker.py b/ocrd_network/ocrd_network/processing_worker.py index b652aab38..c3e92646b 100644 --- a/ocrd_network/ocrd_network/processing_worker.py +++ b/ocrd_network/ocrd_network/processing_worker.py @@ -35,6 +35,17 @@ RMQPublisher ) +try: + # This env variable must be set before importing from Keras + environ['TF_CPP_MIN_LOG_LEVEL'] = '3' + from tensorflow.keras.utils import disable_interactive_logging + # Enabled interactive logging throws an exception + # due to a call of sys.stdout.flush() + disable_interactive_logging() +except Exception: + # Nothing should be handled here if TF is not available + pass + class ProcessingWorker: def __init__(self, rabbitmq_addr, mongodb_addr, processor_name, ocrd_tool: dict, processor_class=None) -> None: @@ -187,16 +198,6 @@ def process_message(self, processing_message: OcrdProcessingMessage) -> None: parameter = processing_message.parameters job_id = processing_message.job_id - # TODO: Find a proper solution for this dirty fix - if self.processor_name == 'ocrd-calamari-recognize': - # This env variable must be set before importing from Keras - environ['TF_CPP_MIN_LOG_LEVEL'] = '3' - from tensorflow.keras.utils import disable_interactive_logging - - # Enabled interactive logging throws an exception - # due to a call of sys.stdout.flush() - disable_interactive_logging() - self.set_job_state(job_id, StateEnum.running) if self.processor_class: self.log.debug(f'Invoking the pythonic processor: {self.processor_name}') diff --git a/ocrd_network/requirements.txt b/ocrd_network/requirements.txt index 3a09ed71a..d11fb430f 100644 --- a/ocrd_network/requirements.txt +++ b/ocrd_network/requirements.txt @@ -4,4 +4,3 @@ docker paramiko pika>=1.2.0 beanie~=1.7 -tensorflow From b3ada1f081f966e8a5fe6db9519835579c32aad9 Mon Sep 17 00:00:00 2001 From: joschrew Date: Thu, 9 Mar 2023 14:41:09 +0100 Subject: [PATCH 156/226] Remove unused function Function was used for tests but it was forgotten to remove it before creating the pr --- ocrd_network/ocrd_network/helpers.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/ocrd_network/ocrd_network/helpers.py b/ocrd_network/ocrd_network/helpers.py index d2e3ce0fc..cf73d3836 100644 --- a/ocrd_network/ocrd_network/helpers.py +++ b/ocrd_network/ocrd_network/helpers.py @@ -44,20 +44,3 @@ def verify_and_parse_rabbitmq_addr(rabbitmq_address: str) -> dict: # The default global vhost is / parsed_data['vhost'] = '/' if len(host_info) == 2 else f'/{host_info[2]}' return parsed_data - - -def get_workspaces_dir() -> str: - """get the path to the workspaces folder - - The processing-workers must have access to the workspaces. First idea is that they are provided - via nfs and always available under $XDG_DATA_HOME/ocrd-workspaces. This function provides the - absolute path to the folder and raises a ValueError if it is not available - """ - if 'XDG_DATA_HOME' in environ: - xdg_data_home = environ['XDG_DATA_HOME'] - else: - xdg_data_home = join(environ['HOME'], '.local', 'share') - res = join(xdg_data_home, 'ocrd-workspaces') - if not exists(res): - raise ValueError('Ocrd-Workspaces directory not found. Expected \'{res}\'') - return res From f154949e7a41f3e7f6810907eb98c8aacfbf7385 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Fri, 10 Mar 2023 16:03:44 +0100 Subject: [PATCH 157/226] Implement custom type check --- ocrd/ocrd/cli/processing_server.py | 22 ++++--- ocrd/ocrd/cli/processing_worker.py | 8 ++- ocrd/ocrd/decorators/ocrd_cli_options.py | 9 ++- ocrd_network/ocrd_network/__init__.py | 5 ++ ocrd_network/ocrd_network/helpers.py | 8 +-- ocrd_network/ocrd_network/param_validators.py | 57 +++++++++++++++++++ 6 files changed, 88 insertions(+), 21 deletions(-) create mode 100644 ocrd_network/ocrd_network/param_validators.py diff --git a/ocrd/ocrd/cli/processing_server.py b/ocrd/ocrd/cli/processing_server.py index ba61453c8..22aeb8434 100644 --- a/ocrd/ocrd/cli/processing_server.py +++ b/ocrd/ocrd/cli/processing_server.py @@ -6,14 +6,21 @@ :nested: full """ import click -from ocrd_utils import initLogging -from ocrd_network import ProcessingServer import logging +from ocrd_utils import initLogging +from ocrd_network import ( + ProcessingServer, + ProcessingServerParamType +) @click.command('processing-server') @click.argument('path_to_config', required=True, type=click.STRING) -@click.option('-a', '--address', help='Host (name/IP) and port to bind the Processing-Server to. Example: localhost:8080', required=True) +@click.option('-a', '--address', + default="localhost:8080", + help='The URL of the Processing server, format: host:port', + type=ProcessingServerParamType(), + required=True) def processing_server_cli(path_to_config, address: str): """ Start and manage processing workers with the processing server @@ -28,10 +35,7 @@ def processing_server_cli(path_to_config, address: str): logging.getLogger('paramiko.transport').setLevel(logging.INFO) logging.getLogger('ocrd.network').setLevel(logging.DEBUG) - try: - host, port = address.split(":") - port_int = int(port) - except ValueError: - raise click.UsageError('The --address option must have the format IP:PORT') - processing_server = ProcessingServer(path_to_config, host, port_int) + # Note, the address is already validated with the type field + host, port = address.split(":") + processing_server = ProcessingServer(path_to_config, host, port) processing_server.start() diff --git a/ocrd/ocrd/cli/processing_worker.py b/ocrd/ocrd/cli/processing_worker.py index 8068a0ce9..3f3db8570 100644 --- a/ocrd/ocrd/cli/processing_worker.py +++ b/ocrd/ocrd/cli/processing_worker.py @@ -11,6 +11,7 @@ initLogging, get_ocrd_tool_json ) +from ocrd_network import QueueServerParamType, DatabaseParamType from ocrd_network.processing_worker import ProcessingWorker @@ -18,11 +19,12 @@ @click.argument('processor_name', required=True, type=click.STRING) @click.option('-q', '--queue', default="admin:admin@localhost:5672/", - help='The username, password, host, port, and virtual host of the RabbitMQ Server. ' - 'Format: username:password@host:port/vhost') + help='The URL of the Queue Server, format: username:password@host:port/vhost', + type=QueueServerParamType()) @click.option('-d', '--database', default="mongodb://localhost:27018", - help='URL of the MongoDB service') + help='The URL of the MongoDB, format: mongodb://host:port', + type=DatabaseParamType()) def processing_worker_cli(processor_name: str, queue: str, database: str): """ Start a processing worker (a specific ocr-d processor) diff --git a/ocrd/ocrd/decorators/ocrd_cli_options.py b/ocrd/ocrd/decorators/ocrd_cli_options.py index 2b512d64b..6245f092b 100644 --- a/ocrd/ocrd/decorators/ocrd_cli_options.py +++ b/ocrd/ocrd/decorators/ocrd_cli_options.py @@ -1,6 +1,8 @@ -from click import option, STRING +from click import option from .parameter_option import parameter_option, parameter_override_option from .loglevel_option import loglevel_option +from ocrd_network import QueueServerParamType, DatabaseParamType + def ocrd_cli_options(f): """ @@ -26,8 +28,8 @@ def cli(mets_url): option('-O', '--output-file-grp', help='File group(s) used as output.', default='OUTPUT'), option('-g', '--page-id', help="ID(s) of the pages to process"), option('--overwrite', help="Overwrite the output file group or a page range (--page-id)", is_flag=True, default=False), - option('--queue', help="The RabbitMQ server address in format: {host}:{port}/{vhost}", type=STRING), - option('--database', help="The MongoDB address in format: mongodb://{host}:{port}", type=STRING), + option('--queue', help="The URL of the Queue Server, format: username:password@host:port/vhost", type=QueueServerParamType()), + option('--database', help="The URL of the MongoDB, format: mongodb://host:port", type=DatabaseParamType()), option('-C', '--show-resource', help='Dump the content of processor resource RESNAME', metavar='RESNAME'), option('-L', '--list-resources', is_flag=True, default=False, help='List names of processor resources'), parameter_option, @@ -43,3 +45,4 @@ def cli(mets_url): for param in params: param(f) return f + diff --git a/ocrd_network/ocrd_network/__init__.py b/ocrd_network/ocrd_network/__init__.py index 97de2b9a2..6cd95dc3c 100644 --- a/ocrd_network/ocrd_network/__init__.py +++ b/ocrd_network/ocrd_network/__init__.py @@ -24,3 +24,8 @@ # the network package. The reason, Mets Server is tightly coupled with the `OcrdWorkspace`. from .processing_server import ProcessingServer from .processing_worker import ProcessingWorker +from .param_validators import ( + DatabaseParamType, + ProcessingServerParamType, + QueueServerParamType +) diff --git a/ocrd_network/ocrd_network/helpers.py b/ocrd_network/ocrd_network/helpers.py index cf73d3836..b4ee39856 100644 --- a/ocrd_network/ocrd_network/helpers.py +++ b/ocrd_network/ocrd_network/helpers.py @@ -1,6 +1,4 @@ -from os import environ -from os.path import join, exists -from re import split +from re import split as re_split def verify_database_url(mongodb_address: str) -> str: @@ -8,9 +6,7 @@ def verify_database_url(mongodb_address: str) -> str: if not mongodb_address.startswith(database_prefix): error_msg = f'The database address must start with a prefix: {database_prefix}' raise ValueError(error_msg) - address_without_prefix = mongodb_address[len(database_prefix):] - print(f'Address without prefix: {address_without_prefix}') elements = address_without_prefix.split(':', 1) if len(elements) != 2: raise ValueError('The database address is in wrong format') @@ -34,7 +30,7 @@ def verify_and_parse_rabbitmq_addr(rabbitmq_address: str) -> dict: parsed_data['username'] = credentials[0] parsed_data['password'] = credentials[1] - host_info = split(pattern=r':|/', string=elements[1]) + host_info = re_split(pattern=r':|/', string=elements[1]) if len(host_info) != 3 and len(host_info) != 2: raise ValueError( 'The RabbitMQ host info is in wrong format. Expected format: username:password@host:port/vhost') diff --git a/ocrd_network/ocrd_network/param_validators.py b/ocrd_network/ocrd_network/param_validators.py new file mode 100644 index 000000000..a0b9726b7 --- /dev/null +++ b/ocrd_network/ocrd_network/param_validators.py @@ -0,0 +1,57 @@ +from click import ParamType +from re import split as re_split + + +class ProcessingServerParamType(ParamType): + name = "Processing server string format" + expected_format = 'host:port' + + def convert(self, value, param, ctx): + try: + elements = value.split(':') + if len(elements) != 2: + raise ValueError('The processing server address is in wrong format') + int(elements[1]) # validate port + except ValueError as error: + self.fail(f'{error}, expected format: {self.expected_format}', param, ctx) + return value + + +class QueueServerParamType(ParamType): + name = "Queue server string format" + expected_format = 'username:password@host:port/vhost' + + def convert(self, value, param, ctx): + try: + elements = value.split('@') + if len(elements) != 2: + raise ValueError('The RabbitMQ address is in wrong format') + credentials = elements[0].split(':') + if len(credentials) != 2: + raise ValueError(f'The RabbitMQ credentials are in wrong format') + host_info = re_split(pattern=r':|/', string=elements[1]) + if len(host_info) != 3 and len(host_info) != 2: + raise ValueError('The RabbitMQ host info is in wrong format') + int(host_info[1]) # validate port + except ValueError as error: + self.fail(f'{error}, expected format: {self.expected_format}', param, ctx) + return value + + +class DatabaseParamType(ParamType): + name = "Database string format" + expected_format = 'mongodb://host:port' + + def convert(self, value, param, ctx): + database_prefix = 'mongodb://' + try: + if not value.startswith(database_prefix): + raise ValueError(f'Wrong database prefix, expected prefix: {database_prefix}') + address_without_prefix = value[len(database_prefix):] + elements = address_without_prefix.split(':') + if len(elements) != 2: + raise ValueError(f'The database host and port are in wrong format') + int(elements[1]) # validate port + except ValueError as error: + self.fail(f'{error}, expected format: {self.expected_format}', param, ctx) + return value From 8a256d5db5db70da8f65073504a1c744b537a43c Mon Sep 17 00:00:00 2001 From: joschrew Date: Wed, 15 Mar 2023 13:42:08 +0100 Subject: [PATCH 158/226] Change native processor startup output handling The output of the processer can be usefull for debugging so it will now be at least written somewhere to read it later instead of ignoring it --- ocrd_network/ocrd_network/deployer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ocrd_network/ocrd_network/deployer.py b/ocrd_network/ocrd_network/deployer.py index 27c8e25ab..a87343272 100644 --- a/ocrd_network/ocrd_network/deployer.py +++ b/ocrd_network/ocrd_network/deployer.py @@ -288,7 +288,10 @@ def start_native_processor(self, client: SSHClient, processor_name: str, queue_u # printed with `echo $!` but it is printed inbetween other output. Because of that I added # `xyz` before and after the code to easily be able to filter out the pid via regex when # returning from the function - stdin.write(f'{cmd} & \n echo xyz$!xyz \n exit \n') + logpath = '/tmp/ocrd-processing-server-startup.log' + stdin.write(f'echo starting processor with \'{cmd}\' >> {logpath} \n') + stdin.write(f'{cmd} >> {logpath} 2>&1 &\n') + stdin.write('echo xyz$!xyz \n exit \n') output = stdout.read().decode('utf-8') stdout.close() stdin.close() From e125a14ad6ae42344f808cf8ac76c7ac325b09fe Mon Sep 17 00:00:00 2001 From: joschrew Date: Wed, 15 Mar 2023 15:14:23 +0100 Subject: [PATCH 159/226] Fix uvicorn logging bug Uvicorn logs its port with a format string which failed --- ocrd_network/ocrd_network/processing_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ocrd_network/ocrd_network/processing_server.py b/ocrd_network/ocrd_network/processing_server.py index 52310978f..6d89f2cc7 100644 --- a/ocrd_network/ocrd_network/processing_server.py +++ b/ocrd_network/ocrd_network/processing_server.py @@ -148,7 +148,7 @@ def start(self) -> None: 'Trying to kill parts of incompletely deployed service') self.deployer.kill_all() raise - uvicorn.run(self, host=self.hostname, port=self.port) + uvicorn.run(self, host=self.hostname, port=int(self.port)) @staticmethod def parse_config(config_path: str) -> ProcessingServerConfig: From b5d3496aa172940edbec1de16250228ff4de4c31 Mon Sep 17 00:00:00 2001 From: joschrew Date: Wed, 15 Mar 2023 15:17:31 +0100 Subject: [PATCH 160/226] Add possibility to run bashlib processors too --- ocrd_network/ocrd_network/deployer.py | 10 +++++++--- ocrd_network/ocrd_network/deployment_utils.py | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/ocrd_network/ocrd_network/deployer.py b/ocrd_network/ocrd_network/deployer.py index a87343272..656b57be8 100644 --- a/ocrd_network/ocrd_network/deployer.py +++ b/ocrd_network/ocrd_network/deployer.py @@ -23,6 +23,7 @@ CustomDockerClient, DeployType, HostData, + is_bashlib_processor, ) from ocrd_network.rabbitmq_utils import RMQPublisher @@ -30,7 +31,7 @@ class Deployer: """Wraps the deployment functionality of the Processing Server - Deployer is the one acting. + Deployer is the one acting. :py:attr:`config` is for representation of the config file only. :py:attr:`hosts` is for managing processor information, not for actually processing. """ @@ -278,11 +279,14 @@ def start_native_processor(self, client: SSHClient, processor_name: str, queue_u Returns: str: pid of running process """ - # TODO: some processors are bashlib. They have to be started differently self.log.info(f'Starting native processor: {processor_name}') channel = client.invoke_shell() stdin, stdout = channel.makefile('wb'), channel.makefile('rb') - cmd = f'{processor_name} --database {database_url} --queue {queue_url}' + if is_bashlib_processor(processor_name): + cmd = f'ocrd processing-worker {processor_name} --database {database_url} ' \ + f'--queue {queue_url}' + else: + cmd = f'{processor_name} --database {database_url} --queue {queue_url}' # the only way (I could find) to make it work to start a process in the background and # return early is this construction. The pid of the last started background process is # printed with `echo $!` but it is printed inbetween other output. Because of that I added diff --git a/ocrd_network/ocrd_network/deployment_utils.py b/ocrd_network/ocrd_network/deployment_utils.py index 7b5c0416e..a310a50f0 100644 --- a/ocrd_network/ocrd_network/deployment_utils.py +++ b/ocrd_network/ocrd_network/deployment_utils.py @@ -1,6 +1,8 @@ from __future__ import annotations from enum import Enum from typing import Union, List +from distutils.spawn import find_executable as which +import re from docker import APIClient, DockerClient from docker.transport import SSHHTTPAdapter @@ -31,6 +33,23 @@ def create_docker_client(address: str, username: str, password: Union[str, None] return CustomDockerClient(username, address, password=password, keypath=keypath) +def is_bashlib_processor(processor_name): + """ Determine if a processor is a bashlib processor + + Returns True if processor_name is available as a program and does not contain a python hashbang + in line 1 """ + if not processor_name.startswith("ocrd"): + return False + program = which(processor_name) + if not program: + return False + with open(program) as fin: + line = fin.readline().strip() + if re.fullmatch('[#][!].*/python[0-9.]*', line): + return False + return True + + class HostData: """class to store runtime information for a host """ From 5ff8860a004a81f6e9fd9d5a93e51040bfc5766c Mon Sep 17 00:00:00 2001 From: joschrew Date: Thu, 16 Mar 2023 09:35:33 +0100 Subject: [PATCH 161/226] Add module docstring for database --- ocrd_network/ocrd_network/database.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ocrd_network/ocrd_network/database.py b/ocrd_network/ocrd_network/database.py index 66e4722e7..0b4a88715 100644 --- a/ocrd_network/ocrd_network/database.py +++ b/ocrd_network/ocrd_network/database.py @@ -1,3 +1,17 @@ +""" The database is used to store information regarding jobs and workspaces. + +Jobs: for every process-request a job is inserted into the database with a uuid, status and +information about the process like parameters and filegroups. It is mainly used to track the status +(`ocrd_network.models.job.StateEnum`) of a job so that the state of a job can be queried. Finished +jobs are not deleted from the database. + +Workspaces: A job or a processor always runs on a workspace. So a processor needs the information +where the workspace is available. This information can be set with providing an absolute path or a +workspace_id. With the latter, the database is used to convert the workspace_id to a path. + +XXX: Currently the information is not preserved after the processing-server shuts down as the +database (runs in docker) currently has no volume set. +""" from beanie import init_beanie from motor.motor_asyncio import AsyncIOMotorClient From 2b5cc3b07b2a006835c9d053c04b0646be257bab Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Thu, 16 Mar 2023 17:24:38 +0100 Subject: [PATCH 162/226] parameters -> arguments in RMQ connector --- ocrd_network/ocrd_network/rabbitmq_utils/connector.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ocrd_network/ocrd_network/rabbitmq_utils/connector.py b/ocrd_network/ocrd_network/rabbitmq_utils/connector.py index 288f8a203..fe778a772 100644 --- a/ocrd_network/ocrd_network/rabbitmq_utils/connector.py +++ b/ocrd_network/ocrd_network/rabbitmq_utils/connector.py @@ -99,16 +99,16 @@ def exchange_bind( destination_exchange: str, source_exchange: str, routing_key: str, - parameters: Optional[Any] = None + arguments: Optional[Any] = None ) -> None: - if parameters is None: - parameters = {} + if arguments is None: + arguments = {} if channel and channel.is_open: channel.exchange_bind( destination=destination_exchange, source=source_exchange, routing_key=routing_key, - parameters=parameters + arguments=arguments ) @staticmethod From ad04cb676af9423b55afec0baf97e7c282676511 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Thu, 16 Mar 2023 18:01:25 +0100 Subject: [PATCH 163/226] Fix quotes --- ocrd/ocrd/cli/processing_server.py | 2 +- ocrd_network/ocrd_network/deployer.py | 13 ++++++------- ocrd_network/ocrd_network/deployment_utils.py | 4 ++-- ocrd_network/ocrd_network/param_validators.py | 6 +++--- ocrd_network/ocrd_network/processing_server.py | 13 ++++++------- ocrd_network/ocrd_network/processing_worker.py | 10 +++++----- 6 files changed, 23 insertions(+), 25 deletions(-) diff --git a/ocrd/ocrd/cli/processing_server.py b/ocrd/ocrd/cli/processing_server.py index 22aeb8434..3baa05607 100644 --- a/ocrd/ocrd/cli/processing_server.py +++ b/ocrd/ocrd/cli/processing_server.py @@ -36,6 +36,6 @@ def processing_server_cli(path_to_config, address: str): logging.getLogger('ocrd.network').setLevel(logging.DEBUG) # Note, the address is already validated with the type field - host, port = address.split(":") + host, port = address.split(':') processing_server = ProcessingServer(path_to_config, host, port) processing_server.start() diff --git a/ocrd_network/ocrd_network/deployer.py b/ocrd_network/ocrd_network/deployer.py index 656b57be8..b33173756 100644 --- a/ocrd_network/ocrd_network/deployer.py +++ b/ocrd_network/ocrd_network/deployer.py @@ -90,8 +90,7 @@ def deploy_hosts(self, rabbitmq_url: str, mongodb_url: str) -> None: def _deploy_processing_worker(self, processor: WorkerConfig, host: HostData, rabbitmq_url: str, mongodb_url: str) -> None: - self.log.debug(f'deploy \'{processor.deploy_type}\' processor: \'{processor}\' on' - f'\'{host.config.address}\'') + self.log.debug(f"deploy '{processor.deploy_type}' processor: '{processor}' on '{host.config.address}'") for _ in range(processor.count): if processor.deploy_type == DeployType.native: @@ -187,8 +186,8 @@ def deploy_mongodb(self, image: str = 'mongo', detach: bool = True, remove: bool ports_mapping: Union[Dict, None] = None) -> str: """ Start mongodb in docker """ - self.log.debug(f'Trying to deploy image[{image}], ' - f'with modes: detach[{detach}], remove[{remove}]') + self.log.debug(f"Trying to deploy '{image}', with modes: " + f"detach='{detach}', remove='{remove}'") if not self.config or not self.config.mongo.address: raise ValueError('Deploying MongoDB has failed - missing configuration.') @@ -257,12 +256,12 @@ def kill_hosts(self) -> None: def kill_processing_worker(self, host: HostData) -> None: for pid in host.pids_native: - self.log.debug(f'Trying to kill/stop native processor: with PID: \'{pid}\'') + self.log.debug(f"Trying to kill/stop native processor: with PID: '{pid}'") host.ssh_client.exec_command(f'kill {pid}') host.pids_native = [] for pid in host.pids_docker: - self.log.debug(f'Trying to kill/stop docker container with PID: {pid}') + self.log.debug(f"Trying to kill/stop docker container with PID: '{pid}'") host.docker_client.containers.get(pid).stop() host.pids_docker = [] @@ -293,7 +292,7 @@ def start_native_processor(self, client: SSHClient, processor_name: str, queue_u # `xyz` before and after the code to easily be able to filter out the pid via regex when # returning from the function logpath = '/tmp/ocrd-processing-server-startup.log' - stdin.write(f'echo starting processor with \'{cmd}\' >> {logpath} \n') + stdin.write(f"echo starting processor with '{cmd}' >> '{logpath}'\n") stdin.write(f'{cmd} >> {logpath} 2>&1 &\n') stdin.write('echo xyz$!xyz \n exit \n') output = stdout.read().decode('utf-8') diff --git a/ocrd_network/ocrd_network/deployment_utils.py b/ocrd_network/ocrd_network/deployment_utils.py index a310a50f0..974052dd8 100644 --- a/ocrd_network/ocrd_network/deployment_utils.py +++ b/ocrd_network/ocrd_network/deployment_utils.py @@ -23,7 +23,7 @@ def create_ssh_client(address: str, username: str, password: Union[str, None], try: client.connect(hostname=address, username=username, password=password, key_filename=keypath) except Exception: - getLogger(__name__).error(f'Error creating SSHClient for host: \'{address}\'') + getLogger(__name__).error(f"Error creating SSHClient for host: '{address}'") raise return client @@ -107,7 +107,7 @@ def __init__(self, base_url, password: Union[str, None] = None, self.password = password self.keypath = keypath if not self.password and not self.keypath: - raise Exception('either "password" or "keypath" must be provided') + raise Exception("either 'password' or 'keypath' must be provided") super().__init__(base_url) def _create_paramiko_client(self, base_url: str) -> None: diff --git a/ocrd_network/ocrd_network/param_validators.py b/ocrd_network/ocrd_network/param_validators.py index a0b9726b7..1ea89c3f2 100644 --- a/ocrd_network/ocrd_network/param_validators.py +++ b/ocrd_network/ocrd_network/param_validators.py @@ -3,7 +3,7 @@ class ProcessingServerParamType(ParamType): - name = "Processing server string format" + name = 'Processing server string format' expected_format = 'host:port' def convert(self, value, param, ctx): @@ -18,7 +18,7 @@ def convert(self, value, param, ctx): class QueueServerParamType(ParamType): - name = "Queue server string format" + name = 'Queue server string format' expected_format = 'username:password@host:port/vhost' def convert(self, value, param, ctx): @@ -39,7 +39,7 @@ def convert(self, value, param, ctx): class DatabaseParamType(ParamType): - name = "Database string format" + name = 'Database string format' expected_format = 'mongodb://host:port' def convert(self, value, param, ctx): diff --git a/ocrd_network/ocrd_network/processing_server.py b/ocrd_network/ocrd_network/processing_server.py index 6d89f2cc7..76ff02d19 100644 --- a/ocrd_network/ocrd_network/processing_server.py +++ b/ocrd_network/ocrd_network/processing_server.py @@ -45,8 +45,8 @@ class ProcessingServer(FastAPI): def __init__(self, config_path: str, host: str, port: int) -> None: super().__init__(on_startup=[self.on_startup], on_shutdown=[self.on_shutdown], - title="OCR-D Processing Server", - description="OCR-D processing and processors") + title='OCR-D Processing Server', + description='OCR-D processing and processors') self.log = getLogger(__name__) self.hostname = host self.port = port @@ -119,7 +119,7 @@ def __init__(self, config_path: str, host: str, port: int) -> None: @self.exception_handler(RequestValidationError) async def validation_exception_handler(request: Request, exc: RequestValidationError): exc_str = f'{exc}'.replace('\n', ' ').replace(' ', ' ') - self.log.error(f"{request}: {exc_str}") + self.log.error(f'{request}: {exc_str}') content = {'status_code': 10422, 'message': exc_str, 'data': None} return JSONResponse(content=content, status_code=status.HTTP_422_UNPROCESSABLE_ENTITY) @@ -228,8 +228,7 @@ async def push_processor_job(self, processor_name: str, data: JobInput) -> JobOu if not ocrd_tool: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=(f'Processor \'{processor_name}\' not available. It\'s ocrd_tool is ' - 'empty or missing') + detail=f"Processor '{processor_name}' not available. It's ocrd_tool is empty or missing" ) validator = ParameterValidator(ocrd_tool) report = validator.validate(data.parameters) @@ -240,14 +239,14 @@ async def push_processor_job(self, processor_name: str, data: JobInput) -> JobOu if bool(data.path) == bool(data.workspace_id): raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail='Either \'path\' or \'workspace_id\' must be set' + detail="Either 'path' or 'workspace_id' must be set" ) elif data.workspace_id: workspace = await Workspace.get(data.workspace_id) if not workspace: raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail=f'Workspace for id \'{data.workspace_id}\' not existing' + detail=f"Workspace for id '{data.workspace_id}' not existing" ) data.path = workspace.workspace_mets_path diff --git a/ocrd_network/ocrd_network/processing_worker.py b/ocrd_network/ocrd_network/processing_worker.py index c3e92646b..bfb0cb5e7 100644 --- a/ocrd_network/ocrd_network/processing_worker.py +++ b/ocrd_network/ocrd_network/processing_worker.py @@ -61,11 +61,11 @@ def __init__(self, rabbitmq_addr, mongodb_addr, processor_name, ocrd_tool: dict, self.db_url = verify_database_url(mongodb_addr) self.log.debug(f'Verified MongoDB URL: {self.db_url}') rmq_data = verify_and_parse_rabbitmq_addr(rabbitmq_addr) - self.rmq_username = rmq_data["username"] - self.rmq_password = rmq_data["password"] - self.rmq_host = rmq_data["host"] - self.rmq_port = rmq_data["port"] - self.rmq_vhost = rmq_data["vhost"] + self.rmq_username = rmq_data['username'] + self.rmq_password = rmq_data['password'] + self.rmq_host = rmq_data['host'] + self.rmq_port = rmq_data['port'] + self.rmq_vhost = rmq_data['vhost'] self.log.debug(f'Verified RabbitMQ Credentials: {self.rmq_username}:{self.rmq_password}') self.log.debug(f'Verified RabbitMQ Server URL: {self.rmq_host}:{self.rmq_port}{self.rmq_vhost}') except ValueError as e: From 8e7b1f84fb8cc954ae591d80b4e474c112dbaf62 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Thu, 16 Mar 2023 18:15:58 +0100 Subject: [PATCH 164/226] better error message --- ocrd_network/ocrd_network/processing_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ocrd_network/ocrd_network/processing_server.py b/ocrd_network/ocrd_network/processing_server.py index 76ff02d19..10d8fd1d7 100644 --- a/ocrd_network/ocrd_network/processing_server.py +++ b/ocrd_network/ocrd_network/processing_server.py @@ -239,7 +239,7 @@ async def push_processor_job(self, processor_name: str, data: JobInput) -> JobOu if bool(data.path) == bool(data.workspace_id): raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail="Either 'path' or 'workspace_id' must be set" + detail="Arguments 'path' and 'workspace_id' are mutually exclusive" ) elif data.workspace_id: workspace = await Workspace.get(data.workspace_id) From 44e1eaaffd2fee18ae7ed6df451bdfeca0cf0059 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Fri, 17 Mar 2023 20:27:51 +0100 Subject: [PATCH 165/226] Improve validation and parsing of URIs --- ocrd/ocrd/cli/processing_worker.py | 4 +- ocrd_network/ocrd_network/deployer.py | 19 ++--- ocrd_network/ocrd_network/helpers.py | 74 +++++++++---------- ocrd_network/ocrd_network/param_validators.py | 41 ++++------ .../ocrd_network/processing_server.py | 5 +- .../ocrd_network/processing_worker.py | 8 +- 6 files changed, 67 insertions(+), 84 deletions(-) diff --git a/ocrd/ocrd/cli/processing_worker.py b/ocrd/ocrd/cli/processing_worker.py index 3f3db8570..a823062af 100644 --- a/ocrd/ocrd/cli/processing_worker.py +++ b/ocrd/ocrd/cli/processing_worker.py @@ -18,8 +18,8 @@ @click.command('processing-worker') @click.argument('processor_name', required=True, type=click.STRING) @click.option('-q', '--queue', - default="admin:admin@localhost:5672/", - help='The URL of the Queue Server, format: username:password@host:port/vhost', + default="amqp://admin:admin@localhost:5672/", + help='The URL of the Queue Server, format: amqp://username:password@host:port/vhost', type=QueueServerParamType()) @click.option('-d', '--database', default="mongodb://localhost:27018", diff --git a/ocrd_network/ocrd_network/deployer.py b/ocrd_network/ocrd_network/deployer.py index b33173756..98408f739 100644 --- a/ocrd_network/ocrd_network/deployer.py +++ b/ocrd_network/ocrd_network/deployer.py @@ -132,7 +132,7 @@ def deploy_rabbitmq(self, image: str = 'rabbitmq:3-management', detach: bool = T self.config.queue.password, self.config.queue.keypath) if not ports_mapping: # 5672, 5671 - used by AMQP 0-9-1 and AMQP 1.0 clients without and with TLS - # 15672, 15671: HTTP API clients, management UI and rabbitmqadmin, without and with TLS + # 15672, 15671: HTTP API clients, management UI and rabbitmq admin, without and with TLS # 25672: used for internode and CLI tools communication and is allocated from # a dynamic range (limited to a single port by default, computed as AMQP port + 20000) ports_mapping = { @@ -157,8 +157,11 @@ def deploy_rabbitmq(self, image: str = 'rabbitmq:3-management', detach: bool = T # Build the RabbitMQ Server URL to return rmq_host = self.config.queue.address - rmq_port = self.config.queue.port - rmq_vhost = '/' # the default virtual host + # note, integer validation is already performed + rmq_port = int(self.config.queue.port) + # the default virtual host since no field is + # provided in the processing server config.yml + rmq_vhost = '/' self.wait_for_rabbitmq_availability(rmq_host, rmq_port, rmq_vhost, self.config.queue.credentials[0], @@ -168,7 +171,7 @@ def deploy_rabbitmq(self, image: str = 'rabbitmq:3-management', detach: bool = T self.log.info(f'The RabbitMQ server was deployed on host: {rabbitmq_hostinfo}') return rabbitmq_hostinfo - def wait_for_rabbitmq_availability(self, host: str, port: str, vhost: str, username: str, + def wait_for_rabbitmq_availability(self, host: str, port: int, vhost: str, username: str, password: str) -> None: max_waiting_steps = 15 while max_waiting_steps > 0: @@ -210,13 +213,11 @@ def deploy_mongodb(self, image: str = 'mongo', detach: bool = True, remove: bool self.mongo_pid = res.id client.close() - # Build the MongoDB URL to return - mongodb_prefix = 'mongodb://' mongodb_host = self.config.mongo.address mongodb_port = self.config.mongo.port - mongodb_url = f'{mongodb_prefix}{mongodb_host}:{mongodb_port}' - self.log.info(f'The MongoDB was deployed on url: {mongodb_url}') - return mongodb_url + mongodb_hostinfo = f'{mongodb_host}:{mongodb_port}' + self.log.info(f'The MongoDB was deployed on host: {mongodb_hostinfo}') + return mongodb_hostinfo def kill_rabbitmq(self) -> None: if not self.mq_pid: diff --git a/ocrd_network/ocrd_network/helpers.py b/ocrd_network/ocrd_network/helpers.py index b4ee39856..b5027c547 100644 --- a/ocrd_network/ocrd_network/helpers.py +++ b/ocrd_network/ocrd_network/helpers.py @@ -1,42 +1,34 @@ -from re import split as re_split - - -def verify_database_url(mongodb_address: str) -> str: - database_prefix = 'mongodb://' - if not mongodb_address.startswith(database_prefix): - error_msg = f'The database address must start with a prefix: {database_prefix}' - raise ValueError(error_msg) - address_without_prefix = mongodb_address[len(database_prefix):] - elements = address_without_prefix.split(':', 1) - if len(elements) != 2: - raise ValueError('The database address is in wrong format') - db_host = elements[0] - db_port = int(elements[1]) - mongodb_url = f'{database_prefix}{db_host}:{db_port}' - return mongodb_url - - -def verify_and_parse_rabbitmq_addr(rabbitmq_address: str) -> dict: - parsed_data = {} - elements = rabbitmq_address.split('@') - if len(elements) != 2: - raise ValueError('The RabbitMQ address is in wrong format. Expected format: username:password@host:port/vhost') - - credentials = elements[0].split(':') - if len(credentials) != 2: - raise ValueError( - 'The RabbitMQ credentials are in wrong format. Expected format: username:password@host:port/vhost') - - parsed_data['username'] = credentials[0] - parsed_data['password'] = credentials[1] - - host_info = re_split(pattern=r':|/', string=elements[1]) - if len(host_info) != 3 and len(host_info) != 2: - raise ValueError( - 'The RabbitMQ host info is in wrong format. Expected format: username:password@host:port/vhost') - - parsed_data['host'] = host_info[0] - parsed_data['port'] = int(host_info[1]) - # The default global vhost is / - parsed_data['vhost'] = '/' if len(host_info) == 2 else f'/{host_info[2]}' +from re import match as re_match +from pika import URLParameters +from pymongo import uri_parser as mongo_uri_parser + + +def verify_database_uri(mongodb_address: str) -> str: + try: + # perform validation check + mongo_uri_parser.parse_uri(uri=mongodb_address, validate=True) + except Exception as error: + raise ValueError(f"The database address '{mongodb_address}' is in wrong format, {error}") + return mongodb_address + + +def verify_and_parse_mq_uri(rabbitmq_address: str): + """ + Check the full list of available parameters in the docs here: + https://pika.readthedocs.io/en/stable/_modules/pika/connection.html#URLParameters + """ + + uri_pattern = r"^(?:([^:\/?#\s]+):\/{2})?(?:([^@\/?#\s]+)@)?([^\/?#\s]+)?(?:\/([^?#\s]*))?(?:[?]([^#\s]+))?\S*$" + match = re_match(pattern=uri_pattern, string=rabbitmq_address) + if not match: + raise ValueError(f"The message queue server address is in wrong format: '{rabbitmq_address}'") + url_params = URLParameters(rabbitmq_address) + + parsed_data = { + 'username': url_params.credentials.username, + 'password': url_params.credentials.password, + 'host': url_params.host, + 'port': url_params.port, + 'vhost': url_params.virtual_host + } return parsed_data diff --git a/ocrd_network/ocrd_network/param_validators.py b/ocrd_network/ocrd_network/param_validators.py index 1ea89c3f2..baf4d2452 100644 --- a/ocrd_network/ocrd_network/param_validators.py +++ b/ocrd_network/ocrd_network/param_validators.py @@ -1,5 +1,9 @@ from click import ParamType -from re import split as re_split + +from ocrd_network.helpers import ( + verify_database_uri, + verify_and_parse_mq_uri +) class ProcessingServerParamType(ParamType): @@ -18,40 +22,25 @@ def convert(self, value, param, ctx): class QueueServerParamType(ParamType): - name = 'Queue server string format' - expected_format = 'username:password@host:port/vhost' + name = 'Message queue server string format' def convert(self, value, param, ctx): try: - elements = value.split('@') - if len(elements) != 2: - raise ValueError('The RabbitMQ address is in wrong format') - credentials = elements[0].split(':') - if len(credentials) != 2: - raise ValueError(f'The RabbitMQ credentials are in wrong format') - host_info = re_split(pattern=r':|/', string=elements[1]) - if len(host_info) != 3 and len(host_info) != 2: - raise ValueError('The RabbitMQ host info is in wrong format') - int(host_info[1]) # validate port - except ValueError as error: - self.fail(f'{error}, expected format: {self.expected_format}', param, ctx) + # perform validation check only + verify_and_parse_mq_uri(value) + except Exception as error: + # TODO: remove the identifier my_error + self.fail(f'my_error: {error}', param, ctx) return value class DatabaseParamType(ParamType): name = 'Database string format' - expected_format = 'mongodb://host:port' def convert(self, value, param, ctx): - database_prefix = 'mongodb://' try: - if not value.startswith(database_prefix): - raise ValueError(f'Wrong database prefix, expected prefix: {database_prefix}') - address_without_prefix = value[len(database_prefix):] - elements = address_without_prefix.split(':') - if len(elements) != 2: - raise ValueError(f'The database host and port are in wrong format') - int(elements[1]) # validate port - except ValueError as error: - self.fail(f'{error}, expected format: {self.expected_format}', param, ctx) + # perform validation check only + verify_database_uri(value) + except Exception as error: + self.fail(f'{error}', param, ctx) return value diff --git a/ocrd_network/ocrd_network/processing_server.py b/ocrd_network/ocrd_network/processing_server.py index 10d8fd1d7..795b80c08 100644 --- a/ocrd_network/ocrd_network/processing_server.py +++ b/ocrd_network/ocrd_network/processing_server.py @@ -129,9 +129,10 @@ def start(self) -> None: try: rabbitmq_hostinfo = self.deployer.deploy_rabbitmq() # Assign the credentials to the rabbitmq url parameter - rabbitmq_url = f'{self.rmq_username}:{self.rmq_password}@{rabbitmq_hostinfo}' + rabbitmq_url = f'amqp://{self.rmq_username}:{self.rmq_password}@{rabbitmq_hostinfo}' - self.mongodb_url = self.deployer.deploy_mongodb() + mongodb_hostinfo = self.deployer.deploy_mongodb() + self.mongodb_url = f'mongodb://{mongodb_hostinfo}' # The RMQPublisher is initialized and a connection to the RabbitMQ is performed self.connect_publisher() diff --git a/ocrd_network/ocrd_network/processing_worker.py b/ocrd_network/ocrd_network/processing_worker.py index bfb0cb5e7..4c96010cb 100644 --- a/ocrd_network/ocrd_network/processing_worker.py +++ b/ocrd_network/ocrd_network/processing_worker.py @@ -24,8 +24,8 @@ run_processor ) from ocrd_network.helpers import ( - verify_database_url, - verify_and_parse_rabbitmq_addr + verify_database_uri, + verify_and_parse_mq_uri ) from ocrd_network.models.job import StateEnum from ocrd_network.rabbitmq_utils import ( @@ -58,9 +58,9 @@ def __init__(self, rabbitmq_addr, mongodb_addr, processor_name, ocrd_tool: dict, self.log.addHandler(file_handler) try: - self.db_url = verify_database_url(mongodb_addr) + self.db_url = verify_database_uri(mongodb_addr) self.log.debug(f'Verified MongoDB URL: {self.db_url}') - rmq_data = verify_and_parse_rabbitmq_addr(rabbitmq_addr) + rmq_data = verify_and_parse_mq_uri(rabbitmq_addr) self.rmq_username = rmq_data['username'] self.rmq_password = rmq_data['password'] self.rmq_host = rmq_data['host'] From c7912603c733d929cc80c8a80d9bcd0e08cf01b9 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Fri, 17 Mar 2023 21:14:35 +0100 Subject: [PATCH 166/226] Clean, refactor, and add small TODOs --- ocrd_network/ocrd_network/database.py | 2 +- ocrd_network/ocrd_network/deployer.py | 19 +++++++++---------- ocrd_network/ocrd_network/models/ocrd_tool.py | 6 +----- ocrd_network/ocrd_network/param_validators.py | 3 +-- .../ocrd_network/processing_server.py | 8 ++++++-- .../ocrd_network/rabbitmq_utils/__init__.py | 3 --- ocrd_network/ocrd_network/web_api/__init__.py | 2 -- ocrd_network/setup.py | 1 + 8 files changed, 19 insertions(+), 25 deletions(-) delete mode 100644 ocrd_network/ocrd_network/web_api/__init__.py diff --git a/ocrd_network/ocrd_network/database.py b/ocrd_network/ocrd_network/database.py index 0b4a88715..369f13426 100644 --- a/ocrd_network/ocrd_network/database.py +++ b/ocrd_network/ocrd_network/database.py @@ -1,7 +1,7 @@ """ The database is used to store information regarding jobs and workspaces. Jobs: for every process-request a job is inserted into the database with a uuid, status and -information about the process like parameters and filegroups. It is mainly used to track the status +information about the process like parameters and file groups. It is mainly used to track the status (`ocrd_network.models.job.StateEnum`) of a job so that the state of a job can be queried. Finished jobs are not deleted from the database. diff --git a/ocrd_network/ocrd_network/deployer.py b/ocrd_network/ocrd_network/deployer.py index 98408f739..dbb875d5f 100644 --- a/ocrd_network/ocrd_network/deployer.py +++ b/ocrd_network/ocrd_network/deployer.py @@ -8,9 +8,8 @@ """ from __future__ import annotations -from typing import Dict, Union, Optional +from typing import Dict, Union from paramiko import SSHClient -from pathlib import Path from re import search as re_search from time import sleep @@ -114,16 +113,16 @@ def _deploy_processing_worker(self, processor: WorkerConfig, host: HostData, host.pids_docker.append(pid) sleep(0.1) - def deploy_rabbitmq(self, image: str = 'rabbitmq:3-management', detach: bool = True, - remove: bool = True, ports_mapping: Union[Dict, None] = None) -> str: + def deploy_rabbitmq(self, image: str, detach: bool, remove: bool, + ports_mapping: Union[Dict, None] = None) -> str: """Start docker-container with rabbitmq This method deploys the RabbitMQ Server. Handling of creation of queues, submitting messages to queues, and receiving messages from queues is part of the RabbitMQ Library which is part of the OCR-D WebAPI implementation. """ - self.log.debug(f'Trying to deploy image[{image}], ' - f'with modes: detach[{detach}], remove[{remove}]') + self.log.debug(f"Trying to deploy '{image}', with modes: " + f"detach='{detach}', remove='{remove}'") if not self.config or not self.config.queue.address: raise ValueError('Deploying RabbitMQ has failed - missing configuration.') @@ -145,6 +144,7 @@ def deploy_rabbitmq(self, image: str = 'rabbitmq:3-management', detach: bool = T detach=detach, remove=remove, ports=ports_mapping, + # The default credentials to be used by the processing workers environment=[ f'RABBITMQ_DEFAULT_USER={self.config.queue.credentials[0]}', f'RABBITMQ_DEFAULT_PASS={self.config.queue.credentials[1]}' @@ -182,10 +182,11 @@ def wait_for_rabbitmq_availability(self, host: str, port: int, vhost: str, usern max_waiting_steps -= 1 sleep(2) else: + # TODO: Disconnect the dummy_publisher here before returning... return raise RuntimeError('Error waiting for queue startup: timeout exceeded') - def deploy_mongodb(self, image: str = 'mongo', detach: bool = True, remove: bool = True, + def deploy_mongodb(self, image: str, detach: bool, remove: bool, ports_mapping: Union[Dict, None] = None) -> str: """ Start mongodb in docker """ @@ -213,9 +214,7 @@ def deploy_mongodb(self, image: str = 'mongo', detach: bool = True, remove: bool self.mongo_pid = res.id client.close() - mongodb_host = self.config.mongo.address - mongodb_port = self.config.mongo.port - mongodb_hostinfo = f'{mongodb_host}:{mongodb_port}' + mongodb_hostinfo = f'{self.config.mongo.address}:{self.config.mongo.port}' self.log.info(f'The MongoDB was deployed on host: {mongodb_hostinfo}') return mongodb_hostinfo diff --git a/ocrd_network/ocrd_network/models/ocrd_tool.py b/ocrd_network/ocrd_network/models/ocrd_tool.py index a60b547ab..6e87a9710 100644 --- a/ocrd_network/ocrd_network/models/ocrd_tool.py +++ b/ocrd_network/ocrd_network/models/ocrd_tool.py @@ -1,9 +1,5 @@ -# This model is directly taken from the Triet's implementation: -# REST API wrapper for the processor #884 - -from typing import List, Optional - from pydantic import BaseModel +from typing import List, Optional class OcrdTool(BaseModel): diff --git a/ocrd_network/ocrd_network/param_validators.py b/ocrd_network/ocrd_network/param_validators.py index baf4d2452..3b3483386 100644 --- a/ocrd_network/ocrd_network/param_validators.py +++ b/ocrd_network/ocrd_network/param_validators.py @@ -29,8 +29,7 @@ def convert(self, value, param, ctx): # perform validation check only verify_and_parse_mq_uri(value) except Exception as error: - # TODO: remove the identifier my_error - self.fail(f'my_error: {error}', param, ctx) + self.fail(f'{error}', param, ctx) return value diff --git a/ocrd_network/ocrd_network/processing_server.py b/ocrd_network/ocrd_network/processing_server.py index 795b80c08..d21345b2e 100644 --- a/ocrd_network/ocrd_network/processing_server.py +++ b/ocrd_network/ocrd_network/processing_server.py @@ -127,11 +127,15 @@ def start(self) -> None: """ deploy agents (db, queue, workers) and start the processing server with uvicorn """ try: - rabbitmq_hostinfo = self.deployer.deploy_rabbitmq() + rabbitmq_hostinfo = self.deployer.deploy_rabbitmq( + image='rabbitmq:3-management', detach=True, remove=True) + # Assign the credentials to the rabbitmq url parameter rabbitmq_url = f'amqp://{self.rmq_username}:{self.rmq_password}@{rabbitmq_hostinfo}' - mongodb_hostinfo = self.deployer.deploy_mongodb() + mongodb_hostinfo = self.deployer.deploy_mongodb( + image='mongo', detach=True, remove=True) + self.mongodb_url = f'mongodb://{mongodb_hostinfo}' # The RMQPublisher is initialized and a connection to the RabbitMQ is performed diff --git a/ocrd_network/ocrd_network/rabbitmq_utils/__init__.py b/ocrd_network/ocrd_network/rabbitmq_utils/__init__.py index 2ef092fa8..2d5f55e62 100644 --- a/ocrd_network/ocrd_network/rabbitmq_utils/__init__.py +++ b/ocrd_network/ocrd_network/rabbitmq_utils/__init__.py @@ -1,6 +1,3 @@ -# The RabbitMQ utils are directly copied from the OCR-D WebAPI implementation repo -# https://github.com/OCR-D/ocrd-webapi-implementation/tree/main/ocrd_webapi/rabbitmq - __all__ = [ 'RMQConsumer', 'RMQConnector', diff --git a/ocrd_network/ocrd_network/web_api/__init__.py b/ocrd_network/ocrd_network/web_api/__init__.py deleted file mode 100644 index f3265b436..000000000 --- a/ocrd_network/ocrd_network/web_api/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# This web_api package is supposed to contain the code that is currently available under: -# https://github.com/OCR-D/ocrd-webapi-implementation diff --git a/ocrd_network/setup.py b/ocrd_network/setup.py index eca56aa32..ac8099a6b 100644 --- a/ocrd_network/setup.py +++ b/ocrd_network/setup.py @@ -7,6 +7,7 @@ install_requires.append('ocrd_utils == %s' % VERSION) install_requires.append('ocrd_validators == %s' % VERSION) +# TODO: This needs to be revisited! Seems badly adapted from ocrd/setup.py setup( name='ocrd_network', version=VERSION, From 494c0abc6dcb6fcb819d702a25e4f0152e565dcc Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Mon, 20 Mar 2023 10:16:35 +0100 Subject: [PATCH 167/226] chdir to ws as in #987 --- tests/processor/test_ocrd_dummy.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/processor/test_ocrd_dummy.py b/tests/processor/test_ocrd_dummy.py index 532f6e668..c98a6d481 100644 --- a/tests/processor/test_ocrd_dummy.py +++ b/tests/processor/test_ocrd_dummy.py @@ -2,6 +2,7 @@ # pylint: disable=invalid-name,line-too-long from io import BytesIO +import os from pathlib import Path from PIL import Image @@ -18,6 +19,7 @@ class TestDummyProcessor(TestCase): def test_copies_ok(self): with copy_of_directory(assets.url_of('SBB0000F29300010000/data')) as wsdir: workspace = Workspace(Resolver(), wsdir) + os.chdir(workspace.directory) input_files = workspace.mets.find_all_files(fileGrp='OCR-D-IMG') self.assertEqual(len(input_files), 3) output_files = workspace.mets.find_all_files(fileGrp='OUTPUT') @@ -53,6 +55,7 @@ def test_copies_ok(self): def test_copy_file_false(tmpdir): workspace = Resolver().workspace_from_nothing(directory=tmpdir) + os.chdir(workspace.directory) for i in range(10): pil_image = Image.new('RGB', (100, 100)) bhandle = BytesIO() From d784a75f5210b1539713a2dc501a7035ba42035d Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Mon, 20 Mar 2023 10:18:18 +0100 Subject: [PATCH 168/226] Call getcwd before get_processor --- ocrd/ocrd/processor/helpers.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ocrd/ocrd/processor/helpers.py b/ocrd/ocrd/processor/helpers.py index 7fe560836..173f308f1 100644 --- a/ocrd/ocrd/processor/helpers.py +++ b/ocrd/ocrd/processor/helpers.py @@ -84,6 +84,7 @@ def run_processor( log = getLogger('ocrd.processor.helpers.run_processor') log.debug("Running processor %s", processorClass) + old_cwd = getcwd() processor = get_processor( processor_class=processorClass, parameter=parameter, @@ -94,8 +95,6 @@ def run_processor( output_file_grp=output_file_grp, instance_caching=instance_caching ) - - old_cwd = getcwd() chdir(processor.workspace.directory) ocrd_tool = processor.ocrd_tool From 43a566015109fe3d23a8f8a272fcf692fa8f9c8b Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Mon, 20 Mar 2023 11:35:50 +0100 Subject: [PATCH 169/226] Set ws outside the constructor --- ocrd/ocrd/processor/helpers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ocrd/ocrd/processor/helpers.py b/ocrd/ocrd/processor/helpers.py index 173f308f1..547edd6ed 100644 --- a/ocrd/ocrd/processor/helpers.py +++ b/ocrd/ocrd/processor/helpers.py @@ -88,13 +88,14 @@ def run_processor( processor = get_processor( processor_class=processorClass, parameter=parameter, - workspace=workspace, + workspace=None, ocrd_tool=ocrd_tool, page_id=page_id, input_file_grp=input_file_grp, output_file_grp=output_file_grp, instance_caching=instance_caching ) + processor.workspace = workspace chdir(processor.workspace.directory) ocrd_tool = processor.ocrd_tool From e396b2b02e69397be3ad184674965cedbfb73677 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Tue, 21 Mar 2023 13:38:28 +0100 Subject: [PATCH 170/226] Modify OcrdResultMessage --- ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py b/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py index a270dbecb..3615ad2c7 100644 --- a/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py +++ b/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py @@ -86,13 +86,17 @@ def from_job(job: Job) -> OcrdProcessingMessage: class OcrdResultMessage: - def __init__(self, job_id: str, status: str, workspace_id: str, path_to_mets: str) -> None: + def __init__(self, job_id: str, status: str, workspace_id: Optional[str] = None, + path_to_mets: Optional[str] = None) -> None: self.job_id = job_id self.status = status # Either of these two below self.workspace_id = workspace_id self.path_to_mets = path_to_mets + if not (workspace_id or path_to_mets): + raise ValueError('Either `workspace_id` or `path_to_mets` must be set') + @staticmethod def encode_yml(ocrd_result_message: OcrdResultMessage) -> bytes: """convert OcrdResultMessage to yml From 57d47913de1b9ebd0f2538a2e328447df7bcada1 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Tue, 21 Mar 2023 13:58:49 +0100 Subject: [PATCH 171/226] Move check before assignments --- ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py b/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py index 3615ad2c7..01dd7995f 100644 --- a/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py +++ b/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py @@ -88,15 +88,15 @@ def from_job(job: Job) -> OcrdProcessingMessage: class OcrdResultMessage: def __init__(self, job_id: str, status: str, workspace_id: Optional[str] = None, path_to_mets: Optional[str] = None) -> None: + if not (workspace_id or path_to_mets): + raise ValueError('Either `workspace_id` or `path_to_mets` must be set') + self.job_id = job_id self.status = status # Either of these two below self.workspace_id = workspace_id self.path_to_mets = path_to_mets - if not (workspace_id or path_to_mets): - raise ValueError('Either `workspace_id` or `path_to_mets` must be set') - @staticmethod def encode_yml(ocrd_result_message: OcrdResultMessage) -> bytes: """convert OcrdResultMessage to yml From 7288d6d0edc595b1f8902a88586b307971526860 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Tue, 21 Mar 2023 17:05:27 +0100 Subject: [PATCH 172/226] Remove pymongo, add call_sync wrapper --- ocrd_network/ocrd_network/database.py | 34 ++++++++++++++++++- .../ocrd_network/processing_worker.py | 23 +++++-------- .../rabbitmq_utils/ocrd_messages.py | 2 +- ocrd_network/ocrd_network/utils.py | 14 ++++++++ 4 files changed, 56 insertions(+), 17 deletions(-) create mode 100644 ocrd_network/ocrd_network/utils.py diff --git a/ocrd_network/ocrd_network/database.py b/ocrd_network/ocrd_network/database.py index 369f13426..b85fc3429 100644 --- a/ocrd_network/ocrd_network/database.py +++ b/ocrd_network/ocrd_network/database.py @@ -14,9 +14,11 @@ """ from beanie import init_beanie from motor.motor_asyncio import AsyncIOMotorClient +from typing import Any -from ocrd_network.models.job import Job +from ocrd_network.models.job import Job, JobOutput, StateEnum from ocrd_network.models.workspace import Workspace +from ocrd_network.utils import call_sync async def initiate_database(db_url: str): @@ -25,3 +27,33 @@ async def initiate_database(db_url: str): database=client.get_default_database(default='ocrd'), document_models=[Job, Workspace] ) + + +@call_sync +async def sync_initiate_database(db_url: str): + await initiate_database(db_url) + + +async def set_processing_job_state(job_id: Any, job_state: StateEnum): + job = await Job.get(job_id) + if not job: + raise ValueError(f'Processing job with id "{job_id}" not available in the DB.') + job.state = job_state + await job.save() + + +@call_sync +async def sync_set_processing_job_state(job_id: Any, job_state: StateEnum): + await set_processing_job_state(job_id, job_state) + + +async def get_processing_job_state(job_id: Any) -> StateEnum: + job = await Job.get(job_id) + if not job: + raise ValueError(f'Processing job with id "{job_id}" not available in the DB.') + return job.state + + +@call_sync +async def sync_get_processing_job_state(job_id: Any) -> StateEnum: + return await get_processing_job_state(job_id) diff --git a/ocrd_network/ocrd_network/processing_worker.py b/ocrd_network/ocrd_network/processing_worker.py index 4c96010cb..cb263ae25 100644 --- a/ocrd_network/ocrd_network/processing_worker.py +++ b/ocrd_network/ocrd_network/processing_worker.py @@ -15,13 +15,13 @@ import pika.spec import pika.adapters.blocking_connection -import pymongo from ocrd import Resolver from ocrd_utils import getLogger -from ocrd.processor.helpers import ( - run_cli, - run_processor +from ocrd.processor.helpers import run_cli, run_processor +from ocrd_network.database import ( + sync_initiate_database, + sync_set_processing_job_state ) from ocrd_network.helpers import ( verify_database_uri, @@ -34,6 +34,7 @@ RMQConsumer, RMQPublisher ) +from ocrd_network.utils import call_sync try: # This env variable must be set before importing from Keras @@ -56,6 +57,7 @@ def __init__(self, rabbitmq_addr, mongodb_addr, processor_name, ocrd_tool: dict, file_handler.setFormatter(logging.Formatter(logging_format)) file_handler.setLevel(logging.DEBUG) self.log.addHandler(file_handler) + sync_initiate_database(mongodb_addr) # Database client try: self.db_url = verify_database_uri(mongodb_addr) @@ -198,7 +200,7 @@ def process_message(self, processing_message: OcrdProcessingMessage) -> None: parameter = processing_message.parameters job_id = processing_message.job_id - self.set_job_state(job_id, StateEnum.running) + sync_set_processing_job_state(job_id=job_id, job_state=StateEnum.running) if self.processor_class: self.log.debug(f'Invoking the pythonic processor: {self.processor_name}') return_status = self.run_processor_from_worker( @@ -220,7 +222,7 @@ def process_message(self, processing_message: OcrdProcessingMessage) -> None: parameter=parameter ) job_state = StateEnum.success if return_status else StateEnum.failed - self.set_job_state(job_id, job_state) + sync_set_processing_job_state(job_id=job_id, job_state=job_state) # If the result_queue field is set, send the job status to a result queue if processing_message.result_queue: @@ -306,12 +308,3 @@ def run_cli_from_worker( else: self.log.debug(f'{executable} exited with success.') return True - - def set_job_state(self, job_id: Any, state: StateEnum): - """Set the job status in mongodb - """ - # TODO: the way to interact with mongodb needs to be thought about. Beanie seems not - # suitable as the worker is not async, thus using pymongo here - with pymongo.MongoClient(self.db_url) as client: - db = client['ocrd'] - db.Job.update_one({'_id': job_id}, {'$set': {'state': state}}, upsert=False) diff --git a/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py b/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py index 01dd7995f..850f2903f 100644 --- a/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py +++ b/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py @@ -90,7 +90,7 @@ def __init__(self, job_id: str, status: str, workspace_id: Optional[str] = None, path_to_mets: Optional[str] = None) -> None: if not (workspace_id or path_to_mets): raise ValueError('Either `workspace_id` or `path_to_mets` must be set') - + self.job_id = job_id self.status = status # Either of these two below diff --git a/ocrd_network/ocrd_network/utils.py b/ocrd_network/ocrd_network/utils.py new file mode 100644 index 000000000..9285ec913 --- /dev/null +++ b/ocrd_network/ocrd_network/utils.py @@ -0,0 +1,14 @@ +import functools + + +# Based on: https://gist.github.com/phizaz/20c36c6734878c6ec053245a477572ec +def call_sync(func): + import asyncio + + @functools.wraps(func) + def func_wrapper(*args, **kwargs): + result = func(*args, **kwargs) + if asyncio.iscoroutine(result): + return asyncio.get_event_loop().run_until_complete(result) + return result + return func_wrapper From 5e6d0ade7524f4278880519449faee9845c7bcc0 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Tue, 21 Mar 2023 17:12:03 +0100 Subject: [PATCH 173/226] Refactor helpers -> utils --- ocrd_network/ocrd_network/helpers.py | 34 ----------------- ocrd_network/ocrd_network/param_validators.py | 2 +- .../ocrd_network/processing_worker.py | 9 ++--- ocrd_network/ocrd_network/utils.py | 38 ++++++++++++++++++- 4 files changed, 41 insertions(+), 42 deletions(-) delete mode 100644 ocrd_network/ocrd_network/helpers.py diff --git a/ocrd_network/ocrd_network/helpers.py b/ocrd_network/ocrd_network/helpers.py deleted file mode 100644 index b5027c547..000000000 --- a/ocrd_network/ocrd_network/helpers.py +++ /dev/null @@ -1,34 +0,0 @@ -from re import match as re_match -from pika import URLParameters -from pymongo import uri_parser as mongo_uri_parser - - -def verify_database_uri(mongodb_address: str) -> str: - try: - # perform validation check - mongo_uri_parser.parse_uri(uri=mongodb_address, validate=True) - except Exception as error: - raise ValueError(f"The database address '{mongodb_address}' is in wrong format, {error}") - return mongodb_address - - -def verify_and_parse_mq_uri(rabbitmq_address: str): - """ - Check the full list of available parameters in the docs here: - https://pika.readthedocs.io/en/stable/_modules/pika/connection.html#URLParameters - """ - - uri_pattern = r"^(?:([^:\/?#\s]+):\/{2})?(?:([^@\/?#\s]+)@)?([^\/?#\s]+)?(?:\/([^?#\s]*))?(?:[?]([^#\s]+))?\S*$" - match = re_match(pattern=uri_pattern, string=rabbitmq_address) - if not match: - raise ValueError(f"The message queue server address is in wrong format: '{rabbitmq_address}'") - url_params = URLParameters(rabbitmq_address) - - parsed_data = { - 'username': url_params.credentials.username, - 'password': url_params.credentials.password, - 'host': url_params.host, - 'port': url_params.port, - 'vhost': url_params.virtual_host - } - return parsed_data diff --git a/ocrd_network/ocrd_network/param_validators.py b/ocrd_network/ocrd_network/param_validators.py index 3b3483386..7cda9feaf 100644 --- a/ocrd_network/ocrd_network/param_validators.py +++ b/ocrd_network/ocrd_network/param_validators.py @@ -1,6 +1,6 @@ from click import ParamType -from ocrd_network.helpers import ( +from ocrd_network.utils import ( verify_database_uri, verify_and_parse_mq_uri ) diff --git a/ocrd_network/ocrd_network/processing_worker.py b/ocrd_network/ocrd_network/processing_worker.py index cb263ae25..1ed255863 100644 --- a/ocrd_network/ocrd_network/processing_worker.py +++ b/ocrd_network/ocrd_network/processing_worker.py @@ -23,10 +23,6 @@ sync_initiate_database, sync_set_processing_job_state ) -from ocrd_network.helpers import ( - verify_database_uri, - verify_and_parse_mq_uri -) from ocrd_network.models.job import StateEnum from ocrd_network.rabbitmq_utils import ( OcrdProcessingMessage, @@ -34,7 +30,10 @@ RMQConsumer, RMQPublisher ) -from ocrd_network.utils import call_sync +from ocrd_network.utils import ( + verify_database_uri, + verify_and_parse_mq_uri +) try: # This env variable must be set before importing from Keras diff --git a/ocrd_network/ocrd_network/utils.py b/ocrd_network/ocrd_network/utils.py index 9285ec913..245fde241 100644 --- a/ocrd_network/ocrd_network/utils.py +++ b/ocrd_network/ocrd_network/utils.py @@ -1,14 +1,48 @@ -import functools +from functools import wraps +from re import match as re_match +from pika import URLParameters +from pymongo import uri_parser as mongo_uri_parser # Based on: https://gist.github.com/phizaz/20c36c6734878c6ec053245a477572ec def call_sync(func): import asyncio - @functools.wraps(func) + @wraps(func) def func_wrapper(*args, **kwargs): result = func(*args, **kwargs) if asyncio.iscoroutine(result): return asyncio.get_event_loop().run_until_complete(result) return result return func_wrapper + + +def verify_database_uri(mongodb_address: str) -> str: + try: + # perform validation check + mongo_uri_parser.parse_uri(uri=mongodb_address, validate=True) + except Exception as error: + raise ValueError(f"The database address '{mongodb_address}' is in wrong format, {error}") + return mongodb_address + + +def verify_and_parse_mq_uri(rabbitmq_address: str): + """ + Check the full list of available parameters in the docs here: + https://pika.readthedocs.io/en/stable/_modules/pika/connection.html#URLParameters + """ + + uri_pattern = r"^(?:([^:\/?#\s]+):\/{2})?(?:([^@\/?#\s]+)@)?([^\/?#\s]+)?(?:\/([^?#\s]*))?(?:[?]([^#\s]+))?\S*$" + match = re_match(pattern=uri_pattern, string=rabbitmq_address) + if not match: + raise ValueError(f"The message queue server address is in wrong format: '{rabbitmq_address}'") + url_params = URLParameters(rabbitmq_address) + + parsed_data = { + 'username': url_params.credentials.username, + 'password': url_params.credentials.password, + 'host': url_params.host, + 'port': url_params.port, + 'vhost': url_params.virtual_host + } + return parsed_data From 66851d4d54e215faffaed6d0ee4a972d604ce4ba Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Tue, 21 Mar 2023 18:59:38 +0100 Subject: [PATCH 174/226] Add callback_url, improve models --- ocrd_network/ocrd_network/models/job.py | 4 + .../ocrd_network/processing_server.py | 21 ++++- .../ocrd_network/processing_worker.py | 82 +++++++++++-------- .../rabbitmq_utils/ocrd_messages.py | 11 ++- 4 files changed, 80 insertions(+), 38 deletions(-) diff --git a/ocrd_network/ocrd_network/models/job.py b/ocrd_network/ocrd_network/models/job.py index 116fe9e86..f42b74a87 100644 --- a/ocrd_network/ocrd_network/models/job.py +++ b/ocrd_network/ocrd_network/models/job.py @@ -23,6 +23,8 @@ class JobInput(BaseModel): output_file_grps: Optional[List[str]] page_id: Optional[str] = None parameters: dict = {} # Always set to empty dict when None, otherwise it fails ocr-d-validation + result_queue: Optional[str] = None + callback_url: Optional[str] = None class Config: schema_extra = { @@ -59,6 +61,8 @@ class Job(Document): output_file_grps: Optional[List[str]] page_id: Optional[str] parameters: Optional[dict] + result_queue_name: Optional[str] + callback_url: Optional[str] start_time: Optional[datetime] end_time: Optional[datetime] diff --git a/ocrd_network/ocrd_network/processing_server.py b/ocrd_network/ocrd_network/processing_server.py index d21345b2e..4619f1d9c 100644 --- a/ocrd_network/ocrd_network/processing_server.py +++ b/ocrd_network/ocrd_network/processing_server.py @@ -255,8 +255,25 @@ async def push_processor_job(self, processor_name: str, data: JobInput) -> JobOu ) data.path = workspace.workspace_mets_path - job = Job(**data.dict(exclude_unset=True, exclude_none=True), processor_name=processor_name, - state=StateEnum.queued) + result_queue_name = None + if data.result_queue and data.result_queue.lower() in ["true", "t", "yes", "1"]: + result_queue_name = processor_name + '-result' + with open('/home/mm/Desktop/a.txt', 'w+') as file_debugger: + file_debugger.write(f'The results for "{processor_name}" will ' + f'be posted to queue with id "{processor_name}-queue"') + self.log.info(f'The results for "{processor_name}" will ' + f'be posted to queue with id "{processor_name}-queue"') + else: + with open('/home/mm/Desktop/a.txt', 'w+') as file_debugger: + file_debugger.write('The result queue is not set') + self.log.info('The result queue is not set') + + job = Job( + **data.dict(exclude_unset=True, exclude_none=True), + processor_name=processor_name, + state=StateEnum.queued, + result_queue_name=result_queue_name + ) await job.insert() processing_message = OcrdProcessingMessage.from_job(job) encoded_processing_message = OcrdProcessingMessage.encode_yml(processing_message) diff --git a/ocrd_network/ocrd_network/processing_worker.py b/ocrd_network/ocrd_network/processing_worker.py index 1ed255863..4df0dc777 100644 --- a/ocrd_network/ocrd_network/processing_worker.py +++ b/ocrd_network/ocrd_network/processing_worker.py @@ -11,6 +11,7 @@ import json import logging from os import environ, getpid +import requests from typing import Any, List import pika.spec @@ -161,7 +162,7 @@ def on_consumed_message( channel.basic_nack(delivery_tag=delivery_tag, multiple=False, requeue=False) raise Exception(f'Failed to process processing message with tag: {delivery_tag}, reason: {e}') - self.log.info(f'Successfully processed message ') + self.log.info(f'Successfully processed RabbitMQ message') self.log.debug(f'Acking message with tag: {delivery_tag}') channel.basic_ack(delivery_tag=delivery_tag, multiple=False) @@ -188,63 +189,80 @@ def process_message(self, processing_message: OcrdProcessingMessage) -> None: # This can be path if invoking `run_processor` # but must be ocrd.Workspace if invoking `run_cli`. - workspace_path = processing_message.path_to_mets - + path_to_mets = processing_message.path_to_mets # Build the workspace from the workspace_path - workspace = Resolver().workspace_from_url(workspace_path) + workspace = Resolver().workspace_from_url(path_to_mets) - page_id = processing_message.page_id - input_file_grps = processing_message.input_file_grps - output_file_grps = processing_message.output_file_grps - parameter = processing_message.parameters job_id = processing_message.job_id - sync_set_processing_job_state(job_id=job_id, job_state=StateEnum.running) if self.processor_class: self.log.debug(f'Invoking the pythonic processor: {self.processor_name}') return_status = self.run_processor_from_worker( processor_class=self.processor_class, workspace=workspace, - page_id=page_id, - input_file_grps=input_file_grps, - output_file_grps=output_file_grps, - parameter=parameter + page_id=processing_message.page_id, + input_file_grps=processing_message.input_file_grps, + output_file_grps=processing_message.output_file_grps, + parameter=processing_message.parameters ) else: self.log.debug(f'Invoking the cli: {self.processor_name}') return_status = self.run_cli_from_worker( executable=self.processor_name, workspace=workspace, - page_id=page_id, - input_file_grps=input_file_grps, - output_file_grps=output_file_grps, - parameter=parameter + page_id=processing_message.page_id, + input_file_grps=processing_message.input_file_grps, + output_file_grps=processing_message.output_file_grps, + parameter=processing_message.parameters ) job_state = StateEnum.success if return_status else StateEnum.failed sync_set_processing_job_state(job_id=job_id, job_state=job_state) - # If the result_queue field is set, send the job status to a result queue - if processing_message.result_queue: - if self.rmq_publisher is None: - self.connect_publisher() + result_queue_name = processing_message.result_queue_name + callback_url = processing_message.callback_url - # create_queue method is idempotent - nothing happens if - # a queue with the specified name already exists - self.rmq_publisher.create_queue(queue_name=processing_message.result_queue) - self.log.info(f'Publishing result message to queue: {processing_message.result_queue}') + if result_queue_name or callback_url: result_message = OcrdResultMessage( job_id=str(job_id), status=job_state.value, # Either path_to_mets or workspace_id must be set (mutually exclusive) - path_to_mets=processing_message.path_to_mets, + path_to_mets=path_to_mets, workspace_id=None ) - self.log.debug(f'Result message: {result_message}') - encoded_result_message = OcrdResultMessage.encode_yml(result_message) - self.rmq_publisher.publish_to_queue( - queue_name=processing_message.result_queue, - message=encoded_result_message - ) + self.log.info(f'Result message: {result_message}') + + # If the result_queue field is set, send the result message to a result queue + if result_queue_name: + self.publish_to_result_queue(result_queue_name, result_message) + + # If the callback_url field is set, post the result message to a callback url + if callback_url: + self.post_to_callback_url(processing_message.callback_url, result_message) + + def publish_to_result_queue(self, result_queue: str, result_message: OcrdResultMessage): + if self.rmq_publisher is None: + self.connect_publisher() + # create_queue method is idempotent - nothing happens if + # a queue with the specified name already exists + self.rmq_publisher.create_queue(queue_name=result_queue) + self.log.info(f'Publishing result message to queue: {result_queue}') + encoded_result_message = OcrdResultMessage.encode_yml(result_message) + self.rmq_publisher.publish_to_queue( + queue_name=result_queue, + message=encoded_result_message + ) + + def post_to_callback_url(self, callback_url: str, result_message: OcrdResultMessage): + self.log.info(f'Posting result message to callback_url "{callback_url}"') + headers = {"accept": "application/json"} + json_data = { + "job_id": result_message.job_id, + "status": result_message.status, + "path_to_mets": result_message.path_to_mets, + "workspace_id": result_message.workspace_id + } + response = requests.post(url=callback_url, headers=headers, json=json_data) + self.log.info(f'Response from callback_url "{response}"') def run_processor_from_worker( self, diff --git a/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py b/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py index 850f2903f..820192ea6 100644 --- a/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py +++ b/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py @@ -1,4 +1,3 @@ -# Check here for more details: Message structure #139 from __future__ import annotations from datetime import datetime from typing import Any, Dict, List, Optional @@ -19,6 +18,7 @@ def __init__( page_id: str = None, parameters: Dict[str, Any] = None, result_queue_name: str = None, + callback_url: str = None ) -> None: if not job_id: raise ValueError('job_id must be set') @@ -41,10 +41,10 @@ def __init__( self.output_file_grps = output_file_grps # e.g., "PHYS_0005..PHYS_0010" will process only pages between 5-10 self.page_id = page_id if page_id else None - # e.g., "ocrd-cis-ocropy-binarize-result" - self.result_queue = result_queue_name if result_queue_name else (self.processor_name + "-result") # processor parameters self.parameters = parameters if parameters else None + self.result_queue_name = result_queue_name + self.callback_url = callback_url self.created_time = created_time @staticmethod @@ -69,7 +69,8 @@ def decode_yml(ocrd_processing_message: bytes) -> OcrdProcessingMessage: output_file_grps=data.get('output_file_grps', None), page_id=data.get('page_id', None), parameters=data.get('parameters', None), - result_queue_name=data.get('result_queue', None), + result_queue_name=data.get('result_queue_name', None), + callback_url=data.get('callback_url', None) ) @staticmethod @@ -82,6 +83,8 @@ def from_job(job: Job) -> OcrdProcessingMessage: output_file_grps=job.output_file_grps, page_id=job.page_id, parameters=job.parameters, + result_queue_name=job.result_queue_name, + callback_url=job.callback_url ) From 07828a68a08a3782e0e9dc8ff7cd3f375b7a2385 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Tue, 21 Mar 2023 19:16:09 +0100 Subject: [PATCH 175/226] Remove the debugging file --- ocrd_network/ocrd_network/processing_server.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/ocrd_network/ocrd_network/processing_server.py b/ocrd_network/ocrd_network/processing_server.py index 4619f1d9c..c246e8601 100644 --- a/ocrd_network/ocrd_network/processing_server.py +++ b/ocrd_network/ocrd_network/processing_server.py @@ -258,14 +258,9 @@ async def push_processor_job(self, processor_name: str, data: JobInput) -> JobOu result_queue_name = None if data.result_queue and data.result_queue.lower() in ["true", "t", "yes", "1"]: result_queue_name = processor_name + '-result' - with open('/home/mm/Desktop/a.txt', 'w+') as file_debugger: - file_debugger.write(f'The results for "{processor_name}" will ' - f'be posted to queue with id "{processor_name}-queue"') self.log.info(f'The results for "{processor_name}" will ' f'be posted to queue with id "{processor_name}-queue"') else: - with open('/home/mm/Desktop/a.txt', 'w+') as file_debugger: - file_debugger.write('The result queue is not set') self.log.info('The result queue is not set') job = Job( From 0eb74b97f32a29560c8351f388752ecf49e57625 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Tue, 21 Mar 2023 20:00:44 +0100 Subject: [PATCH 176/226] Remove result queue related info logs --- ocrd_network/ocrd_network/processing_server.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ocrd_network/ocrd_network/processing_server.py b/ocrd_network/ocrd_network/processing_server.py index c246e8601..6a55add03 100644 --- a/ocrd_network/ocrd_network/processing_server.py +++ b/ocrd_network/ocrd_network/processing_server.py @@ -258,10 +258,6 @@ async def push_processor_job(self, processor_name: str, data: JobInput) -> JobOu result_queue_name = None if data.result_queue and data.result_queue.lower() in ["true", "t", "yes", "1"]: result_queue_name = processor_name + '-result' - self.log.info(f'The results for "{processor_name}" will ' - f'be posted to queue with id "{processor_name}-queue"') - else: - self.log.info('The result queue is not set') job = Job( **data.dict(exclude_unset=True, exclude_none=True), From da2e7e1b34919f86b6fe7f057c4fd966cbbe66b5 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Wed, 22 Mar 2023 13:51:52 +0100 Subject: [PATCH 177/226] Initiate DB client after db addr verification --- ocrd_network/ocrd_network/processing_worker.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/ocrd_network/ocrd_network/processing_worker.py b/ocrd_network/ocrd_network/processing_worker.py index 4df0dc777..bdd6ad03e 100644 --- a/ocrd_network/ocrd_network/processing_worker.py +++ b/ocrd_network/ocrd_network/processing_worker.py @@ -57,7 +57,6 @@ def __init__(self, rabbitmq_addr, mongodb_addr, processor_name, ocrd_tool: dict, file_handler.setFormatter(logging.Formatter(logging_format)) file_handler.setLevel(logging.DEBUG) self.log.addHandler(file_handler) - sync_initiate_database(mongodb_addr) # Database client try: self.db_url = verify_database_uri(mongodb_addr) @@ -73,19 +72,16 @@ def __init__(self, rabbitmq_addr, mongodb_addr, processor_name, ocrd_tool: dict, except ValueError as e: raise ValueError(e) + sync_initiate_database(mongodb_addr) # Database client self.ocrd_tool = ocrd_tool - # The str name of the OCR-D processor instance to be started self.processor_name = processor_name - # The processor class to be used to instantiate the processor # Think of this as a func pointer to the constructor of the respective OCR-D processor self.processor_class = processor_class - # Gets assigned when `connect_consumer` is called on the worker object # Used to consume OcrdProcessingMessage from the queue with name {processor_name} self.rmq_consumer = None - # Gets assigned when the `connect_publisher` is called on the worker object # The publisher is connected when the `result_queue` field of the OcrdProcessingMessage is set for first time # Used to publish OcrdResultMessage type message to the queue with name {processor_name}-result From 31d3a085076041645e7b93993cdb660558dc60d6 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Wed, 22 Mar 2023 13:53:04 +0100 Subject: [PATCH 178/226] Improve exception message --- ocrd_network/ocrd_network/processing_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ocrd_network/ocrd_network/processing_server.py b/ocrd_network/ocrd_network/processing_server.py index 6a55add03..a142b0045 100644 --- a/ocrd_network/ocrd_network/processing_server.py +++ b/ocrd_network/ocrd_network/processing_server.py @@ -244,7 +244,7 @@ async def push_processor_job(self, processor_name: str, data: JobInput) -> JobOu if bool(data.path) == bool(data.workspace_id): raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail="Arguments 'path' and 'workspace_id' are mutually exclusive" + detail="Either 'path' or 'workspace_id' must be provided, but not both" ) elif data.workspace_id: workspace = await Workspace.get(data.workspace_id) From b82446c42223e1e7ba31420d44e282b84c710a66 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Wed, 22 Mar 2023 13:55:18 +0100 Subject: [PATCH 179/226] refactor result_queue, callback_url checks --- ocrd_network/ocrd_network/processing_worker.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ocrd_network/ocrd_network/processing_worker.py b/ocrd_network/ocrd_network/processing_worker.py index bdd6ad03e..a0a58ec1c 100644 --- a/ocrd_network/ocrd_network/processing_worker.py +++ b/ocrd_network/ocrd_network/processing_worker.py @@ -227,13 +227,13 @@ def process_message(self, processing_message: OcrdProcessingMessage) -> None: ) self.log.info(f'Result message: {result_message}') - # If the result_queue field is set, send the result message to a result queue - if result_queue_name: - self.publish_to_result_queue(result_queue_name, result_message) + # If the result_queue field is set, send the result message to a result queue + if result_queue_name: + self.publish_to_result_queue(result_queue_name, result_message) - # If the callback_url field is set, post the result message to a callback url - if callback_url: - self.post_to_callback_url(processing_message.callback_url, result_message) + # If the callback_url field is set, post the result message to a callback url + if callback_url: + self.post_to_callback_url(processing_message.callback_url, result_message) def publish_to_result_queue(self, result_queue: str, result_message: OcrdResultMessage): if self.rmq_publisher is None: From 0f0a6d3d71abf2d2d1125c36388254dea0034871 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Wed, 22 Mar 2023 14:00:01 +0100 Subject: [PATCH 180/226] Set workspace_id when available --- ocrd_network/ocrd_network/processing_worker.py | 4 ++-- ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py | 4 ---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/ocrd_network/ocrd_network/processing_worker.py b/ocrd_network/ocrd_network/processing_worker.py index a0a58ec1c..4c0712816 100644 --- a/ocrd_network/ocrd_network/processing_worker.py +++ b/ocrd_network/ocrd_network/processing_worker.py @@ -221,9 +221,9 @@ def process_message(self, processing_message: OcrdProcessingMessage) -> None: result_message = OcrdResultMessage( job_id=str(job_id), status=job_state.value, - # Either path_to_mets or workspace_id must be set (mutually exclusive) path_to_mets=path_to_mets, - workspace_id=None + # May not be always available + workspace_id=processing_message.workspace_id ) self.log.info(f'Result message: {result_message}') diff --git a/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py b/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py index 820192ea6..4666d05a3 100644 --- a/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py +++ b/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py @@ -91,12 +91,8 @@ def from_job(job: Job) -> OcrdProcessingMessage: class OcrdResultMessage: def __init__(self, job_id: str, status: str, workspace_id: Optional[str] = None, path_to_mets: Optional[str] = None) -> None: - if not (workspace_id or path_to_mets): - raise ValueError('Either `workspace_id` or `path_to_mets` must be set') - self.job_id = job_id self.status = status - # Either of these two below self.workspace_id = workspace_id self.path_to_mets = path_to_mets From 44b6b6e5012fdc45f271088c95e69200a1201ed3 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Wed, 22 Mar 2023 14:16:49 +0100 Subject: [PATCH 181/226] minor variable and comment improvements --- ocrd/ocrd/cli/processing_server.py | 2 +- ocrd/ocrd/cli/processing_worker.py | 2 +- ocrd_network/ocrd_network/processing_worker.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ocrd/ocrd/cli/processing_server.py b/ocrd/ocrd/cli/processing_server.py index 3baa05607..a65e02a71 100644 --- a/ocrd/ocrd/cli/processing_server.py +++ b/ocrd/ocrd/cli/processing_server.py @@ -1,7 +1,7 @@ """ OCR-D CLI: start the processing server -.. click:: ocrd.cli.processing_server:zip_cli +.. click:: ocrd.cli.processing_server:processing_server_cli :prog: ocrd processing-server :nested: full """ diff --git a/ocrd/ocrd/cli/processing_worker.py b/ocrd/ocrd/cli/processing_worker.py index a823062af..2b1914874 100644 --- a/ocrd/ocrd/cli/processing_worker.py +++ b/ocrd/ocrd/cli/processing_worker.py @@ -1,7 +1,7 @@ """ OCR-D CLI: start the processing worker -.. click:: ocrd.cli.processing_worker:zip_cli +.. click:: ocrd.cli.processing_worker:processing_worker_cli :prog: ocrd processing-worker :nested: full """ diff --git a/ocrd_network/ocrd_network/processing_worker.py b/ocrd_network/ocrd_network/processing_worker.py index 4c0712816..d8edc74fe 100644 --- a/ocrd_network/ocrd_network/processing_worker.py +++ b/ocrd_network/ocrd_network/processing_worker.py @@ -59,8 +59,8 @@ def __init__(self, rabbitmq_addr, mongodb_addr, processor_name, ocrd_tool: dict, self.log.addHandler(file_handler) try: - self.db_url = verify_database_uri(mongodb_addr) - self.log.debug(f'Verified MongoDB URL: {self.db_url}') + verify_database_uri(mongodb_addr) + self.log.debug(f'Verified MongoDB URL: {mongodb_addr}') rmq_data = verify_and_parse_mq_uri(rabbitmq_addr) self.rmq_username = rmq_data['username'] self.rmq_password = rmq_data['password'] From 8e0d7d8a8d292b6644a270e05e001911fd167dc9 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Wed, 22 Mar 2023 14:47:32 +0100 Subject: [PATCH 182/226] fix result queue related things --- ocrd_network/ocrd_network/models/job.py | 2 +- .../ocrd_network/processing_server.py | 22 ++++--------------- .../ocrd_network/processing_worker.py | 2 +- .../rabbitmq_utils/ocrd_messages.py | 2 +- 4 files changed, 7 insertions(+), 21 deletions(-) diff --git a/ocrd_network/ocrd_network/models/job.py b/ocrd_network/ocrd_network/models/job.py index f42b74a87..3815a4d81 100644 --- a/ocrd_network/ocrd_network/models/job.py +++ b/ocrd_network/ocrd_network/models/job.py @@ -23,7 +23,7 @@ class JobInput(BaseModel): output_file_grps: Optional[List[str]] page_id: Optional[str] = None parameters: dict = {} # Always set to empty dict when None, otherwise it fails ocr-d-validation - result_queue: Optional[str] = None + result_queue_name: Optional[str] = None callback_url: Optional[str] = None class Config: diff --git a/ocrd_network/ocrd_network/processing_server.py b/ocrd_network/ocrd_network/processing_server.py index a142b0045..d3e2c1629 100644 --- a/ocrd_network/ocrd_network/processing_server.py +++ b/ocrd_network/ocrd_network/processing_server.py @@ -8,21 +8,12 @@ from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse -from ocrd_utils import ( - getLogger, - get_ocrd_tool_json, -) -from ocrd_validators import ( - ParameterValidator, - ProcessingServerValidator -) +from ocrd_utils import getLogger, get_ocrd_tool_json +from ocrd_validators import ParameterValidator, ProcessingServerValidator from ocrd_network.database import initiate_database from ocrd_network.deployer import Deployer from ocrd_network.deployment_config import ProcessingServerConfig -from ocrd_network.rabbitmq_utils import ( - RMQPublisher, - OcrdProcessingMessage -) +from ocrd_network.rabbitmq_utils import RMQPublisher, OcrdProcessingMessage from ocrd_network.models.job import ( Job, JobInput, @@ -255,15 +246,10 @@ async def push_processor_job(self, processor_name: str, data: JobInput) -> JobOu ) data.path = workspace.workspace_mets_path - result_queue_name = None - if data.result_queue and data.result_queue.lower() in ["true", "t", "yes", "1"]: - result_queue_name = processor_name + '-result' - job = Job( **data.dict(exclude_unset=True, exclude_none=True), processor_name=processor_name, - state=StateEnum.queued, - result_queue_name=result_queue_name + state=StateEnum.queued ) await job.insert() processing_message = OcrdProcessingMessage.from_job(job) diff --git a/ocrd_network/ocrd_network/processing_worker.py b/ocrd_network/ocrd_network/processing_worker.py index d8edc74fe..3c2cc1dbc 100644 --- a/ocrd_network/ocrd_network/processing_worker.py +++ b/ocrd_network/ocrd_network/processing_worker.py @@ -219,7 +219,7 @@ def process_message(self, processing_message: OcrdProcessingMessage) -> None: if result_queue_name or callback_url: result_message = OcrdResultMessage( - job_id=str(job_id), + job_id=job_id, status=job_state.value, path_to_mets=path_to_mets, # May not be always available diff --git a/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py b/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py index 4666d05a3..32da9faaf 100644 --- a/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py +++ b/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py @@ -76,7 +76,7 @@ def decode_yml(ocrd_processing_message: bytes) -> OcrdProcessingMessage: @staticmethod def from_job(job: Job) -> OcrdProcessingMessage: return OcrdProcessingMessage( - job_id=job.id, + job_id=str(job.id), processor_name=job.processor_name, path_to_mets=job.path, input_file_grps=job.input_file_grps, From d1e7ff976de470dfaa86688934dce3791a2be471 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Wed, 22 Mar 2023 15:20:11 +0100 Subject: [PATCH 183/226] path -> path_to_mets in Job --- ocrd_network/ocrd_network/models/job.py | 6 +++--- ocrd_network/ocrd_network/processing_server.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ocrd_network/ocrd_network/models/job.py b/ocrd_network/ocrd_network/models/job.py index 3815a4d81..3434d1793 100644 --- a/ocrd_network/ocrd_network/models/job.py +++ b/ocrd_network/ocrd_network/models/job.py @@ -16,7 +16,7 @@ class StateEnum(str, Enum): class JobInput(BaseModel): """ Wraps the parameters required to make a run-processor-request """ - path: Optional[str] = None + path_to_mets: Optional[str] = None workspace_id: Optional[str] = None description: Optional[str] = None input_file_grps: List[str] @@ -53,7 +53,7 @@ class Job(Document): """ Job representation in the database """ processor_name: str - path: str + path_to_mets: str workspace_id: Optional[str] description: Optional[str] state: StateEnum @@ -74,6 +74,6 @@ def to_job_output(self) -> JobOutput: job_id=str(self.id), processor_name=self.processor_name, state=self.state, - workspace_path=self.path if not self.workspace_id else None, + workspace_path=self.path_to_mets if not self.workspace_id else None, workspace_id=self.workspace_id, ) diff --git a/ocrd_network/ocrd_network/processing_server.py b/ocrd_network/ocrd_network/processing_server.py index d3e2c1629..7e5177da6 100644 --- a/ocrd_network/ocrd_network/processing_server.py +++ b/ocrd_network/ocrd_network/processing_server.py @@ -232,7 +232,7 @@ async def push_processor_job(self, processor_name: str, data: JobInput) -> JobOu raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=report.errors) # determine path to mets if workspace_id is provided - if bool(data.path) == bool(data.workspace_id): + if bool(data.path_to_mets) == bool(data.workspace_id): raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="Either 'path' or 'workspace_id' must be provided, but not both" @@ -244,7 +244,7 @@ async def push_processor_job(self, processor_name: str, data: JobInput) -> JobOu status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=f"Workspace for id '{data.workspace_id}' not existing" ) - data.path = workspace.workspace_mets_path + data.path_to_mets = workspace.workspace_mets_path job = Job( **data.dict(exclude_unset=True, exclude_none=True), From c6be9746f133b83f09d4155329fe411a1e97260f Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Wed, 22 Mar 2023 16:12:16 +0100 Subject: [PATCH 184/226] Fix job.path -> job.path_to_mets, refactor --- .../ocrd_network/processing_worker.py | 2 +- .../rabbitmq_utils/ocrd_messages.py | 29 ++++++++++--------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/ocrd_network/ocrd_network/processing_worker.py b/ocrd_network/ocrd_network/processing_worker.py index 3c2cc1dbc..e22a8d14e 100644 --- a/ocrd_network/ocrd_network/processing_worker.py +++ b/ocrd_network/ocrd_network/processing_worker.py @@ -250,7 +250,7 @@ def publish_to_result_queue(self, result_queue: str, result_message: OcrdResultM def post_to_callback_url(self, callback_url: str, result_message: OcrdResultMessage): self.log.info(f'Posting result message to callback_url "{callback_url}"') - headers = {"accept": "application/json"} + headers = {"Content-Type": "application/json"} json_data = { "job_id": result_message.job_id, "status": result_message.status, diff --git a/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py b/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py index 32da9faaf..27d339c6e 100644 --- a/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py +++ b/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py @@ -10,27 +10,27 @@ def __init__( self, job_id: str = None, processor_name: str = None, - created_time: int = None, path_to_mets: str = None, - workspace_id: str = None, + workspace_id: Optional[str] = None, input_file_grps: List[str] = None, output_file_grps: Optional[List[str]] = None, - page_id: str = None, + page_id: Optional[str] = None, parameters: Dict[str, Any] = None, - result_queue_name: str = None, - callback_url: str = None + result_queue_name: Optional[str] = None, + callback_url: Optional[str] = None, + created_time: Optional[int] = None ) -> None: if not job_id: - raise ValueError('job_id must be set') + raise ValueError('job_id must be provided') if not processor_name: - raise ValueError('processor_name must be set') + raise ValueError('processor_name must be provided') + if not input_file_grps or len(input_file_grps) == 0: + raise ValueError('input_file_grps must be provided and contain at least 1 element') + if not (workspace_id or path_to_mets): + raise ValueError('Either "workspace_id" or "path_to_mets" must be provided') if not created_time: # We should not raise a ValueError but just calculate it created_time = int(datetime.utcnow().timestamp()) - if not input_file_grps or len(input_file_grps) == 0: - raise ValueError('input_file_grps must be set and contain at least 1 element') - if not (workspace_id or path_to_mets): - raise ValueError('Either `workspace_id` or `path_to_mets` must be set') self.job_id = job_id # uuid self.processor_name = processor_name # "ocrd-.*" @@ -78,7 +78,7 @@ def from_job(job: Job) -> OcrdProcessingMessage: return OcrdProcessingMessage( job_id=str(job.id), processor_name=job.processor_name, - path_to_mets=job.path, + path_to_mets=job.path_to_mets, input_file_grps=job.input_file_grps, output_file_grps=job.output_file_grps, page_id=job.page_id, @@ -89,8 +89,9 @@ def from_job(job: Job) -> OcrdProcessingMessage: class OcrdResultMessage: - def __init__(self, job_id: str, status: str, workspace_id: Optional[str] = None, - path_to_mets: Optional[str] = None) -> None: + def __init__(self, job_id: str, status: str, + path_to_mets: Optional[str] = None, + workspace_id: Optional[str] = None) -> None: self.job_id = job_id self.status = status self.workspace_id = workspace_id From 4dd0ea925950031898f1d7f75f3b778a07551692 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Wed, 22 Mar 2023 17:35:15 +0100 Subject: [PATCH 185/226] Create job_id, don't rely on document_id --- ocrd_network/ocrd_network/database.py | 20 +++++++++------ ocrd_network/ocrd_network/models/job.py | 3 ++- .../ocrd_network/processing_server.py | 25 +++++++++++-------- .../rabbitmq_utils/ocrd_messages.py | 2 +- ocrd_network/ocrd_network/utils.py | 10 ++++++++ 5 files changed, 41 insertions(+), 19 deletions(-) diff --git a/ocrd_network/ocrd_network/database.py b/ocrd_network/ocrd_network/database.py index b85fc3429..8a340edb6 100644 --- a/ocrd_network/ocrd_network/database.py +++ b/ocrd_network/ocrd_network/database.py @@ -14,7 +14,6 @@ """ from beanie import init_beanie from motor.motor_asyncio import AsyncIOMotorClient -from typing import Any from ocrd_network.models.job import Job, JobOutput, StateEnum from ocrd_network.models.workspace import Workspace @@ -34,8 +33,15 @@ async def sync_initiate_database(db_url: str): await initiate_database(db_url) -async def set_processing_job_state(job_id: Any, job_state: StateEnum): - job = await Job.get(job_id) +async def get_processing_job(job_id: str) -> Job: + job = await Job.find_one(Job.job_id == job_id) + if not job: + raise ValueError(f'Processing job with id "{job_id}" not available in the DB.') + return job + + +async def set_processing_job_state(job_id: str, job_state: StateEnum): + job = await Job.find_one(Job.job_id == job_id) if not job: raise ValueError(f'Processing job with id "{job_id}" not available in the DB.') job.state = job_state @@ -43,17 +49,17 @@ async def set_processing_job_state(job_id: Any, job_state: StateEnum): @call_sync -async def sync_set_processing_job_state(job_id: Any, job_state: StateEnum): +async def sync_set_processing_job_state(job_id: str, job_state: StateEnum): await set_processing_job_state(job_id, job_state) -async def get_processing_job_state(job_id: Any) -> StateEnum: - job = await Job.get(job_id) +async def get_processing_job_state(job_id: str) -> StateEnum: + job = await Job.find_one(Job.job_id == job_id) if not job: raise ValueError(f'Processing job with id "{job_id}" not available in the DB.') return job.state @call_sync -async def sync_get_processing_job_state(job_id: Any) -> StateEnum: +async def sync_get_processing_job_state(job_id: str) -> StateEnum: return await get_processing_job_state(job_id) diff --git a/ocrd_network/ocrd_network/models/job.py b/ocrd_network/ocrd_network/models/job.py index 3434d1793..933bd1278 100644 --- a/ocrd_network/ocrd_network/models/job.py +++ b/ocrd_network/ocrd_network/models/job.py @@ -52,6 +52,7 @@ class JobOutput(BaseModel): class Job(Document): """ Job representation in the database """ + job_id: str processor_name: str path_to_mets: str workspace_id: Optional[str] @@ -71,7 +72,7 @@ class Settings: def to_job_output(self) -> JobOutput: return JobOutput( - job_id=str(self.id), + job_id=self.job_id, processor_name=self.processor_name, state=self.state, workspace_path=self.path_to_mets if not self.workspace_id else None, diff --git a/ocrd_network/ocrd_network/processing_server.py b/ocrd_network/ocrd_network/processing_server.py index 7e5177da6..c287e0d2d 100644 --- a/ocrd_network/ocrd_network/processing_server.py +++ b/ocrd_network/ocrd_network/processing_server.py @@ -10,7 +10,7 @@ from ocrd_utils import getLogger, get_ocrd_tool_json from ocrd_validators import ParameterValidator, ProcessingServerValidator -from ocrd_network.database import initiate_database +import ocrd_network.database as db from ocrd_network.deployer import Deployer from ocrd_network.deployment_config import ProcessingServerConfig from ocrd_network.rabbitmq_utils import RMQPublisher, OcrdProcessingMessage @@ -21,6 +21,7 @@ StateEnum ) from ocrd_network.models.workspace import Workspace +from ocrd_network.utils import generate_id class ProcessingServer(FastAPI): @@ -156,7 +157,7 @@ def parse_config(config_path: str) -> ProcessingServerConfig: return ProcessingServerConfig(obj) async def on_startup(self): - await initiate_database(db_url=self.mongodb_url) + await db.initiate_database(db_url=self.mongodb_url) async def on_shutdown(self) -> None: """ @@ -248,6 +249,7 @@ async def push_processor_job(self, processor_name: str, data: JobInput) -> JobOu job = Job( **data.dict(exclude_unset=True, exclude_none=True), + job_id=generate_id(), processor_name=processor_name, state=StateEnum.queued ) @@ -270,16 +272,19 @@ async def get_processor_info(self, processor_name) -> Dict: ) return get_ocrd_tool_json(processor_name) - async def get_job(self, _processor_name: str, job_id: PydanticObjectId) -> JobOutput: - """ Return job-information from the database + async def get_job(self, _processor_name: str, job_id: str) -> JobOutput: + """ Return processing job-information from the database """ - job = await Job.get(job_id) - if job: + try: + job = await db.get_processing_job(job_id) return job.to_job_output() - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail='Job not found.' - ) + except Exception as error: + # Write the "error" to a log file + # for internal debugging + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail='Job not found.' + ) async def list_processors(self) -> str: """ Return a list of all available processors diff --git a/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py b/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py index 27d339c6e..7e75b75b0 100644 --- a/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py +++ b/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py @@ -76,7 +76,7 @@ def decode_yml(ocrd_processing_message: bytes) -> OcrdProcessingMessage: @staticmethod def from_job(job: Job) -> OcrdProcessingMessage: return OcrdProcessingMessage( - job_id=str(job.id), + job_id=job.job_id, processor_name=job.processor_name, path_to_mets=job.path_to_mets, input_file_grps=job.input_file_grps, diff --git a/ocrd_network/ocrd_network/utils.py b/ocrd_network/ocrd_network/utils.py index 245fde241..51297aa12 100644 --- a/ocrd_network/ocrd_network/utils.py +++ b/ocrd_network/ocrd_network/utils.py @@ -2,6 +2,7 @@ from re import match as re_match from pika import URLParameters from pymongo import uri_parser as mongo_uri_parser +from uuid import uuid4 # Based on: https://gist.github.com/phizaz/20c36c6734878c6ec053245a477572ec @@ -46,3 +47,12 @@ def verify_and_parse_mq_uri(rabbitmq_address: str): 'vhost': url_params.virtual_host } return parsed_data + + +def generate_id() -> str: + """ + Generate the id to be used for processing job ids. + Note, workspace_id and workflow_id in the reference + WebAPI implementation are produced in the same manner + """ + return str(uuid4()) From 69399fbc878e9e196a4c79773661e015c5e75f51 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Wed, 22 Mar 2023 17:45:16 +0100 Subject: [PATCH 186/226] Refactor status -> state to have same standard --- ocrd_network/ocrd_network/processing_server.py | 1 - ocrd_network/ocrd_network/processing_worker.py | 4 ++-- ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py | 6 +++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/ocrd_network/ocrd_network/processing_server.py b/ocrd_network/ocrd_network/processing_server.py index c287e0d2d..69987f8c5 100644 --- a/ocrd_network/ocrd_network/processing_server.py +++ b/ocrd_network/ocrd_network/processing_server.py @@ -3,7 +3,6 @@ import uvicorn from yaml import safe_load -from beanie import PydanticObjectId from fastapi import FastAPI, status, Request, HTTPException from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse diff --git a/ocrd_network/ocrd_network/processing_worker.py b/ocrd_network/ocrd_network/processing_worker.py index e22a8d14e..92d812662 100644 --- a/ocrd_network/ocrd_network/processing_worker.py +++ b/ocrd_network/ocrd_network/processing_worker.py @@ -220,7 +220,7 @@ def process_message(self, processing_message: OcrdProcessingMessage) -> None: if result_queue_name or callback_url: result_message = OcrdResultMessage( job_id=job_id, - status=job_state.value, + state=job_state.value, path_to_mets=path_to_mets, # May not be always available workspace_id=processing_message.workspace_id @@ -253,7 +253,7 @@ def post_to_callback_url(self, callback_url: str, result_message: OcrdResultMess headers = {"Content-Type": "application/json"} json_data = { "job_id": result_message.job_id, - "status": result_message.status, + "state": result_message.state, "path_to_mets": result_message.path_to_mets, "workspace_id": result_message.workspace_id } diff --git a/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py b/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py index 7e75b75b0..8f7758417 100644 --- a/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py +++ b/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py @@ -89,11 +89,11 @@ def from_job(job: Job) -> OcrdProcessingMessage: class OcrdResultMessage: - def __init__(self, job_id: str, status: str, + def __init__(self, job_id: str, state: str, path_to_mets: Optional[str] = None, workspace_id: Optional[str] = None) -> None: self.job_id = job_id - self.status = status + self.state = state self.workspace_id = workspace_id self.path_to_mets = path_to_mets @@ -111,7 +111,7 @@ def decode_yml(ocrd_result_message: bytes) -> OcrdResultMessage: data = yaml.load(msg, Loader=yaml.Loader) return OcrdResultMessage( job_id=data.get('job_id', None), - status=data.get('status', None), + state=data.get('state', None), path_to_mets=data.get('path_to_mets', None), workspace_id=data.get('workspace_id', None), ) From d1d85c62b84ff0d9030f8964e51892485c82f21a Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Wed, 22 Mar 2023 17:51:27 +0100 Subject: [PATCH 187/226] Remove json.dumps --- ocrd_network/ocrd_network/processing_server.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ocrd_network/ocrd_network/processing_server.py b/ocrd_network/ocrd_network/processing_server.py index 69987f8c5..b1f19525f 100644 --- a/ocrd_network/ocrd_network/processing_server.py +++ b/ocrd_network/ocrd_network/processing_server.py @@ -1,4 +1,3 @@ -import json from typing import Dict import uvicorn from yaml import safe_load @@ -288,4 +287,4 @@ async def get_job(self, _processor_name: str, job_id: str) -> JobOutput: async def list_processors(self) -> str: """ Return a list of all available processors """ - return json.dumps(self.processor_list) + return self.processor_list From 93b7f5b117c17f4765993e98c1b1d06d508988c3 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Wed, 22 Mar 2023 18:08:50 +0100 Subject: [PATCH 188/226] Refactor the setup.py --- ocrd_network/setup.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/ocrd_network/setup.py b/ocrd_network/setup.py index ac8099a6b..06cbb00b1 100644 --- a/ocrd_network/setup.py +++ b/ocrd_network/setup.py @@ -1,26 +1,23 @@ # -*- coding: utf-8 -*- -from setuptools import setup, find_packages - +from setuptools import setup from ocrd_utils import VERSION install_requires = open('requirements.txt').read().split('\n') install_requires.append('ocrd_utils == %s' % VERSION) install_requires.append('ocrd_validators == %s' % VERSION) -# TODO: This needs to be revisited! Seems badly adapted from ocrd/setup.py setup( name='ocrd_network', version=VERSION, - description='OCR-D framework - web API', + description='OCR-D framework - network', long_description=open('README.md').read(), long_description_content_type='text/markdown', author='Konstantin Baierer', author_email='unixprog@gmail.com', url='https://github.com/OCR-D/core', license='Apache License 2.0', - packages=find_packages(exclude=('tests', 'docs')), - include_package_data=True, install_requires=install_requires, + packages=['ocrd_network'], package_data={ '': ['*.yml', '*.xsd'] }, From 8f320044fd0e7d719892220722b9b1d49a46d718 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Wed, 22 Mar 2023 19:44:25 +0100 Subject: [PATCH 189/226] Big refactoring - use relative imports --- ocrd/ocrd/cli/processing_worker.py | 7 +++- ocrd_network/ocrd_network/database.py | 37 ++++++++++++------ ocrd_network/ocrd_network/deployer.py | 6 +-- .../ocrd_network/deployment_config.py | 2 +- ocrd_network/ocrd_network/deployment_utils.py | 9 ++++- ocrd_network/ocrd_network/models/__init__.py | 22 +++++++++++ ocrd_network/ocrd_network/models/job.py | 10 ++--- ocrd_network/ocrd_network/models/ocrd_tool.py | 2 +- ocrd_network/ocrd_network/models/workspace.py | 4 +- ocrd_network/ocrd_network/param_validators.py | 2 +- .../ocrd_network/processing_server.py | 39 ++++++++++--------- .../ocrd_network/processing_worker.py | 9 +++-- .../ocrd_network/rabbitmq_utils/connector.py | 2 +- .../ocrd_network/rabbitmq_utils/consumer.py | 4 +- .../rabbitmq_utils/ocrd_messages.py | 4 +- .../ocrd_network/rabbitmq_utils/publisher.py | 4 +- 16 files changed, 106 insertions(+), 57 deletions(-) diff --git a/ocrd/ocrd/cli/processing_worker.py b/ocrd/ocrd/cli/processing_worker.py index 2b1914874..e9311e061 100644 --- a/ocrd/ocrd/cli/processing_worker.py +++ b/ocrd/ocrd/cli/processing_worker.py @@ -11,8 +11,11 @@ initLogging, get_ocrd_tool_json ) -from ocrd_network import QueueServerParamType, DatabaseParamType -from ocrd_network.processing_worker import ProcessingWorker +from ocrd_network import ( + DatabaseParamType, + ProcessingWorker, + QueueServerParamType, +) @click.command('processing-worker') diff --git a/ocrd_network/ocrd_network/database.py b/ocrd_network/ocrd_network/database.py index 8a340edb6..a3d27bc4a 100644 --- a/ocrd_network/ocrd_network/database.py +++ b/ocrd_network/ocrd_network/database.py @@ -15,16 +15,19 @@ from beanie import init_beanie from motor.motor_asyncio import AsyncIOMotorClient -from ocrd_network.models.job import Job, JobOutput, StateEnum -from ocrd_network.models.workspace import Workspace -from ocrd_network.utils import call_sync +from .models import ( + DBProcessorJob, + DBWorkspace, + StateEnum +) +from .utils import call_sync async def initiate_database(db_url: str): client = AsyncIOMotorClient(db_url) await init_beanie( database=client.get_default_database(default='ocrd'), - document_models=[Job, Workspace] + document_models=[DBProcessorJob, DBWorkspace] ) @@ -33,17 +36,28 @@ async def sync_initiate_database(db_url: str): await initiate_database(db_url) -async def get_processing_job(job_id: str) -> Job: - job = await Job.find_one(Job.job_id == job_id) +async def db_get_workspace(workspace_id: str): + workspace = await DBWorkspace.find_one( + DBWorkspace.workspace_id == workspace_id + ) + if not workspace: + raise ValueError(f'Workspace with id "{workspace_id}" not in the DB.') + return workspace + + +async def db_get_processing_job(job_id: str) -> DBProcessorJob: + job = await DBProcessorJob.find_one( + DBProcessorJob.job_id == job_id) if not job: - raise ValueError(f'Processing job with id "{job_id}" not available in the DB.') + raise ValueError(f'Processing job with id "{job_id}" not in the DB.') return job async def set_processing_job_state(job_id: str, job_state: StateEnum): - job = await Job.find_one(Job.job_id == job_id) + job = await DBProcessorJob.find_one( + DBProcessorJob.job_id == job_id) if not job: - raise ValueError(f'Processing job with id "{job_id}" not available in the DB.') + raise ValueError(f'Processing job with id "{job_id}" not in the DB.') job.state = job_state await job.save() @@ -54,9 +68,10 @@ async def sync_set_processing_job_state(job_id: str, job_state: StateEnum): async def get_processing_job_state(job_id: str) -> StateEnum: - job = await Job.find_one(Job.job_id == job_id) + job = await DBProcessorJob.find_one( + DBProcessorJob.job_id == job_id) if not job: - raise ValueError(f'Processing job with id "{job_id}" not available in the DB.') + raise ValueError(f'Processing job with id "{job_id}" not in the DB.') return job.state diff --git a/ocrd_network/ocrd_network/deployer.py b/ocrd_network/ocrd_network/deployer.py index dbb875d5f..50046a1e1 100644 --- a/ocrd_network/ocrd_network/deployer.py +++ b/ocrd_network/ocrd_network/deployer.py @@ -15,8 +15,8 @@ from ocrd_utils import getLogger -from ocrd_network.deployment_config import * -from ocrd_network.deployment_utils import ( +from .deployment_config import * +from .deployment_utils import ( create_docker_client, create_ssh_client, CustomDockerClient, @@ -24,7 +24,7 @@ HostData, is_bashlib_processor, ) -from ocrd_network.rabbitmq_utils import RMQPublisher +from .rabbitmq_utils import RMQPublisher class Deployer: diff --git a/ocrd_network/ocrd_network/deployment_config.py b/ocrd_network/ocrd_network/deployment_config.py index 8cedf4644..6061aa158 100644 --- a/ocrd_network/ocrd_network/deployment_config.py +++ b/ocrd_network/ocrd_network/deployment_config.py @@ -1,6 +1,6 @@ from typing import Dict -from ocrd_network.deployment_utils import DeployType +from .deployment_utils import DeployType __all__ = [ 'ProcessingServerConfig', diff --git a/ocrd_network/ocrd_network/deployment_utils.py b/ocrd_network/ocrd_network/deployment_utils.py index 974052dd8..8b4f2356d 100644 --- a/ocrd_network/ocrd_network/deployment_utils.py +++ b/ocrd_network/ocrd_network/deployment_utils.py @@ -9,10 +9,15 @@ from paramiko import AutoAddPolicy, SSHClient from ocrd_utils import getLogger -from ocrd_network.deployment_config import * +from .deployment_config import * __all__ = [ - 'DeployType' + 'create_docker_client', + 'create_ssh_client', + 'CustomDockerClient', + 'DeployType', + 'HostData', + 'is_bashlib_processor' ] diff --git a/ocrd_network/ocrd_network/models/__init__.py b/ocrd_network/ocrd_network/models/__init__.py index e69de29bb..365e794be 100644 --- a/ocrd_network/ocrd_network/models/__init__.py +++ b/ocrd_network/ocrd_network/models/__init__.py @@ -0,0 +1,22 @@ +""" +DB prefix stands for Database Models +PY prefix stands for Pydantic Models +""" + +__all__ = [ + 'DBProcessorJob', + 'DBWorkspace', + 'PYJobInput', + 'PYJobOutput', + 'PYOcrdTool', + 'StateEnum', +] + +from .job import ( + DBProcessorJob, + PYJobInput, + PYJobOutput, + StateEnum +) +from .ocrd_tool import PYOcrdTool +from .workspace import DBWorkspace diff --git a/ocrd_network/ocrd_network/models/job.py b/ocrd_network/ocrd_network/models/job.py index 933bd1278..0863dfc26 100644 --- a/ocrd_network/ocrd_network/models/job.py +++ b/ocrd_network/ocrd_network/models/job.py @@ -13,7 +13,7 @@ class StateEnum(str, Enum): failed = 'FAILED' -class JobInput(BaseModel): +class PYJobInput(BaseModel): """ Wraps the parameters required to make a run-processor-request """ path_to_mets: Optional[str] = None @@ -39,7 +39,7 @@ class Config: } -class JobOutput(BaseModel): +class PYJobOutput(BaseModel): """ Wraps output information for a job-response """ job_id: str @@ -49,7 +49,7 @@ class JobOutput(BaseModel): workspace_id: Optional[str] -class Job(Document): +class DBProcessorJob(Document): """ Job representation in the database """ job_id: str @@ -70,8 +70,8 @@ class Job(Document): class Settings: use_enum_values = True - def to_job_output(self) -> JobOutput: - return JobOutput( + def to_job_output(self) -> PYJobOutput: + return PYJobOutput( job_id=self.job_id, processor_name=self.processor_name, state=self.state, diff --git a/ocrd_network/ocrd_network/models/ocrd_tool.py b/ocrd_network/ocrd_network/models/ocrd_tool.py index 6e87a9710..b3e2ceaea 100644 --- a/ocrd_network/ocrd_network/models/ocrd_tool.py +++ b/ocrd_network/ocrd_network/models/ocrd_tool.py @@ -2,7 +2,7 @@ from typing import List, Optional -class OcrdTool(BaseModel): +class PYOcrdTool(BaseModel): executable: str categories: List[str] description: str diff --git a/ocrd_network/ocrd_network/models/workspace.py b/ocrd_network/ocrd_network/models/workspace.py index ce0e91a29..2a597b15b 100644 --- a/ocrd_network/ocrd_network/models/workspace.py +++ b/ocrd_network/ocrd_network/models/workspace.py @@ -2,7 +2,7 @@ from typing import Optional -class Workspace(Document): +class DBWorkspace(Document): """ Model to store a workspace in the mongo-database. @@ -16,7 +16,7 @@ class Workspace(Document): bag_info_adds bag-info.txt can also (optionally) contain additional key-value-pairs which are saved here """ - id: str + workspace_id: str workspace_mets_path: str ocrd_identifier: str bagit_profile_identifier: str diff --git a/ocrd_network/ocrd_network/param_validators.py b/ocrd_network/ocrd_network/param_validators.py index 7cda9feaf..8e4669451 100644 --- a/ocrd_network/ocrd_network/param_validators.py +++ b/ocrd_network/ocrd_network/param_validators.py @@ -1,6 +1,6 @@ from click import ParamType -from ocrd_network.utils import ( +from .utils import ( verify_database_uri, verify_and_parse_mq_uri ) diff --git a/ocrd_network/ocrd_network/processing_server.py b/ocrd_network/ocrd_network/processing_server.py index b1f19525f..6d87ed5e5 100644 --- a/ocrd_network/ocrd_network/processing_server.py +++ b/ocrd_network/ocrd_network/processing_server.py @@ -8,18 +8,21 @@ from ocrd_utils import getLogger, get_ocrd_tool_json from ocrd_validators import ParameterValidator, ProcessingServerValidator -import ocrd_network.database as db -from ocrd_network.deployer import Deployer -from ocrd_network.deployment_config import ProcessingServerConfig -from ocrd_network.rabbitmq_utils import RMQPublisher, OcrdProcessingMessage -from ocrd_network.models.job import ( - Job, - JobInput, - JobOutput, +from .database import ( + db_get_processing_job, + db_get_workspace, + initiate_database +) +from .deployer import Deployer +from .deployment_config import ProcessingServerConfig +from .rabbitmq_utils import RMQPublisher, OcrdProcessingMessage +from .models import ( + DBProcessorJob, + PYJobInput, + PYJobOutput, StateEnum ) -from ocrd_network.models.workspace import Workspace -from ocrd_network.utils import generate_id +from .utils import generate_id class ProcessingServer(FastAPI): @@ -71,7 +74,7 @@ def __init__(self, config_path: str, host: str, port: int) -> None: tags=['processing'], status_code=status.HTTP_200_OK, summary='Submit a job to this processor', - response_model=JobOutput, + response_model=PYJobOutput, response_model_exclude_unset=True, response_model_exclude_none=True ) @@ -83,7 +86,7 @@ def __init__(self, config_path: str, host: str, port: int) -> None: tags=['processing'], status_code=status.HTTP_200_OK, summary='Get information about a job based on its ID', - response_model=JobOutput, + response_model=PYJobOutput, response_model_exclude_unset=True, response_model_exclude_none=True ) @@ -155,7 +158,7 @@ def parse_config(config_path: str) -> ProcessingServerConfig: return ProcessingServerConfig(obj) async def on_startup(self): - await db.initiate_database(db_url=self.mongodb_url) + await initiate_database(db_url=self.mongodb_url) async def on_shutdown(self) -> None: """ @@ -208,7 +211,7 @@ def processor_list(self): self._processor_list = list(res) return self._processor_list - async def push_processor_job(self, processor_name: str, data: JobInput) -> JobOutput: + async def push_processor_job(self, processor_name: str, data: PYJobInput) -> PYJobOutput: """ Queue a processor job """ if processor_name not in self.processor_list: @@ -237,7 +240,7 @@ async def push_processor_job(self, processor_name: str, data: JobInput) -> JobOu detail="Either 'path' or 'workspace_id' must be provided, but not both" ) elif data.workspace_id: - workspace = await Workspace.get(data.workspace_id) + workspace = await db_get_workspace(data.workspace_id) if not workspace: raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, @@ -245,7 +248,7 @@ async def push_processor_job(self, processor_name: str, data: JobInput) -> JobOu ) data.path_to_mets = workspace.workspace_mets_path - job = Job( + job = DBProcessorJob( **data.dict(exclude_unset=True, exclude_none=True), job_id=generate_id(), processor_name=processor_name, @@ -270,11 +273,11 @@ async def get_processor_info(self, processor_name) -> Dict: ) return get_ocrd_tool_json(processor_name) - async def get_job(self, _processor_name: str, job_id: str) -> JobOutput: + async def get_job(self, _processor_name: str, job_id: str) -> PYJobOutput: """ Return processing job-information from the database """ try: - job = await db.get_processing_job(job_id) + job = await db_get_processing_job(job_id) return job.to_job_output() except Exception as error: # Write the "error" to a log file diff --git a/ocrd_network/ocrd_network/processing_worker.py b/ocrd_network/ocrd_network/processing_worker.py index 92d812662..76ba7f52e 100644 --- a/ocrd_network/ocrd_network/processing_worker.py +++ b/ocrd_network/ocrd_network/processing_worker.py @@ -20,18 +20,19 @@ from ocrd import Resolver from ocrd_utils import getLogger from ocrd.processor.helpers import run_cli, run_processor -from ocrd_network.database import ( + +from .database import ( sync_initiate_database, sync_set_processing_job_state ) -from ocrd_network.models.job import StateEnum -from ocrd_network.rabbitmq_utils import ( +from .models import StateEnum +from .rabbitmq_utils import ( OcrdProcessingMessage, OcrdResultMessage, RMQConsumer, RMQPublisher ) -from ocrd_network.utils import ( +from .utils import ( verify_database_uri, verify_and_parse_mq_uri ) diff --git a/ocrd_network/ocrd_network/rabbitmq_utils/connector.py b/ocrd_network/ocrd_network/rabbitmq_utils/connector.py index fe778a772..76257048c 100644 --- a/ocrd_network/ocrd_network/rabbitmq_utils/connector.py +++ b/ocrd_network/ocrd_network/rabbitmq_utils/connector.py @@ -13,7 +13,7 @@ ) from pika.adapters.blocking_connection import BlockingChannel -from ocrd_network.rabbitmq_utils.constants import ( +from .constants import ( DEFAULT_EXCHANGER_NAME, DEFAULT_EXCHANGER_TYPE, DEFAULT_QUEUE, diff --git a/ocrd_network/ocrd_network/rabbitmq_utils/consumer.py b/ocrd_network/ocrd_network/rabbitmq_utils/consumer.py index a235475b2..bf282264a 100644 --- a/ocrd_network/ocrd_network/rabbitmq_utils/consumer.py +++ b/ocrd_network/ocrd_network/rabbitmq_utils/consumer.py @@ -9,14 +9,14 @@ from pika import PlainCredentials -from ocrd_network.rabbitmq_utils.constants import ( +from .constants import ( DEFAULT_QUEUE, LOG_LEVEL, RABBIT_MQ_HOST as HOST, RABBIT_MQ_PORT as PORT, RABBIT_MQ_VHOST as VHOST ) -from ocrd_network.rabbitmq_utils.connector import RMQConnector +from .connector import RMQConnector class RMQConsumer(RMQConnector): diff --git a/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py b/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py index 8f7758417..2868f3408 100644 --- a/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py +++ b/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py @@ -2,7 +2,7 @@ from datetime import datetime from typing import Any, Dict, List, Optional import yaml -from ocrd_network.models.job import Job +from ..models import DBProcessorJob class OcrdProcessingMessage: @@ -74,7 +74,7 @@ def decode_yml(ocrd_processing_message: bytes) -> OcrdProcessingMessage: ) @staticmethod - def from_job(job: Job) -> OcrdProcessingMessage: + def from_job(job: DBProcessorJob) -> OcrdProcessingMessage: return OcrdProcessingMessage( job_id=job.job_id, processor_name=job.processor_name, diff --git a/ocrd_network/ocrd_network/rabbitmq_utils/publisher.py b/ocrd_network/ocrd_network/rabbitmq_utils/publisher.py index 806cdb426..9a3d080f9 100644 --- a/ocrd_network/ocrd_network/rabbitmq_utils/publisher.py +++ b/ocrd_network/ocrd_network/rabbitmq_utils/publisher.py @@ -12,7 +12,7 @@ PlainCredentials ) -from ocrd_network.rabbitmq_utils.constants import ( +from .constants import ( DEFAULT_EXCHANGER_NAME, DEFAULT_ROUTER, LOG_FORMAT, @@ -21,7 +21,7 @@ RABBIT_MQ_PORT as PORT, RABBIT_MQ_VHOST as VHOST ) -from ocrd_network.rabbitmq_utils.connector import RMQConnector +from .connector import RMQConnector class RMQPublisher(RMQConnector): From 3c3273b0922c5d4db978ec91c16d23fe7a31e5c6 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Wed, 22 Mar 2023 19:56:00 +0100 Subject: [PATCH 190/226] Fix setup.py --- ocrd_network/setup.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ocrd_network/setup.py b/ocrd_network/setup.py index 06cbb00b1..07759a2f3 100644 --- a/ocrd_network/setup.py +++ b/ocrd_network/setup.py @@ -17,7 +17,11 @@ url='https://github.com/OCR-D/core', license='Apache License 2.0', install_requires=install_requires, - packages=['ocrd_network'], + packages=[ + 'ocrd_network', + 'ocrd_network.models', + 'ocrd_network.rabbitmq_utils' + ], package_data={ '': ['*.yml', '*.xsd'] }, From e080a0fe81da05ed2315d7e4d0ce3e7ea6a050a1 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Thu, 23 Mar 2023 12:01:42 +0100 Subject: [PATCH 191/226] Refactor processing config validation --- ocrd_network/ocrd_network/processing_server.py | 4 ++-- ocrd_validators/ocrd_validators/__init__.py | 4 ++-- ocrd_validators/ocrd_validators/constants.py | 2 ++ ...fig.schema.yml => processing_server_config.schema.yml} | 0 ...validator.py => processing_server_config_validator.py} | 8 +++----- 5 files changed, 9 insertions(+), 9 deletions(-) rename ocrd_validators/ocrd_validators/{config.schema.yml => processing_server_config.schema.yml} (100%) rename ocrd_validators/ocrd_validators/{processing_server_validator.py => processing_server_config_validator.py} (74%) diff --git a/ocrd_network/ocrd_network/processing_server.py b/ocrd_network/ocrd_network/processing_server.py index 6d87ed5e5..6159ce187 100644 --- a/ocrd_network/ocrd_network/processing_server.py +++ b/ocrd_network/ocrd_network/processing_server.py @@ -7,7 +7,7 @@ from fastapi.responses import JSONResponse from ocrd_utils import getLogger, get_ocrd_tool_json -from ocrd_validators import ParameterValidator, ProcessingServerValidator +from ocrd_validators import ParameterValidator, ProcessingServerConfigValidator from .database import ( db_get_processing_job, db_get_workspace, @@ -152,7 +152,7 @@ def start(self) -> None: def parse_config(config_path: str) -> ProcessingServerConfig: with open(config_path) as fin: obj = safe_load(fin) - report = ProcessingServerValidator.validate(obj) + report = ProcessingServerConfigValidator.validate(obj) if not report.is_valid: raise Exception(f'Processing-Server configuration file is invalid:\n{report.errors}') return ProcessingServerConfig(obj) diff --git a/ocrd_validators/ocrd_validators/__init__.py b/ocrd_validators/ocrd_validators/__init__.py index b461c897c..8a46d3e60 100644 --- a/ocrd_validators/ocrd_validators/__init__.py +++ b/ocrd_validators/ocrd_validators/__init__.py @@ -11,7 +11,7 @@ 'XsdValidator', 'XsdMetsValidator', 'XsdPageValidator', - 'ProcessingServerValidator', + 'ProcessingServerConfigValidator', ] from .parameter_validator import ParameterValidator @@ -23,4 +23,4 @@ from .xsd_validator import XsdValidator from .xsd_mets_validator import XsdMetsValidator from .xsd_page_validator import XsdPageValidator -from .processing_server_validator import ProcessingServerValidator +from .processing_server_config_validator import ProcessingServerConfigValidator diff --git a/ocrd_validators/ocrd_validators/constants.py b/ocrd_validators/ocrd_validators/constants.py index a963d89fe..2d761147d 100644 --- a/ocrd_validators/ocrd_validators/constants.py +++ b/ocrd_validators/ocrd_validators/constants.py @@ -5,6 +5,7 @@ from ocrd_utils.package_resources import resource_string, resource_filename __all__ = [ + 'PROCESSING_SERVER_CONFIG_SCHEMA', 'OCRD_TOOL_SCHEMA', 'RESOURCE_LIST_SCHEMA', 'OCRD_BAGIT_PROFILE', @@ -18,6 +19,7 @@ 'XSD_PATHS', ] +PROCESSING_SERVER_CONFIG_SCHEMA = yaml.safe_load(resource_string(__name__, 'processing_server_config.schema.yml')) OCRD_TOOL_SCHEMA = yaml.safe_load(resource_string(__name__, 'ocrd_tool.schema.yml')) RESOURCE_LIST_SCHEMA = { 'type': 'object', diff --git a/ocrd_validators/ocrd_validators/config.schema.yml b/ocrd_validators/ocrd_validators/processing_server_config.schema.yml similarity index 100% rename from ocrd_validators/ocrd_validators/config.schema.yml rename to ocrd_validators/ocrd_validators/processing_server_config.schema.yml diff --git a/ocrd_validators/ocrd_validators/processing_server_validator.py b/ocrd_validators/ocrd_validators/processing_server_config_validator.py similarity index 74% rename from ocrd_validators/ocrd_validators/processing_server_validator.py rename to ocrd_validators/ocrd_validators/processing_server_config_validator.py index 896782430..667fdbd9f 100644 --- a/ocrd_validators/ocrd_validators/processing_server_validator.py +++ b/ocrd_validators/ocrd_validators/processing_server_config_validator.py @@ -1,24 +1,22 @@ """ Validating configuration file for the Processing-Server """ +from .constants import PROCESSING_SERVER_CONFIG_SCHEMA from .json_validator import JsonValidator -import yaml -from ocrd_utils.package_resources import resource_string # TODO: provide a link somewhere in this file as it is done in ocrd_tool.schema.yml but best with a # working link. Currently it is here: # https://github.com/OCR-D/spec/pull/222/files#diff-a71bf71cbc7d9ce94fded977f7544aba4df9e7bdb8fc0cf1014e14eb67a9b273 # But that is a PR not merged yet -class ProcessingServerValidator(JsonValidator): +class ProcessingServerConfigValidator(JsonValidator): """ JsonValidator validating against the schema for the Processing Server """ @staticmethod - def validate(obj): + def validate(obj, schema=PROCESSING_SERVER_CONFIG_SCHEMA): """ Validate against schema for Processing-Server """ - schema = yaml.safe_load(resource_string(__name__, 'config.schema.yml')) return JsonValidator.validate(obj, schema) From 59a06abdb30ef5d945c41f1120dca7c56c4a02ef Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Thu, 23 Mar 2023 14:54:12 +0100 Subject: [PATCH 192/226] Validate processing message against message schema --- .../ocrd_network/deployment_config.py | 13 ++- .../ocrd_network/processing_server.py | 40 +++++---- .../ocrd_network/processing_worker.py | 30 ++++--- .../rabbitmq_utils/ocrd_messages.py | 90 ++++++++----------- ocrd_network/ocrd_network/utils.py | 23 +++-- ocrd_validators/ocrd_validators/__init__.py | 2 + ocrd_validators/ocrd_validators/constants.py | 4 + .../message_processing.schema.yml | 66 ++++++++++++++ .../ocrd_validators/message_result.schema.yml | 31 +++++++ .../ocrd_network_message_validator.py | 22 +++++ 10 files changed, 228 insertions(+), 93 deletions(-) create mode 100644 ocrd_validators/ocrd_validators/message_processing.schema.yml create mode 100644 ocrd_validators/ocrd_validators/message_result.schema.yml create mode 100644 ocrd_validators/ocrd_validators/ocrd_network_message_validator.py diff --git a/ocrd_network/ocrd_network/deployment_config.py b/ocrd_network/ocrd_network/deployment_config.py index 6061aa158..48b123d1a 100644 --- a/ocrd_network/ocrd_network/deployment_config.py +++ b/ocrd_network/ocrd_network/deployment_config.py @@ -1,5 +1,6 @@ from typing import Dict - +from yaml import safe_load +from ocrd_validators import ProcessingServerConfigValidator from .deployment_utils import DeployType __all__ = [ @@ -12,7 +13,15 @@ class ProcessingServerConfig: - def __init__(self, config: dict) -> None: + def __init__(self, config_path: str) -> None: + # Load and validate the config + with open(config_path) as fin: + config = safe_load(fin) + report = ProcessingServerConfigValidator.validate(config) + if not report.is_valid: + raise Exception(f'Processing-Server configuration file is invalid:\n{report.errors}') + + # Split the configurations self.mongo = MongoConfig(config['database']) self.queue = QueueConfig(config['process_queue']) self.hosts = [] diff --git a/ocrd_network/ocrd_network/processing_server.py b/ocrd_network/ocrd_network/processing_server.py index 6159ce187..8c1c42d17 100644 --- a/ocrd_network/ocrd_network/processing_server.py +++ b/ocrd_network/ocrd_network/processing_server.py @@ -1,13 +1,12 @@ from typing import Dict import uvicorn -from yaml import safe_load from fastapi import FastAPI, status, Request, HTTPException from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse from ocrd_utils import getLogger, get_ocrd_tool_json -from ocrd_validators import ParameterValidator, ProcessingServerConfigValidator +from ocrd_validators import ParameterValidator from .database import ( db_get_processing_job, db_get_workspace, @@ -22,7 +21,7 @@ PYJobOutput, StateEnum ) -from .utils import generate_id +from .utils import generate_created_time, generate_id class ProcessingServer(FastAPI): @@ -43,7 +42,7 @@ def __init__(self, config_path: str, host: str, port: int) -> None: self.log = getLogger(__name__) self.hostname = host self.port = port - self.config = ProcessingServer.parse_config(config_path) + self.config = ProcessingServerConfig(config_path) self.deployer = Deployer(self.config) self.mongodb_url = None self.rmq_host = self.config.queue.address @@ -148,15 +147,6 @@ def start(self) -> None: raise uvicorn.run(self, host=self.hostname, port=int(self.port)) - @staticmethod - def parse_config(config_path: str) -> ProcessingServerConfig: - with open(config_path) as fin: - obj = safe_load(fin) - report = ProcessingServerConfigValidator.validate(obj) - if not report.is_valid: - raise Exception(f'Processing-Server configuration file is invalid:\n{report.errors}') - return ProcessingServerConfig(obj) - async def on_startup(self): await initiate_database(db_url=self.mongodb_url) @@ -211,6 +201,23 @@ def processor_list(self): self._processor_list = list(res) return self._processor_list + @staticmethod + def create_processing_message(job: DBProcessorJob) -> OcrdProcessingMessage: + processing_message = OcrdProcessingMessage( + job_id=job.job_id, + processor_name=job.processor_name, + created_time=generate_created_time(), + path_to_mets=job.path_to_mets, + workspace_id=job.workspace_id, + input_file_grps=job.input_file_grps, + output_file_grps=job.output_file_grps, + page_id=job.page_id, + parameters=job.parameters, + result_queue_name=job.result_queue_name, + callback_url=job.callback_url, + ) + return processing_message + async def push_processor_job(self, processor_name: str, data: PYJobInput) -> PYJobOutput: """ Queue a processor job """ @@ -226,10 +233,9 @@ async def push_processor_job(self, processor_name: str, data: PYJobInput) -> PYJ if not ocrd_tool: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Processor '{processor_name}' not available. It's ocrd_tool is empty or missing" + detail=f"Processor '{processor_name}' not available. Empty or missing ocrd_tool" ) - validator = ParameterValidator(ocrd_tool) - report = validator.validate(data.parameters) + report = ParameterValidator(ocrd_tool).validate(data.parameters) if not report.is_valid: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=report.errors) @@ -255,7 +261,7 @@ async def push_processor_job(self, processor_name: str, data: PYJobInput) -> PYJ state=StateEnum.queued ) await job.insert() - processing_message = OcrdProcessingMessage.from_job(job) + processing_message = self.create_processing_message(job) encoded_processing_message = OcrdProcessingMessage.encode_yml(processing_message) if self.rmq_publisher: self.rmq_publisher.publish_to_queue(processor_name, encoded_processing_message) diff --git a/ocrd_network/ocrd_network/processing_worker.py b/ocrd_network/ocrd_network/processing_worker.py index 76ba7f52e..3292f2108 100644 --- a/ocrd_network/ocrd_network/processing_worker.py +++ b/ocrd_network/ocrd_network/processing_worker.py @@ -184,10 +184,17 @@ def process_message(self, processing_message: OcrdProcessingMessage) -> None: raise ValueError(f'Processor name is not matching. Expected: {self.processor_name},' f'Got: {processing_message.processor_name}') - # This can be path if invoking `run_processor` - # but must be ocrd.Workspace if invoking `run_cli`. - path_to_mets = processing_message.path_to_mets - # Build the workspace from the workspace_path + # All of this is needed because the OcrdProcessingMessage object + # may not contain certain keys. Simply passing None in the OcrdProcessingMessage constructor + # breaks the message validator schema which expects String, but not None due to the Optional[] wrapper. + pm_keys = processing_message.__dict__.keys() + output_file_grps = processing_message.output_file_grps if 'output_file_grps' in pm_keys else None + path_to_mets = processing_message.path_to_mets if 'path_to_mets' in pm_keys else None + workspace_id = processing_message.workspace_id if 'workspace_id' in pm_keys else None + page_id = processing_message.page_id if 'page_id' in pm_keys else None + result_queue_name = processing_message.result_queue_name if 'result_queue_name' in pm_keys else None + callback_url = processing_message.callback_url if 'callback_url' in pm_keys else None + workspace = Resolver().workspace_from_url(path_to_mets) job_id = processing_message.job_id @@ -197,9 +204,9 @@ def process_message(self, processing_message: OcrdProcessingMessage) -> None: return_status = self.run_processor_from_worker( processor_class=self.processor_class, workspace=workspace, - page_id=processing_message.page_id, + page_id=page_id, input_file_grps=processing_message.input_file_grps, - output_file_grps=processing_message.output_file_grps, + output_file_grps=output_file_grps, parameter=processing_message.parameters ) else: @@ -207,24 +214,21 @@ def process_message(self, processing_message: OcrdProcessingMessage) -> None: return_status = self.run_cli_from_worker( executable=self.processor_name, workspace=workspace, - page_id=processing_message.page_id, + page_id=page_id, input_file_grps=processing_message.input_file_grps, - output_file_grps=processing_message.output_file_grps, + output_file_grps=output_file_grps, parameter=processing_message.parameters ) job_state = StateEnum.success if return_status else StateEnum.failed sync_set_processing_job_state(job_id=job_id, job_state=job_state) - result_queue_name = processing_message.result_queue_name - callback_url = processing_message.callback_url - if result_queue_name or callback_url: result_message = OcrdResultMessage( job_id=job_id, state=job_state.value, path_to_mets=path_to_mets, # May not be always available - workspace_id=processing_message.workspace_id + workspace_id=workspace_id ) self.log.info(f'Result message: {result_message}') @@ -234,7 +238,7 @@ def process_message(self, processing_message: OcrdProcessingMessage) -> None: # If the callback_url field is set, post the result message to a callback url if callback_url: - self.post_to_callback_url(processing_message.callback_url, result_message) + self.post_to_callback_url(callback_url, result_message) def publish_to_result_queue(self, result_queue: str, result_message: OcrdResultMessage): if self.rmq_publisher is None: diff --git a/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py b/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py index 2868f3408..80f5e253a 100644 --- a/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py +++ b/ocrd_network/ocrd_network/rabbitmq_utils/ocrd_messages.py @@ -1,64 +1,65 @@ from __future__ import annotations -from datetime import datetime from typing import Any, Dict, List, Optional import yaml -from ..models import DBProcessorJob + +from ocrd_validators import OcrdNetworkMessageValidator class OcrdProcessingMessage: def __init__( self, - job_id: str = None, - processor_name: str = None, - path_to_mets: str = None, - workspace_id: Optional[str] = None, - input_file_grps: List[str] = None, - output_file_grps: Optional[List[str]] = None, - page_id: Optional[str] = None, + job_id: str, + processor_name: str, + created_time: int, + input_file_grps: List[str], + output_file_grps: Optional[List[str]], + path_to_mets: Optional[str], + workspace_id: Optional[str], + page_id: Optional[str], + result_queue_name: Optional[str], + callback_url: Optional[str], parameters: Dict[str, Any] = None, - result_queue_name: Optional[str] = None, - callback_url: Optional[str] = None, - created_time: Optional[int] = None ) -> None: if not job_id: raise ValueError('job_id must be provided') if not processor_name: raise ValueError('processor_name must be provided') + if not created_time: + raise ValueError('created time must be provided') if not input_file_grps or len(input_file_grps) == 0: raise ValueError('input_file_grps must be provided and contain at least 1 element') if not (workspace_id or path_to_mets): raise ValueError('Either "workspace_id" or "path_to_mets" must be provided') - if not created_time: - # We should not raise a ValueError but just calculate it - created_time = int(datetime.utcnow().timestamp()) - self.job_id = job_id # uuid - self.processor_name = processor_name # "ocrd-.*" - # Either of these two below - self.workspace_id = workspace_id # uuid - self.path_to_mets = path_to_mets # absolute path - self.input_file_grps = input_file_grps - self.output_file_grps = output_file_grps - # e.g., "PHYS_0005..PHYS_0010" will process only pages between 5-10 - self.page_id = page_id if page_id else None - # processor parameters - self.parameters = parameters if parameters else None - self.result_queue_name = result_queue_name - self.callback_url = callback_url + self.job_id = job_id + self.processor_name = processor_name self.created_time = created_time + self.input_file_grps = input_file_grps + if output_file_grps: + self.output_file_grps = output_file_grps + if path_to_mets: + self.path_to_mets = path_to_mets + if workspace_id: + self.workspace_id = workspace_id + if page_id: + self.page_id = page_id + if result_queue_name: + self.result_queue_name = result_queue_name + if callback_url: + self.callback_url = callback_url + self.parameters = parameters if parameters else {} @staticmethod def encode_yml(ocrd_processing_message: OcrdProcessingMessage) -> bytes: - """convert OcrdProcessingMessage to yml - """ return yaml.dump(ocrd_processing_message.__dict__, indent=2).encode('utf-8') @staticmethod def decode_yml(ocrd_processing_message: bytes) -> OcrdProcessingMessage: - """Parse OcrdProcessingMessage from yml - """ msg = ocrd_processing_message.decode('utf-8') - data = yaml.load(msg, Loader=yaml.Loader) + data = yaml.safe_load(msg) + report = OcrdNetworkMessageValidator.validate_message_processing(data) + if not report.is_valid: + raise ValueError(f'Validating the processing message has failed:\n{report.errors}') return OcrdProcessingMessage( job_id=data.get('job_id', None), processor_name=data.get('processor_name', None), @@ -73,20 +74,6 @@ def decode_yml(ocrd_processing_message: bytes) -> OcrdProcessingMessage: callback_url=data.get('callback_url', None) ) - @staticmethod - def from_job(job: DBProcessorJob) -> OcrdProcessingMessage: - return OcrdProcessingMessage( - job_id=job.job_id, - processor_name=job.processor_name, - path_to_mets=job.path_to_mets, - input_file_grps=job.input_file_grps, - output_file_grps=job.output_file_grps, - page_id=job.page_id, - parameters=job.parameters, - result_queue_name=job.result_queue_name, - callback_url=job.callback_url - ) - class OcrdResultMessage: def __init__(self, job_id: str, state: str, @@ -99,16 +86,15 @@ def __init__(self, job_id: str, state: str, @staticmethod def encode_yml(ocrd_result_message: OcrdResultMessage) -> bytes: - """convert OcrdResultMessage to yml - """ return yaml.dump(ocrd_result_message.__dict__, indent=2).encode('utf-8') @staticmethod def decode_yml(ocrd_result_message: bytes) -> OcrdResultMessage: - """Parse OcrdResultMessage from yml - """ msg = ocrd_result_message.decode('utf-8') - data = yaml.load(msg, Loader=yaml.Loader) + data = yaml.safe_load(msg) + report = OcrdNetworkMessageValidator.validate_message_result(data) + if not report.is_valid: + raise ValueError(f'Validating the result message has failed:\n{report.errors}') return OcrdResultMessage( job_id=data.get('job_id', None), state=data.get('state', None), diff --git a/ocrd_network/ocrd_network/utils.py b/ocrd_network/ocrd_network/utils.py index 51297aa12..a75dd5f4f 100644 --- a/ocrd_network/ocrd_network/utils.py +++ b/ocrd_network/ocrd_network/utils.py @@ -1,3 +1,4 @@ +from datetime import datetime from functools import wraps from re import match as re_match from pika import URLParameters @@ -18,6 +19,19 @@ def func_wrapper(*args, **kwargs): return func_wrapper +def generate_created_time() -> int: + return int(datetime.utcnow().timestamp()) + + +def generate_id() -> str: + """ + Generate the id to be used for processing job ids. + Note, workspace_id and workflow_id in the reference + WebAPI implementation are produced in the same manner + """ + return str(uuid4()) + + def verify_database_uri(mongodb_address: str) -> str: try: # perform validation check @@ -47,12 +61,3 @@ def verify_and_parse_mq_uri(rabbitmq_address: str): 'vhost': url_params.virtual_host } return parsed_data - - -def generate_id() -> str: - """ - Generate the id to be used for processing job ids. - Note, workspace_id and workflow_id in the reference - WebAPI implementation are produced in the same manner - """ - return str(uuid4()) diff --git a/ocrd_validators/ocrd_validators/__init__.py b/ocrd_validators/ocrd_validators/__init__.py index 8a46d3e60..48e5b3046 100644 --- a/ocrd_validators/ocrd_validators/__init__.py +++ b/ocrd_validators/ocrd_validators/__init__.py @@ -12,6 +12,7 @@ 'XsdMetsValidator', 'XsdPageValidator', 'ProcessingServerConfigValidator', + 'OcrdNetworkMessageValidator' ] from .parameter_validator import ParameterValidator @@ -24,3 +25,4 @@ from .xsd_mets_validator import XsdMetsValidator from .xsd_page_validator import XsdPageValidator from .processing_server_config_validator import ProcessingServerConfigValidator +from .ocrd_network_message_validator import OcrdNetworkMessageValidator diff --git a/ocrd_validators/ocrd_validators/constants.py b/ocrd_validators/ocrd_validators/constants.py index 2d761147d..b3834f7eb 100644 --- a/ocrd_validators/ocrd_validators/constants.py +++ b/ocrd_validators/ocrd_validators/constants.py @@ -6,6 +6,8 @@ __all__ = [ 'PROCESSING_SERVER_CONFIG_SCHEMA', + 'MESSAGE_SCHEMA_PROCESSING', + 'MESSAGE_SCHEMA_RESULT', 'OCRD_TOOL_SCHEMA', 'RESOURCE_LIST_SCHEMA', 'OCRD_BAGIT_PROFILE', @@ -20,6 +22,8 @@ ] PROCESSING_SERVER_CONFIG_SCHEMA = yaml.safe_load(resource_string(__name__, 'processing_server_config.schema.yml')) +MESSAGE_SCHEMA_PROCESSING = yaml.safe_load(resource_string(__name__, 'message_processing.schema.yml')) +MESSAGE_SCHEMA_RESULT = yaml.safe_load(resource_string(__name__, 'message_result.schema.yml')) OCRD_TOOL_SCHEMA = yaml.safe_load(resource_string(__name__, 'ocrd_tool.schema.yml')) RESOURCE_LIST_SCHEMA = { 'type': 'object', diff --git a/ocrd_validators/ocrd_validators/message_processing.schema.yml b/ocrd_validators/ocrd_validators/message_processing.schema.yml new file mode 100644 index 000000000..3a8042bf4 --- /dev/null +++ b/ocrd_validators/ocrd_validators/message_processing.schema.yml @@ -0,0 +1,66 @@ +$schema: https://json-schema.org/draft/2020-12/schema +$id: https://ocr-d.de/spec/web-api/processing-message.schema.yml +description: Schema for Processing Messages +type: object +additionalProperties: false +required: + - job_id + - processor_name + - created_time + - input_file_grps +oneOf: + - required: + - path_to_mets + - required: + - workspace_id +properties: + job_id: + description: The ID of the job + type: string + format: uuid + processor_name: + description: Name of the processor + type: string + pattern: "^ocrd-.*$" + examples: + - ocrd-cis-ocropy-binarize + - ocrd-olena-binarize + path_to_mets: + description: Path to a METS file + type: string + workspace_id: + description: ID of a workspace + type: string + input_file_grps: + description: A list of file groups for input + type: array + minItems: 1 + items: + type: string + output_file_grps: + description: A list of file groups for output + type: array + minItems: 1 + items: + type: string + page_id: + description: ID of pages to be processed + type: string + examples: + - PHYS_0001,PHYS_0002,PHYS_0003 + - PHYS_0001..PHYS_0005,PHYS_0007,PHYS_0009 + parameters: + description: Parameters for the used model + type: object + result_queue_name: + description: Name of the queue to which result is published + type: string + callback_url: + description: The URL where the result message will be POST-ed to + type: string + format: uri, + pattern: "^https?://" + created_time: + description: The Unix timestamp when the message was created + type: integer + minimum: 0 diff --git a/ocrd_validators/ocrd_validators/message_result.schema.yml b/ocrd_validators/ocrd_validators/message_result.schema.yml new file mode 100644 index 000000000..aef62821e --- /dev/null +++ b/ocrd_validators/ocrd_validators/message_result.schema.yml @@ -0,0 +1,31 @@ +$schema: https://json-schema.org/draft/2020-12/schema +$id: https://ocr-d.de/spec/web-api/result-message.schema.yml +description: Schema for Result Messages +type: object +additionalProperties: false +required: + - job_id + - status +oneOf: + - required: + - path_to_mets + - required: + - workspace_id +properties: + job_id: + description: The ID of the job + type: string + format: uuid + status: + description: The current status of the job + type: string + enum: + - SUCCESS + - RUNNING + - FAILED + path_to_mets: + description: Path to a METS file + type: string + workspace_id: + description: ID of a workspace + type: string diff --git a/ocrd_validators/ocrd_validators/ocrd_network_message_validator.py b/ocrd_validators/ocrd_validators/ocrd_network_message_validator.py new file mode 100644 index 000000000..486efea43 --- /dev/null +++ b/ocrd_validators/ocrd_validators/ocrd_network_message_validator.py @@ -0,0 +1,22 @@ +""" +Validating ocrd-network messages +""" +from .constants import ( + MESSAGE_SCHEMA_PROCESSING, + MESSAGE_SCHEMA_RESULT +) +from .json_validator import JsonValidator + + +class OcrdNetworkMessageValidator(JsonValidator): + """ + JsonValidator validating against the ocrd network message schemas + """ + + @staticmethod + def validate_message_processing(obj): + return JsonValidator.validate(obj, schema=MESSAGE_SCHEMA_PROCESSING) + + @staticmethod + def validate_message_result(obj): + return JsonValidator.validate(obj, schema=MESSAGE_SCHEMA_RESULT) From 787510ef6f8faa60b13a4815781ff0ac0d79fbdd Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Fri, 24 Mar 2023 13:32:58 +0100 Subject: [PATCH 193/226] Fix: catch db workspace exception --- ocrd_network/ocrd_network/processing_server.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ocrd_network/ocrd_network/processing_server.py b/ocrd_network/ocrd_network/processing_server.py index 8c1c42d17..af530e528 100644 --- a/ocrd_network/ocrd_network/processing_server.py +++ b/ocrd_network/ocrd_network/processing_server.py @@ -246,8 +246,9 @@ async def push_processor_job(self, processor_name: str, data: PYJobInput) -> PYJ detail="Either 'path' or 'workspace_id' must be provided, but not both" ) elif data.workspace_id: - workspace = await db_get_workspace(data.workspace_id) - if not workspace: + try: + workspace = await db_get_workspace(data.workspace_id) + except ValueError: raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=f"Workspace for id '{data.workspace_id}' not existing" @@ -285,9 +286,7 @@ async def get_job(self, _processor_name: str, job_id: str) -> PYJobOutput: try: job = await db_get_processing_job(job_id) return job.to_job_output() - except Exception as error: - # Write the "error" to a log file - # for internal debugging + except ValueError: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail='Job not found.' From 16793823297ec67e5d5f47d55e565e9be4a8f2af Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Fri, 24 Mar 2023 13:48:35 +0100 Subject: [PATCH 194/226] impove: match the returned http status code --- ocrd_network/ocrd_network/processing_server.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ocrd_network/ocrd_network/processing_server.py b/ocrd_network/ocrd_network/processing_server.py index af530e528..c7d7fba6a 100644 --- a/ocrd_network/ocrd_network/processing_server.py +++ b/ocrd_network/ocrd_network/processing_server.py @@ -251,7 +251,7 @@ async def push_processor_job(self, processor_name: str, data: PYJobInput) -> PYJ except ValueError: raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail=f"Workspace for id '{data.workspace_id}' not existing" + detail=f"Workspace with id '{data.workspace_id}' not existing" ) data.path_to_mets = workspace.workspace_mets_path @@ -280,7 +280,7 @@ async def get_processor_info(self, processor_name) -> Dict: ) return get_ocrd_tool_json(processor_name) - async def get_job(self, _processor_name: str, job_id: str) -> PYJobOutput: + async def get_job(self, processor_name: str, job_id: str) -> PYJobOutput: """ Return processing job-information from the database """ try: @@ -288,8 +288,8 @@ async def get_job(self, _processor_name: str, job_id: str) -> PYJobOutput: return job.to_job_output() except ValueError: raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail='Job not found.' + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=f"Processing job with id '{job_id}' of processor type '{processor_name}' not existing" ) async def list_processors(self) -> str: From 709c46c397b1003e60a3adf0b630702f090a39f4 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Fri, 24 Mar 2023 14:45:28 +0100 Subject: [PATCH 195/226] fix: resolve mets path in worker, not server --- ocrd_network/ocrd_network/database.py | 7 ++++++- ocrd_network/ocrd_network/processing_server.py | 6 +++--- ocrd_network/ocrd_network/processing_worker.py | 3 +++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/ocrd_network/ocrd_network/database.py b/ocrd_network/ocrd_network/database.py index a3d27bc4a..01e95874f 100644 --- a/ocrd_network/ocrd_network/database.py +++ b/ocrd_network/ocrd_network/database.py @@ -36,7 +36,7 @@ async def sync_initiate_database(db_url: str): await initiate_database(db_url) -async def db_get_workspace(workspace_id: str): +async def db_get_workspace(workspace_id: str) -> DBWorkspace: workspace = await DBWorkspace.find_one( DBWorkspace.workspace_id == workspace_id ) @@ -45,6 +45,11 @@ async def db_get_workspace(workspace_id: str): return workspace +@call_sync +async def sync_db_get_workspace(workspace_id: str) -> DBWorkspace: + return await db_get_workspace(workspace_id) + + async def db_get_processing_job(job_id: str) -> DBProcessorJob: job = await DBProcessorJob.find_one( DBProcessorJob.job_id == job_id) diff --git a/ocrd_network/ocrd_network/processing_server.py b/ocrd_network/ocrd_network/processing_server.py index c7d7fba6a..6fae5cc1d 100644 --- a/ocrd_network/ocrd_network/processing_server.py +++ b/ocrd_network/ocrd_network/processing_server.py @@ -239,21 +239,21 @@ async def push_processor_job(self, processor_name: str, data: PYJobInput) -> PYJ if not report.is_valid: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=report.errors) - # determine path to mets if workspace_id is provided if bool(data.path_to_mets) == bool(data.workspace_id): raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="Either 'path' or 'workspace_id' must be provided, but not both" ) + # This check is done to return early in case + # the workspace_id is provided but not existing in the DB elif data.workspace_id: try: - workspace = await db_get_workspace(data.workspace_id) + await db_get_workspace(data.workspace_id) except ValueError: raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=f"Workspace with id '{data.workspace_id}' not existing" ) - data.path_to_mets = workspace.workspace_mets_path job = DBProcessorJob( **data.dict(exclude_unset=True, exclude_none=True), diff --git a/ocrd_network/ocrd_network/processing_worker.py b/ocrd_network/ocrd_network/processing_worker.py index 3292f2108..67d033d53 100644 --- a/ocrd_network/ocrd_network/processing_worker.py +++ b/ocrd_network/ocrd_network/processing_worker.py @@ -195,6 +195,9 @@ def process_message(self, processing_message: OcrdProcessingMessage) -> None: result_queue_name = processing_message.result_queue_name if 'result_queue_name' in pm_keys else None callback_url = processing_message.callback_url if 'callback_url' in pm_keys else None + if not path_to_mets and workspace_id: + path_to_mets = sync_db_get_workspace(workspace_id).workspace_mets_path + workspace = Resolver().workspace_from_url(path_to_mets) job_id = processing_message.job_id From 39840e2b744e9623eb3e7356f5080abc7c6327d2 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Fri, 24 Mar 2023 15:00:12 +0100 Subject: [PATCH 196/226] fix: process job model --- ocrd_network/ocrd_network/database.py | 5 +++++ ocrd_network/ocrd_network/models/job.py | 2 +- ocrd_network/ocrd_network/processing_worker.py | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ocrd_network/ocrd_network/database.py b/ocrd_network/ocrd_network/database.py index 01e95874f..751c8e4cb 100644 --- a/ocrd_network/ocrd_network/database.py +++ b/ocrd_network/ocrd_network/database.py @@ -58,6 +58,11 @@ async def db_get_processing_job(job_id: str) -> DBProcessorJob: return job +@call_sync +async def sync_db_get_processing_job(job_id: str) -> DBProcessorJob: + return await db_get_processing_job(job_id) + + async def set_processing_job_state(job_id: str, job_state: StateEnum): job = await DBProcessorJob.find_one( DBProcessorJob.job_id == job_id) diff --git a/ocrd_network/ocrd_network/models/job.py b/ocrd_network/ocrd_network/models/job.py index 0863dfc26..b7a1d6bcf 100644 --- a/ocrd_network/ocrd_network/models/job.py +++ b/ocrd_network/ocrd_network/models/job.py @@ -54,7 +54,7 @@ class DBProcessorJob(Document): """ job_id: str processor_name: str - path_to_mets: str + path_to_mets: Optional[str] workspace_id: Optional[str] description: Optional[str] state: StateEnum diff --git a/ocrd_network/ocrd_network/processing_worker.py b/ocrd_network/ocrd_network/processing_worker.py index 67d033d53..8ef26b4c1 100644 --- a/ocrd_network/ocrd_network/processing_worker.py +++ b/ocrd_network/ocrd_network/processing_worker.py @@ -23,6 +23,7 @@ from .database import ( sync_initiate_database, + sync_db_get_workspace, sync_set_processing_job_state ) from .models import StateEnum From 9ba05e00deb09fa00bdc915581eaf65e57c34435 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Fri, 24 Mar 2023 17:23:27 +0100 Subject: [PATCH 197/226] db single update method --- ocrd_network/ocrd_network/database.py | 42 +++++++++++++++------------ 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/ocrd_network/ocrd_network/database.py b/ocrd_network/ocrd_network/database.py index 751c8e4cb..2daf71761 100644 --- a/ocrd_network/ocrd_network/database.py +++ b/ocrd_network/ocrd_network/database.py @@ -17,8 +17,7 @@ from .models import ( DBProcessorJob, - DBWorkspace, - StateEnum + DBWorkspace ) from .utils import call_sync @@ -63,28 +62,33 @@ async def sync_db_get_processing_job(job_id: str) -> DBProcessorJob: return await db_get_processing_job(job_id) -async def set_processing_job_state(job_id: str, job_state: StateEnum): +async def db_update_processing_job(job_id: str, **kwargs): job = await DBProcessorJob.find_one( DBProcessorJob.job_id == job_id) if not job: raise ValueError(f'Processing job with id "{job_id}" not in the DB.') - job.state = job_state - await job.save() - - -@call_sync -async def sync_set_processing_job_state(job_id: str, job_state: StateEnum): - await set_processing_job_state(job_id, job_state) - -async def get_processing_job_state(job_id: str) -> StateEnum: - job = await DBProcessorJob.find_one( - DBProcessorJob.job_id == job_id) - if not job: - raise ValueError(f'Processing job with id "{job_id}" not in the DB.') - return job.state + # TODO: This may not be the best Pythonic way to do it. However, it works! + # There must be a shorter way with Pydantic. Suggest an improvement. + job_keys = list(job.__dict__.keys()) + for key, value in kwargs.items(): + if key not in job_keys: + raise ValueError(f'Field "{key}" is not available.') + if key == 'state': + job.state = value + elif key == 'start_time': + job.start_time = value + elif key == 'end_time': + job.end_time = value + elif key == 'path_to_mets': + job.path_to_mets = value + elif key == 'exec_time': + job.exec_time = value + else: + raise ValueError(f'Field "{key}" is not updatable.') + await job.save() @call_sync -async def sync_get_processing_job_state(job_id: str) -> StateEnum: - return await get_processing_job_state(job_id) +async def sync_db_update_processing_job(job_id: str, **kwargs): + await db_update_processing_job(job_id=job_id, **kwargs) From 0fe51f2984e98da0371695edb64ab088195ffe4e Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Fri, 24 Mar 2023 17:23:56 +0100 Subject: [PATCH 198/226] calculate execution time in ms --- ocrd_network/ocrd_network/models/job.py | 1 + .../ocrd_network/processing_worker.py | 23 ++++++++++++++++--- ocrd_network/ocrd_network/utils.py | 8 +++++++ 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/ocrd_network/ocrd_network/models/job.py b/ocrd_network/ocrd_network/models/job.py index b7a1d6bcf..3c0857c37 100644 --- a/ocrd_network/ocrd_network/models/job.py +++ b/ocrd_network/ocrd_network/models/job.py @@ -66,6 +66,7 @@ class DBProcessorJob(Document): callback_url: Optional[str] start_time: Optional[datetime] end_time: Optional[datetime] + exec_time: Optional[str] class Settings: use_enum_values = True diff --git a/ocrd_network/ocrd_network/processing_worker.py b/ocrd_network/ocrd_network/processing_worker.py index 8ef26b4c1..de6b1c7c9 100644 --- a/ocrd_network/ocrd_network/processing_worker.py +++ b/ocrd_network/ocrd_network/processing_worker.py @@ -8,6 +8,7 @@ is a single OCR-D Processor instance. """ +from datetime import datetime import json import logging from os import environ, getpid @@ -24,7 +25,7 @@ from .database import ( sync_initiate_database, sync_db_get_workspace, - sync_set_processing_job_state + sync_db_update_processing_job, ) from .models import StateEnum from .rabbitmq_utils import ( @@ -34,6 +35,7 @@ RMQPublisher ) from .utils import ( + calculate_execution_time, verify_database_uri, verify_and_parse_mq_uri ) @@ -202,7 +204,14 @@ def process_message(self, processing_message: OcrdProcessingMessage) -> None: workspace = Resolver().workspace_from_url(path_to_mets) job_id = processing_message.job_id - sync_set_processing_job_state(job_id=job_id, job_state=StateEnum.running) + + start_time = datetime.now() + sync_db_update_processing_job( + job_id=job_id, + state=StateEnum.running, + path_to_mets=path_to_mets, + start_time=start_time + ) if self.processor_class: self.log.debug(f'Invoking the pythonic processor: {self.processor_name}') return_status = self.run_processor_from_worker( @@ -223,8 +232,16 @@ def process_message(self, processing_message: OcrdProcessingMessage) -> None: output_file_grps=output_file_grps, parameter=processing_message.parameters ) + end_time = datetime.now() + # Execution duration in ms + execution_duration = calculate_execution_time(start_time, end_time) job_state = StateEnum.success if return_status else StateEnum.failed - sync_set_processing_job_state(job_id=job_id, job_state=job_state) + sync_db_update_processing_job( + job_id=job_id, + state=job_state, + end_time=end_time, + exec_time=f'{execution_duration} ms' + ) if result_queue_name or callback_url: result_message = OcrdResultMessage( diff --git a/ocrd_network/ocrd_network/utils.py b/ocrd_network/ocrd_network/utils.py index a75dd5f4f..759a31597 100644 --- a/ocrd_network/ocrd_network/utils.py +++ b/ocrd_network/ocrd_network/utils.py @@ -19,6 +19,14 @@ def func_wrapper(*args, **kwargs): return func_wrapper +def calculate_execution_time(start: datetime, end: datetime) -> int: + """ + Calculates the difference between `start` and `end` datetime. + Returns the result in milliseconds + """ + return int((end - start).total_seconds() * 1000) + + def generate_created_time() -> int: return int(datetime.utcnow().timestamp()) From 9910482eae6a3f78d778fa5b530fe3e034ec6cb5 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Fri, 24 Mar 2023 18:19:10 +0100 Subject: [PATCH 199/226] resolve: suggestions by kba --- ocrd_network/ocrd_network/processing_worker.py | 1 + ocrd_network/setup.py | 6 +----- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/ocrd_network/ocrd_network/processing_worker.py b/ocrd_network/ocrd_network/processing_worker.py index de6b1c7c9..8b3681f18 100644 --- a/ocrd_network/ocrd_network/processing_worker.py +++ b/ocrd_network/ocrd_network/processing_worker.py @@ -40,6 +40,7 @@ verify_and_parse_mq_uri ) +# TODO: Check this again when the logging is refactored try: # This env variable must be set before importing from Keras environ['TF_CPP_MIN_LOG_LEVEL'] = '3' diff --git a/ocrd_network/setup.py b/ocrd_network/setup.py index 07759a2f3..63678f191 100644 --- a/ocrd_network/setup.py +++ b/ocrd_network/setup.py @@ -3,7 +3,6 @@ from ocrd_utils import VERSION install_requires = open('requirements.txt').read().split('\n') -install_requires.append('ocrd_utils == %s' % VERSION) install_requires.append('ocrd_validators == %s' % VERSION) setup( @@ -12,7 +11,7 @@ description='OCR-D framework - network', long_description=open('README.md').read(), long_description_content_type='text/markdown', - author='Konstantin Baierer', + author='Mehmed Mustafa, Jonas Schrewe, Triet Doan', author_email='unixprog@gmail.com', url='https://github.com/OCR-D/core', license='Apache License 2.0', @@ -22,8 +21,5 @@ 'ocrd_network.models', 'ocrd_network.rabbitmq_utils' ], - package_data={ - '': ['*.yml', '*.xsd'] - }, keywords=['OCR', 'OCR-D'] ) From 4774f51ac41ab11e5ebc9ab94cc35181d83e6549 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Fri, 24 Mar 2023 23:38:18 +0100 Subject: [PATCH 200/226] bashlib: implement Processing Worker args --- ocrd/ocrd/lib.bash | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/ocrd/ocrd/lib.bash b/ocrd/ocrd/lib.bash index a34263637..11417c570 100644 --- a/ocrd/ocrd/lib.bash +++ b/ocrd/ocrd/lib.bash @@ -143,34 +143,45 @@ ocrd__parse_argv () { --profile) ocrd__argv[profile]=true ;; --profile-file) ocrd__argv[profile_file]=$(realpath "$2") ; shift ;; -V|--version) ocrd ocrd-tool "$OCRD_TOOL_JSON" version; exit ;; + --queue) ocrd__worker_queue="$2" ; shift ;; + --database) ocrd__worker_database="$2" ; shift ;; *) ocrd__raise "Unknown option '$1'" ;; esac shift done - if [[ ! -e "${ocrd__argv[mets_file]}" ]];then + if [ -v ocrd__worker_queue -a -v ocrd__worker_database ]; then + ocrd processing-worker $OCRD_TOOL_NAME --queue "${ocrd__worker_queue}" --database "${ocrd__worker_database}" + exit + elif [ -v ocrd__worker_queue ]; then + ocrd__raise "Processing Worker also requires a --database argument" + elif [ -v ocrd__worker_database ]; then + ocrd__raise "Processing Worker also requires a --queue argument" + fi + + if [[ ! -e "${ocrd__argv[mets_file]}" ]]; then ocrd__raise "METS file '${ocrd__argv[mets_file]}' not found" fi - if [[ ! -d "${ocrd__argv[working_dir]:=$(dirname "${ocrd__argv[mets_file]}")}" ]];then + if [[ ! -d "${ocrd__argv[working_dir]:=$(dirname "${ocrd__argv[mets_file]}")}" ]]; then ocrd__raise "workdir '${ocrd__argv[working_dir]}' not a directory. Use -w/--working-dir to set correctly" fi - if [[ ! "${ocrd__argv[log_level]:=INFO}" =~ OFF|ERROR|WARN|INFO|DEBUG|TRACE ]];then + if [[ ! "${ocrd__argv[log_level]:=INFO}" =~ OFF|ERROR|WARN|INFO|DEBUG|TRACE ]]; then ocrd__raise "log level '${ocrd__argv[log_level]}' is invalid" fi - if [[ -z "${ocrd__argv[input_file_grp]:=}" ]];then + if [[ -z "${ocrd__argv[input_file_grp]:=}" ]]; then ocrd__raise "Provide --input-file-grp/-I explicitly!" fi - if [[ -z "${ocrd__argv[output_file_grp]:=}" ]];then + if [[ -z "${ocrd__argv[output_file_grp]:=}" ]]; then ocrd__raise "Provide --output-file-grp/-O explicitly!" fi # enable profiling (to be extended/acted upon by caller) - if [[ ${ocrd__argv[profile]} = true ]];then - if [[ -n "${ocrd__argv[profile_file]}" ]];then + if [[ ${ocrd__argv[profile]} = true ]]; then + if [[ -n "${ocrd__argv[profile_file]}" ]]; then exec 3> "${ocrd__argv[profile_file]}" else exec 3>&2 From 6741d55b66f13db685172379f06ef958b2a41eab Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Sat, 25 Mar 2023 00:10:05 +0100 Subject: [PATCH 201/226] improve generate_processor_help: - for docstring extraction, add newlines after each block - remove useless "wiring" of fileGrps - add default values in brackets where applicable - re-order logically (mets before grps) - group processing and non-processing options for clarity - fix the Processing Worker queue format - improve formulations --- ocrd/ocrd/processor/helpers.py | 44 ++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/ocrd/ocrd/processor/helpers.py b/ocrd/ocrd/processor/helpers.py index bee6a087c..96ded1781 100644 --- a/ocrd/ocrd/processor/helpers.py +++ b/ocrd/ocrd/processor/helpers.py @@ -240,11 +240,11 @@ def wrap(s): if processor_instance: module = inspect.getmodule(processor_instance) if module and module.__doc__: - doc_help += '\n' + inspect.cleandoc(module.__doc__) + doc_help += '\n' + inspect.cleandoc(module.__doc__) + '\n' if processor_instance.__doc__: - doc_help += '\n' + inspect.cleandoc(processor_instance.__doc__) + doc_help += '\n' + inspect.cleandoc(processor_instance.__doc__) + '\n' if processor_instance.process.__doc__: - doc_help += '\n' + inspect.cleandoc(processor_instance.process.__doc__) + doc_help += '\n' + inspect.cleandoc(processor_instance.process.__doc__) + '\n' if doc_help: doc_help = '\n\n' + wrap_text(doc_help, width=72, initial_indent=' > ', @@ -255,43 +255,47 @@ def wrap(s): %s%s -Options: +Options for processing: + -m, --mets URL-PATH URL or file path of METS to process [./mets.xml] + -w, --working-dir PATH Working directory of local workspace [dirname(URL-PATH)] -I, --input-file-grp USE File group(s) used as input -O, --output-file-grp USE File group(s) used as output - -g, --page-id ID Physical page ID(s) to process + -g, --page-id ID Physical page ID(s) to process [all] --overwrite Remove existing output pages/images - (with --page-id, remove only those) - --queue The RabbitMQ server address in format: {host}:{port}/{vhost}" - --database The MongoDB address in format: mongodb://{host}:{port}" + (with "--page-id", remove only those) --profile Enable profiling - --profile-file Write cProfile stats to this file. Implies --profile + --profile-file PROF-PATH Write cProfile stats to PROF-PATH. Implies "--profile" -p, --parameter JSON-PATH Parameters, either verbatim JSON string or JSON file path -P, --param-override KEY VAL Override a single JSON object key-value pair, - taking precedence over --parameter - -m, --mets URL-PATH URL or file path of METS to process - -w, --working-dir PATH Working directory of local workspace + taking precedence over "--parameter" -l, --log-level [OFF|ERROR|WARN|INFO|DEBUG|TRACE] - Log level + Override log level globally [INFO] + +Options for Processing Worker server: + --queue The RabbitMQ server address in format + "amqp://{user}:{pass}@{host}:{port}/{vhost}" + [amqp://admin:admin@localhost:5672] + --database The MongoDB server address in format + "mongodb://{host}:{port}" + [mongodb://localhost:27018] + +Options for information: -C, --show-resource RESNAME Dump the content of processor resource RESNAME -L, --list-resources List names of processor resources - -J, --dump-json Dump tool description as JSON and exit - -D, --dump-module-dir Output the 'module' directory with resources for this processor - -h, --help This help message + -J, --dump-json Dump tool description as JSON + -D, --dump-module-dir Show the 'module' resource location path for this processor + -h, --help Show this message -V, --version Show version Parameters: %s -Default Wiring: - %s -> %s ''' % ( ocrd_tool['executable'], ocrd_tool['description'], doc_help, parameter_help, - ocrd_tool.get('input_file_grp', 'NONE'), - ocrd_tool.get('output_file_grp', 'NONE') ) From 0ccc9633fb0007e0fab1046005a3573ae926a5e0 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Sat, 25 Mar 2023 00:22:50 +0100 Subject: [PATCH 202/226] ocrd_cli_options: remove redundant help kwarg and add `type=click.Path` for `--profile-file`. --- ocrd/ocrd/decorators/ocrd_cli_options.py | 38 ++++++++++++------------ 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/ocrd/ocrd/decorators/ocrd_cli_options.py b/ocrd/ocrd/decorators/ocrd_cli_options.py index 6245f092b..2ba4bf8ae 100644 --- a/ocrd/ocrd/decorators/ocrd_cli_options.py +++ b/ocrd/ocrd/decorators/ocrd_cli_options.py @@ -1,4 +1,4 @@ -from click import option +from click import option, Path from .parameter_option import parameter_option, parameter_override_option from .loglevel_option import loglevel_option from ocrd_network import QueueServerParamType, DatabaseParamType @@ -19,28 +19,28 @@ def cli(mets_url): """ # XXX Note that the `--help` output is statically generate_processor_help params = [ - option('-m', '--mets', help="METS to process", default="mets.xml"), - option('-w', '--working-dir', help="Working Directory"), + option('-m', '--mets', default="mets.xml"), + option('-w', '--working-dir'), # TODO OCR-D/core#274 - # option('-I', '--input-file-grp', help='File group(s) used as input. **required**'), - # option('-O', '--output-file-grp', help='File group(s) used as output. **required**'), - option('-I', '--input-file-grp', help='File group(s) used as input.', default='INPUT'), - option('-O', '--output-file-grp', help='File group(s) used as output.', default='OUTPUT'), - option('-g', '--page-id', help="ID(s) of the pages to process"), - option('--overwrite', help="Overwrite the output file group or a page range (--page-id)", is_flag=True, default=False), - option('--queue', help="The URL of the Queue Server, format: username:password@host:port/vhost", type=QueueServerParamType()), - option('--database', help="The URL of the MongoDB, format: mongodb://host:port", type=DatabaseParamType()), - option('-C', '--show-resource', help='Dump the content of processor resource RESNAME', metavar='RESNAME'), - option('-L', '--list-resources', is_flag=True, default=False, help='List names of processor resources'), + # option('-I', '--input-file-grp', required=True), + # option('-O', '--output-file-grp', required=True), + option('-I', '--input-file-grp', default='INPUT'), + option('-O', '--output-file-grp', default='OUTPUT'), + option('-g', '--page-id'), + option('--overwrite', is_flag=True, default=False), + option('--profile', is_flag=True, default=False), + option('--profile-file', type=Path(dir_okay=False, writable=True)), parameter_option, parameter_override_option, - option('-J', '--dump-json', help="Dump tool description as JSON and exit", is_flag=True, default=False), - option('-D', '--dump-module-dir', help="Print processor's 'moduledir' of resourcess", is_flag=True, default=False), loglevel_option, - option('-V', '--version', help="Show version", is_flag=True, default=False), - option('-h', '--help', help="This help message", is_flag=True, default=False), - option('--profile', help="Enable profiling", is_flag=True, default=False), - option('--profile-file', help="Write cProfile stats to this file. Implies --profile"), + option('--queue', type=QueueServerParamType()), + option('--database', type=DatabaseParamType()), + option('-C', '--show-resource'), + option('-L', '--list-resources', is_flag=True, default=False), + option('-J', '--dump-json', is_flag=True, default=False), + option('-D', '--dump-module-dir', is_flag=True, default=False), + option('-h', '--help', is_flag=True, default=False), + option('-V', '--version', is_flag=True, default=False), ] for param in params: param(f) From e2e6e69fe22da43b4fed999f4c397b61e16e2925 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Sat, 25 Mar 2023 00:25:43 +0100 Subject: [PATCH 203/226] start native cli: no need for is_bashlib_processor --- ocrd_network/ocrd_network/deployer.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/ocrd_network/ocrd_network/deployer.py b/ocrd_network/ocrd_network/deployer.py index 50046a1e1..27ac323fa 100644 --- a/ocrd_network/ocrd_network/deployer.py +++ b/ocrd_network/ocrd_network/deployer.py @@ -22,7 +22,6 @@ CustomDockerClient, DeployType, HostData, - is_bashlib_processor, ) from .rabbitmq_utils import RMQPublisher @@ -281,11 +280,7 @@ def start_native_processor(self, client: SSHClient, processor_name: str, queue_u self.log.info(f'Starting native processor: {processor_name}') channel = client.invoke_shell() stdin, stdout = channel.makefile('wb'), channel.makefile('rb') - if is_bashlib_processor(processor_name): - cmd = f'ocrd processing-worker {processor_name} --database {database_url} ' \ - f'--queue {queue_url}' - else: - cmd = f'{processor_name} --database {database_url} --queue {queue_url}' + cmd = f'{processor_name} --database {database_url} --queue {queue_url}' # the only way (I could find) to make it work to start a process in the background and # return early is this construction. The pid of the last started background process is # printed with `echo $!` but it is printed inbetween other output. Because of that I added From 4ab2657bfb42a81e416859740a876f7bfc38bcca Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Sat, 25 Mar 2023 00:26:10 +0100 Subject: [PATCH 204/226] rm is_bashlib_processor --- ocrd_network/ocrd_network/deployment_utils.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/ocrd_network/ocrd_network/deployment_utils.py b/ocrd_network/ocrd_network/deployment_utils.py index 8b4f2356d..6b943127b 100644 --- a/ocrd_network/ocrd_network/deployment_utils.py +++ b/ocrd_network/ocrd_network/deployment_utils.py @@ -38,22 +38,6 @@ def create_docker_client(address: str, username: str, password: Union[str, None] return CustomDockerClient(username, address, password=password, keypath=keypath) -def is_bashlib_processor(processor_name): - """ Determine if a processor is a bashlib processor - - Returns True if processor_name is available as a program and does not contain a python hashbang - in line 1 """ - if not processor_name.startswith("ocrd"): - return False - program = which(processor_name) - if not program: - return False - with open(program) as fin: - line = fin.readline().strip() - if re.fullmatch('[#][!].*/python[0-9.]*', line): - return False - return True - class HostData: """class to store runtime information for a host From a4b9c96ff0488d4ffd12da61271713d43e75bb65 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Sat, 25 Mar 2023 00:29:40 +0100 Subject: [PATCH 205/226] =?UTF-8?q?rm=20Py36=20test=20=E2=80=93=20no=20lon?= =?UTF-8?q?ger=20supported?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .circleci/config.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 556d52388..eea93f8fb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -19,17 +19,6 @@ jobs: - run: make install - run: make deps-test test benchmark - test-python36: - docker: - - image: python:3.6.15 - working_directory: ~/ocrd-core - steps: - - checkout - - run: apt-get -y update - - run: pip install -U pip - - run: make deps-ubuntu install - - run: make deps-test test benchmark - test-python37: docker: - image: python:3.7.16 @@ -104,7 +93,6 @@ workflows: only: master test-pull-request: jobs: - - test-python36 - test-python37 - test-python38 - test-python39 From 3f13747f9b0978c08d95c83ba110de2f10aa7d68 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Sat, 25 Mar 2023 00:32:18 +0100 Subject: [PATCH 206/226] CI: use Python images from CircleCI instead of DH (because they are cached/faster) --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index eea93f8fb..181e9d17a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -21,7 +21,7 @@ jobs: test-python37: docker: - - image: python:3.7.16 + - image: cimg/python:3.7.16 working_directory: ~/ocrd-core steps: - checkout @@ -31,7 +31,7 @@ jobs: test-python38: docker: - - image: python:3.8.16 + - image: cimg/python:3.8.16 working_directory: ~/ocrd-core steps: - checkout @@ -41,7 +41,7 @@ jobs: test-python39: docker: - - image: python:3.9.16 + - image: cimg/python:3.9.16 working_directory: ~/ocrd-core steps: - checkout @@ -51,7 +51,7 @@ jobs: test-python310: docker: - - image: python:3.10.10 + - image: cimg/python:3.10.10 working_directory: ~/ocrd-core steps: - checkout @@ -61,7 +61,7 @@ jobs: test-python311: docker: - - image: python:3.11.2 + - image: cimg/python:3.11.2 working_directory: ~/ocrd-core steps: - checkout From 6ead599dce67bca8bee0792e2c5ca6241f2be5da Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Sat, 25 Mar 2023 00:33:24 +0100 Subject: [PATCH 207/226] require Py37+ --- ocrd_utils/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ocrd_utils/setup.py b/ocrd_utils/setup.py index 88d319573..d3e43b033 100644 --- a/ocrd_utils/setup.py +++ b/ocrd_utils/setup.py @@ -14,6 +14,7 @@ url='https://github.com/OCR-D/core', license='Apache License 2.0', packages=['ocrd_utils'], + python_requires=">=3.7", install_requires=install_requires, package_data={'': ['*.json', '*.yml', '*.xml']}, keywords=['OCR', 'OCR-D'] From 89570a79bab5f30fe8ed6ae352c40d19a8f8a5d4 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Sat, 25 Mar 2023 00:33:59 +0100 Subject: [PATCH 208/226] require Py37+ --- ocrd/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ocrd/setup.py b/ocrd/setup.py index 67536620f..be28ba0d6 100644 --- a/ocrd/setup.py +++ b/ocrd/setup.py @@ -22,6 +22,7 @@ license='Apache License 2.0', packages=find_packages(exclude=('tests', 'docs')), include_package_data=True, + python_requires=">=3.7", install_requires=install_requires, entry_points={ 'console_scripts': [ From e1f6ed8c572952cccd766767fed32c7be777351d Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Sat, 25 Mar 2023 00:34:33 +0100 Subject: [PATCH 209/226] require Py37+ --- ocrd_network/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ocrd_network/setup.py b/ocrd_network/setup.py index 63678f191..f79081fa0 100644 --- a/ocrd_network/setup.py +++ b/ocrd_network/setup.py @@ -15,6 +15,7 @@ author_email='unixprog@gmail.com', url='https://github.com/OCR-D/core', license='Apache License 2.0', + python_requires=">=3.7", install_requires=install_requires, packages=[ 'ocrd_network', From fed4128a89f9cf2dbab6b1492691cfcbfda863e1 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Sat, 25 Mar 2023 00:34:56 +0100 Subject: [PATCH 210/226] require Py37+ --- ocrd_modelfactory/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ocrd_modelfactory/setup.py b/ocrd_modelfactory/setup.py index 16bf4c6cc..a25146073 100644 --- a/ocrd_modelfactory/setup.py +++ b/ocrd_modelfactory/setup.py @@ -17,6 +17,7 @@ author_email='unixprog@gmail.com', url='https://github.com/OCR-D/core', license='Apache License 2.0', + python_requires=">=3.7", install_requires=install_requires, packages=['ocrd_modelfactory'], package_data={'': ['*.json', '*.yml', '*.xml']}, From af08d1eaa85efcfda0312a8ef4b89a9ea03f9ef3 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Sat, 25 Mar 2023 00:35:17 +0100 Subject: [PATCH 211/226] require Py37+ --- ocrd_models/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ocrd_models/setup.py b/ocrd_models/setup.py index 3d37f4a33..b34212603 100644 --- a/ocrd_models/setup.py +++ b/ocrd_models/setup.py @@ -16,6 +16,7 @@ author_email='unixprog@gmail.com', url='https://github.com/OCR-D/core', license='Apache License 2.0', + python_requires=">=3.7", install_requires=install_requires, packages=['ocrd_models'], package_data={'': ['*.json', '*.yml', '*.xml']}, From 21c46c1de30bc2d0409b8d6afb7d70bda8b25d34 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Sat, 25 Mar 2023 00:35:33 +0100 Subject: [PATCH 212/226] require Py37+ --- ocrd_validators/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ocrd_validators/setup.py b/ocrd_validators/setup.py index 121071e91..fc8c23e37 100644 --- a/ocrd_validators/setup.py +++ b/ocrd_validators/setup.py @@ -18,6 +18,7 @@ author_email='unixprog@gmail.com', url='https://github.com/OCR-D/core', license='Apache License 2.0', + python_requires=">=3.7", install_requires=install_requires, packages=['ocrd_validators'], package_data={ From fe3303e7edbe9a43307f9fd59c547a2e2cbcb40f Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Sat, 25 Mar 2023 01:09:19 +0100 Subject: [PATCH 213/226] fix CI: - cimg needs sudo - don't specify Python subminors, so it always picks latest - activate docker_layer_caching for deploy to speed up --- .circleci/config.yml | 50 +++++++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 181e9d17a..1d1be701c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,58 +16,63 @@ jobs: steps: - checkout - run: HOMEBREW_NO_AUTO_UPDATE=1 brew install imagemagick geos - - run: make install - - run: make deps-test test benchmark + - run: make install deps-test + - run: make test benchmark test-python37: docker: - - image: cimg/python:3.7.16 + - image: cimg/python:3.7 working_directory: ~/ocrd-core steps: - checkout - - run: apt-get -y update - - run: make deps-ubuntu install - - run: make deps-test test benchmark + - run: sudo apt-get -y update + - run: sudo make deps-ubuntu + - run: make install deps-test + - run: make test benchmark test-python38: docker: - - image: cimg/python:3.8.16 + - image: cimg/python:3.8 working_directory: ~/ocrd-core steps: - checkout - - run: apt-get -y update - - run: make deps-ubuntu install - - run: make deps-test test benchmark + - run: sudo apt-get -y update + - run: sudo make deps-ubuntu + - run: make install deps-test + - run: make test benchmark test-python39: docker: - - image: cimg/python:3.9.16 + - image: cimg/python:3.9 working_directory: ~/ocrd-core steps: - checkout - - run: apt-get -y update - - run: make deps-ubuntu install - - run: make deps-test test benchmark + - run: sudo apt-get -y update + - run: sudo make deps-ubuntu + - run: make install deps-test + - run: make test benchmark test-python310: docker: - - image: cimg/python:3.10.10 + - image: cimg/python:3.10 working_directory: ~/ocrd-core steps: - checkout - - run: apt-get -y update - - run: make deps-ubuntu install - - run: make deps-test test benchmark + - run: sudo apt-get -y update + - run: sudo make deps-ubuntu + - run: make install deps-test + - run: make test benchmark test-python311: docker: - - image: cimg/python:3.11.2 + - image: cimg/python:3.11 working_directory: ~/ocrd-core steps: - checkout - - run: apt-get -y update - - run: make deps-ubuntu install - - run: make deps-test test benchmark + - run: sudo apt-get -y update + - run: sudo make deps-ubuntu + - run: make install deps-test + - run: make test benchmark deploy: docker: @@ -75,6 +80,7 @@ jobs: steps: - checkout - setup_remote_docker # https://circleci.com/docs/2.0/building-docker-images/ + docker_layer_caching: true - run: make docker - run: make docker-cuda - run: From df7bfee37ecacca2817a3323a797bfbe6ca68fd3 Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Sun, 26 Mar 2023 15:28:06 +0200 Subject: [PATCH 214/226] --help: clearer description of --page-id Co-authored-by: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> --- ocrd/ocrd/processor/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ocrd/ocrd/processor/helpers.py b/ocrd/ocrd/processor/helpers.py index 96ded1781..bc3cce637 100644 --- a/ocrd/ocrd/processor/helpers.py +++ b/ocrd/ocrd/processor/helpers.py @@ -260,7 +260,7 @@ def wrap(s): -w, --working-dir PATH Working directory of local workspace [dirname(URL-PATH)] -I, --input-file-grp USE File group(s) used as input -O, --output-file-grp USE File group(s) used as output - -g, --page-id ID Physical page ID(s) to process [all] + -g, --page-id ID Physical page ID(s) to process instead of full document [] --overwrite Remove existing output pages/images (with "--page-id", remove only those) --profile Enable profiling From 378400e17e0776f1e24eb967312c91aadced8203 Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Sun, 26 Mar 2023 15:30:54 +0200 Subject: [PATCH 215/226] .circleci/config.yml: fix syntax --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1d1be701c..e018e389f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -79,7 +79,8 @@ jobs: - image: circleci/buildpack-deps:stretch steps: - checkout - - setup_remote_docker # https://circleci.com/docs/2.0/building-docker-images/ + # https://circleci.com/docs/2.0/building-docker-images/ + - setup_remote_docker docker_layer_caching: true - run: make docker - run: make docker-cuda From dbae7c42b6de64a027d86e0f2dfd4cf7dc8ca36e Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Sun, 26 Mar 2023 15:30:54 +0200 Subject: [PATCH 216/226] .circleci/config.yml: fix syntax --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1d1be701c..ba90d83f3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -79,7 +79,7 @@ jobs: - image: circleci/buildpack-deps:stretch steps: - checkout - - setup_remote_docker # https://circleci.com/docs/2.0/building-docker-images/ + - setup_remote_docker docker_layer_caching: true - run: make docker - run: make docker-cuda From 25cc6fcf58b6975f75711c2f76e0a26c66472ec1 Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Sun, 26 Mar 2023 15:33:38 +0200 Subject: [PATCH 217/226] .circleci/config.yml: fix syntax --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e018e389f..ba90d83f3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -79,7 +79,6 @@ jobs: - image: circleci/buildpack-deps:stretch steps: - checkout - # https://circleci.com/docs/2.0/building-docker-images/ - setup_remote_docker docker_layer_caching: true - run: make docker From 176cf73da04e32b3b5e3ce0d539accc10812f69c Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Sun, 26 Mar 2023 15:35:10 +0200 Subject: [PATCH 218/226] .circleci/config.yml: fix syntax --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ba90d83f3..3490bd467 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -79,7 +79,7 @@ jobs: - image: circleci/buildpack-deps:stretch steps: - checkout - - setup_remote_docker + - setup_remote_docker: # https://circleci.com/docs/2.0/building-docker-images/ docker_layer_caching: true - run: make docker - run: make docker-cuda From 2fa674f4a13e8b8a3efb9ba46d4982bfddc5f97a Mon Sep 17 00:00:00 2001 From: joschrew Date: Mon, 27 Mar 2023 11:53:55 +0200 Subject: [PATCH 219/226] Make receiving job info for procesor work again --- ocrd_network/ocrd_network/processing_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ocrd_network/ocrd_network/processing_server.py b/ocrd_network/ocrd_network/processing_server.py index 6fae5cc1d..fff1be274 100644 --- a/ocrd_network/ocrd_network/processing_server.py +++ b/ocrd_network/ocrd_network/processing_server.py @@ -79,7 +79,7 @@ def __init__(self, config_path: str, host: str, port: int) -> None: ) self.router.add_api_route( - path='/processor/{_processor_name}/{job_id}', + path='/processor/{processor_name}/{job_id}', endpoint=self.get_job, methods=['GET'], tags=['processing'], From 2fc385669e3308a6eb395b3235558c11afb97109 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Mon, 27 Mar 2023 17:37:42 +0200 Subject: [PATCH 220/226] provide flexible queue checks --- .../ocrd_network/processing_server.py | 33 +++++++++++++++---- .../ocrd_network/rabbitmq_utils/publisher.py | 6 ++-- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/ocrd_network/ocrd_network/processing_server.py b/ocrd_network/ocrd_network/processing_server.py index fff1be274..870ff4940 100644 --- a/ocrd_network/ocrd_network/processing_server.py +++ b/ocrd_network/ocrd_network/processing_server.py @@ -5,6 +5,8 @@ from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse +from pika.exceptions import ChannelClosedByBroker + from ocrd_utils import getLogger, get_ocrd_tool_json from ocrd_validators import ParameterValidator from .database import ( @@ -221,11 +223,24 @@ def create_processing_message(job: DBProcessorJob) -> OcrdProcessingMessage: async def push_processor_job(self, processor_name: str, data: PYJobInput) -> PYJobOutput: """ Queue a processor job """ + if not self.rmq_publisher or not self.rmq_publisher._connection or not self.rmq_publisher._channel: + self.log.error('RMQPublisher is not connected') + raise Exception('RMQPublisher is not connected') + if processor_name not in self.processor_list: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail='Processor not available' - ) + try: + # Only checks if the process queue exists, if not raises ValueError + self.rmq_publisher.create_queue(processor_name, passive=True) + except ChannelClosedByBroker as error: + self.log.warning(f"Process queue with id '{processor_name}' not existing: {error}") + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=f"Process queue with id '{processor_name}' not existing" + ) + finally: + # Reconnect publisher - not efficient, but works + # TODO: Revisit when reconnection strategy is implemented + self.connect_publisher(enable_acks=True) # validate additional parameters if data.parameters: @@ -264,10 +279,14 @@ async def push_processor_job(self, processor_name: str, data: PYJobInput) -> PYJ await job.insert() processing_message = self.create_processing_message(job) encoded_processing_message = OcrdProcessingMessage.encode_yml(processing_message) - if self.rmq_publisher: + + try: self.rmq_publisher.publish_to_queue(processor_name, encoded_processing_message) - else: - raise Exception('RMQPublisher is not connected') + except Exception as error: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f'RMQPublisher has failed: {error}' + ) return job.to_job_output() async def get_processor_info(self, processor_name) -> Dict: diff --git a/ocrd_network/ocrd_network/rabbitmq_utils/publisher.py b/ocrd_network/ocrd_network/rabbitmq_utils/publisher.py index 9a3d080f9..1d8474ab2 100644 --- a/ocrd_network/ocrd_network/rabbitmq_utils/publisher.py +++ b/ocrd_network/ocrd_network/rabbitmq_utils/publisher.py @@ -62,7 +62,8 @@ def create_queue( self, queue_name: str, exchange_name: Optional[str] = None, - exchange_type: Optional[str] = None + exchange_type: Optional[str] = None, + passive: bool = False ) -> None: if exchange_name is None: exchange_name = DEFAULT_EXCHANGER_NAME @@ -76,7 +77,8 @@ def create_queue( ) RMQConnector.queue_declare( channel=self._channel, - queue_name=queue_name + queue_name=queue_name, + passive=passive ) RMQConnector.queue_bind( channel=self._channel, From 7835154080b984f4d1ff55507de6812d234304e7 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Mon, 27 Mar 2023 17:39:02 +0200 Subject: [PATCH 221/226] remove unnecessary checks --- ocrd_network/ocrd_network/processing_server.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ocrd_network/ocrd_network/processing_server.py b/ocrd_network/ocrd_network/processing_server.py index 870ff4940..988480ee3 100644 --- a/ocrd_network/ocrd_network/processing_server.py +++ b/ocrd_network/ocrd_network/processing_server.py @@ -223,8 +223,7 @@ def create_processing_message(job: DBProcessorJob) -> OcrdProcessingMessage: async def push_processor_job(self, processor_name: str, data: PYJobInput) -> PYJobOutput: """ Queue a processor job """ - if not self.rmq_publisher or not self.rmq_publisher._connection or not self.rmq_publisher._channel: - self.log.error('RMQPublisher is not connected') + if not self.rmq_publisher: raise Exception('RMQPublisher is not connected') if processor_name not in self.processor_list: From 1c37ad5dc95f2364d43b96ac9be852f2c72079c9 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Tue, 28 Mar 2023 11:00:45 +0200 Subject: [PATCH 222/226] basics of the procesor server --- ocrd/ocrd/cli/__init__.py | 2 + ocrd/ocrd/cli/processor_server.py | 48 ++++++ ocrd_network/ocrd_network/__init__.py | 1 + ocrd_network/ocrd_network/deployer.py | 6 + ocrd_network/ocrd_network/processor_server.py | 151 ++++++++++++++++++ 5 files changed, 208 insertions(+) create mode 100644 ocrd/ocrd/cli/processor_server.py create mode 100644 ocrd_network/ocrd_network/processor_server.py diff --git a/ocrd/ocrd/cli/__init__.py b/ocrd/ocrd/cli/__init__.py index d645daddf..5d0070640 100644 --- a/ocrd/ocrd/cli/__init__.py +++ b/ocrd/ocrd/cli/__init__.py @@ -33,6 +33,7 @@ def get_help(self, ctx): from .log import log_cli from .processing_server import processing_server_cli from .processing_worker import processing_worker_cli +from .processor_server import processor_server_cli @click.group() @@ -53,3 +54,4 @@ def cli(**kwargs): # pylint: disable=unused-argument cli.add_command(resmgr_cli) cli.add_command(processing_server_cli) cli.add_command(processing_worker_cli) +cli.add_command(processor_server_cli) diff --git a/ocrd/ocrd/cli/processor_server.py b/ocrd/ocrd/cli/processor_server.py new file mode 100644 index 000000000..b8a812b1e --- /dev/null +++ b/ocrd/ocrd/cli/processor_server.py @@ -0,0 +1,48 @@ +""" +OCR-D CLI: start the processor server + +.. click:: ocrd.cli.processor_server:processor_server_cli + :prog: ocrd processor-server + :nested: full +""" +import click +import logging +from ocrd_utils import initLogging +from ocrd_network import ( + DatabaseParamType, + ProcessingServerParamType, + ProcessorServer, +) + + +@click.command('processor-server') +@click.argument('processor_name', required=True, type=click.STRING) +@click.option('-d', '--address', + help='The URL of the processor server, format: host:port', + type=ProcessingServerParamType(), + required=True) +@click.option('-d', '--database', + default="mongodb://localhost:27018", + help='The URL of the MongoDB, format: mongodb://host:port', + type=DatabaseParamType()) +def processor_server_cli(processor_name: str, address: str, database: str): + """ + Start ocr-d processor as a server + """ + initLogging() + # TODO: Remove before the release + logging.getLogger('ocrd.network').setLevel(logging.DEBUG) + + # Note, the address is already validated with the type field + host, port = address.split(':') + + try: + processor_server = ProcessorServer( + processor_name=processor_name, + mongodb_addr=database, + processor_class=None, # For readability purposes assigned here + ) + processor_server.run_server(host=host, port=port, access_log=False) + + except Exception as e: + raise Exception("Processor server has failed with error") from e diff --git a/ocrd_network/ocrd_network/__init__.py b/ocrd_network/ocrd_network/__init__.py index 6cd95dc3c..e751ecbef 100644 --- a/ocrd_network/ocrd_network/__init__.py +++ b/ocrd_network/ocrd_network/__init__.py @@ -24,6 +24,7 @@ # the network package. The reason, Mets Server is tightly coupled with the `OcrdWorkspace`. from .processing_server import ProcessingServer from .processing_worker import ProcessingWorker +from .processor_server import ProcessorServer from .param_validators import ( DatabaseParamType, ProcessingServerParamType, diff --git a/ocrd_network/ocrd_network/deployer.py b/ocrd_network/ocrd_network/deployer.py index 27ac323fa..75d7852bc 100644 --- a/ocrd_network/ocrd_network/deployer.py +++ b/ocrd_network/ocrd_network/deployer.py @@ -77,6 +77,8 @@ def deploy_hosts(self, rabbitmq_url: str, mongodb_url: str) -> None: host.config.keypath ) + # TODO: Call the _deploy_processor_server() here and adapt accordingly + for processor in host.config.processors: self._deploy_processing_worker(processor, host, rabbitmq_url, mongodb_url) @@ -112,6 +114,10 @@ def _deploy_processing_worker(self, processor: WorkerConfig, host: HostData, host.pids_docker.append(pid) sleep(0.1) + def _deploy_processor_server(self, mongodb_url: str) -> None: + # TODO: Method for deploying a processor server + pass + def deploy_rabbitmq(self, image: str, detach: bool, remove: bool, ports_mapping: Union[Dict, None] = None) -> str: """Start docker-container with rabbitmq diff --git a/ocrd_network/ocrd_network/processor_server.py b/ocrd_network/ocrd_network/processor_server.py new file mode 100644 index 000000000..7b8325930 --- /dev/null +++ b/ocrd_network/ocrd_network/processor_server.py @@ -0,0 +1,151 @@ +from contextlib import redirect_stdout +from io import StringIO +from subprocess import run, PIPE +import uvicorn + +from fastapi import FastAPI, HTTPException, status, BackgroundTasks + +from ocrd import Resolver +# from ocrd.processor.helpers import run_processor_from_api, run_cli_from_api +from ocrd_validators import ParameterValidator +from ocrd_utils import ( + get_ocrd_tool_json, + parse_json_string_with_comments, + set_json_key_value_overrides +) + +from .database import ( + DBProcessorJob, + db_get_processing_job, + initiate_database +) +from .models import ( + PYJobInput, + PYJobOutput, + PYOcrdTool, + StateEnum +) + + +class ProcessorServer(FastAPI): + + def __init__(self, processor_name: str, mongodb_addr: str, processor_class=None): + self.processor_name = processor_name + self.db_url = mongodb_addr + self.ProcessorClass = processor_class + self.ocrd_tool = None + self.version = None + + self.version = self.get_version() + self.ocrd_tool = self.get_ocrd_tool() + + if not self.ocrd_tool: + raise Exception(f"The ocrd_tool is empty or missing") + + tags_metadata = [ + { + 'name': 'Processing', + 'description': 'OCR-D Processor Server' + } + ] + + super().__init__( + title=self.ocrd_tool['executable'], + description=self.ocrd_tool['description'], + version=self.version, + openapi_tags=tags_metadata, + on_startup=[self.startup] + ) + + # Create routes + self.router.add_api_route( + path='/', + endpoint=self.get_processor_info, + methods=['GET'], + tags=['Processing'], + status_code=status.HTTP_200_OK, + summary='Get information about this processor.', + response_model=PYOcrdTool, + response_model_exclude_unset=True, + response_model_exclude_none=True + ) + + self.router.add_api_route( + path='/', + endpoint=self.process, + methods=['POST'], + tags=['Processing'], + status_code=status.HTTP_202_ACCEPTED, + summary='Submit a job to this processor.', + response_model=PYJobOutput, + response_model_exclude_unset=True, + response_model_exclude_none=True + ) + + self.router.add_api_route( + path='/{job_id}', + endpoint=self.get_job, + methods=['GET'], + tags=['Processing'], + status_code=status.HTTP_200_OK, + summary='Get information about a job based on its ID', + response_model=PYJobOutput, + response_model_exclude_unset=True, + response_model_exclude_none=True + ) + + async def startup(self): + await initiate_database(db_url=self.db_url) + DBProcessorJob.Settings.name = self.processor_name + + async def get_processor_info(self): + return self.ocrd_tool + + async def process(self, data: PYJobInput, background_tasks: BackgroundTasks): + # TODO: Adapt from #884 + pass + + async def get_job(self, processor_name: str, job_id: str) -> PYJobOutput: + """ Return processing job-information from the database + """ + try: + job = await db_get_processing_job(job_id) + return job.to_job_output() + except ValueError: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=f"Processing job with id '{job_id}' of processor type '{processor_name}' not existing" + ) + + def get_ocrd_tool(self): + if self.ocrd_tool: + return self.ocrd_tool + if self.ProcessorClass: + str_out = StringIO() + with redirect_stdout(str_out): + self.ProcessorClass(workspace=None, dump_json=True) + ocrd_tool = parse_json_string_with_comments(str_out.getvalue()) + else: + ocrd_tool = get_ocrd_tool_json(self.processor_name) + return ocrd_tool + + def get_version(self) -> str: + if self.version: + return self.version + if self.ProcessorClass: + str_out = StringIO() + with redirect_stdout(str_out): + self.ProcessorClass(workspace=None, show_version=True) + version_str = str_out.getvalue() + else: + version_str = run( + [self.processor_name, '--version'], + stdout=PIPE, + check=True, + universal_newlines=True + ).stdout + # the version string is in format: Version %s, ocrd/core %s + return version_str + + def run_server(self, host, port): + uvicorn.run(self, host=host, port=port, access_log=False) From 42f16ab5f0e14b71cadd6ec8f2f37616f8adc5c1 Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Tue, 28 Mar 2023 12:08:44 +0200 Subject: [PATCH 223/226] Adapt the fix from #974 --- ocrd_network/ocrd_network/processing_server.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ocrd_network/ocrd_network/processing_server.py b/ocrd_network/ocrd_network/processing_server.py index 988480ee3..3e01c63e1 100644 --- a/ocrd_network/ocrd_network/processing_server.py +++ b/ocrd_network/ocrd_network/processing_server.py @@ -228,18 +228,17 @@ async def push_processor_job(self, processor_name: str, data: PYJobInput) -> PYJ if processor_name not in self.processor_list: try: - # Only checks if the process queue exists, if not raises ValueError + # Only checks if the process queue exists, if not raises ChannelClosedByBroker self.rmq_publisher.create_queue(processor_name, passive=True) except ChannelClosedByBroker as error: self.log.warning(f"Process queue with id '{processor_name}' not existing: {error}") + # Reconnect publisher - not efficient, but works + # TODO: Revisit when reconnection strategy is implemented + self.connect_publisher(enable_acks=True) raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=f"Process queue with id '{processor_name}' not existing" ) - finally: - # Reconnect publisher - not efficient, but works - # TODO: Revisit when reconnection strategy is implemented - self.connect_publisher(enable_acks=True) # validate additional parameters if data.parameters: From ba92c877f06e671e5adf845ee2d8d178bc2bae3e Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Tue, 28 Mar 2023 12:27:25 +0200 Subject: [PATCH 224/226] check defaults, pass shallow copy --- .../ocrd_network/processing_server.py | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/ocrd_network/ocrd_network/processing_server.py b/ocrd_network/ocrd_network/processing_server.py index 3e01c63e1..582c1134c 100644 --- a/ocrd_network/ocrd_network/processing_server.py +++ b/ocrd_network/ocrd_network/processing_server.py @@ -240,17 +240,16 @@ async def push_processor_job(self, processor_name: str, data: PYJobInput) -> PYJ detail=f"Process queue with id '{processor_name}' not existing" ) - # validate additional parameters - if data.parameters: - ocrd_tool = get_ocrd_tool_json(processor_name) - if not ocrd_tool: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Processor '{processor_name}' not available. Empty or missing ocrd_tool" - ) - report = ParameterValidator(ocrd_tool).validate(data.parameters) - if not report.is_valid: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=report.errors) + # validate parameters + ocrd_tool = get_ocrd_tool_json(processor_name) + if not ocrd_tool: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Processor '{processor_name}' not available. Empty or missing ocrd_tool" + ) + report = ParameterValidator(ocrd_tool).validate(dict(data.parameters)) + if not report.is_valid: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=report.errors) if bool(data.path_to_mets) == bool(data.workspace_id): raise HTTPException( From 0644b9e187f311fccbfadd3598732d68ab4e32dc Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Tue, 28 Mar 2023 17:38:22 +0200 Subject: [PATCH 225/226] complete processor server internals --- ocrd/ocrd/cli/processor_server.py | 2 +- ocrd_network/ocrd_network/deployer.py | 8 +- ocrd_network/ocrd_network/processor_server.py | 218 +++++++++++++++++- 3 files changed, 215 insertions(+), 13 deletions(-) diff --git a/ocrd/ocrd/cli/processor_server.py b/ocrd/ocrd/cli/processor_server.py index b8a812b1e..2665fdc97 100644 --- a/ocrd/ocrd/cli/processor_server.py +++ b/ocrd/ocrd/cli/processor_server.py @@ -38,8 +38,8 @@ def processor_server_cli(processor_name: str, address: str, database: str): try: processor_server = ProcessorServer( - processor_name=processor_name, mongodb_addr=database, + processor_name=processor_name, processor_class=None, # For readability purposes assigned here ) processor_server.run_server(host=host, port=port, access_log=False) diff --git a/ocrd_network/ocrd_network/deployer.py b/ocrd_network/ocrd_network/deployer.py index 75d7852bc..16b89c66b 100644 --- a/ocrd_network/ocrd_network/deployer.py +++ b/ocrd_network/ocrd_network/deployer.py @@ -90,7 +90,7 @@ def deploy_hosts(self, rabbitmq_url: str, mongodb_url: str) -> None: def _deploy_processing_worker(self, processor: WorkerConfig, host: HostData, rabbitmq_url: str, mongodb_url: str) -> None: - self.log.debug(f"deploy '{processor.deploy_type}' processor: '{processor}' on '{host.config.address}'") + self.log.debug(f"deploy '{processor.deploy_type}' worker: '{processor}' on '{host.config.address}'") for _ in range(processor.count): if processor.deploy_type == DeployType.native: @@ -114,7 +114,11 @@ def _deploy_processing_worker(self, processor: WorkerConfig, host: HostData, host.pids_docker.append(pid) sleep(0.1) - def _deploy_processor_server(self, mongodb_url: str) -> None: + def _deploy_processor_server(self, processor: WorkerConfig, host: HostData, mongodb_url: str) -> None: + self.log.debug(f"deploy '{processor.deploy_type}' processor server: '{processor}' on '{host.config.address}'") + + + # TODO: Method for deploying a processor server pass diff --git a/ocrd_network/ocrd_network/processor_server.py b/ocrd_network/ocrd_network/processor_server.py index 7b8325930..03f6cadb7 100644 --- a/ocrd_network/ocrd_network/processor_server.py +++ b/ocrd_network/ocrd_network/processor_server.py @@ -1,22 +1,29 @@ from contextlib import redirect_stdout +from datetime import datetime from io import StringIO +import json +import logging +from os import environ, getpid from subprocess import run, PIPE +from typing import List import uvicorn from fastapi import FastAPI, HTTPException, status, BackgroundTasks +from ocrd.processor.helpers import run_cli, run_processor from ocrd import Resolver -# from ocrd.processor.helpers import run_processor_from_api, run_cli_from_api from ocrd_validators import ParameterValidator from ocrd_utils import ( + getLogger, get_ocrd_tool_json, - parse_json_string_with_comments, - set_json_key_value_overrides + parse_json_string_with_comments ) from .database import ( DBProcessorJob, db_get_processing_job, + db_get_workspace, + db_update_processing_job, initiate_database ) from .models import ( @@ -25,13 +32,36 @@ PYOcrdTool, StateEnum ) +from .utils import calculate_execution_time, generate_id + +# TODO: Check this again when the logging is refactored +try: + # This env variable must be set before importing from Keras + environ['TF_CPP_MIN_LOG_LEVEL'] = '3' + from tensorflow.keras.utils import disable_interactive_logging + # Enabled interactive logging throws an exception + # due to a call of sys.stdout.flush() + disable_interactive_logging() +except Exception: + # Nothing should be handled here if TF is not available + pass class ProcessorServer(FastAPI): + def __init__(self, mongodb_addr: str, processor_name: str = "", processor_class=None): + if not (processor_name or processor_class): + raise ValueError('Either "processor_name" or "processor_class" must be provided') + + self.log = getLogger(__name__) + # TODO: Provide more flexibility for configuring file logging (i.e. via ENV variables) + file_handler = logging.FileHandler(f'/tmp/server_{processor_name}_{getpid()}.log', mode='a') + logging_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' + file_handler.setFormatter(logging.Formatter(logging_format)) + file_handler.setLevel(logging.DEBUG) + self.log.addHandler(file_handler) - def __init__(self, processor_name: str, mongodb_addr: str, processor_class=None): - self.processor_name = processor_name self.db_url = mongodb_addr + self.processor_name = processor_name self.ProcessorClass = processor_class self.ocrd_tool = None self.version = None @@ -42,6 +72,9 @@ def __init__(self, processor_name: str, mongodb_addr: str, processor_class=None) if not self.ocrd_tool: raise Exception(f"The ocrd_tool is empty or missing") + if not self.processor_name: + self.processor_name = self.ocrd_tool['executable'] + tags_metadata = [ { 'name': 'Processing', @@ -50,7 +83,7 @@ def __init__(self, processor_name: str, mongodb_addr: str, processor_class=None) ] super().__init__( - title=self.ocrd_tool['executable'], + title=self.processor_name, description=self.ocrd_tool['description'], version=self.version, openapi_tags=tags_metadata, @@ -99,11 +132,73 @@ async def startup(self): DBProcessorJob.Settings.name = self.processor_name async def get_processor_info(self): + if not self.ocrd_tool: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f'Empty or missing ocrd_tool' + ) return self.ocrd_tool - async def process(self, data: PYJobInput, background_tasks: BackgroundTasks): - # TODO: Adapt from #884 - pass + # Note: The Processing server pushes to a queue, while + # the Processor Server creates (pushes to) a background task + async def push_processor_job(self, data: PYJobInput, background_tasks: BackgroundTasks): + if not self.ocrd_tool: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f'Empty or missing ocrd_tool' + ) + report = ParameterValidator(self.ocrd_tool).validate(dict(data.parameters)) + if not report.is_valid: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=report.errors) + + if bool(data.path_to_mets) == bool(data.workspace_id): + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail="Either 'path' or 'workspace_id' must be provided, but not both" + ) + + # This check is done to return early in case + # the workspace_id is provided but not existing in the DB + elif data.workspace_id: + try: + await db_get_workspace(data.workspace_id) + except ValueError: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=f"Workspace with id '{data.workspace_id}' not existing" + ) + + job = DBProcessorJob( + **data.dict(exclude_unset=True, exclude_none=True), + job_id=generate_id(), + processor_name=self.processor_name, + state=StateEnum.queued + ) + await job.insert() + + if self.ProcessorClass: + # Run the processor in the background + background_tasks.add_task( + self.run_processor_from_server, + job_id=job.id, + workspace_id=data.workspace_id, + page_id=data.page_id, + parameter=data.parameters, + input_file_grps=data.input_file_grps, + output_file_grps=data.output_file_grps, + ) + else: + # Run the CLI in the background + background_tasks.add_task( + self.run_cli_from_server, + job_id=job.id, + workspace_id=data.workspace_id, + page_id=data.page_id, + input_file_grps=data.input_file_grps, + output_file_grps=data.output_file_grps, + parameter=data.parameters + ) + return job.to_job_output() async def get_job(self, processor_name: str, job_id: str) -> PYJobOutput: """ Return processing job-information from the database @@ -144,8 +239,111 @@ def get_version(self) -> str: check=True, universal_newlines=True ).stdout - # the version string is in format: Version %s, ocrd/core %s return version_str def run_server(self, host, port): uvicorn.run(self, host=host, port=port, access_log=False) + + async def run_cli_from_server( + self, + job_id: str, + processor_name: str, + workspace_id: str, + input_file_grps: List[str], + output_file_grps: List[str], + page_id: str, + parameters: dict + ): + log = getLogger('ocrd.processor.helpers.run_cli_from_api') + + # Turn input/output file groups into a comma separated string + input_file_grps_str = ','.join(input_file_grps) + output_file_grps_str = ','.join(output_file_grps) + + workspace_db = await db_get_workspace(workspace_id) + path_to_mets = workspace_db.workspace_mets_path + workspace = Resolver().workspace_from_url(path_to_mets) + + start_time = datetime.now() + await db_update_processing_job( + job_id=job_id, + state=StateEnum.running, + start_time=start_time + ) + # Execute the processor + return_code = run_cli( + executable=processor_name, + workspace=workspace, + page_id=page_id, + input_file_grp=input_file_grps_str, + output_file_grp=output_file_grps_str, + parameter=json.dumps(parameters), + mets_url=workspace.mets_target + ) + end_time = datetime.now() + # Execution duration in ms + execution_duration = calculate_execution_time(start_time, end_time) + + if return_code != 0: + job_state = StateEnum.failed + log.error(f'{self.processor_name} exited with non-zero return value {return_code}.') + else: + job_state = StateEnum.success + + await db_update_processing_job( + job_id=job_id, + state=job_state, + end_time=end_time, + exec_time=f'{execution_duration} ms' + ) + + async def run_processor_from_server( + self, + job_id: str, + workspace_id: str, + input_file_grps: List[str], + output_file_grps: List[str], + page_id: str, + parameters: dict, + ): + log = getLogger('ocrd.processor.helpers.run_processor_from_api') + + # Turn input/output file groups into a comma separated string + input_file_grps_str = ','.join(input_file_grps) + output_file_grps_str = ','.join(output_file_grps) + + workspace_db = await db_get_workspace(workspace_id) + path_to_mets = workspace_db.workspace_mets_path + workspace = Resolver().workspace_from_url(path_to_mets) + + is_success = True + start_time = datetime.now() + await db_update_processing_job( + job_id=job_id, + state=StateEnum.running, + start_time=start_time + ) + try: + run_processor( + processorClass=self.ProcessorClass, + workspace=workspace, + page_id=page_id, + parameter=parameters, + input_file_grp=input_file_grps_str, + output_file_grp=output_file_grps_str, + instance_caching=True + ) + except Exception as e: + is_success = False + log.exception(e) + + end_time = datetime.now() + # Execution duration in ms + execution_duration = calculate_execution_time(start_time, end_time) + job_state = StateEnum.success if is_success else StateEnum.failed + await db_update_processing_job( + job_id=job_id, + state=job_state, + end_time=end_time, + exec_time=f'{execution_duration} ms' + ) From cdb73d5b53620fad609b2960a5c78f59e369c2be Mon Sep 17 00:00:00 2001 From: mehmedGIT Date: Wed, 29 Mar 2023 01:13:15 +0200 Subject: [PATCH 226/226] implement most of the logic for everything --- ocrd/ocrd/cli/processing_worker.py | 13 +- ocrd/ocrd/cli/processor_server.py | 20 +- ocrd/ocrd/decorators/__init__.py | 215 ++++++++++-------- ocrd/ocrd/decorators/ocrd_cli_options.py | 4 +- ocrd_network/ocrd_network/deployer.py | 134 ++++++++--- .../ocrd_network/deployment_config.py | 16 ++ ocrd_network/ocrd_network/deployment_utils.py | 9 +- ocrd_network/ocrd_network/models/job.py | 3 + .../ocrd_network/processing_server.py | 166 +++++++++++--- .../ocrd_network/processing_worker.py | 3 +- ocrd_network/ocrd_network/processor_server.py | 38 ++-- .../processing_server_config.schema.yml | 34 ++- 12 files changed, 467 insertions(+), 188 deletions(-) diff --git a/ocrd/ocrd/cli/processing_worker.py b/ocrd/ocrd/cli/processing_worker.py index e9311e061..3e568b268 100644 --- a/ocrd/ocrd/cli/processing_worker.py +++ b/ocrd/ocrd/cli/processing_worker.py @@ -20,15 +20,22 @@ @click.command('processing-worker') @click.argument('processor_name', required=True, type=click.STRING) +@click.option('--agent_type', + help='The type of this network agent', + default="worker", + type=click.STRING, + required=True) @click.option('-q', '--queue', default="amqp://admin:admin@localhost:5672/", help='The URL of the Queue Server, format: amqp://username:password@host:port/vhost', - type=QueueServerParamType()) + type=QueueServerParamType(), + required=True) @click.option('-d', '--database', default="mongodb://localhost:27018", help='The URL of the MongoDB, format: mongodb://host:port', - type=DatabaseParamType()) -def processing_worker_cli(processor_name: str, queue: str, database: str): + type=DatabaseParamType(), + required=True) +def processing_worker_cli(processor_name: str, agent_type: str, queue: str, database: str): """ Start a processing worker (a specific ocr-d processor) """ diff --git a/ocrd/ocrd/cli/processor_server.py b/ocrd/ocrd/cli/processor_server.py index 2665fdc97..cbff6a8f9 100644 --- a/ocrd/ocrd/cli/processor_server.py +++ b/ocrd/ocrd/cli/processor_server.py @@ -17,15 +17,21 @@ @click.command('processor-server') @click.argument('processor_name', required=True, type=click.STRING) -@click.option('-d', '--address', +@click.option('--agent_type', + help='The type of this network agent', + default="server", + type=click.STRING, + required=True) +@click.option('--agent_address', help='The URL of the processor server, format: host:port', type=ProcessingServerParamType(), required=True) @click.option('-d', '--database', default="mongodb://localhost:27018", help='The URL of the MongoDB, format: mongodb://host:port', - type=DatabaseParamType()) -def processor_server_cli(processor_name: str, address: str, database: str): + type=DatabaseParamType(), + required=True) +def processor_server_cli(processor_name: str, agent_type: str, agent_address: str, database: str): """ Start ocr-d processor as a server """ @@ -33,16 +39,14 @@ def processor_server_cli(processor_name: str, address: str, database: str): # TODO: Remove before the release logging.getLogger('ocrd.network').setLevel(logging.DEBUG) - # Note, the address is already validated with the type field - host, port = address.split(':') - try: + # TODO: Better validate that inside the ProcessorServer itself + host, port = agent_address.split(':') processor_server = ProcessorServer( mongodb_addr=database, processor_name=processor_name, processor_class=None, # For readability purposes assigned here ) - processor_server.run_server(host=host, port=port, access_log=False) - + processor_server.run_server(host=host, port=int(port)) except Exception as e: raise Exception("Processor server has failed with error") from e diff --git a/ocrd/ocrd/decorators/__init__.py b/ocrd/ocrd/decorators/__init__.py index 2cffe12fe..1aaeedaf1 100644 --- a/ocrd/ocrd/decorators/__init__.py +++ b/ocrd/ocrd/decorators/__init__.py @@ -15,7 +15,7 @@ from ocrd_utils import getLogger, initLogging, parse_json_string_with_comments from ocrd_validators import WorkspaceValidator -from ocrd_network import ProcessingWorker +from ocrd_network import ProcessingWorker, ProcessorServer from ..resolver import Resolver from ..processor.base import run_processor @@ -38,8 +38,12 @@ def ocrd_cli_wrap_processor( overwrite=False, show_resource=None, list_resources=False, + # ocrd_network params start # + agent_type=None, + agent_address=None, queue=None, database=None, + # ocrd_network params end # **kwargs ): if not sys.argv[1:]: @@ -56,96 +60,123 @@ def ocrd_cli_wrap_processor( list_resources=list_resources ) sys.exit() - # If either of these two is provided but not both - if bool(queue) != bool(database): - raise Exception("Options --queue and --database require each other.") - # If both of these are provided - start the processing worker instead of the processor - processorClass - if queue and database: - initLogging() - # TODO: Remove before the release - # We are importing the logging here because it's not the ocrd logging but python one - import logging - logging.getLogger('ocrd.network').setLevel(logging.DEBUG) - # Get the ocrd_tool dictionary - processor = processorClass(workspace=None, dump_json=True) - ocrd_tool = processor.ocrd_tool + # Used for checking/starting network agents for the WebAPI architecture + # Has no side effects if neither of the 4 ocrd_network parameters are passed + check_and_run_network_agent(processorClass, agent_type, agent_address, database, queue) - try: - processing_worker = ProcessingWorker( - rabbitmq_addr=queue, - mongodb_addr=database, - processor_name=ocrd_tool['executable'], - ocrd_tool=ocrd_tool, - processor_class=processorClass, - ) - # The RMQConsumer is initialized and a connection to the RabbitMQ is performed - processing_worker.connect_consumer() - # Start consuming from the queue with name `processor_name` - processing_worker.start_consuming() - except Exception as e: - raise Exception("Processing worker has failed with error") from e - else: - initLogging() - LOG = getLogger('ocrd_cli_wrap_processor') - # LOG.info('kwargs=%s' % kwargs) - # Merge parameter overrides and parameters - if 'parameter_override' in kwargs: - set_json_key_value_overrides(kwargs['parameter'], *kwargs['parameter_override']) - # TODO OCR-D/core#274 - # Assert -I / -O - # if not kwargs['input_file_grp']: - # raise ValueError('-I/--input-file-grp is required') - # if not kwargs['output_file_grp']: - # raise ValueError('-O/--output-file-grp is required') - resolver = Resolver() - working_dir, mets, _ = resolver.resolve_mets_arguments(working_dir, mets, None) - workspace = resolver.workspace_from_url(mets, working_dir) - page_id = kwargs.get('page_id') - # XXX not possible while processors do not adhere to # https://github.com/OCR-D/core/issues/505 - # if overwrite - # if 'output_file_grp' not in kwargs or not kwargs['output_file_grp']: - # raise Exception("--overwrite requires --output-file-grp") - # LOG.info("Removing files because of --overwrite") - # for grp in kwargs['output_file_grp'].split(','): - # if page_id: - # for one_page_id in kwargs['page_id'].split(','): - # LOG.debug("Removing files in output file group %s with page ID %s", grp, one_page_id) - # for file in workspace.mets.find_files(pageId=one_page_id, fileGrp=grp): - # workspace.remove_file(file, force=True, keep_file=False, page_recursive=True) - # else: - # LOG.debug("Removing all files in output file group %s ", grp) - # # TODO: can be reduced to `page_same_group=True` as soon as core#505 has landed (in all processors) - # workspace.remove_file_group(grp, recursive=True, force=True, keep_files=False, page_recursive=True, page_same_group=False) - # workspace.save_mets() - # XXX While https://github.com/OCR-D/core/issues/505 is open, set 'overwrite_mode' globally on the workspace - if overwrite: - workspace.overwrite_mode = True - report = WorkspaceValidator.check_file_grp(workspace, kwargs['input_file_grp'], '' if overwrite else kwargs['output_file_grp'], page_id) - if not report.is_valid: - raise Exception("Invalid input/output file grps:\n\t%s" % '\n\t'.join(report.errors)) - # Set up profiling behavior from environment variables/flags - if not profile and 'OCRD_PROFILE' in environ: - if 'CPU' in environ['OCRD_PROFILE']: - profile = True - if not profile_file and 'OCRD_PROFILE_FILE' in environ: - profile_file = environ['OCRD_PROFILE_FILE'] - if profile or profile_file: - import cProfile - import pstats - import io - import atexit - print("Profiling...") - pr = cProfile.Profile() - pr.enable() - def exit(): - pr.disable() - print("Profiling completed") - if profile_file: - with open(profile_file, 'wb') as f: - pr.dump_stats(profile_file) - s = io.StringIO() - pstats.Stats(pr, stream=s).sort_stats("cumulative").print_stats() - print(s.getvalue()) - atexit.register(exit) - run_processor(processorClass, mets_url=mets, workspace=workspace, **kwargs) + initLogging() + LOG = getLogger('ocrd_cli_wrap_processor') + # LOG.info('kwargs=%s' % kwargs) + # Merge parameter overrides and parameters + if 'parameter_override' in kwargs: + set_json_key_value_overrides(kwargs['parameter'], *kwargs['parameter_override']) + # TODO OCR-D/core#274 + # Assert -I / -O + # if not kwargs['input_file_grp']: + # raise ValueError('-I/--input-file-grp is required') + # if not kwargs['output_file_grp']: + # raise ValueError('-O/--output-file-grp is required') + resolver = Resolver() + working_dir, mets, _ = resolver.resolve_mets_arguments(working_dir, mets, None) + workspace = resolver.workspace_from_url(mets, working_dir) + page_id = kwargs.get('page_id') + # XXX not possible while processors do not adhere to # https://github.com/OCR-D/core/issues/505 + # if overwrite + # if 'output_file_grp' not in kwargs or not kwargs['output_file_grp']: + # raise Exception("--overwrite requires --output-file-grp") + # LOG.info("Removing files because of --overwrite") + # for grp in kwargs['output_file_grp'].split(','): + # if page_id: + # for one_page_id in kwargs['page_id'].split(','): + # LOG.debug("Removing files in output file group %s with page ID %s", grp, one_page_id) + # for file in workspace.mets.find_files(pageId=one_page_id, fileGrp=grp): + # workspace.remove_file(file, force=True, keep_file=False, page_recursive=True) + # else: + # LOG.debug("Removing all files in output file group %s ", grp) + # # TODO: can be reduced to `page_same_group=True` as soon as core#505 has landed (in all processors) + # workspace.remove_file_group(grp, recursive=True, force=True, keep_files=False, page_recursive=True, page_same_group=False) + # workspace.save_mets() + # XXX While https://github.com/OCR-D/core/issues/505 is open, set 'overwrite_mode' globally on the workspace + if overwrite: + workspace.overwrite_mode = True + report = WorkspaceValidator.check_file_grp(workspace, kwargs['input_file_grp'], '' if overwrite else kwargs['output_file_grp'], page_id) + if not report.is_valid: + raise Exception("Invalid input/output file grps:\n\t%s" % '\n\t'.join(report.errors)) + # Set up profiling behavior from environment variables/flags + if not profile and 'OCRD_PROFILE' in environ: + if 'CPU' in environ['OCRD_PROFILE']: + profile = True + if not profile_file and 'OCRD_PROFILE_FILE' in environ: + profile_file = environ['OCRD_PROFILE_FILE'] + if profile or profile_file: + import cProfile + import pstats + import io + import atexit + print("Profiling...") + pr = cProfile.Profile() + pr.enable() + def exit(): + pr.disable() + print("Profiling completed") + if profile_file: + with open(profile_file, 'wb') as f: + pr.dump_stats(profile_file) + s = io.StringIO() + pstats.Stats(pr, stream=s).sort_stats("cumulative").print_stats() + print(s.getvalue()) + atexit.register(exit) + run_processor(processorClass, mets_url=mets, workspace=workspace, **kwargs) + + +def check_and_run_network_agent(ProcessorClass, agent_type: str, agent_address: str, database: str, queue: str): + if not agent_type and (agent_address or database or queue): + raise ValueError("Options '--database', '--queue', and 'agent_address' are valid only with '--agent_type'") + if agent_type: + if not database: + raise ValueError("Options '--agent_type' and '--database' are mutually inclusive") + allowed_agent_types = ['server', 'worker'] + if agent_type not in allowed_agent_types: + agents_str = ', '.join(allowed_agent_types) + raise ValueError(f"Wrong agent type parameter. Allowed agent types: {agents_str}") + if agent_type == 'server': + if not agent_address: + raise ValueError("Options '--agent_type=server' and '--agent_address' are mutually inclusive") + if queue: + raise ValueError("Options '--agent_type=server' and '--queue' are mutually exclusive") + if agent_type == 'worker': + if not queue: + raise ValueError("Options '--agent_type=worker' and '--queue' are mutually inclusive") + if agent_address: + raise ValueError("Options '--agent_type=worker' and '--agent_address' are mutually exclusive") + + processor = ProcessorClass(workspace=None, dump_json=True) + if agent_type == 'worker': + try: + # TODO: Passing processor_name and ocrd_tool is reduntant + processing_worker = ProcessingWorker( + rabbitmq_addr=queue, + mongodb_addr=database, + processor_name=processor.ocrd_tool['executable'], + ocrd_tool=processor.ocrd_tool, + processor_class=ProcessorClass, + ) + # The RMQConsumer is initialized and a connection to the RabbitMQ is performed + processing_worker.connect_consumer() + # Start consuming from the queue with name `processor_name` + processing_worker.start_consuming() + except Exception as e: + raise Exception("Processing worker has failed with error") from e + if agent_type == 'server': + try: + # TODO: Better validate that inside the ProcessorServer itself + host, port = agent_address.split(':') + processor_server = ProcessorServer( + mongodb_addr=database, + processor_name=processor.ocrd_tool['executable'], + processor_class=ProcessorClass, + ) + processor_server.run_server(host=host, port=int(port)) + except Exception as e: + raise Exception("Processor server has failed with error") from e diff --git a/ocrd/ocrd/decorators/ocrd_cli_options.py b/ocrd/ocrd/decorators/ocrd_cli_options.py index 2ba4bf8ae..42bed275b 100644 --- a/ocrd/ocrd/decorators/ocrd_cli_options.py +++ b/ocrd/ocrd/decorators/ocrd_cli_options.py @@ -1,3 +1,4 @@ +import click from click import option, Path from .parameter_option import parameter_option, parameter_override_option from .loglevel_option import loglevel_option @@ -33,6 +34,8 @@ def cli(mets_url): parameter_option, parameter_override_option, loglevel_option, + option('--agent_type', type=click.STRING), + option('--agent_address', type=click.STRING), option('--queue', type=QueueServerParamType()), option('--database', type=DatabaseParamType()), option('-C', '--show-resource'), @@ -45,4 +48,3 @@ def cli(mets_url): for param in params: param(f) return f - diff --git a/ocrd_network/ocrd_network/deployer.py b/ocrd_network/ocrd_network/deployer.py index 16b89c66b..c0b8c39ed 100644 --- a/ocrd_network/ocrd_network/deployer.py +++ b/ocrd_network/ocrd_network/deployer.py @@ -11,6 +11,7 @@ from typing import Dict, Union from paramiko import SSHClient from re import search as re_search +from os import getpid from time import sleep @@ -77,11 +78,30 @@ def deploy_hosts(self, rabbitmq_url: str, mongodb_url: str) -> None: host.config.keypath ) - # TODO: Call the _deploy_processor_server() here and adapt accordingly - for processor in host.config.processors: self._deploy_processing_worker(processor, host, rabbitmq_url, mongodb_url) + # TODO: This is not optimal - the entire method should be refactored! + if (any(s.deploy_type == DeployType.native for s in host.config.servers) + and not host.ssh_client): + host.ssh_client = create_ssh_client( + host.config.address, + host.config.username, + host.config.password, + host.config.keypath + ) + if (any(s.deploy_type == DeployType.docker for s in host.config.servers) + and not host.docker_client): + host.docker_client = create_docker_client( + host.config.address, + host.config.username, + host.config.password, + host.config.keypath + ) + + for server in host.config.servers: + self._deploy_processor_server(server, host, mongodb_url) + if host.ssh_client: host.ssh_client.close() if host.docker_client: @@ -89,8 +109,7 @@ def deploy_hosts(self, rabbitmq_url: str, mongodb_url: str) -> None: def _deploy_processing_worker(self, processor: WorkerConfig, host: HostData, rabbitmq_url: str, mongodb_url: str) -> None: - - self.log.debug(f"deploy '{processor.deploy_type}' worker: '{processor}' on '{host.config.address}'") + self.log.debug(f"deploy '{processor.deploy_type}' processing worker: '{processor.name}' on '{host.config.address}'") for _ in range(processor.count): if processor.deploy_type == DeployType.native: @@ -114,13 +133,28 @@ def _deploy_processing_worker(self, processor: WorkerConfig, host: HostData, host.pids_docker.append(pid) sleep(0.1) - def _deploy_processor_server(self, processor: WorkerConfig, host: HostData, mongodb_url: str) -> None: - self.log.debug(f"deploy '{processor.deploy_type}' processor server: '{processor}' on '{host.config.address}'") - - - - # TODO: Method for deploying a processor server - pass + # TODO: Revisit this to remove code duplications of deploy_* methods + def _deploy_processor_server(self, server: ProcessorServerConfig, host: HostData, mongodb_url: str) -> None: + self.log.debug(f"deploy '{server.deploy_type}' processor server: '{server.name}' on '{host.config.address}'") + if server.deploy_type == DeployType.native: + assert host.ssh_client + pid = self.start_native_processor_server( + client=host.ssh_client, + processor_name=server.name, + agent_address=f'{host.config.address}:{server.port}', + database_url=mongodb_url, + ) + host.processor_server_pids_native.append(pid) + + if server.name in host.processor_server_ports: + if host.processor_server_ports[server.name]: + host.processor_server_ports[server.name] = host.processor_server_ports[server.name].append(server.port) + else: + host.processor_server_ports[server.name] = [server.port] + else: + host.processor_server_ports[server.name] = [server.port] + else: + raise Exception("Deploying docker processor server is not supported yet!") def deploy_rabbitmq(self, image: str, detach: bool, remove: bool, ports_mapping: Union[Dict, None] = None) -> str: @@ -261,18 +295,43 @@ def kill_hosts(self) -> None: host.docker_client = create_docker_client(host.config.address, host.config.username, host.config.password, host.config.keypath) # Kill deployed OCR-D processor instances on this Processing worker host - self.kill_processing_worker(host) - - def kill_processing_worker(self, host: HostData) -> None: - for pid in host.pids_native: - self.log.debug(f"Trying to kill/stop native processor: with PID: '{pid}'") - host.ssh_client.exec_command(f'kill {pid}') - host.pids_native = [] - - for pid in host.pids_docker: - self.log.debug(f"Trying to kill/stop docker container with PID: '{pid}'") - host.docker_client.containers.get(pid).stop() - host.pids_docker = [] + self.kill_processing_workers(host) + + # Kill deployed Processor Server instances on this host + self.kill_processor_servers(host) + + # TODO: Optimize the code duplication from start_* and kill_* methods + def kill_processing_workers(self, host: HostData) -> None: + amount = len(host.pids_native) + if amount: + self.log.info(f"Trying to kill/stop {amount} native processing workers:") + for pid in host.pids_native: + self.log.info(f"Native with PID: '{pid}'") + host.ssh_client.exec_command(f'kill {pid}') + host.pids_native = [] + amount = len(host.pids_docker) + if amount: + self.log.info(f"Trying to kill/stop {amount} docker processing workers:") + for pid in host.pids_docker: + self.log.info(f"Docker with PID: '{pid}'") + host.docker_client.containers.get(pid).stop() + host.pids_docker = [] + + def kill_processor_servers(self, host: HostData) -> None: + amount = len(host.processor_server_pids_native) + if amount: + self.log.info(f"Trying to kill/stop {amount} native processor servers:") + for pid in host.processor_server_pids_native: + self.log.info(f"Native with PID: '{pid}'") + host.ssh_client.exec_command(f'kill {pid}') + host.processor_server_pids_native = [] + amount = len(host.processor_server_pids_docker) + if amount: + self.log.info(f"Trying to kill/stop {amount} docker processor servers:") + for pid in host.processor_server_pids_docker: + self.log.info(f"Docker with PID: '{pid}'") + host.docker_client.containers.get(pid).stop() + host.processor_server_pids_docker = [] def start_native_processor(self, client: SSHClient, processor_name: str, queue_url: str, database_url: str) -> str: @@ -287,17 +346,17 @@ def start_native_processor(self, client: SSHClient, processor_name: str, queue_u Returns: str: pid of running process """ - self.log.info(f'Starting native processor: {processor_name}') + self.log.info(f'Starting native processing worker: {processor_name}') channel = client.invoke_shell() stdin, stdout = channel.makefile('wb'), channel.makefile('rb') - cmd = f'{processor_name} --database {database_url} --queue {queue_url}' + cmd = f'{processor_name} --agent_type worker --database {database_url} --queue {queue_url}' # the only way (I could find) to make it work to start a process in the background and # return early is this construction. The pid of the last started background process is # printed with `echo $!` but it is printed inbetween other output. Because of that I added # `xyz` before and after the code to easily be able to filter out the pid via regex when # returning from the function logpath = '/tmp/ocrd-processing-server-startup.log' - stdin.write(f"echo starting processor with '{cmd}' >> '{logpath}'\n") + stdin.write(f"echo starting processing worker with '{cmd}' >> '{logpath}'\n") stdin.write(f'{cmd} >> {logpath} 2>&1 &\n') stdin.write('echo xyz$!xyz \n exit \n') output = stdout.read().decode('utf-8') @@ -307,8 +366,31 @@ def start_native_processor(self, client: SSHClient, processor_name: str, queue_u def start_docker_processor(self, client: CustomDockerClient, processor_name: str, queue_url: str, database_url: str) -> str: + + # TODO: Raise an exception here as well? + # raise Exception("Deploying docker processing worker is not supported yet!") + self.log.info(f'Starting docker container processor: {processor_name}') # TODO: add real command here to start processing server in docker here res = client.containers.run('debian', 'sleep 500s', detach=True, remove=True) assert res and res.id, f'Running processor: {processor_name} in docker-container failed' return res.id + + # TODO: Just a copy of the above start_native_processor() method. + # Far from being great... But should be good as a starting point + def start_native_processor_server(self, client: SSHClient, processor_name: str, agent_address: str, database_url: str) -> str: + self.log.info(f"Starting native processor server: {processor_name} on {agent_address}") + channel = client.invoke_shell() + stdin, stdout = channel.makefile('wb'), channel.makefile('rb') + cmd = f'{processor_name} --agent_type server --agent_address {agent_address} --database {database_url}' + port = agent_address.split(':')[1] + logpath = f'/tmp/server_{processor_name}_{port}_{getpid()}.log' + # TODO: This entire stdin/stdout thing is broken with servers! + stdin.write(f"echo starting processor server with '{cmd}' >> '{logpath}'\n") + stdin.write(f'{cmd} >> {logpath} 2>&1 &\n') + stdin.write('echo xyz$!xyz \n exit \n') + output = stdout.read().decode('utf-8') + stdout.close() + stdin.close() + return re_search(r'xyz([0-9]+)xyz', output).group(1) # type: ignore + pass diff --git a/ocrd_network/ocrd_network/deployment_config.py b/ocrd_network/ocrd_network/deployment_config.py index 48b123d1a..de5465a91 100644 --- a/ocrd_network/ocrd_network/deployment_config.py +++ b/ocrd_network/ocrd_network/deployment_config.py @@ -8,6 +8,7 @@ 'HostConfig', 'WorkerConfig', 'MongoConfig', + 'ProcessorServerConfig', 'QueueConfig', ] @@ -49,6 +50,12 @@ def __init__(self, config: dict) -> None: self.processors.append( WorkerConfig(worker['name'], worker['number_of_instance'], deploy_type) ) + self.servers = [] + for server in config['servers']: + deploy_type = DeployType.from_str(server['deploy_type']) + self.servers.append( + ProcessorServerConfig(server['name'], deploy_type, server['port']) + ) class WorkerConfig: @@ -61,6 +68,15 @@ def __init__(self, name: str, count: int, deploy_type: DeployType) -> None: self.deploy_type = deploy_type +# TODO: Not a big fan of the way these configs work... +# Implemented this way to fit the general logic of previous impl +class ProcessorServerConfig: + def __init__(self, name: str, deploy_type: DeployType, port: int): + self.name = name + self.deploy_type = deploy_type + self.port = port + + class MongoConfig: """ Class to hold information for Mongodb-Docker container """ diff --git a/ocrd_network/ocrd_network/deployment_utils.py b/ocrd_network/ocrd_network/deployment_utils.py index 6b943127b..8f6c5f272 100644 --- a/ocrd_network/ocrd_network/deployment_utils.py +++ b/ocrd_network/ocrd_network/deployment_utils.py @@ -16,8 +16,7 @@ 'create_ssh_client', 'CustomDockerClient', 'DeployType', - 'HostData', - 'is_bashlib_processor' + 'HostData' ] @@ -38,7 +37,6 @@ def create_docker_client(address: str, username: str, password: Union[str, None] return CustomDockerClient(username, address, password=password, keypath=keypath) - class HostData: """class to store runtime information for a host """ @@ -48,6 +46,11 @@ def __init__(self, config: HostConfig) -> None: self.docker_client: Union[CustomDockerClient, None] = None self.pids_native: List[str] = [] self.pids_docker: List[str] = [] + # TODO: Revisit this, currently just mimicking the old impl + self.processor_server_pids_native: List[str] = [] + self.processor_server_pids_docker: List[str] = [] + # Key: processor_name, Value: list of ports + self.processor_server_ports: dict = {} @staticmethod def from_config(config: List[HostConfig]) -> List[HostData]: diff --git a/ocrd_network/ocrd_network/models/job.py b/ocrd_network/ocrd_network/models/job.py index 3c0857c37..1ccc82fe9 100644 --- a/ocrd_network/ocrd_network/models/job.py +++ b/ocrd_network/ocrd_network/models/job.py @@ -25,6 +25,9 @@ class PYJobInput(BaseModel): parameters: dict = {} # Always set to empty dict when None, otherwise it fails ocr-d-validation result_queue_name: Optional[str] = None callback_url: Optional[str] = None + # Used to toggle between sending requests to 'worker and 'server', + # i.e., Processing Worker and Processor Server, respectively + agent_type: Optional[str] = 'worker' class Config: schema_extra = { diff --git a/ocrd_network/ocrd_network/processing_server.py b/ocrd_network/ocrd_network/processing_server.py index 582c1134c..43fa5776f 100644 --- a/ocrd_network/ocrd_network/processing_server.py +++ b/ocrd_network/ocrd_network/processing_server.py @@ -1,4 +1,5 @@ -from typing import Dict +import requests +from typing import Dict, List import uvicorn from fastapi import FastAPI, status, Request, HTTPException @@ -56,8 +57,14 @@ def __init__(self, config_path: str, host: str, port: int) -> None: # Gets assigned when `connect_publisher` is called on the working object self.rmq_publisher = None - # This list holds all processors mentioned in the config file - self._processor_list = None + # TODO: These will change dynamically + # according to the new requirements + # This list holds a set of all processing worker + # names mentioned in the config file + self._processing_workers_list = None + # This list holds a set of all processor server + # names mentioned in the config file + self._processor_servers_list = None # Create routes self.router.add_api_route( @@ -193,15 +200,27 @@ def create_message_queues(self) -> None: self.rmq_publisher.create_queue(queue_name=processor.name) @property - def processor_list(self): - if self._processor_list: - return self._processor_list + def processing_workers_list(self): + if self._processing_workers_list: + return self._processing_workers_list res = set([]) for host in self.config.hosts: for processor in host.processors: res.add(processor.name) - self._processor_list = list(res) - return self._processor_list + self._processing_workers_list = list(res) + return self._processing_workers_list + + # TODO: Revisit. This is just mimicking the method above. + @property + def processor_servers_list(self): + if self._processor_servers_list: + return self._processor_servers_list + res = set([]) + for host in self.config.hosts: + for processor_server in host.servers: + res.add(processor_server.name) + self._processor_servers_list = list(res) + return self._processor_server_list @staticmethod def create_processing_message(job: DBProcessorJob) -> OcrdProcessingMessage: @@ -221,12 +240,47 @@ def create_processing_message(job: DBProcessorJob) -> OcrdProcessingMessage: return processing_message async def push_processor_job(self, processor_name: str, data: PYJobInput) -> PYJobOutput: - """ Queue a processor job - """ + if data.agent_type not in ['worker', 'server']: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=f"Unknown network agent with value: {data.agent_type}" + ) + + job_output = None + if data.agent_type == 'worker': + job_output = await self.push_to_processing_queue(processor_name, data) + if data.agent_type == 'server': + job_output = await self.push_to_processor_server(processor_name, data) + if not job_output: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to create job output" + ) + return job_output + + # TODO: Revisit and remove duplications between push_to_* methods + async def push_to_processing_queue(self, processor_name: str, data: PYJobInput) -> PYJobOutput: + # Validate existence of the Workspace in the DB + if bool(data.path_to_mets) == bool(data.workspace_id): + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail="Either 'path' or 'workspace_id' must be provided, but not both" + ) + # This check is done to return early in case + # the workspace_id is provided but not existing in the DB + elif data.workspace_id: + try: + await db_get_workspace(data.workspace_id) + except ValueError: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=f"Workspace with id '{data.workspace_id}' not existing" + ) + if not self.rmq_publisher: raise Exception('RMQPublisher is not connected') - if processor_name not in self.processor_list: + if processor_name not in self._processing_workers_list: try: # Only checks if the process queue exists, if not raises ChannelClosedByBroker self.rmq_publisher.create_queue(processor_name, passive=True) @@ -240,7 +294,7 @@ async def push_processor_job(self, processor_name: str, data: PYJobInput) -> PYJ detail=f"Process queue with id '{processor_name}' not existing" ) - # validate parameters + # TODO: Getting the tool shall be adapted to the change in #1028 ocrd_tool = get_ocrd_tool_json(processor_name) if not ocrd_tool: raise HTTPException( @@ -251,6 +305,27 @@ async def push_processor_job(self, processor_name: str, data: PYJobInput) -> PYJ if not report.is_valid: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=report.errors) + job = DBProcessorJob( + **data.dict(exclude_unset=True, exclude_none=True), + job_id=generate_id(), + processor_name=processor_name, + state=StateEnum.queued + ) + await job.insert() + processing_message = self.create_processing_message(job) + encoded_processing_message = OcrdProcessingMessage.encode_yml(processing_message) + + try: + self.rmq_publisher.publish_to_queue(processor_name, encoded_processing_message) + except Exception as error: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f'RMQPublisher has failed: {error}' + ) + return job.to_job_output() + + async def push_to_processor_server(self, processor_name: str, data: PYJobInput) -> PYJobOutput: + # Validate existence of the Workspace in the DB if bool(data.path_to_mets) == bool(data.workspace_id): raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, @@ -267,29 +342,53 @@ async def push_processor_job(self, processor_name: str, data: PYJobInput) -> PYJ detail=f"Workspace with id '{data.workspace_id}' not existing" ) - job = DBProcessorJob( - **data.dict(exclude_unset=True, exclude_none=True), - job_id=generate_id(), - processor_name=processor_name, - state=StateEnum.queued - ) - await job.insert() - processing_message = self.create_processing_message(job) - encoded_processing_message = OcrdProcessingMessage.encode_yml(processing_message) + processor_server_url = None - try: - self.rmq_publisher.publish_to_queue(processor_name, encoded_processing_message) - except Exception as error: + # Check if a processor server with processor_name was deployed + # TODO: Revisit when the config file classes are refactored (made more abstract). + # This is such a mess now due to the bad abstraction and bad naming conventions! + for host_config in self.config.hosts: + for processor_server in host_config.servers: + if processor_server.name == processor_name: + processor_server_url = f"http://{host_config.address}:{processor_server.port}/" + + if not processor_server_url: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f'RMQPublisher has failed: {error}' + detail=f"Processor Server of '{processor_name}' is not available" ) - return job.to_job_output() + + # Request the tool json from the Processor Server + response = requests.get(processor_server_url, headers={'Accept': 'application/json'}) + if not response.status_code == 200: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to retrieve '{processor_name}' from: {processor_server_url}" + ) + ocrd_tool = response.json() + if not ocrd_tool: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to retrieve ocrd tool json of '{processor_name}' from: {processor_server_url}" + ) + report = ParameterValidator(ocrd_tool).validate(dict(data.parameters)) + if not report.is_valid: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=report.errors) + + # Post a processing job to the Processor Server + response = requests.post(processor_server_url, headers={'Accept': 'application/json'}, json=data) + if not response.status_code == 202: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to post '{processor_name}' job to: {processor_server_url}" + ) + job_output = response.json + return job_output async def get_processor_info(self, processor_name) -> Dict: """ Return a processor's ocrd-tool.json """ - if processor_name not in self.processor_list: + if processor_name not in self._processing_workers_list: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail='Processor not available' @@ -308,7 +407,16 @@ async def get_job(self, processor_name: str, job_id: str) -> PYJobOutput: detail=f"Processing job with id '{job_id}' of processor type '{processor_name}' not existing" ) - async def list_processors(self) -> str: + async def list_processors(self) -> List[str]: """ Return a list of all available processors """ - return self.processor_list + processor_names_list = [] + + # TODO: 1) Revisit this. Currently, it adds labels in + # front of the names for differentiation purposes + # TODO: 2) This could be optimized by holding a dynamic list + for worker_name in self._processing_workers_list: + processor_names_list.append(f'worker {worker_name}') + for server_name in self._processor_servers_list: + processor_names_list.append(f'server {server_name}') + return processor_names_list diff --git a/ocrd_network/ocrd_network/processing_worker.py b/ocrd_network/ocrd_network/processing_worker.py index 8b3681f18..ef385bffa 100644 --- a/ocrd_network/ocrd_network/processing_worker.py +++ b/ocrd_network/ocrd_network/processing_worker.py @@ -19,7 +19,7 @@ import pika.adapters.blocking_connection from ocrd import Resolver -from ocrd_utils import getLogger +from ocrd_utils import getLogger, initLogging from ocrd.processor.helpers import run_cli, run_processor from .database import ( @@ -55,6 +55,7 @@ class ProcessingWorker: def __init__(self, rabbitmq_addr, mongodb_addr, processor_name, ocrd_tool: dict, processor_class=None) -> None: + initLogging() self.log = getLogger(__name__) # TODO: Provide more flexibility for configuring file logging (i.e. via ENV variables) file_handler = logging.FileHandler(f'/tmp/worker_{processor_name}_{getpid()}.log', mode='a') diff --git a/ocrd_network/ocrd_network/processor_server.py b/ocrd_network/ocrd_network/processor_server.py index 03f6cadb7..50d94972c 100644 --- a/ocrd_network/ocrd_network/processor_server.py +++ b/ocrd_network/ocrd_network/processor_server.py @@ -1,6 +1,4 @@ -from contextlib import redirect_stdout from datetime import datetime -from io import StringIO import json import logging from os import environ, getpid @@ -14,9 +12,9 @@ from ocrd import Resolver from ocrd_validators import ParameterValidator from ocrd_utils import ( + initLogging, getLogger, - get_ocrd_tool_json, - parse_json_string_with_comments + get_ocrd_tool_json ) from .database import ( @@ -51,14 +49,8 @@ class ProcessorServer(FastAPI): def __init__(self, mongodb_addr: str, processor_name: str = "", processor_class=None): if not (processor_name or processor_class): raise ValueError('Either "processor_name" or "processor_class" must be provided') - + initLogging() self.log = getLogger(__name__) - # TODO: Provide more flexibility for configuring file logging (i.e. via ENV variables) - file_handler = logging.FileHandler(f'/tmp/server_{processor_name}_{getpid()}.log', mode='a') - logging_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' - file_handler.setFormatter(logging.Formatter(logging_format)) - file_handler.setLevel(logging.DEBUG) - self.log.addHandler(file_handler) self.db_url = mongodb_addr self.processor_name = processor_name @@ -105,7 +97,7 @@ def __init__(self, mongodb_addr: str, processor_name: str = "", processor_class= self.router.add_api_route( path='/', - endpoint=self.process, + endpoint=self.create_processor_job_task, methods=['POST'], tags=['Processing'], status_code=status.HTTP_202_ACCEPTED, @@ -141,7 +133,7 @@ async def get_processor_info(self): # Note: The Processing server pushes to a queue, while # the Processor Server creates (pushes to) a background task - async def push_processor_job(self, data: PYJobInput, background_tasks: BackgroundTasks): + async def create_processor_job_task(self, data: PYJobInput, background_tasks: BackgroundTasks): if not self.ocrd_tool: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, @@ -216,10 +208,7 @@ def get_ocrd_tool(self): if self.ocrd_tool: return self.ocrd_tool if self.ProcessorClass: - str_out = StringIO() - with redirect_stdout(str_out): - self.ProcessorClass(workspace=None, dump_json=True) - ocrd_tool = parse_json_string_with_comments(str_out.getvalue()) + ocrd_tool = self.ProcessorClass(workspace=None, version=True).ocrd_tool else: ocrd_tool = get_ocrd_tool_json(self.processor_name) return ocrd_tool @@ -228,10 +217,7 @@ def get_version(self) -> str: if self.version: return self.version if self.ProcessorClass: - str_out = StringIO() - with redirect_stdout(str_out): - self.ProcessorClass(workspace=None, show_version=True) - version_str = str_out.getvalue() + version_str = self.ProcessorClass(workspace=None, version=True).version else: version_str = run( [self.processor_name, '--version'], @@ -241,8 +227,14 @@ def get_version(self) -> str: ).stdout return version_str - def run_server(self, host, port): - uvicorn.run(self, host=host, port=port, access_log=False) + def run_server(self, host, port, access_log=False): + # TODO: Provide more flexibility for configuring file logging (i.e. via ENV variables) + file_handler = logging.FileHandler(f'/tmp/server_{self.processor_name}_{port}_{getpid()}.log', mode='a') + logging_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' + file_handler.setFormatter(logging.Formatter(logging_format)) + file_handler.setLevel(logging.DEBUG) + self.log.addHandler(file_handler) + uvicorn.run(self, host=host, port=port, access_log=access_log) async def run_cli_from_server( self, diff --git a/ocrd_validators/ocrd_validators/processing_server_config.schema.yml b/ocrd_validators/ocrd_validators/processing_server_config.schema.yml index d28b63a3d..b2fe06cb4 100644 --- a/ocrd_validators/ocrd_validators/processing_server_config.schema.yml +++ b/ocrd_validators/ocrd_validators/processing_server_config.schema.yml @@ -58,6 +58,7 @@ properties: - address - username - workers + - servers oneOf: - required: - password @@ -75,7 +76,7 @@ properties: description: Path to private key file type: string workers: - description: List of workers which will be deployed + description: List of processing workers that will be deployed type: array minItems: 1 items: @@ -97,12 +98,41 @@ properties: minimum: 1 default: 1 deploy_type: - description: Should the processor be deployed natively or with Docker + description: Should the processing worker be deployed natively or with Docker type: string enum: - native - docker default: native + servers: + description: List of processor servers that will be deployed + type: array + minItems: 1 + items: + type: object + additionalProperties: false + required: + - name + - port + properties: + name: + description: Name of the processor + type: string + pattern: "^ocrd-.*$" + examples: + - ocrd-cis-ocropy-binarize + - ocrd-olena-binarize + deploy_type: + description: Should the processor server natively or with Docker + type: string + enum: + - native + - docker + default: native + port: + description: The port number to be deployed on the host + $ref: "#/$defs/port" + $defs: address: type: string