From d2ce099074aa4bf8811d7670ca64ba23b8959663 Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Wed, 6 Sep 2023 16:42:13 +0200 Subject: [PATCH] replace --agent-type with subcommand, #1032, #1085 --- ocrd/ocrd/decorators/__init__.py | 92 +++++++++++------------- ocrd/ocrd/decorators/ocrd_cli_options.py | 9 ++- ocrd/ocrd/processor/helpers.py | 7 +- 3 files changed, 54 insertions(+), 54 deletions(-) diff --git a/ocrd/ocrd/decorators/__init__.py b/ocrd/ocrd/decorators/__init__.py index 3e7b5cae0..5d318974f 100644 --- a/ocrd/ocrd/decorators/__init__.py +++ b/ocrd/ocrd/decorators/__init__.py @@ -20,6 +20,8 @@ from .ocrd_cli_options import ocrd_cli_options from .mets_find_options import mets_find_options +SUBCOMMANDS = ['worker', 'server'] + def ocrd_cli_wrap_processor( processorClass, mets=None, @@ -35,7 +37,7 @@ def ocrd_cli_wrap_processor( show_resource=None, list_resources=False, # ocrd_network params start # - agent_type=None, + subcommand=None, agent_address=None, queue=None, database=None, @@ -45,6 +47,9 @@ def ocrd_cli_wrap_processor( if not sys.argv[1:]: processorClass(workspace=None, show_help=True) sys.exit(1) + if subcommand: + # Used for checking/starting network agents for the WebAPI architecture + check_and_run_network_agent(processorClass, subcommand, agent_address, database, queue) if dump_json or dump_module_dir or help or version or show_resource or list_resources: processorClass( workspace=None, @@ -59,10 +64,6 @@ def ocrd_cli_wrap_processor( initLogging() - # Used for checking/starting network agents for the WebAPI architecture - # Has no side effects if neither of the 4 ocrd_network parameters are passed - check_and_run_network_agent(processorClass, agent_type, agent_address, database, queue) - LOG = getLogger('ocrd_cli_wrap_processor') # LOG.info('kwargs=%s' % kwargs) # Merge parameter overrides and parameters @@ -128,59 +129,50 @@ def exit(): run_processor(processorClass, mets_url=mets, workspace=workspace, **kwargs) -def check_and_run_network_agent(ProcessorClass, agent_type: str, agent_address: str, database: str, queue: str): - if not agent_type and (agent_address or database or queue): - raise ValueError("Options '--database', '--queue', and '--address' are valid only with '--type'") - if not agent_type: - return +def check_and_run_network_agent(ProcessorClass, subcommand: str, agent_address: str, database: str, queue: str, strict : bool = False): + """ + """ + if subcommand not in SUBCOMMANDS: + raise ValueError(f"SUBCOMMAND can only be one of {SUBCOMMANDS}") if not database: - raise ValueError("Options '--type' and '--database' are mutually inclusive") - allowed_agent_types = ['server', 'worker'] - if agent_type not in allowed_agent_types: - agents_str = ', '.join(allowed_agent_types) - raise ValueError(f"Wrong type parameter. Allowed types: {agents_str}") - if agent_type == 'server': + raise ValueError("Option '--database' is invalid for subcommand {subcommand}") + + if subcommand == 'server': if not agent_address: - raise ValueError("Options '--type=server' and '--address' are mutually inclusive") + raise ValueError("Option '--address' required for subcommand {subcommand}") if queue: - raise ValueError("Options '--type=server' and '--queue' are mutually exclusive") - if agent_type == 'worker': - if not queue: - raise ValueError("Options '--type=worker' and '--queue' are mutually inclusive") + raise ValueError("Option '--queue' invalid for subcommand {subcommand}") + if subcommand == 'worker': if agent_address: - raise ValueError("Options '--type=worker' and '--address' are mutually exclusive") + raise ValueError("Option '--address' invalid for subcommand {subcommand}") + if not queue: + raise ValueError("Option '--queue' required for subcommand {subcommand}") import logging logging.getLogger('ocrd.network').setLevel(logging.DEBUG) processor = ProcessorClass(workspace=None, dump_json=True) - if agent_type == 'worker': - try: - # TODO: Passing processor_name and ocrd_tool is reduntant - processing_worker = ProcessingWorker( - rabbitmq_addr=queue, - mongodb_addr=database, - processor_name=processor.ocrd_tool['executable'], - ocrd_tool=processor.ocrd_tool, - processor_class=ProcessorClass, - ) - # The RMQConsumer is initialized and a connection to the RabbitMQ is performed - processing_worker.connect_consumer() - # Start consuming from the queue with name `processor_name` - processing_worker.start_consuming() - except Exception as e: - sys.exit(f"Processing worker has failed with error: {e}") - if agent_type == 'server': - try: - # TODO: Better validate that inside the ProcessorServer itself - host, port = agent_address.split(':') - processor_server = ProcessorServer( - mongodb_addr=database, - processor_name=processor.ocrd_tool['executable'], - processor_class=ProcessorClass, - ) - processor_server.run_server(host=host, port=int(port)) - except Exception as e: - sys.exit(f"Processor server has failed with error: {e}") + if subcommand == 'worker': + # TODO: Passing processor_name and ocrd_tool is reduntant + processing_worker = ProcessingWorker( + rabbitmq_addr=queue, + mongodb_addr=database, + processor_name=processor.ocrd_tool['executable'], + ocrd_tool=processor.ocrd_tool, + processor_class=ProcessorClass, + ) + # The RMQConsumer is initialized and a connection to the RabbitMQ is performed + processing_worker.connect_consumer() + # Start consuming from the queue with name `processor_name` + processing_worker.start_consuming() + elif subcommand == 'server': + # TODO: Better validate that inside the ProcessorServer itself + host, port = agent_address.split(':') + processor_server = ProcessorServer( + mongodb_addr=database, + processor_name=processor.ocrd_tool['executable'], + processor_class=ProcessorClass, + ) + processor_server.run_server(host=host, port=int(port)) sys.exit(0) diff --git a/ocrd/ocrd/decorators/ocrd_cli_options.py b/ocrd/ocrd/decorators/ocrd_cli_options.py index cd6664c23..efff3d347 100644 --- a/ocrd/ocrd/decorators/ocrd_cli_options.py +++ b/ocrd/ocrd/decorators/ocrd_cli_options.py @@ -1,5 +1,5 @@ import click -from click import option, Path +from click import option, Path, group, command, argument from .parameter_option import parameter_option, parameter_override_option from .loglevel_option import loglevel_option from ocrd_network import ( @@ -39,7 +39,6 @@ def cli(mets_url): parameter_option, parameter_override_option, loglevel_option, - option('--type', 'agent_type', type=click.Choice(['worker', 'server'])), option('--address', 'agent_address', type=ServerAddressParamType()), option('--queue', type=QueueServerParamType()), option('--database', type=DatabaseParamType()), @@ -49,6 +48,12 @@ def cli(mets_url): option('-D', '--dump-module-dir', is_flag=True, default=False), option('-h', '--help', is_flag=True, default=False), option('-V', '--version', is_flag=True, default=False), + # Subcommand, only used for 'worker'/'server'. Cannot be handled in + # click because processors use the @command decorator and even if they + # were using `group`, you cannot combine have a command with + # subcommands. So we have to work around that by creating a + # pseudo-subcommand handled in ocrd_cli_wrap_processor + argument('subcommand', nargs=1, required=False, type=click.Choice(['worker', 'server'])), ] for param in params: param(f) diff --git a/ocrd/ocrd/processor/helpers.py b/ocrd/ocrd/processor/helpers.py index 679491e6e..2089cb7ef 100644 --- a/ocrd/ocrd/processor/helpers.py +++ b/ocrd/ocrd/processor/helpers.py @@ -253,10 +253,14 @@ def wrap(s): subsequent_indent=' > ', preserve_paragraphs=True) return ''' -Usage: %s [OPTIONS] +Usage: %s [worker] [OPTIONS] %s%s +Subcommands: + worker Start a processing worker rather than do local processing + server Start a processor server rather than do local processing + Options for processing: -m, --mets URL-PATH URL or file path of METS to process [./mets.xml] -w, --working-dir PATH Working directory of local workspace [dirname(URL-PATH)] @@ -286,7 +290,6 @@ def wrap(s): --database The MongoDB server address in format "mongodb://{host}:{port}" [mongodb://localhost:27018] - --type type of processing: either "worker" or "server" Options for information: -C, --show-resource RESNAME Dump the content of processor resource RESNAME