From 7e8e756c1728ec189b6c3d183e52f11b43c5d72d Mon Sep 17 00:00:00 2001 From: Han Xiao Date: Tue, 3 Aug 2021 23:46:20 +0800 Subject: [PATCH 1/2] feat(executor): add uses-requests cli to override requests in Executor --- jina/jaml/__init__.py | 34 ++++++------- jina/parsers/peapods/runtimes/zed.py | 11 ++++- jina/peapods/runtimes/zmq/zed.py | 1 + .../zed/test_override_config_params.py | 49 +++++++++++++++++++ 4 files changed, 76 insertions(+), 19 deletions(-) diff --git a/jina/jaml/__init__.py b/jina/jaml/__init__.py index 2aa082607756d..452332ad237c1 100644 --- a/jina/jaml/__init__.py +++ b/jina/jaml/__init__.py @@ -454,6 +454,7 @@ def load_config( context: Optional[Dict[str, Any]] = None, override_with: Optional[Dict] = None, override_metas: Optional[Dict] = None, + override_requests: Optional[Dict] = None, **kwargs, ) -> 'JAMLCompatible': """A high-level interface for loading configuration with features @@ -500,8 +501,9 @@ def load_config( :param allow_py_modules: allow importing plugins specified by ``py_modules`` in YAML at any levels :param substitute: substitute environment, internal reference and context variables. :param context: context replacement variables in a dict, the value of the dict is the replacement. - :param override_with: dictionary of parameters to overwrite from the default config - :param override_metas: dictionary of parameters to overwrite from the default config + :param override_with: dictionary of parameters to overwrite from the default config's with field + :param override_metas: dictionary of parameters to overwrite from the default config's metas field + :param override_requests: dictionary of parameters to overwrite from the default config's requests field :param kwargs: kwargs for parse_config_source :return: :class:`JAMLCompatible` object """ @@ -511,21 +513,9 @@ def load_config( no_tag_yml = JAML.load_no_tags(fp) if no_tag_yml: no_tag_yml.update(**kwargs) - if override_with is not None: - with_params = no_tag_yml.get('with', None) - if with_params: - with_params.update(**override_with) - no_tag_yml.update(with_params) - else: - no_tag_yml['with'] = override_with - if override_metas is not None: - metas_params = no_tag_yml.get('metas', None) - if metas_params: - metas_params.update(**override_metas) - no_tag_yml.update(metas_params) - else: - no_tag_yml['metas'] = override_metas - + cls._override_yml_params(no_tag_yml, 'with', override_with) + cls._override_yml_params(no_tag_yml, 'metas', override_metas) + cls._override_yml_params(no_tag_yml, 'requests', override_requests) else: raise BadConfigSource( f'can not construct {cls} from an empty {source}. nothing to read from there' @@ -564,3 +554,13 @@ def load_config( tag_yml = JAML.unescape(JAML.dump(no_tag_yml)) # load into object, no more substitute return JAML.load(tag_yml, substitute=False) + + @classmethod + def _override_yml_params(cls, raw_yaml, field_name, override_field): + if override_field is not None: + field_params = raw_yaml.get(field_name, None) + if field_params: + field_params.update(**override_field) + raw_yaml.update(field_params) + else: + raw_yaml[field_name] = override_field diff --git a/jina/parsers/peapods/runtimes/zed.py b/jina/parsers/peapods/runtimes/zed.py index ad291333e3801..c6f6dccf5c6a1 100644 --- a/jina/parsers/peapods/runtimes/zed.py +++ b/jina/parsers/peapods/runtimes/zed.py @@ -34,7 +34,6 @@ def mixin_zed_runtime_parser(parser): ) gp.add_argument( '--uses-with', - '--override-with', action=KVAppendAction, metavar='KEY: VALUE', nargs='*', @@ -44,7 +43,6 @@ def mixin_zed_runtime_parser(parser): ) gp.add_argument( '--uses-metas', - '--override-metas', action=KVAppendAction, metavar='KEY: VALUE', nargs='*', @@ -52,6 +50,15 @@ def mixin_zed_runtime_parser(parser): Dictionary of keyword arguments that will override the `metas` configuration in `uses` ''', ) + gp.add_argument( + '--uses-requests', + action=KVAppendAction, + metavar='KEY: VALUE', + nargs='*', + help=''' + Dictionary of keyword arguments that will override the `requests` configuration in `uses` + ''', + ) gp.add_argument( '--py-modules', type=str, diff --git a/jina/peapods/runtimes/zmq/zed.py b/jina/peapods/runtimes/zmq/zed.py index dca0764f9722c..abad3b5d2c624 100644 --- a/jina/peapods/runtimes/zmq/zed.py +++ b/jina/peapods/runtimes/zmq/zed.py @@ -88,6 +88,7 @@ def _load_executor(self): self.args.uses, override_with=self.args.uses_with, override_metas=self.args.uses_metas, + override_requests=self.args.uses_requests, runtime_args=vars(self.args), ) except BadConfigSource as ex: diff --git a/tests/integration/override_config_params/zed/test_override_config_params.py b/tests/integration/override_config_params/zed/test_override_config_params.py index 987e6604cf5e8..3a4655269ffa8 100644 --- a/tests/integration/override_config_params/zed/test_override_config_params.py +++ b/tests/integration/override_config_params/zed/test_override_config_params.py @@ -1,6 +1,8 @@ import os + import pytest +from jina import Executor, requests from jina import Flow, Document cur_dir = os.path.dirname(os.path.abspath(__file__)) @@ -54,3 +56,50 @@ def test_override_config_params_parallel(): assert doc.tags['param3'] == 10 # not overriden assert doc.tags['name'] == 'name' # not override assert doc.tags['workspace'] == 'different_workspace' + + +def test_override_requests(): + class MyExec(Executor): + @requests + def foo(self, docs, **kwargs): + for d in docs: + d.text = 'foo' + + def bar(self, docs, **kwargs): + for d in docs: + d.text = 'bar' + + @requests(on=['/1', '/2']) + def foobar(self, docs, **kwargs): + for d in docs: + d.text = 'foobar' + + # original + f = Flow().add(uses=MyExec) + with f: + req = f.post('/index', Document(), return_results=True) + assert req[0].docs[0].text == 'foo' + + # change bind to bar() + f = Flow().add(uses=MyExec, uses_requests={'/index': 'bar'}) + with f: + req = f.post('/index', Document(), return_results=True) + assert req[0].docs[0].text == 'bar' + + req = f.post('/1', Document(), return_results=True) + assert req[0].docs[0].text == 'foobar' + + # change bind to foobar() + f = Flow().add(uses=MyExec, uses_requests={'/index': 'foobar'}) + with f: + req = f.post('/index', Document(), return_results=True) + assert req[0].docs[0].text == 'foobar' + + req = f.post('/index-blah', Document(), return_results=True) + assert req[0].docs[0].text == 'foo' + + # change default bind to foo() + f = Flow().add(uses=MyExec, uses_requests={'/default': 'bar'}) + with f: + req = f.post('/index', Document(), return_results=True) + assert req[0].docs[0].text == 'bar' From 737815751b91966206e109d76eb19962e8019a99 Mon Sep 17 00:00:00 2001 From: Jina Dev Bot Date: Tue, 3 Aug 2021 15:47:54 +0000 Subject: [PATCH 2/2] style: fix overload and cli autocomplete --- cli/autocomplete.py | 12 ++++-------- jina/flow/base.py | 4 ++++ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cli/autocomplete.py b/cli/autocomplete.py index 25d4cfebe849a..6cee182943e5f 100644 --- a/cli/autocomplete.py +++ b/cli/autocomplete.py @@ -65,9 +65,8 @@ '--ssh-password', '--uses', '--uses-with', - '--override-with', '--uses-metas', - '--override-metas', + '--uses-requests', '--py-modules', '--port-in', '--port-out', @@ -134,9 +133,8 @@ '--ssh-password', '--uses', '--uses-with', - '--override-with', '--uses-metas', - '--override-metas', + '--uses-requests', '--py-modules', '--port-in', '--port-out', @@ -216,9 +214,8 @@ '--ssh-password', '--uses', '--uses-with', - '--override-with', '--uses-metas', - '--override-metas', + '--uses-requests', '--py-modules', '--port-in', '--port-out', @@ -293,9 +290,8 @@ '--ssh-password', '--uses', '--uses-with', - '--override-with', '--uses-metas', - '--override-metas', + '--uses-requests', '--py-modules', '--port-in', '--port-out', diff --git a/jina/flow/base.py b/jina/flow/base.py index 373046241b6a2..f3dc4d8683a2b 100644 --- a/jina/flow/base.py +++ b/jina/flow/base.py @@ -132,6 +132,7 @@ def __init__( title: Optional[str] = None, uses: Optional[Union[str, Type['BaseExecutor'], dict]] = 'BaseExecutor', uses_metas: Optional[dict] = None, + uses_requests: Optional[dict] = None, uses_with: Optional[dict] = None, workspace: Optional[str] = None, zmq_identity: Optional[str] = None, @@ -218,6 +219,7 @@ def __init__( - a Python dict that represents the config - a text file stream has `.read()` interface :param uses_metas: Dictionary of keyword arguments that will override the `metas` configuration in `uses` + :param uses_requests: Dictionary of keyword arguments that will override the `requests` configuration in `uses` :param uses_with: Dictionary of keyword arguments that will override the `with` configuration in `uses` :param workspace: The working directory for any IO operations in this object. If not set, then derive from its parent `workspace`. :param zmq_identity: The identity of a ZMQRuntime. It is used for unique socket identification towards other ZMQRuntimes. @@ -479,6 +481,7 @@ def add( uses_after: Optional[Union[str, Type['BaseExecutor'], dict]] = None, uses_before: Optional[Union[str, Type['BaseExecutor'], dict]] = None, uses_metas: Optional[dict] = None, + uses_requests: Optional[dict] = None, uses_with: Optional[dict] = None, volumes: Optional[List[str]] = None, workspace: Optional[str] = None, @@ -584,6 +587,7 @@ def add( :param uses_after: The executor attached after the Peas described by --uses, typically used for receiving from all parallels, accepted type follows `--uses` :param uses_before: The executor attached after the Peas described by --uses, typically before sending to all parallels, accepted type follows `--uses` :param uses_metas: Dictionary of keyword arguments that will override the `metas` configuration in `uses` + :param uses_requests: Dictionary of keyword arguments that will override the `requests` configuration in `uses` :param uses_with: Dictionary of keyword arguments that will override the `with` configuration in `uses` :param volumes: The path on the host to be mounted inside the container.