From ee80f65e75cda83a2405c343268ceb4015ea51df Mon Sep 17 00:00:00 2001 From: Han Xiao Date: Sun, 10 Jan 2021 17:11:02 +0100 Subject: [PATCH 1/4] fix(daemon): fix some helper text for redocs --- .github/workflows/cd.yml | 1 + .github/workflows/tag.yml | 2 ++ daemon/api/endpoints/__init__.py | 6 ++--- daemon/api/endpoints/flow.py | 38 +++++++++++++++----------------- daemon/api/endpoints/pea.py | 4 ++-- daemon/api/endpoints/pod.py | 6 ++--- daemon/config.py | 22 +++++++++--------- daemon/store.py | 3 +-- 8 files changed, 41 insertions(+), 41 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index d8daff0a8de59..b81adc62d018f 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -59,6 +59,7 @@ jobs: - name: push-to-api-repo run: | cp jinad.html schema/jinad/${{env.JINA_VERSION}}.html + cp jinad.html schema/daemon/${{env.JINA_VERSION}}.html cd schema git config --local user.email "dev-bot@jina.ai" git config --local user.name "Jina Dev Bot" diff --git a/.github/workflows/tag.yml b/.github/workflows/tag.yml index 681d186e593bc..76dd1bfba999e 100644 --- a/.github/workflows/tag.yml +++ b/.github/workflows/tag.yml @@ -38,6 +38,8 @@ jobs: run: | cp jinad.html schema/jinad/index.html cp jinad.html schema/jinad/${{env.JINA_VERSION}}.html + cp jinad.html schema/daemon/index.html + cp jinad.html schema/daemon/${{env.JINA_VERSION}}.html cd schema git config --local user.email "dev-bot@jina.ai" git config --local user.name "Jina Dev Bot" diff --git a/daemon/api/endpoints/__init__.py b/daemon/api/endpoints/__init__.py index 6a07547fe48e0..0ab64ac3ee4aa 100644 --- a/daemon/api/endpoints/__init__.py +++ b/daemon/api/endpoints/__init__.py @@ -9,13 +9,13 @@ @common_router.on_event('startup') async def startup(): - daemon_logger.success('Welcome to Jina daemon - the manager of distributed Jina') - daemon_logger.success(f'Uvicorn + FastAPI running on {server_config.HOST}:{server_config.PORT}') + daemon_logger.success(f'Welcome to Jina daemon - the manager of distributed Jina\n' + f'Uvicorn + FastAPI running on {server_config.HOST}:{server_config.PORT}') @common_router.get( path='/alive', - summary='Get status of jinad', + summary='Check if daemon is alive', status_code=status.HTTP_200_OK ) async def _status(): diff --git a/daemon/api/endpoints/flow.py b/daemon/api/endpoints/flow.py index f9dab5adfb59c..b47cb91f87a13 100644 --- a/daemon/api/endpoints/flow.py +++ b/daemon/api/endpoints/flow.py @@ -10,7 +10,6 @@ from jina.parsers import set_client_cli_parser from ...excepts import FlowYamlParseException, FlowCreationException, FlowStartException from ...models import SinglePodModel -from ...models.custom import build_pydantic_model from ...store import flow_store router = APIRouter() @@ -18,7 +17,7 @@ @router.put( path='/flow/pods', - summary='Build & start a Flow using Pods', + summary='Start a Flow from list of Pods', ) async def _create_from_pods( pods: Union[List[Dict]] = Body(..., example=json.loads(SinglePodModel().json())) @@ -59,7 +58,7 @@ async def _create_from_pods( @router.put( path='/flow/yaml', - summary='Build & start a Flow using YAML', + summary='Start a Flow from a YAML config', ) async def _create_from_yaml( yamlspec: UploadFile = File(...), @@ -69,26 +68,24 @@ async def _create_from_yaml( """ Build a flow using [Flow YAML](https://docs.jina.ai/chapters/yaml/yaml.html#flow-yaml-sytanx) - > Upload Flow yamlspec (`yamlspec`) - - > Yamls that Pods use (`uses_files`) (Optional) - - > Python modules (`pymodules_files`) that the Pods use (Optional) + - Upload Flow yamlspec (`yamlspec`) + - Yamls that Pods use (`uses_files`) (Optional) + - Python modules (`pymodules_files`) that the Pods use (Optional) **yamlspec**: !Flow + version: 1.0 with: - rest_api: true - compress_hwm: 1024 + restful: true pods: - encode: - uses: helloworld.encoder.yml - parallel: 2 - index: - uses: helloworld.indexer.yml - shards: 2 - separated_workspace: true + - name: encode + uses: helloworld.encoder.yml + parallel: 2 + - name: index + uses: helloworld.indexer.yml + shards: 2 + separated_workspace: true **uses_files**: `helloworld.encoder.yml` @@ -158,7 +155,7 @@ def __init__(self): @router.get( path='/flow/{flow_id}', - summary='Get Flow information', + summary='Get the status of a running Flow', ) async def _fetch( flow_id: uuid.UUID, @@ -193,7 +190,7 @@ async def _fetch( @router.get( path='/ping', - summary='Connect to Flow gateway', + summary='Check if the Flow is alive', ) async def _ping( host: str, @@ -208,6 +205,7 @@ async def _ping( _, args, _ = ArgNamespace.get_parsed_args(kwargs, set_client_cli_parser()) client = Client(args) try: + # TODO: this introduces side-effect, need to be refactored. (2020.01.10) client.index(input_fn=['abc']) return { 'status_code': status.HTTP_200_OK, @@ -220,7 +218,7 @@ async def _ping( @router.delete( path='/flow', - summary='Close Flow Context', + summary='Terminate a running Flow', ) async def _delete( flow_id: uuid.UUID diff --git a/daemon/api/endpoints/pea.py b/daemon/api/endpoints/pea.py index d03a6978398b9..30c59ce343450 100644 --- a/daemon/api/endpoints/pea.py +++ b/daemon/api/endpoints/pea.py @@ -15,7 +15,7 @@ @router.put( path='/pea/upload', - summary='Upload pod context yamls & pymodules', + summary='Upload YAML & py_modules required by a Pea', ) async def _upload( uses_files: List[UploadFile] = File(()), @@ -70,7 +70,7 @@ async def _create( @router.delete( path='/pea', - summary='Close Pea Context', + summary='Terminate a running Pea', ) async def _delete( pea_id: uuid.UUID diff --git a/daemon/api/endpoints/pod.py b/daemon/api/endpoints/pod.py index 5d573f9bdc7d0..8aafc31f5ec6e 100644 --- a/daemon/api/endpoints/pod.py +++ b/daemon/api/endpoints/pod.py @@ -15,7 +15,7 @@ @router.put( path='/upload', - summary='Upload pod context yamls & pymodules', + summary='Upload YAML & py_modules required by a Pod', ) async def _upload( uses_files: List[UploadFile] = File(()), @@ -41,7 +41,7 @@ async def _upload( @router.put( path='/pod', - summary='Create a Pod via Flow or CLI', + summary='Create a Pod', ) async def _create( pod_arguments: Union[SinglePodModel, ParallelPodModel] @@ -71,7 +71,7 @@ async def _create( @router.delete( path='/pod', - summary='Close Pod Context', + summary='Terminate a running Pod', ) async def _delete( pod_id: uuid.UUID diff --git a/daemon/config.py b/daemon/config.py index f983a1d999090..843b82f64b123 100644 --- a/daemon/config.py +++ b/daemon/config.py @@ -11,40 +11,40 @@ class Config: class FastAPIConfig(BaseConfig): - NAME: str = 'Jina Remote Manager' - DESCRIPTION: str = 'REST API for managing Jina on Remote' + NAME: str = 'JinaD (Daemon)' + DESCRIPTION: str = 'The REST API of the daemon for managing distributed Jina' VERSION: str = __version__ PREFIX: str = '/' + __prefix__ class OpenAPITags(BaseConfig): API_TAGS: list = [{ - 'name': 'Jina Remote Management', - 'description': 'API to invoke remote Flows/Pods/Peas', + 'name': 'Distributed Jina management', + 'description': 'API to invoke distributed Flows/Pods/Peas', 'externalDocs': { - 'description': 'Jina Remote Context Manager', + 'description': 'Jina Documentation', 'url': 'https://docs.jina.ai/', }, }] FLOW_API_TAGS: list = [{ - 'name': 'Remote Flow Manager', - 'description': 'API to invoke local/remote Flows', + 'name': 'Managing distributed Flow', + 'description': 'API to invoke distributed Flows', 'externalDocs': { 'description': 'Jina Flow Context Manager', 'url': 'https://docs.jina.ai/chapters/flow/index.html', }, }] POD_API_TAGS: list = [{ - 'name': 'Remote Pod Manager', - 'description': 'API to invoke remote Pods (__should be used by Flow APIs only__)', + 'name': 'Managing distributed Pod', + 'description': 'API to invoke distributed Pods (__should be used by Flow APIs only__)', 'externalDocs': { 'description': 'Jina 101', 'url': 'https://docs.jina.ai/chapters/101/.sphinx.html', }, }] PEA_API_TAGS: list = [{ - 'name': 'Remote Pea Manager', - 'description': 'API to invoke remote Peas', + 'name': 'Managing distributed Pea', + 'description': 'API to invoke distributed Peas', 'externalDocs': { 'description': 'Jina 101', 'url': 'https://docs.jina.ai/chapters/101/.sphinx.html', diff --git a/daemon/store.py b/daemon/store.py index 3dd2f8cc2dd95..52c95c5b0a2f8 100644 --- a/daemon/store.py +++ b/daemon/store.py @@ -79,8 +79,7 @@ def _create(self, if isinstance(config, str) or isinstance(config, SpooledTemporaryFile): yamlspec = config.read().decode() if isinstance(config, SpooledTemporaryFile) else config try: - JAML.register(Flow) - flow = JAML.load(yamlspec) + Flow.load_config(yamlspec) except Exception as e: self.logger.error(f'Got error while loading from yaml {e!r}') raise FlowYamlParseException From 94afe18448cfa34ecca6f65674428855d8014833 Mon Sep 17 00:00:00 2001 From: Han Xiao Date: Sun, 10 Jan 2021 17:28:36 +0100 Subject: [PATCH 2/4] fix(daemon): fix some helper text for redocs --- daemon/__init__.py | 2 +- daemon/config.py | 3 --- daemon/store.py | 9 ++++----- jina/peapods/runtimes/jinad/api.py | 7 +++---- tests/jinad/integration/distributed/helpers.py | 2 +- .../distributed/test_index_query/test_integration.py | 8 ++++---- .../test_index_query_with_shards/test_integration.py | 8 ++++---- .../test_simple_distributed/test_integration.py | 4 ++-- .../test_integration.py | 4 ++-- .../distributed/test_simple_hub_pods/test_integration.py | 4 ++-- 10 files changed, 23 insertions(+), 28 deletions(-) diff --git a/daemon/__init__.py b/daemon/__init__.py index 8283281cd8756..16fb2a875052e 100644 --- a/daemon/__init__.py +++ b/daemon/__init__.py @@ -10,7 +10,7 @@ from jina.logging import JinaLogger from .parser import get_main_parser -daemon_logger = JinaLogger(context='👻 JINAD') +daemon_logger = JinaLogger(context='👻 JinaD') def _get_app(): diff --git a/daemon/config.py b/daemon/config.py index 843b82f64b123..3245e0ea3b5ef 100644 --- a/daemon/config.py +++ b/daemon/config.py @@ -2,8 +2,6 @@ from jina import __version__ -__prefix__ = 'v1' - class BaseConfig(BaseSettings): class Config: @@ -14,7 +12,6 @@ class FastAPIConfig(BaseConfig): NAME: str = 'JinaD (Daemon)' DESCRIPTION: str = 'The REST API of the daemon for managing distributed Jina' VERSION: str = __version__ - PREFIX: str = '/' + __prefix__ class OpenAPITags(BaseConfig): diff --git a/daemon/store.py b/daemon/store.py index 52c95c5b0a2f8..d473071b75437 100644 --- a/daemon/store.py +++ b/daemon/store.py @@ -6,12 +6,11 @@ from fastapi import UploadFile +from jina.enums import PodRoleType from jina.flow import Flow -from jina.jaml import JAML from jina.helper import colored -from jina.enums import PodRoleType -from jina.logging import JinaLogger from jina.peapods import Pea, Pod +from . import daemon_logger from .excepts import FlowYamlParseException, FlowCreationException, \ FlowStartException, PodStartException, PeaStartException, FlowBadInputException from .helper import create_meta_files_from_upload, delete_meta_files_from_upload @@ -24,7 +23,7 @@ class InMemoryStore: # https://github.com/jina-ai/jinad/issues/4 credentials = 'foo:bar' _session_token = None - logger = JinaLogger(context='🏪 STORE') + logger = daemon_logger @contextmanager def _session(self): @@ -79,7 +78,7 @@ def _create(self, if isinstance(config, str) or isinstance(config, SpooledTemporaryFile): yamlspec = config.read().decode() if isinstance(config, SpooledTemporaryFile) else config try: - Flow.load_config(yamlspec) + flow = Flow.load_config(yamlspec) except Exception as e: self.logger.error(f'Got error while loading from yaml {e!r}') raise FlowYamlParseException diff --git a/jina/peapods/runtimes/jinad/api.py b/jina/peapods/runtimes/jinad/api.py index a620f333d6f6b..8ac7843120175 100644 --- a/jina/peapods/runtimes/jinad/api.py +++ b/jina/peapods/runtimes/jinad/api.py @@ -74,7 +74,6 @@ class JinadAPI: def __init__(self, host: str, port: int, - version: str = 'v1', logger: 'JinaLogger' = None, timeout: int = 5, **kwargs): """ @@ -92,9 +91,9 @@ def __init__(self, # TODO: for https, the jinad server would need a tls certificate. # no changes would be required in terms of how the api gets invoked, # as requests does ssl verfication. we'd need to add some exception handling logic though - url = f'{host}:{port}/{version}' - rest_url = f'http://{url}' - websocket_url = f'ws://{url}' + base_url = f'{host}:{port}' + rest_url = f'http://{base_url}' + websocket_url = f'ws://{base_url}' self.alive_url = f'{rest_url}/alive' self.upload_url = f'{rest_url}/upload' self.pea_url = f'{rest_url}/pea' diff --git a/tests/jinad/integration/distributed/helpers.py b/tests/jinad/integration/distributed/helpers.py index 631014f57836a..5e80fa86837ca 100644 --- a/tests/jinad/integration/distributed/helpers.py +++ b/tests/jinad/integration/distributed/helpers.py @@ -29,7 +29,7 @@ def get_results(query: str, def create_flow(flow_yaml: str, pod_dir: Optional[str] = None, - url: str = 'http://localhost:8000/v1/flow/yaml'): + url: str = 'http://localhost:8000/flow/yaml'): with ExitStack() as file_stack: pymodules_files = [] uses_files = [] diff --git a/tests/jinad/integration/distributed/test_index_query/test_integration.py b/tests/jinad/integration/distributed/test_index_query/test_integration.py index 69dcbaaff1da6..14eff85f0b8d5 100644 --- a/tests/jinad/integration/distributed/test_index_query/test_integration.py +++ b/tests/jinad/integration/distributed/test_index_query/test_integration.py @@ -23,12 +23,12 @@ def test_index_query(docker_compose): assert text_indexed == 'text:hey, dude' r = invoke_requests(method='get', - url=f'http://localhost:8000/v1/flow/{flow_id}') + url=f'http://localhost:8000/flow/{flow_id}') assert r is not None assert r['status_code'] == 200 r = invoke_requests(method='delete', - url=f'http://localhost:8000/v1/flow?flow_id={flow_id}') + url=f'http://localhost:8000/flow?flow_id={flow_id}') assert r is not None assert r['status_code'] == 200 @@ -40,12 +40,12 @@ def test_index_query(docker_compose): assert text_matched == 'text:hey, dude' r = invoke_requests(method='get', - url=f'http://localhost:8000/v1/flow/{flow_id}') + url=f'http://localhost:8000/flow/{flow_id}') assert r is not None assert r['status_code'] == 200 r = invoke_requests(method='delete', - url=f'http://localhost:8000/v1/flow?flow_id={flow_id}') + url=f'http://localhost:8000/flow?flow_id={flow_id}') assert r is not None assert r['status_code'] == 200 expected_text = 'text:hey, dude' diff --git a/tests/jinad/integration/distributed/test_index_query_with_shards/test_integration.py b/tests/jinad/integration/distributed/test_index_query_with_shards/test_integration.py index 45a201193c2c0..e10a354fedb30 100644 --- a/tests/jinad/integration/distributed/test_index_query_with_shards/test_integration.py +++ b/tests/jinad/integration/distributed/test_index_query_with_shards/test_integration.py @@ -27,12 +27,12 @@ def test_flow(docker_compose): assert text_indexed == text r = invoke_requests(method='get', - url=f'http://localhost:8000/v1/flow/{flow_id}') + url=f'http://localhost:8000/flow/{flow_id}') assert r is not None assert r['status_code'] == 200 r = invoke_requests(method='delete', - url=f'http://localhost:8000/v1/flow?flow_id={flow_id}') + url=f'http://localhost:8000/flow?flow_id={flow_id}') assert r is not None assert r['status_code'] == 200 @@ -45,11 +45,11 @@ def test_flow(docker_compose): assert len(texts_matched['search']['docs'][0]['matches']) == 10 r = invoke_requests(method='get', - url=f'http://localhost:8000/v1/flow/{flow_id}') + url=f'http://localhost:8000/flow/{flow_id}') assert r is not None assert r['status_code'] == 200 r = invoke_requests(method='delete', - url=f'http://localhost:8000/v1/flow?flow_id={flow_id}') + url=f'http://localhost:8000/flow?flow_id={flow_id}') assert r is not None assert r['status_code'] == 200 diff --git a/tests/jinad/integration/distributed/test_simple_distributed/test_integration.py b/tests/jinad/integration/distributed/test_simple_distributed/test_integration.py index a904995e2aaad..e092be044cd53 100644 --- a/tests/jinad/integration/distributed/test_simple_distributed/test_integration.py +++ b/tests/jinad/integration/distributed/test_simple_distributed/test_integration.py @@ -23,11 +23,11 @@ def test_flow(docker_compose): print(f'Got response text_indexed: {text_indexed}') assert text_indexed == 'text:cats rulessss' - r = invoke_requests(method='get', url=f'http://localhost:8000/v1/flow/{flow_id}') + r = invoke_requests(method='get', url=f'http://localhost:8000/flow/{flow_id}') assert r is not None assert r['status_code'] == 200 r = invoke_requests(method='delete', - url=f'http://localhost:8000/v1/flow?flow_id={flow_id}') + url=f'http://localhost:8000/flow?flow_id={flow_id}') assert r is not None assert r['status_code'] == 200 diff --git a/tests/jinad/integration/distributed/test_simple_distributed_with_shards/test_integration.py b/tests/jinad/integration/distributed/test_simple_distributed_with_shards/test_integration.py index 2ecef883749ce..8d1c75a01feea 100644 --- a/tests/jinad/integration/distributed/test_simple_distributed_with_shards/test_integration.py +++ b/tests/jinad/integration/distributed/test_simple_distributed_with_shards/test_integration.py @@ -24,11 +24,11 @@ def test_flow(docker_compose): assert text_indexed == 'text:cats rulessss' r = invoke_requests(method='get', - url=f'http://localhost:8000/v1/flow/{flow_id}') + url=f'http://localhost:8000/flow/{flow_id}') assert r is not None assert r['status_code'] == 200 r = invoke_requests(method='delete', - url=f'http://localhost:8000/v1/flow?flow_id={flow_id}') + url=f'http://localhost:8000/flow?flow_id={flow_id}') assert r is not None assert r['status_code'] == 200 diff --git a/tests/jinad/integration/distributed/test_simple_hub_pods/test_integration.py b/tests/jinad/integration/distributed/test_simple_hub_pods/test_integration.py index 4433cc0e71296..e1c0f38085188 100644 --- a/tests/jinad/integration/distributed/test_simple_hub_pods/test_integration.py +++ b/tests/jinad/integration/distributed/test_simple_hub_pods/test_integration.py @@ -25,11 +25,11 @@ def test_simple_hub_pods(docker_compose): print(f'Returned document has the text: {text_matched}') r = invoke_requests(method='get', - url=f'http://0.0.0.0:8000/v1/flow/{flow_id}') + url=f'http://0.0.0.0:8000/flow/{flow_id}') assert r['status_code'] == 200 r = invoke_requests(method='delete', - url=f'http://0.0.0.0:8000/v1/flow?flow_id={flow_id}') + url=f'http://0.0.0.0:8000/flow?flow_id={flow_id}') assert r['status_code'] == 200 assert expected_text == text_matched From 1eac6f487117d0601287cda61c4026eebf81a0e5 Mon Sep 17 00:00:00 2001 From: Han Xiao Date: Sun, 10 Jan 2021 17:57:03 +0100 Subject: [PATCH 3/4] fix(daemon): fix some helper text for redocs --- daemon/__init__.py | 12 +++----- daemon/api/endpoints/logs.py | 3 +- tests/conftest.py | 30 +++++++++---------- .../api/endpoints/test_logging_endpoint.py | 8 ++--- tests/jinad/unit/test_config.py | 1 - 5 files changed, 24 insertions(+), 30 deletions(-) diff --git a/daemon/__init__.py b/daemon/__init__.py index 16fb2a875052e..25d6740c08664 100644 --- a/daemon/__init__.py +++ b/daemon/__init__.py @@ -34,21 +34,17 @@ def _get_app(): description=fastapi_config.DESCRIPTION, version=fastapi_config.VERSION ) - app.include_router(router=common_router, - prefix=fastapi_config.PREFIX) - app.include_router(router=logs.router, - prefix=fastapi_config.PREFIX) + app.include_router(router=common_router) + app.include_router(router=logs.router) if jinad_config.CONTEXT == 'all': for _current_router in _all_routers.values(): app.include_router(router=_current_router.router, - tags=_current_router.tags, - prefix=fastapi_config.PREFIX) + tags=_current_router.tags) else: _current_router = _all_routers[jinad_config.CONTEXT] app.openapi_tags = _current_router.openapi_tags app.include_router(router=_current_router.router, - tags=_current_router.tags, - prefix=fastapi_config.PREFIX) + tags=_current_router.tags) return app diff --git a/daemon/api/endpoints/logs.py b/daemon/api/endpoints/logs.py index 1a3eca0ac9071..2367c4bfb499f 100644 --- a/daemon/api/endpoints/logs.py +++ b/daemon/api/endpoints/logs.py @@ -104,6 +104,7 @@ async def on_disconnect(self, websocket: WebSocket, close_code: int) -> None: self.active_clients.remove(websocket) daemon_logger.info(f'Client {self.client_details} got disconnected!') - +# TODO: adding websocket in this way do not generate any docs +# see: https://github.com/tiangolo/fastapi/issues/1983 router.add_websocket_route(path='/logstream/{log_id}', endpoint=LogStreamingEndpoint) diff --git a/tests/conftest.py b/tests/conftest.py index 070b66dadf71a..eb69fbe116839 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,11 +4,9 @@ import time import pytest -from daemon.config import fastapi_config from fastapi.testclient import TestClient -from jina.executors.metas import get_default_metas -PREFIX = fastapi_config.PREFIX +from jina.executors.metas import get_default_metas @pytest.fixture(scope='function') @@ -42,37 +40,37 @@ def common_endpoints(): ('swagger_ui_html', '/docs'), ('swagger_ui_redirect', '/docs/oauth2-redirect'), ('redoc_html', '/redoc'), - ('_status', f'{PREFIX}/alive'), - ('LogStreamingEndpoint', f'{PREFIX}/logstream/{{log_id}}') + ('_status', f'/alive'), + ('LogStreamingEndpoint', f'/logstream/{{log_id}}') ] @pytest.fixture(scope='session') def flow_endpoints(): return [ - ('_create_from_pods', f'{PREFIX}/flow/pods'), - ('_create_from_yaml', f'{PREFIX}/flow/yaml'), - ('_fetch', f'{PREFIX}/flow/{{flow_id}}'), - ('_ping', f'{PREFIX}/ping'), - ('_delete', f'{PREFIX}/flow'), + ('_create_from_pods', f'/flow/pods'), + ('_create_from_yaml', f'/flow/yaml'), + ('_fetch', f'/flow/{{flow_id}}'), + ('_ping', f'/ping'), + ('_delete', f'/flow'), ] @pytest.fixture(scope='session') def pod_endpoints(): return [ - ('_upload', f'{PREFIX}/upload'), - ('_create', f'{PREFIX}/pod'), - ('_delete', f'{PREFIX}/pod') + ('_upload', f'upload'), + ('_create', f'pod'), + ('_delete', f'pod') ] @pytest.fixture(scope='session') def pea_endpoints(): return [ - ('_upload', f'{PREFIX}/pea/upload'), - ('_create', f'{PREFIX}/pea'), - ('_delete', f'{PREFIX}/pea') + ('_upload', f'pea/upload'), + ('_create', f'pea'), + ('_delete', f'pea') ] diff --git a/tests/jinad/integration/api/endpoints/test_logging_endpoint.py b/tests/jinad/integration/api/endpoints/test_logging_endpoint.py index 01c05c5571fd9..d2775bedc409c 100644 --- a/tests/jinad/integration/api/endpoints/test_logging_endpoint.py +++ b/tests/jinad/integration/api/endpoints/test_logging_endpoint.py @@ -34,7 +34,7 @@ def feed_path_logs(filepath, total_lines, sleep, mp_event=None): async def test_logging_endpoint_invalid_id(fastapi_client): log_id = uuid.uuid1() with pytest.raises(NoSuchFileException) as response: - with fastapi_client.websocket_connect(f'{fastapi_config.PREFIX}/logstream/{log_id}') as websocket: + with fastapi_client.websocket_connect(f'logstream/{log_id}') as websocket: pass @@ -65,7 +65,7 @@ async def test_logging_dashboard(fastapi_client, total_lines, sleep): # sleeping for 2 secs to allow the process to write logs time.sleep(2) - with fastapi_client.websocket_connect(f'{fastapi_config.PREFIX}/logstream/{log_id}?timeout=1') as websocket: + with fastapi_client.websocket_connect(f'logstream/{log_id}?timeout=1') as websocket: websocket.send_json({'from': 0}) while True: data = websocket.receive_json() @@ -121,7 +121,7 @@ async def test_logging_core(fastapi_client, total_lines, sleep, timeout): # sleeping for 2 secs to allow the process to write logs time.sleep(2) - with fastapi_client.websocket_connect(f'{fastapi_config.PREFIX}/logstream/{log_id}?timeout={timeout}') as websocket: + with fastapi_client.websocket_connect(f'logstream/{log_id}?timeout={timeout}') as websocket: current_line_number = -1 # During tests, these are usual function calls not awaitables # https://www.starlette.io/testclient/#testing-websocket-sessions @@ -166,7 +166,7 @@ async def test_logging_client_disconnection(fastapi_client, total_lines, sleep, # sleeping for 2 secs to allow the process to write logs time.sleep(2) - with fastapi_client.websocket_connect(f'{fastapi_config.PREFIX}/logstream/{log_id}?timeout=1') as websocket: + with fastapi_client.websocket_connect(f'logstream/{log_id}?timeout=1') as websocket: websocket.send_json({'from': 0}) while True: data = websocket.receive_json() diff --git a/tests/jinad/unit/test_config.py b/tests/jinad/unit/test_config.py index 050c6f79b6207..8d3f20a5bacc1 100644 --- a/tests/jinad/unit/test_config.py +++ b/tests/jinad/unit/test_config.py @@ -7,7 +7,6 @@ def test_valid_fastapi_config(): assert FastAPIConfig(NAME='blah').NAME == 'blah' assert FastAPIConfig(DESCRIPTION='blah').DESCRIPTION == 'blah' assert FastAPIConfig(VERSION='blah').VERSION == 'blah' - assert FastAPIConfig(PREFIX='blah').PREFIX == 'blah' def test_invalid_fastapi_config(): From 00a1eced0a1cb07cbd51bb09d30beec071014e9e Mon Sep 17 00:00:00 2001 From: Han Xiao Date: Sun, 10 Jan 2021 18:24:20 +0100 Subject: [PATCH 4/4] fix(daemon): fix some helper text for redocs --- daemon/__init__.py | 5 ++++- daemon/api/endpoints/logs.py | 1 + jina/resources/logging.daemon.yml | 30 ++++++++++++++++++++++++++++++ tests/conftest.py | 12 ++++++------ 4 files changed, 41 insertions(+), 7 deletions(-) create mode 100644 jina/resources/logging.daemon.yml diff --git a/daemon/__init__.py b/daemon/__init__.py index 25d6740c08664..f40cd651f9a21 100644 --- a/daemon/__init__.py +++ b/daemon/__init__.py @@ -1,4 +1,5 @@ import json +import os import subprocess import threading from collections import namedtuple @@ -10,7 +11,9 @@ from jina.logging import JinaLogger from .parser import get_main_parser -daemon_logger = JinaLogger(context='👻 JinaD') +daemon_logger = JinaLogger(context='👻', log_config=os.getenv('JINAD_LOG_CONFIG', + pkg_resources.resource_filename('jina', '/'.join( + ('resources', 'logging.daemon.yml'))))) def _get_app(): diff --git a/daemon/api/endpoints/logs.py b/daemon/api/endpoints/logs.py index 2367c4bfb499f..da26e12f04a91 100644 --- a/daemon/api/endpoints/logs.py +++ b/daemon/api/endpoints/logs.py @@ -104,6 +104,7 @@ async def on_disconnect(self, websocket: WebSocket, close_code: int) -> None: self.active_clients.remove(websocket) daemon_logger.info(f'Client {self.client_details} got disconnected!') + # TODO: adding websocket in this way do not generate any docs # see: https://github.com/tiangolo/fastapi/issues/1983 router.add_websocket_route(path='/logstream/{log_id}', diff --git a/jina/resources/logging.daemon.yml b/jina/resources/logging.daemon.yml new file mode 100644 index 0000000000000..d91219aa46e56 --- /dev/null +++ b/jina/resources/logging.daemon.yml @@ -0,0 +1,30 @@ +handlers: # enabled handlers, order does not matter + - StreamHandler + - SysLogHandler + - FluentHandler +level: INFO # set verbose level +configs: + FileHandler: + format: '%(asctime)s:{name}|%(module)s/%(filename)s:L%(lineno)d@%(process)2d[%(levelname).1s]:%(message)s' + output: 'daemon-{uptime}.log' + formatter: JsonFormatter + StreamHandler: + format: '{name} %(module)s/%(filename)s:L%(lineno)d@%(process)2d[%(levelname).1s]:%(message)s' + formatter: ColorFormatter + SysLogHandler: + ident: # this will be prepend to all messages + format: '{name} %(module)s/%(filename)s:L%(lineno)d@%(process)2d[%(levelname).1s]:%(message)s' + host: # when not given then record it locally, /dev/log on linux /var/run/syslog on mac + port: # when not given then record it locally, /dev/log on linux /var/run/syslog on mac + formatter: PlainFormatter + FluentHandler: + # this configuration describes where is the fluentD daemon running and waiting for logs to be emitted. + # FluentD then will have its own configuration to forward the messages according to its own syntax + # prefix will help fluentD filter data. This will be prepended for FluentD to easily filter incoming messages + tag: daemon + host: 0.0.0.0 + port: 24224 + format: + host: '%(hostname)s' + process: '%(process)s' + type: '%(levelname)s' diff --git a/tests/conftest.py b/tests/conftest.py index eb69fbe116839..913bef3db77a3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -59,18 +59,18 @@ def flow_endpoints(): @pytest.fixture(scope='session') def pod_endpoints(): return [ - ('_upload', f'upload'), - ('_create', f'pod'), - ('_delete', f'pod') + ('_upload', f'/upload'), + ('_create', f'/pod'), + ('_delete', f'/pod') ] @pytest.fixture(scope='session') def pea_endpoints(): return [ - ('_upload', f'pea/upload'), - ('_create', f'pea'), - ('_delete', f'pea') + ('_upload', f'/pea/upload'), + ('_create', f'/pea'), + ('_delete', f'/pea') ]