8000 fix(daemon): fix some helper text for redocs by hanxiao · Pull Request #1642 · jina-ai/serve · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

fix(daemon): fix some helper text for redocs #1642

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Jan 10, 2021
Merged

fix(daemon): fix some helper text for redocs #1642

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ee80f65
fix(daemon): fix some helper text for redocs
hanxiao Jan 10, 2021
94afe18
fix(daemon): fix some helper text for redocs
hanxiao Jan 10, 2021
1eac6f4
fix(daemon): fix some helper text for redocs
hanxiao Jan 10, 2021
00a1ece
fix(daemon): fix some helper text for redocs
hanxiao Jan 10, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/cd.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ jobs:
- name: push-to-api-repo
run: |
cp jinad.html schema/jinad/${{env.JINA_VERSION}}.html
cp jinad.html schema/daemon/${{env.JINA_VERSION}}.html
cd schema
git config --local user.email "dev-bot@jina.ai"
git config --local user.name "Jina Dev Bot"
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/tag.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ jobs:
run: |
cp jinad.html schema/jinad/index.html
cp jinad.html schema/jinad/${{env.JINA_VERSION}}.html
cp jinad.html schema/daemon/index.html
cp jinad.html schema/daemon/${{env.JINA_VERSION}}.html
cd schema
git config --local user.email "dev-bot@jina.ai"
git config --local user.name "Jina Dev Bot"
Expand Down
17 changes: 8 additions & 9 deletions daemon/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import json
import os
import subprocess
import threading
from collections import namedtuple
Expand All @@ -10,7 +11,9 @@
from jina.logging import JinaLogger
from .parser import get_main_parser

daemon_logger = JinaLogger(context='👻 JINAD')
daemon_logger = JinaLogger(context='👻', log_config=os.getenv('JINAD_LOG_CONFIG',
pkg_resources.resource_filename('jina', '/'.join(
('resources', 'logging.daemon.yml')))))


def _get_app():
Expand All @@ -34,21 +37,17 @@ def _get_app():
description=fastapi_config.DESCRIPTION,
version=fastapi_config.VERSION
)
app.include_router(router=common_router,
prefix=fastapi_config.PREFIX)
app.include_router(router=logs.router,
prefix=fastapi_config.PREFIX)
app.include_router(router=common_router)
app.include_router(router=logs.router)
if jinad_config.CONTEXT == 'all':
for _current_router in _all_routers.values():
app.include_router(router=_current_router.router,
tags=_current_router.tags,
prefix=fastapi_config.PREFIX)
tags=_current_router.tags)
else:
_current_router = _all_routers[jinad_config.CONTEXT]
app.openapi_tags = _current_router.openapi_tags
app.include_router(router=_current_router.router,
tags=_current_router.tags,
prefix=fastapi_config.PREFIX)
tags=_current_router.tags)
return app


Expand Down
6 changes: 3 additions & 3 deletions daemon/api/endpoints/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,13 @@

@common_router.on_event('startup')
async def startup():
daemon_logger.success('Welcome to Jina daemon - the manager of distributed Jina')
daemon_logger.success(f'Uvicorn + FastAPI running on {server_config.HOST}:{server_config.PORT}')
daemon_logger.success(f'Welcome to Jina daemon - the manager of distributed Jina\n'
f'Uvicorn + FastAPI running on {server_config.HOST}:{server_config.PORT}')


@common_router.get(
path='/alive',
summary='Get status of jinad',
summary='Check if daemon is alive',
status_code=status.HTTP_200_OK
)
async def _status():
Expand Down
38 changes: 18 additions & 20 deletions daemon/api/endpoints/flow.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,14 @@
from jina.parsers import set_client_cli_parser
from ...excepts import FlowYamlParseException, FlowCreationException, FlowStartException
from ...models import SinglePodModel
from ...models.custom import build_pydantic_model
from ...store import flow_store

router = APIRouter()


@router.put(
path='/flow/pods',
summary='Build & start a Flow using Pods',
summary='Start a Flow from list of Pods',
)
async def _create_from_pods(
pods: Union[List[Dict]] = Body(..., example=json.loads(SinglePodModel().json()))
Expand Down Expand Up @@ -59,7 +58,7 @@ async def _create_from_pods(

@router.put(
path='/flow/yaml',
summary='Build & start a Flow using YAML',
summary='Start a Flow from a YAML config',
)
async def _create_from_yaml(
yamlspec: UploadFile = File(...),
Expand All @@ -69,26 +68,24 @@ async def _create_from_yaml(
"""
Build a flow using [Flow YAML](https://docs.jina.ai/chapters/yaml/yaml.html#flow-yaml-sytanx)

> Upload Flow yamlspec (`yamlspec`)

> Yamls that Pods use (`uses_files`) (Optional)

> Python modules (`pymodules_files`) that the Pods use (Optional)
- Upload Flow yamlspec (`yamlspec`)
- Yamls that Pods use (`uses_files`) (Optional)
- Python modules (`pymodules_files`) that the Pods use (Optional)

**yamlspec**:

!Flow
version: 1.0
with:
rest_api: true
compress_hwm: 1024
restful: true
pods:
encode:
uses: helloworld.encoder.yml
parallel: 2
index:
uses: helloworld.indexer.yml
shards: 2
separated_workspace: true
- name: encode
uses: helloworld.encoder.yml
parallel: 2
- name: index
uses: helloworld.indexer.yml
shards: 2
separated_workspace: true

**uses_files**: `helloworld.encoder.yml`

Expand Down Expand Up @@ -158,7 +155,7 @@ def __init__(self):

@router.get(
path='/flow/{flow_id}',
summary='Get Flow information',
summary='Get the status of a running Flow',
)
async def _fetch(
flow_id: uuid.UUID,
Expand Down Expand Up @@ -193,7 +190,7 @@ async def _fetch(

@router.get(
path='/ping',
summary='Connect to Flow gateway',
summary='Check if the Flow is alive',
)
async def _ping(
host: str,
Expand All @@ -208,6 +205,7 @@ async def _ping(
_, args, _ = ArgNamespace.get_parsed_args(kwargs, set_client_cli_parser())
client = Client(args)
try:
# TODO: this introduces side-effect, need to be refactored. (2020.01.10)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Client & WebsocketClient need to be refactored to work as a context manager. This interface can tell the daemon client if the gateway server is alive.

client.index(input_fn=['abc'])
return {
'status_code': status.HTTP_200_OK,
Expand All @@ -220,7 +218,7 @@ async def _ping(

@router.delete(
path='/flow',
summary='Close Flow Context',
summary='Terminate a running Flow',
)
async def _delete(
flow_id: uuid.UUID
Expand Down
2 changes: 2 additions & 0 deletions daemon/api/endpoints/logs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,5 +105,7 @@ async def on_disconnect(self, websocket: WebSocket, close_code: int) -> None:
daemon_logger.info(f'Client {self.client_details} got disconnected!')


# TODO: adding websocket in this way do not generate any docs
# see: https://github.com/tiangolo/fastapi/issues/1983
router.add_websocket_route(path='/logstream/{log_id}',
endpoint=LogStreamingEndpoint)
4 changes: 2 additions & 2 deletions daemon/api/endpoints/pea.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@

@router.put(
path='/pea/upload',
summary='Upload pod context yamls & pymodules',
summary='Upload YAML & py_modules required by a Pea',
)
async def _upload(
uses_files: List[UploadFile] = File(()),
Expand Down Expand Up @@ -70,7 +70,7 @@ async def _create(

@router.delete(
path='/pea',
summary='Close Pea Context',
summary='Terminate a running Pea',
)
async def _delete(
pea_id: uuid.UUID
Expand Down
6 changes: 3 additions & 3 deletions daemon/api/endpoints/pod.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@

@router.put(
path='/upload',
summary='Upload pod context yamls & pymodules',
summary='Upload YAML & py_modules required by a Pod',
)
async def _upload(
uses_files: List[UploadFile] = File(()),
Expand All @@ -41,7 +41,7 @@ async def _upload(

@router.put(
path='/pod',
summary='Create a Pod via Flow or CLI',
summary='Create a Pod',
)
async def _create(
pod_arguments: Union[SinglePodModel, ParallelPodModel]
Expand Down Expand Up @@ -71,7 +71,7 @@ async def _create(

@router.delete(
path='/pod',
summary='Close Pod Context',
summary='Terminate a running Pod',
)
async def _delete(
pod_id: uuid.UUID
Expand Down
25 changes: 11 additions & 14 deletions daemon/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,49 +2,46 @@

from jina import __version__

__prefix__ = 'v1'


class BaseConfig(BaseSettings):
class Config:
env_prefix = 'JINAD_'


class FastAPIConfig(BaseConfig):
NAME: str = 'Jina Remote Manager'
DESCRIPTION: str = 'REST API for managing Jina on Remote'
NAME: str = 'JinaD (Daemon)'
DESCRIPTION: str = 'The REST API of the daemon for managing distributed Jina'
VERSION: str = __version__
PREFIX: str = '/' + __prefix__


class OpenAPITags(BaseConfig):
API_TAGS: list = [{
'name': 'Jina Remote Management',
'description': 'API to invoke remote Flows/Pods/Peas',
'name': 'Distributed Jina management',
'description': 'API to invoke distributed Flows/Pods/Peas',
'externalDocs': {
'description': 'Jina Remote Context Manager',
'description': 'Jina Documentation',
'url': 'https://docs.jina.ai/',
},
}]
FLOW_API_TAGS: list = [{
'name': 'Remote Flow Manager',
'description': 'API to invoke local/remote Flows',
'name': 'Managing distributed Flow',
'description': 'API to invoke distributed Flows',
'externalDocs': {
'description': 'Jina Flow Context Manager',
'url': 'https://docs.jina.ai/chapters/flow/index.html',
},
}]
POD_API_TAGS: list = [{
'name': 'Remote Pod Manager',
'description': 'API to invoke remote Pods (__should be used by Flow APIs only__)',
'name': 'Managing distributed Pod',
'description': 'API to invoke distributed Pods (__should be used by Flow APIs only__)',
'externalDocs': {
'description': 'Jina 101',
'url': 'https://docs.jina.ai/chapters/101/.sphinx.html',
},
}]
PEA_API_TAGS: list = [{
'name': 'Remote Pea Manager',
'description': 'API to invoke remote Peas',
'name': 'Managing distributed Pea',
'description': 'API to invoke distributed Peas',
'externalDocs': {
'description': 'Jina 101',
'url': 'https://docs.jina.ai/chapters/101/.sphinx.html',
Expand Down
10 changes: 4 additions & 6 deletions daemon/store.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,11 @@

from fastapi import UploadFile

from jina.enums import PodRoleType
from jina.flow import Flow
from jina.jaml import JAML
from jina.helper import colored
from jina.enums import PodRoleType
from jina.logging import JinaLogger
from jina.peapods import Pea, Pod
from . import daemon_logger
from .excepts import FlowYamlParseException, FlowCreationException, \
FlowStartException, PodStartException, PeaStartException, FlowBadInputException
from .helper import create_meta_files_from_upload, delete_meta_files_from_upload
Expand All @@ -24,7 +23,7 @@ class InMemoryStore:
# https://github.com/jina-ai/jinad/issues/4
credentials = 'foo:bar'
_session_token = None
logger = JinaLogger(context='🏪 STORE')
logger = daemon_logger

@contextmanager
def _session(self):
Expand Down Expand Up @@ -79,8 +78,7 @@ def _create(self,
if isinstance(config, str) or isinstance(config, SpooledTemporaryFile):
yamlspec = config.read().decode() if isinstance(config, SpooledTemporaryFile) else config
try:
JAML.register(Flow)
flow = JAML.load(yamlspec)
flow = Flow.load_config(yamlspec)
except Exception as e:
self.logger.error(f'Got error while loading from yaml {e!r}')
raise FlowYamlParseException
Expand Down
7 changes: 3 additions & 4 deletions jina/peapods/runtimes/jinad/api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,6 @@ class JinadAPI:
def __init__(self,
host: str,
port: int,
version: str = 'v1',
logger: 'JinaLogger' = None,
timeout: int = 5, **kwargs):
"""
Expand All @@ -92,9 +91,9 @@ def __init__(self,
# TODO: for https, the jinad server would need a tls certificate.
# no changes would be required in terms of how the api gets invoked,
# as requests does ssl verfication. we'd need to add some exception handling logic though
url = f'{host}:{port}/{version}'
rest_url = f'http://{url}'
websocket_url = f'ws://{url}'
base_url = f'{host}:{port}'
rest_url = f'http://{base_url}'
websocket_url = f'ws://{base_url}'
self.alive_url = f'{rest_url}/alive'
self.upload_url = f'{rest_url}/upload'
self.pea_url = f'{rest_url}/pea'
Expand Down
30 changes: 30 additions & 0 deletions jina/resources/logging.daemon.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
handlers: # enabled handlers, order does not matter
- StreamHandler
- SysLogHandler
- FluentHandler
level: INFO # set verbose level
configs:
FileHandler:
format: '%(asctime)s:{name}|%(module)s/%(filename)s:L%(lineno)d@%(process)2d[%(levelname).1s]:%(message)s'
output: 'daemon-{uptime}.log'
formatter: JsonFormatter
StreamHandler:
format: '{name} %(module)s/%(filename)s:L%(lineno)d@%(process)2d[%(levelname).1s]:%(message)s'
formatter: ColorFormatter
SysLogHandler:
ident: # this will be prepend to all messages
format: 6278 '{name} %(module)s/%(filename)s:L%(lineno)d@%(process)2d[%(levelname).1s]:%(message)s'
host: # when not given then record it locally, /dev/log on linux /var/run/syslog on mac
port: # when not given then record it locally, /dev/log on linux /var/run/syslog on mac
formatter: PlainFormatter
FluentHandler:
# this configuration describes where is the fluentD daemon running and waiting for logs to be emitted.
# FluentD then will have its own configuration to forward the messages according to its own syntax
# prefix will help fluentD filter data. This will be prepended for FluentD to easily filter incoming messages
tag: daemon
host: 0.0.0.0
port: 24224
format:
host: '%(hostname)s'
process: '%(process)s'
type: '%(levelname)s'
Loading
0