jina-ai · hanxiao · Jan 11, 2021 · Jan 8, 2021 · Jan 9, 2021 · Jan 9, 2021
diff --git a/jina/drivers/craft.py b/jina/drivers/craft.py
@@ -1,10 +1,9 @@
 __copyright__ = "Copyright (c) 2020 Jina AI Limited. All rights reserved."
 __license__ = "Apache-2.0"
 
-from typing import Tuple
+from typing import Optional
 
 from . import BaseExecutableDriver
-from ..types.document import Document
 
 if False:
     from .. import DocumentSet
@@ -13,7 +12,7 @@
 class CraftDriver(BaseExecutableDriver):
     """Drivers inherited from this Driver will bind :meth:`craft` by default """
 
-    def __init__(self, executor: str = None, method: str = 'craft', *args, **kwargs):
+    def __init__(self, executor: Optional[str] = None, method: str = 'craft', *args, **kwargs):
         super().__init__(executor, method, *args, **kwargs)
 
     def _apply_all(self, docs: 'DocumentSet', *args, **kwargs):
@@ -26,24 +25,3 @@ def _apply_all(self, docs: 'DocumentSet', *args, **kwargs):
     @staticmethod
     def update(doc, ret):
         doc.set_attrs(**ret)
-
-
-class SegmentDriver(CraftDriver):
-    """Segment document into chunks using the executor
-    """
-
-    def __init__(
-            self,
-            traversal_paths: Tuple[str] = ('r',),
-            *args,
-            **kwargs
-    ):
-        super().__init__(traversal_paths=traversal_paths, *args, **kwargs)
-
-    @staticmethod
-    def update(doc, ret):
-        for r in ret:
-            with Document(length=len(ret), **r) as c:
-                if not c.mime_type:
-                    c.mime_type = doc.mime_type
-            doc.chunks.append(c)
diff --git a/jina/drivers/segment.py b/jina/drivers/segment.py
@@ -0,0 +1,39 @@
+__copyright__ = "Copyright (c) 2020 Jina AI Limited. All rights reserved."
+__license__ = "Apache-2.0"
+
+from typing import Tuple, Optional
+
+from . import BaseExecutableDriver
+from ..types.document import Document
+
+if False:
+    from .. import DocumentSet
+
+
+class SegmentDriver(BaseExecutableDriver):
+    """Drivers inherited from this Driver will bind :meth:`segment` by default """
+
+    def __init__(
+            self,
+            executor: Optional[str] = None,
+            method: str = 'segment',
+            traversal_paths: Tuple[str] = ('r',),
+            *args,
+            **kwargs
+    ):
+        super().__init__(executor, method, traversal_paths=traversal_paths, *args, **kwargs)
+
+    def _apply_all(self, docs: 'DocumentSet', *args, **kwargs):
+        for doc in docs:
+            _args_dict = doc.get_attrs(*self.exec.required_keys)
+            ret = self.exec_fn(**_args_dict)
+            if ret:
+                self.update(doc, ret)
+
+    @staticmethod
+    def update(doc, ret):
+        for r in ret:
+            with Document(length=len(ret), **r) as c:
+                if not c.mime_type:
+                    c.mime_type = doc.mime_type
+            doc.chunks.append(c)
diff --git a/jina/executors/__init__.py b/jina/executors/__init__.py
@@ -112,7 +112,7 @@ def __init__(awesomeness = 5):
 
     """
     store_args_kwargs = False  #: set this to ``True`` to save ``args`` (in a list) and ``kwargs`` (in a map) in YAML config
-    exec_methods = ['encode', 'add', 'query', 'craft', 'score', 'evaluate', 'predict', 'query_by_id', 'delete', 'update']
+    exec_methods = ['encode', 'add', 'query', 'craft', 'segment', 'score', 'evaluate', 'predict', 'query_by_id', 'delete', 'update']
 
     def __init__(self, *args, **kwargs):
         if isinstance(args, tuple) and len(args) > 0:

diff --git a/jina/executors/crafters/__init__.py b/jina/executors/crafters/__init__.py
@@ -2,7 +2,7 @@
 __license__ = "Apache-2.0"
 
 import inspect
-from typing import Dict, List
+from typing import Dict
 
 from .. import BaseExecutor
 from ...helper import typename
@@ -28,18 +28,3 @@ def craft(self, *args, **kwargs) -> Dict:
         in the protobuf.
         """
         raise NotImplementedError
-
-
-class BaseSegmenter(BaseCrafter):
-    """:class:`BaseSegmenter` works on doc-level,
-        it receives value on the doc-level and returns new value on the chunk-level """
-
-    def craft(self, *args, **kwargs) -> List[Dict]:
-        """The apply function of this executor.
-        Unlike :class:`BaseCrafter`, the :func:`craft` here works on doc-level info and the output is defined on
-        chunk-level. Therefore the name of the arguments should be always valid keys defined
-        in the doc-level protobuf whereas the output dict keys should always be valid keys defined in the chunk-level
-        protobuf.
-        :return: a list of chunks-level info represented by a dict
-        """
-        raise NotImplementedError
diff --git a/jina/executors/segmenters/__init__.py b/jina/executors/segmenters/__init__.py
@@ -0,0 +1,26 @@
+import inspect
+from typing import Dict, List
+
+from .. import BaseExecutor
+from ...helper import typename
+
+
+class BaseSegmenter(BaseExecutor):
+    """:class:`BaseSegmenter` works on doc-level,
+        it chunks Documents into set of Chunks """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.required_keys = {k for k in inspect.getfullargspec(self.segment).args if k != 'self'}
+        if not self.required_keys:
+            self.logger.warning(f'{typename(self)} works on keys, but no keys are specified')
+
+    def segment(self, *args, **kwargs) -> List[Dict]:
+        """The apply function of this executor.
+        Unlike :class:`BaseCrafter`, the :func:`craft` here works on doc-level info and the output is defined on
+        chunk-level. Therefore the name of the arguments should be always valid keys defined
+        in the doc-level protobuf whereas the output dict keys should always be valid keys defined in the chunk-level
+        protobuf.
+        :return: a list of chunks-level info represented by a dict
+        """
+        raise NotImplementedError
diff --git a/tests/integration/issues/github_1546/bad1/custom_crafter.py b/tests/integration/issues/github_1546/bad1/custom_crafter.py
@@ -1,8 +1,8 @@
-from jina.executors.crafters import BaseSegmenter
+from jina.executors.crafters import BaseCrafter
 from .helper import helper_function
 
 
-class CustomCrafter1(BaseSegmenter):
+class CustomCrafter1(BaseCrafter):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         print(helper_function)
diff --git a/tests/integration/issues/github_1546/bad2/__init__.py b/tests/integration/issues/github_1546/bad2/__init__.py
@@ -1,9 +1,9 @@
 from .helper import helper_function
 
-from jina.executors.crafters import BaseSegmenter
+from jina.executors.crafters import BaseCrafter
 
 
-class CustomCrafter2(BaseSegmenter):
+class CustomCrafter2(BaseCrafter):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         print(helper_function)
diff --git a/tests/integration/issues/github_1546/good1/custom_crafter.py b/tests/integration/issues/github_1546/good1/custom_crafter.py
@@ -1,9 +1,9 @@
 from jinahub.helper import helper_function
 
-from jina.executors.crafters import BaseSegmenter
+from jina.executors.crafters import BaseCrafter
 
 
-class CustomCrafter3(BaseSegmenter):
+class CustomCrafter3(BaseCrafter):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         print(helper_function)
diff --git a/tests/integration/issues/github_1546/good2/__init__.py b/tests/integration/issues/github_1546/good2/__init__.py
@@ -1,9 +1,9 @@
 from .helper import helper_function
 
-from jina.executors.crafters import BaseSegmenter
+from jina.executors.crafters import BaseCrafter
 
 
-class CustomCrafter4(BaseSegmenter):
+class CustomCrafter4(BaseCrafter):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         print(helper_function)
diff --git a/tests/integration/issues/github_1546/good3/__init__.py b/tests/integration/issues/github_1546/good3/__init__.py
@@ -1,7 +1,7 @@
 from typing import Tuple, Dict, List, Union
 
 import numpy as np
-from jina.executors.crafters import BaseSegmenter
+from jina.executors.segmenters import BaseSegmenter
 
 from .helper import _crop_image, _move_channel_axis, _load_image
 
@@ -25,7 +25,7 @@ def __init__(self,
         self.target_size = target_size
         self.channel_axis = channel_axis
 
-    def craft(self, blob: 'np.ndarray', *args, **kwargs) -> List[Dict]:
+    def segment(self, blob: 'np.ndarray', *args, **kwargs) -> List[Dict]:
         """
         Crop the input image array.
 

diff --git a/tests/integration/level_depth/yaml/custom_executor.py b/tests/integration/level_depth/yaml/custom_executor.py
@@ -0,0 +1,37 @@
+import re
+import string
+from typing import Dict, List
+
+from jina.executors.segmenters import BaseSegmenter
+
+
+class DummySentencizer(BaseSegmenter):
+
+    def __init__(self,
+                 *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        punct_chars = [',']
+        self._slit_pat = re.compile('\s*([^{0}]+)(?<!\s)[{0}]*'.format(''.join(set(punct_chars))))
+
+    def segment(self, text: str, *args, **kwargs) -> List[Dict]:
+        """
+        Split the text into sentences.
+
+        :param text: the raw text
+        :return: a list of chunk dicts with the cropped images
+        """
+        results = []
+        ret = [(m.group(0), m.start(), m.end()) for m in
+               re.finditer(self._slit_pat, text)]
+        if not ret:
+            ret = [(text, 0, len(text))]
+        for ci, (r, s, e) in enumerate(ret):
+            f = ''.join(filter(lambda x: x in string.printable, r))
+            f = re.sub('\n+', ' ', f).strip()
+            f = f[:100]
+            if len(f) > 3:
+                results.append(dict(
+                    text=f
+                ))
+        return results
diff --git a/tests/integration/level_depth/yaml/segmenter.yml b/tests/integration/level_depth/yaml/segmenter.yml
@@ -1,10 +1,12 @@
-!Sentencizer
+!DummySentencizer
 with:
   min_sent_len: 3
   max_sent_len: 128
   punct_chars: '.,;!?:'
 metas:
   name: sentencizer
+  py_modules:
+    - custom_executor.py
 requests:
   on:
     [SearchRequest, IndexRequest]:

diff --git a/tests/unit/yaml/dummy-seg-not-random.yml → ...integration/mime/dummy-seg-not-random.yml b/tests/unit/yaml/dummy-seg-not-random.yml → ...integration/mime/dummy-seg-not-random.yml
@@ -6,4 +6,4 @@ requests:
     [SearchRequest, TrainRequest, IndexRequest]:
     - !SegmentDriver
       with:
-        method: craft
+        method: segment
diff --git a/tests/unit/yaml/dummy-seg-random.yml → tests/integration/mime/dummy-seg-random.yml b/tests/unit/yaml/dummy-seg-random.yml → tests/integration/mime/dummy-seg-random.yml
@@ -6,4 +6,4 @@ requests:
     [SearchRequest, TrainRequest, IndexRequest]:
     - !SegmentDriver
       with:
-        method: craft
+        method: segment
diff --git a/tests/integration/mime/test_segmenter.py b/tests/integration/mime/test_segmenter.py
@@ -2,15 +2,15 @@
 
 import pytest
 
-from jina.executors.crafters import BaseSegmenter
+from jina.executors.segmenters import BaseSegmenter
 from jina.flow import Flow
 from tests import random_docs
 
 cur_dir = os.path.dirname(os.path.abspath(__file__))
 

 class DummySegment(BaseSegmenter):
-    def craft(self):
+    def segment(self):
         return [dict(buffer=b'aa'), dict(buffer=b'bb')]
 
 
@@ -36,7 +36,7 @@ def test_dummy_seg(mocker, restful):
 @pytest.mark.parametrize('restful', [False, True])
 def test_dummy_seg_random(mocker, restful):
     mock = mocker.Mock()
-    f = Flow(restful=restful).add(uses=os.path.join(cur_dir, '../../unit/yaml/dummy-seg-random.yml'))
+    f = Flow(restful=restful).add(uses=os.path.join(cur_dir, 'dummy-seg-random.yml'))
     with f:
         f.index(input_fn=random_docs(10, chunks_per_doc=0), on_done=validate_factory(mock))
     mock.assert_called_once()
@@ -45,7 +45,7 @@ def test_dummy_seg_random(mocker, restful):
 @pytest.mark.parametrize('restful', [False, True])
 def test_dummy_seg_not_random(mocker, restful):
     mock = mocker.Mock()
-    f = Flow(restful=restful).add(uses=os.path.join(cur_dir, '../../unit/yaml/dummy-seg-not-random.yml'))
+    f = Flow(restful=restful).add(uses=os.path.join(cur_dir, 'dummy-seg-not-random.yml'))
     with f:
         f.index(input_fn=random_docs(10, chunks_per_doc=0), on_done=validate_factory(mock))
     mock.assert_called_once()
diff --git a/tests/unit/drivers/querylang/test_querylang_drivers.py b/tests/unit/drivers/querylang/test_querylang_drivers.py
@@ -1,6 +1,6 @@
 from google.protobuf import json_format
 
-from jina.executors.crafters import BaseSegmenter
+from jina.executors.segmenters import BaseSegmenter
 from jina.flow import Flow
 from jina.proto import jina_pb2
 
@@ -45,13 +45,13 @@ def random_docs_with_tags():
 
 class DummySegmenter(BaseSegmenter):
 
-    def craft(self, text, *args, **kwargs):
+    def segment(self, text, *args, **kwargs):
         return [{'text': 'adasd' * (j + 1), 'tags': {'id': j}} for j in range(10)]
 
 
 class DummyModeIdSegmenter(BaseSegmenter):
 
-    def craft(self, text, *args, **kwargs):
+    def segment(self, text, *args, **kwargs):
         if 'chunk3' not in text:
             return [{'text': f'chunk{j + 1}', 'modality': f'mode{j + 1}'} for j in range(2)]
         elif 'chunk3' in text:

diff --git a/tests/unit/drivers/test_reduce_all_driver.py b/tests/unit/drivers/test_reduce_all_driver.py
@@ -5,7 +5,7 @@
 import pytest
 
 from jina import Document
-from jina.executors.crafters import BaseSegmenter
+from jina.executors.segmenters import BaseSegmenter
 from jina.executors.encoders import BaseEncoder
 from jina.flow import Flow
 
@@ -24,7 +24,7 @@ def docs():
 
 class MockSegmenterReduce(BaseSegmenter):
 
-    def craft(self, text: str, *args, **kwargs) -> List[Dict]:
+    def segment(self, text: str, *args, **kwargs) -> List[Dict]:
         split = text.split(',')
         chunks = [dict(text=split[0], offset=0, weight=1.0, modality='mode1'),
                   dict(text=split[1], offset=1, weight=1.0, modality='mode2')]
@@ -57,9 +57,9 @@ def validate(req):
 
     response_mock = mocker.Mock(wrap=validate)
 
-    flow = Flow().add(name='crafter', uses='MockSegmenterReduce'). \
+    flow = Flow().add(name='segmenter', uses='MockSegmenterReduce'). \
         add(name='encoder1', uses=os.path.join(cur_dir, 'yaml/mockencoder-mode1.yml')). \
-        add(name='encoder2', uses=os.path.join(cur_dir, 'yaml/mockencoder-mode2.yml'), needs=['crafter']). \
+        add(name='encoder2', uses=os.path.join(cur_dir, 'yaml/mockencoder-mode2.yml'), needs=['segmenter']). \
         add(name='reducer', uses='- !ReduceAllDriver | {traversal_paths: [c]}',
             needs=['encoder1', 'encoder2'])
 

diff --git a/tests/unit/drivers/test_segmenter_driver.py b/tests/unit/drivers/test_segmenter_driver.py
@@ -4,8 +4,8 @@
 import pytest
 
 from jina import Document
-from jina.drivers.craft import SegmentDriver
-from jina.executors.crafters import BaseSegmenter
+from jina.drivers.segment import SegmentDriver
+from jina.executors.segmenters import BaseSegmenter
 from jina.types.sets import DocumentSet
 
 
@@ -14,7 +14,7 @@ def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.required_keys = {'text'}
 
-    def craft(self, text: str, *args, **kwargs) -> List[Dict]:
+    def segment(self, text: str, *args, **kwargs) -> List[Dict]:
         if text == 'valid':
             # length, parent_id and id are protected keys that won't affect the segments
             return [{'blob': np.array([0.0, 0.0, 0.0]), 'weight': 0.0, 'mime_type': 'text/plain', 'tags': {'id': 3}},
@@ -26,6 +26,9 @@ def craft(self, text: str, *args, **kwargs) -> List[Dict]:
 
 class SimpleSegmentDriver(SegmentDriver):
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
     @property
     def exec_fn(self):
         return self._exec_fn