8000 fix: fix sparse pipeline test by JoanFM · Pull Request #2641 · jina-ai/serve · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

fix: fix sparse pipeline test #2641

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jun 14, 2021
Merged

fix: fix sparse pipeline test #2641

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 19 additions & 20 deletions tests/integration/sparse_pipeline/test_sparse_pipeline.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from typing import Any, Iterable
from typing import Any
import os

import pytest
Expand All @@ -10,6 +10,7 @@
from tests import validate_callback

cur_dir = os.path.dirname(os.path.abspath(__file__))
TOP_K = 3


@pytest.fixture(scope='function')
Expand All @@ -27,32 +28,28 @@ def docs_to_index(num_docs):


class DummyCSRSparseIndexEncoder(Executor):
embedding_cls_type = 'scipy_csr'

def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.docs = DocumentArray()
self.vectors = {}

@requests(on='index')
@requests(on='/index')
def encode(self, docs: 'DocumentArray', *args, **kwargs) -> Any:
for i, doc in enumerate(docs):
doc.embedding = sparse.coo_matrix(doc.content)
self.docs.extend(docs)
for i, doc in enumerate(self.docs):
doc.embedding = sparse.csr_matrix(doc.content)
self.vectors[doc.id] = doc.embedding.getrow(i)

@requests(on='search')
def query(self, parameters, *args, **kwargs):
top_k = parameters['top_k']
doc = parameters['doc']
distances = [item for item in range(0, min(top_k, len(self.docs)))]
return [self.docs[:top_k]], np.array([distances])
@requests(on='/search')
def query(self, docs: 'DocumentArray', parameters, *args, **kwargs):
top_k = int(parameters['top_k'])
for doc in docs:
doc.matches = self.docs[:top_k]


def test_sparse_pipeline(mocker, docs_to_index):
def validate(response):
assert len(response.data.docs) == 10
for doc in response.data.docs:
assert len(response.docs) == 1
for doc in response.docs:
assert len(doc.matches) == TOP_K
for i, match in enumerate(doc.matches):
assert match.id == docs_to_index[i].id
assert isinstance(match.embedding, sparse.coo_matrix)
Expand All @@ -63,13 +60,15 @@ def validate(response):
error_mock = mocker.Mock()

with f:
f.index(
f.post(
on='/index',
5D79 inputs=docs_to_index,
on_done=mock,
on_error=error_mock,
)
f.search(
f.post(
on='/search',
inputs=docs_to_index[0],
parameters={'doc': docs_to_index[0], 'top_k': 1},
parameters={'top_k': TOP_K},
on_done=mock,
on_error=error_mock,
)
Expand Down
0