From 05dec19437c9e1005f3e8437f49d768ea712d0e1 Mon Sep 17 00:00:00 2001 From: Lauren-D Date: Fri, 26 May 2023 17:07:40 +0200 Subject: [PATCH 01/14] local entity: create new resource * Adds basic `local_entity` resource. * Adds `pro_entity_manager` permission. * Adds `invenio-records-rest` config for unified search. * Adds custom routing converter defining 'dummypid' type. * Writes tests. Co-Authored-by: Lauren-D --- data/local_entities.json | 27 ++++ data/role_policies.json | 25 ++- data/system_role_policies.json | 6 + pyproject.toml | 14 +- rero_ils/config.py | 76 ++++++++- rero_ils/converters.py | 57 +++++++ rero_ils/es_templates/v7/record.json | 1 + rero_ils/modules/commons/dumpers.py | 2 +- .../mappings/v7/entities/entity-v0.0.1.json | 3 + rero_ils/modules/local_entities/__init__.py | 19 +++ rero_ils/modules/local_entities/api.py | 117 ++++++++++++++ .../local_entities/dumpers/__init__.py | 40 +++++ .../modules/local_entities/dumpers/indexer.py | 33 ++++ .../local_entities/extensions/__init__.py | 24 +++ .../extensions/authorized_access_point.py | 42 +++++ .../local_entities/jsonschemas/__init__.py | 19 +++ .../local_entities/local_entity-v0.0.1.json | 45 ++++++ .../local_entities/mappings/__init__.py | 19 +++ .../local_entities/mappings/v7/__init__.py | 19 +++ .../local_entities/local_entity-v0.0.1.json | 76 +++++++++ rero_ils/modules/local_entities/models.py | 54 +++++++ .../modules/local_entities/permissions.py | 45 ++++++ .../jsonschemas/patrons/patron-v0.0.1.json | 7 +- rero_ils/modules/unified_entities/__init__.py | 19 +++ rero_ils/modules/unified_entities/fetchers.py | 46 ++++++ rero_ils/modules/unified_entities/minters.py | 37 +++++ .../unified_entities/serializers/__init__.py | 31 ++++ .../unified_entities/serializers/base.py | 37 +++++ rero_ils/modules/users/models.py | 3 +- rero_ils/modules/utils.py | 26 +++- scripts/setup | 6 + setup.py | 8 + .../test_local_entities_permissions.py | 92 +++++++++++ .../test_local_entities_rest.py | 144 ++++++++++++++++++ tests/api/test_monitoring_rest.py | 4 + .../test_unified_entities_rest.py | 86 +++++++++++ .../test_unified_entities_search.py | 58 +++++++ tests/data/data.json | 15 +- tests/data/policies/role_policies.json | 12 ++ tests/data/policies/system_role_policies.json | 6 + tests/fixtures/metadata.py | 38 +++++ 41 files changed, 1422 insertions(+), 16 deletions(-) create mode 100644 data/local_entities.json create mode 100644 rero_ils/converters.py create mode 100644 rero_ils/modules/local_entities/__init__.py create mode 100644 rero_ils/modules/local_entities/api.py create mode 100644 rero_ils/modules/local_entities/dumpers/__init__.py create mode 100644 rero_ils/modules/local_entities/dumpers/indexer.py create mode 100644 rero_ils/modules/local_entities/extensions/__init__.py create mode 100644 rero_ils/modules/local_entities/extensions/authorized_access_point.py create mode 100644 rero_ils/modules/local_entities/jsonschemas/__init__.py create mode 100644 rero_ils/modules/local_entities/jsonschemas/local_entities/local_entity-v0.0.1.json create mode 100644 rero_ils/modules/local_entities/mappings/__init__.py create mode 100644 rero_ils/modules/local_entities/mappings/v7/__init__.py create mode 100644 rero_ils/modules/local_entities/mappings/v7/local_entities/local_entity-v0.0.1.json create mode 100644 rero_ils/modules/local_entities/models.py create mode 100644 rero_ils/modules/local_entities/permissions.py create mode 100644 rero_ils/modules/unified_entities/__init__.py create mode 100644 rero_ils/modules/unified_entities/fetchers.py create mode 100644 rero_ils/modules/unified_entities/minters.py create mode 100644 rero_ils/modules/unified_entities/serializers/__init__.py create mode 100644 rero_ils/modules/unified_entities/serializers/base.py create mode 100644 tests/api/local_entities/test_local_entities_permissions.py create mode 100644 tests/api/local_entities/test_local_entities_rest.py create mode 100644 tests/api/unified_entities/test_unified_entities_rest.py create mode 100644 tests/api/unified_entities/test_unified_entities_search.py diff --git a/data/local_entities.json b/data/local_entities.json new file mode 100644 index 0000000000..0411ec2c85 --- /dev/null +++ b/data/local_entities.json @@ -0,0 +1,27 @@ +[ + { + "pid": "1", + "preferred_name": "Shakespeare, William", + "type": "bf:Person" + }, + { + "pid": "2", + "preferred_name": "Eiffel, Gustave", + "type": "bf:Person" + }, + { + "pid": "3", + "preferred_name": "Jobs, Steve", + "type": "bf:Person" + }, + { + "pid": "4", + "preferred_name": "Dylan, Bob", + "type": "bf:Person" + }, + { + "pid": "5", + "preferred_name": "UCLouvain", + "type": "bf:Organisation" + } +] diff --git a/data/role_policies.json b/data/role_policies.json index 3eb2555852..fbba7ceb92 100644 --- a/data/role_policies.json +++ b/data/role_policies.json @@ -783,7 +783,8 @@ "pro_circulation_manager", "pro_user_manager", "pro_acquisition_manager", - "pro_library_administrator" + "pro_library_administrator", + "pro_entity_manager" ], "access-circulation": [ "pro_circulation_manager", @@ -797,5 +798,27 @@ "permission-management": [ "admin", "pro_full_permissions" + ], + "locent-access": [ + "pro_full_permissions", + "pro_read_only", + "pro_catalog_manager", + "pro_circulation_manager", + "pro_user_manager", + "pro_acquisition_manager", + "pro_library_administrator", + "pro_entity_manager" + ], + "locent-update": [ + "pro_full_permissions", + "pro_entity_manager" + ], + "locent-create": [ + "pro_full_permissions", + "pro_entity_manager" + ], + "locent-delete": [ + "pro_full_permissions", + "pro_entity_manager" ] } diff --git a/data/system_role_policies.json b/data/system_role_policies.json index 48f7cd1262..a2dddc7a1b 100644 --- a/data/system_role_policies.json +++ b/data/system_role_policies.json @@ -55,5 +55,11 @@ ], "ptre-read": [ "authenticated_user" + ], + "locent-search": [ + "any_user" + ], + "locent-read": [ + "any_user" ] } diff --git a/pyproject.toml b/pyproject.toml index 26c7d73075..0a4cb592c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -199,6 +199,9 @@ templates = "rero_ils.modules.templates.views:blueprint" theme = "rero_ils.theme.views:blueprint" users = "rero_ils.modules.users.views:blueprint" +[tool.poetry.plugins."invenio_base.api_converters"] +nooppid = "rero_ils.converters:NoopPIDConverter" + [tool.poetry.plugins."invenio_celery.tasks"] apiharvester = "rero_ils.modules.apiharvester.tasks" collections = "rero_ils.modules.collections.tasks" @@ -239,6 +242,7 @@ libraries = "rero_ils.modules.libraries.models" local_fields = "rero_ils.modules.local_fields.models" locations = "rero_ils.modules.locations.models" entities = "rero_ils.modules.entities.models" +local_entities = "rero_ils.modules.local_entities.models" notifications = "rero_ils.modules.notifications.models" organisations = "rero_ils.modules.organisations.models" patron_transaction_events = "rero_ils.modules.patron_transaction_events.models" @@ -273,6 +277,7 @@ item_types = "rero_ils.modules.item_types.jsonschemas" items = "rero_ils.modules.items.jsonschemas" libraries = "rero_ils.modules.libraries.jsonschemas" loans = "rero_ils.modules.loans.jsonschemas" +local_entities = "rero_ils.modules.local_entities.jsonschemas" local_fields = "rero_ils.modules.local_fields.jsonschemas" locations = "rero_ils.modules.locations.jsonschemas" notifications = "rero_ils.modules.notifications.jsonschemas" @@ -305,13 +310,14 @@ acq_receipt_line_id = "rero_ils.modules.acquisition.acq_receipt_lines.api:acq_re budget_id = "rero_ils.modules.acquisition.budgets.api:budget_id_fetcher" circ_policy_id = "rero_ils.modules.circ_policies.api:circ_policy_id_fetcher" collection_id = "rero_ils.modules.collections.api:collection_id_fetcher" -entity_id = "rero_ils.modules.entities.api:entity_id_fetcher" document_id = "rero_ils.modules.documents.api:document_id_fetcher" +entity_id = "rero_ils.modules.entities.api:entity_id_fetcher" holding_id = "rero_ils.modules.holdings.api:holding_id_fetcher" ill_request_id = "rero_ils.modules.ill_requests.api:ill_request_id_fetcher" item_id = "rero_ils.modules.items.api:item_id_fetcher" item_type_id = "rero_ils.modules.item_types.api:item_type_id_fetcher" library_id = "rero_ils.modules.libraries.api:library_id_fetcher" +local_entity_id = "rero_ils.modules.local_entities.api:local_entity_id_fetcher" local_field_id = "rero_ils.modules.local_fields.api:local_field_id_fetcher" location_id = "rero_ils.modules.locations.api:location_id_fetcher" notification_id = "rero_ils.modules.notifications.api:notification_id_fetcher" @@ -323,6 +329,7 @@ patron_transaction_id = "rero_ils.modules.patron_transactions.api:patron_transac patron_type_id = "rero_ils.modules.patron_types.api:patron_type_id_fetcher" stat_id = "rero_ils.modules.stats.api:stat_id_fetcher" template_id = "rero_ils.modules.templates.api:template_id_fetcher" +unified_entity_id = "rero_ils.modules.unified_entities.fetchers:id_fetcher" vendor_id = "rero_ils.modules.vendors.api:vendor_id_fetcher" [tool.poetry.plugins."invenio_pidstore.minters"] @@ -335,13 +342,14 @@ acq_receipt_line_id = "rero_ils.modules.acquisition.acq_receipt_lines.api:acq_re budget_id = "rero_ils.modules.acquisition.budgets.api:budget_id_minter" circ_policy_id = "rero_ils.modules.circ_policies.api:circ_policy_id_minter" collection_id = "rero_ils.modules.collections.api:collection_id_minter" -entity_id = "rero_ils.modules.entities.api:entity_id_minter" document_id = "rero_ils.modules.documents.api:document_id_minter" +entity_id = "rero_ils.modules.entities.api:entity_id_minter" holding_id = "rero_ils.modules.holdings.api:holding_id_minter" ill_request_id = "rero_ils.modules.ill_requests.api:ill_request_id_minter" item_id = "rero_ils.modules.items.api:item_id_minter" item_type_id = "rero_ils.modules.item_types.api:item_type_id_minter" library_id = "rero_ils.modules.libraries.api:library_id_minter" +local_entity_id = "rero_ils.modules.local_entities.api:local_entity_id_minter" local_field_id = "rero_ils.modules.local_fields.api:local_field_id_minter" location_id = "rero_ils.modules.locations.api:location_id_minter" notification_id = "rero_ils.modules.notifications.api:notification_id_minter" @@ -352,6 +360,7 @@ patron_transaction_id = "rero_ils.modules.patron_transactions.api:patron_transac patron_type_id = "rero_ils.modules.patron_types.api:patron_type_id_minter" stat_id = "rero_ils.modules.stats.api:stat_id_minter" template_id = "rero_ils.modules.templates.api:template_id_minter" +unified_entity_id = "rero_ils.modules.unified_entities.minters:id_minter" vendor_id = "rero_ils.modules.vendors.api:vendor_id_minter" [tool.poetry.plugins."invenio_records.jsonresolver"] @@ -399,6 +408,7 @@ item_types = "rero_ils.modules.item_types.mappings" items = "rero_ils.modules.items.mappings" libraries = "rero_ils.modules.libraries.mappings" loans = "rero_ils.modules.loans.mappings" +local_entities = "rero_ils.modules.local_entities.mappings" local_fields = "rero_ils.modules.local_fields.mappings" locations = "rero_ils.modules.locations.mappings" notifications = "rero_ils.modules.notifications.mappings" diff --git a/rero_ils/config.py b/rero_ils/config.py index 4b4092e093..de229a92f6 100644 --- a/rero_ils/config.py +++ b/rero_ils/config.py @@ -40,6 +40,7 @@ ItemOnLoanToItemReturned, PendingToItemAtDesk, \ PendingToItemInTransitPickup, ToCancelled, ToItemOnLoan from invenio_records_rest.facets import range_filter, terms_filter +from invenio_records_rest.utils import deny_all, allow_all from rero_ils.modules.acquisition.acq_accounts.api import AcqAccount from rero_ils.modules.acquisition.acq_accounts.permissions import \ @@ -96,6 +97,8 @@ get_extension_params, is_item_available_for_checkout, \ loan_build_document_ref, loan_build_item_ref, loan_build_patron_ref, \ validate_item_pickup_transaction_locations, validate_loan_duration +from .modules.local_entities.api import LocalEntity +from .modules.local_entities.permissions import LocalEntityPermissionPolicy from .modules.local_fields.api import LocalField from .modules.local_fields.permissions import LocalFieldPermissionPolicy from .modules.locations.api import Location @@ -1144,6 +1147,66 @@ def _(x): update_permission_factory_imp=lambda record: EntityPermissionPolicy('update', record=record), delete_permission_factory_imp=lambda record: EntityPermissionPolicy('delete', record=record) ), + locent=dict( + pid_type='locent', + pid_minter='local_entity_id', + pid_fetcher='local_entity_id', + search_class='rero_ils.modules.local_entities.api:LocalEntitiesSearch', + search_index='local_entities', + indexer_class='rero_ils.modules.local_entities.api:LocalEntitiesIndexer', + search_type=None, + record_serializers={ + 'application/json': 'rero_ils.modules.serializers:json_v1_response' + }, + record_serializers_aliases={ + 'json': 'application/json', + }, + search_serializers={ + 'application/json': 'rero_ils.modules.serializers:json_v1_search' + }, + search_serializers_aliases={ + 'json': 'application/json' + }, + list_route='/local_entities/', + record_loaders={ + 'application/json': lambda: LocalEntity(request.get_json()), + }, + record_class='rero_ils.modules.local_entities.api:LocalEntity', + item_route='/local_entities/', + default_media_type='application/json', + max_result_window=MAX_RESULT_WINDOW, + search_factory_imp='rero_ils.query:search_factory', + list_permission_factory_imp=lambda record: LocalEntityPermissionPolicy('search', record=record), + read_permission_factory_imp=lambda record: LocalEntityPermissionPolicy('read', record=record), + create_permission_factory_imp=lambda record: LocalEntityPermissionPolicy('create', record=record), + update_permission_factory_imp=lambda record: LocalEntityPermissionPolicy('update', record=record), + delete_permission_factory_imp=lambda record: LocalEntityPermissionPolicy('delete', record=record) + ), + unient=dict( + pid_type='unient', + pid_minter='unified_entity_id', # This is mandatory for invenio-records-rest but not used + pid_fetcher='unified_entity_id', + search_index='unified_entities', + record_serializers={ + 'application/json': 'rero_ils.modules.serializers:json_v1_response' + }, # This is mandatory for invenio-records-rest but not used + search_serializers={ + 'application/json': 'rero_ils.modules.unified_entities.serializers:json_entities_search' + }, + search_serializers_aliases={ + 'json': 'application/json' + }, + list_route='/unified_entities/', + item_route='/unified_entities/', # mandatory for invenio-records-rest (only used for permissions) + default_media_type='application/json', + max_result_window=MAX_RESULT_WINDOW, + search_factory_imp='rero_ils.query:search_factory', + list_permission_factory_imp=allow_all, + read_permission_factory_imp=deny_all, + create_permission_factory_imp=deny_all, + update_permission_factory_imp=deny_all, + delete_permission_factory_imp=deny_all + ), cipo=dict( pid_type='cipo', pid_minter='circ_policy_id', @@ -2192,15 +2255,16 @@ def _(x): 'budgets', 'circ_policies', 'collections', - 'entities', 'documents', + 'entities', 'holdings', 'items', 'item_types', 'ill_requests', 'libraries', - 'local_fields', 'loans', + 'local_entities', + 'local_fields', 'locations', 'notifications', 'operation_logs', @@ -2210,6 +2274,7 @@ def _(x): 'patron_transactions', 'patron_types', 'templates', + 'unified_entities', 'vendors' ] @@ -2623,6 +2688,12 @@ def _(x): 'rero_ils.modules.local_fields.permissions:create_action', 'rero_ils.modules.local_fields.permissions:update_action', 'rero_ils.modules.local_fields.permissions:delete_action', + 'rero_ils.modules.local_entities.permissions:access_action', + 'rero_ils.modules.local_entities.permissions:search_action', + 'rero_ils.modules.local_entities.permissions:read_action', + 'rero_ils.modules.local_entities.permissions:create_action', + 'rero_ils.modules.local_entities.permissions:update_action', + 'rero_ils.modules.local_entities.permissions:delete_action', 'rero_ils.modules.notifications.permissions:access_action', 'rero_ils.modules.notifications.permissions:search_action', 'rero_ils.modules.notifications.permissions:read_action', @@ -2754,6 +2825,7 @@ def _(x): 'itty': '/item_types/item_type-v0.0.1.json', 'lib': '/libraries/library-v0.0.1.json', 'loc': '/locations/location-v0.0.1.json', + 'locent': '/local_entities/local_entity-v0.0.1.json', 'lofi': '/local_fields/local_field-v0.0.1.json', 'notif': '/notifications/notification-v0.0.1.json', 'org': '/organisations/organisation-v0.0.1.json', diff --git a/rero_ils/converters.py b/rero_ils/converters.py new file mode 100644 index 0000000000..847d51cfac --- /dev/null +++ b/rero_ils/converters.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""General converters utilities module.""" +from invenio_records_rest.utils import PIDConverter +from werkzeug.utils import cached_property + + +class NoopPIDValue(object): + """Noop PID resolver class.""" + + def __init__(self, value): + """Initialize with the PID value. + + :params value: PID value. + :type value: str + """ + self.value = value + + @cached_property + def data(self): + """Resolve PID from a value. + + :returns: A tuple with the PID and an empty record. + """ + return self.value, {} + + +class NoopPIDConverter(PIDConverter): + """Converter for PID values in the route mapping. + + This class is a custom routing converter defining the 'PID' type. + See http://werkzeug.pocoo.org/docs/0.12/routing/#custom-converters. + + Use ``pid`` as a type in the route pattern, e.g.: the use of + route decorator: ``@blueprint.route('/record/')``, + will match and resolve a path: ``/record/123456``. + """ + + def to_python(self, value): + """Resolve PID value.""" + return NoopPIDValue(value) diff --git a/rero_ils/es_templates/v7/record.json b/rero_ils/es_templates/v7/record.json index 2bab1e2674..3d3da29fa0 100644 --- a/rero_ils/es_templates/v7/record.json +++ b/rero_ils/es_templates/v7/record.json @@ -18,6 +18,7 @@ "libraries-*", "loans-*", "loans-*", + "local_entities-*", "local_fields-*", "locations-*", "notifications-*", diff --git a/rero_ils/modules/commons/dumpers.py b/rero_ils/modules/commons/dumpers.py index 0c09049233..f0a248b4b0 100644 --- a/rero_ils/modules/commons/dumpers.py +++ b/rero_ils/modules/commons/dumpers.py @@ -63,7 +63,7 @@ def load(self, data, record_cls): class ReplaceRefsDumper(InvenioRecordsDumper): - """Replace link data in document.""" + """Replace link data in resource.""" def dump(self, record, data): """Dump record data by replacing `$ref` links. diff --git a/rero_ils/modules/entities/mappings/v7/entities/entity-v0.0.1.json b/rero_ils/modules/entities/mappings/v7/entities/entity-v0.0.1.json index d54de204b5..df045f6783 100644 --- a/rero_ils/modules/entities/mappings/v7/entities/entity-v0.0.1.json +++ b/rero_ils/modules/entities/mappings/v7/entities/entity-v0.0.1.json @@ -1,4 +1,7 @@ { + "aliases": { + "unified_entities": {} + }, "settings": { "analysis": { "analyzer": { diff --git a/rero_ils/modules/local_entities/__init__.py b/rero_ils/modules/local_entities/__init__.py new file mode 100644 index 0000000000..49158d4a7a --- /dev/null +++ b/rero_ils/modules/local_entities/__init__.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Local entities Records.""" diff --git a/rero_ils/modules/local_entities/api.py b/rero_ils/modules/local_entities/api.py new file mode 100644 index 0000000000..6ec6a610c2 --- /dev/null +++ b/rero_ils/modules/local_entities/api.py @@ -0,0 +1,117 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""API for manipulating local entities.""" + +from functools import partial + +from rero_ils.modules.api import IlsRecord, IlsRecordsIndexer, IlsRecordsSearch +from rero_ils.modules.documents.api import DocumentsSearch +from rero_ils.modules.utils import sorted_pids +from rero_ils.modules.fetchers import id_fetcher +from rero_ils.modules.minters import id_minter +from rero_ils.modules.providers import Provider + +from .dumpers import replace_refs_dumper +from .extensions import AuthorizedAccessPointExtension +from .models import LocalEntityIdentifier, LocalEntityMetadata + +# provider +LocalEntityProvider = type( + 'LocalEntityProvider', + (Provider,), + dict(identifier=LocalEntityIdentifier, pid_type='locent') +) +# minter +local_entity_id_minter = partial(id_minter, provider=LocalEntityProvider) +# fetcher +local_entity_id_fetcher = partial(id_fetcher, provider=LocalEntityProvider) + + +class LocalEntitiesSearch(IlsRecordsSearch): + """Local entities search.""" + + class Meta: + """Meta class.""" + + index = 'local_entities' + doc_types = None + fields = ('*', ) + facets = {} + + default_filter = None + + +class LocalEntity(IlsRecord): + """Local entity class.""" + + minter = local_entity_id_minter + fetcher = local_entity_id_fetcher + provider = LocalEntityProvider + model_cls = LocalEntityMetadata + # disable legacy replace refs + enable_jsonref = False + + _extensions = [ + AuthorizedAccessPointExtension() + ] + + def resolve(self): + """Resolve references data. + + Uses the dumper to do the job. + Mainly used by the `resolve=1` URL parameter. + + :returns: a fresh copy of the resolved data. + """ + return self.dumps(replace_refs_dumper) + + def get_links_to_me(self, get_pids=False): + """Record links. + + :param get_pids: if True list of linked pids + if False count of linked records + """ + document_query = DocumentsSearch() \ + .filter('term', local_entity__pid=self.pid) + documents = sorted_pids(document_query) if get_pids \ + else document_query.count() + links = { + 'documents': documents + } + return {k: v for k, v in links.items() if v} + + def reasons_not_to_delete(self): + """Get reasons not to delete record.""" + cannot_delete = {} + if links := self.get_links_to_me(): + cannot_delete['links'] = links + return cannot_delete + + +class LocalEntitiesIndexer(IlsRecordsIndexer): + """Local entity indexing class.""" + + record_cls = LocalEntity + + def bulk_index(self, record_id_iterator): + """Bulk index records. + + :param record_id_iterator: Iterator yielding record UUIDs. + """ + super().bulk_index(record_id_iterator, doc_type='locent') diff --git a/rero_ils/modules/local_entities/dumpers/__init__.py b/rero_ils/modules/local_entities/dumpers/__init__.py new file mode 100644 index 0000000000..1ad9efdb9a --- /dev/null +++ b/rero_ils/modules/local_entities/dumpers/__init__.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2022 RERO +# Copyright (C) 2019-2022 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Local entity dumpers.""" + +from invenio_records.dumpers import Dumper + +from rero_ils.modules.commons.dumpers import MultiDumper, ReplaceRefsDumper + +from .indexer import LocalEntityIndexerDumper + +# replace linked data (seems not necessary at this time) +replace_refs_dumper = MultiDumper(dumpers=[ + # make a fresh copy + Dumper(), + ReplaceRefsDumper() +]) + +# dumper used for indexing +indexer_dumper = MultiDumper(dumpers=[ + # make a fresh copy + Dumper(), + ReplaceRefsDumper(), + LocalEntityIndexerDumper() +]) diff --git a/rero_ils/modules/local_entities/dumpers/indexer.py b/rero_ils/modules/local_entities/dumpers/indexer.py new file mode 100644 index 0000000000..977d95c07b --- /dev/null +++ b/rero_ils/modules/local_entities/dumpers/indexer.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2022 RERO +# Copyright (C) 2019-2022 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Indexing dumper.""" + +from invenio_records.dumpers import Dumper + + +class LocalEntityIndexerDumper(Dumper): + """Local entity indexer.""" + + def dump(self, record, data): + """Dump a local entity instance. + + :param record: The record to dump. + :param data: The initial dump data passed in by ``record.dumps()``. + """ + return data diff --git a/rero_ils/modules/local_entities/extensions/__init__.py b/rero_ils/modules/local_entities/extensions/__init__.py new file mode 100644 index 0000000000..cfbc8264ed --- /dev/null +++ b/rero_ils/modules/local_entities/extensions/__init__.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2022 RERO +# Copyright (C) 2022 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Document record extensions.""" +from .authorized_access_point import AuthorizedAccessPointExtension + +__all__ = [ + 'AuthorizedAccessPointExtension' +] diff --git a/rero_ils/modules/local_entities/extensions/authorized_access_point.py b/rero_ils/modules/local_entities/extensions/authorized_access_point.py new file mode 100644 index 0000000000..623cde3183 --- /dev/null +++ b/rero_ils/modules/local_entities/extensions/authorized_access_point.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2021 RERO +# Copyright (C) 2021 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Document record extension to add the MEF pid in the database.""" + + +from invenio_records.extensions import RecordExtension + + +class AuthorizedAccessPointExtension(RecordExtension): + """Adds the authorized access point.""" + + def _generate_authorized_access_point(self, record, data): + """Generate authorized access point. + + :params record: dict - a document record. + """ + data.update({'authorized_access_point': record.get('preferred_name')}) + + def pre_dump(self, record, data, dumper=None): + """Called before a record is dumped. + + :param record: the record to dump + :param data: the data to dump. + :param dumper: the dumper class used to dump the record. + """ + return self._generate_authorized_access_point(record, data) diff --git a/rero_ils/modules/local_entities/jsonschemas/__init__.py b/rero_ils/modules/local_entities/jsonschemas/__init__.py new file mode 100644 index 0000000000..1e838f9c6b --- /dev/null +++ b/rero_ils/modules/local_entities/jsonschemas/__init__.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""JSON schemas for rero-ils.""" diff --git a/rero_ils/modules/local_entities/jsonschemas/local_entities/local_entity-v0.0.1.json b/rero_ils/modules/local_entities/jsonschemas/local_entities/local_entity-v0.0.1.json new file mode 100644 index 0000000000..048f7d5ba7 --- /dev/null +++ b/rero_ils/modules/local_entities/jsonschemas/local_entities/local_entity-v0.0.1.json @@ -0,0 +1,45 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "title": "Local entity", + "description": "JSON schema for a local entity", + "additionalProperties": false, + "required": [ + "$schema", + "pid", + "preferred_name", + "type" + ], + "properties": { + "$schema": { + "title": "Schema", + "description": "Schema to validate local entity records against.", + "type": "string", + "minLength": 9 + }, + "pid": { + "title": "Local entity PID", + "type": "string" + }, + "preferred_name": { + "title": "Preferred name", + "type": "string", + "minLength": 1 + }, + "type": { + "title": "Agent type", + "type": "string", + "enum": [ + "bf:Organisation", + "bf:Person", + "bf:Place", + "bf:Topic", + "bf:Work" + ] + }, + "deleted": { + "title": "Deletion date", + "type": "string" + } + } +} diff --git a/rero_ils/modules/local_entities/mappings/__init__.py b/rero_ils/modules/local_entities/mappings/__init__.py new file mode 100644 index 0000000000..9f61e5023a --- /dev/null +++ b/rero_ils/modules/local_entities/mappings/__init__.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Elasticsearch mappings.""" diff --git a/rero_ils/modules/local_entities/mappings/v7/__init__.py b/rero_ils/modules/local_entities/mappings/v7/__init__.py new file mode 100644 index 0000000000..9f61e5023a --- /dev/null +++ b/rero_ils/modules/local_entities/mappings/v7/__init__.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Elasticsearch mappings.""" diff --git a/rero_ils/modules/local_entities/mappings/v7/local_entities/local_entity-v0.0.1.json b/rero_ils/modules/local_entities/mappings/v7/local_entities/local_entity-v0.0.1.json new file mode 100644 index 0000000000..1c86a1d1ea --- /dev/null +++ b/rero_ils/modules/local_entities/mappings/v7/local_entities/local_entity-v0.0.1.json @@ -0,0 +1,76 @@ +{ + "aliases": { + "unified_entities": {} + }, + "settings": { + "analysis": { + "analyzer": { + "autocomplete": { + "type": "custom", + "tokenizer": "standard", + "filter": [ + "lowercase", + "icu_normalizer", + "icu_folding", + "edge_ngram_filter" + ] + }, + "search_autocomplete": { + "type": "custom", + "tokenizer": "standard", + "filter": [ + "lowercase", + "icu_normalizer", + "icu_folding" + ] + } + } + } + }, + "mappings": { + "date_detection": false, + "numeric_detection": false, + "properties": { + "autocomplete_name": { + "type": "text", + "analyzer": "autocomplete", + "search_analyzer": "search_autocomplete" + }, + "authorized_access_point_sort": { + "type": "keyword", + "normalizer": "sort_normalizer" + }, + "$schema": { + "type": "keyword" + }, + "pid": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "preferred_name": { + "type": "text" + }, + "authorized_access_point": { + "type": "text", + "copy_to": [ + "autocomplete_name", + "authorized_access_point_sort" + ] + }, + "document_organisation_pids": { + "type": "keyword" + }, + "deleted": { + "type": "date" + }, + "_created": { + "type": "date" + }, + "_updated": { + "type": "date" + } + } + } +} diff --git a/rero_ils/modules/local_entities/models.py b/rero_ils/modules/local_entities/models.py new file mode 100644 index 0000000000..b7bc833bbf --- /dev/null +++ b/rero_ils/modules/local_entities/models.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Define relation between records and buckets.""" + +from invenio_db import db +from invenio_pidstore.models import RecordIdentifier +from invenio_records.models import RecordMetadataBase + + +class LocalEntityIdentifier(RecordIdentifier): + """Sequence generator for `Entity` identifiers.""" + + __tablename__ = 'local_entity_id' + __mapper_args__ = {'concrete': True} + + recid = db.Column( + db.BigInteger().with_variant(db.Integer, 'sqlite'), + primary_key=True, + autoincrement=True, + ) + + +class LocalEntityMetadata(db.Model, RecordMetadataBase): + """Entity record metadata.""" + + __tablename__ = 'local_entity_metadata' + + +class LocalEntityType: + """Class holding all available entity types.""" + + AGENT = 'bf:Agent' + ORGANISATION = 'bf:Organisation' + PERSON = 'bf:Person' + PLACE = 'bf:Place' + TEMPORAL = 'bf:Temporal' + TOPIC = 'bf:Topic' + WORK = 'bf:Work' diff --git a/rero_ils/modules/local_entities/permissions.py b/rero_ils/modules/local_entities/permissions.py new file mode 100644 index 0000000000..1a10524e94 --- /dev/null +++ b/rero_ils/modules/local_entities/permissions.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Permissions for `Local Entity` records.""" +from invenio_access import action_factory + +from rero_ils.modules.permissions import RecordPermissionPolicy, \ + AllowedByAction + +# Actions to control local entity policies for CRUD operations +search_action = action_factory('locent-search') +read_action = action_factory('locent-read') +create_action = action_factory('locent-create') +update_action = action_factory('locent-update') +delete_action = action_factory('locent-delete') +access_action = action_factory('locent-access') + + +class LocalEntityPermissionPolicy(RecordPermissionPolicy): + """Local entity Permission Policy used by the CRUD operations. + + Only search and read is allowed for all users. + Other operations are denied far anybody. + """ + + can_search = [AllowedByAction(search_action)] + can_read = [AllowedByAction(read_action)] + can_create = [AllowedByAction(create_action)] + can_update = [AllowedByAction(update_action)] + can_delete = [AllowedByAction(delete_action)] diff --git a/rero_ils/modules/patrons/jsonschemas/patrons/patron-v0.0.1.json b/rero_ils/modules/patrons/jsonschemas/patrons/patron-v0.0.1.json index 7837f60346..5e6dac49bc 100644 --- a/rero_ils/modules/patrons/jsonschemas/patrons/patron-v0.0.1.json +++ b/rero_ils/modules/patrons/jsonschemas/patrons/patron-v0.0.1.json @@ -438,7 +438,8 @@ "pro_circulation_manager", "pro_user_manager", "pro_acquisition_manager", - "pro_library_administrator" + "pro_library_administrator", + "pro_entity_manager" ] }, "form": { @@ -476,6 +477,10 @@ { "label": "pro_library_administrator", "value": "pro_library_administrator" + }, + { + "label": "pro_entity_manager", + "value": "pro_entity_manager" } ], "wrappers": [ diff --git a/rero_ils/modules/unified_entities/__init__.py b/rero_ils/modules/unified_entities/__init__.py new file mode 100644 index 0000000000..6afed31d9b --- /dev/null +++ b/rero_ils/modules/unified_entities/__init__.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Unified entities Records.""" diff --git a/rero_ils/modules/unified_entities/fetchers.py b/rero_ils/modules/unified_entities/fetchers.py new file mode 100644 index 0000000000..b22ed76a77 --- /dev/null +++ b/rero_ils/modules/unified_entities/fetchers.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Persistent identifier fetchers.""" + + +from __future__ import absolute_import, print_function + +from collections import namedtuple + +from rero_ils.modules.utils import get_pid_type_from_schema + +FetchedPID = namedtuple('FetchedPID', ['pid_type', 'pid_value']) +"""A pid fetcher.""" + + +def id_fetcher(record_uuid, data): + """Fetch a record's identifier. + + :param record_uuid: The record UUID. + :param data: The record metadata. + :return: A :data:`rero_ils.modules.fetchers.FetchedPID` instance. + """ + pid_type = 'unient' + # try to extract pid type from schema + if schema := data.get('$schema'): + pid_type = get_pid_type_from_schema(schema) + return FetchedPID( + pid_type=pid_type, + pid_value=data['pid'] + ) diff --git a/rero_ils/modules/unified_entities/minters.py b/rero_ils/modules/unified_entities/minters.py new file mode 100644 index 0000000000..a867883389 --- /dev/null +++ b/rero_ils/modules/unified_entities/minters.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Persistent identifier minters.""" +from collections import namedtuple + +EntityMinter = namedtuple( + 'EntityMinter', + ['pid_type', 'pid_value', 'object_uuid', 'object_type']) + + +def id_minter(record_uuid, data, provider, pid_key='pid', object_type='rec'): + """RERO ILS dummy minter.""" + # DEV NOTES: + # A minter is required for invenio-records-rest. + # This return a dummy PersistentIdentifier + return EntityMinter( + pid_type=object_type, + pid_value=data['pid'], + object_uuid=record_uuid, + object_type=object_type + ) diff --git a/rero_ils/modules/unified_entities/serializers/__init__.py b/rero_ils/modules/unified_entities/serializers/__init__.py new file mode 100644 index 0000000000..14c3a11f45 --- /dev/null +++ b/rero_ils/modules/unified_entities/serializers/__init__.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""RERO Unified entities serialization.""" + +from rero_ils.modules.serializers import RecordSchemaJSONV1, \ + search_responsify +from .base import EntityJSONSerializer + +# Serializers +# =========== +_json = EntityJSONSerializer(RecordSchemaJSONV1) + +# Records-REST serializers +# ======================== +json_entities_search = search_responsify(_json, 'application/json') diff --git a/rero_ils/modules/unified_entities/serializers/base.py b/rero_ils/modules/unified_entities/serializers/base.py new file mode 100644 index 0000000000..ce61610412 --- /dev/null +++ b/rero_ils/modules/unified_entities/serializers/base.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2022 RERO +# Copyright (C) 2019-2022 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""RERO Entity JSON serialization.""" + +from rero_ils.modules.serializers import JSONSerializer + + +class EntityJSONSerializer(JSONSerializer): + """Serializer for RERO-ILS `Document` records as JSON.""" + + def _postprocess_search_links(self, search_results, pid_fetcher) -> None: + """Post-process search links. + + :param search_results: Elasticsearch search result. + :param pid_fetcher: Persistent identifier fetcher related to records + into the search result. + """ + # DEV NOTES : + # We need to override this method to remove the `create` link from + # search results. + # See `rero_ils.modules.serializers.mixins.PostprocessorMixin` diff --git a/rero_ils/modules/users/models.py b/rero_ils/modules/users/models.py index 7c3bc7b39b..03ffd68c4e 100644 --- a/rero_ils/modules/users/models.py +++ b/rero_ils/modules/users/models.py @@ -30,11 +30,12 @@ class UserRole: CIRCULATION_MANAGER = 'pro_circulation_manager' LIBRARY_ADMINISTRATOR = 'pro_library_administrator' USER_MANAGER = 'pro_user_manager' + PRO_ENTITY_MANAGER = 'pro_entity_manager' LIBRARIAN_ROLES = [ PROFESSIONAL_READ_ONLY, ACQUISITION_MANAGER, CATALOG_MANAGER, CIRCULATION_MANAGER, - LIBRARY_ADMINISTRATOR, USER_MANAGER + LIBRARY_ADMINISTRATOR, USER_MANAGER, PRO_ENTITY_MANAGER ] PROFESSIONAL_ROLES = [FULL_PERMISSIONS] + LIBRARIAN_ROLES diff --git a/rero_ils/modules/utils.py b/rero_ils/modules/utils.py index 55fc5ac2f6..a39e4a1bb7 100644 --- a/rero_ils/modules/utils.py +++ b/rero_ils/modules/utils.py @@ -542,19 +542,31 @@ def get_record_class_from_schema_or_pid_type(schema=None, pid_type=None): :return: the record class. """ if schema: - try: - pid_type_schema_value = schema.split('schemas')[1] - schemas = current_app.config.get('RERO_ILS_DEFAULT_JSON_SCHEMA') - pid_type = [key for key, value in schemas.items() - if value == pid_type_schema_value][0] - except IndexError: - pass + pid_type = get_pid_type_from_schema(schema) return obj_or_import_string( current_app.config .get('RECORDS_REST_ENDPOINTS') .get(pid_type, {}).get('record_class')) +def get_pid_type_from_schema(schema): + """Get the pid_type from a given schema or a pid type. + + If both the schema and pid_type are given, the record_class of the + schema will be returned. + + :param schema: record schema. + :return: the pid type. + """ + try: + pid_type_schema_value = schema.split('schemas')[1] + schemas = current_app.config.get('RERO_ILS_DEFAULT_JSON_SCHEMA') + return [key for key, value in schemas.items() + if value == pid_type_schema_value][0] + except IndexError: + pass + + def get_patron_from_arguments(**kwargs): """Try to load a patron from potential arguments.""" from .patrons.api import Patron diff --git a/scripts/setup b/scripts/setup index 3b99d08912..9c38e22e4a 100755 --- a/scripts/setup +++ b/scripts/setup @@ -246,6 +246,7 @@ eval ${PREFIX} "invenio roles create -d 'Professional: Catalog manager' pro_cata eval ${PREFIX} "invenio roles create -d 'Professional: Circulation manager' pro_circulation_manager" eval ${PREFIX} "invenio roles create -d 'Professional: Library administrator' pro_library_administrator" eval ${PREFIX} "invenio roles create -d 'Professional: User manager' pro_user_manager" +eval ${PREFIX} "invenio roles create -d 'Professional: Entity manager' pro_entity_manager" eval ${PREFIX} "invenio roles create -d 'Documentation Editor' editor" eval ${PREFIX} "invenio roles create -d 'Document Importing' document_importer" @@ -328,6 +329,11 @@ info_msg "- ILL requests: ${DATA_PATH}/ill_request.json" eval ${PREFIX} invenio reroils fixtures create_ill_requests -f ${DATA_PATH}/ill_requests.json eval ${PREFIX} invenio reroils index reindex -t illr --yes-i-know +info_msg "- Local entities ${DATA_PATH}/local_entities.json ${CREATE_LAZY} ${DONT_STOP}" +eval ${PREFIX} invenio reroils fixtures create --pid_type locent --schema 'https://bib.rero.ch/schemas/local_entities/local_entity-v0.0.1.json' ${DATA_PATH}/local_entities.json --append ${CREATE_LAZY} ${DONT_STOP} +eval ${PREFIX} invenio reroils index reindex -t locent --yes-i-know +eval ${PREFIX} invenio reroils index run --raise-on-error + #: - xml to json transformation for rero marcxml -------------------------------- #: don't forget to recreat the documents json schema files: # SIZE=big # SIZE=small diff --git a/setup.py b/setup.py index df1df89194..e691c29c05 100644 --- a/setup.py +++ b/setup.py @@ -88,6 +88,9 @@ def run(self): 'rero-ils = rero_ils.modules.ext:REROILSAPP', 'invenio_i18n = invenio_i18n:InvenioI18N' ], + 'invenio_base.api_converters': [ + 'dummypid = rero_ils.converters:NoopPIDConverter', + ], 'invenio_base.blueprints': [ 'collections = rero_ils.modules.collections.views:blueprint', 'entities = rero_ils.modules.entities.views:blueprint', @@ -177,6 +180,7 @@ def run(self): 'item_types = rero_ils.modules.item_types.models', 'items = rero_ils.modules.items.models', 'libraries = rero_ils.modules.libraries.models', + 'local_entity = rero_ils.modules.local_entities.models', 'local_fields = rero_ils.modules.local_fields.models', 'locations = rero_ils.modules.locations.models', 'entity = rero_ils.modules.entities.models', @@ -208,6 +212,7 @@ def run(self): 'item_id = rero_ils.modules.items.api:item_id_minter', 'item_type_id = rero_ils.modules.item_types.api:item_type_id_minter', 'library_id = rero_ils.modules.libraries.api:library_id_minter', + 'local_entity_id = rero_ils.modules.local_entities.api:local_entity_id_minter', 'local_field_id = rero_ils.modules.local_fields.api:local_field_id_minter', 'location_id = rero_ils.modules.locations.api:location_id_minter', 'notification_id = rero_ils.modules.notifications.api:notification_id_minter', @@ -232,6 +237,7 @@ def run(self): 'collection_id = rero_ils.modules.collections.api:collection_id_fetcher', 'document_id = rero_ils.modules.documents.api:document_id_fetcher', 'entity_id = rero_ils.modules.entities.api:entity_id_fetcher', + 'local_entity_id = rero_ils.modules.local_entities.api:local_entity_id_fetcher', 'holding_id = rero_ils.modules.holdings.api:holding_id_fetcher', 'ill_request_id = rero_ils.modules.ill_requests.api:ill_request_id_fetcher', 'item_id = rero_ils.modules.items.api:item_id_fetcher', @@ -269,6 +275,7 @@ def run(self): 'items = rero_ils.modules.items.jsonschemas', 'libraries = rero_ils.modules.libraries.jsonschemas', 'loans = rero_ils.modules.loans.jsonschemas', + 'local_entities = rero_ils.modules.local_entities.jsonschemas', 'local_fields = rero_ils.modules.local_fields.jsonschemas', 'locations = rero_ils.modules.locations.jsonschemas', 'notifications = rero_ils.modules.notifications.jsonschemas', @@ -301,6 +308,7 @@ def run(self): 'items = rero_ils.modules.items.mappings', 'libraries = rero_ils.modules.libraries.mappings', 'loans = rero_ils.modules.loans.mappings', + 'local_entities = rero_ils.modules.local_entities.mappings', 'local_fields = rero_ils.modules.local_fields.mappings', 'locations = rero_ils.modules.locations.mappings', 'notifications = rero_ils.modules.notifications.mappings', diff --git a/tests/api/local_entities/test_local_entities_permissions.py b/tests/api/local_entities/test_local_entities_permissions.py new file mode 100644 index 0000000000..be41a455e0 --- /dev/null +++ b/tests/api/local_entities/test_local_entities_permissions.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from flask import current_app +from flask_principal import AnonymousIdentity, identity_changed +from flask_security.utils import login_user +from utils import check_permission + +from rero_ils.modules.local_entities.permissions import \ + LocalEntityPermissionPolicy + + +def test_local_entity_permissions(patron_martigny, + librarian_martigny, + librarian2_martigny, + system_librarian_martigny, + local_entity_person): + """Test entity permissions class.""" + permission_policy = LocalEntityPermissionPolicy + + # Anonymous user + # - Allow search/read actions on any local entity + # - Deny create/update/delete actions on any local entity + identity_changed.send( + current_app._get_current_object(), identity=AnonymousIdentity() + ) + check_permission(permission_policy, { + 'search': True, + 'read': True, + 'create': False, + 'update': False, + 'delete': False + }, {}) + # Patron user + # - Allow search/read actions on any local entity + # - Deny create/update/delete actions on any local entity + login_user(patron_martigny.user) + check_permission(permission_policy, { + 'search': True, + 'read': True, + 'create': False, + 'update': False, + 'delete': False + }, local_entity_person) + # As staff member without `pro_entity_manager` role : + # - Allow search/read actions on any local entity + # - Deny create/update/delete actions on any local entity + login_user(librarian2_martigny.user) + check_permission(permission_policy, { + 'search': True, + 'read': True, + 'create': False, + 'update': False, + 'delete': False + }, local_entity_person) + # As staff member with `pro_entity_manager` role : + # - Allow search/read actions on any local entity + # - Allow create/update/delete actions on any local entity + login_user(librarian_martigny.user) + check_permission(permission_policy, { + 'search': True, + 'read': True, + 'create': True, + 'update': True, + 'delete': True + }, local_entity_person) + # Full permission user + # - Allow search/read actions on any local entity + # - Allow create/update/delete actions on any local entity + login_user(system_librarian_martigny.user) + check_permission(permission_policy, { + 'search': True, + 'read': True, + 'create': True, + 'update': True, + 'delete': True + }, local_entity_person) diff --git a/tests/api/local_entities/test_local_entities_rest.py b/tests/api/local_entities/test_local_entities_rest.py new file mode 100644 index 0000000000..4b8a56073d --- /dev/null +++ b/tests/api/local_entities/test_local_entities_rest.py @@ -0,0 +1,144 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Tests `LocalEntity` resource REST API.""" + +import json +import mock +from flask import url_for +from utils import get_json, postdata, to_relative_url, \ + VerifyRecordPermissionPatch + +from rero_ils.modules.entities.models import EntityType +from rero_ils.modules.local_entities.api import LocalEntity + + +def test_local_entities_permissions(client, roles, local_entity_person, + json_header): + """Test record retrieval.""" + item_url = url_for('invenio_records_rest.locent_item', + pid_value='locent_pers') + res = client.get(item_url) + assert res.status_code == 200 + + res, _ = postdata(client, 'invenio_records_rest.locent_list', {}) + assert res.status_code == 401 + + client.put( + url_for('invenio_records_rest.ent_item', pid_value='locent_pers'), + data={}, + headers=json_header + ) + assert res.status_code == 401 + + res = client.delete(item_url) + assert res.status_code == 401 + + +@mock.patch('invenio_records_rest.views.verify_record_permission', + mock.MagicMock(return_value=VerifyRecordPermissionPatch)) +def test_local_entities_get(client, local_entity_person): + """Test record retrieval.""" + item_url = url_for('invenio_records_rest.locent_item', + pid_value='locent_pers') + + res = client.get(item_url) + assert res.status_code == 200 + assert res.headers['ETag'] == f'"{local_entity_person.revision_id}"' + + data = get_json(res) + assert local_entity_person.dumps() == data['metadata'] + + # Check metadata + for k in ['created', 'updated', 'metadata', 'links']: + assert k in data + + # Check self links + res = client.get(to_relative_url(data['links']['self'])) + assert res.status_code == 200 + assert data == get_json(res) + + assert local_entity_person.dumps() == data['metadata'] + + list_url = url_for('invenio_records_rest.locent_list', pid='locent_pers') + res = client.get(list_url) + assert res.status_code == 200 + data = get_json(res) + entity_person = local_entity_person.replace_refs() + entity_person['type'] = EntityType.PERSON + assert data['hits']['hits'][0]['metadata'] == entity_person.dumps() + + +@mock.patch('invenio_records_rest.views.verify_record_permission', + mock.MagicMock(return_value=VerifyRecordPermissionPatch)) +def test_local_entities_post_put_delete(client, local_entity_person_data, + json_header): + """Test record api post, put and delete.""" + item_url = url_for('invenio_records_rest.locent_item', pid_value='1') + list_url = url_for('invenio_records_rest.locent_list', q='pid:1') + local_entity_data = local_entity_person_data + # Create record / POST + local_entity_data['pid'] = '1' + res, data = postdata( + client, + 'invenio_records_rest.locent_list', + local_entity_data + ) + assert res.status_code == 201 + local_entity = LocalEntity.get_record_by_pid(data['metadata']['pid']) + # Check that the returned record matches the given data + assert local_entity.dumps() == data['metadata'] + + res = client.get(item_url) + assert res.status_code == 200 + data = get_json(res) + + assert local_entity.dumps() == data['metadata'] + + # Update record/PUT + data = local_entity_data + data['preferred_name'] = 'Test Name' + res = client.put( + item_url, + data=json.dumps(data), + headers=json_header + ) + assert res.status_code == 200 + + # Check that the returned record matches the given data + data = get_json(res) + assert data['metadata']['preferred_name'] == 'Test Name' + + res = client.get(item_url) + assert res.status_code == 200 + + data = get_json(res) + assert data['metadata']['preferred_name'] == 'Test Name' + + res = client.get(list_url) + assert res.status_code == 200 + + data = get_json(res)['hits']['hits'][0] + assert data['metadata']['preferred_name'] == 'Test Name' + + # Delete record/DELETE + res = client.delete(item_url) + assert res.status_code == 204 + + res = client.get(item_url) + assert res.status_code == 410 diff --git a/tests/api/test_monitoring_rest.py b/tests/api/test_monitoring_rest.py index 5c26059465..ad0d26b1d9 100644 --- a/tests/api/test_monitoring_rest.py +++ b/tests/api/test_monitoring_rest.py @@ -56,6 +56,8 @@ def test_monitoring_es_db_counts(client): 'lib': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'libraries'}, 'loanid': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'loans'}, 'loc': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'locations'}, + 'locent': {'db': 0, 'db-es': 0, 'es': 0, + 'index': 'local_entities'}, 'lofi': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'local_fields'}, 'notif': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'notifications'}, 'oplg': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'operation_logs'}, @@ -68,6 +70,8 @@ def test_monitoring_es_db_counts(client): 'ptty': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'patron_types'}, 'stat': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'stats'}, 'tmpl': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'templates'}, + 'unient': {'db': 0, 'db-es': 0, 'es': 0, + 'index': 'unified_entities'}, 'vndr': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'vendors'}, } } diff --git a/tests/api/unified_entities/test_unified_entities_rest.py b/tests/api/unified_entities/test_unified_entities_rest.py new file mode 100644 index 0000000000..e76e4c5a26 --- /dev/null +++ b/tests/api/unified_entities/test_unified_entities_rest.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Tests `LocalEntity` resource REST API.""" + +from flask import url_for +from utils import get_json, postdata + + +def test_unified_entities_permissions(client, entity_person, + local_entity_person, json_header): + """Test record retrieval.""" + item_url = url_for('invenio_records_rest.unient_item', + pid_value='locent_pers') + res = client.get(item_url) + assert res.status_code == 401 + + item_url = url_for('invenio_records_rest.unient_item', + pid_value='ent_pers') + res = client.get(item_url) + assert res.status_code == 401 + + res = client.get(url_for('invenio_records_rest.unient_list')) + assert res.status_code == 200 + + res, _ = postdata(client, 'invenio_records_rest.unient_list', {}) + assert res.status_code == 401 + + client.put( + url_for('invenio_records_rest.unient_item', pid_value='unient_pers'), + data={}, + headers=json_header + ) + + res = client.delete(item_url) + assert res.status_code == 401 + + +def test_unified_entities_get(client, entity_person, local_entity_person): + """Test record retrieval.""" + item_url = url_for('invenio_records_rest.unient_item', + pid_value='locent_pers') + res = client.get(item_url) + assert res.status_code == 401 + + item_url = url_for('invenio_records_rest.unient_item', + pid_value='ent_pers') + res = client.get(item_url) + assert res.status_code == 401 + + res = client.get(url_for('invenio_records_rest.unient_list')) + assert res.status_code == 200 + + # Check remote/local entities self links + data = get_json(res) + pid_link_map = { + 'ent_pers': 'http://localhost/entities/ent_pers', + 'locent_pers': 'http://localhost/local_entities/locent_pers' + } + for hit in data['hits']['hits']: + assert hit['links']['self'] == pid_link_map.get(hit['id']) + + # search entity record + list_url = url_for('invenio_records_rest.unient_list', pid='ent_pers') + res = client.get(list_url) + assert res.status_code == 200 + + # search local entity record + list_url = url_for('invenio_records_rest.unient_list', pid='locent_pers') + res = client.get(list_url) + assert res.status_code == 200 diff --git a/tests/api/unified_entities/test_unified_entities_search.py b/tests/api/unified_entities/test_unified_entities_search.py new file mode 100644 index 0000000000..7bd9fe1302 --- /dev/null +++ b/tests/api/unified_entities/test_unified_entities_search.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019 RERO +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Search tests.""" +import mock +from flask import url_for +from utils import VerifyRecordPermissionPatch, get_json + + +@mock.patch('invenio_records_rest.views.verify_record_permission', + mock.MagicMock(return_value=VerifyRecordPermissionPatch)) +def test_unified_entity_search(client, entity_person, local_entity_person, + entity_organisation): + """Test unified entity search queries.""" + + # unified entity search + list_url = url_for( + 'invenio_records_rest.unient_list', + q='"Loy, Georg"', + simple='1' + ) + res = client.get(list_url) + hits = get_json(res)['hits'] + assert hits['total']['value'] == 2 + + # unified entity search organisation + list_url = url_for( + 'invenio_records_rest.unient_list', + q='"Convegno internazionale di italianistica Craiova"', + simple='1' + ) + res = client.get(list_url) + hits = get_json(res)['hits'] + assert hits['total']['value'] == 1 + + # empty search + list_url = url_for( + 'invenio_records_rest.unient_list', + q='"Nebehay, Christian Michael"', + simple='1' + ) + res = client.get(list_url) + hits = get_json(res)['hits'] + assert hits['total']['value'] == 0 diff --git a/tests/data/data.json b/tests/data/data.json index 2e280a7463..809b57d15f 100644 --- a/tests/data/data.json +++ b/tests/data/data.json @@ -1843,6 +1843,18 @@ ], "viaf_pid": "108158722" }, + "locent_pers": { + "$schema": "https://bib.rero.ch/schemas/local_entities/local_entity-v0.0.1.json", + "pid": "locent_pers", + "preferred_name": "Loy, Georg", + "type": "bf:Person" + }, + "locent_org": { + "$schema": "https://bib.rero.ch/schemas/local_entities/local_entity-v0.0.1.json", + "pid": "locent_org", + "preferred_name": "Convegno internazionale di Italianistica", + "type": "bf:Organisation" + }, "doc1": { "$schema": "https://bib.rero.ch/schemas/documents/document-v0.0.1.json", "pid": "doc1", @@ -4110,7 +4122,8 @@ "pro_circulation_manager", "pro_user_manager", "pro_acquisition_manager", - "pro_library_administrator" + "pro_library_administrator", + "pro_entity_manager" ], "password": "Pw123456" }, diff --git a/tests/data/policies/role_policies.json b/tests/data/policies/role_policies.json index 6c51cf886f..2b54a1a7c9 100644 --- a/tests/data/policies/role_policies.json +++ b/tests/data/policies/role_policies.json @@ -667,5 +667,17 @@ ], "permission-management": [ "pro_full_permissions" + ], + "locent-update": [ + "pro_full_permissions", + "pro_entity_manager" + ], + "locent-create": [ + "pro_full_permissions", + "pro_entity_manager" + ], + "locent-delete": [ + "pro_full_permissions", + "pro_entity_manager" ] } diff --git a/tests/data/policies/system_role_policies.json b/tests/data/policies/system_role_policies.json index 2db3b0af3d..8a3ab44dc2 100644 --- a/tests/data/policies/system_role_policies.json +++ b/tests/data/policies/system_role_policies.json @@ -49,5 +49,11 @@ ], "ptre-read": [ "authenticated_user" + ], + "locent-search": [ + "any_user" + ], + "locent-read": [ + "any_user" ] } diff --git a/tests/fixtures/metadata.py b/tests/fixtures/metadata.py index 85931eaa08..ec73a0176f 100644 --- a/tests/fixtures/metadata.py +++ b/tests/fixtures/metadata.py @@ -30,6 +30,8 @@ from rero_ils.modules.entities.api import EntitiesSearch, Entity from rero_ils.modules.holdings.api import Holding, HoldingsSearch from rero_ils.modules.items.api import Item, ItemsSearch +from rero_ils.modules.local_entities.api import LocalEntitiesSearch, \ + LocalEntity from rero_ils.modules.local_fields.api import LocalField, LocalFieldsSearch from rero_ils.modules.operation_logs.api import OperationLog from rero_ils.modules.templates.api import Template, TemplatesSearch @@ -456,6 +458,42 @@ def person2(app, person2_data): return pers +@pytest.fixture(scope="module") +def local_entity_person_data(data): + """Load mef contribution person data.""" + return deepcopy(data.get('locent_pers')) + + +@pytest.fixture(scope="module") +def local_entity_org_data(data): + """Load mef contribution person data.""" + return deepcopy(data.get('locent_org')) + + +@pytest.fixture(scope="module") +def local_entity_person(app, local_entity_person_data): + """Create mef person record.""" + pers = LocalEntity.create( + data=local_entity_person_data, + delete_pid=False, + dbcommit=True, + reindex=True) + flush_index(LocalEntitiesSearch.Meta.index) + return pers + + +@pytest.fixture(scope="module") +def local_entity_org(app, local_entity_org_data): + """Create mef person record.""" + org = LocalEntity.create( + data=local_entity_org_data, + delete_pid=False, + dbcommit=True, + reindex=True) + flush_index(LocalEntitiesSearch.Meta.index) + return org + + @pytest.fixture(scope="module") @mock.patch('requests.Session.get') def document_ref(mock_contributions_mef_get, From 2cc0722864b4d5d02ff8550c073a95dd8e0eff7f Mon Sep 17 00:00:00 2001 From: Lauren-D Date: Tue, 6 Jun 2023 20:11:40 +0200 Subject: [PATCH 02/14] entities: refactoring of resources * Renames `entities` to `remote_entities`. * Changes remote_entities pid_type from `ent` to `rement`. * Renames `unified_entities` to `entities`. * Groups local_entities and remote_entities under entities. Co-Authored-by: Lauren-D --- ...ties_big.json => remote_entities_big.json} | 0 ..._small.json => remote_entities_small.json} | 0 pyproject.toml | 30 ++-- rero_ils/config.py | 86 +++++------ rero_ils/es_templates/v7/record.json | 2 +- rero_ils/modules/cli/reroils.py | 2 +- rero_ils/modules/cli/utils.py | 4 +- rero_ils/modules/documents/api.py | 9 +- .../dojson/contrib/jsontodc/model.py | 3 +- .../dojson/contrib/jsontomarc21/model.py | 9 +- .../modules/documents/dumpers/replace_refs.py | 17 ++- .../documents/extensions/add_mef_pid.py | 10 +- .../extensions/provision_activities.py | 2 +- .../modules/documents/serializers/base.py | 2 +- .../modules/documents/serializers/marc.py | 4 +- rero_ils/modules/documents/utils.py | 2 +- rero_ils/modules/documents/views.py | 9 +- rero_ils/modules/entities/__init__.py | 5 +- .../fetchers.py | 2 +- .../{ => entities}/local_entities/__init__.py | 0 .../{ => entities}/local_entities/api.py | 0 .../local_entities/dumpers/__init__.py | 0 .../local_entities/dumpers/indexer.py | 0 .../local_entities/extensions/__init__.py | 0 .../extensions/authorized_access_point.py | 0 .../local_entities/jsonschemas/__init__.py | 0 .../local_entities/local_entity-v0.0.1.json | 0 .../local_entities/mappings/__init__.py | 0 .../local_entities/mappings/v7/__init__.py | 0 .../local_entities/local_entity-v0.0.1.json | 2 +- .../{ => entities}/local_entities/models.py | 12 -- .../local_entities/permissions.py | 0 .../{unified_entities => entities}/minters.py | 0 rero_ils/modules/entities/models.py | 35 +---- .../remote_entities}/__init__.py | 5 +- .../entities/{ => remote_entities}/api.py | 35 ++--- .../entities/{ => remote_entities}/cli.py | 0 .../jsonschemas/__init__.py | 0 .../remote_entity-v0.0.1.json} | 0 .../{ => remote_entities}/listener.py | 12 +- .../mappings/__init__.py | 0 .../mappings/v7/__init__.py | 0 .../remote_entity-v0.0.1.json} | 2 +- .../entities/remote_entities/models.py | 52 +++++++ .../{ => remote_entities}/permissions.py | 2 +- .../entities/{ => remote_entities}/proxy.py | 0 .../entities/{ => remote_entities}/sync.py | 28 ++-- .../entities/{ => remote_entities}/tasks.py | 4 +- .../templates/rero_ils/_entity_by_source.html | 0 .../rero_ils/_entity_by_source_data.html | 0 .../templates/rero_ils/_entity_unified.html | 0 .../rero_ils/detailed_view_entity.html | 0 .../templates/rero_ils/macros/entity.html | 0 .../entities/{ => remote_entities}/utils.py | 0 .../entities/{ => remote_entities}/views.py | 26 ++-- .../serializers/__init__.py | 0 .../serializers/base.py | 0 rero_ils/modules/ext.py | 5 +- rero_ils/modules/indexer_utils.py | 4 +- rero_ils/query.py | 2 +- scripts/setup | 22 +-- setup.py | 20 +-- tests/api/entities/test_entities_rest.py | 134 +++++------------ .../test_entities_search.py} | 6 +- .../test_local_entities_permissions.py | 2 +- .../test_local_entities_rest.py | 4 +- .../test_remote_entities_permissions.py} | 21 +-- .../test_remote_entities_rest.py | 141 ++++++++++++++++++ tests/api/test_monitoring_rest.py | 25 ++-- .../test_unified_entities_rest.py | 86 ----------- tests/data/data.json | 6 +- tests/data/mef.json | 2 +- tests/fixtures/mef.py | 9 +- tests/fixtures/metadata.py | 21 +-- tests/ui/documents/test_documents_api.py | 32 ++-- .../test_entities_api.py | 72 ++++----- .../test_entities_filter.py | 6 +- .../test_entities_mapping.py | 19 +-- .../test_entities_ui.py | 8 +- .../test_entities_utils.py | 3 +- tests/ui/test_indexer_utils.py | 2 +- tests/unit/conftest.py | 31 ++-- .../documents/test_documents_dojson_marc21.py | 3 +- tests/unit/test_contributions_jsonschema.py | 8 +- 84 files changed, 563 insertions(+), 544 deletions(-) rename data/{entities_big.json => remote_entities_big.json} (100%) rename data/{entities_small.json => remote_entities_small.json} (100%) rename rero_ils/modules/{unified_entities => entities}/fetchers.py (98%) rename rero_ils/modules/{ => entities}/local_entities/__init__.py (100%) rename rero_ils/modules/{ => entities}/local_entities/api.py (100%) rename rero_ils/modules/{ => entities}/local_entities/dumpers/__init__.py (100%) rename rero_ils/modules/{ => entities}/local_entities/dumpers/indexer.py (100%) rename rero_ils/modules/{ => entities}/local_entities/extensions/__init__.py (100%) rename rero_ils/modules/{ => entities}/local_entities/extensions/authorized_access_point.py (100%) rename rero_ils/modules/{ => entities}/local_entities/jsonschemas/__init__.py (100%) rename rero_ils/modules/{ => entities}/local_entities/jsonschemas/local_entities/local_entity-v0.0.1.json (100%) rename rero_ils/modules/{ => entities}/local_entities/mappings/__init__.py (100%) rename rero_ils/modules/{ => entities}/local_entities/mappings/v7/__init__.py (100%) rename rero_ils/modules/{ => entities}/local_entities/mappings/v7/local_entities/local_entity-v0.0.1.json (98%) rename rero_ils/modules/{ => entities}/local_entities/models.py (84%) rename rero_ils/modules/{ => entities}/local_entities/permissions.py (100%) rename rero_ils/modules/{unified_entities => entities}/minters.py (100%) rename rero_ils/modules/{unified_entities => entities/remote_entities}/__init__.py (86%) rename rero_ils/modules/entities/{ => remote_entities}/api.py (92%) rename rero_ils/modules/entities/{ => remote_entities}/cli.py (100%) rename rero_ils/modules/entities/{ => remote_entities}/jsonschemas/__init__.py (100%) rename rero_ils/modules/entities/{jsonschemas/entities/entity-v0.0.1.json => remote_entities/jsonschemas/remote_entities/remote_entity-v0.0.1.json} (100%) rename rero_ils/modules/entities/{ => remote_entities}/listener.py (73%) rename rero_ils/modules/entities/{ => remote_entities}/mappings/__init__.py (100%) rename rero_ils/modules/entities/{ => remote_entities}/mappings/v7/__init__.py (100%) rename rero_ils/modules/entities/{mappings/v7/entities/entity-v0.0.1.json => remote_entities/mappings/v7/remote_entities/remote_entity-v0.0.1.json} (99%) create mode 100644 rero_ils/modules/entities/remote_entities/models.py rename rero_ils/modules/entities/{ => remote_entities}/permissions.py (94%) rename rero_ils/modules/entities/{ => remote_entities}/proxy.py (100%) rename rero_ils/modules/entities/{ => remote_entities}/sync.py (96%) rename rero_ils/modules/entities/{ => remote_entities}/tasks.py (97%) rename rero_ils/modules/entities/{ => remote_entities}/templates/rero_ils/_entity_by_source.html (100%) rename rero_ils/modules/entities/{ => remote_entities}/templates/rero_ils/_entity_by_source_data.html (100%) rename rero_ils/modules/entities/{ => remote_entities}/templates/rero_ils/_entity_unified.html (100%) rename rero_ils/modules/entities/{ => remote_entities}/templates/rero_ils/detailed_view_entity.html (100%) rename rero_ils/modules/entities/{ => remote_entities}/templates/rero_ils/macros/entity.html (100%) rename rero_ils/modules/entities/{ => remote_entities}/utils.py (100%) rename rero_ils/modules/entities/{ => remote_entities}/views.py (90%) rename rero_ils/modules/{unified_entities => entities}/serializers/__init__.py (100%) rename rero_ils/modules/{unified_entities => entities}/serializers/base.py (100%) rename tests/api/{unified_entities/test_unified_entities_search.py => entities/test_entities_search.py} (93%) rename tests/api/{entities/test_entities_permissions.py => remote_entities/test_remote_entities_permissions.py} (83%) create mode 100644 tests/api/remote_entities/test_remote_entities_rest.py delete mode 100644 tests/api/unified_entities/test_unified_entities_rest.py rename tests/ui/{entities => remote_entities}/test_entities_api.py (87%) rename tests/ui/{entities => remote_entities}/test_entities_filter.py (94%) rename tests/ui/{entities => remote_entities}/test_entities_mapping.py (81%) rename tests/ui/{entities => remote_entities}/test_entities_ui.py (82%) rename tests/ui/{entities => remote_entities}/test_entities_utils.py (94%) diff --git a/data/entities_big.json b/data/remote_entities_big.json similarity index 100% rename from data/entities_big.json rename to data/remote_entities_big.json diff --git a/data/entities_small.json b/data/remote_entities_small.json similarity index 100% rename from data/entities_small.json rename to data/remote_entities_small.json diff --git a/pyproject.toml b/pyproject.toml index 0a4cb592c1..352c23dcf9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -163,7 +163,7 @@ acq_orders = "rero_ils.modules.acquisition.acq_orders.views:api_blueprint" acq_receipts = "rero_ils.modules.acquisition.acq_receipts.views:api_blueprint" api_documents = "rero_ils.modules.documents.views:api_blueprint" circ_policies = "rero_ils.modules.circ_policies.views:blueprint" -entities = "rero_ils.modules.entities.views:api_blueprint" +remote_entities = "rero_ils.modules.entities.remote_entities.views:api_blueprint" documents = "rero_ils.modules.documents.views:api_blueprint" holdings = "rero_ils.modules.holdings.api_views:api_blueprint" imports = "rero_ils.modules.imports.views:api_blueprint" @@ -185,7 +185,7 @@ rero-ils = "rero_ils.modules.ext:REROILSAPP" [tool.poetry.plugins."invenio_base.blueprints"] circ_policies = "rero_ils.modules.circ_policies.views:blueprint" collections = "rero_ils.modules.collections.views:blueprint" -entities = "rero_ils.modules.entities.views:blueprint" +remote_entities = "rero_ils.modules.entities.remote_entities.views:blueprint" documents = "rero_ils.modules.documents.views:blueprint" holdings = "rero_ils.modules.holdings.views:blueprint" ill_requests = "rero_ils.modules.ill_requests.views:blueprint" @@ -206,7 +206,7 @@ nooppid = "rero_ils.converters:NoopPIDConverter" apiharvester = "rero_ils.modules.apiharvester.tasks" collections = "rero_ils.modules.collections.tasks" documents = "rero_ils.modules.documents.tasks" -entities = "rero_ils.modules.entities.tasks" +remote_entities = "rero_ils.modules.entities.remote_entities.tasks" ebooks = "rero_ils.modules.ebooks.tasks" holdings = "rero_ils.modules.holdings.tasks" items = "rero_ils.modules.items.tasks" @@ -241,8 +241,8 @@ items = "rero_ils.modules.items.models" libraries = "rero_ils.modules.libraries.models" local_fields = "rero_ils.modules.local_fields.models" locations = "rero_ils.modules.locations.models" -entities = "rero_ils.modules.entities.models" -local_entities = "rero_ils.modules.local_entities.models" +remote_entities = "rero_ils.modules.entities.remote_entities.models" +local_entities = "rero_ils.modules.entities.local_entities.models" notifications = "rero_ils.modules.notifications.models" organisations = "rero_ils.modules.organisations.models" patron_transaction_events = "rero_ils.modules.patron_transaction_events.models" @@ -269,7 +269,7 @@ budgets = "rero_ils.modules.acquisition.budgets.jsonschemas" circ_policies = "rero_ils.modules.circ_policies.jsonschemas" collections = "rero_ils.modules.collections.jsonschemas" common = "rero_ils.jsonschemas" -entities = "rero_ils.modules.entities.jsonschemas" +remote_entities = "rero_ils.modules.entities.remote_entities.jsonschemas" documents = "rero_ils.modules.documents.jsonschemas" holdings = "rero_ils.modules.holdings.jsonschemas" ill_requests = "rero_ils.modules.ill_requests.jsonschemas" @@ -277,7 +277,7 @@ item_types = "rero_ils.modules.item_types.jsonschemas" items = "rero_ils.modules.items.jsonschemas" libraries = "rero_ils.modules.libraries.jsonschemas" loans = "rero_ils.modules.loans.jsonschemas" -local_entities = "rero_ils.modules.local_entities.jsonschemas" +local_entities = "rero_ils.modules.entities.local_entities.jsonschemas" local_fields = "rero_ils.modules.local_fields.jsonschemas" locations = "rero_ils.modules.locations.jsonschemas" notifications = "rero_ils.modules.notifications.jsonschemas" @@ -311,13 +311,13 @@ budget_id = "rero_ils.modules.acquisition.budgets.api:budget_id_fetcher" circ_policy_id = "rero_ils.modules.circ_policies.api:circ_policy_id_fetcher" collection_id = "rero_ils.modules.collections.api:collection_id_fetcher" document_id = "rero_ils.modules.documents.api:document_id_fetcher" -entity_id = "rero_ils.modules.entities.api:entity_id_fetcher" +remote_entity_id = "rero_ils.modules.entities.remote_entities.api:remote_entity_id_fetcher" holding_id = "rero_ils.modules.holdings.api:holding_id_fetcher" ill_request_id = "rero_ils.modules.ill_requests.api:ill_request_id_fetcher" item_id = "rero_ils.modules.items.api:item_id_fetcher" item_type_id = "rero_ils.modules.item_types.api:item_type_id_fetcher" library_id = "rero_ils.modules.libraries.api:library_id_fetcher" -local_entity_id = "rero_ils.modules.local_entities.api:local_entity_id_fetcher" +local_entity_id = "rero_ils.modules.entities.local_entities.api:local_entity_id_fetcher" local_field_id = "rero_ils.modules.local_fields.api:local_field_id_fetcher" location_id = "rero_ils.modules.locations.api:location_id_fetcher" notification_id = "rero_ils.modules.notifications.api:notification_id_fetcher" @@ -329,7 +329,7 @@ patron_transaction_id = "rero_ils.modules.patron_transactions.api:patron_transac patron_type_id = "rero_ils.modules.patron_types.api:patron_type_id_fetcher" stat_id = "rero_ils.modules.stats.api:stat_id_fetcher" template_id = "rero_ils.modules.templates.api:template_id_fetcher" -unified_entity_id = "rero_ils.modules.unified_entities.fetchers:id_fetcher" +entity_id = "rero_ils.modules.entities.fetchers:id_fetcher" vendor_id = "rero_ils.modules.vendors.api:vendor_id_fetcher" [tool.poetry.plugins."invenio_pidstore.minters"] @@ -343,13 +343,13 @@ budget_id = "rero_ils.modules.acquisition.budgets.api:budget_id_minter" circ_policy_id = "rero_ils.modules.circ_policies.api:circ_policy_id_minter" collection_id = "rero_ils.modules.collections.api:collection_id_minter" document_id = "rero_ils.modules.documents.api:document_id_minter" -entity_id = "rero_ils.modules.entities.api:entity_id_minter" +remote_entity_id = "rero_ils.modules.entities.remote_entities.api:remote_entity_id_minter" holding_id = "rero_ils.modules.holdings.api:holding_id_minter" ill_request_id = "rero_ils.modules.ill_requests.api:ill_request_id_minter" item_id = "rero_ils.modules.items.api:item_id_minter" item_type_id = "rero_ils.modules.item_types.api:item_type_id_minter" library_id = "rero_ils.modules.libraries.api:library_id_minter" -local_entity_id = "rero_ils.modules.local_entities.api:local_entity_id_minter" +local_entity_id = "rero_ils.modules.entities.local_entities.api:local_entity_id_minter" local_field_id = "rero_ils.modules.local_fields.api:local_field_id_minter" location_id = "rero_ils.modules.locations.api:location_id_minter" notification_id = "rero_ils.modules.notifications.api:notification_id_minter" @@ -360,7 +360,7 @@ patron_transaction_id = "rero_ils.modules.patron_transactions.api:patron_transac patron_type_id = "rero_ils.modules.patron_types.api:patron_type_id_minter" stat_id = "rero_ils.modules.stats.api:stat_id_minter" template_id = "rero_ils.modules.templates.api:template_id_minter" -unified_entity_id = "rero_ils.modules.unified_entities.minters:id_minter" +entity_id = "rero_ils.modules.entities.minters:id_minter" vendor_id = "rero_ils.modules.vendors.api:vendor_id_minter" [tool.poetry.plugins."invenio_records.jsonresolver"] @@ -400,7 +400,7 @@ acq_receipts = "rero_ils.modules.acquisition.acq_receipts.mappings" budgets = "rero_ils.modules.acquisition.budgets.mappings" circ_policies = "rero_ils.modules.circ_policies.mappings" collections = "rero_ils.modules.collections.mappings" -entities = "rero_ils.modules.entities.mappings" +remote_entities = "rero_ils.modules.entities.remote_entities.mappings" documents = "rero_ils.modules.documents.mappings" holdings = "rero_ils.modules.holdings.mappings" ill_requests = "rero_ils.modules.ill_requests.mappings" @@ -408,7 +408,7 @@ item_types = "rero_ils.modules.item_types.mappings" items = "rero_ils.modules.items.mappings" libraries = "rero_ils.modules.libraries.mappings" loans = "rero_ils.modules.loans.mappings" -local_entities = "rero_ils.modules.local_entities.mappings" +local_entities = "rero_ils.modules.entities.local_entities.mappings" local_fields = "rero_ils.modules.local_fields.mappings" locations = "rero_ils.modules.locations.mappings" notifications = "rero_ils.modules.notifications.mappings" diff --git a/rero_ils/config.py b/rero_ils/config.py index de229a92f6..7f4493710b 100644 --- a/rero_ils/config.py +++ b/rero_ils/config.py @@ -73,8 +73,9 @@ from .modules.documents.permissions import DocumentPermissionPolicy from .modules.documents.query import acquisition_filter, \ nested_identified_filter -from .modules.entities.api import Entity -from .modules.entities.permissions import EntityPermissionPolicy +from rero_ils.modules.entities.remote_entities.api import RemoteEntity +from rero_ils.modules.entities.remote_entities.permissions import \ + RemoteEntityPermissionPolicy from .modules.holdings.api import Holding from .modules.holdings.models import HoldingCirculationAction from .modules.holdings.permissions import HoldingsPermissionPolicy @@ -97,8 +98,9 @@ get_extension_params, is_item_available_for_checkout, \ loan_build_document_ref, loan_build_item_ref, loan_build_patron_ref, \ validate_item_pickup_transaction_locations, validate_loan_duration -from .modules.local_entities.api import LocalEntity -from .modules.local_entities.permissions import LocalEntityPermissionPolicy +from rero_ils.modules.entities.local_entities.api import LocalEntity +from rero_ils.modules.entities.local_entities.permissions import \ + LocalEntityPermissionPolicy from .modules.local_fields.api import LocalField from .modules.local_fields.permissions import LocalFieldPermissionPolicy from .modules.locations.api import Location @@ -425,7 +427,7 @@ def _(x): 'enabled': False, }, 'sync-entities': { - 'task': 'rero_ils.modules.entities.tasks.sync_entities', + 'task': 'rero_ils.modules.entities.remote_entities.tasks.sync_entities', 'schedule': crontab(minute=0, hour=1), # Every day at 01:00 UTC, 'enabled': False, }, @@ -1112,13 +1114,13 @@ def _(x): update_permission_factory_imp=lambda record: LocationPermissionPolicy('update', record=record), delete_permission_factory_imp=lambda record: LocationPermissionPolicy('delete', record=record) ), - ent=dict( - pid_type='ent', - pid_minter='entity_id', - pid_fetcher='entity_id', - search_class='rero_ils.modules.entities.api:EntitiesSearch', - search_index='entities', - indexer_class='rero_ils.modules.entities.api:EntitiesIndexer', + rement=dict( + pid_type='rement', + pid_minter='remote_entity_id', + pid_fetcher='remote_entity_id', + search_class='rero_ils.modules.entities.remote_entities.api:RemoteEntitiesSearch', + search_index='remote_entities', + indexer_class='rero_ils.modules.entities.remote_entities.api:RemoteEntitiesIndexer', search_type=None, record_serializers={ 'application/json': 'rero_ils.modules.serializers:json_v1_response' @@ -1132,28 +1134,28 @@ def _(x): search_serializers_aliases={ 'json': 'application/json' }, - list_route='/entities/', + list_route='/remote_entities/', record_loaders={ - 'application/json': lambda: Entity(request.get_json()), + 'application/json': lambda: RemoteEntity(request.get_json()), }, - record_class='rero_ils.modules.entities.api:Entity', - item_route='/entities/', + record_class='rero_ils.modules.entities.remote_entities.api:RemoteEntity', + item_route='/remote_entities/', default_media_type='application/json', max_result_window=MAX_RESULT_WINDOW, - search_factory_imp='rero_ils.query:entity_view_search_factory', - list_permission_factory_imp=lambda record: EntityPermissionPolicy('search', record=record), - read_permission_factory_imp=lambda record: EntityPermissionPolicy('read', record=record), - create_permission_factory_imp=lambda record: EntityPermissionPolicy('create', record=record), - update_permission_factory_imp=lambda record: EntityPermissionPolicy('update', record=record), - delete_permission_factory_imp=lambda record: EntityPermissionPolicy('delete', record=record) + search_factory_imp='rero_ils.query:remote_entity_view_search_factory', + list_permission_factory_imp=lambda record: RemoteEntityPermissionPolicy('search', record=record), + read_permission_factory_imp=lambda record: RemoteEntityPermissionPolicy('read', record=record), + create_permission_factory_imp=lambda record: RemoteEntityPermissionPolicy('create', record=record), + update_permission_factory_imp=lambda record: RemoteEntityPermissionPolicy('update', record=record), + delete_permission_factory_imp=lambda record: RemoteEntityPermissionPolicy('delete', record=record) ), locent=dict( pid_type='locent', pid_minter='local_entity_id', pid_fetcher='local_entity_id', - search_class='rero_ils.modules.local_entities.api:LocalEntitiesSearch', + search_class='rero_ils.modules.entities.local_entities.api:LocalEntitiesSearch', search_index='local_entities', - indexer_class='rero_ils.modules.local_entities.api:LocalEntitiesIndexer', + indexer_class='rero_ils.modules.entities.local_entities.api:LocalEntitiesIndexer', search_type=None, record_serializers={ 'application/json': 'rero_ils.modules.serializers:json_v1_response' @@ -1171,8 +1173,8 @@ def _(x): record_loaders={ 'application/json': lambda: LocalEntity(request.get_json()), }, - record_class='rero_ils.modules.local_entities.api:LocalEntity', - item_route='/local_entities/', + record_class='rero_ils.modules.entities.local_entities.api:LocalEntity', + item_route='/local_entities/', default_media_type='application/json', max_result_window=MAX_RESULT_WINDOW, search_factory_imp='rero_ils.query:search_factory', @@ -1182,22 +1184,22 @@ def _(x): update_permission_factory_imp=lambda record: LocalEntityPermissionPolicy('update', record=record), delete_permission_factory_imp=lambda record: LocalEntityPermissionPolicy('delete', record=record) ), - unient=dict( - pid_type='unient', - pid_minter='unified_entity_id', # This is mandatory for invenio-records-rest but not used - pid_fetcher='unified_entity_id', - search_index='unified_entities', + ent=dict( + pid_type='ent', + pid_minter='entity_id', # This is mandatory for invenio-records-rest but not used + pid_fetcher='entity_id', + search_index='entities', record_serializers={ 'application/json': 'rero_ils.modules.serializers:json_v1_response' }, # This is mandatory for invenio-records-rest but not used search_serializers={ - 'application/json': 'rero_ils.modules.unified_entities.serializers:json_entities_search' + 'application/json': 'rero_ils.modules.entities.serializers:json_entities_search' }, search_serializers_aliases={ 'json': 'application/json' }, - list_route='/unified_entities/', - item_route='/unified_entities/', # mandatory for invenio-records-rest (only used for permissions) + list_route='/entities/', + item_route='/entities/', # mandatory for invenio-records-rest (only used for permissions) default_media_type='application/json', max_result_window=MAX_RESULT_WINDOW, search_factory_imp='rero_ils.query:search_factory', @@ -2273,8 +2275,8 @@ def _(x): 'patron_transaction_events', 'patron_transactions', 'patron_types', + 'remote_entities', 'templates', - 'unified_entities', 'vendors' ] @@ -2643,6 +2645,12 @@ def _(x): 'rero_ils.modules.documents.permissions:create_action', 'rero_ils.modules.documents.permissions:update_action', 'rero_ils.modules.documents.permissions:delete_action', + 'rero_ils.modules.entities.local_entities.permissions.access_action', + 'rero_ils.modules.entities.local_entities.permissions.search_action', + 'rero_ils.modules.entities.local_entities.permissions.read_action', + 'rero_ils.modules.entities.local_entities.permissions.create_action', + 'rero_ils.modules.entities.local_entities.permissions.update_action', + 'rero_ils.modules.entities.local_entities.permissions.delete_action', 'rero_ils.modules.holdings.permissions:access_action', 'rero_ils.modules.holdings.permissions:search_action', 'rero_ils.modules.holdings.permissions:read_action', @@ -2688,12 +2696,6 @@ def _(x): 'rero_ils.modules.local_fields.permissions:create_action', 'rero_ils.modules.local_fields.permissions:update_action', 'rero_ils.modules.local_fields.permissions:delete_action', - 'rero_ils.modules.local_entities.permissions:access_action', - 'rero_ils.modules.local_entities.permissions:search_action', - 'rero_ils.modules.local_entities.permissions:read_action', - 'rero_ils.modules.local_entities.permissions:create_action', - 'rero_ils.modules.local_entities.permissions:update_action', - 'rero_ils.modules.local_entities.permissions:delete_action', 'rero_ils.modules.notifications.permissions:access_action', 'rero_ils.modules.notifications.permissions:search_action', 'rero_ils.modules.notifications.permissions:read_action', @@ -2817,7 +2819,7 @@ def _(x): 'budg': '/budgets/budget-v0.0.1.json', 'cipo': '/circ_policies/circ_policy-v0.0.1.json', 'coll': '/collections/collection-v0.0.1.json', - 'ent': '/entities/entity-v0.0.1.json', + 'rement': '/remote_entities/remote_entity-v0.0.1.json', 'doc': '/documents/document-v0.0.1.json', 'hold': '/holdings/holding-v0.0.1.json', 'illr': '/ill_requests/ill_request-v0.0.1.json', diff --git a/rero_ils/es_templates/v7/record.json b/rero_ils/es_templates/v7/record.json index 3d3da29fa0..c1c233389d 100644 --- a/rero_ils/es_templates/v7/record.json +++ b/rero_ils/es_templates/v7/record.json @@ -10,7 +10,6 @@ "circ_policies-*", "collections-*", "documents-*", - "entities-*", "holdings-*", "ill_requests-*", "item_types-*", @@ -27,6 +26,7 @@ "patron_transactions-*", "patron_types-*", "patrons-*", + "remote_entities-*", "stats-*", "templates-*", "vendors-*" diff --git a/rero_ils/modules/cli/reroils.py b/rero_ils/modules/cli/reroils.py index ccf03bb583..83d5938a5e 100644 --- a/rero_ils/modules/cli/reroils.py +++ b/rero_ils/modules/cli/reroils.py @@ -25,7 +25,7 @@ from rero_ils.modules.acquisition.cli import acquisition from rero_ils.modules.apiharvester.cli import apiharvester from rero_ils.modules.ebooks.cli import oaiharvester -from rero_ils.modules.entities.cli import entity +from rero_ils.modules.entities.remote_entities.cli import entity from rero_ils.modules.monitoring.cli import monitoring from rero_ils.modules.notifications.cli import notifications from rero_ils.modules.stats.cli import stats diff --git a/rero_ils/modules/cli/utils.py b/rero_ils/modules/cli/utils.py index 481c763951..cabacf734e 100644 --- a/rero_ils/modules/cli/utils.py +++ b/rero_ils/modules/cli/utils.py @@ -59,7 +59,7 @@ from rero_ils.modules.documents.api import Document, DocumentsSearch from rero_ils.modules.documents.dojson.contrib.marc21tojson.rero import marc21 from rero_ils.modules.documents.views import get_cover_art -from rero_ils.modules.entities.api import Entity +from rero_ils.modules.entities.remote_entities.api import RemoteEntity from rero_ils.modules.items.api import Item from rero_ils.modules.libraries.api import Library from rero_ils.modules.loans.tasks import \ @@ -1580,7 +1580,7 @@ def export(verbose, pid_type, outfile_name, pidfile, indent, schema): f'{count: <8} {pid_type} export {rec.pid}:{rec.id}') if not schema: rec.pop('$schema', None) - if isinstance(rec, Entity): + if isinstance(rec, RemoteEntity): for agent_source in agents_sources: rec.get(agent_source, {}).pop('$schema', None) outfile.write(rec) diff --git a/rero_ils/modules/documents/api.py b/rero_ils/modules/documents/api.py index 1cf31cdc7d..170b8fcaa1 100644 --- a/rero_ils/modules/documents/api.py +++ b/rero_ils/modules/documents/api.py @@ -242,20 +242,21 @@ def reasons_not_to_delete(self): def index_contributions(self, bulk=False): """Index all attached contributions.""" - from ..entities.api import EntitiesIndexer, Entity + from rero_ils.modules.entities.remote_entities.api import \ + RemoteEntitiesIndexer, RemoteEntity from ..tasks import process_bulk_queue contributions_ids = [] for contribution in self.get('contribution', []): ref = contribution['entity'].get('$ref') if not ref and (cont_pid := contribution['entity'].get('pid')): if bulk: - uid = Entity.get_id_by_pid(cont_pid) + uid = RemoteEntity.get_id_by_pid(cont_pid) contributions_ids.append(uid) else: - contrib = Entity.get_record_by_pid(cont_pid) + contrib = RemoteEntity.get_record_by_pid(cont_pid) contrib.reindex() if contributions_ids: - EntitiesIndexer().bulk_index(contributions_ids) + RemoteEntitiesIndexer().bulk_index(contributions_ids) process_bulk_queue.apply_async() @classmethod diff --git a/rero_ils/modules/documents/dojson/contrib/jsontodc/model.py b/rero_ils/modules/documents/dojson/contrib/jsontodc/model.py index abc6be1bcc..69685a1dad 100644 --- a/rero_ils/modules/documents/dojson/contrib/jsontodc/model.py +++ b/rero_ils/modules/documents/dojson/contrib/jsontodc/model.py @@ -22,7 +22,8 @@ from rero_ils.modules.documents.extensions import TitleExtension from rero_ils.modules.entities.models import EntityType -from rero_ils.modules.entities.utils import get_entity_localized_value +from rero_ils.modules.entities.remote_entities.utils import \ + get_entity_localized_value class DublinCoreOverdo(Overdo): diff --git a/rero_ils/modules/documents/dojson/contrib/jsontomarc21/model.py b/rero_ils/modules/documents/dojson/contrib/jsontomarc21/model.py index d5204b9985..8425537921 100644 --- a/rero_ils/modules/documents/dojson/contrib/jsontomarc21/model.py +++ b/rero_ils/modules/documents/dojson/contrib/jsontomarc21/model.py @@ -25,8 +25,9 @@ from rero_ils.modules.documents.utils import display_alternate_graphic_first from rero_ils.modules.documents.views import create_title_responsibilites -from rero_ils.modules.entities.api import EntitiesSearch, Entity from rero_ils.modules.entities.models import EntityType +from rero_ils.modules.entities.remote_entities.api import RemoteEntity, \ + RemoteEntitiesSearch from rero_ils.modules.holdings.api import Holding, HoldingsSearch from rero_ils.modules.items.api import Item, ItemsSearch from rero_ils.modules.libraries.api import Library @@ -116,7 +117,7 @@ def do_contribution(contribution, source_order): if pid := entity.get('pid'): # we have a $ref, get the real entity ref = entity.get('$ref') - if entity_db := Entity.get_record_by_pid(pid): + if entity_db := RemoteEntity.get_record_by_pid(pid): contribution = replace_contribution_sources( contribution={'entity': entity_db}, source_order=source_order @@ -180,7 +181,7 @@ def do_concept(entity, source_order): if pid := entity.get('pid'): ref = entity.get('$ref') # we have a $ref, get the real entity - if entity := Entity.get_record_by_pid(pid): + if entity := RemoteEntity.get_record_by_pid(pid): entity = replace_concept_sources( concept=entity, source_order=source_order @@ -718,7 +719,7 @@ def add_identified_by(result, identified_by): tag = None entity_type = entity.get('type') or entity.get('bf:Agent') if entity_pid := entity.get('pid'): - query = EntitiesSearch().filter('term', pid=entity_pid) + query = RemoteEntitiesSearch().filter('term', pid=entity_pid) if query.count(): entity_type = next(query.source('type').scan()).type if entity_type in [EntityType.PERSON, EntityType.ORGANISATION]: diff --git a/rero_ils/modules/documents/dumpers/replace_refs.py b/rero_ils/modules/documents/dumpers/replace_refs.py index aedfa06848..b89953565c 100644 --- a/rero_ils/modules/documents/dumpers/replace_refs.py +++ b/rero_ils/modules/documents/dumpers/replace_refs.py @@ -20,7 +20,8 @@ from invenio_records.dumpers import Dumper from rero_ils.modules.commons.exceptions import RecordNotFound -from rero_ils.modules.entities.utils import extract_data_from_mef_uri +from rero_ils.modules.entities.remote_entities.utils import \ + extract_data_from_mef_uri class ReplaceRefsEntitiesDumperMixin(Dumper): @@ -29,9 +30,9 @@ class ReplaceRefsEntitiesDumperMixin(Dumper): @staticmethod def _replace_entity(data): """Replace the `$ref` linked contributions.""" - from rero_ils.modules.entities.api import Entity - if not (entity := Entity.get_record_by_pid(data['pid'])): - raise RecordNotFound(Entity, data['pid']) + from rero_ils.modules.entities.remote_entities.api import RemoteEntity + if not (entity := RemoteEntity.get_record_by_pid(data['pid'])): + raise RecordNotFound(RemoteEntity, data['pid']) _, _type, _ = extract_data_from_mef_uri(data['$ref']) entity = entity.dumps_for_document() entity.update({ @@ -84,11 +85,11 @@ def dump(self, record, data): :return a dict with dumped data. """ for field_name in self.field_names: - entities = [] + remote_entities = [] for entity in [d['entity'] for d in data.get(field_name, [])]: if entity.get('$ref'): entity = self._replace_entity(entity) - entities.append({'entity': entity}) - if entities: - data[field_name] = entities + remote_entities.append({'entity': entity}) + if remote_entities: + data[field_name] = remote_entities return data diff --git a/rero_ils/modules/documents/extensions/add_mef_pid.py b/rero_ils/modules/documents/extensions/add_mef_pid.py index f6df1d1625..b0198b8b5d 100644 --- a/rero_ils/modules/documents/extensions/add_mef_pid.py +++ b/rero_ils/modules/documents/extensions/add_mef_pid.py @@ -40,8 +40,8 @@ def add_mef_pid(self, record): :params record: dict - a document record. """ - from rero_ils.modules.entities.api import Entity - entities = [] + from rero_ils.modules.entities.remote_entities.api import RemoteEntity + remote_entities = [] # Search about all entities present in the document through fields # defined for this extension @@ -49,14 +49,14 @@ def add_mef_pid(self, record): fields = record.get(field_name, []) if not isinstance(fields, list): fields = [fields] - entities.extend([ + remote_entities.extend([ field['entity'] for field in fields if 'entity' in field ]) # For each found entity, add its PID into the entity data. - for entity_data in entities: + for entity_data in remote_entities: if ref := entity_data.get('$ref'): - entity, _ = Entity.get_record_by_ref(ref) + entity, _ = RemoteEntity.get_record_by_ref(ref) if entity: # inject mef pid entity_data['pid'] = entity['pid'] diff --git a/rero_ils/modules/documents/extensions/provision_activities.py b/rero_ils/modules/documents/extensions/provision_activities.py index d446491e61..ff8e6ab616 100644 --- a/rero_ils/modules/documents/extensions/provision_activities.py +++ b/rero_ils/modules/documents/extensions/provision_activities.py @@ -24,7 +24,7 @@ from rero_ils.dojson.utils import remove_trailing_punctuation from ..utils import display_alternate_graphic_first -from ...entities.models import EntityType +from rero_ils.modules.entities.models import EntityType class ProvisionActivitiesExtension(RecordExtension): diff --git a/rero_ils/modules/documents/serializers/base.py b/rero_ils/modules/documents/serializers/base.py index 87b9e4316f..d4cebcd233 100644 --- a/rero_ils/modules/documents/serializers/base.py +++ b/rero_ils/modules/documents/serializers/base.py @@ -27,7 +27,7 @@ from rero_ils.modules.utils import get_base_url from ..api import DocumentsSearch -from ...entities.models import EntityType +from rero_ils.modules.entities.models import EntityType CREATOR_ROLES = [ 'aut', 'cmp', 'cre', 'dub', 'pht', 'ape', 'aqt', 'arc', 'art', 'aus', diff --git a/rero_ils/modules/documents/serializers/marc.py b/rero_ils/modules/documents/serializers/marc.py index 1f98dec06c..bc35ab297f 100644 --- a/rero_ils/modules/documents/serializers/marc.py +++ b/rero_ils/modules/documents/serializers/marc.py @@ -31,7 +31,7 @@ from rero_ils.modules.documents.dojson.contrib.jsontomarc21 import to_marc21 from rero_ils.modules.documents.dojson.contrib.jsontomarc21.model import \ replace_contribution_sources -from rero_ils.modules.entities.api import EntitiesSearch +from rero_ils.modules.entities.remote_entities.api import RemoteEntitiesSearch from rero_ils.modules.serializers import JSONSerializer from rero_ils.modules.utils import strip_chars @@ -114,7 +114,7 @@ def transform_records(self, hits, pid_fetcher, language, contribution_pid = contribution.get('entity', {}).get('pid') if contribution_pid: contribution_pids.append(contribution_pid) - search = EntitiesSearch() \ + search = RemoteEntitiesSearch() \ .filter('terms', pid=list(set(contribution_pids))) es_contributions = {} for hit in search.scan(): diff --git a/rero_ils/modules/documents/utils.py b/rero_ils/modules/documents/utils.py index 0973550d3a..f9472d4934 100644 --- a/rero_ils/modules/documents/utils.py +++ b/rero_ils/modules/documents/utils.py @@ -245,7 +245,7 @@ def create_authorized_access_point(agent): if not agent: return None authorized_access_point = agent.get('preferred_name') - from ..entities.models import EntityType + from rero_ils.modules.entities.models import EntityType if agent.get('type') == EntityType.PERSON: date_parts = [agent.get('date_of_birth'), agent.get('date_of_death')] date = '-'.join(filter(None, date_parts)) diff --git a/rero_ils/modules/documents/views.py b/rero_ils/modules/documents/views.py index e7cf0eb2c0..fd647f046a 100644 --- a/rero_ils/modules/documents/views.py +++ b/rero_ils/modules/documents/views.py @@ -37,8 +37,8 @@ title_format_text, title_format_text_alternate_graphic, \ title_variant_format_text from ..collections.api import CollectionsSearch -from ..entities.api import Entity -from ..entities.models import EntityType +from rero_ils.modules.entities.remote_entities.api import RemoteEntity +from rero_ils.modules.entities.models import EntityType from ..holdings.models import HoldingNoteTypes from ..items.models import ItemCirculationAction from ..libraries.api import Library @@ -263,7 +263,8 @@ def contribution_format(contributions, language, viewcode, with_roles=False): """ output = [] for contrib in filter(lambda c: c.get('entity'), contributions): - if entity := Entity.get_record_by_pid(contrib['entity'].get('pid')): + if entity := RemoteEntity \ + .get_record_by_pid(contrib['entity'].get('pid')): text = entity.get_authorized_access_point(language=language) entity_type = 'persons' if entity.get('type') == EntityType.ORGANISATION: @@ -296,7 +297,7 @@ def doc_entity_label(entity, language=None, part_separator=' - ') -> str: """ parts = [] if 'pid' in entity: - entity = Entity.get_record_by_pid(entity['pid']) + entity = RemoteEntity.get_record_by_pid(entity['pid']) parts.append(entity.get_authorized_access_point(language=language)) else: default_key = 'authorized_access_point' diff --git a/rero_ils/modules/entities/__init__.py b/rero_ils/modules/entities/__init__.py index ff7065b145..2d4735e61e 100644 --- a/rero_ils/modules/entities/__init__.py +++ b/rero_ils/modules/entities/__init__.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- # # RERO ILS -# Copyright (C) 2019-2022 RERO +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -15,4 +16,4 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -"""Mef Contributions Records.""" +"""Entities module.""" diff --git a/rero_ils/modules/unified_entities/fetchers.py b/rero_ils/modules/entities/fetchers.py similarity index 98% rename from rero_ils/modules/unified_entities/fetchers.py rename to rero_ils/modules/entities/fetchers.py index b22ed76a77..457321b950 100644 --- a/rero_ils/modules/unified_entities/fetchers.py +++ b/rero_ils/modules/entities/fetchers.py @@ -36,7 +36,7 @@ def id_fetcher(record_uuid, data): :param data: The record metadata. :return: A :data:`rero_ils.modules.fetchers.FetchedPID` instance. """ - pid_type = 'unient' + pid_type = 'ent' # try to extract pid type from schema if schema := data.get('$schema'): pid_type = get_pid_type_from_schema(schema) diff --git a/rero_ils/modules/local_entities/__init__.py b/rero_ils/modules/entities/local_entities/__init__.py similarity index 100% rename from rero_ils/modules/local_entities/__init__.py rename to rero_ils/modules/entities/local_entities/__init__.py diff --git a/rero_ils/modules/local_entities/api.py b/rero_ils/modules/entities/local_entities/api.py similarity index 100% rename from rero_ils/modules/local_entities/api.py rename to rero_ils/modules/entities/local_entities/api.py diff --git a/rero_ils/modules/local_entities/dumpers/__init__.py b/rero_ils/modules/entities/local_entities/dumpers/__init__.py similarity index 100% rename from rero_ils/modules/local_entities/dumpers/__init__.py rename to rero_ils/modules/entities/local_entities/dumpers/__init__.py diff --git a/rero_ils/modules/local_entities/dumpers/indexer.py b/rero_ils/modules/entities/local_entities/dumpers/indexer.py similarity index 100% rename from rero_ils/modules/local_entities/dumpers/indexer.py rename to rero_ils/modules/entities/local_entities/dumpers/indexer.py diff --git a/rero_ils/modules/local_entities/extensions/__init__.py b/rero_ils/modules/entities/local_entities/extensions/__init__.py similarity index 100% rename from rero_ils/modules/local_entities/extensions/__init__.py rename to rero_ils/modules/entities/local_entities/extensions/__init__.py diff --git a/rero_ils/modules/local_entities/extensions/authorized_access_point.py b/rero_ils/modules/entities/local_entities/extensions/authorized_access_point.py similarity index 100% rename from rero_ils/modules/local_entities/extensions/authorized_access_point.py rename to rero_ils/modules/entities/local_entities/extensions/authorized_access_point.py diff --git a/rero_ils/modules/local_entities/jsonschemas/__init__.py b/rero_ils/modules/entities/local_entities/jsonschemas/__init__.py similarity index 100% rename from rero_ils/modules/local_entities/jsonschemas/__init__.py rename to rero_ils/modules/entities/local_entities/jsonschemas/__init__.py diff --git a/rero_ils/modules/local_entities/jsonschemas/local_entities/local_entity-v0.0.1.json b/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity-v0.0.1.json similarity index 100% rename from rero_ils/modules/local_entities/jsonschemas/local_entities/local_entity-v0.0.1.json rename to rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity-v0.0.1.json diff --git a/rero_ils/modules/local_entities/mappings/__init__.py b/rero_ils/modules/entities/local_entities/mappings/__init__.py similarity index 100% rename from rero_ils/modules/local_entities/mappings/__init__.py rename to rero_ils/modules/entities/local_entities/mappings/__init__.py diff --git a/rero_ils/modules/local_entities/mappings/v7/__init__.py b/rero_ils/modules/entities/local_entities/mappings/v7/__init__.py similarity index 100% rename from rero_ils/modules/local_entities/mappings/v7/__init__.py rename to rero_ils/modules/entities/local_entities/mappings/v7/__init__.py diff --git a/rero_ils/modules/local_entities/mappings/v7/local_entities/local_entity-v0.0.1.json b/rero_ils/modules/entities/local_entities/mappings/v7/local_entities/local_entity-v0.0.1.json similarity index 98% rename from rero_ils/modules/local_entities/mappings/v7/local_entities/local_entity-v0.0.1.json rename to rero_ils/modules/entities/local_entities/mappings/v7/local_entities/local_entity-v0.0.1.json index 1c86a1d1ea..3f17a16c4c 100644 --- a/rero_ils/modules/local_entities/mappings/v7/local_entities/local_entity-v0.0.1.json +++ b/rero_ils/modules/entities/local_entities/mappings/v7/local_entities/local_entity-v0.0.1.json @@ -1,6 +1,6 @@ { "aliases": { - "unified_entities": {} + "entities": {} }, "settings": { "analysis": { diff --git a/rero_ils/modules/local_entities/models.py b/rero_ils/modules/entities/local_entities/models.py similarity index 84% rename from rero_ils/modules/local_entities/models.py rename to rero_ils/modules/entities/local_entities/models.py index b7bc833bbf..3ccd081320 100644 --- a/rero_ils/modules/local_entities/models.py +++ b/rero_ils/modules/entities/local_entities/models.py @@ -40,15 +40,3 @@ class LocalEntityMetadata(db.Model, RecordMetadataBase): """Entity record metadata.""" __tablename__ = 'local_entity_metadata' - - -class LocalEntityType: - """Class holding all available entity types.""" - - AGENT = 'bf:Agent' - ORGANISATION = 'bf:Organisation' - PERSON = 'bf:Person' - PLACE = 'bf:Place' - TEMPORAL = 'bf:Temporal' - TOPIC = 'bf:Topic' - WORK = 'bf:Work' diff --git a/rero_ils/modules/local_entities/permissions.py b/rero_ils/modules/entities/local_entities/permissions.py similarity index 100% rename from rero_ils/modules/local_entities/permissions.py rename to rero_ils/modules/entities/local_entities/permissions.py diff --git a/rero_ils/modules/unified_entities/minters.py b/rero_ils/modules/entities/minters.py similarity index 100% rename from rero_ils/modules/unified_entities/minters.py rename to rero_ils/modules/entities/minters.py diff --git a/rero_ils/modules/entities/models.py b/rero_ils/modules/entities/models.py index 5b5130407d..e7b16af88f 100644 --- a/rero_ils/modules/entities/models.py +++ b/rero_ils/modules/entities/models.py @@ -16,32 +16,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -"""Define relation between records and buckets.""" - -from __future__ import absolute_import - -from invenio_db import db -from invenio_pidstore.models import RecordIdentifier -from invenio_records.models import RecordMetadataBase - - -class EntityIdentifier(RecordIdentifier): - """Sequence generator for `Entity` identifiers.""" - - __tablename__ = 'entity_id' - __mapper_args__ = {'concrete': True} - - recid = db.Column( - db.BigInteger().with_variant(db.Integer, 'sqlite'), - primary_key=True, - autoincrement=True, - ) - - -class EntityMetadata(db.Model, RecordMetadataBase): - """Entity record metadata.""" - - __tablename__ = 'entity_metadata' +"""Entities model class.""" class EntityType: @@ -54,11 +29,3 @@ class EntityType: TEMPORAL = 'bf:Temporal' TOPIC = 'bf:Topic' WORK = 'bf:Work' - - -class EntityUpdateAction: - """Class holding all available agent record creation actions.""" - - REPLACE = 'replace' - UPTODATE = 'uptodate' - ERROR = 'error' diff --git a/rero_ils/modules/unified_entities/__init__.py b/rero_ils/modules/entities/remote_entities/__init__.py similarity index 86% rename from rero_ils/modules/unified_entities/__init__.py rename to rero_ils/modules/entities/remote_entities/__init__.py index 6afed31d9b..ff7065b145 100644 --- a/rero_ils/modules/unified_entities/__init__.py +++ b/rero_ils/modules/entities/remote_entities/__init__.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- # # RERO ILS -# Copyright (C) 2019-2023 RERO -# Copyright (C) 2019-2023 UCLouvain +# Copyright (C) 2019-2022 RERO # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -16,4 +15,4 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -"""Unified entities Records.""" +"""Mef Contributions Records.""" diff --git a/rero_ils/modules/entities/api.py b/rero_ils/modules/entities/remote_entities/api.py similarity index 92% rename from rero_ils/modules/entities/api.py rename to rero_ils/modules/entities/remote_entities/api.py index 48f9f37185..e3b9261c96 100644 --- a/rero_ils/modules/entities/api.py +++ b/rero_ils/modules/entities/remote_entities/api.py @@ -33,28 +33,29 @@ from rero_ils.modules.providers import Provider from rero_ils.utils import get_i18n_supported_languages -from .models import EntityIdentifier, EntityMetadata, EntityUpdateAction +from .models import RemoteEntityIdentifier, RemoteEntityMetadata, \ + EntityUpdateAction from .utils import extract_data_from_mef_uri, get_mef_data_by_type # provider -EntityProvider = type( +RemoteEntityProvider = type( 'EntityProvider', (Provider,), - dict(identifier=EntityIdentifier, pid_type='ent') + dict(identifier=RemoteEntityIdentifier, pid_type='rement') ) # minter -entity_id_minter = partial(id_minter, provider=EntityProvider) +remote_entity_id_minter = partial(id_minter, provider=RemoteEntityProvider) # fetcher -entity_id_fetcher = partial(id_fetcher, provider=EntityProvider) +remote_entity_id_fetcher = partial(id_fetcher, provider=RemoteEntityProvider) -class EntitiesSearch(IlsRecordsSearch): +class RemoteEntitiesSearch(IlsRecordsSearch): """Mef contribution search.""" class Meta: """Meta class.""" - index = 'entities' + index = 'remote_entities' doc_types = None fields = ('*', ) facets = {} @@ -62,13 +63,13 @@ class Meta: default_filter = None -class Entity(IlsRecord): +class RemoteEntity(IlsRecord): """Mef contribution class.""" - minter = entity_id_minter - fetcher = entity_id_fetcher - provider = EntityProvider - model_cls = EntityMetadata + minter = remote_entity_id_minter + fetcher = remote_entity_id_fetcher + provider = RemoteEntityProvider + model_cls = RemoteEntityMetadata @classmethod def get_entity(cls, ref_type, ref_pid): @@ -81,7 +82,7 @@ def get_entity(cls, ref_type, ref_pid): es_filter = Q('term', viaf_pid=ref_pid) # in case of multiple results get the more recent - query = EntitiesSearch() \ + query = RemoteEntitiesSearch() \ .params(preserve_order=True) \ .sort({'_created': {'order': 'desc'}})\ .filter(es_filter) @@ -113,7 +114,7 @@ def get_record_by_ref(cls, ref): )): raise Exception('NO DATA') # Try to get the contribution from DB maybe it was not indexed. - if entity := Entity.get_record_by_pid(data['pid']): + if entity := RemoteEntity.get_record_by_pid(data['pid']): entity = entity.replace(data) else: entity = cls.create(data) @@ -290,14 +291,14 @@ def type(self): return entity_types.get(self['type']) -class EntitiesIndexer(IlsRecordsIndexer): +class RemoteEntitiesIndexer(IlsRecordsIndexer): """Entity indexing class.""" - record_cls = Entity + record_cls = RemoteEntity def bulk_index(self, record_id_iterator): """Bulk index records. :param record_id_iterator: Iterator yielding record UUIDs. """ - super().bulk_index(record_id_iterator, doc_type='ent') + super().bulk_index(record_id_iterator, doc_type='rement') diff --git a/rero_ils/modules/entities/cli.py b/rero_ils/modules/entities/remote_entities/cli.py similarity index 100% rename from rero_ils/modules/entities/cli.py rename to rero_ils/modules/entities/remote_entities/cli.py diff --git a/rero_ils/modules/entities/jsonschemas/__init__.py b/rero_ils/modules/entities/remote_entities/jsonschemas/__init__.py similarity index 100% rename from rero_ils/modules/entities/jsonschemas/__init__.py rename to rero_ils/modules/entities/remote_entities/jsonschemas/__init__.py diff --git a/rero_ils/modules/entities/jsonschemas/entities/entity-v0.0.1.json b/rero_ils/modules/entities/remote_entities/jsonschemas/remote_entities/remote_entity-v0.0.1.json similarity index 100% rename from rero_ils/modules/entities/jsonschemas/entities/entity-v0.0.1.json rename to rero_ils/modules/entities/remote_entities/jsonschemas/remote_entities/remote_entity-v0.0.1.json diff --git a/rero_ils/modules/entities/listener.py b/rero_ils/modules/entities/remote_entities/listener.py similarity index 73% rename from rero_ils/modules/entities/listener.py rename to rero_ils/modules/entities/remote_entities/listener.py index c6877604b2..daea79f0b7 100644 --- a/rero_ils/modules/entities/listener.py +++ b/rero_ils/modules/entities/remote_entities/listener.py @@ -18,11 +18,11 @@ """Signals connector for `Entity` records.""" -from .api import EntitiesSearch, Entity +from .api import RemoteEntitiesSearch, RemoteEntity -def enrich_entities_data(sender, json=None, record=None, index=None, - doc_type=None, arguments=None, **dummy_kwargs): +def enrich_remote_entities_data(sender, json=None, record=None, index=None, + doc_type=None, arguments=None, **dummy_kwargs): """Signal sent before a record is indexed. :param json: The dumped record dictionary which can be modified. @@ -30,7 +30,7 @@ def enrich_entities_data(sender, json=None, record=None, index=None, :param index: The index in which the record will be indexed. :param doc_type: The doc_type for the record. """ - if index.split('-')[0] == EntitiesSearch.Meta.index: - if not isinstance(record, Entity): - record = Entity.get_record_by_pid(record.get('pid')) + if index.split('-')[0] == RemoteEntitiesSearch.Meta.index: + if not isinstance(record, RemoteEntity): + record = RemoteEntity.get_record_by_pid(record.get('pid')) json['organisations'] = record.organisation_pids diff --git a/rero_ils/modules/entities/mappings/__init__.py b/rero_ils/modules/entities/remote_entities/mappings/__init__.py similarity index 100% rename from rero_ils/modules/entities/mappings/__init__.py rename to rero_ils/modules/entities/remote_entities/mappings/__init__.py diff --git a/rero_ils/modules/entities/mappings/v7/__init__.py b/rero_ils/modules/entities/remote_entities/mappings/v7/__init__.py similarity index 100% rename from rero_ils/modules/entities/mappings/v7/__init__.py rename to rero_ils/modules/entities/remote_entities/mappings/v7/__init__.py diff --git a/rero_ils/modules/entities/mappings/v7/entities/entity-v0.0.1.json b/rero_ils/modules/entities/remote_entities/mappings/v7/remote_entities/remote_entity-v0.0.1.json similarity index 99% rename from rero_ils/modules/entities/mappings/v7/entities/entity-v0.0.1.json rename to rero_ils/modules/entities/remote_entities/mappings/v7/remote_entities/remote_entity-v0.0.1.json index df045f6783..c363fd7d9b 100644 --- a/rero_ils/modules/entities/mappings/v7/entities/entity-v0.0.1.json +++ b/rero_ils/modules/entities/remote_entities/mappings/v7/remote_entities/remote_entity-v0.0.1.json @@ -1,6 +1,6 @@ { "aliases": { - "unified_entities": {} + "entities": {} }, "settings": { "analysis": { diff --git a/rero_ils/modules/entities/remote_entities/models.py b/rero_ils/modules/entities/remote_entities/models.py new file mode 100644 index 0000000000..075dc96e29 --- /dev/null +++ b/rero_ils/modules/entities/remote_entities/models.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Define relation between records and buckets.""" + +from __future__ import absolute_import + +from invenio_db import db +from invenio_pidstore.models import RecordIdentifier +from invenio_records.models import RecordMetadataBase + + +class RemoteEntityIdentifier(RecordIdentifier): + """Sequence generator for `Remote Entity` identifiers.""" + + __tablename__ = 'remote_entity_id' + __mapper_args__ = {'concrete': True} + + recid = db.Column( + db.BigInteger().with_variant(db.Integer, 'sqlite'), + primary_key=True, + autoincrement=True, + ) + + +class RemoteEntityMetadata(db.Model, RecordMetadataBase): + """Remote Entity record metadata.""" + + __tablename__ = 'remote_entity_metadata' + + +class EntityUpdateAction: + """Class holding all available agent record creation actions.""" + + REPLACE = 'replace' + UPTODATE = 'uptodate' + ERROR = 'error' diff --git a/rero_ils/modules/entities/permissions.py b/rero_ils/modules/entities/remote_entities/permissions.py similarity index 94% rename from rero_ils/modules/entities/permissions.py rename to rero_ils/modules/entities/remote_entities/permissions.py index 179606fdc4..d487d25664 100644 --- a/rero_ils/modules/entities/permissions.py +++ b/rero_ils/modules/entities/remote_entities/permissions.py @@ -22,7 +22,7 @@ from rero_ils.modules.permissions import RecordPermissionPolicy -class EntityPermissionPolicy(RecordPermissionPolicy): +class RemoteEntityPermissionPolicy(RecordPermissionPolicy): """Entity Permission Policy used by the CRUD operations. Only search and read is allowed for all users. diff --git a/rero_ils/modules/entities/proxy.py b/rero_ils/modules/entities/remote_entities/proxy.py similarity index 100% rename from rero_ils/modules/entities/proxy.py rename to rero_ils/modules/entities/remote_entities/proxy.py diff --git a/rero_ils/modules/entities/sync.py b/rero_ils/modules/entities/remote_entities/sync.py similarity index 96% rename from rero_ils/modules/entities/sync.py rename to rero_ils/modules/entities/remote_entities/sync.py index 870b530319..93fbc8b9f1 100644 --- a/rero_ils/modules/entities/sync.py +++ b/rero_ils/modules/entities/remote_entities/sync.py @@ -29,8 +29,8 @@ from invenio_db import db from rero_ils.modules.documents.api import Document, DocumentsSearch -from rero_ils.modules.entities.api import EntitiesSearch, Entity -from rero_ils.modules.entities.logger import create_logger +from rero_ils.modules.entities.remote_entities.api import \ + RemoteEntitiesSearch, RemoteEntity from rero_ils.modules.utils import get_mef_url, get_timestamp, \ requests_retry_session, set_timestamp @@ -140,22 +140,22 @@ def _update_entities_in_document(self, doc_pid, pids_to_replace): # get all entities from the document over all entity fields: # contribution and subjects - entities = [ + remote_entities = [ subject for subject in doc.get('subjects', []) if subject.get('$ref') ] - entities += [ + remote_entities += [ contrib['entity'] for contrib in doc.get('contribution', []) if contrib.get('entity', {}).get('$ref') ] - entities += [ + remote_entities += [ genre_form for genre_form in doc.get('genreForm', []) if genre_form.get('$ref') ] - if not entities: + if not remote_entities: self.logger.debug(f'No entity to update for document {doc.pid}') # update the $ref entity URL and MEF pid @@ -163,7 +163,7 @@ def _update_entities_in_document(self, doc_pid, pids_to_replace): old_entity_url = f'{mef_url}/{old_pid}' new_entity_url = f'{mef_url}/{new_pid}' entities_to_update = filter( - lambda c: c.get('$ref') == old_entity_url, entities) + lambda c: c.get('$ref') == old_entity_url, remote_entities) for entity in entities_to_update: if old_entity_url != new_entity_url: self.logger.info( @@ -201,7 +201,7 @@ def get_entities_pids(self, query='*', from_date=None): :returns: the list of the contribution identifiers. :rtype: list of strings. """ - es_query = EntitiesSearch().filter('query_string', query=query) + es_query = RemoteEntitiesSearch().filter('query_string', query=query) total = es_query.count() if not from_date and self.from_date: from_date = self.from_date @@ -299,7 +299,7 @@ def sync_record(self, pid): updated = error = False try: # get contribution in db - entity = Entity.get_record_by_pid(pid) + entity = RemoteEntity.get_record_by_pid(pid) if not entity: raise Exception(f'ERROR MEF {pid} does not exists in db.') self.logger.debug(f'Processing {entity["type"]} MEF(pid: {pid})') @@ -342,7 +342,7 @@ def sync_record(self, pid): f'{old_mef_pid} to {new_mef_pid} ' f'for {source} (pid:{old_entity_pid})' ) - if Entity.get_record_by_pid(new_mef_pid): + if RemoteEntity.get_record_by_pid(new_mef_pid): # update the new MEF - recursion self.logger.info( f'{entity["type"]} MEF(pid: {entity.pid}) ' @@ -356,7 +356,7 @@ def sync_record(self, pid): else: # if the MEF record does not exist create it if not self.dry_run: - Entity.create( + RemoteEntity.create( data=new_mef_data, dbcommit=True, reindex=True @@ -370,14 +370,14 @@ def sync_record(self, pid): 'content has been updated') if not self.dry_run: if old_mef_pid == new_mef_pid: - Entity.get_record(entity.id).replace( + RemoteEntity.get_record(entity.id).replace( new_mef_data, dbcommit=True, reindex=True) else: # as we have only the last mef but not the old one # we need get it from the MEF server # this is important as it can still be used by # other entities - Entity.get_record_by_pid(pid)\ + RemoteEntity.get_record_by_pid(pid)\ .update_online(dbcommit=True, reindex=True) updated = True @@ -468,7 +468,7 @@ def remove_unused_record(self, pid): doc_pids = SyncEntity._get_documents_pids_from_mef(pid) if len(doc_pids) == 0: # get the contribution for the database - entity = Entity.get_record_by_pid(pid) + entity = RemoteEntity.get_record_by_pid(pid) if not self.dry_run: # remove from the database and the index: no tombstone entity.delete(True, True, True) diff --git a/rero_ils/modules/entities/tasks.py b/rero_ils/modules/entities/remote_entities/tasks.py similarity index 97% rename from rero_ils/modules/entities/tasks.py rename to rero_ils/modules/entities/remote_entities/tasks.py index d2668cabee..1ed159dd2f 100644 --- a/rero_ils/modules/entities/tasks.py +++ b/rero_ils/modules/entities/remote_entities/tasks.py @@ -24,7 +24,7 @@ from celery import shared_task from flask import current_app -from .api import Entity +from .api import RemoteEntity from .replace import ReplaceIdentifiedBy from .sync import SyncEntity @@ -38,7 +38,7 @@ def delete_records(records, verbose=False): :return: count of records """ for record in records: - status = Entity.delete( + status = RemoteEntity.delete( record, force=False, dbcommit=True, diff --git a/rero_ils/modules/entities/templates/rero_ils/_entity_by_source.html b/rero_ils/modules/entities/remote_entities/templates/rero_ils/_entity_by_source.html similarity index 100% rename from rero_ils/modules/entities/templates/rero_ils/_entity_by_source.html rename to rero_ils/modules/entities/remote_entities/templates/rero_ils/_entity_by_source.html diff --git a/rero_ils/modules/entities/templates/rero_ils/_entity_by_source_data.html b/rero_ils/modules/entities/remote_entities/templates/rero_ils/_entity_by_source_data.html similarity index 100% rename from rero_ils/modules/entities/templates/rero_ils/_entity_by_source_data.html rename to rero_ils/modules/entities/remote_entities/templates/rero_ils/_entity_by_source_data.html diff --git a/rero_ils/modules/entities/templates/rero_ils/_entity_unified.html b/rero_ils/modules/entities/remote_entities/templates/rero_ils/_entity_unified.html similarity index 100% rename from rero_ils/modules/entities/templates/rero_ils/_entity_unified.html rename to rero_ils/modules/entities/remote_entities/templates/rero_ils/_entity_unified.html diff --git a/rero_ils/modules/entities/templates/rero_ils/detailed_view_entity.html b/rero_ils/modules/entities/remote_entities/templates/rero_ils/detailed_view_entity.html similarity index 100% rename from rero_ils/modules/entities/templates/rero_ils/detailed_view_entity.html rename to rero_ils/modules/entities/remote_entities/templates/rero_ils/detailed_view_entity.html diff --git a/rero_ils/modules/entities/templates/rero_ils/macros/entity.html b/rero_ils/modules/entities/remote_entities/templates/rero_ils/macros/entity.html similarity index 100% rename from rero_ils/modules/entities/templates/rero_ils/macros/entity.html rename to rero_ils/modules/entities/remote_entities/templates/rero_ils/macros/entity.html diff --git a/rero_ils/modules/entities/utils.py b/rero_ils/modules/entities/remote_entities/utils.py similarity index 100% rename from rero_ils/modules/entities/utils.py rename to rero_ils/modules/entities/remote_entities/utils.py diff --git a/rero_ils/modules/entities/views.py b/rero_ils/modules/entities/remote_entities/views.py similarity index 90% rename from rero_ils/modules/entities/views.py rename to rero_ils/modules/entities/remote_entities/views.py index ef76f7cff4..1f991c4604 100644 --- a/rero_ils/modules/entities/views.py +++ b/rero_ils/modules/entities/remote_entities/views.py @@ -30,12 +30,12 @@ from rero_ils.modules.organisations.api import Organisation from rero_ils.theme.views import url_active -from .api import Entity -from .models import EntityType +from .api import RemoteEntity +from ..models import EntityType from .proxy import MEFProxyFactory blueprint = Blueprint( - 'entities', + 'remote_entities', __name__, url_prefix='/', template_folder='templates', @@ -43,12 +43,12 @@ ) api_blueprint = Blueprint( - 'api_entities', + 'api_remote_entities', __name__ ) -def entity_proxy(viewcode, pid, entity_type): +def remote_entity_proxy(viewcode, pid, entity_type): """Proxy for entities. :param viewcode: viewcode of html request @@ -56,10 +56,10 @@ def entity_proxy(viewcode, pid, entity_type): :param entity_type: type of the entity :returns: entity template """ - entity = Entity.get_record_by_pid(pid) + entity = RemoteEntity.get_record_by_pid(pid) if not entity or entity.get('type') != entity_type: abort(404, 'Record not found') - return entity_view_method( + return remote_entity_view_method( pid=entity.persistent_identifier, record=entity, template='rero_ils/detailed_view_entity.html', @@ -67,7 +67,7 @@ def entity_proxy(viewcode, pid, entity_type): ) -def entity_view_method(pid, record, template=None, **kwargs): +def remote_entity_view_method(pid, record, template=None, **kwargs): """Display default view. Sends record_viewed signal and renders template. @@ -99,19 +99,19 @@ def entity_view_method(pid, record, template=None, **kwargs): @blueprint.route('/persons/', methods=['GET']) def persons_proxy(viewcode, pid): """Proxy person for entity.""" - return entity_proxy(viewcode, pid, EntityType.PERSON) + return remote_entity_proxy(viewcode, pid, EntityType.PERSON) @blueprint.route('/corporate-bodies/', methods=['GET']) def corporate_bodies_proxy(viewcode, pid): """Proxy corporate bodies for entity.""" - return entity_proxy(viewcode, pid, EntityType.ORGANISATION) + return remote_entity_proxy(viewcode, pid, EntityType.ORGANISATION) -@api_blueprint.route('/entities/remote/search/', +@api_blueprint.route('/remote_entities/search/', defaults={'entity_type': 'agents'}) -@api_blueprint.route('/entities/remote/search//') -@api_blueprint.route('/entities/remote/search///') +@api_blueprint.route('/remote_entities/search//') +@api_blueprint.route('/remote_entities/search///') @check_logged_as_librarian def remote_search_proxy(entity_type, term): """Proxy to search entities on remote server. diff --git a/rero_ils/modules/unified_entities/serializers/__init__.py b/rero_ils/modules/entities/serializers/__init__.py similarity index 100% rename from rero_ils/modules/unified_entities/serializers/__init__.py rename to rero_ils/modules/entities/serializers/__init__.py diff --git a/rero_ils/modules/unified_entities/serializers/base.py b/rero_ils/modules/entities/serializers/base.py similarity index 100% rename from rero_ils/modules/unified_entities/serializers/base.py rename to rero_ils/modules/entities/serializers/base.py diff --git a/rero_ils/modules/ext.py b/rero_ils/modules/ext.py index 44591aea32..1f83c5d375 100644 --- a/rero_ils/modules/ext.py +++ b/rero_ils/modules/ext.py @@ -57,7 +57,8 @@ budget_is_active_changed from rero_ils.modules.collections.listener import enrich_collection_data from rero_ils.modules.ebooks.receivers import publish_harvested_records -from rero_ils.modules.entities.listener import enrich_entities_data +from rero_ils.modules.entities.remote_entities.listener import \ + enrich_remote_entities_data from rero_ils.modules.holdings.listener import enrich_holding_data, \ update_items_locations_and_types from rero_ils.modules.ill_requests.listener import enrich_ill_request_data @@ -290,7 +291,7 @@ def register_signals(self, app): before_record_index.connect(enrich_acq_order_line_data, sender=app) before_record_index.connect(enrich_collection_data, sender=app) before_record_index.connect(enrich_loan_data, sender=app) - before_record_index.connect(enrich_entities_data, sender=app) + before_record_index.connect(enrich_remote_entities_data, sender=app) before_record_index.connect(enrich_item_data, sender=app) before_record_index.connect(enrich_patron_data, sender=app) before_record_index.connect(enrich_holding_data, sender=app) diff --git a/rero_ils/modules/indexer_utils.py b/rero_ils/modules/indexer_utils.py index e818c9821e..c90e0d368a 100644 --- a/rero_ils/modules/indexer_utils.py +++ b/rero_ils/modules/indexer_utils.py @@ -41,8 +41,8 @@ def record_to_index(record): # authorities specific transformation if re.search(r'/mef/', schema): - schema = re.sub(r'/mef/', '/entities/', schema) - schema = re.sub(r'mef-contribution', 'entity', schema) + schema = re.sub(r'/mef/', '/remote_entities/', schema) + schema = re.sub(r'mef-contribution', 'remote_entity', schema) index, doc_type = schema_to_index(schema, index_names=index_names) if index and doc_type: diff --git a/rero_ils/query.py b/rero_ils/query.py index d5feafa39a..6ef10a17ac 100644 --- a/rero_ils/query.py +++ b/rero_ils/query.py @@ -204,7 +204,7 @@ def search_factory_for_holdings_and_items(view, search): return search -def entity_view_search_factory(self, search, query_parser=None): +def remote_entity_view_search_factory(self, search, query_parser=None): """Search factory with view code parameter.""" view = request.args.get( 'view', current_app.config.get('RERO_ILS_SEARCH_GLOBAL_VIEW_CODE')) diff --git a/scripts/setup b/scripts/setup index 9c38e22e4a..707309aece 100755 --- a/scripts/setup +++ b/scripts/setup @@ -78,7 +78,7 @@ invert_warning_option() { DEPLOYMENT=false -LOADENTITIES=false +LOAD_REMOTE_ENTITIES=false CREATE_ITEMS_HOLDINGS_SMALL=false CREATE_ITEMS_HOLDINGS_BIG=false STOP_EXECUTION=true @@ -117,7 +117,7 @@ do LOADEXTRAFILES=true ;; -E|--entities) - LOADENTITIES=true + LOAD_REMOTE_ENTITIES=true ;; -s|--create_items_holdings_small) CREATE_ITEMS_HOLDINGS_SMALL=true @@ -340,33 +340,33 @@ eval ${PREFIX} invenio reroils index run --raise-on-error # export RERO_ILS_MEF_AGENTS_URL=https://mef.rero.ch/api/agents # invenio reroils utils marc21tojson -t rero ${DATA_PATH}/documents_${SIZE}.xml ${DATA_PATH}/documents_${SIZE}.json ${DATA_PATH}/documents_${SIZE}_errors.xml -v -r # Save the entities after setup for later use. -# invenio reroils utils export -t ent -o ${DATA_PATH}/entities_${SIZE}.json -v +# invenio reroils utils export -t rement -o ${DATA_PATH}/remote_entities_${SIZE}.json -v if ${DEPLOYMENT} then DOCUMENTS=${DATA_PATH}/documents_big.json ITEMS=${DATA_PATH}/items_big.json HOLDINGS=${DATA_PATH}/holdings_big.json - ENTITIES=${DATA_PATH}/entities_big.json + REMOTE_ENTITIES=${DATA_PATH}/remote_entities_big.json else DOCUMENTS=${DATA_PATH}/documents_small.json ITEMS=${DATA_PATH}/items_small.json HOLDINGS=${DATA_PATH}/holdings_small.json - ENTITIES=${DATA_PATH}/entities_small.json + REMOTE_ENTITIES=${DATA_PATH}/remote_entities_small.json fi if ${LOADCONTRIBUTIONS} then - info_msg "- ENTITIES: ${ENTITIES} ${CREATE_LAZY} ${DONT_STOP}" - eval ${PREFIX} invenio reroils fixtures create --pid_type ent --schema 'https://bib.rero.ch/schemas/entities/entity-v0.0.1.json' ${ENTITIES} --append ${CREATE_LAZY} ${DONT_STOP} + info_msg "- REMOTE REMOTE_ENTITIES: ${REMOTE_ENTITIES} ${CREATE_LAZY} ${DONT_STOP}" + eval ${PREFIX} invenio reroils fixtures create --pid_type rement --schema 'https://bib.rero.ch/schemas/remote_entities/remote_entity-v0.0.1.json' ${REMOTE_ENTITIES} --append ${CREATE_LAZY} ${DONT_STOP} info_msg "Indexing Entities:" - eval ${PREFIX} invenio reroils index reindex -t ent --yes-i-know + eval ${PREFIX} invenio reroils index reindex -t rement --yes-i-know if [ ${INDEX_PARALLEL} -gt 0 ]; then eval ${PREFIX} invenio reroils index run -d -c ${INDEX_PARALLEL} --raise-on-error fi eval ${PREFIX} invenio reroils index run --raise-on-error if [ ${INDEX_PARALLEL} -gt 0 ]; then - eval ${PREFIX} invenio reroils index reindex_missing -t ent -v + eval ${PREFIX} invenio reroils index reindex_missing -t rement -v fi # else # info_msg "- Contributions from MEF: ${DOCUMENTS} ${CREATE_LAZY} ${ENQUEUE}" @@ -439,13 +439,13 @@ fi # index entities # We have to reindex entities to get the organisations pids indexed correctly. -eval ${PREFIX} invenio reroils index reindex -t ent --yes-i-know +eval ${PREFIX} invenio reroils index reindex -t rement --yes-i-know if [ ${INDEX_PARALLEL} -gt 0 ]; then eval ${PREFIX} invenio reroils index run -d -c ${INDEX_PARALLEL} --raise-on-error fi eval ${PREFIX} invenio reroils index run --raise-on-error if [ ${INDEX_PARALLEL} -gt 0 ]; then - eval ${PREFIX} invenio reroils index reindex_missing -t ent -v + eval ${PREFIX} invenio reroils index reindex_missing -t rement -v fi info_msg "- Local fields ${DATA_PATH}/local_fields.json ${CREATE_LAZY} ${DONT_STOP}" diff --git a/setup.py b/setup.py index e691c29c05..dd1f2e033c 100644 --- a/setup.py +++ b/setup.py @@ -89,11 +89,11 @@ def run(self): 'invenio_i18n = invenio_i18n:InvenioI18N' ], 'invenio_base.api_converters': [ - 'dummypid = rero_ils.converters:NoopPIDConverter', + 'nooppid = rero_ils.converters:NoopPIDConverter', ], 'invenio_base.blueprints': [ 'collections = rero_ils.modules.collections.views:blueprint', - 'entities = rero_ils.modules.entities.views:blueprint', + 'remote_entities = rero_ils.modules.entities.remote_entities.views:blueprint', 'documents = rero_ils.modules.documents.views:blueprint', 'holdings = rero_ils.modules.holdings.views:blueprint', 'ill_requests = rero_ils.modules.ill_requests.views:blueprint', @@ -110,7 +110,7 @@ def run(self): 'acq_receipts = rero_ils.modules.acquisition.acq_receipts.views:api_blueprint', 'api_documents = rero_ils.modules.documents.views:api_blueprint', 'circ_policies = rero_ils.modules.circ_policies.views:blueprint', - 'entities = rero_ils.modules.entities.views:api_blueprint', + 'remote_entities = rero_ils.modules.entities.remote_entities.views:api_blueprint', 'holdings = rero_ils.modules.holdings.api_views:api_blueprint', 'item_types = rero_ils.modules.item_types.views:blueprint', 'items = rero_ils.modules.items.views:api_blueprint', @@ -152,7 +152,7 @@ def run(self): ], 'flask.commands': [ 'apiharvester = rero_ils.modules.apiharvester.cli:apiharvester', - 'entity = rero_ils.modules.entities.cli:entity', + 'remote_entity = rero_ils.modules.entities.remote_entities.cli:remote_entity', 'monitoring = rero_ils.modules.monitoring.cli:monitoring', 'notifications = rero_ils.modules.notifications.cli:notifications', 'oaiharvester = rero_ils.modules.ebooks.cli:oaiharvester', @@ -183,7 +183,7 @@ def run(self): 'local_entity = rero_ils.modules.local_entities.models', 'local_fields = rero_ils.modules.local_fields.models', 'locations = rero_ils.modules.locations.models', - 'entity = rero_ils.modules.entities.models', + 'remote_entity = rero_ils.modules.entities.remote_entities.models', 'notifications = rero_ils.modules.notifications.models', 'organisations = rero_ils.modules.organisations.models', 'patron_transaction_events = rero_ils.modules.patron_transaction_events.models', @@ -205,7 +205,7 @@ def run(self): 'budget_id = rero_ils.modules.acquisition.budgets.api:budget_id_minter', 'circ_policy_id = rero_ils.modules.circ_policies.api:circ_policy_id_minter', 'collection_id = rero_ils.modules.collections.api:collection_id_minter', - 'entity_id = rero_ils.modules.entities.api:entity_id_minter', + 'remote_entity_id = rero_ils.modules.entities.remote_entities.api:remote_entity_id_minter', 'document_id = rero_ils.modules.documents.api:document_id_minter', 'holding_id = rero_ils.modules.holdings.api:holding_id_minter', 'ill_request_id = rero_ils.modules.ill_requests.api:ill_request_id_minter', @@ -236,7 +236,7 @@ def run(self): 'circ_policy_id = rero_ils.modules.circ_policies.api:circ_policy_id_fetcher', 'collection_id = rero_ils.modules.collections.api:collection_id_fetcher', 'document_id = rero_ils.modules.documents.api:document_id_fetcher', - 'entity_id = rero_ils.modules.entities.api:entity_id_fetcher', + 'remote_entity_id = rero_ils.modules.entities.remote_entities.api:remote_entity_id_fetcher', 'local_entity_id = rero_ils.modules.local_entities.api:local_entity_id_fetcher', 'holding_id = rero_ils.modules.holdings.api:holding_id_fetcher', 'ill_request_id = rero_ils.modules.ill_requests.api:ill_request_id_fetcher', @@ -267,7 +267,7 @@ def run(self): 'circ_policies = rero_ils.modules.circ_policies.jsonschemas', 'collections = rero_ils.modules.collections.jsonschemas', 'common = rero_ils.jsonschemas', - 'entities = rero_ils.modules.entities.jsonschemas', + 'remote_entities = rero_ils.modules.entities.remote_entities.jsonschemas', 'documents = rero_ils.modules.documents.jsonschemas', 'holdings = rero_ils.modules.holdings.jsonschemas', 'ill_requests = rero_ils.modules.ill_requests.jsonschemas', @@ -300,7 +300,7 @@ def run(self): 'budgets = rero_ils.modules.acquisition.budgets.mappings', 'circ_policies = rero_ils.modules.circ_policies.mappings', 'collections = rero_ils.modules.collections.mappings', - 'entities = rero_ils.modules.entities.mappings', + 'remote_entities = rero_ils.modules.entities.remote_entities.mappings', 'documents = rero_ils.modules.documents.mappings', 'holdings = rero_ils.modules.holdings.mappings', 'ill_requests = rero_ils.modules.ill_requests.mappings', @@ -347,7 +347,7 @@ def run(self): 'acq_receipt_lines = rero_ils.modules.acquisition.acq_receipt_lines.jsonresolver', 'budgets = rero_ils.modules.acquisition.budgets.jsonresolver', 'collections = rero_ils.modules.collections.jsonresolver', - 'entities = rero_ils.modules.entities.jsonresolver', + 'remote_entities = rero_ils.modules.entities.remote_entities.jsonresolver', 'documents = rero_ils.modules.documents.jsonresolver', 'holdings = rero_ils.modules.holdings.jsonresolver', 'ill_requests = rero_ils.modules.ill_requests.jsonresolver', diff --git a/tests/api/entities/test_entities_rest.py b/tests/api/entities/test_entities_rest.py index 49fd90f7d6..4954fe49d2 100644 --- a/tests/api/entities/test_entities_rest.py +++ b/tests/api/entities/test_entities_rest.py @@ -16,19 +16,26 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -"""Tests `Entity` resource REST API.""" +"""Tests `LocalEntity` resource REST API.""" -import mock from flask import url_for -from utils import get_json, mock_response, postdata, to_relative_url +from utils import get_json, postdata -from rero_ils.modules.entities.models import EntityType - -def test_entities_permissions(client, entity_person, json_header): +def test_entities_permissions(client, entity_person, + local_entity_person, json_header): """Test record retrieval.""" - item_url = url_for('invenio_records_rest.ent_item', pid_value='ent_pers') + item_url = url_for('invenio_records_rest.ent_item', + pid_value='locent_pers') res = client.get(item_url) + assert res.status_code == 401 + + item_url = url_for('invenio_records_rest.ent_item', + pid_value='ent_pers') + res = client.get(item_url) + assert res.status_code == 401 + + res = client.get(url_for('invenio_records_rest.ent_list')) assert res.status_code == 200 res, _ = postdata(client, 'invenio_records_rest.ent_list', {}) @@ -44,103 +51,36 @@ def test_entities_permissions(client, entity_person, json_header): assert res.status_code == 401 -def test_entities_get(client, entity_person): +def test_entities_get(client, entity_person, local_entity_person): """Test record retrieval.""" - item_url = url_for('invenio_records_rest.ent_item', pid_value='ent_pers') - + item_url = url_for('invenio_records_rest.ent_item', + pid_value='locent_pers') res = client.get(item_url) - assert res.status_code == 200 - assert res.headers['ETag'] == f'"{entity_person.revision_id}"' - - data = get_json(res) - assert entity_person.dumps() == data['metadata'] - assert entity_person.dumps() == data['metadata'] + assert res.status_code == 401 - # Check metadata - for k in ['created', 'updated', 'metadata', 'links']: - assert k in data + item_url = url_for('invenio_records_rest.ent_item', + pid_value='ent_pers') + res = client.get(item_url) + assert res.status_code == 401 - # Check self links - res = client.get(to_relative_url(data['links']['self'])) + res = client.get(url_for('invenio_records_rest.ent_list')) assert res.status_code == 200 - assert data == get_json(res) - assert entity_person.dumps() == data['metadata'] + # Check remote/local entities self links + data = get_json(res) + pid_link_map = { + 'ent_pers': 'http://localhost/remote_entities/ent_pers', + 'locent_pers': 'http://localhost/local_entities/locent_pers' + } + for hit in data['hits']['hits']: + assert hit['links']['self'] == pid_link_map.get(hit['id']) + + # search entity record list_url = url_for('invenio_records_rest.ent_list', pid='ent_pers') res = client.get(list_url) assert res.status_code == 200 - data = get_json(res) - entity_person = entity_person.replace_refs() - entity_person['organisations'] = entity_person.organisation_pids - entity_person['type'] = EntityType.PERSON - entity_person['type'] = EntityType.PERSON - assert data['hits']['hits'][0]['metadata'] == entity_person.replace_refs() - - -@mock.patch('rero_ils.modules.decorators.login_and_librarian', - mock.MagicMock()) -@mock.patch('requests.request') -def test_remote_search_proxy( - mock_es_concept_get, app, client, - mef_concept2_es_response, mef_agents1_es_response -): - """Test entities search on remote servers.""" - # TEST#1 :: Concepts - # All results must include a `type` key if a root `metadata` field - # exists. - mock_es_concept_get.return_value = mock_response( - json_data=mef_concept2_es_response) - - response = client.get(url_for( - 'api_entities.remote_search_proxy', - entity_type='concepts-genreForm', - term='side-car' - )) - assert response.status_code == 200 - assert all( - hit.get('metadata', {}).get('type') == EntityType.TOPIC - for hit in response.json['hits']['hits'] - if 'metadata' in hit - ) - assert all( - hit.get('metadata', {}).get('type') == EntityType.TOPIC - for hit in response.json['hits']['hits'] - if 'metadata' in hit - ) - # TEST#2 :: Agents - # All result must include a `identifiedBy` object if a root - mock_es_concept_get.return_value = mock_response( - json_data=mef_agents1_es_response) - response = client.get(url_for( - 'api_entities.remote_search_proxy', - entity_type='agents', - term='UCLouvain' - )) - identifier = mef_agents1_es_response['hits']['hits'][0][ - 'metadata']['idref']['identifier'] - assert identifier == response.json['hits']['hits'][0][ - 'metadata']['idref']['identifiedBy'][0]['value'] - - # TEST#3 :: Unknown MEF search type - # Try to execute a search on a not-configured MEF category. It should be - # raised a `ValueError` caught by flask to return an HTTP 400 response - category = 'unknown_category' - response = client.get(url_for( - 'api_entities.remote_search_proxy', - entity_type=category, - term='search_term' - )) - assert response.status_code == 400 - assert response.json['message'] == \ - f'Unable to find a MEF factory for {category}' - - # TEST#4 :: Simulate MEF errors - # Simulate than MEF call return an HTTP error and check the response. - mock_es_concept_get.return_value = mock_response(status=404) - response = client.get(url_for( - 'api_entities.remote_search_proxy', - entity_type='agents', - term='UCLouvain' - )) - assert response.status_code == 404 + # search local entity record + list_url = url_for('invenio_records_rest.ent_list', pid='locent_pers') + res = client.get(list_url) + assert res.status_code == 200 diff --git a/tests/api/unified_entities/test_unified_entities_search.py b/tests/api/entities/test_entities_search.py similarity index 93% rename from tests/api/unified_entities/test_unified_entities_search.py rename to tests/api/entities/test_entities_search.py index 7bd9fe1302..3694940420 100644 --- a/tests/api/unified_entities/test_unified_entities_search.py +++ b/tests/api/entities/test_entities_search.py @@ -29,7 +29,7 @@ def test_unified_entity_search(client, entity_person, local_entity_person, # unified entity search list_url = url_for( - 'invenio_records_rest.unient_list', + 'invenio_records_rest.ent_list', q='"Loy, Georg"', simple='1' ) @@ -39,7 +39,7 @@ def test_unified_entity_search(client, entity_person, local_entity_person, # unified entity search organisation list_url = url_for( - 'invenio_records_rest.unient_list', + 'invenio_records_rest.ent_list', q='"Convegno internazionale di italianistica Craiova"', simple='1' ) @@ -49,7 +49,7 @@ def test_unified_entity_search(client, entity_person, local_entity_person, # empty search list_url = url_for( - 'invenio_records_rest.unient_list', + 'invenio_records_rest.ent_list', q='"Nebehay, Christian Michael"', simple='1' ) diff --git a/tests/api/local_entities/test_local_entities_permissions.py b/tests/api/local_entities/test_local_entities_permissions.py index be41a455e0..344e20b1f6 100644 --- a/tests/api/local_entities/test_local_entities_permissions.py +++ b/tests/api/local_entities/test_local_entities_permissions.py @@ -21,7 +21,7 @@ from flask_security.utils import login_user from utils import check_permission -from rero_ils.modules.local_entities.permissions import \ +from rero_ils.modules.entities.local_entities.permissions import \ LocalEntityPermissionPolicy diff --git a/tests/api/local_entities/test_local_entities_rest.py b/tests/api/local_entities/test_local_entities_rest.py index 4b8a56073d..37d4d3636d 100644 --- a/tests/api/local_entities/test_local_entities_rest.py +++ b/tests/api/local_entities/test_local_entities_rest.py @@ -25,7 +25,7 @@ VerifyRecordPermissionPatch from rero_ils.modules.entities.models import EntityType -from rero_ils.modules.local_entities.api import LocalEntity +from rero_ils.modules.entities.local_entities.api import LocalEntity def test_local_entities_permissions(client, roles, local_entity_person, @@ -40,7 +40,7 @@ def test_local_entities_permissions(client, roles, local_entity_person, assert res.status_code == 401 client.put( - url_for('invenio_records_rest.ent_item', pid_value='locent_pers'), + url_for('invenio_records_rest.locent_item', pid_value='locent_pers'), data={}, headers=json_header ) diff --git a/tests/api/entities/test_entities_permissions.py b/tests/api/remote_entities/test_remote_entities_permissions.py similarity index 83% rename from tests/api/entities/test_entities_permissions.py rename to tests/api/remote_entities/test_remote_entities_permissions.py index eda3df34ef..c71d50e4b8 100644 --- a/tests/api/entities/test_entities_permissions.py +++ b/tests/api/remote_entities/test_remote_entities_permissions.py @@ -22,20 +22,21 @@ from invenio_accounts.testutils import login_user_via_session from utils import check_permission, get_json -from rero_ils.modules.entities.permissions import EntityPermissionPolicy +from rero_ils.modules.entities.remote_entities.permissions import \ + RemoteEntityPermissionPolicy -def test_entity_permissions_api(client, patron_martigny, - entity_person, - librarian_martigny): +def test_remote_entity_permissions_api(client, patron_martigny, + entity_person, + librarian_martigny): """Test entities permissions api.""" prs_permissions_url = url_for( 'api_blueprint.permissions', - route_name='entities' + route_name='remote_entities' ) prs_real_permission_url = url_for( 'api_blueprint.permissions', - route_name='entities', + route_name='remote_entities', record_pid=entity_person.pid ) @@ -60,11 +61,11 @@ def test_entity_permissions_api(client, patron_martigny, assert not data['delete']['can'] -def test_entity_permissions(patron_martigny, - librarian_martigny, - system_librarian_martigny): +def test_remote_entity_permissions(patron_martigny, + librarian_martigny, + system_librarian_martigny): """Test entity permissions class.""" - permission_policy = EntityPermissionPolicy + permission_policy = RemoteEntityPermissionPolicy # Anonymous user # - Allow search/read actions on any entity diff --git a/tests/api/remote_entities/test_remote_entities_rest.py b/tests/api/remote_entities/test_remote_entities_rest.py new file mode 100644 index 0000000000..aae71d0b0d --- /dev/null +++ b/tests/api/remote_entities/test_remote_entities_rest.py @@ -0,0 +1,141 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Tests `Entity` resource REST API.""" + +import mock +from flask import url_for +from utils import get_json, mock_response, postdata, to_relative_url + +from rero_ils.modules.entities.models import EntityType + + +def test_remote_entities_permissions(client, entity_person, json_header): + """Test record retrieval.""" + item_url = url_for('invenio_records_rest.rement_item', + pid_value='ent_pers') + res = client.get(item_url) + assert res.status_code == 200 + + res, _ = postdata(client, 'invenio_records_rest.rement_list', {}) + assert res.status_code == 401 + + client.put( + url_for('invenio_records_rest.rement_item', pid_value='ent_pers'), + data={}, + headers=json_header + ) + + res = client.delete(item_url) + assert res.status_code == 401 + + +def test_remote_entities_get(client, entity_person): + """Test record retrieval.""" + item_url = url_for('invenio_records_rest.rement_item', + pid_value='ent_pers') + + res = client.get(item_url) + assert res.status_code == 200 + assert res.headers['ETag'] == f'"{entity_person.revision_id}"' + + data = get_json(res) + assert entity_person.dumps() == data['metadata'] + + # Check metadata + for k in ['created', 'updated', 'metadata', 'links']: + assert k in data + + # Check self links + res = client.get(to_relative_url(data['links']['self'])) + assert res.status_code == 200 + assert data == get_json(res) + assert entity_person.dumps() == data['metadata'] + + list_url = url_for('invenio_records_rest.rement_list', pid='ent_pers') + res = client.get(list_url) + assert res.status_code == 200 + data = get_json(res) + entity_person = entity_person.replace_refs() + entity_person['organisations'] = entity_person.organisation_pids + entity_person['type'] = EntityType.PERSON + assert data['hits']['hits'][0]['metadata'] == entity_person.replace_refs() + + +@mock.patch('rero_ils.modules.decorators.login_and_librarian', + mock.MagicMock()) +@mock.patch('requests.request') +def test_remote_search_proxy( + mock_es_concept_get, app, client, + mef_concept2_es_response, mef_agents1_es_response +): + """Test entities search on remote servers.""" + # TEST#1 :: Concepts + # All results must include a `type` key if a root `metadata` field + # exists. + mock_es_concept_get.return_value = mock_response( + json_data=mef_concept2_es_response) + + response = client.get(url_for( + 'api_remote_entities.remote_search_proxy', + entity_type='concepts-genreForm', + term='side-car' + )) + assert response.status_code == 200 + assert all( + hit.get('metadata', {}).get('type') == EntityType.TOPIC + for hit in response.json['hits']['hits'] + if 'metadata' in hit + ) + + # TEST#2 :: Agents + # All result must include a `identifiedBy` object if a root + mock_es_concept_get.return_value = mock_response( + json_data=mef_agents1_es_response) + response = client.get(url_for( + 'api_remote_entities.remote_search_proxy', + entity_type='agents', + term='UCLouvain' + )) + identifier = mef_agents1_es_response['hits']['hits'][0][ + 'metadata']['idref']['identifier'] + assert identifier == response.json['hits']['hits'][0][ + 'metadata']['idref']['identifiedBy'][0]['value'] + + # TEST#3 :: Unknown MEF search type + # Try to execute a search on a not-configured MEF category. It should be + # raised a `ValueError` caught by flask to return an HTTP 400 response + category = 'unknown_category' + response = client.get(url_for( + 'api_remote_entities.remote_search_proxy', + entity_type=category, + term='search_term' + )) + assert response.status_code == 400 + assert response.json['message'] == \ + f'Unable to find a MEF factory for {category}' + + # TEST#4 :: Simulate MEF errors + # Simulate than MEF call return an HTTP error and check the response. + mock_es_concept_get.return_value = mock_response(status=404) + response = client.get(url_for( + 'api_remote_entities.remote_search_proxy', + entity_type='agents', + term='UCLouvain' + )) + assert response.status_code == 404 diff --git a/tests/api/test_monitoring_rest.py b/tests/api/test_monitoring_rest.py index ad0d26b1d9..5f9ea35187 100644 --- a/tests/api/test_monitoring_rest.py +++ b/tests/api/test_monitoring_rest.py @@ -27,7 +27,8 @@ from invenio_db import db from utils import flush_index, get_json -from rero_ils.modules.entities.api import EntitiesSearch, Entity +from rero_ils.modules.entities.remote_entities.api import \ + RemoteEntitiesSearch, RemoteEntity from rero_ils.modules.utils import get_timestamp, set_timestamp @@ -47,7 +48,8 @@ def test_monitoring_es_db_counts(client): 'budg': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'budgets'}, 'cipo': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'circ_policies'}, 'coll': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'collections'}, - 'ent': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'entities'}, + 'rement': {'db': 0, 'db-es': 0, 'es': 0, + 'index': 'remote_entities'}, 'doc': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'documents'}, 'hold': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'holdings'}, 'illr': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'ill_requests'}, @@ -70,8 +72,7 @@ def test_monitoring_es_db_counts(client): 'ptty': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'patron_types'}, 'stat': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'stats'}, 'tmpl': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'templates'}, - 'unient': {'db': 0, 'db-es': 0, 'es': 0, - 'index': 'unified_entities'}, + 'ent': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'entities'}, 'vndr': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'vendors'}, } } @@ -84,34 +85,34 @@ def test_monitoring_check_es_db_counts(app, client, entity_person_data, assert res.status_code == 200 assert get_json(res) == {'data': {'status': 'green'}} - pers = Entity.create( + pers = RemoteEntity.create( data=entity_person_data, delete_pid=False, dbcommit=True, reindex=False) - flush_index(EntitiesSearch.Meta.index) + flush_index(RemoteEntitiesSearch.Meta.index) res = client.get(url_for('api_monitoring.check_es_db_counts', delay=0)) assert res.status_code == 200 assert get_json(res) == { 'data': {'status': 'red'}, 'errors': [{ 'code': 'DB_ES_COUNTER_MISMATCH', - 'details': 'There are 1 items from ent missing in ES.', + 'details': 'There are 1 items from rement missing in ES.', 'id': 'DB_ES_COUNTER_MISMATCH', 'links': { 'about': 'http://localhost/monitoring/check_es_db_counts', - 'ent': 'http://localhost/monitoring/missing_pids/ent' + 'rement': 'http://localhost/monitoring/missing_pids/rement' }, 'title': "DB items counts don't match ES items count." }] } # this view is only accessible by monitoring - res = client.get(url_for('api_monitoring.missing_pids', doc_type='ent')) + res = client.get(url_for('api_monitoring.missing_pids', doc_type='rement')) assert res.status_code == 401 login_user_via_session(client, system_librarian_martigny.user) - res = client.get(url_for('api_monitoring.missing_pids', doc_type='ent')) + res = client.get(url_for('api_monitoring.missing_pids', doc_type='rement')) assert res.status_code == 403 # give user superuser admin rights @@ -123,12 +124,12 @@ def test_monitoring_check_es_db_counts(app, client, entity_person_data, ) db.session.commit() res = client.get(url_for( - 'api_monitoring.missing_pids', doc_type='ent', delay=0)) + 'api_monitoring.missing_pids', doc_type='rement', delay=0)) assert res.status_code == 200 assert get_json(res) == { 'data': { 'DB': [], - 'ES': ['http://localhost/entities/ent_pers'], + 'ES': ['http://localhost/remote_entities/ent_pers'], 'ES duplicate': [] } } diff --git a/tests/api/unified_entities/test_unified_entities_rest.py b/tests/api/unified_entities/test_unified_entities_rest.py deleted file mode 100644 index e76e4c5a26..0000000000 --- a/tests/api/unified_entities/test_unified_entities_rest.py +++ /dev/null @@ -1,86 +0,0 @@ -# -*- coding: utf-8 -*- -# -# RERO ILS -# Copyright (C) 2019-2023 RERO -# Copyright (C) 2019-2023 UCLouvain -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, version 3 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -"""Tests `LocalEntity` resource REST API.""" - -from flask import url_for -from utils import get_json, postdata - - -def test_unified_entities_permissions(client, entity_person, - local_entity_person, json_header): - """Test record retrieval.""" - item_url = url_for('invenio_records_rest.unient_item', - pid_value='locent_pers') - res = client.get(item_url) - assert res.status_code == 401 - - item_url = url_for('invenio_records_rest.unient_item', - pid_value='ent_pers') - res = client.get(item_url) - assert res.status_code == 401 - - res = client.get(url_for('invenio_records_rest.unient_list')) - assert res.status_code == 200 - - res, _ = postdata(client, 'invenio_records_rest.unient_list', {}) - assert res.status_code == 401 - - client.put( - url_for('invenio_records_rest.unient_item', pid_value='unient_pers'), - data={}, - headers=json_header - ) - - res = client.delete(item_url) - assert res.status_code == 401 - - -def test_unified_entities_get(client, entity_person, local_entity_person): - """Test record retrieval.""" - item_url = url_for('invenio_records_rest.unient_item', - pid_value='locent_pers') - res = client.get(item_url) - assert res.status_code == 401 - - item_url = url_for('invenio_records_rest.unient_item', - pid_value='ent_pers') - res = client.get(item_url) - assert res.status_code == 401 - - res = client.get(url_for('invenio_records_rest.unient_list')) - assert res.status_code == 200 - - # Check remote/local entities self links - data = get_json(res) - pid_link_map = { - 'ent_pers': 'http://localhost/entities/ent_pers', - 'locent_pers': 'http://localhost/local_entities/locent_pers' - } - for hit in data['hits']['hits']: - assert hit['links']['self'] == pid_link_map.get(hit['id']) - - # search entity record - list_url = url_for('invenio_records_rest.unient_list', pid='ent_pers') - res = client.get(list_url) - assert res.status_code == 200 - - # search local entity record - list_url = url_for('invenio_records_rest.unient_list', pid='locent_pers') - res = client.get(list_url) - assert res.status_code == 200 diff --git a/tests/data/data.json b/tests/data/data.json index 809b57d15f..bd22d89a92 100644 --- a/tests/data/data.json +++ b/tests/data/data.json @@ -1523,7 +1523,7 @@ ] }, "ent_pers": { - "$schema": "https://bib.rero.ch/schemas/entities/entity-v0.0.1.json", + "$schema": "https://bib.rero.ch/schemas/remote_entities/remote_entity-v0.0.1.json", "gnd": { "$schema": "https://mef.test.rero.ch/schemas/gnd/gnd-contribution-v0.0.1.json", "bf:Agent": "bf:Person", @@ -1604,7 +1604,7 @@ "type": "bf:Person" }, "ent_org": { - "$schema": "https://bib.rero.ch/schemas/entities/entity-v0.0.1.json", + "$schema": "https://bib.rero.ch/schemas/remote_entities/remote_entity-v0.0.1.json", "gnd": { "$schema": "https://mef.rero.ch/schemas/gnd/gnd-contribution-v0.0.1.json", "bf:Agent": "bf:Organisation", @@ -1638,7 +1638,7 @@ "viaf_pid": "3117153063217219320007" }, "ent_topic": { - "$schema": "https://bib.rero.ch/schemas/entities/entity-v0.0.1.json", + "$schema": "https://bib.rero.ch/schemas/remote_entities/remote_entity-v0.0.1.json", "idref": { "$schema": "https://mef.rero.ch/schemas/concepts_idref/idref-concept-v0.0.1.json", "authorized_access_point": "Swing (jazz)", diff --git a/tests/data/mef.json b/tests/data/mef.json index 24a8d58bcb..9b03dbbd83 100644 --- a/tests/data/mef.json +++ b/tests/data/mef.json @@ -1,6 +1,6 @@ { "concept_1": { - "$schema": "https://bib.rero.ch/schemas/entities/entity-v0.0.1.json", + "$schema": "https://bib.rero.ch/schemas/remote_entities/remote_entity-v0.0.1.json", "type": "bf:Topic", "idref": { "$schema": "https://mef.rero.ch/schemas/concepts_idref/idref-concept-v0.0.1.json", diff --git a/tests/fixtures/mef.py b/tests/fixtures/mef.py index 24ec9ef3bd..dfe90981c9 100644 --- a/tests/fixtures/mef.py +++ b/tests/fixtures/mef.py @@ -20,9 +20,8 @@ from copy import deepcopy import pytest - -from rero_ils.modules.entities.api import EntitiesSearch, Entity - +from rero_ils.modules.entities.remote_entities.api import RemoteEntity, \ + RemoteEntitiesSearch @pytest.fixture(scope="module") def mef_concept1_data(mef_entities): @@ -33,13 +32,13 @@ def mef_concept1_data(mef_entities): @pytest.fixture(scope="module") def mef_concept1(mef_concept1_data): """Load MEF concept_1 data.""" - entity = Entity.create( + entity = RemoteEntity.create( data=mef_concept1_data, dbcommit=True, reindex=True, delete_pid=False ) - EntitiesSearch.flush_and_refresh() + RemoteEntitiesSearch.flush_and_refresh() return entity diff --git a/tests/fixtures/metadata.py b/tests/fixtures/metadata.py index ec73a0176f..744b091acf 100644 --- a/tests/fixtures/metadata.py +++ b/tests/fixtures/metadata.py @@ -27,10 +27,11 @@ from utils import flush_index, mock_response from rero_ils.modules.documents.api import Document, DocumentsSearch -from rero_ils.modules.entities.api import EntitiesSearch, Entity +from rero_ils.modules.entities.remote_entities.api import \ + RemoteEntitiesSearch, RemoteEntity from rero_ils.modules.holdings.api import Holding, HoldingsSearch from rero_ils.modules.items.api import Item, ItemsSearch -from rero_ils.modules.local_entities.api import LocalEntitiesSearch, \ +from rero_ils.modules.entities.local_entities.api import LocalEntitiesSearch, \ LocalEntity from rero_ils.modules.local_fields.api import LocalField, LocalFieldsSearch from rero_ils.modules.operation_logs.api import OperationLog @@ -292,12 +293,12 @@ def entity_person_response_data(entity_topic_data): @pytest.fixture(scope="module") def entity_topic(app, entity_topic_data): """Load contribution person record.""" - cont = Entity.create( + cont = RemoteEntity.create( data=entity_topic_data, delete_pid=False, dbcommit=True, reindex=True) - flush_index(EntitiesSearch.Meta.index) + flush_index(RemoteEntitiesSearch.Meta.index) return cont @@ -335,12 +336,12 @@ def entity_person_response_data(entity_person_data): @pytest.fixture(scope="module") def entity_person(app, entity_person_data): """Load contribution person record.""" - cont = Entity.create( + cont = RemoteEntity.create( data=entity_person_data, delete_pid=False, dbcommit=True, reindex=True) - flush_index(EntitiesSearch.Meta.index) + flush_index(RemoteEntitiesSearch.Meta.index) return cont @@ -410,12 +411,12 @@ def entity_organisation_response_data(entity_organisation_data): @pytest.fixture(scope="module") def entity_organisation(app, entity_organisation_data): """Create mef contribution organisation record.""" - org = Entity.create( + org = RemoteEntity.create( data=entity_organisation_data, delete_pid=False, dbcommit=True, reindex=True) - flush_index(EntitiesSearch.Meta.index) + flush_index(RemoteEntitiesSearch.Meta.index) return org @@ -449,12 +450,12 @@ def person2_response_data(person2_data): @pytest.fixture(scope="module") def person2(app, person2_data): """Create mef person record.""" - pers = Entity.create( + pers = RemoteEntity.create( data=person2_data, delete_pid=False, dbcommit=True, reindex=True) - flush_index(EntitiesSearch.Meta.index) + flush_index(RemoteEntitiesSearch.Meta.index) return pers diff --git a/tests/ui/documents/test_documents_api.py b/tests/ui/documents/test_documents_api.py index d116badcb1..f70a896f90 100644 --- a/tests/ui/documents/test_documents_api.py +++ b/tests/ui/documents/test_documents_api.py @@ -32,9 +32,11 @@ document_id_fetcher from rero_ils.modules.documents.models import DocumentIdentifier from rero_ils.modules.ebooks.tasks import create_records -from rero_ils.modules.entities.api import EntitiesSearch, Entity from rero_ils.modules.entities.models import EntityType -from rero_ils.modules.entities.utils import extract_data_from_mef_uri +from rero_ils.modules.entities.remote_entities.api import \ + RemoteEntitiesSearch, RemoteEntity +from rero_ils.modules.entities.remote_entities.utils import \ + extract_data_from_mef_uri from rero_ils.modules.tasks import process_bulk_queue @@ -82,7 +84,7 @@ def test_document_create_with_mef( mock_contributions_mef_get.return_value = mock_response( json_data=entity_person_response_data ) - assert EntitiesSearch().count() == 0 + assert RemoteEntitiesSearch().count() == 0 doc = Document.create( data=deepcopy(document_data_ref), delete_pid=False, dbcommit=False, reindex=False) @@ -94,15 +96,15 @@ def test_document_create_with_mef( assert hit['contribution'][0]['entity']['pid'] == entity_person_data['pid'] assert hit['contribution'][0]['entity']['primary_source'] == 'rero' - assert EntitiesSearch().count() == 1 - contrib = Entity.get_record_by_pid(entity_person_data['pid']) + assert RemoteEntitiesSearch().count() == 1 + contrib = RemoteEntity.get_record_by_pid(entity_person_data['pid']) contrib.delete_from_index() doc.delete_from_index() db.session.rollback() assert not Document.get_record_by_pid(doc.get('pid')) - assert not Entity.get_record_by_pid(entity_person_data['pid']) - assert EntitiesSearch().count() == 0 + assert not RemoteEntity.get_record_by_pid(entity_person_data['pid']) + assert RemoteEntitiesSearch().count() == 0 with pytest.raises(ValidationError): doc = Document.create( @@ -110,8 +112,8 @@ def test_document_create_with_mef( delete_pid=False, dbcommit=True, reindex=True) assert not Document.get_record_by_pid(doc.get('pid')) - assert not Entity.get_record_by_pid(entity_person_data['pid']) - assert EntitiesSearch().count() == 0 + assert not RemoteEntity.get_record_by_pid(entity_person_data['pid']) + assert RemoteEntitiesSearch().count() == 0 data = deepcopy(document_data_ref) contrib = data.pop('contribution') doc = Document.create( @@ -125,15 +127,15 @@ def test_document_create_with_mef( doc.pop('type') doc.update(doc, commit=True, dbcommit=True, reindex=True) assert Document.get_record_by_pid(doc.get('pid')) - assert not Entity.get_record_by_pid(entity_person_data['pid']) - assert EntitiesSearch().count() == 0 + assert not RemoteEntity.get_record_by_pid(entity_person_data['pid']) + assert RemoteEntitiesSearch().count() == 0 data = deepcopy(document_data_ref) doc.update(data, commit=True, dbcommit=False, reindex=False) doc.reindex() assert Document.get_record_by_pid(doc.get('pid')) - assert Entity.get_record_by_pid(entity_person_data['pid']) - assert EntitiesSearch().count() == 1 + assert RemoteEntity.get_record_by_pid(entity_person_data['pid']) + assert RemoteEntitiesSearch().count() == 1 doc.delete_from_index() db.session.rollback() @@ -161,10 +163,10 @@ def test_document_linked_subject( # - Check if the entity has been created # - Check if ES mapping is correct for this entity _, _type, _id = extract_data_from_mef_uri(entity_uri) - entity = Entity.get_entity(_type, _id) + entity = RemoteEntity.get_entity(_type, _id) assert _type in entity.get('sources') - es_record = EntitiesSearch().get_record_by_pid(entity.pid) + es_record = RemoteEntitiesSearch().get_record_by_pid(entity.pid) assert es_record['type'] == EntityType.TOPIC assert es_record[_type]['pid'] == _id diff --git a/tests/ui/entities/test_entities_api.py b/tests/ui/remote_entities/test_entities_api.py similarity index 87% rename from tests/ui/entities/test_entities_api.py rename to tests/ui/remote_entities/test_entities_api.py index 2c76abda4a..e920f2fce2 100644 --- a/tests/ui/entities/test_entities_api.py +++ b/tests/ui/remote_entities/test_entities_api.py @@ -27,17 +27,17 @@ from utils import flush_index, mock_response from rero_ils.modules.documents.api import Document, DocumentsSearch -from rero_ils.modules.entities.api import EntitiesSearch, Entity, \ - entity_id_fetcher +from rero_ils.modules.entities.remote_entities.api import \ + RemoteEntitiesSearch, RemoteEntity, remote_entity_id_fetcher from rero_ils.modules.entities.replace import ReplaceIdentifiedBy -from rero_ils.modules.entities.sync import SyncEntity +from rero_ils.modules.entities.remote_entities.sync import SyncEntity -def test_entity_create(app, entity_person_data_tmp, caplog): +def test_remote_entity_create(app, entity_person_data_tmp, caplog): """Test MEF entity creation.""" - pers = Entity.get_record_by_pid('1') + pers = RemoteEntity.get_record_by_pid('1') assert not pers - pers = Entity.create( + pers = RemoteEntity.create( entity_person_data_tmp, dbcommit=True, delete_pid=True @@ -45,15 +45,15 @@ def test_entity_create(app, entity_person_data_tmp, caplog): assert pers == entity_person_data_tmp assert pers.get('pid') == '1' - pers = Entity.get_record_by_pid('1') + pers = RemoteEntity.get_record_by_pid('1') assert pers == entity_person_data_tmp - fetched_pid = entity_id_fetcher(pers.id, pers) + fetched_pid = remote_entity_id_fetcher(pers.id, pers) assert fetched_pid.pid_value == '1' - assert fetched_pid.pid_type == 'ent' + assert fetched_pid.pid_type == 'rement' entity_person_data_tmp['viaf_pid'] = '1234' - Entity.create(entity_person_data_tmp, dbcommit=True, delete_pid=True) - pers = Entity.get_record_by_pid('2') + RemoteEntity.create(entity_person_data_tmp, dbcommit=True, delete_pid=True) + pers = RemoteEntity.get_record_by_pid('2') assert pers.get('viaf_pid') == '1234' assert pers.organisation_pids == [] @@ -62,35 +62,35 @@ def test_entity_create(app, entity_person_data_tmp, caplog): # test the messages from current_app.logger assert caplog.records[0].name == 'elasticsearch' assert caplog.record_tuples[1] == ( - 'invenio', 30, 'Can not delete from index Entity: 2' + 'invenio', 30, 'Can not delete from index RemoteEntity: 2' ) @mock.patch('requests.Session.get') -def test_entity_mef_create( +def test_remote_entity_mef_create( mock_contributions_mef_get, app, mef_agents_url, entity_person_data_tmp, entity_person_response_data ): """Test MEF contribution creation.""" - count = Entity.count() + count = RemoteEntity.count() mock_contributions_mef_get.return_value = mock_response( json_data=entity_person_response_data ) - pers_mef, online = Entity.get_record_by_ref( + pers_mef, online = RemoteEntity.get_record_by_ref( f'{mef_agents_url}/rero/A017671081') - flush_index(EntitiesSearch.Meta.index) + flush_index(RemoteEntitiesSearch.Meta.index) assert pers_mef == entity_person_data_tmp assert online - assert Entity.count() == count + 1 + assert RemoteEntity.count() == count + 1 pers_mef.pop('idref') pers_mef['sources'] = ['gnd'] pers_mef.replace(pers_mef, dbcommit=True) - pers_db, online = Entity.get_record_by_ref( + pers_db, online = RemoteEntity.get_record_by_ref( f'{mef_agents_url}/gnd/13343771X') assert pers_db['sources'] == ['gnd'] assert not online # remove created contribution - Entity.get_record_by_pid(entity_person_data_tmp['pid']).delete( + RemoteEntity.get_record_by_pid(entity_person_data_tmp['pid']).delete( True, True, True) @@ -104,13 +104,13 @@ def test_sync_contribution( sync_entity = SyncEntity(log_dir=log_path) assert sync_entity - pers = Entity.create( + pers = RemoteEntity.create( entity_person_data_tmp, dbcommit=True, reindex=True, delete_pid=True ) - flush_index(EntitiesSearch.Meta.index) + flush_index(RemoteEntitiesSearch.Meta.index) idref_pid = pers['idref']['pid'] document_data_ref['contribution'][0]['entity']['$ref'] = \ @@ -151,7 +151,7 @@ def test_sync_contribution( flush_index(DocumentsSearch.Meta.index) # contribution and document should be changed - assert Entity.get_record_by_pid( + assert RemoteEntity.get_record_by_pid( pers.pid)['idref']['authorized_access_point'] == 'foo' assert DocumentsSearch().query( 'term', contribution__entity__authorized_access_point_fr='foo').count() @@ -175,8 +175,8 @@ def test_sync_contribution( assert (1, 1, set()) == sync_entity.sync(f'{pers.pid}') flush_index(DocumentsSearch.Meta.index) # new contribution has been created - assert Entity.get_record_by_pid('foo_mef') - assert Entity.get_record_by_ref( + assert RemoteEntity.get_record_by_pid('foo_mef') + assert RemoteEntity.get_record_by_ref( f'{mef_agents_url}/idref/{idref_pid}')[0] db_agent = Document.get_record_by_pid( doc.pid).get('contribution')[0]['entity'] @@ -184,7 +184,7 @@ def test_sync_contribution( # the old MEF has been removed assert (1, []) == sync_entity.remove_unused(f'{pers.pid}') # should not exists anymore - assert not Entity.get_record_by_pid(pers.pid) + assert not RemoteEntity.get_record_by_pid(pers.pid) # === Update the MEF links content data = deepcopy(entity_person_data_tmp) @@ -205,7 +205,7 @@ def test_sync_contribution( assert (1, 1, set()) == sync_entity.sync(f'{data["pid"]}') flush_index(DocumentsSearch.Meta.index) # new contribution has been created - assert Entity.get_record_by_pid('foo_mef') + assert RemoteEntity.get_record_by_pid('foo_mef') # document has been updated with the new MEF and IDREF pid assert DocumentsSearch().query( 'term', contribution__entity__pid='foo_mef').count() @@ -224,7 +224,7 @@ def test_sync_contribution( # the MEF record can be removed assert (1, []) == sync_entity.remove_unused() # should not exists anymore - assert not Entity.get_record_by_pid('foo_mef') + assert not RemoteEntity.get_record_by_pid('foo_mef') @mock.patch('requests.Session.get') @@ -238,13 +238,13 @@ def test_sync_concept( sync_entity = SyncEntity(log_dir=log_path) assert sync_entity - topic = Entity.create( + topic = RemoteEntity.create( entity_topic_data, dbcommit=True, reindex=True, delete_pid=True ) - flush_index(EntitiesSearch.Meta.index) + flush_index(RemoteEntitiesSearch.Meta.index) idref_pid = topic['idref']['pid'] document_data_subject_ref['subjects'][0]['entity']['$ref'] = \ @@ -286,7 +286,7 @@ def test_sync_concept( flush_index(DocumentsSearch.Meta.index) # contribution and document should be changed - assert Entity.get_record_by_pid( + assert RemoteEntity.get_record_by_pid( topic.pid)['idref']['authorized_access_point'] == 'foo' assert DocumentsSearch().query( 'term', subjects__entity__authorized_access_point_fr='foo').count() @@ -301,10 +301,10 @@ def test_sync_concept( # the MEF record can be removed assert (1, []) == sync_entity.remove_unused() # should not exists anymore - assert not Entity.get_record_by_pid('foo_mef') + assert not RemoteEntity.get_record_by_pid('foo_mef') -def test_entity_properties( +def test_remote_entity_properties( entity_person, item_lib_martigny, document, document_data ): """Test entity properties.""" @@ -324,8 +324,8 @@ def test_entity_properties( assert str(document.id) in entity_person.documents_ids() assert item.organisation_pid in entity_person.organisation_pids - assert entity_person == Entity.get_entity('mef', entity_person.pid) - assert entity_person == Entity.get_entity('viaf', '70119347') + assert entity_person == RemoteEntity.get_entity('mef', entity_person.pid) + assert entity_person == RemoteEntity.get_entity('viaf', '70119347') sources_pids = entity_person.source_pids() assert sources_pids['idref'] == '223977268' @@ -339,10 +339,10 @@ def test_entity_properties( # Simulate an exception into the entity creation to test the exception # catching block statement. with mock.patch( - 'rero_ils.modules.entities.api.Entity.create', + 'rero_ils.modules.entities.remote_entities.api.RemoteEntity.create', side_effect=Exception() ): - entity, _ = Entity.get_record_by_ref( + entity, _ = RemoteEntity.get_record_by_ref( 'https://bib.rero.ch/api/documents/dummy_doc') assert entity is None diff --git a/tests/ui/entities/test_entities_filter.py b/tests/ui/remote_entities/test_entities_filter.py similarity index 94% rename from tests/ui/entities/test_entities_filter.py rename to tests/ui/remote_entities/test_entities_filter.py index cac9ab565d..6080a26cd0 100644 --- a/tests/ui/entities/test_entities_filter.py +++ b/tests/ui/remote_entities/test_entities_filter.py @@ -18,11 +18,11 @@ """Jinja2 filters tests.""" -from rero_ils.modules.entities.views import entity_label, \ +from rero_ils.modules.entities.remote_entities.views import entity_label, \ entity_merge_data_values -def test_entity_label(app, entity_person_data): +def test_remote_entity_label(app, entity_person_data): """Test entity label.""" app.config['RERO_ILS_AGENTS_LABEL_ORDER'] = { 'fallback': 'fr', @@ -35,7 +35,7 @@ def test_entity_label(app, entity_person_data): assert label == 'Loy, Georg, 1885-19..' -def test_entity_merge_data_values(app, entity_person_data): +def test_remote_entity_merge_data_values(app, entity_person_data): """Test entities merge data.""" app.config['RERO_ILS_AGENTS_SOURCES'] = ['idref', 'gnd', 'rero'] data = entity_merge_data_values(entity_person_data) diff --git a/tests/ui/entities/test_entities_mapping.py b/tests/ui/remote_entities/test_entities_mapping.py similarity index 81% rename from tests/ui/entities/test_entities_mapping.py rename to tests/ui/remote_entities/test_entities_mapping.py index f270aa2aa1..e048a60507 100644 --- a/tests/ui/entities/test_entities_mapping.py +++ b/tests/ui/remote_entities/test_entities_mapping.py @@ -20,15 +20,16 @@ from utils import get_mapping -from rero_ils.modules.entities.api import EntitiesSearch, Entity +from rero_ils.modules.entities.remote_entities.api import \ + RemoteEntitiesSearch, RemoteEntity -def test_entity_es_mapping(es_clear, db, entity_person_data_tmp): +def test_remote_entity_es_mapping(es_clear, db, entity_person_data_tmp): """Test contribution entity elasticsearch mapping.""" - search = EntitiesSearch() + search = RemoteEntitiesSearch() mapping = get_mapping(search.Meta.index) assert mapping - Entity.create( + RemoteEntity.create( entity_person_data_tmp, dbcommit=True, reindex=True, @@ -39,10 +40,10 @@ def test_entity_es_mapping(es_clear, db, entity_person_data_tmp): def test_concept_entity_es_mapping(es_clear, db, mef_concept1_data_tmp): """Test concept entity elasticsearch mapping.""" - search = EntitiesSearch() + search = RemoteEntitiesSearch() mapping = get_mapping(search.Meta.index) assert mapping - Entity.create( + RemoteEntity.create( mef_concept1_data_tmp, dbcommit=True, reindex=True, @@ -53,12 +54,12 @@ def test_concept_entity_es_mapping(es_clear, db, mef_concept1_data_tmp): def test_entities_search_mapping(app, entity_person): """Test Mef entities search mapping.""" - assert EntitiesSearch()\ + assert RemoteEntitiesSearch()\ .query('query_string', query='philosophische Fakultät')\ .count() == 1 - assert EntitiesSearch()\ + assert RemoteEntitiesSearch()\ .query('match', **{'gnd.preferred_name': 'Loy'})\ .count() == 1 - assert EntitiesSearch()\ + assert RemoteEntitiesSearch()\ .query('match', **{'gnd.variant_name': 'Madeiros'})\ .count() == 1 diff --git a/tests/ui/entities/test_entities_ui.py b/tests/ui/remote_entities/test_entities_ui.py similarity index 82% rename from tests/ui/entities/test_entities_ui.py rename to tests/ui/remote_entities/test_entities_ui.py index e8fa2cd13f..3303ff713d 100644 --- a/tests/ui/entities/test_entities_ui.py +++ b/tests/ui/remote_entities/test_entities_ui.py @@ -21,17 +21,17 @@ from flask import url_for -def test_entity_person_detailed_view(client, entity_person): +def test_remote_entity_person_detailed_view(client, entity_person): """Test entity person detailed view.""" res = client.get(url_for( - 'entities.persons_proxy', + 'remote_entities.persons_proxy', viewcode='global', pid=entity_person.pid)) assert res.status_code == 200 -def test_entity_organisation_detailed_view(client, entity_organisation): +def test_remote_entity_organisation_detailed_view(client, entity_organisation): """Test entity organisation detailed view.""" res = client.get(url_for( - 'entities.corporate_bodies_proxy', + 'remote_entities.corporate_bodies_proxy', viewcode='global', pid='ent_org')) assert res.status_code == 200 diff --git a/tests/ui/entities/test_entities_utils.py b/tests/ui/remote_entities/test_entities_utils.py similarity index 94% rename from tests/ui/entities/test_entities_utils.py rename to tests/ui/remote_entities/test_entities_utils.py index 0c4d01e97c..5795092048 100644 --- a/tests/ui/entities/test_entities_utils.py +++ b/tests/ui/remote_entities/test_entities_utils.py @@ -22,7 +22,8 @@ from requests import RequestException from utils import mock_response -from rero_ils.modules.entities.utils import get_mef_data_by_type +from rero_ils.modules.entities.remote_entities.utils import \ + get_mef_data_by_type @mock.patch('requests.Session.get') diff --git a/tests/ui/test_indexer_utils.py b/tests/ui/test_indexer_utils.py index 23641e6fc1..adcf30fe30 100644 --- a/tests/ui/test_indexer_utils.py +++ b/tests/ui/test_indexer_utils.py @@ -69,7 +69,7 @@ def test_record_to_index(app): assert record_to_index({ '$schema': 'https://mef.rero.ch/schemas/' 'mef/mef-contribution-v0.0.1.json' - }) == ('entities-entity-v0.0.1', '_doc') + }) == ('remote_entities-remote_entity-v0.0.1', '_doc') # for others assert record_to_index({ diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 3f70f8f8f0..d658d97e7b 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -25,8 +25,9 @@ from pkg_resources import resource_string from utils import get_schema -from rero_ils.modules.entities.api import EntitiesSearch, Entity from rero_ils.modules.patrons.api import Patron +from rero_ils.modules.entities.remote_entities.api import RemoteEntity, \ + RemoteEntitiesSearch @pytest.fixture(scope='module') @@ -215,8 +216,8 @@ def patron_martigny_data_tmp_with_id(patron_martigny_data_tmp): def entities_schema(monkeypatch): """Entity Jsonschema for records.""" schema_in_bytes = resource_string( - 'rero_ils.modules.entities.jsonschemas', - '/entities/entity-v0.0.1.json' + 'rero_ils.modules.entities.remote_entities.jsonschemas', + '/remote_entities/remote_entity-v0.0.1.json' ) return get_schema(monkeypatch, schema_in_bytes) @@ -302,7 +303,7 @@ def mef_record_with_idref_rero_data(): """Mef record with idref rero.""" return { '$schema': 'https://bib.rero.ch/schemas/' - 'entities/entity-v0.0.1.json', + 'remote_entities/remote_entity-v0.0.1.json', 'idref': { '$schema': 'https://mef.rero.ch/schemas/' 'agents_idref/idref-agent-v0.0.1.json', @@ -337,16 +338,16 @@ def mef_record_with_idref_rero_data(): @pytest.fixture() def mef_record_with_idref_rero(mef_record_with_idref_rero_data): """Mef record with idref rero.""" - if entity := Entity.get_record_by_pid( + if entity := RemoteEntity.get_record_by_pid( mef_record_with_idref_rero_data['pid']): return entity - entity = Entity.create( + entity = RemoteEntity.create( data=mef_record_with_idref_rero_data, dbcommit=True, reindex=True, delete_pid=False ) - EntitiesSearch.flush_and_refresh() + RemoteEntitiesSearch.flush_and_refresh() return entity @@ -355,7 +356,7 @@ def mef_record_with_idref_gnd_data(): """Mef record with idref gnd.""" return { '$schema': 'https://bib.rero.ch/schemas/' - 'entities/entity-v0.0.1.json', + 'remote_entities/remote_entity-v0.0.1.json', 'gnd': { '$schema': 'https://mef.rero.ch/schemas/' 'agents_gnd/gnd-agent-v0.0.1.json', @@ -439,16 +440,16 @@ def mef_record_with_idref_gnd_data(): @pytest.fixture() def mef_record_with_idref_gnd(mef_record_with_idref_gnd_data): """Mef record with idref rero.""" - if entity := Entity.get_record_by_pid( + if entity := RemoteEntity.get_record_by_pid( mef_record_with_idref_gnd_data['pid']): return entity - entity = Entity.create( + entity = RemoteEntity.create( data=mef_record_with_idref_gnd_data, dbcommit=True, reindex=True, delete_pid=False ) - EntitiesSearch.flush_and_refresh() + RemoteEntitiesSearch.flush_and_refresh() return entity @@ -457,7 +458,7 @@ def mef_record_with_idref_gnd_rero_data(): """Mef record with idref gnd rero is conference.""" return { '$schema': 'https://bib.rero.ch/schemas/' - 'entities/entity-v0.0.1.json', + 'remote_entities/remote_entity-v0.0.1.json', 'gnd': { '$schema': 'https://mef.rero.ch/schemas/' 'agents_gnd/gnd-agent-v0.0.1.json', @@ -539,14 +540,14 @@ def mef_record_with_idref_gnd_rero_data(): @pytest.fixture() def mef_record_with_idref_gnd_rero(mef_record_with_idref_gnd_rero_data): """Mef record with idref rero.""" - if entity := Entity.get_record_by_pid( + if entity := RemoteEntity.get_record_by_pid( mef_record_with_idref_gnd_rero_data['pid']): return entity - entity = Entity.create( + entity = RemoteEntity.create( data=mef_record_with_idref_gnd_rero_data, dbcommit=True, reindex=True, delete_pid=False ) - EntitiesSearch.flush_and_refresh() + RemoteEntitiesSearch.flush_and_refresh() return entity diff --git a/tests/unit/documents/test_documents_dojson_marc21.py b/tests/unit/documents/test_documents_dojson_marc21.py index b12d324f57..53157007e4 100644 --- a/tests/unit/documents/test_documents_dojson_marc21.py +++ b/tests/unit/documents/test_documents_dojson_marc21.py @@ -618,7 +618,8 @@ def test_contribution_to_marc21(app, mef_agents_url, marc21_record, }] } with mock.patch( - 'rero_ils.modules.entities.api.Entity.get_entity', + 'rero_ils.modules.entities.remote_entities.api.' + 'RemoteEntity.get_entity', side_effect=[mef_record_with_idref_rero, mef_record_with_idref_gnd, mef_record_with_idref_gnd_rero] ): diff --git a/tests/unit/test_contributions_jsonschema.py b/tests/unit/test_contributions_jsonschema.py index 3543c8485f..b0c3c94871 100644 --- a/tests/unit/test_contributions_jsonschema.py +++ b/tests/unit/test_contributions_jsonschema.py @@ -44,8 +44,8 @@ def test_required(entities_schema, entity_person_data_tmp): with pytest.raises(ValidationError): validate({ - '$schema': 'https://bib.rero.ch/schemas/entities/' - 'entity-v0.0.1.json', + '$schema': 'https://bib.rero.ch/schemas/remote_entities/' + 'remote_entity-v0.0.1.json', 'viaf_pid': '56597999', 'sources': [ 'rero', @@ -55,8 +55,8 @@ def test_required(entities_schema, entity_person_data_tmp): with pytest.raises(ValidationError): validate({ - '$schema': 'https://bib.rero.ch/schemas/entities/' - 'entity-v0.0.1.json', + '$schema': 'https://bib.rero.ch/schemas/remote_entities/' + 'remote_entity-v0.0.1.json', 'pid': 'ent_pers', 'viaf_pid': '56597999' }, entities_schema) From 3fa31bebbfbcc852aba17267495397a1c9f61d81 Mon Sep 17 00:00:00 2001 From: Renaud Michotte Date: Mon, 3 Jul 2023 16:15:30 +0200 Subject: [PATCH 03/14] settings: remove obsolete settings. Removes obsolete `agentAgentTypes` setting from logged user API response. Co-Authored-by: Renaud Michotte --- rero_ils/config.py | 4 ---- rero_ils/modules/patrons/views.py | 1 - 2 files changed, 5 deletions(-) diff --git a/rero_ils/config.py b/rero_ils/config.py index 7f4493710b..3130e66399 100644 --- a/rero_ils/config.py +++ b/rero_ils/config.py @@ -3017,10 +3017,6 @@ def _(x): #: Entities RERO_ILS_AGENTS_SOURCES = ['idref', 'gnd', 'rero'] -RERO_ILS_AGENTS_AGENT_TYPES = { - 'bf:Person': 'persons', - 'bf:Organisation': 'corporate-bodies' -} RERO_ILS_AGENTS_LABEL_ORDER = { 'fallback': 'fr', 'fr': ['idref', 'rero', 'gnd'], diff --git a/rero_ils/modules/patrons/views.py b/rero_ils/modules/patrons/views.py index 5acb2b0a25..621321d1b8 100644 --- a/rero_ils/modules/patrons/views.py +++ b/rero_ils/modules/patrons/views.py @@ -131,7 +131,6 @@ def logged_user(): 'language': current_i18n.locale.language, 'globalView': config.get('RERO_ILS_SEARCH_GLOBAL_VIEW_CODE'), 'baseUrl': get_base_url(), - 'agentAgentTypes': config.get('RERO_ILS_AGENTS_AGENT_TYPES', {}), 'agentLabelOrder': config.get('RERO_ILS_AGENTS_LABEL_ORDER', {}), 'agentSources': config.get('RERO_ILS_AGENTS_SOURCES', []), 'operationLogs': config.get('RERO_ILS_ENABLE_OPERATION_LOG', []), From b48b54bdd1556e3a67946bad766de39130d296b9 Mon Sep 17 00:00:00 2001 From: Lauren-D Date: Wed, 14 Jun 2023 16:32:13 +0200 Subject: [PATCH 04/14] entities: complete local entities data model * Adapts `local_entities` jsonschema. * Adapts `local_entities` elasticsearch mapping. * Adds `document_dumper`. * Adds `authorized_access_point` extension. * Creates facets. * Adapts elasicsearch sort. Co-Authored-by: Lauren-D --- data/local_entities.json | 106 ++++++++++- pyproject.toml | 1 + rero_ils/config.py | 85 ++++++--- .../common/identifier-v0.0.1.json} | 0 .../modules/documents/dumpers/replace_refs.py | 3 +- .../document_contribution_local-v0.0.1.json | 2 +- ...ment_contribution_organisation-v0.0.1.json | 2 +- .../document_contribution_person-v0.0.1.json | 2 +- .../document_entity_local-v0.0.1.json | 4 +- .../document_genre_form_local-v0.0.1.json | 2 +- .../document_provision_activity-v0.0.1.json | 2 +- .../document_work_access_point-v0.0.1.json | 2 +- rero_ils/modules/entities/api.py | 34 ++++ rero_ils/modules/entities/dumpers/__init__.py | 26 +++ .../dumpers/authorized_acces_point.py | 37 ++++ rero_ils/modules/entities/helpers.py | 51 ++++++ .../modules/entities/local_entities/api.py | 17 +- .../local_entities/dumpers/__init__.py | 14 +- .../local_entities/dumpers/indexer.py | 3 + .../local_entities/extensions/__init__.py | 10 +- .../extensions/authorized_access_point.py | 36 ++-- .../extensions/local_entity_factory.py | 54 ++++++ .../entities/local_entities/jsonresolver.py | 29 +++ .../local_entities/local_entity-v0.0.1.json | 50 ++--- .../local_entity_organisation-v0.0.1.json | 164 +++++++++++++++++ .../local_entity_person-v0.0.1.json | 173 ++++++++++++++++++ .../local_entity_place-v0.0.1.json | 96 ++++++++++ .../local_entity_temporal-v0.0.1.json | 96 ++++++++++ .../local_entity_topic-v0.0.1.json | 103 +++++++++++ .../local_entity_work-v0.0.1.json | 90 +++++++++ .../local_entities/local_entity-v0.0.1.json | 128 ++++++++++++- .../local_entities/subclasses/__init__.py | 35 ++++ .../local_entities/subclasses/organisation.py | 44 +++++ .../local_entities/subclasses/person.py | 41 +++++ .../local_entities/subclasses/place.py | 32 ++++ .../local_entities/subclasses/temporal.py | 32 ++++ .../local_entities/subclasses/topic.py | 32 ++++ .../local_entities/subclasses/work.py | 37 ++++ rero_ils/modules/entities/models.py | 7 + .../modules/entities/remote_entities/api.py | 48 ++--- .../remote_entities/dumpers/__init__.py | 41 +++++ .../remote_entities/dumpers/document.py | 51 ++++++ .../remote_entities/dumpers/indexer.py | 37 ++++ .../remote_entities/remote_entity-v0.0.1.json | 3 +- .../remote_entities/remote_entity-v0.0.1.json | 66 +++++-- .../entities/{ => remote_entities}/replace.py | 11 +- rero_ils/modules/entities/serializers/base.py | 4 +- rero_ils/modules/ext.py | 3 - scripts/setup | 2 +- setup.py | 10 +- .../test_local_entities_extensions.py | 40 ++++ .../test_local_entities_rest.py | 16 +- .../test_remote_entities_rest.py | 4 +- tests/data/data.json | 37 +++- tests/fixtures/metadata.py | 42 +++++ .../test_local_entities_dumpers.py | 36 ++++ .../test_local_entities_jsonresolver.py | 47 +++++ .../test_local_entities_mapping.py | 36 ++++ tests/ui/remote_entities/test_entities_api.py | 2 +- tests/unit/conftest.py | 14 +- tests/unit/test_contributions_jsonschema.py | 20 +- tests/unit/test_local_entities_jsonschema.py | 31 ++++ tests/utils.py | 14 +- 63 files changed, 2110 insertions(+), 187 deletions(-) rename rero_ils/{modules/documents/jsonschemas/documents/document_identifier-v0.0.1.json => jsonschemas/common/identifier-v0.0.1.json} (100%) create mode 100644 rero_ils/modules/entities/api.py create mode 100644 rero_ils/modules/entities/dumpers/__init__.py create mode 100644 rero_ils/modules/entities/dumpers/authorized_acces_point.py create mode 100644 rero_ils/modules/entities/helpers.py create mode 100644 rero_ils/modules/entities/local_entities/extensions/local_entity_factory.py create mode 100644 rero_ils/modules/entities/local_entities/jsonresolver.py create mode 100644 rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_organisation-v0.0.1.json create mode 100644 rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_person-v0.0.1.json create mode 100644 rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_place-v0.0.1.json create mode 100644 rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_temporal-v0.0.1.json create mode 100644 rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_topic-v0.0.1.json create mode 100644 rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_work-v0.0.1.json create mode 100644 rero_ils/modules/entities/local_entities/subclasses/__init__.py create mode 100644 rero_ils/modules/entities/local_entities/subclasses/organisation.py create mode 100644 rero_ils/modules/entities/local_entities/subclasses/person.py create mode 100644 rero_ils/modules/entities/local_entities/subclasses/place.py create mode 100644 rero_ils/modules/entities/local_entities/subclasses/temporal.py create mode 100644 rero_ils/modules/entities/local_entities/subclasses/topic.py create mode 100644 rero_ils/modules/entities/local_entities/subclasses/work.py create mode 100644 rero_ils/modules/entities/remote_entities/dumpers/__init__.py create mode 100644 rero_ils/modules/entities/remote_entities/dumpers/document.py create mode 100644 rero_ils/modules/entities/remote_entities/dumpers/indexer.py rename rero_ils/modules/entities/{ => remote_entities}/replace.py (97%) create mode 100644 tests/api/local_entities/test_local_entities_extensions.py create mode 100644 tests/ui/local_entities/test_local_entities_dumpers.py create mode 100644 tests/ui/local_entities/test_local_entities_jsonresolver.py create mode 100644 tests/ui/local_entities/test_local_entities_mapping.py create mode 100644 tests/unit/test_local_entities_jsonschema.py diff --git a/data/local_entities.json b/data/local_entities.json index 0411ec2c85..8dbb5f72fd 100644 --- a/data/local_entities.json +++ b/data/local_entities.json @@ -1,27 +1,123 @@ [ { "pid": "1", - "preferred_name": "Shakespeare, William", + "name": "Shakespeare, William", + "date_of_birth": "1564", + "date_of_death": "1616", + "identifier": { + "source": "OCLC", + "type": "bf:Local", + "value": "https://id.oclc.org/worldcat/entity/E39PBJxx96qPfyhwWrJChP9kXd" + }, "type": "bf:Person" }, { "pid": "2", - "preferred_name": "Eiffel, Gustave", + "name": "Allies, Mary H.", + "date_of_birth": "1852", + "date_of_death": "1927", + "fuller_form_of_name": "Mary Helen", "type": "bf:Person" }, { "pid": "3", - "preferred_name": "Jobs, Steve", + "name": "Frankel, Valerie Estelle", + "date_of_birth": "1980", "type": "bf:Person" }, { "pid": "4", - "preferred_name": "Dylan, Bob", + "name": "Simberg, Hugo", + "date_of_birth": "1873", + "date_of_death": "1917", + "source_catalog": "lcsh", "type": "bf:Person" }, { "pid": "5", - "preferred_name": "UCLouvain", + "name": "Vesalius, Andreas", + "date_of_birth": "1514", + "date_of_death": "1564", + "source_catalog": "lcsh", + "type": "bf:Person" + }, + { + "pid": "6", + "name": "William", + "date_of_birth": "1650", + "date_of_death": "1702", + "numeration": "III", + "qualifier": "King of England", + "type": "bf:Person" + }, + { + "pid": "7", + "name": "Medicine in Art.", + "source_catalog": "mesh", + "genreForm": false, + "type": "bf:Topic" + }, + { + "pid": "8", + "name": "Medical Illustration.", + "genreForm": false, + "source_catalog": "mesh", + "type": "bf:Topic" + }, + { + "pid": "9", + "name": "UCLouvain", + "conference": false, + "type": "bf:Organisation" + }, + { + "pid": "10", + "name": "Forum on Bilateral Conversation", + "conference_numbering": "5th", + "conference_date": "1990", + "conference_place": "Budapest, Hungary", + "conference": true, "type": "bf:Organisation" + }, + { + "pid": "11", + "name": "Catholic Church", + "subordinate_units": [ + "Concilium Plenarium Americae Latinae" + ], + "conference_numbering": "5th", + "conference_date": "1899", + "conference_place": "Rome, Italy", + "conference": false, + "type": "bf:Organisation" + }, + { + "pid": "12", + "title": "Chevalier au cygne", + "type": "bf:Work" + }, + { + "pid": "13", + "title": "Sonnets", + "creator": "Shakespeare, William (1564-1616)", + "type": "bf:Work" + }, + { + "pid": "14", + "title": "Preludes", + "creator": "Chopin, Fre\u0301de\u0301ric (1810-1849)", + "type": "bf:Work" + }, + { + "pid": "15", + "name": "1800-1899", + "type": "bf:Temporal" + }, + { + "pid": "16", + "name": "Personal correspondence", + "source_catalog": "lcgft", + "genreForm": true, + "type": "bf:Topic" } ] diff --git a/pyproject.toml b/pyproject.toml index 352c23dcf9..34e40a1dc7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -380,6 +380,7 @@ items = "rero_ils.modules.items.jsonresolver" libraries = "rero_ils.modules.libraries.jsonresolver" loans = "rero_ils.modules.loans.jsonresolver" local_fields = "rero_ils.modules.local_fields.jsonresolver" +local_entities = "rero_ils.modules.entities.local_entities.jsonresolver" locations = "rero_ils.modules.locations.jsonresolver" notifications = "rero_ils.modules.notifications.jsonresolver" organisations = "rero_ils.modules.organisations.jsonresolver" diff --git a/rero_ils/config.py b/rero_ils/config.py index 3130e66399..a0d541a03a 100644 --- a/rero_ils/config.py +++ b/rero_ils/config.py @@ -1658,12 +1658,14 @@ def _(x): RERO_ILS_AGGREGATION_SIZE = { 'documents': 50, 'organisations': 10, - 'collections': 20 + 'collections': 20, + 'entities': 20 } DOCUMENTS_AGGREGATION_SIZE = RERO_ILS_AGGREGATION_SIZE.get('documents', RERO_ILS_DEFAULT_AGGREGATION_SIZE) PTRE_AGGREGATION_SIZE = RERO_ILS_AGGREGATION_SIZE.get('patron_transaction_events', RERO_ILS_DEFAULT_AGGREGATION_SIZE) ACQ_ORDER_AGGREGATION_SIZE = RERO_ILS_AGGREGATION_SIZE.get('acq_orders', RERO_ILS_DEFAULT_AGGREGATION_SIZE) +ENTITIES_AGGREGATION_SIZE = RERO_ILS_AGGREGATION_SIZE.get('entities', RERO_ILS_DEFAULT_AGGREGATION_SIZE) FICTIONS_TERMS = ['Fictions', 'Films de fiction'] @@ -2031,23 +2033,64 @@ def _(x): }, ), entities=dict( + aggs=dict( + resource_type=dict( + terms=dict( + field='resource_type', + size=ENTITIES_AGGREGATION_SIZE + ) + ), + type=dict( + terms=dict( + field='type', + size=ENTITIES_AGGREGATION_SIZE + ) + ), + source_catalog=dict( + terms=dict( + field='source_catalog', + size=ENTITIES_AGGREGATION_SIZE + ) + ), + ), + filters={ + _('resource_type'): and_term_filter('resource_type'), + _('type'): and_term_filter('type'), + _('source_catalog'): and_term_filter('source_catalog'), + } + ), + remote_entities=dict( aggs=dict( sources=dict( terms=dict( field='sources', - # This does not take into account - # env variable or instance config file - size=RERO_ILS_AGGREGATION_SIZE.get( - 'entity', RERO_ILS_DEFAULT_AGGREGATION_SIZE) + size=ENTITIES_AGGREGATION_SIZE ) ), type=dict( terms=dict( field='type', - # This does not take into account - # env variable or instance config file - size=RERO_ILS_AGGREGATION_SIZE.get( - 'entity', RERO_ILS_DEFAULT_AGGREGATION_SIZE) + size=ENTITIES_AGGREGATION_SIZE + ) + ) + ), + filters={ + _('sources'): and_term_filter('sources'), + _('type'): and_term_filter('type') + } + ), + local_entities=dict( + aggs=dict( + source_catalog=dict( + terms=dict( + field='source_catalog', + size=ENTITIES_AGGREGATION_SIZE + ) + ), + type=dict( + terms=dict( + field='type', + size=ENTITIES_AGGREGATION_SIZE ) ) ), @@ -2381,23 +2424,21 @@ def _(x): RECORDS_REST_DEFAULT_SORT['collections'] = dict( query='bestmatch', noquery='start_date') -# ------ CONTRIBUTIONS SORT +# ------ ENTITIES SORT RECORDS_REST_SORT_OPTIONS['entities']['fr_name'] = dict( - fields=[ - 'idref_authorized_access_point_sort', - 'rero_authorized_access_point_sort', - 'gnd_authorized_access_point_sort', - ], - title='Collection french name', + fields=['authorized_access_point_fr.sort'], default_order='asc' ) RECORDS_REST_SORT_OPTIONS['entities']['de_name'] = dict( - fields=[ - 'gnd_authorized_access_point_sort', - 'idref_authorized_access_point_sort', - 'rero_authorized_access_point_sort' - ], - title='Collection german name', + fields=['authorized_access_point_de.sort'], + default_order='asc' +) +RECORDS_REST_SORT_OPTIONS['entities']['en_name'] = dict( + fields=['authorized_access_point_en.sort'], + default_order='asc' +) +RECORDS_REST_SORT_OPTIONS['entities']['it_name'] = dict( + fields=['authorized_access_point_it.sort'], default_order='asc' ) diff --git a/rero_ils/modules/documents/jsonschemas/documents/document_identifier-v0.0.1.json b/rero_ils/jsonschemas/common/identifier-v0.0.1.json similarity index 100% rename from rero_ils/modules/documents/jsonschemas/documents/document_identifier-v0.0.1.json rename to rero_ils/jsonschemas/common/identifier-v0.0.1.json diff --git a/rero_ils/modules/documents/dumpers/replace_refs.py b/rero_ils/modules/documents/dumpers/replace_refs.py index b89953565c..26f05a660f 100644 --- a/rero_ils/modules/documents/dumpers/replace_refs.py +++ b/rero_ils/modules/documents/dumpers/replace_refs.py @@ -20,6 +20,7 @@ from invenio_records.dumpers import Dumper from rero_ils.modules.commons.exceptions import RecordNotFound +from rero_ils.modules.entities.remote_entities.dumpers import document_dumper from rero_ils.modules.entities.remote_entities.utils import \ extract_data_from_mef_uri @@ -34,7 +35,7 @@ def _replace_entity(data): if not (entity := RemoteEntity.get_record_by_pid(data['pid'])): raise RecordNotFound(RemoteEntity, data['pid']) _, _type, _ = extract_data_from_mef_uri(data['$ref']) - entity = entity.dumps_for_document() + entity = entity.dumps(document_dumper) entity.update({ 'primary_source': _type, 'pid': data['pid'] diff --git a/rero_ils/modules/documents/jsonschemas/documents/document_contribution_local-v0.0.1.json b/rero_ils/modules/documents/jsonschemas/documents/document_contribution_local-v0.0.1.json index eb0477fb86..2d2e38667c 100644 --- a/rero_ils/modules/documents/jsonschemas/documents/document_contribution_local-v0.0.1.json +++ b/rero_ils/modules/documents/jsonschemas/documents/document_contribution_local-v0.0.1.json @@ -48,7 +48,7 @@ } }, "identifiedBy": { - "$ref": "https://bib.rero.ch/schemas/documents/document_identifier-v0.0.1.json#/identifier" + "$ref": "https://bib.rero.ch/schemas/common/identifier-v0.0.1.json#/identifier" } }, "form": { diff --git a/rero_ils/modules/documents/jsonschemas/documents/document_contribution_organisation-v0.0.1.json b/rero_ils/modules/documents/jsonschemas/documents/document_contribution_organisation-v0.0.1.json index d54075a6e9..ad35cab3de 100644 --- a/rero_ils/modules/documents/jsonschemas/documents/document_contribution_organisation-v0.0.1.json +++ b/rero_ils/modules/documents/jsonschemas/documents/document_contribution_organisation-v0.0.1.json @@ -116,7 +116,7 @@ } }, { - "$ref": "https://bib.rero.ch/schemas/documents/document_identifier-v0.0.1.json#/identifier" + "$ref": "https://bib.rero.ch/schemas/common/identifier-v0.0.1.json#/identifier" } ] } diff --git a/rero_ils/modules/documents/jsonschemas/documents/document_contribution_person-v0.0.1.json b/rero_ils/modules/documents/jsonschemas/documents/document_contribution_person-v0.0.1.json index d03c076a57..41eadee1f9 100644 --- a/rero_ils/modules/documents/jsonschemas/documents/document_contribution_person-v0.0.1.json +++ b/rero_ils/modules/documents/jsonschemas/documents/document_contribution_person-v0.0.1.json @@ -101,7 +101,7 @@ } }, "identifiedBy": { - "$ref": "https://bib.rero.ch/schemas/documents/document_identifier-v0.0.1.json#/identifier" + "$ref": "https://bib.rero.ch/schemas/common/identifier-v0.0.1.json#/identifier" } }, "form": { diff --git a/rero_ils/modules/documents/jsonschemas/documents/document_entity_local-v0.0.1.json b/rero_ils/modules/documents/jsonschemas/documents/document_entity_local-v0.0.1.json index 7ece94b463..f5b3b385c4 100644 --- a/rero_ils/modules/documents/jsonschemas/documents/document_entity_local-v0.0.1.json +++ b/rero_ils/modules/documents/jsonschemas/documents/document_entity_local-v0.0.1.json @@ -70,7 +70,7 @@ } }, "identifiedBy": { - "$ref": "https://bib.rero.ch/schemas/documents/document_identifier-v0.0.1.json#/identifier" + "$ref": "https://bib.rero.ch/schemas/common/identifier-v0.0.1.json#/identifier" }, "source": { "title": "Source", @@ -130,7 +130,7 @@ "$ref": "#/properties/authorized_access_point" }, "identifiedBy": { - "$ref": "https://bib.rero.ch/schemas/documents/document_identifier-v0.0.1.json#/identifier" + "$ref": "https://bib.rero.ch/schemas/common/identifier-v0.0.1.json#/identifier" } }, "form": { diff --git a/rero_ils/modules/documents/jsonschemas/documents/document_genre_form_local-v0.0.1.json b/rero_ils/modules/documents/jsonschemas/documents/document_genre_form_local-v0.0.1.json index 249b8c2df3..7ca4995025 100644 --- a/rero_ils/modules/documents/jsonschemas/documents/document_genre_form_local-v0.0.1.json +++ b/rero_ils/modules/documents/jsonschemas/documents/document_genre_form_local-v0.0.1.json @@ -44,7 +44,7 @@ "minLength": 3 }, "identifiedBy": { - "$ref": "https://bib.rero.ch/schemas/documents/document_identifier-v0.0.1.json#/identifier" + "$ref": "https://bib.rero.ch/schemas/common/identifier-v0.0.1.json#/identifier" } }, "form": { diff --git a/rero_ils/modules/documents/jsonschemas/documents/document_provision_activity-v0.0.1.json b/rero_ils/modules/documents/jsonschemas/documents/document_provision_activity-v0.0.1.json index 198933f050..8756d30c9a 100644 --- a/rero_ils/modules/documents/jsonschemas/documents/document_provision_activity-v0.0.1.json +++ b/rero_ils/modules/documents/jsonschemas/documents/document_provision_activity-v0.0.1.json @@ -104,7 +104,7 @@ } }, { - "$ref": "https://bib.rero.ch/schemas/documents/document_identifier-v0.0.1.json#/identifier" + "$ref": "https://bib.rero.ch/schemas/common/identifier-v0.0.1.json#/identifier" } ] } diff --git a/rero_ils/modules/documents/jsonschemas/documents/document_work_access_point-v0.0.1.json b/rero_ils/modules/documents/jsonschemas/documents/document_work_access_point-v0.0.1.json index 57f2b969c9..5b763f779c 100644 --- a/rero_ils/modules/documents/jsonschemas/documents/document_work_access_point-v0.0.1.json +++ b/rero_ils/modules/documents/jsonschemas/documents/document_work_access_point-v0.0.1.json @@ -114,7 +114,7 @@ "minLength": 1 }, "identifiedBy": { - "$ref": "https://bib.rero.ch/schemas/documents/document_identifier-v0.0.1.json#/identifier" + "$ref": "https://bib.rero.ch/schemas/common/identifier-v0.0.1.json#/identifier" } } }, diff --git a/rero_ils/modules/entities/api.py b/rero_ils/modules/entities/api.py new file mode 100644 index 0000000000..fd6074cf38 --- /dev/null +++ b/rero_ils/modules/entities/api.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""API for manipulating entities.""" + +from abc import ABC, abstractmethod + + +class Entity(ABC): + """Entity class.""" + + @abstractmethod + def get_authorized_access_point(self, language): + """Get localized authorized_access_point. + + :param language: language for authorized access point. + :returns: authorized access point in given language. + """ + raise NotImplementedError diff --git a/rero_ils/modules/entities/dumpers/__init__.py b/rero_ils/modules/entities/dumpers/__init__.py new file mode 100644 index 0000000000..d26a089995 --- /dev/null +++ b/rero_ils/modules/entities/dumpers/__init__.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Common entity dumpers.""" + +from rero_ils.modules.entities.dumpers.authorized_acces_point import \ + LocalizedAuthorizedAccessPointDumper + +__all__ = [ + 'LocalizedAuthorizedAccessPointDumper' +] diff --git a/rero_ils/modules/entities/dumpers/authorized_acces_point.py b/rero_ils/modules/entities/dumpers/authorized_acces_point.py new file mode 100644 index 0000000000..c300ffc4cc --- /dev/null +++ b/rero_ils/modules/entities/dumpers/authorized_acces_point.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""LocalizedAuthorizedAccessPoint dumper.""" +from invenio_records.dumpers import Dumper + +from rero_ils.utils import get_i18n_supported_languages + + +class LocalizedAuthorizedAccessPointDumper(Dumper): + """Localized entity authorized access point dumper.""" + + def dump(self, record, data): + """Dump a local entity by adding localized authorized access point. + + :param record: The record to dump. + :param data: The initial dump data passed in by ``record.dumps()``. + """ + for language in get_i18n_supported_languages(): + data[f'authorized_access_point_{language}'] = \ + record.get_authorized_access_point(language) + return data diff --git a/rero_ils/modules/entities/helpers.py b/rero_ils/modules/entities/helpers.py new file mode 100644 index 0000000000..b4514970b6 --- /dev/null +++ b/rero_ils/modules/entities/helpers.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Helpers for entities.""" + + +def str_builder(field_values, prefix='', suffix='', delimiter=''): + """String builder method. + + This builder is used to format string depending on given arguments + First, it joins all values with the delimiter and add a prefix or suffix. + + :param field_values: values to format. + :type field_values: str or an array of str + :param prefix: value to add before field_values. + :type prefix: str + :param suffix: value to add after field_values. + :type suffix: str + :param delimiter: delimiter used to concatenate multiple field_values. + :type field_values: str + :return: formatted string or an empty string. + :rtype: str + """ + if not isinstance(field_values, list): + field_values = [field_values] + + # DEV NOTES: + # We use any() instead bool() because we can have a list of empty string. + # => see OrganisationLocalEntity subclass + # If all values in list are empty we don't want to process field. + # Ex: + # any(field_values['','']) == False + # bool(field_values['','']) == True + if any(field_values): + return f'{prefix}{delimiter.join(field_values)}{suffix}' + return '' diff --git a/rero_ils/modules/entities/local_entities/api.py b/rero_ils/modules/entities/local_entities/api.py index 6ec6a610c2..2c8c1567dd 100644 --- a/rero_ils/modules/entities/local_entities/api.py +++ b/rero_ils/modules/entities/local_entities/api.py @@ -27,10 +27,13 @@ from rero_ils.modules.minters import id_minter from rero_ils.modules.providers import Provider -from .dumpers import replace_refs_dumper -from .extensions import AuthorizedAccessPointExtension +from .dumpers import indexer_dumper, replace_refs_dumper +from .extensions import AuthorizedAccessPointExtension, \ + LocalEntityFactoryExtension from .models import LocalEntityIdentifier, LocalEntityMetadata +from ..api import Entity + # provider LocalEntityProvider = type( 'LocalEntityProvider', @@ -57,7 +60,7 @@ class Meta: default_filter = None -class LocalEntity(IlsRecord): +class LocalEntity(IlsRecord, Entity): """Local entity class.""" minter = local_entity_id_minter @@ -68,9 +71,15 @@ class LocalEntity(IlsRecord): enable_jsonref = False _extensions = [ + LocalEntityFactoryExtension(), AuthorizedAccessPointExtension() ] + @property + def type(self): + """Shortcut for local entity type.""" + return self.get('type') + def resolve(self): """Resolve references data. @@ -108,6 +117,8 @@ class LocalEntitiesIndexer(IlsRecordsIndexer): """Local entity indexing class.""" record_cls = LocalEntity + # data dumper for indexing + record_dumper = indexer_dumper def bulk_index(self, record_id_iterator): """Bulk index records. diff --git a/rero_ils/modules/entities/local_entities/dumpers/__init__.py b/rero_ils/modules/entities/local_entities/dumpers/__init__.py index 1ad9efdb9a..62e95c09f5 100644 --- a/rero_ils/modules/entities/local_entities/dumpers/__init__.py +++ b/rero_ils/modules/entities/local_entities/dumpers/__init__.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- # # RERO ILS -# Copyright (C) 2019-2022 RERO -# Copyright (C) 2019-2022 UCLouvain +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -23,6 +23,7 @@ from rero_ils.modules.commons.dumpers import MultiDumper, ReplaceRefsDumper from .indexer import LocalEntityIndexerDumper +from ...dumpers import LocalizedAuthorizedAccessPointDumper # replace linked data (seems not necessary at this time) replace_refs_dumper = MultiDumper(dumpers=[ @@ -36,5 +37,12 @@ # make a fresh copy Dumper(), ReplaceRefsDumper(), - LocalEntityIndexerDumper() + LocalEntityIndexerDumper(), + LocalizedAuthorizedAccessPointDumper() +]) + +document_dumper = MultiDumper(dumpers=[ + # make a fresh copy + Dumper(), + LocalizedAuthorizedAccessPointDumper(), ]) diff --git a/rero_ils/modules/entities/local_entities/dumpers/indexer.py b/rero_ils/modules/entities/local_entities/dumpers/indexer.py index 977d95c07b..f5514b7970 100644 --- a/rero_ils/modules/entities/local_entities/dumpers/indexer.py +++ b/rero_ils/modules/entities/local_entities/dumpers/indexer.py @@ -20,6 +20,8 @@ from invenio_records.dumpers import Dumper +from rero_ils.modules.entities.models import EntityResourceType + class LocalEntityIndexerDumper(Dumper): """Local entity indexer.""" @@ -30,4 +32,5 @@ def dump(self, record, data): :param record: The record to dump. :param data: The initial dump data passed in by ``record.dumps()``. """ + data['resource_type'] = EntityResourceType.LOCAL return data diff --git a/rero_ils/modules/entities/local_entities/extensions/__init__.py b/rero_ils/modules/entities/local_entities/extensions/__init__.py index cfbc8264ed..9c91dc9a99 100644 --- a/rero_ils/modules/entities/local_entities/extensions/__init__.py +++ b/rero_ils/modules/entities/local_entities/extensions/__init__.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- # # RERO ILS -# Copyright (C) 2022 RERO -# Copyright (C) 2022 UCLouvain +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -16,9 +16,11 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -"""Document record extensions.""" +"""Local entity record extensions.""" from .authorized_access_point import AuthorizedAccessPointExtension +from .local_entity_factory import LocalEntityFactoryExtension __all__ = [ - 'AuthorizedAccessPointExtension' + 'AuthorizedAccessPointExtension', + 'LocalEntityFactoryExtension' ] diff --git a/rero_ils/modules/entities/local_entities/extensions/authorized_access_point.py b/rero_ils/modules/entities/local_entities/extensions/authorized_access_point.py index 623cde3183..3025e44b0c 100644 --- a/rero_ils/modules/entities/local_entities/extensions/authorized_access_point.py +++ b/rero_ils/modules/entities/local_entities/extensions/authorized_access_point.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- # # RERO ILS -# Copyright (C) 2021 RERO -# Copyright (C) 2021 UCLouvain +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -16,8 +16,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -"""Document record extension to add the MEF pid in the database.""" - +"""Local entity extension to add authorized access point pid in the DB.""" from invenio_records.extensions import RecordExtension @@ -25,18 +24,27 @@ class AuthorizedAccessPointExtension(RecordExtension): """Adds the authorized access point.""" - def _generate_authorized_access_point(self, record, data): - """Generate authorized access point. + def _get_authorized_access_point(self, record): + """.""" + # there is no language for local entities + language = None + return record.get_authorized_access_point(language) + + def pre_create(self, record): + """Called before a record is created. - :params record: dict - a document record. + :param record: the record metadata. """ - data.update({'authorized_access_point': record.get('preferred_name')}) + record['authorized_access_point'] = \ + self._get_authorized_access_point(record) + # required for validation + if record.model: + record.model.data = record - def pre_dump(self, record, data, dumper=None): - """Called before a record is dumped. + def pre_commit(self, record): + """Called before a record is committed. - :param record: the record to dump - :param data: the data to dump. - :param dumper: the dumper class used to dump the record. + :param record: the record metadata. """ - return self._generate_authorized_access_point(record, data) + record['authorized_access_point'] = \ + self._get_authorized_access_point(record) diff --git a/rero_ils/modules/entities/local_entities/extensions/local_entity_factory.py b/rero_ils/modules/entities/local_entities/extensions/local_entity_factory.py new file mode 100644 index 0000000000..d28323fc38 --- /dev/null +++ b/rero_ils/modules/entities/local_entities/extensions/local_entity_factory.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Local entity record factory extensions.""" + +from invenio_records.extensions import RecordExtension + +from ...models import EntityType + + +class LocalEntityFactoryExtension(RecordExtension): + """Local entity factory extension class. + + Choose the best local entity subclass based on `type` attributes. + """ + + @staticmethod + def _get_local_entity_class(record): + """Get the Local entity class to use based on record data.""" + from ..api import LocalEntity + from ..subclasses import OrganisationLocalEntity, PersonLocalEntity, \ + PlaceLocalEntity, TemporalLocalEntity, TopicLocalEntity, \ + WorkLocalEntity + + mapping = { + EntityType.PERSON: PersonLocalEntity, + EntityType.ORGANISATION: OrganisationLocalEntity, + EntityType.TOPIC: TopicLocalEntity, + EntityType.PLACE: PlaceLocalEntity, + EntityType.TEMPORAL: TemporalLocalEntity, + EntityType.WORK: WorkLocalEntity, + } + + return mapping.get(record.type, LocalEntity) + + def post_init(self, record, data, model=None, **kwargs): + """Called after a record is initialized.""" + cls = LocalEntityFactoryExtension._get_local_entity_class(record) + record.__class__ = cls diff --git a/rero_ils/modules/entities/local_entities/jsonresolver.py b/rero_ils/modules/entities/local_entities/jsonresolver.py new file mode 100644 index 0000000000..6a9397e10b --- /dev/null +++ b/rero_ils/modules/entities/local_entities/jsonresolver.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Local entity resolver.""" + +import jsonresolver + +from rero_ils.modules.jsonresolver import resolve_json_refs + + +@jsonresolver.route('/api/local_entities/', host='bib.rero.ch') +def local_entities_resolver(pid): + """Resolver for acq_account record.""" + return resolve_json_refs('locent', pid) diff --git a/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity-v0.0.1.json b/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity-v0.0.1.json index 048f7d5ba7..f9619547f2 100644 --- a/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity-v0.0.1.json +++ b/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity-v0.0.1.json @@ -1,45 +1,25 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", "title": "Local entity", "description": "JSON schema for a local entity", - "additionalProperties": false, - "required": [ - "$schema", - "pid", - "preferred_name", - "type" - ], - "properties": { - "$schema": { - "title": "Schema", - "description": "Schema to validate local entity records against.", - "type": "string", - "minLength": 9 + "oneOf": [ + { + "$ref": "https://bib.rero.ch/schemas/local_entities/local_entity_person-v0.0.1.json" }, - "pid": { - "title": "Local entity PID", - "type": "string" + { + "$ref": "https://bib.rero.ch/schemas/local_entities/local_entity_organisation-v0.0.1.json" }, - "preferred_name": { - "title": "Preferred name", - "type": "string", - "minLength": 1 + { + "$ref": "https://bib.rero.ch/schemas/local_entities/local_entity_topic-v0.0.1.json" }, - "type": { - "title": "Agent type", - "type": "string", - "enum": [ - "bf:Organisation", - "bf:Person", - "bf:Place", - "bf:Topic", - "bf:Work" - ] + { + "$ref": "https://bib.rero.ch/schemas/local_entities/local_entity_place-v0.0.1.json" }, - "deleted": { - "title": "Deletion date", - "type": "string" + { + "$ref": "https://bib.rero.ch/schemas/local_entities/local_entity_temporal-v0.0.1.json" + }, + { + "$ref": "https://bib.rero.ch/schemas/local_entities/local_entity_work-v0.0.1.json" } - } + ] } diff --git a/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_organisation-v0.0.1.json b/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_organisation-v0.0.1.json new file mode 100644 index 0000000000..f920e56539 --- /dev/null +++ b/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_organisation-v0.0.1.json @@ -0,0 +1,164 @@ +{ + "type": "object", + "title": "Organisation", + "additionalProperties": false, + "required": [ + "$schema", + "pid", + "type", + "authorized_access_point", + "name", + "conference" + ], + "propertiesOrder": [ + "type", + "name", + "subordinate_units", + "conference", + "conference_place", + "conference_numbering", + "conference_date", + "start_date", + "end_date", + "alternative_names", + "source_catalog", + "identifier" + ], + "properties": { + "$schema": { + "title": "Schema", + "description": "Schema to validate local entity records against.", + "type": "string", + "minLength": 9 + }, + "type": { + "title": "Type", + "type": "string", + "readOnly": true, + "default": "bf:Organisation", + "const": "bf:Organisation", + "form": { + "templateOptions": { + "wrappers": [ + "hide" + ] + } + } + }, + "pid": { + "title": "Local entity PID", + "type": "string", + "minLength": 1 + }, + "deleted": { + "title": "Deletion date", + "type": "string", + "format": "date-time" + }, + "name": { + "title": "Name", + "type": "string", + "minLength": 1, + "form": { + "placeholder": "Example: Universit\u00e9 de Bordeaux III" + } + }, + "authorized_access_point": { + "title": "Authorized access point", + "type": "string", + "minLength": 1, + "readOnly": true, + "form": { + "templateOptions": { + "wrappers": [ + "hide" + ] + } + } + }, + "subordinate_units": { + "title": "Subordinate units", + "type": "array", + "minItems": 1, + "items": { + "title": "Subordinate unit", + "type": "string", + "minLength": 1, + "form": { + "placeholder": "Example: Institut d'histoire" + } + } + }, + "start_date": { + "title": "Start date", + "type": "string", + "minLength": 1, + "form": { + "placeholder": "Example: 1955" + } + }, + "end_date": { + "title": "End date", + "type": "string", + "minLength": 1, + "form": { + "placeholder": "Example: 2015" + } + }, + "conference": { + "title": "Conference", + "description": "Mark as conference if the access point is directly at the conference name and not at the corporate body name", + "type": "boolean", + "default": false + }, + "conference_place": { + "title": "Location of conference", + "type": "string", + "minLength": 1, + "form": { + "placeholder": "Example: Paris", + "hideExpression": "!field?.parent?.model || !field.parent.model.conference || field.parent.model.conference == false" + } + }, + "conference_numbering": { + "title": "Number of conference", + "type": "string", + "minLength": 1, + "form": { + "placeholder": "Example: 23e", + "hideExpression": "!field?.parent?.model || !field.parent.model.conference || field.parent.model.conference == false" + } + }, + "conference_date": { + "title": "Date of conference", + "description": "Record only years.", + "type": "string", + "minLength": 1, + "form": { + "placeholder": "Example: 2022", + "hideExpression": "!field?.parent?.model || !field.parent.model.conference || field.parent.model.conference == false" + } + }, + "alternative_names": { + "title": "Alternative names", + "type": "array", + "minItems": 1, + "items": { + "title": "Alternative name", + "type": "string", + "minLength": 1 + } + }, + "source_catalog": { + "title": "Source catalog", + "type": "string", + "minLength": 1, + "form": { + "placeholder": "Example: lcsh, mesh, rameau" + } + }, + "identifier": { + "$ref": "https://bib.rero.ch/schemas/common/identifier-v0.0.1.json#/identifier" + } + } +} diff --git a/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_person-v0.0.1.json b/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_person-v0.0.1.json new file mode 100644 index 0000000000..644e4b99e7 --- /dev/null +++ b/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_person-v0.0.1.json @@ -0,0 +1,173 @@ +{ + "type": "object", + "title": "Person", + "additionalProperties": false, + "required": [ + "$schema", + "pid", + "type", + "authorized_access_point", + "name" + ], + "propertiesOrder": [ + "type", + "name", + "date_of_birth", + "date_of_death", + "numeration", + "qualifier", + "fuller_form_of_name", + "alternative_names", + "gender", + "source_catalog", + "identifier" + ], + "properties": { + "$schema": { + "title": "Schema", + "description": "Schema to validate local entity records against.", + "type": "string", + "minLength": 9 + }, + "type": { + "title": "Type", + "type": "string", + "readOnly": true, + "default": "bf:Person", + "const": "bf:Person", + "form": { + "templateOptions": { + "wrappers": [ + "hide" + ] + } + } + }, + "pid": { + "title": "Local entity PID", + "type": "string", + "minLength": 1 + }, + "deleted": { + "title": "Deletion date", + "type": "string", + "format": "date-time" + }, + "name": { + "title": "Name", + "type": "string", + "minLength": 1, + "form": { + "placeholder": "Example: Miller, John" + } + }, + "authorized_access_point": { + "title": "Authorized access point", + "type": "string", + "minLength": 1, + "readOnly": true, + "form": { + "templateOptions": { + "wrappers": [ + "hide" + ] + } + } + }, + "date_of_birth": { + "title": "Birth date", + "description": "Record only years.", + "type": "string", + "minLength": 1, + "form": { + "placeholder": "Example: 1955" + } + }, + "date_of_death": { + "title": "Death date", + "description": "Record only years.", + "type": "string", + "minLength": 1, + "form": { + "placeholder": "Example: 2012" + } + }, + "qualifier": { + "title": "Qualifier", + "type": "string", + "minLength": 1, + "form": { + "placeholder": "Example: physicist" + } + }, + "numeration": { + "title": "Numeration", + "description": "Record only a number.", + "type": "string", + "minLength": 1, + "form": { + "placeholder": "Example: XXIII" + } + }, + "fuller_form_of_name": { + "title": "Fuller form of Name", + "description": "Used only for import. Fuller form of a name represented only by an initial, abbreviation, or other variant not included in the form chosen as the preferred name", + "type": "string", + "minLength": 1 + }, + "alternative_names": { + "title": "Alternative names", + "type": "array", + "minItems": 1, + "items": { + "title": "Alternative name", + "type": "string", + "minLength": 1, + "form": { + "placeholder": "Example: Beyer, Adam" + } + } + }, + "gender": { + "title": "Gender", + "type": "string", + "minLength": 1, + "maxLength": 1, + "enum": [ + "F", + "M", + "X" + ], + "form": { + "options": [ + { + "value": "M", + "label": "male" + }, + { + "value": "F", + "label": "female" + }, + { + "value": "X", + "label": "other" + } + ], + "templateOptions": { + "itemCssClass": "col-lg-6" + } + } + }, + "source_catalog": { + "title": "Source catalog", + "type": "string", + "minLength": 1, + "form": { + "placeholder": "Example: lcsh, mesh, rameau" + } + }, + "identifier": { + "$ref": "https://bib.rero.ch/schemas/common/identifier-v0.0.1.json#/identifier" + } + } +} diff --git a/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_place-v0.0.1.json b/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_place-v0.0.1.json new file mode 100644 index 0000000000..9423eb3845 --- /dev/null +++ b/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_place-v0.0.1.json @@ -0,0 +1,96 @@ +{ + "type": "object", + "title": "Place", + "additionalProperties": false, + "required": [ + "$schema", + "pid", + "type", + "authorized_access_point", + "name" + ], + "propertiesOrder": [ + "type", + "name", + "alternative_names", + "source_catalog", + "identifier" + ], + "properties": { + "$schema": { + "title": "Schema", + "description": "Schema to validate local entity records against.", + "type": "string", + "minLength": 9 + }, + "type": { + "title": "Type", + "type": "string", + "readOnly": true, + "default": "bf:Place", + "const": "bf:Place", + "form": { + "templateOptions": { + "wrappers": [ + "hide" + ] + } + } + }, + "pid": { + "title": "Local entity PID", + "type": "string", + "minLength": 1 + }, + "deleted": { + "title": "Deletion date", + "type": "string", + "format": "date-time" + }, + "name": { + "title": "Name", + "type": "string", + "minLength": 1, + "form": { + "placeholder": "Example: Parc Natural de l'Albufera (Spain)" + } + }, + "authorized_access_point": { + "title": "Authorized access point", + "type": "string", + "minLength": 1, + "readOnly": true, + "form": { + "templateOptions": { + "wrappers": [ + "hide" + ] + } + } + }, + "alternative_names": { + "title": "Alternative names", + "type": "array", + "minItems": 1, + "items": { + "title": "Alternative name", + "type": "string", + "minLength": 1, + "form": { + "placeholder": "Example: Albufera, Parc Natural de l' (Spain)" + } + } + }, + "source_catalog": { + "title": "Source catalog", + "type": "string", + "minLength": 1, + "form": { + "placeholder": "Example: lcsh, mesh, rameau" + } + }, + "identifier": { + "$ref": "https://bib.rero.ch/schemas/common/identifier-v0.0.1.json#/identifier" + } + } +} diff --git a/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_temporal-v0.0.1.json b/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_temporal-v0.0.1.json new file mode 100644 index 0000000000..4b999870b1 --- /dev/null +++ b/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_temporal-v0.0.1.json @@ -0,0 +1,96 @@ +{ + "type": "object", + "title": "Temporal", + "additionalProperties": false, + "required": [ + "$schema", + "pid", + "type", + "authorized_access_point", + "name" + ], + "propertiesOrder": [ + "type", + "name", + "alternative_names", + "source_catalog", + "identifier" + ], + "properties": { + "$schema": { + "title": "Schema", + "description": "Schema to validate local entity records against.", + "type": "string", + "minLength": 9 + }, + "type": { + "title": "Type", + "type": "string", + "readOnly": true, + "default": "bf:Temporal", + "const": "bf:Temporal", + "form": { + "templateOptions": { + "wrappers": [ + "hide" + ] + } + } + }, + "pid": { + "title": "Local entity PID", + "type": "string", + "minLength": 1 + }, + "deleted": { + "title": "Deletion date", + "type": "string", + "format": "date-time" + }, + "name": { + "title": "Name", + "type": "string", + "minLength": 1, + "form": { + "placeholder": "Example: 21st century" + } + }, + "authorized_access_point": { + "title": "Authorized access point", + "type": "string", + "minLength": 1, + "readOnly": true, + "form": { + "templateOptions": { + "wrappers": [ + "hide" + ] + } + } + }, + "alternative_names": { + "title": "Alternative names", + "type": "array", + "minItems": 1, + "items": { + "title": "Alternative name", + "type": "string", + "minLength": 1, + "form": { + "placeholder": "Example: Beyer, Adam" + } + } + }, + "source_catalog": { + "title": "Source catalog", + "type": "string", + "minLength": 1, + "form": { + "placeholder": "Example: lcsh, mesh, rameau" + } + }, + "identifier": { + "$ref": "https://bib.rero.ch/schemas/common/identifier-v0.0.1.json#/identifier" + } + } +} diff --git a/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_topic-v0.0.1.json b/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_topic-v0.0.1.json new file mode 100644 index 0000000000..233ee03771 --- /dev/null +++ b/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_topic-v0.0.1.json @@ -0,0 +1,103 @@ +{ + "type": "object", + "title": "Topic", + "additionalProperties": false, + "required": [ + "$schema", + "pid", + "type", + "authorized_access_point", + "name", + "genreForm" + ], + "propertiesOrder": [ + "type", + "name", + "genreForm", + "alternative_names", + "source_catalog", + "identifier" + ], + "properties": { + "$schema": { + "title": "Schema", + "description": "Schema to validate local entity records against.", + "type": "string", + "minLength": 9 + }, + "type": { + "title": "Type", + "type": "string", + "readOnly": true, + "default": "bf:Topic", + "const": "bf:Topic", + "form": { + "templateOptions": { + "wrappers": [ + "hide" + ] + } + } + }, + "pid": { + "title": "Local entity PID", + "type": "string", + "minLength": 1 + }, + "deleted": { + "title": "Deletion date", + "type": "string", + "format": "date-time" + }, + "name": { + "title": "Name", + "type": "string", + "minLength": 1, + "form": { + "placeholder": "Example: Industrial archaeology" + } + }, + "authorized_access_point": { + "title": "Authorized access point", + "type": "string", + "minLength": 1, + "readOnly": true, + "form": { + "templateOptions": { + "wrappers": [ + "hide" + ] + } + } + }, + "genreForm": { + "title": "Genre form", + "type": "boolean", + "default": false + }, + "alternative_names": { + "title": "Alternative names", + "type": "array", + "minItems": 1, + "items": { + "title": "Alternative name", + "type": "string", + "minLength": 1, + "form": { + "placeholder": "Example: Antiquities, Industrial" + } + } + }, + "source_catalog": { + "title": "Source catalog", + "type": "string", + "minLength": 1, + "form": { + "placeholder": "Example: lcsh, mesh, rameau" + } + }, + "identifier": { + "$ref": "https://bib.rero.ch/schemas/common/identifier-v0.0.1.json#/identifier" + } + } +} diff --git a/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_work-v0.0.1.json b/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_work-v0.0.1.json new file mode 100644 index 0000000000..70d09602fe --- /dev/null +++ b/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_work-v0.0.1.json @@ -0,0 +1,90 @@ +{ + "type": "object", + "title": "Work", + "additionalProperties": false, + "required": [ + "$schema", + "pid", + "type", + "authorized_access_point", + "title" + ], + "propertiesOrder": [ + "type", + "title", + "creator", + "identifier" + ], + "properties": { + "$schema": { + "title": "Schema", + "description": "Schema to validate local entity records against.", + "type": "string", + "minLength": 9 + }, + "type": { + "title": "Type", + "type": "string", + "readOnly": true, + "default": "bf:Work", + "const": "bf:Work", + "form": { + "templateOptions": { + "wrappers": [ + "hide" + ] + } + } + }, + "pid": { + "title": "Local entity PID", + "type": "string", + "minLength": 1 + }, + "deleted": { + "title": "Deletion date", + "type": "string", + "format": "date-time" + }, + "title": { + "title": "Title", + "type": "string", + "minLength": 1, + "form": { + "placeholder": "Example: Walk\u00fcre" + } + }, + "creator": { + "title": "Creator", + "type": "string", + "minLength": 1, + "form": { + "placeholder": "Example: Wagner, Richard, 1813-1883" + } + }, + "authorized_access_point": { + "title": "Authorized access point", + "type": "string", + "minLength": 1, + "readOnly": true, + "form": { + "templateOptions": { + "wrappers": [ + "hide" + ] + } + } + }, + "source_catalog": { + "title": "Source catalog", + "type": "string", + "minLength": 1, + "form": { + "placeholder": "Example: lcsh, mesh, rameau" + } + }, + "identifier": { + "$ref": "https://bib.rero.ch/schemas/common/identifier-v0.0.1.json#/identifier" + } + } +} diff --git a/rero_ils/modules/entities/local_entities/mappings/v7/local_entities/local_entity-v0.0.1.json b/rero_ils/modules/entities/local_entities/mappings/v7/local_entities/local_entity-v0.0.1.json index 3f17a16c4c..7c96955e5c 100644 --- a/rero_ils/modules/entities/local_entities/mappings/v7/local_entities/local_entity-v0.0.1.json +++ b/rero_ils/modules/entities/local_entities/mappings/v7/local_entities/local_entity-v0.0.1.json @@ -49,22 +49,140 @@ "type": { "type": "keyword" }, - "preferred_name": { - "type": "text" - }, "authorized_access_point": { "type": "text", "copy_to": [ - "autocomplete_name", - "authorized_access_point_sort" + "autocomplete_name" ] }, + "authorized_access_point_en": { + "type": "text", + "fields": { + "raw": { + "type": "keyword" + }, + "sort": { + "type": "keyword", + "normalizer": "sort_normalizer" + } + } + }, + "authorized_access_point_fr": { + "type": "text", + "fields": { + "raw": { + "type": "keyword" + }, + "sort": { + "type": "keyword", + "normalizer": "sort_normalizer" + } + } + }, + "authorized_access_point_de": { + "type": "text", + "fields": { + "raw": { + "type": "keyword" + }, + "sort": { + "type": "keyword", + "normalizer": "sort_normalizer" + } + } + }, + "authorized_access_point_it": { + "type": "text", + "fields": { + "raw": { + "type": "keyword" + }, + "sort": { + "type": "keyword", + "normalizer": "sort_normalizer" + } + } + }, + "identifier": { + "type": "object", + "properties": { + "type": { + "type": "keyword" + }, + "source": { + "type": "keyword" + }, + "value": { + "type": "text" + } + } + }, + "source_catalog": { + "type": "keyword" + }, + "name": { + "type": "text" + }, + "alternative_names": { + "type": "text" + }, + "fuller_form_of_name": { + "type": "text" + }, + "gender": { + "type": "keyword" + }, + "date_of_birth": { + "type": "keyword" + }, + "date_of_death": { + "type": "keyword" + }, + "qualifier": { + "type": "keyword" + }, + "numeration": { + "type": "keyword" + }, + "conference": { + "type": "keyword" + }, + "subordinate_unit": { + "type": "keyword" + }, + "conference_place": { + "type": "keyword" + }, + "conference_numbering": { + "type": "keyword" + }, + "conference_date": { + "type": "keyword" + }, + "start_date": { + "type": "keyword" + }, + "end_date": { + "type": "keyword" + }, + "genreForm": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "creator": { + "type": "text" + }, "document_organisation_pids": { "type": "keyword" }, "deleted": { "type": "date" }, + "resource_type": { + "type": "keyword" + }, "_created": { "type": "date" }, diff --git a/rero_ils/modules/entities/local_entities/subclasses/__init__.py b/rero_ils/modules/entities/local_entities/subclasses/__init__.py new file mode 100644 index 0000000000..1f16eb23dc --- /dev/null +++ b/rero_ils/modules/entities/local_entities/subclasses/__init__.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Local entity record subclasses.""" + +from .organisation import OrganisationLocalEntity +from .person import PersonLocalEntity +from .place import PlaceLocalEntity +from .temporal import TemporalLocalEntity +from .topic import TopicLocalEntity +from .work import WorkLocalEntity + +__all__ = [ + 'OrganisationLocalEntity', + 'PersonLocalEntity', + 'PlaceLocalEntity', + 'TemporalLocalEntity', + 'TopicLocalEntity', + 'WorkLocalEntity' +] diff --git a/rero_ils/modules/entities/local_entities/subclasses/organisation.py b/rero_ils/modules/entities/local_entities/subclasses/organisation.py new file mode 100644 index 0000000000..bd6713f6ce --- /dev/null +++ b/rero_ils/modules/entities/local_entities/subclasses/organisation.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""API for manipulating "organisation" local entities.""" + +from ..api import LocalEntity +from ...helpers import str_builder as builder + + +class OrganisationLocalEntity(LocalEntity): + """Person local entity class.""" + + def get_authorized_access_point(self, language=None): + """Get the authorized access point this local entity. + + :return return the calculated authorized access point to use. + """ + conference = [ + self.get('conference_numbering', ''), + self.get('conference_date', ''), + self.get('conference_place', ''), + ] + field_builders = [ + self.get('name'), + builder(self.get('subordinate_units'), + prefix='. ', delimiter='. '), + builder(conference, delimiter=' ; ', prefix=' (', suffix=')') + ] + return ''.join(field_builders) diff --git a/rero_ils/modules/entities/local_entities/subclasses/person.py b/rero_ils/modules/entities/local_entities/subclasses/person.py new file mode 100644 index 0000000000..e6002008bd --- /dev/null +++ b/rero_ils/modules/entities/local_entities/subclasses/person.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""API for manipulating "person" local entities.""" + +from ..api import LocalEntity +from ...helpers import str_builder as builder + + +class PersonLocalEntity(LocalEntity): + """Person local entity class.""" + + def get_authorized_access_point(self, language=None): + """Get the authorized access point this local entity. + + :return return the calculated authorized access point to use. + """ + dates = [self.get('date_of_birth', ''), self.get('date_of_death', '')] + field_builders = [ + self.get('name'), + builder(self.get('numeration'), prefix=' '), + builder(self.get('qualifier'), prefix=', '), + builder(self.get('fuller_form_of_name'), prefix=' (', suffix=')'), + builder(dates, delimiter='-', prefix=' (', suffix=')') + ] + return ''.join(field_builders) diff --git a/rero_ils/modules/entities/local_entities/subclasses/place.py b/rero_ils/modules/entities/local_entities/subclasses/place.py new file mode 100644 index 0000000000..ff29dd4fa3 --- /dev/null +++ b/rero_ils/modules/entities/local_entities/subclasses/place.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""API for manipulating "place" local entities.""" + +from ..api import LocalEntity + + +class PlaceLocalEntity(LocalEntity): + """Place local entity class.""" + + def get_authorized_access_point(self, language=None): + """Get the authorized access point this local entity. + + :return return the calculated authorized access point to use. + """ + return self.get('name') diff --git a/rero_ils/modules/entities/local_entities/subclasses/temporal.py b/rero_ils/modules/entities/local_entities/subclasses/temporal.py new file mode 100644 index 0000000000..ec8190e0b7 --- /dev/null +++ b/rero_ils/modules/entities/local_entities/subclasses/temporal.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""API for manipulating "temporal" local entities.""" + +from ..api import LocalEntity + + +class TemporalLocalEntity(LocalEntity): + """Temporal local entity class.""" + + def get_authorized_access_point(self, language=None): + """Get the authorized access point this local entity. + + :return return the calculated authorized access point to use. + """ + return self.get('name') diff --git a/rero_ils/modules/entities/local_entities/subclasses/topic.py b/rero_ils/modules/entities/local_entities/subclasses/topic.py new file mode 100644 index 0000000000..b1116a42b9 --- /dev/null +++ b/rero_ils/modules/entities/local_entities/subclasses/topic.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""API for manipulating "topic" local entities.""" + +from ..api import LocalEntity + + +class TopicLocalEntity(LocalEntity): + """Topic local entity class.""" + + def get_authorized_access_point(self, language=None): + """Get the authorized access point this local entity. + + :return return the calculated authorized access point to use. + """ + return self.get('name') diff --git a/rero_ils/modules/entities/local_entities/subclasses/work.py b/rero_ils/modules/entities/local_entities/subclasses/work.py new file mode 100644 index 0000000000..1cf65b6632 --- /dev/null +++ b/rero_ils/modules/entities/local_entities/subclasses/work.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""API for manipulating "work" local entities.""" + +from ..api import LocalEntity +from ...helpers import str_builder as builder + + +class WorkLocalEntity(LocalEntity): + """Work local entity class.""" + + def get_authorized_access_point(self, language=None): + """Get the authorized access point this local entity. + + :return return the calculated authorized access point to use. + """ + field_builders = [ + builder(self.get('creator'), suffix='. '), + self.get('title'), + ] + return ''.join(field_builders) diff --git a/rero_ils/modules/entities/models.py b/rero_ils/modules/entities/models.py index e7b16af88f..24d60f4021 100644 --- a/rero_ils/modules/entities/models.py +++ b/rero_ils/modules/entities/models.py @@ -29,3 +29,10 @@ class EntityType: TEMPORAL = 'bf:Temporal' TOPIC = 'bf:Topic' WORK = 'bf:Work' + + +class EntityResourceType: + """Class holding all available resource entity types.""" + + REMOTE = 'remote' + LOCAL = 'local' diff --git a/rero_ils/modules/entities/remote_entities/api.py b/rero_ils/modules/entities/remote_entities/api.py index e3b9261c96..5fc81418f6 100644 --- a/rero_ils/modules/entities/remote_entities/api.py +++ b/rero_ils/modules/entities/remote_entities/api.py @@ -16,7 +16,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -"""API for manipulating entities.""" +"""API for manipulating remote entities.""" import contextlib from functools import partial @@ -31,13 +31,16 @@ from rero_ils.modules.fetchers import id_fetcher from rero_ils.modules.minters import id_minter from rero_ils.modules.providers import Provider -from rero_ils.utils import get_i18n_supported_languages +from .dumpers import indexer_dumper from .models import RemoteEntityIdentifier, RemoteEntityMetadata, \ EntityUpdateAction from .utils import extract_data_from_mef_uri, get_mef_data_by_type # provider +from ..api import Entity +from ..local_entities.dumpers import replace_refs_dumper + RemoteEntityProvider = type( 'EntityProvider', (Provider,), @@ -63,13 +66,25 @@ class Meta: default_filter = None -class RemoteEntity(IlsRecord): +class RemoteEntity(IlsRecord, Entity): """Mef contribution class.""" minter = remote_entity_id_minter fetcher = remote_entity_id_fetcher provider = RemoteEntityProvider model_cls = RemoteEntityMetadata + # disable legacy replace refs + enable_jsonref = False + + def resolve(self): + """Resolve references data. + + Uses the dumper to do the job. + Mainly used by the `resolve=1` URL parameter. + + :returns: a fresh copy of the resolved data. + """ + return self.dumps(replace_refs_dumper) @classmethod def get_entity(cls, ref_type, ref_pid): @@ -139,31 +154,6 @@ def _get_mef_localized_value(self, key, language): return value return self.get(key, None) - def dumps_for_document(self): - """Transform the record into document contribution format.""" - agent = {'pid': self.pid} - for agency in current_app.config['RERO_ILS_AGENTS_SOURCES']: - if field := self.get(agency): - agent['type'] = field.get('bf:Agent', self['type']) - agent[f'id_{agency}'] = self[agency]['pid'] - - for language in get_i18n_supported_languages(): - value = self._get_mef_localized_value( - 'authorized_access_point', language) - agent[f'authorized_access_point_{language}'] = value - variant_access_points = [] - parallel_access_points = [] - for source in self.get('sources'): - variant_access_points += self[source].get( - 'variant_access_point', []) - parallel_access_points += self[source].get( - 'parallel_access_point', []) - if variant_access_points: - agent['variant_access_point'] = variant_access_points - if parallel_access_points: - agent['parallel_access_point'] = parallel_access_points - return agent - @property def organisation_pids(self): """Get organisations pids.""" @@ -295,6 +285,8 @@ class RemoteEntitiesIndexer(IlsRecordsIndexer): """Entity indexing class.""" record_cls = RemoteEntity + # data dumper for indexing + record_dumper = indexer_dumper def bulk_index(self, record_id_iterator): """Bulk index records. diff --git a/rero_ils/modules/entities/remote_entities/dumpers/__init__.py b/rero_ils/modules/entities/remote_entities/dumpers/__init__.py new file mode 100644 index 0000000000..0d21d759c3 --- /dev/null +++ b/rero_ils/modules/entities/remote_entities/dumpers/__init__.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Remote entity dumpers.""" + +from invenio_records.dumpers import Dumper + +# dumper used for indexing +from rero_ils.modules.commons.dumpers import MultiDumper, ReplaceRefsDumper +from ...dumpers import LocalizedAuthorizedAccessPointDumper +from .document import DocumentEntityDumper +from .indexer import RemoteEntityIndexerDumper + + +indexer_dumper = MultiDumper(dumpers=[ + # make a fresh copy + Dumper(), + ReplaceRefsDumper(), + RemoteEntityIndexerDumper(), + LocalizedAuthorizedAccessPointDumper() +]) + +document_dumper = MultiDumper(dumpers=[ + DocumentEntityDumper(), + LocalizedAuthorizedAccessPointDumper() +]) diff --git a/rero_ils/modules/entities/remote_entities/dumpers/document.py b/rero_ils/modules/entities/remote_entities/dumpers/document.py new file mode 100644 index 0000000000..ae9707e954 --- /dev/null +++ b/rero_ils/modules/entities/remote_entities/dumpers/document.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Indexing dumper.""" +from flask import current_app +from invenio_records.dumpers import Dumper + + +class DocumentEntityDumper(Dumper): + """Remote entity indexer.""" + + def dump(self, record, data): + """Dump a remote entity instance. + + :param record: The record to dump. + :param data: The initial dump data passed in by ``record.dumps()``. + """ + data = {'pid': record.pid} + for agency in current_app.config['RERO_ILS_AGENTS_SOURCES']: + if field := record.get(agency): + data['type'] = field.get('bf:Agent', record['type']) + data[f'id_{agency}'] = record[agency]['pid'] + + variant_access_points = [] + parallel_access_points = [] + for source in record.get('sources'): + variant_access_points += record[source].get( + 'variant_access_point', []) + parallel_access_points += record[source].get( + 'parallel_access_point', []) + if variant_access_points: + data['variant_access_point'] = variant_access_points + if parallel_access_points: + data['parallel_access_point'] = parallel_access_points + + return data diff --git a/rero_ils/modules/entities/remote_entities/dumpers/indexer.py b/rero_ils/modules/entities/remote_entities/dumpers/indexer.py new file mode 100644 index 0000000000..9e037d6bea --- /dev/null +++ b/rero_ils/modules/entities/remote_entities/dumpers/indexer.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Indexing dumper.""" + +from invenio_records.dumpers import Dumper + +from rero_ils.modules.entities.models import EntityResourceType + + +class RemoteEntityIndexerDumper(Dumper): + """Remote entity indexer.""" + + def dump(self, record, data): + """Dump a remote entity instance. + + :param record: The record to dump. + :param data: The initial dump data passed in by ``record.dumps()``. + """ + data['organisations'] = record.organisation_pids + data['resource_type'] = EntityResourceType.REMOTE + return data diff --git a/rero_ils/modules/entities/remote_entities/jsonschemas/remote_entities/remote_entity-v0.0.1.json b/rero_ils/modules/entities/remote_entities/jsonschemas/remote_entities/remote_entity-v0.0.1.json index 8da82005ab..83d391a0fa 100644 --- a/rero_ils/modules/entities/remote_entities/jsonschemas/remote_entities/remote_entity-v0.0.1.json +++ b/rero_ils/modules/entities/remote_entities/jsonschemas/remote_entities/remote_entity-v0.0.1.json @@ -84,7 +84,8 @@ }, "deleted": { "title": "Deletion date", - "type": "string" + "type": "string", + "format": "date-time" } } } diff --git a/rero_ils/modules/entities/remote_entities/mappings/v7/remote_entities/remote_entity-v0.0.1.json b/rero_ils/modules/entities/remote_entities/mappings/v7/remote_entities/remote_entity-v0.0.1.json index c363fd7d9b..d3604a95bd 100644 --- a/rero_ils/modules/entities/remote_entities/mappings/v7/remote_entities/remote_entity-v0.0.1.json +++ b/rero_ils/modules/entities/remote_entities/mappings/v7/remote_entities/remote_entity-v0.0.1.json @@ -36,17 +36,53 @@ "analyzer": "autocomplete", "search_analyzer": "search_autocomplete" }, - "gnd_authorized_access_point_sort": { - "type": "keyword", - "normalizer": "sort_normalizer" + "authorized_access_point_en": { + "type": "text", + "fields": { + "raw": { + "type": "keyword" + }, + "sort": { + "type": "keyword", + "normalizer": "sort_normalizer" + } + } }, - "idref_authorized_access_point_sort": { - "type": "keyword", - "normalizer": "sort_normalizer" + "authorized_access_point_fr": { + "type": "text", + "fields": { + "raw": { + "type": "keyword" + }, + "sort": { + "type": "keyword", + "normalizer": "sort_normalizer" + } + } }, - "rero_authorized_access_point_sort": { - "type": "keyword", - "normalizer": "sort_normalizer" + "authorized_access_point_de": { + "type": "text", + "fields": { + "raw": { + "type": "keyword" + }, + "sort": { + "type": "keyword", + "normalizer": "sort_normalizer" + } + } + }, + "authorized_access_point_it": { + "type": "text", + "fields": { + "raw": { + "type": "keyword" + }, + "sort": { + "type": "keyword", + "normalizer": "sort_normalizer" + } + } }, "$schema": { "type": "keyword" @@ -54,6 +90,9 @@ "pid": { "type": "keyword" }, + "resource_type": { + "type": "keyword" + }, "type": { "type": "keyword" }, @@ -120,8 +159,7 @@ "authorized_access_point": { "type": "text", "copy_to": [ - "autocomplete_name", - "gnd_authorized_access_point_sort" + "autocomplete_name" ] }, "qualifier": { @@ -331,8 +369,7 @@ "authorized_access_point": { "type": "text", "copy_to": [ - "autocomplete_name", - "idref_authorized_access_point_sort" + "autocomplete_name" ] }, "qualifier": { @@ -542,8 +579,7 @@ "authorized_access_point": { "type": "text", "copy_to": [ - "autocomplete_name", - "rero_authorized_access_point_sort" + "autocomplete_name" ] }, "qualifier": { diff --git a/rero_ils/modules/entities/replace.py b/rero_ils/modules/entities/remote_entities/replace.py similarity index 97% rename from rero_ils/modules/entities/replace.py rename to rero_ils/modules/entities/remote_entities/replace.py index 9567db57f5..e34123cafd 100644 --- a/rero_ils/modules/entities/replace.py +++ b/rero_ils/modules/entities/remote_entities/replace.py @@ -26,11 +26,12 @@ from flask import current_app from rero_ils.modules.documents.api import Document, DocumentsSearch -from rero_ils.modules.entities.api import Entity -from rero_ils.modules.entities.logger import create_logger from rero_ils.modules.utils import get_mef_url, get_timestamp, \ requests_retry_session, set_timestamp +from .api import RemoteEntity +from ..logger import create_logger + class ReplaceIdentifiedBy(object): """Entity replace identifiedBy with $ref. @@ -137,15 +138,15 @@ def _create_entity(self, mef_type, mef_data): :param mef_type: MEF type (agent, concept) :param mef_data: MEF data for entity. """ - if not Entity.get_record_by_pid(mef_data['pid']): + if not RemoteEntity.get_record_by_pid(mef_data['pid']): if not self.dry_run: new_mef_data = deepcopy(mef_data) fields_to_remove = ['$schema', '_created', '_updated'] for field in fields_to_remove: new_mef_data.pop(field, None) # TODO: try to optimize with parent commit and reindex - # bulk operation - Entity.create( + # bulk operation + RemoteEntity.create( data=new_mef_data, dbcommit=True, reindex=True diff --git a/rero_ils/modules/entities/serializers/base.py b/rero_ils/modules/entities/serializers/base.py index ce61610412..a92a1f76b8 100644 --- a/rero_ils/modules/entities/serializers/base.py +++ b/rero_ils/modules/entities/serializers/base.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- # # RERO ILS -# Copyright (C) 2019-2022 RERO -# Copyright (C) 2019-2022 UCLouvain +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by diff --git a/rero_ils/modules/ext.py b/rero_ils/modules/ext.py index 1f83c5d375..186083cb1c 100644 --- a/rero_ils/modules/ext.py +++ b/rero_ils/modules/ext.py @@ -57,8 +57,6 @@ budget_is_active_changed from rero_ils.modules.collections.listener import enrich_collection_data from rero_ils.modules.ebooks.receivers import publish_harvested_records -from rero_ils.modules.entities.remote_entities.listener import \ - enrich_remote_entities_data from rero_ils.modules.holdings.listener import enrich_holding_data, \ update_items_locations_and_types from rero_ils.modules.ill_requests.listener import enrich_ill_request_data @@ -291,7 +289,6 @@ def register_signals(self, app): before_record_index.connect(enrich_acq_order_line_data, sender=app) before_record_index.connect(enrich_collection_data, sender=app) before_record_index.connect(enrich_loan_data, sender=app) - before_record_index.connect(enrich_remote_entities_data, sender=app) before_record_index.connect(enrich_item_data, sender=app) before_record_index.connect(enrich_patron_data, sender=app) before_record_index.connect(enrich_holding_data, sender=app) diff --git a/scripts/setup b/scripts/setup index 707309aece..072e3adb0c 100755 --- a/scripts/setup +++ b/scripts/setup @@ -355,7 +355,7 @@ else REMOTE_ENTITIES=${DATA_PATH}/remote_entities_small.json fi -if ${LOADCONTRIBUTIONS} +if ${LOAD_REMOTE_ENTITIES} then info_msg "- REMOTE REMOTE_ENTITIES: ${REMOTE_ENTITIES} ${CREATE_LAZY} ${DONT_STOP}" eval ${PREFIX} invenio reroils fixtures create --pid_type rement --schema 'https://bib.rero.ch/schemas/remote_entities/remote_entity-v0.0.1.json' ${REMOTE_ENTITIES} --append ${CREATE_LAZY} ${DONT_STOP} diff --git a/setup.py b/setup.py index dd1f2e033c..89f30b0028 100644 --- a/setup.py +++ b/setup.py @@ -180,7 +180,7 @@ def run(self): 'item_types = rero_ils.modules.item_types.models', 'items = rero_ils.modules.items.models', 'libraries = rero_ils.modules.libraries.models', - 'local_entity = rero_ils.modules.local_entities.models', + 'local_entity = rero_ils.modules.entities.local_entities.models', 'local_fields = rero_ils.modules.local_fields.models', 'locations = rero_ils.modules.locations.models', 'remote_entity = rero_ils.modules.entities.remote_entities.models', @@ -212,7 +212,7 @@ def run(self): 'item_id = rero_ils.modules.items.api:item_id_minter', 'item_type_id = rero_ils.modules.item_types.api:item_type_id_minter', 'library_id = rero_ils.modules.libraries.api:library_id_minter', - 'local_entity_id = rero_ils.modules.local_entities.api:local_entity_id_minter', + 'local_entity_id = rero_ils.modules.entities.local_entities.api:local_entity_id_minter', 'local_field_id = rero_ils.modules.local_fields.api:local_field_id_minter', 'location_id = rero_ils.modules.locations.api:location_id_minter', 'notification_id = rero_ils.modules.notifications.api:notification_id_minter', @@ -237,7 +237,7 @@ def run(self): 'collection_id = rero_ils.modules.collections.api:collection_id_fetcher', 'document_id = rero_ils.modules.documents.api:document_id_fetcher', 'remote_entity_id = rero_ils.modules.entities.remote_entities.api:remote_entity_id_fetcher', - 'local_entity_id = rero_ils.modules.local_entities.api:local_entity_id_fetcher', + 'local_entity_id = rero_ils.modules.entities.local_entities.api:local_entity_id_fetcher', 'holding_id = rero_ils.modules.holdings.api:holding_id_fetcher', 'ill_request_id = rero_ils.modules.ill_requests.api:ill_request_id_fetcher', 'item_id = rero_ils.modules.items.api:item_id_fetcher', @@ -275,7 +275,7 @@ def run(self): 'items = rero_ils.modules.items.jsonschemas', 'libraries = rero_ils.modules.libraries.jsonschemas', 'loans = rero_ils.modules.loans.jsonschemas', - 'local_entities = rero_ils.modules.local_entities.jsonschemas', + 'local_entities = rero_ils.modules.entities.local_entities.jsonschemas', 'local_fields = rero_ils.modules.local_fields.jsonschemas', 'locations = rero_ils.modules.locations.jsonschemas', 'notifications = rero_ils.modules.notifications.jsonschemas', @@ -308,7 +308,7 @@ def run(self): 'items = rero_ils.modules.items.mappings', 'libraries = rero_ils.modules.libraries.mappings', 'loans = rero_ils.modules.loans.mappings', - 'local_entities = rero_ils.modules.local_entities.mappings', + 'local_entities = rero_ils.modules.entities.local_entities.mappings', 'local_fields = rero_ils.modules.local_fields.mappings', 'locations = rero_ils.modules.locations.mappings', 'notifications = rero_ils.modules.notifications.mappings', diff --git a/tests/api/local_entities/test_local_entities_extensions.py b/tests/api/local_entities/test_local_entities_extensions.py new file mode 100644 index 0000000000..f5c1427ae1 --- /dev/null +++ b/tests/api/local_entities/test_local_entities_extensions.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Tests `LocalEntity` authorized access point.""" + + +def test_local_entities_authorized_access_point(local_entity_person, + local_entity_person2, + local_entity_org, + local_entity_org2): + """Test authorized access point calculation.""" + dumped_record = local_entity_person.dumps() + assert dumped_record['authorized_access_point'] == 'Loy, Georg (1881-1968)' + dumped_record = local_entity_person2.dumps() + assert dumped_record['authorized_access_point'] == \ + 'William III, King of England (1650-1702)' + + dumped_record = local_entity_org.dumps() + assert dumped_record['authorized_access_point'] == \ + 'Convegno internazionale di Italianistica' + # + dumped_record = local_entity_org2.dumps() + assert dumped_record['authorized_access_point'] == \ + 'Catholic Church. Concilium Plenarium Americae ' \ + 'Latinae (5th ; 1899 ; Rome, Italy)' diff --git a/tests/api/local_entities/test_local_entities_rest.py b/tests/api/local_entities/test_local_entities_rest.py index 37d4d3636d..3b0bbf49fa 100644 --- a/tests/api/local_entities/test_local_entities_rest.py +++ b/tests/api/local_entities/test_local_entities_rest.py @@ -26,6 +26,7 @@ from rero_ils.modules.entities.models import EntityType from rero_ils.modules.entities.local_entities.api import LocalEntity +from rero_ils.modules.entities.local_entities.dumpers import indexer_dumper def test_local_entities_permissions(client, roles, local_entity_person, @@ -81,7 +82,8 @@ def test_local_entities_get(client, local_entity_person): data = get_json(res) entity_person = local_entity_person.replace_refs() entity_person['type'] = EntityType.PERSON - assert data['hits']['hits'][0]['metadata'] == entity_person.dumps() + assert data['hits']['hits'][0]['metadata'] == \ + entity_person.dumps(dumper=indexer_dumper) @mock.patch('invenio_records_rest.views.verify_record_permission', @@ -112,7 +114,7 @@ def test_local_entities_post_put_delete(client, local_entity_person_data, # Update record/PUT data = local_entity_data - data['preferred_name'] = 'Test Name' + data['name'] = 'Test Name' res = client.put( item_url, data=json.dumps(data), @@ -122,19 +124,19 @@ def test_local_entities_post_put_delete(client, local_entity_person_data, # Check that the returned record matches the given data data = get_json(res) - assert data['metadata']['preferred_name'] == 'Test Name' + assert data['metadata']['name'] == 'Test Name' + # Check value from record API res = client.get(item_url) assert res.status_code == 200 - data = get_json(res) - assert data['metadata']['preferred_name'] == 'Test Name' + assert data['metadata']['name'] == 'Test Name' + # Check value from Elasticsearch res = client.get(list_url) assert res.status_code == 200 - data = get_json(res)['hits']['hits'][0] - assert data['metadata']['preferred_name'] == 'Test Name' + assert data['metadata']['name'] == 'Test Name' # Delete record/DELETE res = client.delete(item_url) diff --git a/tests/api/remote_entities/test_remote_entities_rest.py b/tests/api/remote_entities/test_remote_entities_rest.py index aae71d0b0d..118c2b4f5a 100644 --- a/tests/api/remote_entities/test_remote_entities_rest.py +++ b/tests/api/remote_entities/test_remote_entities_rest.py @@ -23,6 +23,7 @@ from utils import get_json, mock_response, postdata, to_relative_url from rero_ils.modules.entities.models import EntityType +from rero_ils.modules.entities.remote_entities.dumpers import indexer_dumper def test_remote_entities_permissions(client, entity_person, json_header): @@ -74,7 +75,8 @@ def test_remote_entities_get(client, entity_person): entity_person = entity_person.replace_refs() entity_person['organisations'] = entity_person.organisation_pids entity_person['type'] = EntityType.PERSON - assert data['hits']['hits'][0]['metadata'] == entity_person.replace_refs() + assert data['hits']['hits'][0]['metadata'] == \ + entity_person.dumps(indexer_dumper) @mock.patch('rero_ils.modules.decorators.login_and_librarian', diff --git a/tests/data/data.json b/tests/data/data.json index bd22d89a92..5f6a63259d 100644 --- a/tests/data/data.json +++ b/tests/data/data.json @@ -1846,15 +1846,48 @@ "locent_pers": { "$schema": "https://bib.rero.ch/schemas/local_entities/local_entity-v0.0.1.json", "pid": "locent_pers", - "preferred_name": "Loy, Georg", + "name": "Loy, Georg", + "date_of_birth": "1881", + "date_of_death": "1968", + "type": "bf:Person" + }, + "locent_pers2": { + "$schema": "https://bib.rero.ch/schemas/local_entities/local_entity-v0.0.1.json", + "pid": "locent_pers2", + "name": "William", + "date_of_birth": "1650", + "date_of_death": "1702", + "numeration": "III", + "qualifier": "King of England", "type": "bf:Person" }, "locent_org": { "$schema": "https://bib.rero.ch/schemas/local_entities/local_entity-v0.0.1.json", "pid": "locent_org", - "preferred_name": "Convegno internazionale di Italianistica", + "name": "Convegno internazionale di Italianistica", + "conference": false, + "type": "bf:Organisation" + }, + "locent_org2": { + "$schema": "https://bib.rero.ch/schemas/local_entities/local_entity-v0.0.1.json", + "pid": "locent_org2", + "name": "Catholic Church", + "subordinate_units": [ + "Concilium Plenarium Americae Latinae" + ], + "conference_numbering": "5th", + "conference_date": "1899", + "conference_place": "Rome, Italy", + "conference": false, "type": "bf:Organisation" }, + "locent_work": { + "$schema": "https://bib.rero.ch/schemas/local_entities/local_entity-v0.0.1.json", + "pid": "locent_work", + "title": "Preludes", + "creator": "Chopin, Fre\u0301de\u0301ric (1810-1849)", + "type": "bf:Work" + }, "doc1": { "$schema": "https://bib.rero.ch/schemas/documents/document-v0.0.1.json", "pid": "doc1", diff --git a/tests/fixtures/metadata.py b/tests/fixtures/metadata.py index 744b091acf..4767066a25 100644 --- a/tests/fixtures/metadata.py +++ b/tests/fixtures/metadata.py @@ -465,12 +465,30 @@ def local_entity_person_data(data): return deepcopy(data.get('locent_pers')) +@pytest.fixture(scope="module") +def local_entity_person2_data(data): + """Load mef contribution person data.""" + return deepcopy(data.get('locent_pers2')) + + @pytest.fixture(scope="module") def local_entity_org_data(data): """Load mef contribution person data.""" return deepcopy(data.get('locent_org')) +@pytest.fixture(scope="module") +def local_entity_org2_data(data): + """Load mef contribution person data.""" + return deepcopy(data.get('locent_org2')) + + +@pytest.fixture(scope="module") +def local_entity_work_data(data): + """Load mef contribution person data.""" + return deepcopy(data.get('locent_work')) + + @pytest.fixture(scope="module") def local_entity_person(app, local_entity_person_data): """Create mef person record.""" @@ -483,6 +501,18 @@ def local_entity_person(app, local_entity_person_data): return pers +@pytest.fixture(scope="module") +def local_entity_person2(app, local_entity_person2_data): + """Create mef person record.""" + pers = LocalEntity.create( + data=local_entity_person2_data, + delete_pid=False, + dbcommit=True, + reindex=True) + flush_index(LocalEntitiesSearch.Meta.index) + return pers + + @pytest.fixture(scope="module") def local_entity_org(app, local_entity_org_data): """Create mef person record.""" @@ -495,6 +525,18 @@ def local_entity_org(app, local_entity_org_data): return org +@pytest.fixture(scope="module") +def local_entity_org2(app, local_entity_org2_data): + """Create mef person record.""" + org = LocalEntity.create( + data=local_entity_org2_data, + delete_pid=False, + dbcommit=True, + reindex=True) + flush_index(LocalEntitiesSearch.Meta.index) + return org + + @pytest.fixture(scope="module") @mock.patch('requests.Session.get') def document_ref(mock_contributions_mef_get, diff --git a/tests/ui/local_entities/test_local_entities_dumpers.py b/tests/ui/local_entities/test_local_entities_dumpers.py new file mode 100644 index 0000000000..45371061a6 --- /dev/null +++ b/tests/ui/local_entities/test_local_entities_dumpers.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2022 RERO +# Copyright (C) 2022 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Items Record dumper tests.""" + +from rero_ils.modules.entities.local_entities.dumpers import document_dumper + + +def test_local_entities_document_dumper(local_entity_person2): + """Test document dumper.""" + + dumped_record = local_entity_person2.dumps(dumper=document_dumper) + authorized_access_point = 'William III, King of England (1650-1702)' + for field in [ + 'authorized_access_point', + 'authorized_access_point_de', + 'authorized_access_point_en', + 'authorized_access_point_fr', + 'authorized_access_point_it' + ]: + assert dumped_record[field] == authorized_access_point diff --git a/tests/ui/local_entities/test_local_entities_jsonresolver.py b/tests/ui/local_entities/test_local_entities_jsonresolver.py new file mode 100644 index 0000000000..f233f37f61 --- /dev/null +++ b/tests/ui/local_entities/test_local_entities_jsonresolver.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019 RERO +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Item JSON Resolver tests.""" + +import pytest +from invenio_records.api import Record +from jsonref import JsonRefError + + +def test_local_entities_jsonresolver(local_entity_person2): + """Test local entity json resolver.""" + rec = Record.create({ + 'local_entity': { + '$ref': 'https://bib.rero.ch/api/local_entities/locent_pers2' + } + }) + assert rec.replace_refs().get('local_entity') == { + 'pid': 'locent_pers2', + 'type': 'locent' + } + + # deleted record + local_entity_person2.delete() + with pytest.raises(JsonRefError): + rec.replace_refs().dumps() + + # non existing record + rec = Record.create({ + 'local_entity': {'$ref': 'https://bib.rero.ch/api/local_entities/n_e'} + }) + with pytest.raises(JsonRefError): + rec.replace_refs().dumps() diff --git a/tests/ui/local_entities/test_local_entities_mapping.py b/tests/ui/local_entities/test_local_entities_mapping.py new file mode 100644 index 0000000000..b25a0b9aed --- /dev/null +++ b/tests/ui/local_entities/test_local_entities_mapping.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019 RERO +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Item record mapping tests.""" +from utils import get_mapping + +from rero_ils.modules.entities.local_entities.api import LocalEntitiesSearch, \ + LocalEntity + + +def test_local_entities_es_mapping(app, local_entity_person2_data): + """Test local entity elasticsearch mapping.""" + search = LocalEntitiesSearch() + mapping = get_mapping(search.Meta.index) + assert mapping + LocalEntity.create( + local_entity_person2_data, + dbcommit=True, + reindex=True, + delete_pid=True + ) + assert mapping == get_mapping(search.Meta.index) diff --git a/tests/ui/remote_entities/test_entities_api.py b/tests/ui/remote_entities/test_entities_api.py index e920f2fce2..451087a276 100644 --- a/tests/ui/remote_entities/test_entities_api.py +++ b/tests/ui/remote_entities/test_entities_api.py @@ -29,7 +29,7 @@ from rero_ils.modules.documents.api import Document, DocumentsSearch from rero_ils.modules.entities.remote_entities.api import \ RemoteEntitiesSearch, RemoteEntity, remote_entity_id_fetcher -from rero_ils.modules.entities.replace import ReplaceIdentifiedBy +from rero_ils.modules.entities.remote_entities.replace import ReplaceIdentifiedBy from rero_ils.modules.entities.remote_entities.sync import SyncEntity diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index d658d97e7b..32b0f29d1f 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -213,8 +213,8 @@ def patron_martigny_data_tmp_with_id(patron_martigny_data_tmp): @pytest.fixture() -def entities_schema(monkeypatch): - """Entity Jsonschema for records.""" +def remote_entities_schema(monkeypatch): + """Remote entity Jsonschema for records.""" schema_in_bytes = resource_string( 'rero_ils.modules.entities.remote_entities.jsonschemas', '/remote_entities/remote_entity-v0.0.1.json' @@ -222,6 +222,16 @@ def entities_schema(monkeypatch): return get_schema(monkeypatch, schema_in_bytes) +@pytest.fixture() +def local_entities_schema(monkeypatch): + """Local entity Jsonschema for records.""" + schema_in_bytes = resource_string( + 'rero_ils.modules.entities.local_entities.jsonschemas', + '/local_entities/local_entity-v0.0.1.json' + ) + return get_schema(monkeypatch, schema_in_bytes) + + @pytest.fixture() def document_schema(monkeypatch): """Jsonschema for documents.""" diff --git a/tests/unit/test_contributions_jsonschema.py b/tests/unit/test_contributions_jsonschema.py index b0c3c94871..32b200cf41 100644 --- a/tests/unit/test_contributions_jsonschema.py +++ b/tests/unit/test_contributions_jsonschema.py @@ -24,13 +24,13 @@ from jsonschema.exceptions import ValidationError -def test_required(entities_schema, entity_person_data_tmp): +def test_required(remote_entities_schema, entity_person_data_tmp): '''Test required for patron jsonschemas.''' - validate(entity_person_data_tmp, entities_schema) + validate(entity_person_data_tmp, remote_entities_schema) with pytest.raises(ValidationError): - validate({}, entities_schema) - validate(entity_person_data_tmp, entities_schema) + validate({}, remote_entities_schema) + validate(entity_person_data_tmp, remote_entities_schema) with pytest.raises(ValidationError): validate({ @@ -39,8 +39,8 @@ def test_required(entities_schema, entity_person_data_tmp): 'sources': [ 'rero', 'gnd' - ]}, entities_schema) - validate(entity_person_data_tmp, entities_schema) + ]}, remote_entities_schema) + validate(entity_person_data_tmp, remote_entities_schema) with pytest.raises(ValidationError): validate({ @@ -50,8 +50,8 @@ def test_required(entities_schema, entity_person_data_tmp): 'sources': [ 'rero', 'gnd' - ]}, entities_schema) - validate(entity_person_data_tmp, entities_schema) + ]}, remote_entities_schema) + validate(entity_person_data_tmp, remote_entities_schema) with pytest.raises(ValidationError): validate({ @@ -59,5 +59,5 @@ def test_required(entities_schema, entity_person_data_tmp): 'remote_entity-v0.0.1.json', 'pid': 'ent_pers', 'viaf_pid': '56597999' - }, entities_schema) - validate(entity_person_data_tmp, entities_schema) + }, remote_entities_schema) + validate(entity_person_data_tmp, remote_entities_schema) diff --git a/tests/unit/test_local_entities_jsonschema.py b/tests/unit/test_local_entities_jsonschema.py new file mode 100644 index 0000000000..3d0e4e3258 --- /dev/null +++ b/tests/unit/test_local_entities_jsonschema.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019 RERO +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""item JSON schema tests.""" + +from __future__ import absolute_import, print_function + +import pytest +from jsonschema import validate +from jsonschema.exceptions import ValidationError + + +def test_validate(local_entities_schema, local_entity_person): + """Test required for item jsonschemas.""" + validate(local_entity_person, local_entities_schema) + with pytest.raises(ValidationError): + validate({}, local_entities_schema) diff --git a/tests/utils.py b/tests/utils.py index 4a503efc53..0c94cdf490 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -267,15 +267,21 @@ def json(self): return self.json_data ref_split = args[0].split('/') + # TODO: find a better way to determine name and path. if ref_split[-2] == 'common': path = 'rero_ils.jsonschemas' name = 'common/{name}'.format( name=ref_split[-1] ) else: - path = 'rero_ils.modules.{type}.jsonschemas'.format( - type=ref_split[-2] - ) + if ref_split[-2] in ['remote_entities', 'local_entities']: + path = 'rero_ils.modules.entities.{type}.jsonschemas'.format( + type=ref_split[-2] + ) + else: + path = 'rero_ils.modules.{type}.jsonschemas'.format( + type=ref_split[-2] + ) name = '{type}/{name}'.format( type=ref_split[-2], name=ref_split[-1] @@ -303,8 +309,8 @@ def get_schema(monkeypatch, schema_in_bytes): """ # apply the monkeypatch for requests.get to mocked_requests_get monkeypatch.setattr(requests, "get", mocked_requests_get) - schema = jsonref.loads(schema_in_bytes.decode('utf8')) + # Replace all remaining $refs while schema != jsonref.loads(jsonref.dumps(schema)): schema = jsonref.loads(jsonref.dumps(schema)) From 5e564a64ef64c5d023f4b1cdc783824272cf63b6 Mon Sep 17 00:00:00 2001 From: Renaud Michotte Date: Wed, 21 Jun 2023 16:45:31 +0200 Subject: [PATCH 05/14] document: update type of agent entity field into document JSON schema * Updates pattern allowed for linked entity. * Updates linked entity widget configuration. Co-Authored-by: Renaud Michotte --- rero_ils/config.py | 2 +- ...ument_contribution_entity_link-v0.0.1.json | 40 +++++++-------- .../document_genre_form_link-v0.0.1.json | 32 ++++++------ .../document_subjects_entity_link-v0.0.1.json | 49 ++++++++++--------- tests/data/data.json | 10 ++-- tests/fixtures/metadata.py | 8 +-- tests/ui/remote_entities/test_entities_api.py | 3 +- .../ui/test_invenio_celery_tasks_endpoints.py | 7 +-- 8 files changed, 77 insertions(+), 74 deletions(-) diff --git a/rero_ils/config.py b/rero_ils/config.py index a0d541a03a..6551bd65db 100644 --- a/rero_ils/config.py +++ b/rero_ils/config.py @@ -432,7 +432,7 @@ def _(x): 'enabled': False, }, 'replace-identified-by': { - 'task': 'rero_ils.modules.entities.tasks.replace_identified_by', + 'task': 'rero_ils.modules.entities.remote_entities.tasks.replace_identified_by', 'schedule': crontab(minute=0, hour=3, day_of_week=6), # Every Saturday at 03:00 UTC, 'enabled': False, }, diff --git a/rero_ils/modules/documents/jsonschemas/documents/document_contribution_entity_link-v0.0.1.json b/rero_ils/modules/documents/jsonschemas/documents/document_contribution_entity_link-v0.0.1.json index 9c9f47c93f..e271ae6263 100644 --- a/rero_ils/modules/documents/jsonschemas/documents/document_contribution_entity_link-v0.0.1.json +++ b/rero_ils/modules/documents/jsonschemas/documents/document_contribution_entity_link-v0.0.1.json @@ -12,28 +12,28 @@ "$ref": { "title": "Agent", "type": "string", - "pattern": "^https://mef.rero.ch/api/agents/(gnd|idref|rero)/.*?$", - "form": { - "type": "remoteTypeahead", - "remoteTypeahead": { - "enableGroupField": true, - "type": "mef", - "filters": { - "default": "bf:Person", - "options": [ - { - "label": "Person", - "value": "bf:Person" - }, - { - "label": "bf:Organisation", - "value": "bf:Organisation" - } + "pattern": "^(https://mef.rero.ch/api/agents/(gnd|idref|rero)/.*|https://bib.rero.ch/api/local_entities/.*?)$", + "widget": { + "formlyConfig": { + "type": "entityTypeahead", + "templateOptions": { + "filters": { + "options": [ + { + "label": "Person", + "value": "bf:Person" + }, + { + "label": "bf:Organisation", + "value": "bf:Organisation" + } + ] + }, + "itemCssClass": "col-12", + "wrappers": [ + "form-field-horizontal" ] } - }, - "templateOptions": { - "itemCssClass": "col-lg-12" } } }, diff --git a/rero_ils/modules/documents/jsonschemas/documents/document_genre_form_link-v0.0.1.json b/rero_ils/modules/documents/jsonschemas/documents/document_genre_form_link-v0.0.1.json index f840f1c728..311cb78d1c 100644 --- a/rero_ils/modules/documents/jsonschemas/documents/document_genre_form_link-v0.0.1.json +++ b/rero_ils/modules/documents/jsonschemas/documents/document_genre_form_link-v0.0.1.json @@ -12,24 +12,24 @@ "$ref": { "title": "Genre, form", "type": "string", - "pattern": "^https://mef.rero.ch/api/concepts/idref/.*?$", - "form": { - "type": "remoteTypeahead", - "remoteTypeahead": { - "enableGroupField": true, - "type": "mef", - "filters": { - "default": "concepts-genreForm", - "options": [ - { - "label": "Genre, form", - "value": "concepts-genreForm" - } + "pattern": "^(https://mef.rero.ch/api/concepts/(gnd|idref|rero)/.*|https://bib.rero.ch/api/local_entities/.*?)$", + "widget": { + "formlyConfig": { + "type": "entityTypeahead", + "templateOptions": { + "filters": { + "options": [ + { + "label": "Genre, form", + "value": "concepts-genreForm" + } + ] + }, + "itemCssClass": "col-12", + "wrappers": [ + "form-field-horizontal" ] } - }, - "templateOptions": { - "itemCssClass": "col-lg-12" } } }, diff --git a/rero_ils/modules/documents/jsonschemas/documents/document_subjects_entity_link-v0.0.1.json b/rero_ils/modules/documents/jsonschemas/documents/document_subjects_entity_link-v0.0.1.json index 8029aaee8f..8b906ada4d 100644 --- a/rero_ils/modules/documents/jsonschemas/documents/document_subjects_entity_link-v0.0.1.json +++ b/rero_ils/modules/documents/jsonschemas/documents/document_subjects_entity_link-v0.0.1.json @@ -12,32 +12,33 @@ "$ref": { "title": "Subject", "type": "string", - "pattern": "^https://mef.rero.ch/api/(agents|concepts)/(gnd|idref|rero)/.*?$", - "form": { - "type": "remoteTypeahead", - "remoteTypeahead": { - "enableGroupField": true, - "type": "mef", - "filters": { - "default": "bf:Topic", - "options": [ - { - "label": "Topic", - "value": "bf:Topic" - }, - { - "label": "Person", - "value": "bf:Person" - }, - { - "label": "bf:Organisation", - "value": "bf:Organisation" - } + "pattern": "^(https://mef.rero.ch/api/(agents|concepts)/(gnd|idref|rero)/.*|https://bib.rero.ch/api/local_entities/.*?)$", + "widget": { + "formlyConfig": { + "type": "entityTypeahead", + "templateOptions": { + "filters": { + "default": "bf:Topic", + "options": [ + { + "label": "Topic", + "value": "bf:Topic" + }, + { + "label": "Person", + "value": "bf:Person" + }, + { + "label": "bf:Organisation", + "value": "bf:Organisation" + } + ] + }, + "itemCssClass": "col-12", + "wrappers": [ + "form-field-horizontal" ] } - }, - "templateOptions": { - "itemCssClass": "col-lg-12" } } }, diff --git a/tests/data/data.json b/tests/data/data.json index 5f6a63259d..76b075fd7b 100644 --- a/tests/data/data.json +++ b/tests/data/data.json @@ -1580,7 +1580,7 @@ "viaf_pid": "70119347" }, "ent_pers2": { - "$schema": "https://bib.rero.ch/schemas/entities/entity-v0.0.1.json", + "$schema": "https://bib.rero.ch/schemas/remote_entities/remote_entity-v0.0.1.json", "pid": "ent_pers2", "rero": { "$schema": "https://mef.rero.ch/schemas/agents_rero/rero-agent-v0.0.1.json", @@ -1725,7 +1725,7 @@ ] }, "ent_topic2": { - "$schema": "https://bib.rero.ch/schemas/entities/entity-v0.0.1.json", + "$schema": "https://bib.rero.ch/schemas/remote_entities/remote_entity-v0.0.1.json", "pid": "ent_topic2", "rero": { "$schema": "https://mef.rero.ch/schemas/concepts_rero/rero-concept-v0.0.1.json", @@ -1766,7 +1766,7 @@ ] }, "ent_pers_all": { - "$schema": "https://bib.rero.ch/schemas/entities/entity-v0.0.1.json", + "$schema": "https://bib.rero.ch/schemas/remote_entities/remote_entity-v0.0.1.json", "type": "bf:Person", "gnd": { "$schema": "https://mef.rero.ch/schemas/agents_gnd/gnd-agent-v0.0.1.json", @@ -1826,7 +1826,7 @@ "viaf_pid": "108158722" }, "ent_pers_rero": { - "$schema": "https://bib.rero.ch/schemas/entities/entity-v0.0.1.json", + "$schema": "https://bib.rero.ch/schemas/remote_entities/remote_entity-v0.0.1.json", "type": "bf:Person", "pid": "ent_pers_rero", "rero": { @@ -5016,4 +5016,4 @@ "home_phone": "+012024561414", "keep_history": true } -} \ No newline at end of file +} diff --git a/tests/fixtures/metadata.py b/tests/fixtures/metadata.py index 4767066a25..0eae8b21a1 100644 --- a/tests/fixtures/metadata.py +++ b/tests/fixtures/metadata.py @@ -354,12 +354,12 @@ def entity_person_data_all(data): @pytest.fixture(scope="module") def entity_person_all(app, entity_person_data_all): """Load contribution person record.""" - cont = Entity.create( + cont = RemoteEntity.create( data=entity_person_data_all, delete_pid=False, dbcommit=True, reindex=True) - flush_index(EntitiesSearch.Meta.index) + flush_index(RemoteEntitiesSearch.Meta.index) return cont @@ -372,12 +372,12 @@ def entity_person_rero_data(data): @pytest.fixture(scope="module") def entity_person_rero(app, entity_person_rero_data): """Create mef person record.""" - pers = Entity.create( + pers = RemoteEntity.create( data=entity_person_rero_data, delete_pid=False, dbcommit=True, reindex=True) - flush_index(EntitiesSearch.Meta.index) + flush_index(RemoteEntitiesSearch.Meta.index) return pers diff --git a/tests/ui/remote_entities/test_entities_api.py b/tests/ui/remote_entities/test_entities_api.py index 451087a276..cfe23313b7 100644 --- a/tests/ui/remote_entities/test_entities_api.py +++ b/tests/ui/remote_entities/test_entities_api.py @@ -29,7 +29,8 @@ from rero_ils.modules.documents.api import Document, DocumentsSearch from rero_ils.modules.entities.remote_entities.api import \ RemoteEntitiesSearch, RemoteEntity, remote_entity_id_fetcher -from rero_ils.modules.entities.remote_entities.replace import ReplaceIdentifiedBy +from rero_ils.modules.entities.remote_entities.replace import \ + ReplaceIdentifiedBy from rero_ils.modules.entities.remote_entities.sync import SyncEntity diff --git a/tests/ui/test_invenio_celery_tasks_endpoints.py b/tests/ui/test_invenio_celery_tasks_endpoints.py index 17d438e787..d36e5c3162 100644 --- a/tests/ui/test_invenio_celery_tasks_endpoints.py +++ b/tests/ui/test_invenio_celery_tasks_endpoints.py @@ -26,9 +26,10 @@ def test_missing_invenio_celery_task_endpoints(app): """Test missing invenio_celery task endpoints.""" celery_extension = app.extensions['invenio-celery'] - celery_entpoints = [] - for e in entry_points(group=celery_extension.entry_point_group): - celery_entpoints.append(e.value) + celery_entpoints = [ + e.value + for e in entry_points(group=celery_extension.entry_point_group) + ] for task, data in app.config['CELERY_BEAT_SCHEDULE'].items(): task_function = data['task'] From 2fbff9e1dc43c1ef6293cc257f362e564f2a9aa9 Mon Sep 17 00:00:00 2001 From: Lauren-D Date: Mon, 26 Jun 2023 10:38:26 +0200 Subject: [PATCH 06/14] entities: configure operation logs for local entities * Uses default operation log extension. Co-Authored-by: Lauren-D --- rero_ils/config.py | 3 ++- rero_ils/modules/entities/local_entities/api.py | 6 ++++-- rero_ils/modules/entities/local_entities/jsonresolver.py | 2 +- .../jsonschemas/operation_logs/operation_log-v0.0.1.json | 3 ++- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/rero_ils/config.py b/rero_ils/config.py index 6551bd65db..0216e5b991 100644 --- a/rero_ils/config.py +++ b/rero_ils/config.py @@ -2889,7 +2889,8 @@ def _(x): 'documents': 'doc', 'holdings': 'hold', 'items': 'item', - 'ill_requests': 'illr' + 'ill_requests': 'illr', + 'local_entities': 'locent' } RERO_ILS_ENABLE_OPERATION_LOG_VALIDATION = False diff --git a/rero_ils/modules/entities/local_entities/api.py b/rero_ils/modules/entities/local_entities/api.py index 2c8c1567dd..ee6213cc74 100644 --- a/rero_ils/modules/entities/local_entities/api.py +++ b/rero_ils/modules/entities/local_entities/api.py @@ -25,13 +25,14 @@ from rero_ils.modules.utils import sorted_pids from rero_ils.modules.fetchers import id_fetcher from rero_ils.modules.minters import id_minter +from rero_ils.modules.operation_logs.extensions import \ + OperationLogObserverExtension from rero_ils.modules.providers import Provider from .dumpers import indexer_dumper, replace_refs_dumper from .extensions import AuthorizedAccessPointExtension, \ LocalEntityFactoryExtension from .models import LocalEntityIdentifier, LocalEntityMetadata - from ..api import Entity # provider @@ -72,7 +73,8 @@ class LocalEntity(IlsRecord, Entity): _extensions = [ LocalEntityFactoryExtension(), - AuthorizedAccessPointExtension() + AuthorizedAccessPointExtension(), + OperationLogObserverExtension() ] @property diff --git a/rero_ils/modules/entities/local_entities/jsonresolver.py b/rero_ils/modules/entities/local_entities/jsonresolver.py index 6a9397e10b..366cc95d20 100644 --- a/rero_ils/modules/entities/local_entities/jsonresolver.py +++ b/rero_ils/modules/entities/local_entities/jsonresolver.py @@ -25,5 +25,5 @@ @jsonresolver.route('/api/local_entities/', host='bib.rero.ch') def local_entities_resolver(pid): - """Resolver for acq_account record.""" + """Resolver for local entity record.""" return resolve_json_refs('locent', pid) diff --git a/rero_ils/modules/operation_logs/jsonschemas/operation_logs/operation_log-v0.0.1.json b/rero_ils/modules/operation_logs/jsonschemas/operation_logs/operation_log-v0.0.1.json index 578aaa94a4..80f9bb837d 100644 --- a/rero_ils/modules/operation_logs/jsonschemas/operation_logs/operation_log-v0.0.1.json +++ b/rero_ils/modules/operation_logs/jsonschemas/operation_logs/operation_log-v0.0.1.json @@ -41,7 +41,8 @@ "item", "loan", "illr", - "notif" + "notif", + "locent" ] }, "value": { From 688b6b7de3a208bb0b317687d1ad0f3db778b7db Mon Sep 17 00:00:00 2001 From: Renaud Michotte Date: Tue, 4 Jul 2023 16:19:18 +0200 Subject: [PATCH 07/14] entities: proxy to search local entities. Creates an API endpoint to search local entities by category (a category could represent multiple ES filters coupling AND/OR/... filters). Co-Authored-by: Renaud Michotte --- pyproject.toml | 1 + rero_ils/config.py | 2 +- .../modules/entities/local_entities/proxy.py | 69 ++++++++++++++++++ .../modules/entities/local_entities/views.py | 70 +++++++++++++++++++ .../modules/entities/remote_entities/proxy.py | 2 +- .../modules/entities/remote_entities/views.py | 5 +- .../test_local_entities_extensions.py | 0 .../test_local_entities_permissions.py | 0 .../test_local_entities_rest.py | 43 ++++++++++++ .../test_remote_entities_permissions.py | 0 .../test_remote_entities_rest.py | 0 tests/data/data.json | 8 +++ tests/fixtures/metadata.py | 18 +++++ .../test_local_entities_dumpers.py | 0 .../test_local_entities_jsonresolver.py | 0 .../test_local_entities_mapping.py | 0 .../remote_entities/test_entities_api.py | 0 .../remote_entities/test_entities_filter.py | 0 .../remote_entities/test_entities_mapping.py | 0 .../remote_entities/test_entities_ui.py | 0 .../remote_entities/test_entities_utils.py | 0 21 files changed, 214 insertions(+), 4 deletions(-) create mode 100644 rero_ils/modules/entities/local_entities/proxy.py create mode 100644 rero_ils/modules/entities/local_entities/views.py rename tests/api/{ => entities}/local_entities/test_local_entities_extensions.py (100%) rename tests/api/{ => entities}/local_entities/test_local_entities_permissions.py (100%) rename tests/api/{ => entities}/local_entities/test_local_entities_rest.py (79%) rename tests/api/{ => entities}/remote_entities/test_remote_entities_permissions.py (100%) rename tests/api/{ => entities}/remote_entities/test_remote_entities_rest.py (100%) rename tests/ui/{ => entities}/local_entities/test_local_entities_dumpers.py (100%) rename tests/ui/{ => entities}/local_entities/test_local_entities_jsonresolver.py (100%) rename tests/ui/{ => entities}/local_entities/test_local_entities_mapping.py (100%) rename tests/ui/{ => entities}/remote_entities/test_entities_api.py (100%) rename tests/ui/{ => entities}/remote_entities/test_entities_filter.py (100%) rename tests/ui/{ => entities}/remote_entities/test_entities_mapping.py (100%) rename tests/ui/{ => entities}/remote_entities/test_entities_ui.py (100%) rename tests/ui/{ => entities}/remote_entities/test_entities_utils.py (100%) diff --git a/pyproject.toml b/pyproject.toml index 34e40a1dc7..655ec836ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -163,6 +163,7 @@ acq_orders = "rero_ils.modules.acquisition.acq_orders.views:api_blueprint" acq_receipts = "rero_ils.modules.acquisition.acq_receipts.views:api_blueprint" api_documents = "rero_ils.modules.documents.views:api_blueprint" circ_policies = "rero_ils.modules.circ_policies.views:blueprint" +local_entities = "rero_ils.modules.entities.local_entities.views:api_blueprint" remote_entities = "rero_ils.modules.entities.remote_entities.views:api_blueprint" documents = "rero_ils.modules.documents.views:api_blueprint" holdings = "rero_ils.modules.holdings.api_views:api_blueprint" diff --git a/rero_ils/config.py b/rero_ils/config.py index 0216e5b991..0890db2b1b 100644 --- a/rero_ils/config.py +++ b/rero_ils/config.py @@ -3044,7 +3044,6 @@ def _(x): 'bf:Organisation': 'agents', 'bf:Topic': 'concepts' } -RERO_ILS_MEF_RESULT_SIZE = 100 # The absolute path to put the agent synchronization logs, default is the # instance path @@ -3064,6 +3063,7 @@ def _(x): 'fr': ['idref', 'rero', 'gnd'], 'de': ['gnd', 'idref', 'rero'], } +RERO_ILS_DEFAULT_SUGGESTION_LIMIT = 10 # ============================================================================= # RERO_ILS PATRON ROLES MANAGEMENT diff --git a/rero_ils/modules/entities/local_entities/proxy.py b/rero_ils/modules/entities/local_entities/proxy.py new file mode 100644 index 0000000000..75ea3b18b0 --- /dev/null +++ b/rero_ils/modules/entities/local_entities/proxy.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Local entity proxies.""" +from elasticsearch_dsl import Q + +from .api import LocalEntitiesSearch +from ..models import EntityType + + +CATEGORY_FILTERS = { + 'agents': Q('terms', type=[EntityType.PERSON, EntityType.ORGANISATION]), + 'person': Q('term', type=EntityType.PERSON), + 'organisation': Q('term', type=EntityType.ORGANISATION), + 'concepts': Q('term', type=EntityType.TOPIC), + 'concepts-genreForm': + Q('term', type=EntityType.TOPIC) & Q('term', genreForm=True) +} + + +class LocalEntityProxy: + """Local entity proxy.""" + + def __init__(self, category): + """Init magic method. + + :param category: the search category ('agents', 'organisation', ...). + """ + self.category = category + + def search(self, search_term, size=10): + """Search for local entities. + + :param search_term: the search term. + :param size: the number of hit to return. + :return: local entities matching the search term. + :rtype: generator. + """ + query = self._create_base_query()[:size]\ + .filter('query_string', query=search_term) + yield from query.execute() + + def _create_base_query(self): + """Build the base ES query object to search `LocalEntity`. + + Either the search_category is key for a predefined configuration, + either the search_category will be used as local entity type search + term. + """ + query = LocalEntitiesSearch() + if self.category in CATEGORY_FILTERS: + return query.filter(CATEGORY_FILTERS[self.category]) + else: + return query.filter('term', type=self.category) diff --git a/rero_ils/modules/entities/local_entities/views.py b/rero_ils/modules/entities/local_entities/views.py new file mode 100644 index 0000000000..9bca18935d --- /dev/null +++ b/rero_ils/modules/entities/local_entities/views.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Blueprint about local entities.""" + +from contextlib import suppress +from functools import wraps + +from flask import Blueprint, current_app, jsonify, request + +from rero_ils.modules.decorators import check_logged_as_librarian +from rero_ils.modules.entities.local_entities.proxy import LocalEntityProxy + +api_blueprint = Blueprint('api_local_entities', __name__) + + +def extract_size_parameter(func): + """Decorator to extract the size parameter from query string.""" + @wraps(func) + def wrapper(*args, **kwargs): + if 'size' not in kwargs: + kwargs['size'] = current_app.config.get( + 'RERO_ILS_DEFAULT_SUGGESTION_LIMIT') + with suppress(ValueError): + kwargs['size'] = int(request.args.get('size') or '') + return func(*args, **kwargs) + return wrapper + + +@api_blueprint.route('/local_entities/search/', + defaults={'entity_type': 'agents'}) +@api_blueprint.route('/local_entities/search//') +@api_blueprint.route('/local_entities/search///') +@check_logged_as_librarian +@extract_size_parameter +def local_search_proxy(entity_type, term, size): + """Proxy to search local entities by entity_type. + + :param entity_type: The type of entities to search. + :param term: the searched term. + :param size: the number of suggestion to retrieve + """ + # DEV NOTES :: Why not using invenio list API + # In some situation, we need to search on multiple key/filters at same + # time. For example, searching for concept[@type=genreForm] requires to + # filter query on @type='bf:Topic' and on @genreForm=true. + # To not introduce specific logic into external consumers, this endpoint + # can analyse the `entity_type` argument to determine the base query to + # apply. + # See same behavior for remote entities search proxy. + + return jsonify([ + hit.to_dict() + for hit in LocalEntityProxy(entity_type).search(term, size) + ]) diff --git a/rero_ils/modules/entities/remote_entities/proxy.py b/rero_ils/modules/entities/remote_entities/proxy.py index 1743cd472b..fe1327913c 100644 --- a/rero_ils/modules/entities/remote_entities/proxy.py +++ b/rero_ils/modules/entities/remote_entities/proxy.py @@ -16,7 +16,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -"""Entity proxies.""" +"""Remote entity proxies.""" import json from urllib.parse import quote_plus diff --git a/rero_ils/modules/entities/remote_entities/views.py b/rero_ils/modules/entities/remote_entities/views.py index 1f991c4604..ac34a6cc9a 100644 --- a/rero_ils/modules/entities/remote_entities/views.py +++ b/rero_ils/modules/entities/remote_entities/views.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- # # RERO ILS -# Copyright (C) 2019-2022 RERO +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -15,7 +16,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -"""Blueprint used for loading templates.""" +"""Blueprint about remote entities.""" from __future__ import absolute_import, print_function diff --git a/tests/api/local_entities/test_local_entities_extensions.py b/tests/api/entities/local_entities/test_local_entities_extensions.py similarity index 100% rename from tests/api/local_entities/test_local_entities_extensions.py rename to tests/api/entities/local_entities/test_local_entities_extensions.py diff --git a/tests/api/local_entities/test_local_entities_permissions.py b/tests/api/entities/local_entities/test_local_entities_permissions.py similarity index 100% rename from tests/api/local_entities/test_local_entities_permissions.py rename to tests/api/entities/local_entities/test_local_entities_permissions.py diff --git a/tests/api/local_entities/test_local_entities_rest.py b/tests/api/entities/local_entities/test_local_entities_rest.py similarity index 79% rename from tests/api/local_entities/test_local_entities_rest.py rename to tests/api/entities/local_entities/test_local_entities_rest.py index 3b0bbf49fa..661b0058dd 100644 --- a/tests/api/local_entities/test_local_entities_rest.py +++ b/tests/api/entities/local_entities/test_local_entities_rest.py @@ -144,3 +144,46 @@ def test_local_entities_post_put_delete(client, local_entity_person_data, res = client.get(item_url) assert res.status_code == 410 + + +@mock.patch('rero_ils.modules.decorators.login_and_librarian', + mock.MagicMock()) +def test_local_search_by_proxy( + client, local_entity_genre_form, local_entity_org +): + """Test local entity search proxy.""" + response = client.get(url_for( + 'api_local_entities.local_search_proxy', + entity_type='concepts-genreForm', + term='personal', + size='dummy_qs_arg' + )) + assert response.status_code == 200 + assert len(response.json) == 1 + assert response.json[0]['pid'] == local_entity_genre_form.pid + + response = client.get(url_for( + 'api_local_entities.local_search_proxy', + entity_type='concepts-genreForm', + term='personal', + size='0' + )) + assert response.status_code == 200 + assert len(response.json) == 0 + + response = client.get(url_for( + 'api_local_entities.local_search_proxy', + entity_type='concepts-genreForm', + term='dummy_key' + )) + assert response.status_code == 200 + assert len(response.json) == 0 + + response = client.get(url_for( + 'api_local_entities.local_search_proxy', + entity_type='bf:Organisation', + term='Convegno' + )) + assert response.status_code == 200 + assert len(response.json) == 1 + assert response.json[0]['pid'] == local_entity_org.pid diff --git a/tests/api/remote_entities/test_remote_entities_permissions.py b/tests/api/entities/remote_entities/test_remote_entities_permissions.py similarity index 100% rename from tests/api/remote_entities/test_remote_entities_permissions.py rename to tests/api/entities/remote_entities/test_remote_entities_permissions.py diff --git a/tests/api/remote_entities/test_remote_entities_rest.py b/tests/api/entities/remote_entities/test_remote_entities_rest.py similarity index 100% rename from tests/api/remote_entities/test_remote_entities_rest.py rename to tests/api/entities/remote_entities/test_remote_entities_rest.py diff --git a/tests/data/data.json b/tests/data/data.json index 76b075fd7b..86cb026a36 100644 --- a/tests/data/data.json +++ b/tests/data/data.json @@ -1888,6 +1888,14 @@ "creator": "Chopin, Fre\u0301de\u0301ric (1810-1849)", "type": "bf:Work" }, + "locent_genreForm": { + "$schema": "https://bib.rero.ch/schemas/local_entities/local_entity-v0.0.1.json", + "name": "Personal correspondence", + "source_catalog": "lcgft", + "genreForm": true, + "type": "bf:Topic", + "pid": "locent_genreForm" + }, "doc1": { "$schema": "https://bib.rero.ch/schemas/documents/document-v0.0.1.json", "pid": "doc1", diff --git a/tests/fixtures/metadata.py b/tests/fixtures/metadata.py index 0eae8b21a1..cc23e158fc 100644 --- a/tests/fixtures/metadata.py +++ b/tests/fixtures/metadata.py @@ -489,6 +489,12 @@ def local_entity_work_data(data): return deepcopy(data.get('locent_work')) +@pytest.fixture(scope="module") +def local_entity_genre_form_data(data): + """Load mef genreForm local entity data.""" + return deepcopy(data.get('locent_genreForm')) + + @pytest.fixture(scope="module") def local_entity_person(app, local_entity_person_data): """Create mef person record.""" @@ -537,6 +543,18 @@ def local_entity_org2(app, local_entity_org2_data): return org +@pytest.fixture(scope="module") +def local_entity_genre_form(app, local_entity_genre_form_data): + """Create mef person record.""" + entity = LocalEntity.create( + data=local_entity_genre_form_data, + delete_pid=False, + dbcommit=True, + reindex=True) + flush_index(LocalEntitiesSearch.Meta.index) + return entity + + @pytest.fixture(scope="module") @mock.patch('requests.Session.get') def document_ref(mock_contributions_mef_get, diff --git a/tests/ui/local_entities/test_local_entities_dumpers.py b/tests/ui/entities/local_entities/test_local_entities_dumpers.py similarity index 100% rename from tests/ui/local_entities/test_local_entities_dumpers.py rename to tests/ui/entities/local_entities/test_local_entities_dumpers.py diff --git a/tests/ui/local_entities/test_local_entities_jsonresolver.py b/tests/ui/entities/local_entities/test_local_entities_jsonresolver.py similarity index 100% rename from tests/ui/local_entities/test_local_entities_jsonresolver.py rename to tests/ui/entities/local_entities/test_local_entities_jsonresolver.py diff --git a/tests/ui/local_entities/test_local_entities_mapping.py b/tests/ui/entities/local_entities/test_local_entities_mapping.py similarity index 100% rename from tests/ui/local_entities/test_local_entities_mapping.py rename to tests/ui/entities/local_entities/test_local_entities_mapping.py diff --git a/tests/ui/remote_entities/test_entities_api.py b/tests/ui/entities/remote_entities/test_entities_api.py similarity index 100% rename from tests/ui/remote_entities/test_entities_api.py rename to tests/ui/entities/remote_entities/test_entities_api.py diff --git a/tests/ui/remote_entities/test_entities_filter.py b/tests/ui/entities/remote_entities/test_entities_filter.py similarity index 100% rename from tests/ui/remote_entities/test_entities_filter.py rename to tests/ui/entities/remote_entities/test_entities_filter.py diff --git a/tests/ui/remote_entities/test_entities_mapping.py b/tests/ui/entities/remote_entities/test_entities_mapping.py similarity index 100% rename from tests/ui/remote_entities/test_entities_mapping.py rename to tests/ui/entities/remote_entities/test_entities_mapping.py diff --git a/tests/ui/remote_entities/test_entities_ui.py b/tests/ui/entities/remote_entities/test_entities_ui.py similarity index 100% rename from tests/ui/remote_entities/test_entities_ui.py rename to tests/ui/entities/remote_entities/test_entities_ui.py diff --git a/tests/ui/remote_entities/test_entities_utils.py b/tests/ui/entities/remote_entities/test_entities_utils.py similarity index 100% rename from tests/ui/remote_entities/test_entities_utils.py rename to tests/ui/entities/remote_entities/test_entities_utils.py From 11e19a760eb34d05f744b30a8ab3d38794c321ab Mon Sep 17 00:00:00 2001 From: Lauren-D Date: Mon, 3 Jul 2023 15:21:35 +0200 Subject: [PATCH 08/14] entities: link to document * Create generic method to load entity. * Adapts jinja filter for document detail view. * Reorganizes dumpers for entities. * Adapts tests. Co-Authored-by: Lauren-D --- pyproject.toml | 1 + rero_ils/config.py | 4 +- rero_ils/modules/api.py | 24 +++++++ .../modules/documents/dumpers/replace_refs.py | 16 ++++- .../v7/documents/document-v0.0.1.json | 9 +++ rero_ils/modules/documents/views.py | 36 +++++----- rero_ils/modules/entities/api.py | 71 +++++++++++++++++++ rero_ils/modules/entities/dumpers/__init__.py | 32 +++++++-- rero_ils/modules/entities/dumpers/document.py | 62 ++++++++++++++++ .../{local_entities => }/dumpers/indexer.py | 11 ++- rero_ils/modules/entities/helpers.py | 22 ++++++ .../modules/entities/local_entities/api.py | 25 ++----- .../local_entities/dumpers/__init__.py | 48 ------------- .../entities/local_entities/indexer.py | 71 +++++++++++++++++++ .../modules/entities/remote_entities/api.py | 64 ++--------------- .../remote_entities/dumpers/__init__.py | 41 ----------- .../remote_entities/dumpers/document.py | 51 ------------- .../remote_entities/dumpers/indexer.py | 37 ---------- .../entities/remote_entities/listener.py | 36 ---------- .../modules/entities/remote_entities/sync.py | 5 +- rero_ils/modules/utils.py | 21 ++++++ tests/api/documents/test_documents_dumpers.py | 18 ++--- .../test_local_entities_rest.py | 2 +- .../test_remote_entities_rest.py | 2 +- tests/ui/documents/test_documents_filter.py | 4 +- .../test_local_entities_dumpers.py | 3 +- 26 files changed, 379 insertions(+), 337 deletions(-) create mode 100644 rero_ils/modules/entities/dumpers/document.py rename rero_ils/modules/entities/{local_entities => }/dumpers/indexer.py (80%) delete mode 100644 rero_ils/modules/entities/local_entities/dumpers/__init__.py create mode 100644 rero_ils/modules/entities/local_entities/indexer.py delete mode 100644 rero_ils/modules/entities/remote_entities/dumpers/__init__.py delete mode 100644 rero_ils/modules/entities/remote_entities/dumpers/document.py delete mode 100644 rero_ils/modules/entities/remote_entities/dumpers/indexer.py delete mode 100644 rero_ils/modules/entities/remote_entities/listener.py diff --git a/pyproject.toml b/pyproject.toml index 655ec836ba..d27487cf6d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -212,6 +212,7 @@ ebooks = "rero_ils.modules.ebooks.tasks" holdings = "rero_ils.modules.holdings.tasks" items = "rero_ils.modules.items.tasks" loans = "rero_ils.modules.loans.tasks" +indexer_locent = "rero_ils.modules.entities.local_entities.indexer" modules = "rero_ils.modules.tasks" notifications = "rero_ils.modules.notifications.tasks" patrons = "rero_ils.modules.patrons.tasks" diff --git a/rero_ils/config.py b/rero_ils/config.py index 0890db2b1b..f8dc528b39 100644 --- a/rero_ils/config.py +++ b/rero_ils/config.py @@ -1155,7 +1155,7 @@ def _(x): pid_fetcher='local_entity_id', search_class='rero_ils.modules.entities.local_entities.api:LocalEntitiesSearch', search_index='local_entities', - indexer_class='rero_ils.modules.entities.local_entities.api:LocalEntitiesIndexer', + indexer_class='rero_ils.modules.entities.local_entities.indexer:LocalEntitiesIndexer', search_type=None, record_serializers={ 'application/json': 'rero_ils.modules.serializers:json_v1_response' @@ -2997,6 +2997,8 @@ def _(x): # Misc INDEXER_REPLACE_REFS = True INDEXER_RECORD_TO_INDEX = 'rero_ils.modules.indexer_utils.record_to_index' +#: Trigger delay for celery tasks to index referenced records. +RERO_ILS_INDEXER_TASK_DELAY = timedelta(seconds=2) RERO_ILS_APP_URL_SCHEME = 'https' RERO_ILS_APP_HOST = 'bib.rero.ch' diff --git a/rero_ils/modules/api.py b/rero_ils/modules/api.py index 27f2852505..0694892b17 100644 --- a/rero_ils/modules/api.py +++ b/rero_ils/modules/api.py @@ -724,3 +724,27 @@ def _prepare_record( **kwargs ) return data + + +class ReferencedRecordsIndexer: + """Referenced records Indexer class.""" + + def index(self, indexer_class, referenced): + """Index record. + + :param indexer_class: record indexer class. + :param referenced: referenced records to index. A list of dicts + containing `pid_type` and `record` keys of the records that will + be indexed. + """ + indexer = indexer_class() + for r in referenced: + try: + record_to_index = r['record'] + indexer.index(record_to_index) + except Exception as err: + pid_type = r['pid_type'], + pid_value = r['record']['pid'] + current_app.logger.error( + f'Record indexing error {pid_type} {pid_value}: {err}' + ) diff --git a/rero_ils/modules/documents/dumpers/replace_refs.py b/rero_ils/modules/documents/dumpers/replace_refs.py index 26f05a660f..609192974d 100644 --- a/rero_ils/modules/documents/dumpers/replace_refs.py +++ b/rero_ils/modules/documents/dumpers/replace_refs.py @@ -20,9 +20,11 @@ from invenio_records.dumpers import Dumper from rero_ils.modules.commons.exceptions import RecordNotFound -from rero_ils.modules.entities.remote_entities.dumpers import document_dumper +from rero_ils.modules.entities.dumpers import \ + document_dumper from rero_ils.modules.entities.remote_entities.utils import \ extract_data_from_mef_uri +from rero_ils.modules.utils import extracted_data_from_ref class ReplaceRefsEntitiesDumperMixin(Dumper): @@ -31,10 +33,20 @@ class ReplaceRefsEntitiesDumperMixin(Dumper): @staticmethod def _replace_entity(data): """Replace the `$ref` linked contributions.""" + from rero_ils.modules.entities.local_entities.api import LocalEntity from rero_ils.modules.entities.remote_entities.api import RemoteEntity + + # try to get entity record + entity = extracted_data_from_ref(data['$ref'], 'record') + # check if local entity + if entity and isinstance(entity, LocalEntity): + # internal resources will be resolved later (see ReplaceRefsDumper) + return entity.dumps(document_dumper) + + _, _type, _ = extract_data_from_mef_uri(data['$ref']) if not (entity := RemoteEntity.get_record_by_pid(data['pid'])): raise RecordNotFound(RemoteEntity, data['pid']) - _, _type, _ = extract_data_from_mef_uri(data['$ref']) + entity = entity.dumps(document_dumper) entity.update({ 'primary_source': _type, diff --git a/rero_ils/modules/documents/mappings/v7/documents/document-v0.0.1.json b/rero_ils/modules/documents/mappings/v7/documents/document-v0.0.1.json index 9ffed8c876..ee2295c41c 100644 --- a/rero_ils/modules/documents/mappings/v7/documents/document-v0.0.1.json +++ b/rero_ils/modules/documents/mappings/v7/documents/document-v0.0.1.json @@ -327,6 +327,9 @@ "primary_source": { "type": "keyword" }, + "unique_key": { + "type": "keyword" + }, "pid": { "type": "keyword" }, @@ -850,6 +853,9 @@ "pid": { "type": "keyword" }, + "unique_key": { + "type": "keyword" + }, "id_gnd": { "type": "keyword" }, @@ -974,6 +980,9 @@ "pid": { "type": "keyword" }, + "unique_key": { + "type": "keyword" + }, "sources": { "type": "keyword" }, diff --git a/rero_ils/modules/documents/views.py b/rero_ils/modules/documents/views.py index fd647f046a..28a3e7872e 100644 --- a/rero_ils/modules/documents/views.py +++ b/rero_ils/modules/documents/views.py @@ -24,7 +24,8 @@ import click from elasticsearch_dsl.query import Q -from flask import Blueprint, abort, current_app, jsonify, render_template +from flask import Blueprint, abort, current_app, jsonify, render_template, \ + url_for from flask import request as flask_request from flask_babelex import gettext as _ from flask_login import current_user @@ -36,16 +37,17 @@ from .utils import display_alternate_graphic_first, get_remote_cover, \ title_format_text, title_format_text_alternate_graphic, \ title_variant_format_text -from ..collections.api import CollectionsSearch +from rero_ils.modules.collections.api import CollectionsSearch from rero_ils.modules.entities.remote_entities.api import RemoteEntity from rero_ils.modules.entities.models import EntityType -from ..holdings.models import HoldingNoteTypes -from ..items.models import ItemCirculationAction -from ..libraries.api import Library -from ..locations.api import Location -from ..organisations.api import Organisation -from ..patrons.api import current_patrons -from ..utils import cached, extracted_data_from_ref +from rero_ils.modules.entities.helpers import get_entity_record_from_data +from rero_ils.modules.holdings.models import HoldingNoteTypes +from rero_ils.modules.items.models import ItemCirculationAction +from rero_ils.modules.libraries.api import Library +from rero_ils.modules.locations.api import Location +from rero_ils.modules.organisations.api import Organisation +from rero_ils.modules.patrons.api import current_patrons +from rero_ils.modules.utils import cached, extracted_data_from_ref def doc_item_view_method(pid, record, template=None, **kwargs): @@ -263,14 +265,16 @@ def contribution_format(contributions, language, viewcode, with_roles=False): """ output = [] for contrib in filter(lambda c: c.get('entity'), contributions): - if entity := RemoteEntity \ - .get_record_by_pid(contrib['entity'].get('pid')): + if entity := get_entity_record_from_data(contrib['entity']): text = entity.get_authorized_access_point(language=language) - entity_type = 'persons' - if entity.get('type') == EntityType.ORGANISATION: - entity_type = 'corporate-bodies' - label = \ - f'{text}' + args = { + 'viewcode': viewcode, + 'recordType': 'documents', + 'q': f'contribution.entity.unique_key:{entity.unique_key}', + 'simple': 0 + } + url = url_for('rero_ils.search', **args) + label = f'{text}' else: default_key = 'authorized_access_point' localized_key = f'{default_key}_{language}' diff --git a/rero_ils/modules/entities/api.py b/rero_ils/modules/entities/api.py index fd6074cf38..661db5b7bf 100644 --- a/rero_ils/modules/entities/api.py +++ b/rero_ils/modules/entities/api.py @@ -20,6 +20,11 @@ from abc import ABC, abstractmethod +from elasticsearch_dsl import Q, A +from flask import current_app + +from rero_ils.modules.documents.api import DocumentsSearch + class Entity(ABC): """Entity class.""" @@ -32,3 +37,69 @@ def get_authorized_access_point(self, language): :returns: authorized access point in given language. """ raise NotImplementedError + + @property + def unique_key(self): + """Get the unique key. + + As entity subclasses doesn't share pid generation sequence, pid + overlapping can occur. To prevent this, use `unique_key` to point + specific entity. + + Example: + => `remote_1` + => `local_1` + """ + return f'{self.resource_type}_{self.pid}' + + def _search_documents(self, with_subjects=True, + with_subjects_imported=True): + """Get search documents.""" + filters = Q('term', contribution__entity__unique_key=self.unique_key) + + if with_subjects: + filters |= \ + Q('term', subjects__entity__unique_key=self.unique_key) + if with_subjects_imported: + filters |= \ + Q('term', + subjects_imported__entity__unique_key=self.unique_key) + return DocumentsSearch().filter(filters) + + def documents_pids(self, with_subjects=True, with_subjects_imported=True): + """Get documents pids.""" + search = self._search_documents( + with_subjects=with_subjects, + with_subjects_imported=with_subjects_imported + ).source('pid') + return [hit.pid for hit in search.scan()] + + def documents_ids(self, with_subjects=True, with_subjects_imported=True): + """Get documents ids.""" + search = self._search_documents( + with_subjects=with_subjects, + with_subjects_imported=with_subjects_imported + ).source() + return [hit.meta.id for hit in search.scan()] + + @property + def organisation_pids(self): + """Get organisations pids.""" + # TODO :: Should be linked also on other fields ? + # ex: subjects, genre_form, ... + # Seems only use to filer entities by viewcode. + search = self._search_documents() + agg = A( + 'terms', + field='holdings.organisation.organisation_pid', + min_doc_count=1, + size=current_app.config + .get('RERO_ILS_AGGREGATION_SIZE') + .get('organisations') + ) + search.aggs.bucket('organisation', agg) + results = search.execute() + return list({ + result.key + for result in results.aggregations.organisation.buckets + }) diff --git a/rero_ils/modules/entities/dumpers/__init__.py b/rero_ils/modules/entities/dumpers/__init__.py index d26a089995..ec9ea48ac6 100644 --- a/rero_ils/modules/entities/dumpers/__init__.py +++ b/rero_ils/modules/entities/dumpers/__init__.py @@ -17,10 +17,32 @@ # along with this program. If not, see . """Common entity dumpers.""" +from invenio_records.dumpers import Dumper -from rero_ils.modules.entities.dumpers.authorized_acces_point import \ - LocalizedAuthorizedAccessPointDumper +from rero_ils.modules.commons.dumpers import MultiDumper, ReplaceRefsDumper -__all__ = [ - 'LocalizedAuthorizedAccessPointDumper' -] +from .authorized_acces_point import LocalizedAuthorizedAccessPointDumper +from .document import BaseDocumentEntityDumper +from .indexer import EntityIndexerDumper + +# replace linked data (seems not necessary at this time) +replace_refs_dumper = MultiDumper(dumpers=[ + # make a fresh copy + Dumper(), + ReplaceRefsDumper() +]) + +# dumper used for indexing +indexer_dumper = MultiDumper(dumpers=[ + # make a fresh copy + Dumper(), + ReplaceRefsDumper(), + EntityIndexerDumper(), + LocalizedAuthorizedAccessPointDumper() +]) + +document_dumper = MultiDumper(dumpers=[ + BaseDocumentEntityDumper(), + EntityIndexerDumper(), + LocalizedAuthorizedAccessPointDumper(), +]) diff --git a/rero_ils/modules/entities/dumpers/document.py b/rero_ils/modules/entities/dumpers/document.py new file mode 100644 index 0000000000..5fd413c468 --- /dev/null +++ b/rero_ils/modules/entities/dumpers/document.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Indexing dumper.""" +from flask import current_app +from invenio_records.dumpers import Dumper + +from rero_ils.modules.entities.models import EntityResourceType + + +class BaseDocumentEntityDumper(Dumper): + """Base document Entity dumper class.""" + + def dump(self, record, data): + """Dump an entity instance. + + This dumper this + + :param record: The record to dump. + :param data: The initial dump data passed in by ``record.dumps()``. + """ + # DEV NOTES: Why using `unique_key` + # Unique key is used to avoid nested implementation in Elasticsearch + data = { + 'pid': record.pid, + 'type': record['type'], + 'unique_key': record.unique_key + } + if record.resource_type == EntityResourceType.REMOTE: + for agency in current_app.config['RERO_ILS_AGENTS_SOURCES']: + if field := record.get(agency): + data['type'] = field.get('bf:Agent', record['type']) + data[f'id_{agency}'] = record[agency]['pid'] + + variant_access_points = [] + parallel_access_points = [] + for source in record.get('sources'): + variant_access_points += record[source].get( + 'variant_access_point', []) + parallel_access_points += record[source].get( + 'parallel_access_point', []) + if variant_access_points: + data['variant_access_point'] = variant_access_points + if parallel_access_points: + data['parallel_access_point'] = parallel_access_points + + return data diff --git a/rero_ils/modules/entities/local_entities/dumpers/indexer.py b/rero_ils/modules/entities/dumpers/indexer.py similarity index 80% rename from rero_ils/modules/entities/local_entities/dumpers/indexer.py rename to rero_ils/modules/entities/dumpers/indexer.py index f5514b7970..43a3ca7706 100644 --- a/rero_ils/modules/entities/local_entities/dumpers/indexer.py +++ b/rero_ils/modules/entities/dumpers/indexer.py @@ -20,17 +20,16 @@ from invenio_records.dumpers import Dumper -from rero_ils.modules.entities.models import EntityResourceType - -class LocalEntityIndexerDumper(Dumper): - """Local entity indexer.""" +class EntityIndexerDumper(Dumper): + """Entity indexer.""" def dump(self, record, data): - """Dump a local entity instance. + """Dump an entity instance. :param record: The record to dump. :param data: The initial dump data passed in by ``record.dumps()``. """ - data['resource_type'] = EntityResourceType.LOCAL + data['resource_type'] = record.resource_type + data['organisations'] = record.organisation_pids return data diff --git a/rero_ils/modules/entities/helpers.py b/rero_ils/modules/entities/helpers.py index b4514970b6..e375f4b274 100644 --- a/rero_ils/modules/entities/helpers.py +++ b/rero_ils/modules/entities/helpers.py @@ -17,6 +17,10 @@ # along with this program. If not, see . """Helpers for entities.""" +from rero_ils.modules.commons.exceptions import RecordNotFound +from rero_ils.modules.entities.local_entities.api import LocalEntity +from rero_ils.modules.entities.remote_entities.api import RemoteEntity +from rero_ils.modules.utils import extracted_data_from_ref def str_builder(field_values, prefix='', suffix='', delimiter=''): @@ -49,3 +53,21 @@ def str_builder(field_values, prefix='', suffix='', delimiter=''): if any(field_values): return f'{prefix}{delimiter.join(field_values)}{suffix}' return '' + + +def get_entity_record_from_data(data): + """Retrieve entity record from data. + + # todo: Add comments + """ + # try to get entity record + if pid := data.get('pid'): + # remote entities have a pid in data + if entity := RemoteEntity.get_record_by_pid(pid): + return entity + raise RecordNotFound(RemoteEntity, data.get('pid')) + if ref := data.get('$ref'): + entity = extracted_data_from_ref(ref, 'record') + # check if local entity + if entity and isinstance(entity, LocalEntity): + return entity diff --git a/rero_ils/modules/entities/local_entities/api.py b/rero_ils/modules/entities/local_entities/api.py index ee6213cc74..28437ad353 100644 --- a/rero_ils/modules/entities/local_entities/api.py +++ b/rero_ils/modules/entities/local_entities/api.py @@ -20,8 +20,7 @@ from functools import partial -from rero_ils.modules.api import IlsRecord, IlsRecordsIndexer, IlsRecordsSearch -from rero_ils.modules.documents.api import DocumentsSearch +from rero_ils.modules.api import IlsRecord, IlsRecordsSearch from rero_ils.modules.utils import sorted_pids from rero_ils.modules.fetchers import id_fetcher from rero_ils.modules.minters import id_minter @@ -29,11 +28,12 @@ OperationLogObserverExtension from rero_ils.modules.providers import Provider -from .dumpers import indexer_dumper, replace_refs_dumper from .extensions import AuthorizedAccessPointExtension, \ LocalEntityFactoryExtension from .models import LocalEntityIdentifier, LocalEntityMetadata from ..api import Entity +from ..dumpers import replace_refs_dumper +from ..models import EntityResourceType # provider LocalEntityProvider = type( @@ -76,6 +76,7 @@ class LocalEntity(IlsRecord, Entity): AuthorizedAccessPointExtension(), OperationLogObserverExtension() ] + resource_type = EntityResourceType.LOCAL @property def type(self): @@ -98,8 +99,7 @@ def get_links_to_me(self, get_pids=False): :param get_pids: if True list of linked pids if False count of linked records """ - document_query = DocumentsSearch() \ - .filter('term', local_entity__pid=self.pid) + document_query = self._search_documents() documents = sorted_pids(document_query) if get_pids \ else document_query.count() links = { @@ -113,18 +113,3 @@ def reasons_not_to_delete(self): if links := self.get_links_to_me(): cannot_delete['links'] = links return cannot_delete - - -class LocalEntitiesIndexer(IlsRecordsIndexer): - """Local entity indexing class.""" - - record_cls = LocalEntity - # data dumper for indexing - record_dumper = indexer_dumper - - def bulk_index(self, record_id_iterator): - """Bulk index records. - - :param record_id_iterator: Iterator yielding record UUIDs. - """ - super().bulk_index(record_id_iterator, doc_type='locent') diff --git a/rero_ils/modules/entities/local_entities/dumpers/__init__.py b/rero_ils/modules/entities/local_entities/dumpers/__init__.py deleted file mode 100644 index 62e95c09f5..0000000000 --- a/rero_ils/modules/entities/local_entities/dumpers/__init__.py +++ /dev/null @@ -1,48 +0,0 @@ -# -*- coding: utf-8 -*- -# -# RERO ILS -# Copyright (C) 2019-2023 RERO -# Copyright (C) 2019-2023 UCLouvain -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, version 3 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -"""Local entity dumpers.""" - -from invenio_records.dumpers import Dumper - -from rero_ils.modules.commons.dumpers import MultiDumper, ReplaceRefsDumper - -from .indexer import LocalEntityIndexerDumper -from ...dumpers import LocalizedAuthorizedAccessPointDumper - -# replace linked data (seems not necessary at this time) -replace_refs_dumper = MultiDumper(dumpers=[ - # make a fresh copy - Dumper(), - ReplaceRefsDumper() -]) - -# dumper used for indexing -indexer_dumper = MultiDumper(dumpers=[ - # make a fresh copy - Dumper(), - ReplaceRefsDumper(), - LocalEntityIndexerDumper(), - LocalizedAuthorizedAccessPointDumper() -]) - -document_dumper = MultiDumper(dumpers=[ - # make a fresh copy - Dumper(), - LocalizedAuthorizedAccessPointDumper(), -]) diff --git a/rero_ils/modules/entities/local_entities/indexer.py b/rero_ils/modules/entities/local_entities/indexer.py new file mode 100644 index 0000000000..fe0cc33b6b --- /dev/null +++ b/rero_ils/modules/entities/local_entities/indexer.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Local entity indexer APIs.""" +from celery import shared_task +from datetime import datetime + +from flask import current_app + +from rero_ils.modules.utils import get_record_class_by_resource, \ + get_indexer_class_by_resource +from rero_ils.modules.api import IlsRecordsIndexer, ReferencedRecordsIndexer +from .api import LocalEntity +from ..dumpers import indexer_dumper + + +@shared_task(ignore_result=True) +def index_referenced_records(entity): + """Index referenced records.""" + indexer = ReferencedRecordsIndexer() + entity = LocalEntity.get_record_by_pid(entity.get('pid')) + if referenced_resources := entity.get_links_to_me(get_pids=True): + for resource, pids in referenced_resources.items(): + record_cls = get_record_class_by_resource(resource) + indexer_cls = get_indexer_class_by_resource(resource) + pid_type = record_cls.provider.pid_type + referenced = [] + for pid in pids: + record = record_cls.get_record_by_pid(pid) + referenced.append(dict( + pid_type=pid_type, + record=record + )) + indexer.index(indexer_cls, referenced) + + +class LocalEntitiesIndexer(IlsRecordsIndexer): + """Local entity indexing class.""" + + record_cls = LocalEntity + # data dumper for indexing + record_dumper = indexer_dumper + + def index(self, entity, arguments=None, **kwargs): + """Index a Local entity record.""" + super().index(entity) + eta = datetime.utcnow() + current_app.config.get( + "RERO_ILS_INDEXER_TASK_DELAY", 0) + index_referenced_records.apply_async((entity,), eta=eta) + + def bulk_index(self, record_id_iterator): + """Bulk index records. + + :param record_id_iterator: Iterator yielding record UUIDs. + """ + super().bulk_index(record_id_iterator, doc_type='locent') diff --git a/rero_ils/modules/entities/remote_entities/api.py b/rero_ils/modules/entities/remote_entities/api.py index 5fc81418f6..57a38d7782 100644 --- a/rero_ils/modules/entities/remote_entities/api.py +++ b/rero_ils/modules/entities/remote_entities/api.py @@ -21,26 +21,25 @@ import contextlib from functools import partial -from elasticsearch_dsl import A from elasticsearch_dsl.query import Q from flask import current_app from invenio_db import db from rero_ils.modules.api import IlsRecord, IlsRecordsIndexer, IlsRecordsSearch -from rero_ils.modules.documents.api import DocumentsIndexer, DocumentsSearch +from rero_ils.modules.documents.api import DocumentsIndexer from rero_ils.modules.fetchers import id_fetcher from rero_ils.modules.minters import id_minter from rero_ils.modules.providers import Provider -from .dumpers import indexer_dumper from .models import RemoteEntityIdentifier, RemoteEntityMetadata, \ EntityUpdateAction from .utils import extract_data_from_mef_uri, get_mef_data_by_type -# provider from ..api import Entity -from ..local_entities.dumpers import replace_refs_dumper +from ..dumpers import indexer_dumper, replace_refs_dumper +from ..models import EntityResourceType +# provider RemoteEntityProvider = type( 'EntityProvider', (Provider,), @@ -76,6 +75,8 @@ class RemoteEntity(IlsRecord, Entity): # disable legacy replace refs enable_jsonref = False + resource_type = EntityResourceType.REMOTE + def resolve(self): """Resolve references data. @@ -154,29 +155,6 @@ def _get_mef_localized_value(self, key, language): return value return self.get(key, None) - @property - def organisation_pids(self): - """Get organisations pids.""" - # TODO :: Should be linked also on other fields ? - # ex: subjects, genre_form, ... - # Seems only use to filer entities by viewcode. - search = DocumentsSearch()\ - .filter('term', contribution__entity__pid=self.pid) - agg = A( - 'terms', - field='holdings.organisation.organisation_pid', - min_doc_count=1, - size=current_app.config - .get('RERO_ILS_AGGREGATION_SIZE') - .get('organisations') - ) - search.aggs.bucket('organisation', agg) - results = search.execute() - return list({ - result.key - for result in results.aggregations.organisation.buckets - }) - def get_authorized_access_point(self, language): """Get localized authorized_access_point. @@ -188,36 +166,6 @@ def get_authorized_access_point(self, language): language=language ) - def _search_documents(self, with_subjects=True, - with_subjects_imported=True): - """Get documents pids.""" - filters = Q('term', contribution__entity__pid=self.pid) - if with_subjects: - filters |= \ - Q('term', subjects__pid=self.pid) & \ - Q('terms', subjects__type=['bf:Person', 'bf:Organisation']) - if with_subjects_imported: - filters |= \ - Q('term', subjects_imported__pid=self.pid) & \ - Q('terms', subjects__type=['bf:Person', 'bf:Organisation']) - return DocumentsSearch().filter(filters) - - def documents_pids(self, with_subjects=True, with_subjects_imported=True): - """Get documents pids.""" - search = self._search_documents( - with_subjects=with_subjects, - with_subjects_imported=with_subjects_imported - ).source('pid') - return [hit.pid for hit in search.scan()] - - def documents_ids(self, with_subjects=True, with_subjects_imported=True): - """Get documents ids.""" - search = self._search_documents( - with_subjects=with_subjects, - with_subjects_imported=with_subjects_imported - ).source('pid') - return [hit.meta.id for hit in search.scan()] - def update_online( self, dbcommit=False, reindex=False, verbose=False, reindex_doc=True diff --git a/rero_ils/modules/entities/remote_entities/dumpers/__init__.py b/rero_ils/modules/entities/remote_entities/dumpers/__init__.py deleted file mode 100644 index 0d21d759c3..0000000000 --- a/rero_ils/modules/entities/remote_entities/dumpers/__init__.py +++ /dev/null @@ -1,41 +0,0 @@ -# -*- coding: utf-8 -*- -# -# RERO ILS -# Copyright (C) 2019-2023 RERO -# Copyright (C) 2019-2023 UCLouvain -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, version 3 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -"""Remote entity dumpers.""" - -from invenio_records.dumpers import Dumper - -# dumper used for indexing -from rero_ils.modules.commons.dumpers import MultiDumper, ReplaceRefsDumper -from ...dumpers import LocalizedAuthorizedAccessPointDumper -from .document import DocumentEntityDumper -from .indexer import RemoteEntityIndexerDumper - - -indexer_dumper = MultiDumper(dumpers=[ - # make a fresh copy - Dumper(), - ReplaceRefsDumper(), - RemoteEntityIndexerDumper(), - LocalizedAuthorizedAccessPointDumper() -]) - -document_dumper = MultiDumper(dumpers=[ - DocumentEntityDumper(), - LocalizedAuthorizedAccessPointDumper() -]) diff --git a/rero_ils/modules/entities/remote_entities/dumpers/document.py b/rero_ils/modules/entities/remote_entities/dumpers/document.py deleted file mode 100644 index ae9707e954..0000000000 --- a/rero_ils/modules/entities/remote_entities/dumpers/document.py +++ /dev/null @@ -1,51 +0,0 @@ -# -*- coding: utf-8 -*- -# -# RERO ILS -# Copyright (C) 2019-2023 RERO -# Copyright (C) 2019-2023 UCLouvain -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, version 3 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -"""Indexing dumper.""" -from flask import current_app -from invenio_records.dumpers import Dumper - - -class DocumentEntityDumper(Dumper): - """Remote entity indexer.""" - - def dump(self, record, data): - """Dump a remote entity instance. - - :param record: The record to dump. - :param data: The initial dump data passed in by ``record.dumps()``. - """ - data = {'pid': record.pid} - for agency in current_app.config['RERO_ILS_AGENTS_SOURCES']: - if field := record.get(agency): - data['type'] = field.get('bf:Agent', record['type']) - data[f'id_{agency}'] = record[agency]['pid'] - - variant_access_points = [] - parallel_access_points = [] - for source in record.get('sources'): - variant_access_points += record[source].get( - 'variant_access_point', []) - parallel_access_points += record[source].get( - 'parallel_access_point', []) - if variant_access_points: - data['variant_access_point'] = variant_access_points - if parallel_access_points: - data['parallel_access_point'] = parallel_access_points - - return data diff --git a/rero_ils/modules/entities/remote_entities/dumpers/indexer.py b/rero_ils/modules/entities/remote_entities/dumpers/indexer.py deleted file mode 100644 index 9e037d6bea..0000000000 --- a/rero_ils/modules/entities/remote_entities/dumpers/indexer.py +++ /dev/null @@ -1,37 +0,0 @@ -# -*- coding: utf-8 -*- -# -# RERO ILS -# Copyright (C) 2019-2023 RERO -# Copyright (C) 2019-2023 UCLouvain -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, version 3 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -"""Indexing dumper.""" - -from invenio_records.dumpers import Dumper - -from rero_ils.modules.entities.models import EntityResourceType - - -class RemoteEntityIndexerDumper(Dumper): - """Remote entity indexer.""" - - def dump(self, record, data): - """Dump a remote entity instance. - - :param record: The record to dump. - :param data: The initial dump data passed in by ``record.dumps()``. - """ - data['organisations'] = record.organisation_pids - data['resource_type'] = EntityResourceType.REMOTE - return data diff --git a/rero_ils/modules/entities/remote_entities/listener.py b/rero_ils/modules/entities/remote_entities/listener.py deleted file mode 100644 index daea79f0b7..0000000000 --- a/rero_ils/modules/entities/remote_entities/listener.py +++ /dev/null @@ -1,36 +0,0 @@ -# -*- coding: utf-8 -*- -# -# RERO ILS -# Copyright (C) 2019-2023 RERO -# Copyright (C) 2019-2023 UCLouvain -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, version 3 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -"""Signals connector for `Entity` records.""" - -from .api import RemoteEntitiesSearch, RemoteEntity - - -def enrich_remote_entities_data(sender, json=None, record=None, index=None, - doc_type=None, arguments=None, **dummy_kwargs): - """Signal sent before a record is indexed. - - :param json: The dumped record dictionary which can be modified. - :param record: The record being indexed. - :param index: The index in which the record will be indexed. - :param doc_type: The doc_type for the record. - """ - if index.split('-')[0] == RemoteEntitiesSearch.Meta.index: - if not isinstance(record, RemoteEntity): - record = RemoteEntity.get_record_by_pid(record.get('pid')) - json['organisations'] = record.organisation_pids diff --git a/rero_ils/modules/entities/remote_entities/sync.py b/rero_ils/modules/entities/remote_entities/sync.py index 93fbc8b9f1..668ee2883a 100644 --- a/rero_ils/modules/entities/remote_entities/sync.py +++ b/rero_ils/modules/entities/remote_entities/sync.py @@ -304,7 +304,6 @@ def sync_record(self, pid): raise Exception(f'ERROR MEF {pid} does not exists in db.') self.logger.debug(f'Processing {entity["type"]} MEF(pid: {pid})') # iterate over all entity sources: rero, gnd, idref - doc_pids = self._get_documents_pids_from_mef(entity.pid) pids_to_replace = {} for source in entity['sources']: mef = self._get_latest( @@ -382,10 +381,12 @@ def sync_record(self, pid): updated = True if updated: - # for each documents + # need to update each documents + doc_pids = entity.documents_pids() self.logger.info( f'MEF {entity["type"]} record(pid: {entity.pid}) ' f' try to update documents: {doc_pids}') + for doc_pid in doc_pids: self._update_entities_in_document( doc_pid=doc_pid, diff --git a/rero_ils/modules/utils.py b/rero_ils/modules/utils.py index a39e4a1bb7..33f61ca259 100644 --- a/rero_ils/modules/utils.py +++ b/rero_ils/modules/utils.py @@ -549,6 +549,27 @@ def get_record_class_from_schema_or_pid_type(schema=None, pid_type=None): .get(pid_type, {}).get('record_class')) +def get_indexer_class_by_resource(resource): + """Get indexer class by resource name. + + :param: resource name + :return: indexer class for resource name + """ + endpoint = get_endpoint_configuration(resource) + if indexer_cls := endpoint.get('indexer_class'): + return obj_or_import_string(indexer_cls) + + +def get_record_class_by_resource(resource): + """Get record class by resource name. + + :param: resource name + :return: record class for resource name + """ + if endpoint := get_endpoint_configuration(resource): + return obj_or_import_string(endpoint["record_class"]) + + def get_pid_type_from_schema(schema): """Get the pid_type from a given schema or a pid type. diff --git a/tests/api/documents/test_documents_dumpers.py b/tests/api/documents/test_documents_dumpers.py index 0499b81bec..7a3d6b6a39 100644 --- a/tests/api/documents/test_documents_dumpers.py +++ b/tests/api/documents/test_documents_dumpers.py @@ -38,18 +38,18 @@ def test_document_dumpers(document, document_data): assert dump_data['title_text'] assert dump_data['identifiers'] - document['contribution'] = [{'entity': {'$ref': 'n/a', 'pid': 'n/a'}}] - with pytest.raises(RecordNotFound): - document.dumps(dumper=document_replace_refs_dumper) - document['contribution'] = document_data['contribution'] - document['subjects'] = [{ + entity_data = { 'entity': { - '$ref': 'n/a', - 'pid': 'n/a', + '$ref': 'https://mef.rero.ch/api/agents/idref/dummy_idref', + 'pid': 'dummy_pid', 'type': EntityType.PERSON - } } - ] + } + document['contribution'] = [entity_data] + with pytest.raises(RecordNotFound): + document.dumps(dumper=document_replace_refs_dumper) + document['contribution'] = document_data['contribution'] + document['subjects'] = [entity_data] with pytest.raises(RecordNotFound): document.dumps(dumper=document_replace_refs_dumper) diff --git a/tests/api/entities/local_entities/test_local_entities_rest.py b/tests/api/entities/local_entities/test_local_entities_rest.py index 661b0058dd..40f8f84a5b 100644 --- a/tests/api/entities/local_entities/test_local_entities_rest.py +++ b/tests/api/entities/local_entities/test_local_entities_rest.py @@ -26,7 +26,7 @@ from rero_ils.modules.entities.models import EntityType from rero_ils.modules.entities.local_entities.api import LocalEntity -from rero_ils.modules.entities.local_entities.dumpers import indexer_dumper +from rero_ils.modules.entities.dumpers import indexer_dumper def test_local_entities_permissions(client, roles, local_entity_person, diff --git a/tests/api/entities/remote_entities/test_remote_entities_rest.py b/tests/api/entities/remote_entities/test_remote_entities_rest.py index 118c2b4f5a..bb0be4bf80 100644 --- a/tests/api/entities/remote_entities/test_remote_entities_rest.py +++ b/tests/api/entities/remote_entities/test_remote_entities_rest.py @@ -23,7 +23,7 @@ from utils import get_json, mock_response, postdata, to_relative_url from rero_ils.modules.entities.models import EntityType -from rero_ils.modules.entities.remote_entities.dumpers import indexer_dumper +from rero_ils.modules.entities.dumpers import indexer_dumper def test_remote_entities_permissions(client, entity_person, json_header): diff --git a/tests/ui/documents/test_documents_filter.py b/tests/ui/documents/test_documents_filter.py index 238bc2a7e2..0497e64e83 100644 --- a/tests/ui/documents/test_documents_filter.py +++ b/tests/ui/documents/test_documents_filter.py @@ -406,7 +406,9 @@ def test_contribution_format(db, entity_organisation): contributions = [{ 'entity': {'pid': entity_organisation.pid} }] - link_part = f'/corporate-bodies/{entity_organisation.pid}' + link_part = f'/global/search/documents?q' \ + f'=contribution.entity.unique_key%3A' \ + f'{entity_organisation.unique_key}' assert link_part in contribution_format(contributions, 'en', 'global') diff --git a/tests/ui/entities/local_entities/test_local_entities_dumpers.py b/tests/ui/entities/local_entities/test_local_entities_dumpers.py index 45371061a6..3ff1395431 100644 --- a/tests/ui/entities/local_entities/test_local_entities_dumpers.py +++ b/tests/ui/entities/local_entities/test_local_entities_dumpers.py @@ -18,7 +18,7 @@ """Items Record dumper tests.""" -from rero_ils.modules.entities.local_entities.dumpers import document_dumper +from rero_ils.modules.entities.dumpers import document_dumper def test_local_entities_document_dumper(local_entity_person2): @@ -27,7 +27,6 @@ def test_local_entities_document_dumper(local_entity_person2): dumped_record = local_entity_person2.dumps(dumper=document_dumper) authorized_access_point = 'William III, King of England (1650-1702)' for field in [ - 'authorized_access_point', 'authorized_access_point_de', 'authorized_access_point_en', 'authorized_access_point_fr', From 957cdf6f0ecb973c30bfe42b2d54004a47ac3ddb Mon Sep 17 00:00:00 2001 From: Renaud Michotte Date: Wed, 19 Jul 2023 11:55:54 +0200 Subject: [PATCH 09/14] entity: merge all ids into `pids` key During serialization of an entity, all identifiers (pids) for all sources will be available into the `pids.{source}` key. This is used for `resolve=1` and also for document indexing. Co-Authored-by: Renaud Michotte --- rero_ils/modules/api.py | 5 +- .../v7/documents/document-v0.0.1.json | 90 +++++++---- rero_ils/modules/documents/views.py | 3 +- rero_ils/modules/entities/api.py | 104 +++++++------ rero_ils/modules/entities/dumpers/document.py | 11 +- .../modules/entities/local_entities/api.py | 27 +++- .../modules/entities/remote_entities/api.py | 143 ++++++++++-------- .../modules/entities/remote_entities/cli.py | 7 +- .../entities/remote_entities/replace.py | 4 +- .../modules/entities/remote_entities/sync.py | 96 +++++------- .../test_local_entities_rest.py | 31 ++++ tests/ui/documents/test_documents_api.py | 2 +- tests/ui/documents/test_documents_filter.py | 7 +- .../local_entities/test_local_entities_api.py | 73 +++++++++ ...ies_api.py => test_remote_entities_api.py} | 67 +++++--- ...lter.py => test_remote_entities_filter.py} | 0 ...ing.py => test_remote_entities_mapping.py} | 0 ...ities_ui.py => test_remote_entities_ui.py} | 0 ...utils.py => test_remote_entities_utils.py} | 0 tests/ui/entities/test_entities_api.py | 46 ++++++ 20 files changed, 463 insertions(+), 253 deletions(-) create mode 100644 tests/ui/entities/local_entities/test_local_entities_api.py rename tests/ui/entities/remote_entities/{test_entities_api.py => test_remote_entities_api.py} (87%) rename tests/ui/entities/remote_entities/{test_entities_filter.py => test_remote_entities_filter.py} (100%) rename tests/ui/entities/remote_entities/{test_entities_mapping.py => test_remote_entities_mapping.py} (100%) rename tests/ui/entities/remote_entities/{test_entities_ui.py => test_remote_entities_ui.py} (100%) rename tests/ui/entities/remote_entities/{test_entities_utils.py => test_remote_entities_utils.py} (100%) create mode 100644 tests/ui/entities/test_entities_api.py diff --git a/rero_ils/modules/api.py b/rero_ils/modules/api.py index 0694892b17..95ee258210 100644 --- a/rero_ils/modules/api.py +++ b/rero_ils/modules/api.py @@ -743,8 +743,7 @@ def index(self, indexer_class, referenced): record_to_index = r['record'] indexer.index(record_to_index) except Exception as err: - pid_type = r['pid_type'], - pid_value = r['record']['pid'] current_app.logger.error( - f'Record indexing error {pid_type} {pid_value}: {err}' + f'Record indexing error {r["pid_type"]} ' + f'{r["record"]["pid"]}: {str(err)}' ) diff --git a/rero_ils/modules/documents/mappings/v7/documents/document-v0.0.1.json b/rero_ils/modules/documents/mappings/v7/documents/document-v0.0.1.json index ee2295c41c..d85f87ee35 100644 --- a/rero_ils/modules/documents/mappings/v7/documents/document-v0.0.1.json +++ b/rero_ils/modules/documents/mappings/v7/documents/document-v0.0.1.json @@ -327,20 +327,28 @@ "primary_source": { "type": "keyword" }, - "unique_key": { - "type": "keyword" - }, "pid": { "type": "keyword" }, - "id_gnd": { - "type": "keyword" - }, - "id_idref": { - "type": "keyword" - }, - "id_rero": { - "type": "keyword" + "pids": { + "type": "object", + "properties": { + "gnd": { + "type": "keyword" + }, + "idref": { + "type": "keyword" + }, + "rero": { + "type": "keyword" + }, + "remote": { + "type": "keyword" + }, + "local": { + "type": "keyword" + } + } }, "authorized_access_point_en": { "type": "text", @@ -853,17 +861,25 @@ "pid": { "type": "keyword" }, - "unique_key": { - "type": "keyword" - }, - "id_gnd": { - "type": "keyword" - }, - "id_idref": { - "type": "keyword" - }, - "id_rero": { - "type": "keyword" + "pids": { + "type": "object", + "properties": { + "gnd": { + "type": "keyword" + }, + "idref": { + "type": "keyword" + }, + "rero": { + "type": "keyword" + }, + "remote": { + "type": "keyword" + }, + "local": { + "type": "keyword" + } + } }, "authorized_access_point_en": { "type": "text", @@ -980,9 +996,6 @@ "pid": { "type": "keyword" }, - "unique_key": { - "type": "keyword" - }, "sources": { "type": "keyword" }, @@ -1789,14 +1802,25 @@ "pid": { "type": "keyword" }, - "id_gnd": { - "type": "keyword" - }, - "id_idref": { - "type": "keyword" - }, - "id_rero": { - "type": "keyword" + "pids": { + "type": "object", + "properties": { + "gnd": { + "type": "keyword" + }, + "idref": { + "type": "keyword" + }, + "rero": { + "type": "keyword" + }, + "remote": { + "type": "keyword" + }, + "local": { + "type": "keyword" + } + } }, "authorized_access_point_en": { "type": "text", diff --git a/rero_ils/modules/documents/views.py b/rero_ils/modules/documents/views.py index 28a3e7872e..6d3335b79c 100644 --- a/rero_ils/modules/documents/views.py +++ b/rero_ils/modules/documents/views.py @@ -270,7 +270,8 @@ def contribution_format(contributions, language, viewcode, with_roles=False): args = { 'viewcode': viewcode, 'recordType': 'documents', - 'q': f'contribution.entity.unique_key:{entity.unique_key}', + 'q': f'contribution.entity.pids.{entity.resource_type}:' + f'{entity.pid}', 'simple': 0 } url = url_for('rero_ils.search', **args) diff --git a/rero_ils/modules/entities/api.py b/rero_ils/modules/entities/api.py index 661db5b7bf..906763202a 100644 --- a/rero_ils/modules/entities/api.py +++ b/rero_ils/modules/entities/api.py @@ -23,10 +23,11 @@ from elasticsearch_dsl import Q, A from flask import current_app +from rero_ils.modules.api import IlsRecord from rero_ils.modules.documents.api import DocumentsSearch -class Entity(ABC): +class Entity(IlsRecord, ABC): """Entity class.""" @abstractmethod @@ -39,63 +40,22 @@ def get_authorized_access_point(self, language): raise NotImplementedError @property - def unique_key(self): - """Get the unique key. - - As entity subclasses doesn't share pid generation sequence, pid - overlapping can occur. To prevent this, use `unique_key` to point - specific entity. - - Example: - => `remote_1` - => `local_1` - """ - return f'{self.resource_type}_{self.pid}' - - def _search_documents(self, with_subjects=True, - with_subjects_imported=True): - """Get search documents.""" - filters = Q('term', contribution__entity__unique_key=self.unique_key) - - if with_subjects: - filters |= \ - Q('term', subjects__entity__unique_key=self.unique_key) - if with_subjects_imported: - filters |= \ - Q('term', - subjects_imported__entity__unique_key=self.unique_key) - return DocumentsSearch().filter(filters) - - def documents_pids(self, with_subjects=True, with_subjects_imported=True): - """Get documents pids.""" - search = self._search_documents( - with_subjects=with_subjects, - with_subjects_imported=with_subjects_imported - ).source('pid') - return [hit.pid for hit in search.scan()] - - def documents_ids(self, with_subjects=True, with_subjects_imported=True): - """Get documents ids.""" - search = self._search_documents( - with_subjects=with_subjects, - with_subjects_imported=with_subjects_imported - ).source() - return [hit.meta.id for hit in search.scan()] + @abstractmethod + def resource_type(self): + """Get the entity type.""" + raise NotImplementedError @property def organisation_pids(self): """Get organisations pids.""" - # TODO :: Should be linked also on other fields ? - # ex: subjects, genre_form, ... - # Seems only use to filer entities by viewcode. search = self._search_documents() agg = A( 'terms', field='holdings.organisation.organisation_pid', min_doc_count=1, size=current_app.config - .get('RERO_ILS_AGGREGATION_SIZE') - .get('organisations') + .get('RERO_ILS_AGGREGATION_SIZE', {}) + .get('organisations', 10) ) search.aggs.bucket('organisation', agg) results = search.execute() @@ -103,3 +63,51 @@ def organisation_pids(self): result.key for result in results.aggregations.organisation.buckets }) + + def _search_documents( + self, with_subjects=True, with_subjects_imported=True, + with_genre_forms=True + ): + """Get ES query to search documents containing this entity. + + :param with_subjects: search also on `subjects` ? + :param with_subjects_imported: search also on `subject_imported` ? + :param with_genre_forms: search also on `genreForm` ? + """ + contribution_key = f'contribution.entity.pids.{self.resource_type}' + filters = Q('term', **{contribution_key: self.pid}) + if with_subjects: + search_field = f'subjects.entity.pids.{self.resource_type}' + filters |= Q('term', **{search_field: self.pid}) + if with_subjects_imported: + search_field = f'subjects_imported.pids.{self.resource_type}' + filters |= Q('term', **{search_field: self.pid}) + if with_genre_forms: + search_field = f'genreForm.entity.pids.{self.resource_type}' + filters |= Q('term', **{search_field: self.pid}) + + return DocumentsSearch().filter(filters) + + def documents_pids( + self, with_subjects=True, with_subjects_imported=True, + with_genre_forms=True + ): + """Get documents pids.""" + search = self._search_documents( + with_subjects=with_subjects, + with_subjects_imported=with_subjects_imported, + with_genre_forms=with_genre_forms + ).source('pid') + return [hit.pid for hit in search.scan()] + + def documents_ids( + self, with_subjects=True, with_subjects_imported=True, + with_genre_forms=True + ): + """Get documents ids.""" + search = self._search_documents( + with_subjects=with_subjects, + with_subjects_imported=with_subjects_imported, + with_genre_forms=with_genre_forms + ).source(False) + return [hit.meta.id for hit in search.scan()] diff --git a/rero_ils/modules/entities/dumpers/document.py b/rero_ils/modules/entities/dumpers/document.py index 5fd413c468..25545694fa 100644 --- a/rero_ils/modules/entities/dumpers/document.py +++ b/rero_ils/modules/entities/dumpers/document.py @@ -34,18 +34,18 @@ def dump(self, record, data): :param record: The record to dump. :param data: The initial dump data passed in by ``record.dumps()``. """ - # DEV NOTES: Why using `unique_key` - # Unique key is used to avoid nested implementation in Elasticsearch data = { - 'pid': record.pid, 'type': record['type'], - 'unique_key': record.unique_key + 'pid': record.pid, + 'pids': { + record.resource_type: record.pid + } } if record.resource_type == EntityResourceType.REMOTE: for agency in current_app.config['RERO_ILS_AGENTS_SOURCES']: if field := record.get(agency): data['type'] = field.get('bf:Agent', record['type']) - data[f'id_{agency}'] = record[agency]['pid'] + data['pids'][agency] = record[agency]['pid'] variant_access_points = [] parallel_access_points = [] @@ -58,5 +58,4 @@ def dump(self, record, data): data['variant_access_point'] = variant_access_points if parallel_access_points: data['parallel_access_point'] = parallel_access_points - return data diff --git a/rero_ils/modules/entities/local_entities/api.py b/rero_ils/modules/entities/local_entities/api.py index 28437ad353..955b1a33f4 100644 --- a/rero_ils/modules/entities/local_entities/api.py +++ b/rero_ils/modules/entities/local_entities/api.py @@ -20,7 +20,7 @@ from functools import partial -from rero_ils.modules.api import IlsRecord, IlsRecordsSearch +from rero_ils.modules.api import IlsRecordsSearch from rero_ils.modules.utils import sorted_pids from rero_ils.modules.fetchers import id_fetcher from rero_ils.modules.minters import id_minter @@ -61,7 +61,7 @@ class Meta: default_filter = None -class LocalEntity(IlsRecord, Entity): +class LocalEntity(Entity): """Local entity class.""" minter = local_entity_id_minter @@ -76,13 +76,28 @@ class LocalEntity(IlsRecord, Entity): AuthorizedAccessPointExtension(), OperationLogObserverExtension() ] - resource_type = EntityResourceType.LOCAL + + @property + def resource_type(self): + """Get entity type.""" + return EntityResourceType.LOCAL @property def type(self): """Shortcut for local entity type.""" return self.get('type') + def get_authorized_access_point(self, language): + """Get localized authorized_access_point. + + For a local entity, no matters `language` parameter, the authorized + access point is always the `authorized_access_point` field content. + + :param language: language for authorized access point. + :returns: authorized access point in given language. + """ + return self.get('authorized_access_point') + def resolve(self): """Resolve references data. @@ -91,6 +106,12 @@ def resolve(self): :returns: a fresh copy of the resolved data. """ + # DEV NOTES :: Why using `replace_refs_dumper` + # Not really required now (because no $ref relation exists into an + # entity resource) but in next development, links between entity will + # be implemented. + # The links will be stored as a `$ref` and `replace_refs_dumper` + # will be used. return self.dumps(replace_refs_dumper) def get_links_to_me(self, get_pids=False): diff --git a/rero_ils/modules/entities/remote_entities/api.py b/rero_ils/modules/entities/remote_entities/api.py index 57a38d7782..7d69f12eca 100644 --- a/rero_ils/modules/entities/remote_entities/api.py +++ b/rero_ils/modules/entities/remote_entities/api.py @@ -24,8 +24,9 @@ from elasticsearch_dsl.query import Q from flask import current_app from invenio_db import db +from urllib3.exceptions import HTTPError -from rero_ils.modules.api import IlsRecord, IlsRecordsIndexer, IlsRecordsSearch +from rero_ils.modules.api import IlsRecordsIndexer, IlsRecordsSearch from rero_ils.modules.documents.api import DocumentsIndexer from rero_ils.modules.fetchers import id_fetcher from rero_ils.modules.minters import id_minter @@ -65,44 +66,35 @@ class Meta: default_filter = None -class RemoteEntity(IlsRecord, Entity): +class RemoteEntity(Entity): """Mef contribution class.""" minter = remote_entity_id_minter fetcher = remote_entity_id_fetcher provider = RemoteEntityProvider model_cls = RemoteEntityMetadata - # disable legacy replace refs - enable_jsonref = False + enable_jsonref = False # disable legacy replace refs - resource_type = EntityResourceType.REMOTE - - def resolve(self): - """Resolve references data. + @classmethod + def get_entity(cls, ref_type, ref_pid): + """Get entity based on type and id. - Uses the dumper to do the job. - Mainly used by the `resolve=1` URL parameter. + In case of multiple entity, we will return the most recent created. - :returns: a fresh copy of the resolved data. + :param ref_type: the type of identifier (mef, viaf, ...) + :param ref_pid: the identifier to search. + :returns: the corresponding `Entity` if exists. """ - return self.dumps(replace_refs_dumper) - - @classmethod - def get_entity(cls, ref_type, ref_pid): - """Get contribution.""" if ref_type == 'mef': return cls.get_record_by_pid(ref_pid) es_filter = Q('term', **{f'{ref_type}.pid': ref_pid}) if ref_type == 'viaf': es_filter = Q('term', viaf_pid=ref_pid) - - # in case of multiple results get the more recent query = RemoteEntitiesSearch() \ .params(preserve_order=True) \ - .sort({'_created': {'order': 'desc'}})\ + .sort({'_created': {'order': 'desc'}}) \ .filter(es_filter) - with contextlib.suppress(StopIteration): pid = next(query.source('pid').scan()).pid return cls.get_record_by_pid(pid) @@ -114,46 +106,67 @@ def get_record_by_ref(cls, ref): If the record dos not exist get it from MEF and create it. :param ref: MEF URI - :returns: the corresponding `Entity` class instance + :returns: the corresponding `Entity` class instance ; if entity has + loaded from remote server. + :rtype: tuple(`Entity`, bool) """ online = False entity_type, ref_type, ref_pid = extract_data_from_mef_uri(ref) - entity = cls.get_entity(ref_type, ref_pid) - if not entity: - # We dit not find the record in DB get it from MEF and create it. - nested = db.session.begin_nested() - try: - if not (data := get_mef_data_by_type( - entity_type=entity_type, - pid_type=ref_type, - pid=ref_pid - )): - raise Exception('NO DATA') - # Try to get the contribution from DB maybe it was not indexed. - if entity := RemoteEntity.get_record_by_pid(data['pid']): - entity = entity.replace(data) - else: - entity = cls.create(data) - online = True - nested.commit() - # TODO: reindex in the document indexing - entity.reindex() - except Exception as err: - nested.rollback() - current_app.logger.error( - f'Get MEF record: {ref_type}:{ref_pid} >>{err}<<' - ) - entity = None + if entity := cls.get_entity(ref_type, ref_pid): + return entity, online + + # Corresponding entity isn't found into database. + # 1) Get it from remote MEF server + # 2) Create the entity from remote data + nested = db.session.begin_nested() + try: + data = get_mef_data_by_type( + entity_type=entity_type, pid_type=ref_type, pid=ref_pid) + if not data: + raise HTTPError('', 404, "Not found") + # Try to get the contribution from DB maybe it was not indexed. + if entity := RemoteEntity.get_record_by_pid(data['pid']): + entity = entity.replace(data) + else: + entity = cls.create(data) + online = True + nested.commit() + # TODO: reindex in the document indexing + entity.reindex() + except Exception as err: + nested.rollback() + current_app.logger.error( + f'Get MEF record: {ref_type}:{ref_pid} >>{err}<<' + ) + entity = None return entity, online - def _get_mef_localized_value(self, key, language): - """Get the 1st localized value for given key among MEF source list.""" - order = current_app.config.get('RERO_ILS_AGENTS_LABEL_ORDER', []) - source_order = order.get(language, order.get(order['fallback'], [])) - for source in source_order: - if value := self.get(source, {}).get(key, None): - return value - return self.get(key, None) + @property + def resource_type(self): + """Get entity type.""" + return EntityResourceType.REMOTE + + @property + def type(self): + """Get entity type.""" + entity_types = current_app.config['RERO_ILS_ENTITY_TYPES'] + return entity_types.get(self['type']) + + def resolve(self): + """Resolve references data. + + Uses the dumper to do the job. + Mainly used by the `resolve=1` URL parameter. + + :returns: a fresh copy of the resolved data. + """ + # DEV NOTES :: Why using `replace_refs_dumper` + # Not really required now (because no $ref relation exists into an + # entity resource) but in next development, links between entity will + # be implemented. + # The links will be stored as a `$ref` and `replace_refs_dumper` + # will be used. + return self.dumps(replace_refs_dumper) def get_authorized_access_point(self, language): """Get localized authorized_access_point. @@ -166,10 +179,8 @@ def get_authorized_access_point(self, language): language=language ) - def update_online( - self, dbcommit=False, reindex=False, verbose=False, - reindex_doc=True - ): + def update_online(self, dbcommit=False, reindex=False, verbose=False, + reindex_doc=True): """Update record online. :param reindex: reindex record by record @@ -222,18 +233,20 @@ def source_pids(self): if source in self } - @property - def type(self): - """Get entity type.""" - entity_types = current_app.config['RERO_ILS_ENTITY_TYPES'] - return entity_types.get(self['type']) + def _get_mef_localized_value(self, key, language): + """Get the 1st localized value for given key among MEF source list.""" + order = current_app.config.get('RERO_ILS_AGENTS_LABEL_ORDER', []) + source_order = order.get(language, order.get(order['fallback'], [])) + for source in source_order: + if value := self.get(source, {}).get(key, None): + return value + return self.get(key, None) class RemoteEntitiesIndexer(IlsRecordsIndexer): """Entity indexing class.""" record_cls = RemoteEntity - # data dumper for indexing record_dumper = indexer_dumper def bulk_index(self, record_id_iterator): diff --git a/rero_ils/modules/entities/remote_entities/cli.py b/rero_ils/modules/entities/remote_entities/cli.py index 7592d46e37..e97b09489d 100644 --- a/rero_ils/modules/entities/remote_entities/cli.py +++ b/rero_ils/modules/entities/remote_entities/cli.py @@ -91,10 +91,9 @@ def clean(query, dry_run, verbose, log_dir): err_pids = [] with click.progressbar(pids, length=total) as bar: for pid in bar: - updated, error = sync_entity.remove_unused_record(pid) - if updated: - n_removed += 1 - if error: + try: + n_removed += int(sync_entity.remove_unused_record(pid)) + except Exception: err_pids.append(pid) click.secho(f'{n_removed} removed MEF records', fg='green') diff --git a/rero_ils/modules/entities/remote_entities/replace.py b/rero_ils/modules/entities/remote_entities/replace.py index e34123cafd..419ad3ad93 100644 --- a/rero_ils/modules/entities/remote_entities/replace.py +++ b/rero_ils/modules/entities/remote_entities/replace.py @@ -91,10 +91,8 @@ def _get_latest(self, entity_type, source, pid): url = f'{self._get_base_url(entity_type)}/mef/latest/{source}:{pid}' res = requests_retry_session().get(url) if res.status_code == requests.codes.ok: - data = res.json() # TODO: could be deleted if MEF is updated. - # If we have a latest record add type to the data. - if data: + if data := res.json(): if data_type := data.get('bf:Agent', data.get('type')): data['type'] = data_type elif entity_type == 'concepts': diff --git a/rero_ils/modules/entities/remote_entities/sync.py b/rero_ils/modules/entities/remote_entities/sync.py index 668ee2883a..1733c7fe76 100644 --- a/rero_ils/modules/entities/remote_entities/sync.py +++ b/rero_ils/modules/entities/remote_entities/sync.py @@ -25,17 +25,17 @@ import requests from deepdiff import DeepDiff -from elasticsearch_dsl import Q from invenio_db import db -from rero_ils.modules.documents.api import Document, DocumentsSearch +from rero_ils.modules.commons.exceptions import RecordNotFound +from rero_ils.modules.documents.api import Document from rero_ils.modules.entities.remote_entities.api import \ RemoteEntitiesSearch, RemoteEntity from rero_ils.modules.utils import get_mef_url, get_timestamp, \ requests_retry_session, set_timestamp -class SyncEntity(object): +class SyncEntity: """Entity MEF synchronization.""" def __init__(self, dry_run=False, verbose=False, log_dir=None, @@ -175,23 +175,6 @@ def _update_entities_in_document(self, doc_pid, pids_to_replace): if not self.dry_run: doc.replace(doc, dbcommit=True, reindex=True) - @staticmethod - def _get_documents_pids_from_mef(pid): - """Retrieve all the linked documents to a MEF record. - - :param pid: (string) a MEF identifier. - :returns: a list of identifiers. - :rtype: list of strings. - """ - # the MEF link can be in contribution or subjects - es_query = DocumentsSearch() - filters = Q('term', contribution__entity__pid=pid) - filters |= Q('term', subjects__entity__pid=pid) - filters |= Q('term', genreForm__entity__pid=pid) - es_query = es_query.filter('bool', must=[filters]).source('pid') - # can be a list as it should not be too big - return [d.pid for d in es_query.params(scroll='30m').scan()] - def get_entities_pids(self, query='*', from_date=None): """Get contributions identifiers. @@ -293,15 +276,14 @@ def sync_record(self, pid): :rtype: integer, boolean, x. """ # close db session to prevent psycopg2.OperationalError. - # a new session will be opend automaticly. + # a new session will be open automatically. db.session.close() doc_updated = set() updated = error = False try: - # get contribution in db - entity = RemoteEntity.get_record_by_pid(pid) - if not entity: - raise Exception(f'ERROR MEF {pid} does not exists in db.') + if not (entity := RemoteEntity.get_record_by_pid(pid)): + raise RecordNotFound(RemoteEntity, pid) + self.logger.debug(f'Processing {entity["type"]} MEF(pid: {pid})') # iterate over all entity sources: rero, gnd, idref pids_to_replace = {} @@ -313,14 +295,14 @@ def sync_record(self, pid): ) # MEF sever failed to retrieve the latest MEF record # for the given entity - if not mef.get('pid'): + if not (mef_pid := mef.get('pid')): raise Exception( f'Error cannot get latest for ' f'{entity["type"]} {source}:{entity[source]["pid"]}') - old_entity_pid = entity[source]["pid"] + old_entity_pid = entity[source]['pid'] new_entity_pid = mef[source]['pid'] - new_mef_pid = mef.get('pid') + new_mef_pid = mef_pid old_mef_pid = entity.pid if old_entity_pid != new_entity_pid: mef_url = f'{get_mef_url(entity.type)}/{source}' @@ -426,7 +408,7 @@ def end_sync(self, n_doc_updated, n_mef_updated, mef_errors): n_mef_updated=n_mef_updated, errors=errors, start_timestamp=self.start_timestamp) - def sync(self, query="*", from_date=None, in_memory=False): + def sync(self, query='*', from_date=None, in_memory=False): """Updated the MEF records and the linked documents. :param query: (string) a query to select the MEF record to be updated. @@ -462,28 +444,19 @@ def remove_unused_record(self, pid): """Removes MEF record if it is not linked to any documents. :param pid: (string) MEF identifier. - :returns: true if the record has been deleted, true if an error occurs. - :rtype: boolean, boolean + :returns: True if the record has been deleted + :rtype: bool + :raises Exception: If a deletion problem occurred """ - try: - doc_pids = SyncEntity._get_documents_pids_from_mef(pid) - if len(doc_pids) == 0: - # get the contribution for the database - entity = RemoteEntity.get_record_by_pid(pid) - if not self.dry_run: - # remove from the database and the index: no tombstone - entity.delete(True, True, True) - self.logger.info( - f'MEF {entity["type"]} record(pid: {entity.pid}) ' - 'has been deleted.') - # removed, no error - return True, False - except Exception as err: - self.logger.error(f'MEF record(pid: {pid}) -> {err}') - # no removed, error - return False, True - # no removed, no error - return False, False + entity = RemoteEntity.get_record_by_pid(pid) + if not entity.documents_pids(): + if not self.dry_run: + # remove from the database and the index: no tombstone + entity.delete(True, True, True) + self.logger.info(f'MEF {entity["type"]} record(pid: {entity.pid}) ' + 'has been deleted.') + return True + return False @classmethod def get_errors(cls): @@ -507,24 +480,23 @@ def start_clean(self): else: self.logger.info('--------- Starting cleaning ---------') - def remove_unused(self, query="*"): + def remove_unused(self, query='*'): """Removes MEF records that are not linked to any documents. :param query: (string) query to limit the record candidates. - :returns: the number of deleted records, the list of pid that + :returns: the number of deleted records; the list of pid that causes an error. - :rtype: integer, list of strings. + :rtype: integer, list. """ self.start_clean() - n_removed = 0 - err_pids = [] + removed_entity_counter = 0 + error_entities = [] pids, _ = self.get_entities_pids(query) for pid in pids: - removed, error = self.remove_unused_record(pid) - if removed: - n_removed += 1 - if error: - err_pids.append(pid) + try: + removed_entity_counter += int(self.remove_unused_record(pid)) + except Exception: + error_entities.append(pid) sys.stdout.flush() - self.logger.info(f'DONE: MEF deleted: {n_removed}') - return n_removed, err_pids + self.logger.info(f'DONE: MEF deleted: {removed_entity_counter}') + return removed_entity_counter, error_entities diff --git a/tests/api/entities/local_entities/test_local_entities_rest.py b/tests/api/entities/local_entities/test_local_entities_rest.py index 40f8f84a5b..4f37ae8d84 100644 --- a/tests/api/entities/local_entities/test_local_entities_rest.py +++ b/tests/api/entities/local_entities/test_local_entities_rest.py @@ -24,9 +24,11 @@ from utils import get_json, postdata, to_relative_url, \ VerifyRecordPermissionPatch +from rero_ils.modules.documents.dumpers import document_replace_refs_dumper from rero_ils.modules.entities.models import EntityType from rero_ils.modules.entities.local_entities.api import LocalEntity from rero_ils.modules.entities.dumpers import indexer_dumper +from rero_ils.modules.utils import get_ref_for_pid def test_local_entities_permissions(client, roles, local_entity_person, @@ -187,3 +189,32 @@ def test_local_search_by_proxy( assert response.status_code == 200 assert len(response.json) == 1 assert response.json[0]['pid'] == local_entity_org.pid + + +@mock.patch('invenio_records_rest.views.verify_record_permission', + mock.MagicMock(return_value=VerifyRecordPermissionPatch)) +def test_local_entities_resolve( + client, mef_agents_url, local_entity_person, document +): + """Test local entity resolver""" + + # LOCAL ENTITY RESOLVER =================================================== + res = client.get(url_for( + 'invenio_records_rest.locent_item', + pid_value=local_entity_person.pid, + resolve='1' + )) + assert res.status_code == 200 + + # LOCAL ENTITY INTO A DOCUMENT RESOLVER =================================== + ent_ref = get_ref_for_pid('locent', local_entity_person.pid) + document.setdefault('contribution', []).append({ + 'entity': {'$ref': ent_ref}, + 'role': ['aut'] + }) + document = document.update(document, dbcommit=True, reindex=True) + data = document.dumps(dumper=document_replace_refs_dumper) + assert any( + contribution['entity'].get('pid') == local_entity_person.pid + for contribution in data['contribution'] + ) diff --git a/tests/ui/documents/test_documents_api.py b/tests/ui/documents/test_documents_api.py index f70a896f90..18125b5b2b 100644 --- a/tests/ui/documents/test_documents_api.py +++ b/tests/ui/documents/test_documents_api.py @@ -175,7 +175,7 @@ def test_document_linked_subject( es_record = DocumentsSearch().get_record_by_pid(doc.pid) subject = es_record['subjects'][0] assert subject['entity']['primary_source'] == _type - assert _id in subject['entity'][f'id_{_type}'] + assert _id in subject['entity']['pids'][_type] assert subject['entity']['authorized_access_point_fr'] == \ 'Antienzymes' assert 'Inhibiteurs enzymatiques' \ diff --git a/tests/ui/documents/test_documents_filter.py b/tests/ui/documents/test_documents_filter.py index 0497e64e83..210e991964 100644 --- a/tests/ui/documents/test_documents_filter.py +++ b/tests/ui/documents/test_documents_filter.py @@ -393,6 +393,7 @@ def test_work_access_point(): def test_contribution_format(db, entity_organisation): """Test contribution format.""" + entity = entity_organisation contributions = [{ 'entity': { 'authorized_access_point': 'author_def', @@ -404,11 +405,11 @@ def test_contribution_format(db, entity_organisation): assert contribution_format(contributions, 'zh', 'global') == 'author_def' contributions = [{ - 'entity': {'pid': entity_organisation.pid} + 'entity': {'pid': entity.pid} }] link_part = f'/global/search/documents?q' \ - f'=contribution.entity.unique_key%3A' \ - f'{entity_organisation.unique_key}' + f'=contribution.entity.pids.{entity.resource_type}%3A' \ + f'{entity.pid}' assert link_part in contribution_format(contributions, 'en', 'global') diff --git a/tests/ui/entities/local_entities/test_local_entities_api.py b/tests/ui/entities/local_entities/test_local_entities_api.py new file mode 100644 index 0000000000..ecfbf9c3f1 --- /dev/null +++ b/tests/ui/entities/local_entities/test_local_entities_api.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Local entities record tests.""" + +from __future__ import absolute_import, print_function + +import time +from datetime import timedelta +from utils import flush_index + +from rero_ils.modules.documents.api import Document, DocumentsSearch +from rero_ils.modules.utils import get_ref_for_pid + + +def test_local_entity_properties(local_entity_person): + """Test local entity property""" + assert local_entity_person.get_authorized_access_point(None) == \ + local_entity_person['authorized_access_point'] + + +def test_local_entity_indexing(app, local_entity_person, document_data_tmp): + """Test local entity indexing.""" + entity = local_entity_person + + # Check relations between local entity and other resources. + data = document_data_tmp + data.setdefault('contribution', []).append({ + 'entity': {'$ref': get_ref_for_pid('locent', entity.pid)}, + 'role': ['aut'] + }) + doc = Document.create(data, delete_pid=True, reindex=True, dbcommit=True) + reasons = entity.reasons_not_to_delete() + assert reasons['links']['documents'] + + # Update the local entity and check if related resources are updated + original_access_point = entity['authorized_access_point'] + entity['name'] = 'my_local_access_point' + entity = entity.update(entity, dbcommit=True, reindex=True, commit=True) + # updating related resource is an asynchronous task (to not block app if + # there are a lot of related resource). We need to wait to the end of the + # task to check id related resources are up-to-date. + delay = app.config.get('RERO_ILS_INDEXER_TASK_DELAY', 0) \ + + timedelta(seconds=2) + time.sleep(delay.seconds) # find a better way to detect task is finished. + + flush_index(DocumentsSearch.Meta.index) + hit = DocumentsSearch().get_record_by_pid(doc.pid) + assert any( + contribution['entity']['authorized_access_point_fr'] == + entity.get_authorized_access_point(language='fr') + for contribution in hit.contribution + ) + + # reset fixtures + entity['authorized_access_point'] = original_access_point + entity.update(entity, dbcommit=True, reindex=True) + doc.delete() diff --git a/tests/ui/entities/remote_entities/test_entities_api.py b/tests/ui/entities/remote_entities/test_remote_entities_api.py similarity index 87% rename from tests/ui/entities/remote_entities/test_entities_api.py rename to tests/ui/entities/remote_entities/test_remote_entities_api.py index cfe23313b7..4ffc13edce 100644 --- a/tests/ui/entities/remote_entities/test_entities_api.py +++ b/tests/ui/entities/remote_entities/test_remote_entities_api.py @@ -209,9 +209,9 @@ def test_sync_contribution( assert RemoteEntity.get_record_by_pid('foo_mef') # document has been updated with the new MEF and IDREF pid assert DocumentsSearch().query( - 'term', contribution__entity__pid='foo_mef').count() + 'term', contribution__entity__pids__remote='foo_mef').count() assert DocumentsSearch().query( - 'term', contribution__entity__id_idref='foo_idref').count() + 'term', contribution__entity__pids__idref='foo_idref').count() db_agent = Document.get_record_by_pid( doc.pid).get('contribution')[0]['entity'] assert db_agent['$ref'] == f'{mef_agents_url}/idref/foo_idref' @@ -233,6 +233,7 @@ def test_sync_concept( mock_get, app, mef_concepts_url, entity_topic_data, document_data_subject_ref ): + # """Test MEF agent synchronization.""" # === setup log_path = tempfile.mkdtemp() @@ -247,9 +248,8 @@ def test_sync_concept( ) flush_index(RemoteEntitiesSearch.Meta.index) - idref_pid = topic['idref']['pid'] - document_data_subject_ref['subjects'][0]['entity']['$ref'] = \ - f'{mef_concepts_url}/idref/{idref_pid}' + entity_url = f'{mef_concepts_url}/idref/{topic["idref"]["pid"]}' + document_data_subject_ref['subjects'][0]['entity']['$ref'] = entity_url doc = Document.create( deepcopy(document_data_subject_ref), @@ -260,14 +260,11 @@ def test_sync_concept( flush_index(DocumentsSearch.Meta.index) # === nothing to update - sync_entity._get_latest = mock.MagicMock( - # TODO: delete pop for MEF v0.12.0 - return_value=entity_topic_data - ) + sync_entity._get_latest = mock.MagicMock(return_value=entity_topic_data) # nothing touched as it is up-to-date - assert (0, 0, set()) == sync_entity.sync(f'{topic.pid}') + assert (0, 0, set()) == sync_entity.sync(f'pid:{topic.pid}') # nothing removed - assert (0, []) == sync_entity.remove_unused(f'{topic.pid}') + assert (0, []) == sync_entity.remove_unused(f'pid:{topic.pid}') # === MEF metadata has been changed data = deepcopy(entity_topic_data) @@ -283,25 +280,25 @@ def test_sync_concept( subjects__entity__authorized_access_point_fr='foo').count() == 0 # synchronization the same document has been updated 3 times, one MEF # record has been updated, no errors - assert (1, 1, set()) == sync_entity.sync(f'{topic.pid}') + assert (1, 1, set()) == sync_entity.sync(f'pid:{topic.pid}') flush_index(DocumentsSearch.Meta.index) # contribution and document should be changed - assert RemoteEntity.get_record_by_pid( - topic.pid)['idref']['authorized_access_point'] == 'foo' - assert DocumentsSearch().query( - 'term', subjects__entity__authorized_access_point_fr='foo').count() + entity = RemoteEntity.get_record_by_pid(topic.pid) + assert entity['idref']['authorized_access_point'] == 'foo' + assert DocumentsSearch()\ + .query('term', subjects__entity__authorized_access_point_fr='foo')\ + .count() # nothing has been removed as only metadata has been changed - assert (0, []) == sync_entity.remove_unused(f'{topic.pid}') + assert (0, []) == sync_entity.remove_unused(topic.pid) - # remove the document + # RESET FIXTURES + # * Remove the document + # * Entity record can be removed ; and should not exist anymore doc = Document.get_record_by_pid(doc.pid) doc.delete(True, True, True) flush_index(DocumentsSearch.Meta.index) - - # the MEF record can be removed assert (1, []) == sync_entity.remove_unused() - # should not exists anymore assert not RemoteEntity.get_record_by_pid('foo_mef') @@ -430,3 +427,31 @@ def test_replace_identified_by( 'bf:Person: Athenagoras (patriarche oecuménique ; 1)', 'rero:A021039750': 'bf:Topic: Bases de données déductives' } + + +def test_entity_get_record_by_ref( + mef_agents_url, entity_person, entity_person_data_tmp +): + """Test remote entity: get record by ref.""" + dummy_ref = f'{mef_agents_url}/idref/dummy_idref_pid' + assert (None, False) == RemoteEntity.get_record_by_ref(dummy_ref) + + # Remote entity from ES index + RemoteEntitiesSearch().filter('term', pid=entity_person.pid).delete() + flush_index(RemoteEntitiesSearch.Meta.index) + ent_ref = f'{mef_agents_url}/idref/{entity_person["idref"]["pid"]}' + with mock.patch( + 'rero_ils.modules.entities.remote_entities.api.get_mef_data_by_type', + return_value=entity_person_data_tmp + ): + entity, online = RemoteEntity.get_record_by_ref(ent_ref) + assert entity and online + flush_index(RemoteEntitiesSearch.Meta.index) + assert RemoteEntitiesSearch().filter('term', pid=entity_person.pid).count() + + +def test_remote_entity_resolve(entity_person): + """Test remote entity resolver.""" + # TODO :: Only for code coverage for now. When relations between entities + # will be implemented, this test should be corrected. + assert entity_person.resolve() diff --git a/tests/ui/entities/remote_entities/test_entities_filter.py b/tests/ui/entities/remote_entities/test_remote_entities_filter.py similarity index 100% rename from tests/ui/entities/remote_entities/test_entities_filter.py rename to tests/ui/entities/remote_entities/test_remote_entities_filter.py diff --git a/tests/ui/entities/remote_entities/test_entities_mapping.py b/tests/ui/entities/remote_entities/test_remote_entities_mapping.py similarity index 100% rename from tests/ui/entities/remote_entities/test_entities_mapping.py rename to tests/ui/entities/remote_entities/test_remote_entities_mapping.py diff --git a/tests/ui/entities/remote_entities/test_entities_ui.py b/tests/ui/entities/remote_entities/test_remote_entities_ui.py similarity index 100% rename from tests/ui/entities/remote_entities/test_entities_ui.py rename to tests/ui/entities/remote_entities/test_remote_entities_ui.py diff --git a/tests/ui/entities/remote_entities/test_entities_utils.py b/tests/ui/entities/remote_entities/test_remote_entities_utils.py similarity index 100% rename from tests/ui/entities/remote_entities/test_entities_utils.py rename to tests/ui/entities/remote_entities/test_remote_entities_utils.py diff --git a/tests/ui/entities/test_entities_api.py b/tests/ui/entities/test_entities_api.py new file mode 100644 index 0000000000..74fd9700d0 --- /dev/null +++ b/tests/ui/entities/test_entities_api.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Entities record tests.""" +import pytest + +from rero_ils.modules.commons.exceptions import RecordNotFound +from rero_ils.modules.entities.api import Entity +from rero_ils.modules.entities.helpers import get_entity_record_from_data +from rero_ils.modules.utils import get_ref_for_pid + + +def test_entities_properties(entity_person_data_tmp): + """Test entity properties.""" + + # These tests are only for code coverage + entity = Entity(entity_person_data_tmp) + with pytest.raises(NotImplementedError): + entity.get_authorized_access_point(None) + with pytest.raises(NotImplementedError): + entity.resource_type + + +def test_entities_helpers(local_entity_org): + """Test entity helpers""" + data = {'pid': 'dummy'} + with pytest.raises(RecordNotFound): + get_entity_record_from_data(data) + + data = {'$ref': get_ref_for_pid('locent', local_entity_org.pid)} + assert get_entity_record_from_data(data) == local_entity_org From 2911af458bc2d4111757f9d4e2f438ed11012bde Mon Sep 17 00:00:00 2001 From: Bertrand Zuchuat Date: Wed, 16 Aug 2023 10:54:54 +0200 Subject: [PATCH 10/14] local entities: reorder the conference fields Co-Authored-by: Bertrand Zuchuat --- .../local_entities/local_entity_organisation-v0.0.1.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_organisation-v0.0.1.json b/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_organisation-v0.0.1.json index f920e56539..fe0b5449ee 100644 --- a/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_organisation-v0.0.1.json +++ b/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_organisation-v0.0.1.json @@ -15,9 +15,9 @@ "name", "subordinate_units", "conference", - "conference_place", "conference_numbering", "conference_date", + "conference_place", "start_date", "end_date", "alternative_names", From c8de3b780b46e74318a6b4a523dda955a2932585 Mon Sep 17 00:00:00 2001 From: Bertrand Zuchuat Date: Mon, 7 Aug 2023 15:36:54 +0200 Subject: [PATCH 11/14] document: add 3 new type of subjects * Adds place, temporal and work type on subject filters. Co-Authored-by: Bertrand Zuchuat --- .../document_subjects_entity_link-v0.0.1.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/rero_ils/modules/documents/jsonschemas/documents/document_subjects_entity_link-v0.0.1.json b/rero_ils/modules/documents/jsonschemas/documents/document_subjects_entity_link-v0.0.1.json index 8b906ada4d..7454bddfc7 100644 --- a/rero_ils/modules/documents/jsonschemas/documents/document_subjects_entity_link-v0.0.1.json +++ b/rero_ils/modules/documents/jsonschemas/documents/document_subjects_entity_link-v0.0.1.json @@ -31,6 +31,18 @@ { "label": "bf:Organisation", "value": "bf:Organisation" + }, + { + "label": "Place", + "value": "bf:Place" + }, + { + "label": "Temporal", + "value": "bf:Temporal" + }, + { + "label": "Work", + "value": "bf:Work" } ] }, From f0c9b5471b20987c1d5784302a17ae99e600776a Mon Sep 17 00:00:00 2001 From: PascalRepond Date: Wed, 16 Aug 2023 09:16:48 +0200 Subject: [PATCH 12/14] entities: adapt labels and descriptions * Adapts entites labels and descriptions to account for local VS textual entities. Co-Authored-by: Pascal Repond --- .../documents/document_contribution-v0.0.1.json | 10 +++++----- .../documents/document_contribution_local-v0.0.1.json | 2 +- .../documents/document_entity_local-v0.0.1.json | 6 +++--- .../documents/document_genre_form-v0.0.1.json | 8 ++++---- .../documents/document_genre_form_local-v0.0.1.json | 4 ++-- .../documents/document_subjects-v0.0.1.json | 8 ++++---- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/rero_ils/modules/documents/jsonschemas/documents/document_contribution-v0.0.1.json b/rero_ils/modules/documents/jsonschemas/documents/document_contribution-v0.0.1.json index f1176be060..b881c83951 100644 --- a/rero_ils/modules/documents/jsonschemas/documents/document_contribution-v0.0.1.json +++ b/rero_ils/modules/documents/jsonschemas/documents/document_contribution-v0.0.1.json @@ -1,18 +1,18 @@ { "contribution": { "title": "Contribution", - "description": "Relationship to an agent associated with the document", + "description": "Relationship with an agent contributing to the document.", "type": "array", "minItems": 1, "items": { "title": "Contribution", - "description": "Person, family or corporate body (including conferences). Always create a link to IdRef or GND, if possible.", + "description": "Person, family or corporate body (including conferences) and their contributing role for the document.", "type": "object", "oneOf": [ { "type": "object", "title": "Link to an entity", - "description": "Person, family or corporate body (including conferences). Always create a link to IdRef or GND, if possible.", + "description": "Link to a remote or local entity of type person, family or corporate body (including conferences).", "additionalProperties": false, "propertiesOrder": [ "entity", @@ -33,8 +33,8 @@ }, { "type": "object", - "title": "Entity (local)", - "description": "Person, family or corporate body (including conferences). Always create a link to IdRef or GND, if possible.", + "title": "Entity (textual)", + "description": "Textual description of a person, family or corporate body (including conferences). Usually used temporarily until a link to an entity can be established.", "additionalProperties": false, "propertiesOrder": [ "entity", diff --git a/rero_ils/modules/documents/jsonschemas/documents/document_contribution_local-v0.0.1.json b/rero_ils/modules/documents/jsonschemas/documents/document_contribution_local-v0.0.1.json index 2d2e38667c..5690c52b30 100644 --- a/rero_ils/modules/documents/jsonschemas/documents/document_contribution_local-v0.0.1.json +++ b/rero_ils/modules/documents/jsonschemas/documents/document_contribution_local-v0.0.1.json @@ -1,5 +1,5 @@ { - "title": "Agent (local)", + "title": "Agent (textual)", "type": "object", "additionalProperties": false, "propertiesOrder": [ diff --git a/rero_ils/modules/documents/jsonschemas/documents/document_entity_local-v0.0.1.json b/rero_ils/modules/documents/jsonschemas/documents/document_entity_local-v0.0.1.json index f5b3b385c4..56bd01fef0 100644 --- a/rero_ils/modules/documents/jsonschemas/documents/document_entity_local-v0.0.1.json +++ b/rero_ils/modules/documents/jsonschemas/documents/document_entity_local-v0.0.1.json @@ -1,5 +1,5 @@ { - "title": "Entity (local)", + "title": "Entity (textual)", "type": "object", "additionalProperties": false, "propertiesOrder": [ @@ -89,8 +89,8 @@ "uniqueItems": true, "items": { "type": "object", - "title": "Entity (local)", - "description": "Topic (including genre/form), place, temporal, person, family or corporate body (including conferences). Always create a link to IdRef or GND, if possible.", + "title": "Entity (textual)", + "description": "Textual description of a topic (including genre and forms), person, organisation (including conferences), place, temporal or work.", "additionalProperties": false, "propertiesOrder": [ "entity" diff --git a/rero_ils/modules/documents/jsonschemas/documents/document_genre_form-v0.0.1.json b/rero_ils/modules/documents/jsonschemas/documents/document_genre_form-v0.0.1.json index 69d08e94c4..9b2d5668bc 100644 --- a/rero_ils/modules/documents/jsonschemas/documents/document_genre_form-v0.0.1.json +++ b/rero_ils/modules/documents/jsonschemas/documents/document_genre_form-v0.0.1.json @@ -7,12 +7,12 @@ "items": { "title": "Genre, form", "type": "object", - "description": "Genre or form of the document. Always create a link to IdRef or GND, if possible.", + "description": "Genre or form of the document.", "oneOf": [ { "title": "Link to an entity", "type": "object", - "description": "Genre or form of the document. Always create a link to IdRef or GND, if possible.", + "description": "Link to a remote or local entity of type topic accepted as a 'genre, form'.", "additionalProperties": false, "propertiesOrder": [ "entity" @@ -27,8 +27,8 @@ } }, { - "title": "Entity (local)", - "description": "Genre or form of the document. Always create a link to IdRef or GND, if possible.", + "title": "Entity (textual)", + "description": "Textual description of a genre or form. Usually used temporarily until a link to an entity can be established.", "type": "object", "additionalProperties": false, "propertiesOrder": [ diff --git a/rero_ils/modules/documents/jsonschemas/documents/document_genre_form_local-v0.0.1.json b/rero_ils/modules/documents/jsonschemas/documents/document_genre_form_local-v0.0.1.json index 7ca4995025..40dcb6cbc8 100644 --- a/rero_ils/modules/documents/jsonschemas/documents/document_genre_form_local-v0.0.1.json +++ b/rero_ils/modules/documents/jsonschemas/documents/document_genre_form_local-v0.0.1.json @@ -1,5 +1,5 @@ { - "title": "Genre, form (local)", + "title": "Genre, form (textual)", "type": "object", "additionalProperties": false, "propertiesOrder": [ @@ -39,7 +39,7 @@ }, "source": { "title": "Source", - "description": "Source of the subject.", + "description": "Source of the genre or form.", "type": "string", "minLength": 3 }, diff --git a/rero_ils/modules/documents/jsonschemas/documents/document_subjects-v0.0.1.json b/rero_ils/modules/documents/jsonschemas/documents/document_subjects-v0.0.1.json index efd7052527..90cd8efdf1 100644 --- a/rero_ils/modules/documents/jsonschemas/documents/document_subjects-v0.0.1.json +++ b/rero_ils/modules/documents/jsonschemas/documents/document_subjects-v0.0.1.json @@ -7,12 +7,12 @@ "items": { "type": "object", "title": "Subject", - "description": "Topic (including genre/form), place, temporal, person, family or corporate body (including conferences). Always create a link to IdRef or GND, if possible.", + "description": "Topic, person, organisation (including conferences), place, temporal or work that is a subject of the document.", "oneOf": [ { "title": "Link to an entity", "type": "object", - "description": "Topic (including genre/form), place, temporal, person, family or corporate body (including conferences). Always create a link to IdRef or GND, if possible.", + "description": "Link to a remote or local entity of type topic, person, organisation (including conferences), place, temporal or work.", "additionalProperties": false, "propertiesOrder": [ "entity" @@ -27,9 +27,9 @@ } }, { - "title": "Entity (local)", + "title": "Entity (textual)", "type": "object", - "description": "Topic (including genre/form), place, temporal, person, family or corporate body (including conferences). Always create a link to IdRef or GND, if possible.", + "description": "Textual description of a topic, person, organisation (including conferences), place, temporal or work. Usually used temporarily until a link to an entity can be established.", "additionalProperties": false, "propertiesOrder": [ "entity" From fc5b1cef61da6b2a5ea693dc4b24780ddbfa9fed Mon Sep 17 00:00:00 2001 From: Bertrand Zuchuat Date: Mon, 7 Aug 2023 14:37:23 +0200 Subject: [PATCH 13/14] editor: fix length to maxItems on simple form Co-Authored-by: Bertrand Zuchuat --- .../local_entities/local_entity_organisation-v0.0.1.json | 4 ++-- .../local_entities/local_entity_person-v0.0.1.json | 2 +- .../jsonschemas/local_entities/local_entity_place-v0.0.1.json | 2 +- .../local_entities/local_entity_temporal-v0.0.1.json | 2 +- .../jsonschemas/local_entities/local_entity_topic-v0.0.1.json | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_organisation-v0.0.1.json b/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_organisation-v0.0.1.json index fe0b5449ee..0a9f437161 100644 --- a/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_organisation-v0.0.1.json +++ b/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_organisation-v0.0.1.json @@ -79,7 +79,7 @@ "subordinate_units": { "title": "Subordinate units", "type": "array", - "minItems": 1, + "minItems": 0, "items": { "title": "Subordinate unit", "type": "string", @@ -142,7 +142,7 @@ "alternative_names": { "title": "Alternative names", "type": "array", - "minItems": 1, + "minItems": 0, "items": { "title": "Alternative name", "type": "string", diff --git a/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_person-v0.0.1.json b/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_person-v0.0.1.json index 644e4b99e7..9574490ae9 100644 --- a/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_person-v0.0.1.json +++ b/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_person-v0.0.1.json @@ -118,7 +118,7 @@ "alternative_names": { "title": "Alternative names", "type": "array", - "minItems": 1, + "minItems": 0, "items": { "title": "Alternative name", "type": "string", diff --git a/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_place-v0.0.1.json b/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_place-v0.0.1.json index 9423eb3845..a48d0360cb 100644 --- a/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_place-v0.0.1.json +++ b/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_place-v0.0.1.json @@ -71,7 +71,7 @@ "alternative_names": { "title": "Alternative names", "type": "array", - "minItems": 1, + "minItems": 0, "items": { "title": "Alternative name", "type": "string", diff --git a/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_temporal-v0.0.1.json b/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_temporal-v0.0.1.json index 4b999870b1..caa6fed138 100644 --- a/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_temporal-v0.0.1.json +++ b/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_temporal-v0.0.1.json @@ -71,7 +71,7 @@ "alternative_names": { "title": "Alternative names", "type": "array", - "minItems": 1, + "minItems": 0, "items": { "title": "Alternative name", "type": "string", diff --git a/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_topic-v0.0.1.json b/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_topic-v0.0.1.json index 233ee03771..434b75a28f 100644 --- a/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_topic-v0.0.1.json +++ b/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_topic-v0.0.1.json @@ -78,7 +78,7 @@ "alternative_names": { "title": "Alternative names", "type": "array", - "minItems": 1, + "minItems": 0, "items": { "title": "Alternative name", "type": "string", From d77acf5d5b448fd7bfe10e842e98a0fff9e590a8 Mon Sep 17 00:00:00 2001 From: Bertrand Zuchuat Date: Mon, 10 Jul 2023 12:06:44 +0200 Subject: [PATCH 14/14] entities: public view for local and remote * Changes english date format to en_GB (DD/MM/YYYY). * Fixes `reasons_not_to_delete` and `get_links_to_me` methods for `Entity` resource. * Adds `alternative_names` as alias for autocomplete search. Co-Authored-by: Bertrand Zuchuat Co-Authored-by: Renaud Michotte --- pyproject.toml | 2 +- rero_ils/config.py | 15 +- rero_ils/filter.py | 21 +++ rero_ils/modules/documents/api.py | 25 +++ .../dojson/contrib/jsontomarc21/model.py | 4 +- .../modules/documents/dumpers/replace_refs.py | 3 +- .../extensions/provision_activities.py | 2 +- .../modules/documents/serializers/base.py | 2 +- .../rero_ils/detailed_view_documents.html | 27 ++- rero_ils/modules/documents/views.py | 59 ++++--- rero_ils/modules/entities/api.py | 120 +++++++++----- .../modules/entities/local_entities/api.py | 22 --- .../entities/local_entities/indexer.py | 7 +- .../local_entity_organisation-v0.0.1.json | 11 +- .../local_entities/local_entity-v0.0.1.json | 10 +- .../entities/local_entities/permissions.py | 4 +- .../modules/entities/local_entities/proxy.py | 1 - .../entities/remote_entities/__init__.py | 5 +- .../modules/entities/remote_entities/api.py | 5 +- .../modules/entities/remote_entities/sync.py | 5 +- .../templates/rero_ils/_entity_by_source.html | 55 ------- .../templates/rero_ils/_entity_unified.html | 45 ----- .../rero_ils/detailed_view_entity.html | 82 --------- .../modules/entities/remote_entities/views.py | 155 +----------------- .../modules/entities/serializers/__init__.py | 4 +- .../rero_ils/_local_organisation.html | 28 ++++ .../templates/rero_ils/_local_person.html | 26 +++ .../templates/rero_ils/_local_place.html | 21 +++ .../templates/rero_ils/_local_temporal.html | 21 +++ .../templates/rero_ils/_local_topic.html | 22 +++ .../templates/rero_ils/_local_work.html | 21 +++ .../rero_ils/_remote_organisation.html} | 26 ++- .../templates/rero_ils/_remote_person.html | 44 +++++ .../templates/rero_ils/_remote_topic.html | 71 ++++++++ .../templates/rero_ils/_search_link.html | 24 +++ .../templates/rero_ils/entity_local.html | 64 ++++++++ .../templates/rero_ils/entity_remote.html | 64 ++++++++ .../templates/rero_ils/macros/entity.html | 28 +--- rero_ils/modules/entities/views.py | 148 +++++++++++++++++ rero_ils/modules/ext.py | 3 +- rero_ils/modules/monitoring/api.py | 2 +- .../rero_ils/address_block/eng.tpl.txt | 2 +- .../rero_ils/address_block/fre.tpl.txt | 2 +- .../rero_ils/address_block/ger.tpl.txt | 2 +- .../rero_ils/address_block/ita.tpl.txt | 2 +- .../test_local_entities_rest.py | 9 +- .../test_remote_entities_rest.py | 2 +- tests/fixtures/mef.py | 6 +- tests/fixtures/metadata.py | 4 +- tests/ui/documents/test_documents_api.py | 3 +- tests/ui/documents/test_documents_filter.py | 102 ++++++++---- .../local_entities/test_local_entities_api.py | 1 + .../test_remote_entities_api.py | 4 + .../test_remote_entities_filter.py | 108 ------------ .../test_remote_entities_ui.py | 29 ++-- tests/ui/entities/test_entities_ui.py | 137 ++++++++++++++++ tests/ui/test_filters.py | 14 +- tests/unit/conftest.py | 4 +- 58 files changed, 1055 insertions(+), 680 deletions(-) delete mode 100644 rero_ils/modules/entities/remote_entities/templates/rero_ils/_entity_by_source.html delete mode 100644 rero_ils/modules/entities/remote_entities/templates/rero_ils/_entity_unified.html delete mode 100644 rero_ils/modules/entities/remote_entities/templates/rero_ils/detailed_view_entity.html create mode 100644 rero_ils/modules/entities/templates/rero_ils/_local_organisation.html create mode 100644 rero_ils/modules/entities/templates/rero_ils/_local_person.html create mode 100644 rero_ils/modules/entities/templates/rero_ils/_local_place.html create mode 100644 rero_ils/modules/entities/templates/rero_ils/_local_temporal.html create mode 100644 rero_ils/modules/entities/templates/rero_ils/_local_topic.html create mode 100644 rero_ils/modules/entities/templates/rero_ils/_local_work.html rename rero_ils/modules/entities/{remote_entities/templates/rero_ils/_entity_by_source_data.html => templates/rero_ils/_remote_organisation.html} (62%) create mode 100644 rero_ils/modules/entities/templates/rero_ils/_remote_person.html create mode 100644 rero_ils/modules/entities/templates/rero_ils/_remote_topic.html create mode 100644 rero_ils/modules/entities/templates/rero_ils/_search_link.html create mode 100644 rero_ils/modules/entities/templates/rero_ils/entity_local.html create mode 100644 rero_ils/modules/entities/templates/rero_ils/entity_remote.html rename rero_ils/modules/entities/{remote_entities => }/templates/rero_ils/macros/entity.html (72%) create mode 100644 rero_ils/modules/entities/views.py delete mode 100644 tests/ui/entities/remote_entities/test_remote_entities_filter.py create mode 100644 tests/ui/entities/test_entities_ui.py diff --git a/pyproject.toml b/pyproject.toml index d27487cf6d..71209ca4db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -186,7 +186,7 @@ rero-ils = "rero_ils.modules.ext:REROILSAPP" [tool.poetry.plugins."invenio_base.blueprints"] circ_policies = "rero_ils.modules.circ_policies.views:blueprint" collections = "rero_ils.modules.collections.views:blueprint" -remote_entities = "rero_ils.modules.entities.remote_entities.views:blueprint" +entities = "rero_ils.modules.entities.views:blueprint" documents = "rero_ils.modules.documents.views:blueprint" holdings = "rero_ils.modules.holdings.views:blueprint" ill_requests = "rero_ils.modules.ill_requests.views:blueprint" diff --git a/rero_ils/config.py b/rero_ils/config.py index f8dc528b39..6fe9bb5053 100644 --- a/rero_ils/config.py +++ b/rero_ils/config.py @@ -40,7 +40,7 @@ ItemOnLoanToItemReturned, PendingToItemAtDesk, \ PendingToItemInTransitPickup, ToCancelled, ToItemOnLoan from invenio_records_rest.facets import range_filter, terms_filter -from invenio_records_rest.utils import deny_all, allow_all +from invenio_records_rest.utils import allow_all, deny_all from rero_ils.modules.acquisition.acq_accounts.api import AcqAccount from rero_ils.modules.acquisition.acq_accounts.permissions import \ @@ -63,6 +63,12 @@ from rero_ils.modules.acquisition.budgets.api import Budget from rero_ils.modules.acquisition.budgets.permissions import \ BudgetPermissionPolicy +from rero_ils.modules.entities.local_entities.api import LocalEntity +from rero_ils.modules.entities.local_entities.permissions import \ + LocalEntityPermissionPolicy +from rero_ils.modules.entities.remote_entities.api import RemoteEntity +from rero_ils.modules.entities.remote_entities.permissions import \ + RemoteEntityPermissionPolicy from .modules.circ_policies.api import CircPolicy from .modules.circ_policies.permissions import \ @@ -73,9 +79,6 @@ from .modules.documents.permissions import DocumentPermissionPolicy from .modules.documents.query import acquisition_filter, \ nested_identified_filter -from rero_ils.modules.entities.remote_entities.api import RemoteEntity -from rero_ils.modules.entities.remote_entities.permissions import \ - RemoteEntityPermissionPolicy from .modules.holdings.api import Holding from .modules.holdings.models import HoldingCirculationAction from .modules.holdings.permissions import HoldingsPermissionPolicy @@ -98,9 +101,6 @@ get_extension_params, is_item_available_for_checkout, \ loan_build_document_ref, loan_build_item_ref, loan_build_patron_ref, \ validate_item_pickup_transaction_locations, validate_loan_duration -from rero_ils.modules.entities.local_entities.api import LocalEntity -from rero_ils.modules.entities.local_entities.permissions import \ - LocalEntityPermissionPolicy from .modules.local_fields.api import LocalField from .modules.local_fields.permissions import LocalFieldPermissionPolicy from .modules.locations.api import Location @@ -3060,6 +3060,7 @@ def _(x): #: Entities RERO_ILS_AGENTS_SOURCES = ['idref', 'gnd', 'rero'] +RERO_ILS_AGENTS_SOURCES_EXCLUDE_LINK = ['rero'] RERO_ILS_AGENTS_LABEL_ORDER = { 'fallback': 'fr', 'fr': ['idref', 'rero', 'gnd'], diff --git a/rero_ils/filter.py b/rero_ils/filter.py index 6c461061f0..162b47fbe3 100644 --- a/rero_ils/filter.py +++ b/rero_ils/filter.py @@ -25,6 +25,7 @@ import dateparser from babel.dates import format_date, format_datetime, format_time from flask import current_app, render_template +from flask_babelex import gettext as _ from invenio_i18n.ext import current_i18n from jinja2 import TemplateNotFound from markupsafe import Markup @@ -102,6 +103,10 @@ def format_date_filter( if not locale: locale = current_i18n.locale.language + # Date formatting in GB English (DD/MM/YYYY) + if locale == 'en': + locale += '_GB' + if timezone: tzinfo = timezone else: @@ -184,3 +189,19 @@ def message_filter(key): :return: none or a json (check structure into the class Message). """ return Message.get(key) + + +def translate(data, prefix='', separator=', '): + """Translate data. + + :param data: the data to translate + :param prefix: A prefix as a character string + :param separator: A character string separator. + :return: The translated string + """ + if data: + if isinstance(data, list): + translated = [_(f'{prefix}{item}') for item in data] + return separator.join(translated) + elif isinstance(data, str): + return _(f'{prefix}{data}') diff --git a/rero_ils/modules/documents/api.py b/rero_ils/modules/documents/api.py index 170b8fcaa1..96bd65e5b5 100644 --- a/rero_ils/modules/documents/api.py +++ b/rero_ils/modules/documents/api.py @@ -73,6 +73,30 @@ class Meta: default_filter = None + def by_entity(self, entity, subjects=True, imported_subjects=True, + genre_forms=True): + """Build a search to get hits related to an entity. + + :param entity: the entity record to search. + :param subjects: search on `subject` field. + :param imported_subjects: search on `imported_subject` field. + :param genre_forms: search on `genre_forms` field. + :returns: An ElasticSearch query to get hits related the entity. + :rtype: `elasticsearch_dsl.Search` + """ + field = f'contribution.entity.pids.{entity.resource_type}' + filters = Q('term', **{field: entity.pid}) + if subjects: + field = f'subjects.entity.pids.{entity.resource_type}' + filters |= Q('term', **{field: entity.pid}) + if imported_subjects: + field = f'subjects_imported.pids.{entity.resource_type}' + filters |= Q('term', **{field: entity.pid}) + if genre_forms: + field = f'genreForm.entity.pids.{entity.resource_type}' + filters |= Q('term', **{field: entity.pid}) + return self.filter(filters) + class Document(IlsRecord): """Document class.""" @@ -244,6 +268,7 @@ def index_contributions(self, bulk=False): """Index all attached contributions.""" from rero_ils.modules.entities.remote_entities.api import \ RemoteEntitiesIndexer, RemoteEntity + from ..tasks import process_bulk_queue contributions_ids = [] for contribution in self.get('contribution', []): diff --git a/rero_ils/modules/documents/dojson/contrib/jsontomarc21/model.py b/rero_ils/modules/documents/dojson/contrib/jsontomarc21/model.py index 8425537921..aa7ab42e03 100644 --- a/rero_ils/modules/documents/dojson/contrib/jsontomarc21/model.py +++ b/rero_ils/modules/documents/dojson/contrib/jsontomarc21/model.py @@ -26,8 +26,8 @@ from rero_ils.modules.documents.utils import display_alternate_graphic_first from rero_ils.modules.documents.views import create_title_responsibilites from rero_ils.modules.entities.models import EntityType -from rero_ils.modules.entities.remote_entities.api import RemoteEntity, \ - RemoteEntitiesSearch +from rero_ils.modules.entities.remote_entities.api import \ + RemoteEntitiesSearch, RemoteEntity from rero_ils.modules.holdings.api import Holding, HoldingsSearch from rero_ils.modules.items.api import Item, ItemsSearch from rero_ils.modules.libraries.api import Library diff --git a/rero_ils/modules/documents/dumpers/replace_refs.py b/rero_ils/modules/documents/dumpers/replace_refs.py index 609192974d..6abc939c96 100644 --- a/rero_ils/modules/documents/dumpers/replace_refs.py +++ b/rero_ils/modules/documents/dumpers/replace_refs.py @@ -20,8 +20,7 @@ from invenio_records.dumpers import Dumper from rero_ils.modules.commons.exceptions import RecordNotFound -from rero_ils.modules.entities.dumpers import \ - document_dumper +from rero_ils.modules.entities.dumpers import document_dumper from rero_ils.modules.entities.remote_entities.utils import \ extract_data_from_mef_uri from rero_ils.modules.utils import extracted_data_from_ref diff --git a/rero_ils/modules/documents/extensions/provision_activities.py b/rero_ils/modules/documents/extensions/provision_activities.py index ff8e6ab616..598f8cd410 100644 --- a/rero_ils/modules/documents/extensions/provision_activities.py +++ b/rero_ils/modules/documents/extensions/provision_activities.py @@ -22,9 +22,9 @@ from invenio_records.extensions import RecordExtension from rero_ils.dojson.utils import remove_trailing_punctuation +from rero_ils.modules.entities.models import EntityType from ..utils import display_alternate_graphic_first -from rero_ils.modules.entities.models import EntityType class ProvisionActivitiesExtension(RecordExtension): diff --git a/rero_ils/modules/documents/serializers/base.py b/rero_ils/modules/documents/serializers/base.py index d4cebcd233..7b510d2963 100644 --- a/rero_ils/modules/documents/serializers/base.py +++ b/rero_ils/modules/documents/serializers/base.py @@ -24,10 +24,10 @@ from rero_ils.modules.commons.identifiers import IdentifierFactory, \ IdentifierStatus, IdentifierType +from rero_ils.modules.entities.models import EntityType from rero_ils.modules.utils import get_base_url from ..api import DocumentsSearch -from rero_ils.modules.entities.models import EntityType CREATOR_ROLES = [ 'aut', 'cmp', 'cre', 'dub', 'pht', 'ape', 'aqt', 'arc', 'art', 'aus', diff --git a/rero_ils/modules/documents/templates/rero_ils/detailed_view_documents.html b/rero_ils/modules/documents/templates/rero_ils/detailed_view_documents.html index 83c1c6bb09..dad7691285 100644 --- a/rero_ils/modules/documents/templates/rero_ils/detailed_view_documents.html +++ b/rero_ils/modules/documents/templates/rero_ils/detailed_view_documents.html @@ -2,6 +2,7 @@ RERO ILS Copyright (C) 2019-2023 RERO + Copyright (C) 2019-2023 UCLouvain This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by @@ -218,7 +219,18 @@

{% for subject in record.subjects %} {% if 'entity' in subject %} - {{ subject.entity | doc_entity_label(language=current_i18n.language) }} + {% set type, value, label = subject.entity | doc_entity_label(language=current_i18n.language) %} + {% if 'textual' == type %} + {% set query = 'subjects.entity.authorized_access_point_' ~ current_i18n.language ~ ':"' ~ value ~ '"' %} + {% elif type in ['local', 'remote'] %} + {% set query = 'subjects.entity.pids.' ~ type ~ ':' ~ value %} + {% endif %} + + {% if query %} + {{ label }} + {% else %} + {{ label }} + {% endif %} {% endif %} {% endfor %} @@ -230,16 +242,25 @@

{% for genreForm in record.genreForm %} {% if 'entity' in genreForm %} + {% set type, value, label = genreForm.entity | doc_entity_label(language=current_i18n.language) %} + {% if 'textual' == type %} + {% set query = 'genreForm.entity.authorized_access_point_' ~ current_i18n.language ~ ':"' ~ value ~ '"' %} + {% elif type in ['local', 'remote'] %} + {% set query = 'genreForm.entity.pids.' ~ type ~ ':' ~ value %} + {% endif %} - {{ genreForm.entity | doc_entity_label(language=current_i18n.language) }} + {% if query %} + {{ label }} + {% else %} + {{ label }} + {% endif %} {% endif %} {% endfor %}
{% endif %} - {% if linked_documents_count and linked_documents_count > 0 %} {text}' else: default_key = 'authorized_access_point' - localized_key = f'{default_key}_{language}' - label = contrib['entity'].get(localized_key) or \ + localized_key = f'authorized_access_point_{language}' + text = contrib['entity'].get(localized_key) or \ contrib['entity'].get(default_key) + args = { + 'viewcode': viewcode, + 'recordType': 'documents', + 'q': f'contribution.entity.{localized_key}:"{text}"', + 'simple': 0 + } + url = url_for('rero_ils.search', **args) + label = f'{text}' if with_roles: if roles := [_(role) for role in contrib.get('role', [])]: @@ -288,7 +295,7 @@ def contribution_format(contributions, language, viewcode, with_roles=False): label += f' ({roles_str})' output.append(label) - return ' ; '.join(output) + return ' ; '.join(output) @blueprint.app_template_filter() @@ -301,20 +308,28 @@ def doc_entity_label(entity, language=None, part_separator=' - ') -> str: :returns: the best possible label to display. """ parts = [] - if 'pid' in entity: - entity = RemoteEntity.get_record_by_pid(entity['pid']) - parts.append(entity.get_authorized_access_point(language=language)) + if '$ref' in entity: + # Local or remote entity + if entity := Entity.get_record_by_ref(entity['$ref']): + entity_type = entity.resource_type + value = entity.pid + parts.append(entity.get_authorized_access_point(language=language)) else: + # Textual entity + entity_type = 'textual' default_key = 'authorized_access_point' localized_key = f'{default_key}_{language}' - parts.append(entity.get(localized_key) or entity.get(default_key)) + value = entity.get(localized_key) or entity.get(default_key) + parts.append(value) + # Subdivisions (only for textual entity) for subdivision in entity.get('subdivisions', []): if sub_entity := subdivision.get('entity'): - parts.append( - doc_entity_label(sub_entity, language, part_separator)) + _, _, label = doc_entity_label( + sub_entity, language, part_separator) + parts.append(label) - return part_separator.join(filter(None, parts)) + return entity_type, value, part_separator.join(filter(None, parts)) @blueprint.app_template_filter() diff --git a/rero_ils/modules/entities/api.py b/rero_ils/modules/entities/api.py index 906763202a..a8b6498a99 100644 --- a/rero_ils/modules/entities/api.py +++ b/rero_ils/modules/entities/api.py @@ -20,16 +20,42 @@ from abc import ABC, abstractmethod -from elasticsearch_dsl import Q, A +from elasticsearch_dsl import A from flask import current_app -from rero_ils.modules.api import IlsRecord +from rero_ils.modules.api import IlsRecord, IlsRecordsSearch from rero_ils.modules.documents.api import DocumentsSearch +from rero_ils.modules.entities.remote_entities.utils import \ + extract_data_from_mef_uri +from rero_ils.modules.utils import extracted_data_from_ref, sorted_pids + + +class EntitiesSearch(IlsRecordsSearch): + """Entities search class.""" + + class Meta: + """Meta class.""" + + index = 'entities' + doc_types = None + fields = ('*', ) + facets = {} + + default_filter = None class Entity(IlsRecord, ABC): """Entity class.""" + @classmethod + def get_record_by_ref(cls, ref): + """.""" + from .remote_entities.api import RemoteEntity + if entity := extracted_data_from_ref(ref, 'record'): + return entity + _, _type, _pid = extract_data_from_mef_uri(ref) + return RemoteEntity.get_entity(_type, _pid) + @abstractmethod def get_authorized_access_point(self, language): """Get localized authorized_access_point. @@ -39,6 +65,30 @@ def get_authorized_access_point(self, language): """ raise NotImplementedError + @abstractmethod + def get_links_to_me(self, get_pids=False): + """Get links to other resources. + + :param get_pids: related resource pids are included into response ; + otherwise the count of related resources are specified. + :returns: list of related resource to this entity. + :rtype: dict. + """ + document_query = DocumentsSearch().by_entity(self) + documents = sorted_pids(document_query) if get_pids \ + else document_query.count() + links = { + 'documents': documents + } + return {k: v for k, v in links.items() if v} + + def reasons_not_to_delete(self): + """Get reasons not to delete record.""" + cannot_delete = {} + if links := self.get_links_to_me(): + cannot_delete['links'] = links + return cannot_delete + @property @abstractmethod def resource_type(self): @@ -47,8 +97,8 @@ def resource_type(self): @property def organisation_pids(self): - """Get organisations pids.""" - search = self._search_documents() + """Get organisation pids related with this entity.""" + search = DocumentsSearch().by_entity(self)[:0] agg = A( 'terms', field='holdings.organisation.organisation_pid', @@ -64,39 +114,25 @@ def organisation_pids(self): for result in results.aggregations.organisation.buckets }) - def _search_documents( + def documents_pids( self, with_subjects=True, with_subjects_imported=True, with_genre_forms=True ): - """Get ES query to search documents containing this entity. + """Get documents pids related to this entity. - :param with_subjects: search also on `subjects` ? - :param with_subjects_imported: search also on `subject_imported` ? - :param with_genre_forms: search also on `genreForm` ? + :param with_subjects: is the document `subject` field must be analyzed. + :param with_subjects_imported: is the document `subject_imported` field + must be analyzed. + :param with_genre_forms: is the document `genre_forms` field must be + analyzed. + :returns: document pids related to this entity. + :rtype: list """ - contribution_key = f'contribution.entity.pids.{self.resource_type}' - filters = Q('term', **{contribution_key: self.pid}) - if with_subjects: - search_field = f'subjects.entity.pids.{self.resource_type}' - filters |= Q('term', **{search_field: self.pid}) - if with_subjects_imported: - search_field = f'subjects_imported.pids.{self.resource_type}' - filters |= Q('term', **{search_field: self.pid}) - if with_genre_forms: - search_field = f'genreForm.entity.pids.{self.resource_type}' - filters |= Q('term', **{search_field: self.pid}) - - return DocumentsSearch().filter(filters) - - def documents_pids( - self, with_subjects=True, with_subjects_imported=True, - with_genre_forms=True - ): - """Get documents pids.""" - search = self._search_documents( - with_subjects=with_subjects, - with_subjects_imported=with_subjects_imported, - with_genre_forms=with_genre_forms + search = DocumentsSearch().by_entity( + self, + subjects=with_subjects, + imported_subjects=with_subjects_imported, + genre_forms=with_genre_forms ).source('pid') return [hit.pid for hit in search.scan()] @@ -104,10 +140,20 @@ def documents_ids( self, with_subjects=True, with_subjects_imported=True, with_genre_forms=True ): - """Get documents ids.""" - search = self._search_documents( - with_subjects=with_subjects, - with_subjects_imported=with_subjects_imported, - with_genre_forms=with_genre_forms + """Get document ID's/UUID related to this entity. + + :param with_subjects: is the document `subject` field must be analyzed. + :param with_subjects_imported: is the document `subject_imported` field + must be analyzed. + :param with_genre_forms: is the document `genre_forms` field must be + analyzed. + :returns: document ID's/UUID related to this entity. + :rtype: list + """ + search = DocumentsSearch().by_entity( + self, + subjects=with_subjects, + imported_subjects=with_subjects_imported, + genre_forms=with_genre_forms ).source(False) return [hit.meta.id for hit in search.scan()] diff --git a/rero_ils/modules/entities/local_entities/api.py b/rero_ils/modules/entities/local_entities/api.py index 955b1a33f4..6d847d6e64 100644 --- a/rero_ils/modules/entities/local_entities/api.py +++ b/rero_ils/modules/entities/local_entities/api.py @@ -21,7 +21,6 @@ from functools import partial from rero_ils.modules.api import IlsRecordsSearch -from rero_ils.modules.utils import sorted_pids from rero_ils.modules.fetchers import id_fetcher from rero_ils.modules.minters import id_minter from rero_ils.modules.operation_logs.extensions import \ @@ -113,24 +112,3 @@ def resolve(self): # The links will be stored as a `$ref` and `replace_refs_dumper` # will be used. return self.dumps(replace_refs_dumper) - - def get_links_to_me(self, get_pids=False): - """Record links. - - :param get_pids: if True list of linked pids - if False count of linked records - """ - document_query = self._search_documents() - documents = sorted_pids(document_query) if get_pids \ - else document_query.count() - links = { - 'documents': documents - } - return {k: v for k, v in links.items() if v} - - def reasons_not_to_delete(self): - """Get reasons not to delete record.""" - cannot_delete = {} - if links := self.get_links_to_me(): - cannot_delete['links'] = links - return cannot_delete diff --git a/rero_ils/modules/entities/local_entities/indexer.py b/rero_ils/modules/entities/local_entities/indexer.py index fe0cc33b6b..0bdfc97ca1 100644 --- a/rero_ils/modules/entities/local_entities/indexer.py +++ b/rero_ils/modules/entities/local_entities/indexer.py @@ -17,14 +17,15 @@ # along with this program. If not, see . """Local entity indexer APIs.""" -from celery import shared_task from datetime import datetime +from celery import shared_task from flask import current_app -from rero_ils.modules.utils import get_record_class_by_resource, \ - get_indexer_class_by_resource from rero_ils.modules.api import IlsRecordsIndexer, ReferencedRecordsIndexer +from rero_ils.modules.utils import get_indexer_class_by_resource, \ + get_record_class_by_resource + from .api import LocalEntity from ..dumpers import indexer_dumper diff --git a/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_organisation-v0.0.1.json b/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_organisation-v0.0.1.json index 0a9f437161..8f24ac20ae 100644 --- a/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_organisation-v0.0.1.json +++ b/rero_ils/modules/entities/local_entities/jsonschemas/local_entities/local_entity_organisation-v0.0.1.json @@ -14,10 +14,10 @@ "type", "name", "subordinate_units", - "conference", "conference_numbering", "conference_date", "conference_place", + "conference", "start_date", "end_date", "alternative_names", @@ -116,8 +116,7 @@ "type": "string", "minLength": 1, "form": { - "placeholder": "Example: Paris", - "hideExpression": "!field?.parent?.model || !field.parent.model.conference || field.parent.model.conference == false" + "placeholder": "Example: Paris" } }, "conference_numbering": { @@ -125,8 +124,7 @@ "type": "string", "minLength": 1, "form": { - "placeholder": "Example: 23e", - "hideExpression": "!field?.parent?.model || !field.parent.model.conference || field.parent.model.conference == false" + "placeholder": "Example: 23e" } }, "conference_date": { @@ -135,8 +133,7 @@ "type": "string", "minLength": 1, "form": { - "placeholder": "Example: 2022", - "hideExpression": "!field?.parent?.model || !field.parent.model.conference || field.parent.model.conference == false" + "placeholder": "Example: 2022" } }, "alternative_names": { diff --git a/rero_ils/modules/entities/local_entities/mappings/v7/local_entities/local_entity-v0.0.1.json b/rero_ils/modules/entities/local_entities/mappings/v7/local_entities/local_entity-v0.0.1.json index 7c96955e5c..f28776ac0c 100644 --- a/rero_ils/modules/entities/local_entities/mappings/v7/local_entities/local_entity-v0.0.1.json +++ b/rero_ils/modules/entities/local_entities/mappings/v7/local_entities/local_entity-v0.0.1.json @@ -124,10 +124,16 @@ "type": "text" }, "alternative_names": { - "type": "text" + "type": "text", + "copy_to": [ + "autocomplete_name" + ] }, "fuller_form_of_name": { - "type": "text" + "type": "text", + "copy_to": [ + "autocomplete_name" + ] }, "gender": { "type": "keyword" diff --git a/rero_ils/modules/entities/local_entities/permissions.py b/rero_ils/modules/entities/local_entities/permissions.py index 1a10524e94..6af2cd6d83 100644 --- a/rero_ils/modules/entities/local_entities/permissions.py +++ b/rero_ils/modules/entities/local_entities/permissions.py @@ -19,8 +19,8 @@ """Permissions for `Local Entity` records.""" from invenio_access import action_factory -from rero_ils.modules.permissions import RecordPermissionPolicy, \ - AllowedByAction +from rero_ils.modules.permissions import AllowedByAction, \ + RecordPermissionPolicy # Actions to control local entity policies for CRUD operations search_action = action_factory('locent-search') diff --git a/rero_ils/modules/entities/local_entities/proxy.py b/rero_ils/modules/entities/local_entities/proxy.py index 75ea3b18b0..ec93ee5a03 100644 --- a/rero_ils/modules/entities/local_entities/proxy.py +++ b/rero_ils/modules/entities/local_entities/proxy.py @@ -22,7 +22,6 @@ from .api import LocalEntitiesSearch from ..models import EntityType - CATEGORY_FILTERS = { 'agents': Q('terms', type=[EntityType.PERSON, EntityType.ORGANISATION]), 'person': Q('term', type=EntityType.PERSON), diff --git a/rero_ils/modules/entities/remote_entities/__init__.py b/rero_ils/modules/entities/remote_entities/__init__.py index ff7065b145..aba233d747 100644 --- a/rero_ils/modules/entities/remote_entities/__init__.py +++ b/rero_ils/modules/entities/remote_entities/__init__.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- # # RERO ILS -# Copyright (C) 2019-2022 RERO +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -15,4 +16,4 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -"""Mef Contributions Records.""" +"""Remote entities Records.""" diff --git a/rero_ils/modules/entities/remote_entities/api.py b/rero_ils/modules/entities/remote_entities/api.py index 7d69f12eca..76fa5d2303 100644 --- a/rero_ils/modules/entities/remote_entities/api.py +++ b/rero_ils/modules/entities/remote_entities/api.py @@ -32,10 +32,9 @@ from rero_ils.modules.minters import id_minter from rero_ils.modules.providers import Provider -from .models import RemoteEntityIdentifier, RemoteEntityMetadata, \ - EntityUpdateAction +from .models import EntityUpdateAction, RemoteEntityIdentifier, \ + RemoteEntityMetadata from .utils import extract_data_from_mef_uri, get_mef_data_by_type - from ..api import Entity from ..dumpers import indexer_dumper, replace_refs_dumper from ..models import EntityResourceType diff --git a/rero_ils/modules/entities/remote_entities/sync.py b/rero_ils/modules/entities/remote_entities/sync.py index 1733c7fe76..2be3f64dfa 100644 --- a/rero_ils/modules/entities/remote_entities/sync.py +++ b/rero_ils/modules/entities/remote_entities/sync.py @@ -29,11 +29,12 @@ from rero_ils.modules.commons.exceptions import RecordNotFound from rero_ils.modules.documents.api import Document -from rero_ils.modules.entities.remote_entities.api import \ - RemoteEntitiesSearch, RemoteEntity from rero_ils.modules.utils import get_mef_url, get_timestamp, \ requests_retry_session, set_timestamp +from .api import RemoteEntitiesSearch, RemoteEntity +from ..logger import create_logger + class SyncEntity: """Entity MEF synchronization.""" diff --git a/rero_ils/modules/entities/remote_entities/templates/rero_ils/_entity_by_source.html b/rero_ils/modules/entities/remote_entities/templates/rero_ils/_entity_by_source.html deleted file mode 100644 index 3eb2fe35ce..0000000000 --- a/rero_ils/modules/entities/remote_entities/templates/rero_ils/_entity_by_source.html +++ /dev/null @@ -1,55 +0,0 @@ -{# -*- coding: utf-8 -*- - - RERO ILS - Copyright (C) 2019-2023 RERO - Copyright (C) 2019-2023 UCLouvain - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, version 3 of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - -#} - - -{% for agency in ["rero", "gnd", "idref"] %} -{% if record[agency] %} -
-
-
-
- - - -
-
-

- {{ _(agency) }} -

-
-
-
-
-
-
-   -
-
-
- {% with data=record[agency], source_name='agency', source=agency %} - {% include('rero_ils/_entity_by_source_data.html') %} - {% endwith %} -
-
-
-
-
-{% endif %} -{% endfor %} diff --git a/rero_ils/modules/entities/remote_entities/templates/rero_ils/_entity_unified.html b/rero_ils/modules/entities/remote_entities/templates/rero_ils/_entity_unified.html deleted file mode 100644 index cf6305d56c..0000000000 --- a/rero_ils/modules/entities/remote_entities/templates/rero_ils/_entity_unified.html +++ /dev/null @@ -1,45 +0,0 @@ -{# -*- coding: utf-8 -*- - - RERO ILS - Copyright (C) 2019-2023 RERO - Copyright (C) 2019-2023 UCLouvain - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, version 3 of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - -#} - -{% from 'rero_ils/macros/entity.html' import dl, dl_bool, dl_permalink %} -{% set record = record %} -{% set data = record | entity_merge_data_values %} - -
- {{ dl(_('Birth date'), data.date_of_birth) }} - {{ dl(_('Death date'), data.date_of_death) }} - {% if data.language %} - {{ dl(_('Language'), data.language|translat_unified('lang_')) }} - {% endif %} - {{ dl(_('Gender'), data.gender) }} - {{ dl(_('Biographical information'), data.biographical_information | biographicaUrl) }} - {{ dl(_('Qualifier'), data.qualifier) }} - {{ dl(_('Numeration'), data.numeration) }} - {{ dl(_('Date establishment'), data.date_of_establishment) }} - {{ dl(_('Death termination'), data.date_of_termination) }} - {{ dl_bool(_('Conference'), data.conference) }} - {% if data.country_associated %} - {{ dl(_('Associated country'), data.country_associated|translat_unified('country_')) }} - {% endif %} - {{ dl_permalink(_('ID'), data) }} - {{ dl(_('Authorized access point'), data.authorized_access_point) }} - {{ dl(_('Variant access point'), data.variant_access_point) }} - {{ dl(_('Parallel access point'), data.parallel_access_point) }} -
diff --git a/rero_ils/modules/entities/remote_entities/templates/rero_ils/detailed_view_entity.html b/rero_ils/modules/entities/remote_entities/templates/rero_ils/detailed_view_entity.html deleted file mode 100644 index 5541cbe30d..0000000000 --- a/rero_ils/modules/entities/remote_entities/templates/rero_ils/detailed_view_entity.html +++ /dev/null @@ -1,82 +0,0 @@ -{# -*- coding: utf-8 -*- - - RERO ILS - Copyright (C) 2019-2022 RERO - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, version 3 of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - -#} - -{%- extends 'rero_ils/page.html' %} - -{%- block body %} -
-
- {% if record.type == "bf:Organisation" %} - - {% else %} - - {% endif %} -
-
-

{{ record | entity_label(current_i18n.language) }}

- MEF ID: {{ record.pid }} -
-
- - -{%- if record.documents %} -
-
-

{{ _('Documents') }}

-
- -
-{% endif %} - -{% endblock body %} diff --git a/rero_ils/modules/entities/remote_entities/views.py b/rero_ils/modules/entities/remote_entities/views.py index ac34a6cc9a..a2b5229d52 100644 --- a/rero_ils/modules/entities/remote_entities/views.py +++ b/rero_ils/modules/entities/remote_entities/views.py @@ -20,95 +20,18 @@ from __future__ import absolute_import, print_function -import typing - -from flask import Blueprint, abort, current_app, render_template -from flask_babelex import gettext as translate -from invenio_records_ui.signals import record_viewed +from flask import Blueprint, abort from rero_ils.modules.decorators import check_logged_as_librarian -from rero_ils.modules.documents.api import DocumentsSearch -from rero_ils.modules.organisations.api import Organisation -from rero_ils.theme.views import url_active -from .api import RemoteEntity -from ..models import EntityType from .proxy import MEFProxyFactory -blueprint = Blueprint( - 'remote_entities', - __name__, - url_prefix='/', - template_folder='templates', - static_folder='static', -) - api_blueprint = Blueprint( 'api_remote_entities', __name__ ) -def remote_entity_proxy(viewcode, pid, entity_type): - """Proxy for entities. - - :param viewcode: viewcode of html request - :param pid: pid of contribution - :param entity_type: type of the entity - :returns: entity template - """ - entity = RemoteEntity.get_record_by_pid(pid) - if not entity or entity.get('type') != entity_type: - abort(404, 'Record not found') - return remote_entity_view_method( - pid=entity.persistent_identifier, - record=entity, - template='rero_ils/detailed_view_entity.html', - viewcode=viewcode - ) - - -def remote_entity_view_method(pid, record, template=None, **kwargs): - """Display default view. - - Sends record_viewed signal and renders template. - - :param pid: PID object. - :param record: the `Entity` record, - :param template: the template to use to render the entity - """ - record_viewed.send( - current_app._get_current_object(), pid=pid, record=record) - - # Get contribution persons documents - search = DocumentsSearch()\ - .filter('term', contribution__entity__pid=pid.pid_value) - - viewcode = kwargs['viewcode'] - if viewcode != current_app.config.get('RERO_ILS_SEARCH_GLOBAL_VIEW_CODE'): - org_pid = Organisation.get_record_by_viewcode(viewcode)['pid'] - search = search \ - .filter('term', holdings__organisation__organisation_pid=org_pid) - search = search \ - .params(preserve_order=True)\ - .sort({'sort_title': {'order': 'asc'}}) - - record['documents'] = list(search.scan()) - return render_template(template, record=record, viewcode=viewcode) - - -@blueprint.route('/persons/', methods=['GET']) -def persons_proxy(viewcode, pid): - """Proxy person for entity.""" - return remote_entity_proxy(viewcode, pid, EntityType.PERSON) - - -@blueprint.route('/corporate-bodies/', methods=['GET']) -def corporate_bodies_proxy(viewcode, pid): - """Proxy corporate bodies for entity.""" - return remote_entity_proxy(viewcode, pid, EntityType.ORGANISATION) - - @api_blueprint.route('/remote_entities/search/', defaults={'entity_type': 'agents'}) @api_blueprint.route('/remote_entities/search//') @@ -128,79 +51,3 @@ def remote_search_proxy(entity_type, term): return MEFProxyFactory.create_proxy(entity_type).search(term) except ValueError as err: abort(400, str(err)) - - -# TEMPLATE JINJA FILTERS ====================================================== -@blueprint.app_template_filter() -def entity_merge_data_values(data): - """Create merged data for values.""" - sources = current_app.config.get('RERO_ILS_AGENTS_SOURCES', []) - result = {} - for source in sources: - if data.get(source): - result[source] = { - 'pid': data[source]['pid'], - 'identifier': data[source]['identifier'] - } - for key, values in data.get(source, {}).items(): - if key == 'conference': - result['conference'] = data[source]['conference'] - else: - if key not in result: - result[key] = {} - if isinstance(values, str): - values = [values] - for value in values: - if isinstance(value, typing.Hashable): - if value in result[key]: - result[key][value].append(source) - else: - result[key][value] = [source] - return result - - -@blueprint.app_template_filter() -def entity_label(data, language): - """Create contribution label.""" - order = current_app.config.get('RERO_ILS_AGENTS_LABEL_ORDER', []) - source_order = order.get(language, order.get(order['fallback'], [])) - for source in source_order: - if label := data.get(source, {}).get('authorized_access_point', None): - return label - return '-' - - -@blueprint.app_template_filter() -def translat_unified(data, prefix=''): - """Translate the keys of an dictionary. - - :param data: dictionary to translate - :param prefix: prefix to add to keys - :returns: dictionary with translated keys - """ - return { - translate(f'{prefix}{key}'): value - for key, value - in data.items() - } - - -@blueprint.app_template_filter() -@api_blueprint.app_template_filter() -def translat(data, prefix='', seperator=', '): - """Translate data.""" - if data: - if isinstance(data, list): - translated = [translate(f'{prefix}{item}') for item in data] - return seperator.join(translated) - elif isinstance(data, str): - return translate(f'{prefix}{data}') - - -@blueprint.app_template_filter('biographicaUrl') -def biographical_url(biographicals): - """Add link url on text if http detected.""" - return { - url_active(biographical, '_blank'): biographicals[biographical] - for biographical in biographicals - } diff --git a/rero_ils/modules/entities/serializers/__init__.py b/rero_ils/modules/entities/serializers/__init__.py index 14c3a11f45..f74f406517 100644 --- a/rero_ils/modules/entities/serializers/__init__.py +++ b/rero_ils/modules/entities/serializers/__init__.py @@ -18,8 +18,8 @@ """RERO Unified entities serialization.""" -from rero_ils.modules.serializers import RecordSchemaJSONV1, \ - search_responsify +from rero_ils.modules.serializers import RecordSchemaJSONV1, search_responsify + from .base import EntityJSONSerializer # Serializers diff --git a/rero_ils/modules/entities/templates/rero_ils/_local_organisation.html b/rero_ils/modules/entities/templates/rero_ils/_local_organisation.html new file mode 100644 index 0000000000..a88976f481 --- /dev/null +++ b/rero_ils/modules/entities/templates/rero_ils/_local_organisation.html @@ -0,0 +1,28 @@ +{# -*- coding: utf-8 -*- + + RERO ILS + Copyright (C) 2019-2023 RERO + Copyright (C) 2019-2023 UCLouvain + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, version 3 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +#} +{{ dl(_('Name'), record.name) }} +{{ dl(_('Subordinate units'), record.subordinate_units) }} +{{ dl(_('Conference place'), record.conference_place) }} +{{ dl(_('Conference numbering'), record.conference_numbering) }} +{{ dl(_('Conference date'), record.conference_date) }} +{{ dl_bool(_('Conference'), record.conference) }} +{{ dl(_('Start date'), record.start_date) }} +{{ dl(_('End date'), record.end_date) }} +{{ dl(_('Alternative names'), record.alternative_names) }} diff --git a/rero_ils/modules/entities/templates/rero_ils/_local_person.html b/rero_ils/modules/entities/templates/rero_ils/_local_person.html new file mode 100644 index 0000000000..89a4ea4067 --- /dev/null +++ b/rero_ils/modules/entities/templates/rero_ils/_local_person.html @@ -0,0 +1,26 @@ +{# -*- coding: utf-8 -*- + + RERO ILS + Copyright (C) 2019-2023 RERO + Copyright (C) 2019-2023 UCLouvain + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, version 3 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +#} +{{ dl(_('Name'), record.name) }} +{{ dl(_('Birth date'), record.date_of_birth) }} +{{ dl(_('Death date'), record.date_of_death) }} +{{ dl(_('Numeration'), record.numeration) }} +{{ dl(_('Qualifier'), record.qualifier) }} +{{ dl(_('Fuller form of name'), record.fuller_form_of_name) }} +{{ dl(_('Alternative names'), record.alternative_names) }} diff --git a/rero_ils/modules/entities/templates/rero_ils/_local_place.html b/rero_ils/modules/entities/templates/rero_ils/_local_place.html new file mode 100644 index 0000000000..8c281c2318 --- /dev/null +++ b/rero_ils/modules/entities/templates/rero_ils/_local_place.html @@ -0,0 +1,21 @@ +{# -*- coding: utf-8 -*- + + RERO ILS + Copyright (C) 2019-2023 RERO + Copyright (C) 2019-2023 UCLouvain + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, version 3 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +#} +{{ dl(_('Name'), record.name) }} +{{ dl(_('Alternative names'), record.alternative_names) }} diff --git a/rero_ils/modules/entities/templates/rero_ils/_local_temporal.html b/rero_ils/modules/entities/templates/rero_ils/_local_temporal.html new file mode 100644 index 0000000000..8c281c2318 --- /dev/null +++ b/rero_ils/modules/entities/templates/rero_ils/_local_temporal.html @@ -0,0 +1,21 @@ +{# -*- coding: utf-8 -*- + + RERO ILS + Copyright (C) 2019-2023 RERO + Copyright (C) 2019-2023 UCLouvain + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, version 3 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +#} +{{ dl(_('Name'), record.name) }} +{{ dl(_('Alternative names'), record.alternative_names) }} diff --git a/rero_ils/modules/entities/templates/rero_ils/_local_topic.html b/rero_ils/modules/entities/templates/rero_ils/_local_topic.html new file mode 100644 index 0000000000..327fe585b6 --- /dev/null +++ b/rero_ils/modules/entities/templates/rero_ils/_local_topic.html @@ -0,0 +1,22 @@ +{# -*- coding: utf-8 -*- + + RERO ILS + Copyright (C) 2019-2023 RERO + Copyright (C) 2019-2023 UCLouvain + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, version 3 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +#} +{{ dl(_('Name'), record.name) }} +{{ dl_bool(_('Genre form'), record.genreForm) }} +{{ dl(_('Alternative names'), record.alternative_names) }} diff --git a/rero_ils/modules/entities/templates/rero_ils/_local_work.html b/rero_ils/modules/entities/templates/rero_ils/_local_work.html new file mode 100644 index 0000000000..5d534f4ef0 --- /dev/null +++ b/rero_ils/modules/entities/templates/rero_ils/_local_work.html @@ -0,0 +1,21 @@ +{# -*- coding: utf-8 -*- + + RERO ILS + Copyright (C) 2019-2023 RERO + Copyright (C) 2019-2023 UCLouvain + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, version 3 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +#} +{{ dl(_('Title'), record.title) }} +{{ dl(_('Creator'), record.creator) }} diff --git a/rero_ils/modules/entities/remote_entities/templates/rero_ils/_entity_by_source_data.html b/rero_ils/modules/entities/templates/rero_ils/_remote_organisation.html similarity index 62% rename from rero_ils/modules/entities/remote_entities/templates/rero_ils/_entity_by_source_data.html rename to rero_ils/modules/entities/templates/rero_ils/_remote_organisation.html index 0086768e61..c25c78cde8 100644 --- a/rero_ils/modules/entities/remote_entities/templates/rero_ils/_entity_by_source_data.html +++ b/rero_ils/modules/entities/templates/rero_ils/_remote_organisation.html @@ -18,24 +18,20 @@ #} -{% from 'rero_ils/macros/entity.html' import dl, dl_bool, dl_permalink_by_source %} - -{{ dl(_('Birth date'), data.date_of_birth) }} -{{ dl(_('Death date'), data.date_of_death) }} -{% if data.language %} - {{ dl(_('Language'), data.language|translat('lang_')) }} -{% endif %} -{{ dl(_('Gender'), data.gender) }} -{{ dl(_('Biographical information'), data.biographical_information | join('\n') | urlActive('_blank') | nl2br | safe ) }} -{{ dl(_('Qualifier'), data.qualifier) }} -{{ dl(_('Numeration'), data.numeration) }} + {{ dl(_('Date establishment'), data.date_of_establishment) }} + {{ dl(_('Death termination'), data.date_of_termination) }} -{{ dl_bool(_('Conference'), data.conference) }} + +{% if data.language %} + {{ dl(_('Language'), data.language|translate('lang_')) }} +{% endif %} + {% if data.country_associated %} - {{ dl(_('Associated country'), data.country_associated|translat('country_')) }} + {{ dl(_('Associated country'), data.country_associated|translate('country_')) }} {% endif %} -{{ dl_permalink_by_source(_('ID'), data, source) }} + {{ dl(_('Authorized access point'), data.authorized_access_point) }} + {{ dl(_('Variant access point'), data.variant_access_point) }} -{{ dl(_('Parallel access point'), data.parallel_access_point) }} + diff --git a/rero_ils/modules/entities/templates/rero_ils/_remote_person.html b/rero_ils/modules/entities/templates/rero_ils/_remote_person.html new file mode 100644 index 0000000000..c2b6eebad1 --- /dev/null +++ b/rero_ils/modules/entities/templates/rero_ils/_remote_person.html @@ -0,0 +1,44 @@ +{# -*- coding: utf-8 -*- + + RERO ILS + Copyright (C) 2019-2023 RERO + Copyright (C) 2019-2023 UCLouvain + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, version 3 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +#} + + +{% if data.date_of_birth %} + {{ dl(_('Birth date'), data.date_of_birth | format_date(date_format='short', time_format=None)) }} +{% endif %} + +{% if data.date_of_death %} + {{ dl(_('Death date'), data.date_of_death | format_date(date_format='short', time_format=None)) }} +{% endif %} + +{% if data.language %} + {{ dl(_('Language'), data.language|translate('lang_')) }} +{% endif %} + +{{ dl(_('Gender'), data.gender) }} + +{% if data.biographical_information %} +{{ dl(_('Biographical information'), data.biographical_information | join('\n') | urlActive('_blank') | nl2br | safe ) }} +{% endif %} + +{{ dl(_('Name'), data.name) }} + +{{ dl(_('Variant access point'), data.variant_access_point) }} + +{{ dl(_('Authorized access point'), data.authorized_access_point) }} diff --git a/rero_ils/modules/entities/templates/rero_ils/_remote_topic.html b/rero_ils/modules/entities/templates/rero_ils/_remote_topic.html new file mode 100644 index 0000000000..e11561c534 --- /dev/null +++ b/rero_ils/modules/entities/templates/rero_ils/_remote_topic.html @@ -0,0 +1,71 @@ +{# -*- coding: utf-8 -*- + + RERO ILS + Copyright (C) 2019-2023 RERO + Copyright (C) 2019-2023 UCLouvain + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, version 3 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +#} + + +{% if data.bnf_type %} + {{ dl(_('Type'), data.bnf_type) }} +{% endif %} + +{% if data.variant_access_point %} + {{ dl(_('Variant access point(s)'), data.variant_access_point) }} +{% endif %} + +{% if data.broader %} +
{{ _('Broader') }}
+
+
    + {% for broader in data.broader %} +
  • {{ broader.authorized_access_point }}
    • + {% endfor %} +
+
+{% endif %} + +{% if data.related %} +
{{ _('Related') }}
+
+
    + {% for related in data.related %} +
  • {{ related.authorized_access_point }}
    • + {% endfor %} +
+
+{% endif %} + +{% if data.classification %} +
{{ _('Classification(s)') }}
+
+
    + {% for classification in data.classification %} +
  • + {% if classification.classificationPortion %} + {{ classification.classificationPortion }} + {% endif %} + {% if classification.name %} + {{ classification.name }} + {% endif %} + {% if classification.type %} + {{ _(classification.type) }} + {% endif %} +
    • + {% endfor %} +
+
+{% endif %} diff --git a/rero_ils/modules/entities/templates/rero_ils/_search_link.html b/rero_ils/modules/entities/templates/rero_ils/_search_link.html new file mode 100644 index 0000000000..67c06af6cc --- /dev/null +++ b/rero_ils/modules/entities/templates/rero_ils/_search_link.html @@ -0,0 +1,24 @@ +{# -*- coding: utf-8 -*- + + RERO ILS + Copyright (C) 2019-2023 RERO + Copyright (C) 2019-2023 UCLouvain + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, version 3 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +#} + + + + {{ _('Search documents') }} + diff --git a/rero_ils/modules/entities/templates/rero_ils/entity_local.html b/rero_ils/modules/entities/templates/rero_ils/entity_local.html new file mode 100644 index 0000000000..e13b98bca5 --- /dev/null +++ b/rero_ils/modules/entities/templates/rero_ils/entity_local.html @@ -0,0 +1,64 @@ +{# -*- coding: utf-8 -*- + + RERO ILS + Copyright (C) 2019-2023 RERO + Copyright (C) 2019-2023 UCLouvain + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, version 3 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +#} +{% from 'rero_ils/macros/entity.html' import dl, dl_bool, dl_permalink_by_source %} + +{% extends 'rero_ils/page.html' %} + +{% set pid = record.pid %} + +{%- block body %} +
+
+

+ + {{ record.authorized_access_point }} +

+ {{ _('LOCAL ID') }}: {{ pid }} +
+
+ {% include 'rero_ils/_search_link.html' %} +
+
+
+ {% if record['type'] == 'bf:Organisation' %} + {% include 'rero_ils/_local_organisation.html' %} + {% elif record['type'] == 'bf:Person' %} + {% include 'rero_ils/_local_person.html' %} + {% elif record['type'] == 'bf:Place' %} + {% include 'rero_ils/_local_place.html' %} + {% elif record['type'] == 'bf:Temporal' %} + {% include 'rero_ils/_local_temporal.html' %} + {% elif record['type'] == 'bf:Topic' %} + {% include 'rero_ils/_local_topic.html' %} + {% elif record['type'] == 'bf:Work' %} + {% include 'rero_ils/_local_work.html' %} + {% endif %} + + + {{ dl(_('Source catalog'), record.source_catalog) }} + {% if record.identifier %} +
{{ _('Identifier') }}
+
+ {{ _(record.identifier.type) }} - {{ record.identifier.value }} + {% if record.identifier.source %}({{ record.identifier.source }}){% endif %} +
+ {% endif %} +
+{%- endblock body %} diff --git a/rero_ils/modules/entities/templates/rero_ils/entity_remote.html b/rero_ils/modules/entities/templates/rero_ils/entity_remote.html new file mode 100644 index 0000000000..97f2efbe88 --- /dev/null +++ b/rero_ils/modules/entities/templates/rero_ils/entity_remote.html @@ -0,0 +1,64 @@ +{# -*- coding: utf-8 -*- + + RERO ILS + Copyright (C) 2019-2023 RERO + Copyright (C) 2019-2023 UCLouvain + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, version 3 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +#} +{% from 'rero_ils/macros/entity.html' import dl, dl_bool, dl_permalink_by_source %} + +{% extends 'rero_ils/page.html' %} + +{% set source, data = record|extract_data_from_remote_entity %} +{% set pid = record.pid %} + +{%- block body %} +
+
+

+ + {{ record | entity_label(current_i18n.language) }} +

+ MEF ID: {{ pid }} +
+
+ {% include 'rero_ils/_search_link.html' %} +
+
+
+ {% if record['type'] == 'bf:Organisation' %} + {% include 'rero_ils/_remote_organisation.html' %} + {% elif record['type'] == 'bf:Person' %} + {% include 'rero_ils/_remote_person.html' %} + {% elif record['type'] == 'bf:Topic' %} + {% include 'rero_ils/_remote_topic.html' %} + {% endif %} + + {% set links = record|sources_link %} + {% if links != {} %} +
{{ _('Source(s)') }}
+
+
    + {% for source, link in links.items() %} +
  • + {{ source }} + {{ "; " if not loop.last else "" }} +
    • + {% endfor %} +
+
+ {% endif %} +
+{%- endblock body %} diff --git a/rero_ils/modules/entities/remote_entities/templates/rero_ils/macros/entity.html b/rero_ils/modules/entities/templates/rero_ils/macros/entity.html similarity index 72% rename from rero_ils/modules/entities/remote_entities/templates/rero_ils/macros/entity.html rename to rero_ils/modules/entities/templates/rero_ils/macros/entity.html index 9bd81f1392..8dd8572b9e 100644 --- a/rero_ils/modules/entities/remote_entities/templates/rero_ils/macros/entity.html +++ b/rero_ils/modules/entities/templates/rero_ils/macros/entity.html @@ -20,10 +20,10 @@ {% macro dl(name, value, prefix) %} {% if value %} -
+
{{ name }}:
-
+
{% if value is string %} {% if prefix %} {{ _(prefix.format(v=value)) }} @@ -56,33 +56,11 @@ {{ name }}:
- +
{% endif %} {% endmacro %} -{% macro dl_permalink(name, record) %} -
- {{ _(name) }}: -
-
-
    - {% for source in config.RERO_ILS_AGENTS_SOURCES %} - {% if record[source] %} -
  • - {% if source != 'rero' %} - {{ record[source].pid }} - {% else %} - {{ record[source].pid }} - {% endif %} - {{ source }} -
    • - {% endif %} - {% endfor %} -
-
-{% endmacro %} - {% macro dl_permalink_by_source(name, data, source) %} {% if data.pid %}
diff --git a/rero_ils/modules/entities/views.py b/rero_ils/modules/entities/views.py new file mode 100644 index 0000000000..b6e9f481f8 --- /dev/null +++ b/rero_ils/modules/entities/views.py @@ -0,0 +1,148 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Blueprint used for entities.""" +from flask import Blueprint, abort, current_app, render_template +from flask_babelex import gettext as _ +from invenio_i18n.ext import current_i18n + +from rero_ils.modules.entities.models import EntityType + +from .local_entities.api import LocalEntity +from .remote_entities.api import RemoteEntity + +blueprint = Blueprint( + 'entities', + __name__, + url_prefix='//entities', + template_folder='templates', + static_folder='static', +) + + +@blueprint.route('//') +def entity_detailed_view(viewcode, type, pid): + """Display entity view (local or remote). + + :param: viewcode: The current view code. + :param: type: Resource type. + :param: pid: Resource PID. + :returns: The html rendering of the resource. + """ + entity_class = LocalEntity if type == 'local' else RemoteEntity + if not (record := entity_class.get_record_by_pid(pid)): + abort(404, _('Entity not found.')) + + return render_template( + f'rero_ils/entity_{type}.html', + record=record, + viewcode=viewcode, + search_link=search_link(record) + ) + + +@blueprint.app_template_filter() +def entity_icon(type): + """Selects the right icon according to type. + + :param: type: Resource type. + :returns: string, The class of the selected icon. + """ + icons = { + EntityType.ORGANISATION: 'fa-building-o', + EntityType.PERSON: 'fa-user-o', + EntityType.PLACE: 'fa-map-marker', + EntityType.TEMPORAL: 'fa-calendar', + EntityType.TOPIC: 'fa-tag', + EntityType.WORK: 'fa-book' + } + return icons.get(type, 'fa-question-circle-o') + + +@blueprint.app_template_filter() +def extract_data_from_remote_entity(record): + """Data extraction based on language and resource type. + + Used only on remote entity. + + :param: record: the json record + :returns: source and the dictionary of the resource selected. + """ + locale = current_i18n.locale.language + agent_order = current_app.config.get('RERO_ILS_AGENTS_LABEL_ORDER') + if locale not in agent_order: + locale = agent_order.get('fallback', {}) + sources = agent_order.get(locale) + for source in sources: + if data := record.get(source): + return source, data + + +@blueprint.app_template_filter() +def entity_label(data, language): + """Create contribution label. + + :param data: The record metadata. + :param language: The current language. + :returns: The contribution label. + """ + order = current_app.config.get('RERO_ILS_AGENTS_LABEL_ORDER', []) + source_order = order.get(language, order.get(order['fallback'], [])) + for source in source_order: + if label := data.get(source, {}).get('authorized_access_point', None): + return label + return '-' + + +@blueprint.app_template_filter() +def sources_link(data): + """Extract sources link. + + :param data: The record metadata. + :returns A dict with the source and link. + """ + links = {} + sources_link = list(filter(lambda source: source not in + current_app.config.get( + 'RERO_ILS_AGENTS_SOURCES_EXCLUDE_LINK', []), + data.get('sources', []))) + + for source in sources_link: + if identifier := data.get(source, {}).get('identifier'): + links[source] = identifier + return links + + +def search_link(metadata): + """Generate Link for search entities. + + :param metadata: the record metadata. + :returns: the search link. + """ + queries = [] + for field in ['contribution', 'subjects', 'genreForm']: + if 'sources' in metadata: + # Remote entities + source, data = extract_data_from_remote_entity(metadata) + entity_id = data.get('pid') + else: + # Local entities + source = 'local' + entity_id = metadata.get('pid') + queries.append(f'{field}.entity.pids.{source}:{entity_id}') + return " OR ".join(queries) + "&simple=0" diff --git a/rero_ils/modules/ext.py b/rero_ils/modules/ext.py index 186083cb1c..cddd0d56db 100644 --- a/rero_ils/modules/ext.py +++ b/rero_ils/modules/ext.py @@ -42,7 +42,7 @@ from rero_ils.filter import address_block, empty_data, format_date_filter, \ get_record_by_ref, jsondumps, message_filter, node_assets, text_to_id, \ - to_pretty_json + to_pretty_json, translate from rero_ils.modules.acquisition.acq_accounts.listener import \ enrich_acq_account_data from rero_ils.modules.acquisition.acq_order_lines.listener import \ @@ -175,6 +175,7 @@ def __init__(self, app=None): app.add_template_filter(address_block) app.add_template_filter(message_filter, name='message') app.add_template_filter(issue_client_reference) + app.add_template_filter(translate) app.jinja_env.add_extension('jinja2.ext.do') app.jinja_env.globals['version'] = __version__ self.register_signals(app) diff --git a/rero_ils/modules/monitoring/api.py b/rero_ils/modules/monitoring/api.py index d72a304f79..f8ebbc7cb6 100644 --- a/rero_ils/modules/monitoring/api.py +++ b/rero_ils/modules/monitoring/api.py @@ -68,7 +68,7 @@ class Monitoring(object): times of selected functions. """ - has_no_db = ['oplg'] + has_no_db = ['oplg', 'ent'] def __init__(self, time_delta=1): """Constructor. diff --git a/rero_ils/theme/templates/rero_ils/address_block/eng.tpl.txt b/rero_ils/theme/templates/rero_ils/address_block/eng.tpl.txt index 16820da98c..c4c01298c5 100644 --- a/rero_ils/theme/templates/rero_ils/address_block/eng.tpl.txt +++ b/rero_ils/theme/templates/rero_ils/address_block/eng.tpl.txt @@ -1,7 +1,7 @@ {{ data.name }} {{ data.address.street }} {{ data.address.zip_code }} - {{ data.address.city }} -{{ data.address.country | translat('country_') }} +{{ data.address.country | translate('country_') }} {%- if data.email %} Email: {{ data.email }} {%- endif %} diff --git a/rero_ils/theme/templates/rero_ils/address_block/fre.tpl.txt b/rero_ils/theme/templates/rero_ils/address_block/fre.tpl.txt index 4bb3d15337..5583059424 100644 --- a/rero_ils/theme/templates/rero_ils/address_block/fre.tpl.txt +++ b/rero_ils/theme/templates/rero_ils/address_block/fre.tpl.txt @@ -1,7 +1,7 @@ {{ data.name }} {{ data.address.street }} {{ data.address.zip_code }} - {{ data.address.city }} -{{ data.address.country | translat('country_') }} +{{ data.address.country | translate('country_') }} {%- if data.email %} E-mail: {{ data.email }} {%- endif %} diff --git a/rero_ils/theme/templates/rero_ils/address_block/ger.tpl.txt b/rero_ils/theme/templates/rero_ils/address_block/ger.tpl.txt index 2dd13d4009..b1e49c29c2 100644 --- a/rero_ils/theme/templates/rero_ils/address_block/ger.tpl.txt +++ b/rero_ils/theme/templates/rero_ils/address_block/ger.tpl.txt @@ -1,7 +1,7 @@ {{ data.name }} {{ data.address.street }} {{ data.address.zip_code }} - {{ data.address.city }} -{{ data.address.country | translat('country_') }} +{{ data.address.country | translate('country_') }} {%- if data.email %} E-mail: {{ data.email }} {%- endif %} diff --git a/rero_ils/theme/templates/rero_ils/address_block/ita.tpl.txt b/rero_ils/theme/templates/rero_ils/address_block/ita.tpl.txt index 3454e4259d..60cca2d6b6 100644 --- a/rero_ils/theme/templates/rero_ils/address_block/ita.tpl.txt +++ b/rero_ils/theme/templates/rero_ils/address_block/ita.tpl.txt @@ -1,7 +1,7 @@ {{ data.name }} {{ data.address.street }} {{ data.address.zip_code }} - {{ data.address.city }} -{{ data.address.country | translat('country_') }} +{{ data.address.country | translate('country_') }} {%- if data.email %} E-mail: {{ data.email }} {%- endif %} diff --git a/tests/api/entities/local_entities/test_local_entities_rest.py b/tests/api/entities/local_entities/test_local_entities_rest.py index 4f37ae8d84..23ae3c21ab 100644 --- a/tests/api/entities/local_entities/test_local_entities_rest.py +++ b/tests/api/entities/local_entities/test_local_entities_rest.py @@ -19,15 +19,16 @@ """Tests `LocalEntity` resource REST API.""" import json + import mock from flask import url_for -from utils import get_json, postdata, to_relative_url, \ - VerifyRecordPermissionPatch +from utils import VerifyRecordPermissionPatch, get_json, postdata, \ + to_relative_url from rero_ils.modules.documents.dumpers import document_replace_refs_dumper -from rero_ils.modules.entities.models import EntityType -from rero_ils.modules.entities.local_entities.api import LocalEntity from rero_ils.modules.entities.dumpers import indexer_dumper +from rero_ils.modules.entities.local_entities.api import LocalEntity +from rero_ils.modules.entities.models import EntityType from rero_ils.modules.utils import get_ref_for_pid diff --git a/tests/api/entities/remote_entities/test_remote_entities_rest.py b/tests/api/entities/remote_entities/test_remote_entities_rest.py index bb0be4bf80..f6a7de963d 100644 --- a/tests/api/entities/remote_entities/test_remote_entities_rest.py +++ b/tests/api/entities/remote_entities/test_remote_entities_rest.py @@ -22,8 +22,8 @@ from flask import url_for from utils import get_json, mock_response, postdata, to_relative_url -from rero_ils.modules.entities.models import EntityType from rero_ils.modules.entities.dumpers import indexer_dumper +from rero_ils.modules.entities.models import EntityType def test_remote_entities_permissions(client, entity_person, json_header): diff --git a/tests/fixtures/mef.py b/tests/fixtures/mef.py index dfe90981c9..d713ddfef0 100644 --- a/tests/fixtures/mef.py +++ b/tests/fixtures/mef.py @@ -20,8 +20,10 @@ from copy import deepcopy import pytest -from rero_ils.modules.entities.remote_entities.api import RemoteEntity, \ - RemoteEntitiesSearch + +from rero_ils.modules.entities.remote_entities.api import \ + RemoteEntitiesSearch, RemoteEntity + @pytest.fixture(scope="module") def mef_concept1_data(mef_entities): diff --git a/tests/fixtures/metadata.py b/tests/fixtures/metadata.py index cc23e158fc..06f2f60286 100644 --- a/tests/fixtures/metadata.py +++ b/tests/fixtures/metadata.py @@ -27,12 +27,12 @@ from utils import flush_index, mock_response from rero_ils.modules.documents.api import Document, DocumentsSearch +from rero_ils.modules.entities.local_entities.api import LocalEntitiesSearch, \ + LocalEntity from rero_ils.modules.entities.remote_entities.api import \ RemoteEntitiesSearch, RemoteEntity from rero_ils.modules.holdings.api import Holding, HoldingsSearch from rero_ils.modules.items.api import Item, ItemsSearch -from rero_ils.modules.entities.local_entities.api import LocalEntitiesSearch, \ - LocalEntity from rero_ils.modules.local_fields.api import LocalField, LocalFieldsSearch from rero_ils.modules.operation_logs.api import OperationLog from rero_ils.modules.templates.api import Template, TemplatesSearch diff --git a/tests/ui/documents/test_documents_api.py b/tests/ui/documents/test_documents_api.py index 18125b5b2b..0ee5a57f2a 100644 --- a/tests/ui/documents/test_documents_api.py +++ b/tests/ui/documents/test_documents_api.py @@ -182,8 +182,9 @@ def test_document_linked_subject( in subject['entity']['variant_access_point'] # reset fixtures - entity.delete() + doc.delete_from_index() doc.delete() + entity.delete() def test_document_add_cover_url(db, document): diff --git a/tests/ui/documents/test_documents_filter.py b/tests/ui/documents/test_documents_filter.py index 210e991964..e7dc724623 100644 --- a/tests/ui/documents/test_documents_filter.py +++ b/tests/ui/documents/test_documents_filter.py @@ -400,15 +400,26 @@ def test_contribution_format(db, entity_organisation): 'authorized_access_point_fr': 'author_fr' } }] - assert contribution_format(contributions, 'en', 'global') == 'author_def' - assert contribution_format(contributions, 'fr', 'global') == 'author_fr' - assert contribution_format(contributions, 'zh', 'global') == 'author_def' + # ---- Textual contribution + # With english language + link_part = '/global/search/documents?q=' \ + 'contribution.entity.authorized_access_point_en%3A' \ + '%22author_def%22' + assert link_part in contribution_format(contributions, 'en', 'global') + + # With french language + link_part = '/global/search/documents?q=' \ + 'contribution.entity.authorized_access_point_fr%3A' \ + '%22author_fr%22' + assert link_part in contribution_format(contributions, 'fr', 'global') + + # ---- Remote contribution contributions = [{ 'entity': {'pid': entity.pid} }] - link_part = f'/global/search/documents?q' \ - f'=contribution.entity.pids.{entity.resource_type}%3A' \ + link_part = f'/global/search/documents?q=' \ + f'contribution.entity.pids.{entity.resource_type}%3A' \ f'{entity.pid}' assert link_part in contribution_format(contributions, 'en', 'global') @@ -550,46 +561,71 @@ def test_main_title_text(): assert extract[0].get('_text') is not None -def test_doc_entity_label_filter(entity_person): +def test_doc_entity_label_filter(entity_person, local_entity_person): """Test entity label filter.""" + + # Remote entity + remote_pid = entity_person['idref']['pid'] data = { 'entity': { - 'authorized_access_point': 'subject topic', - 'type': EntityType.TOPIC + '$ref': f'https://mef.rero.ch/api/concepts/idref/{remote_pid}', + 'pid': remote_pid } } - assert doc_entity_label(data['entity'], None) == 'subject topic' - assert doc_entity_label(data['entity'], 'fr') == 'subject topic' + entity_type, value, label = doc_entity_label(data['entity'], 'fr') + assert 'remote' == entity_type + assert 'ent_pers' == value + assert 'Loy, Georg, 1885-19..' == label + # Local entity + pid = local_entity_person['pid'] data = { 'entity': { - 'authorized_access_point': 'topic_default', - 'authorized_access_point_fr': 'topic_fr', - 'type': EntityType.TOPIC + '$ref': f'https://bib.rero.ch/api/local_entities/{pid}' } } - assert doc_entity_label(data['entity'], 'fr') == 'topic_fr' - assert doc_entity_label(data['entity'], 'en') == 'topic_default' - assert doc_entity_label(data['entity'], None) == 'topic_default' + entity_type, value, label = doc_entity_label(data['entity'], 'fr') + assert 'local' == entity_type + assert 'locent_pers' == value + assert 'Loy, Georg (1881-1968)' == label + entity_type, value, label = doc_entity_label(data['entity'], 'en') + assert 'local' == entity_type + assert 'locent_pers' == value + assert 'Loy, Georg (1881-1968)' == label + + # Textual data = { 'entity': { - 'authorized_access_point': 'topic_default', - 'subdivisions': [{ - 'entity': { - 'authorized_access_point': 'sub_default', - 'authorized_access_point_fr': 'sub_fr' - } - }], - 'type': EntityType.TOPIC + 'authorized_access_point': 'subject topic' } } - assert doc_entity_label(data['entity'], 'fr') == 'topic_default - sub_fr' - assert doc_entity_label( - data['entity'], 'en') == 'topic_default - sub_default' - assert doc_entity_label( - data['entity'], None) == 'topic_default - sub_default' - - data = {'entity': {'pid': entity_person.pid}} - assert doc_entity_label(data['entity'], 'fr') == 'Loy, Georg, 1885-19..' - assert doc_entity_label(data['entity'], 'de') == 'Loy, Georg, 1885' + entity_type, value, label = doc_entity_label(data['entity'], None) + assert 'textual' == entity_type + assert 'subject topic' == value + assert 'subject topic' == label + + entity_type, value, label = doc_entity_label(data['entity'], 'fr') + assert 'textual' == entity_type + assert 'subject topic' == value + assert 'subject topic' == label + + # Textual with subdivision + data['entity']['subdivisions'] = [ + { + 'entity': { + 'authorized_access_point': 'Sub 1', + 'type': EntityType.TOPIC + } + }, + { + 'entity': { + 'authorized_access_point': 'Sub 2', + 'type': EntityType.TOPIC + } + } + ] + entity_type, value, label = doc_entity_label(data['entity'], 'fr') + assert 'textual' == entity_type + assert 'subject topic' == value + assert 'subject topic - Sub 1 - Sub 2' == label diff --git a/tests/ui/entities/local_entities/test_local_entities_api.py b/tests/ui/entities/local_entities/test_local_entities_api.py index ecfbf9c3f1..296c2ea0b7 100644 --- a/tests/ui/entities/local_entities/test_local_entities_api.py +++ b/tests/ui/entities/local_entities/test_local_entities_api.py @@ -22,6 +22,7 @@ import time from datetime import timedelta + from utils import flush_index from rero_ils.modules.documents.api import Document, DocumentsSearch diff --git a/tests/ui/entities/remote_entities/test_remote_entities_api.py b/tests/ui/entities/remote_entities/test_remote_entities_api.py index 4ffc13edce..311a117a3b 100644 --- a/tests/ui/entities/remote_entities/test_remote_entities_api.py +++ b/tests/ui/entities/remote_entities/test_remote_entities_api.py @@ -125,6 +125,10 @@ def test_sync_contribution( ) flush_index(DocumentsSearch.Meta.index) + # Test that entity could not be deleted + assert pers.get_links_to_me(True)['documents'] == [doc.pid] + assert pers.reasons_not_to_delete()['links']['documents'] == 1 + # === nothing to update sync_entity._get_latest = mock.MagicMock( return_value=entity_person_data_tmp diff --git a/tests/ui/entities/remote_entities/test_remote_entities_filter.py b/tests/ui/entities/remote_entities/test_remote_entities_filter.py deleted file mode 100644 index 6080a26cd0..0000000000 --- a/tests/ui/entities/remote_entities/test_remote_entities_filter.py +++ /dev/null @@ -1,108 +0,0 @@ -# -*- coding: utf-8 -*- -# -# RERO ILS -# Copyright (C) 2019-2023 RERO -# Copyright (C) 2019-2023 UCLouvain -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, version 3 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -"""Jinja2 filters tests.""" - -from rero_ils.modules.entities.remote_entities.views import entity_label, \ - entity_merge_data_values - - -def test_remote_entity_label(app, entity_person_data): - """Test entity label.""" - app.config['RERO_ILS_AGENTS_LABEL_ORDER'] = { - 'fallback': 'fr', - 'fr': ['rero', 'idref', 'gnd'], - 'de': ['gnd', 'rero', 'idref'], - } - label = entity_label(entity_person_data, 'fr') - assert label == 'Loy, Georg, 1885-19..' - label = entity_label(entity_person_data, 'it') - assert label == 'Loy, Georg, 1885-19..' - - -def test_remote_entity_merge_data_values(app, entity_person_data): - """Test entities merge data.""" - app.config['RERO_ILS_AGENTS_SOURCES'] = ['idref', 'gnd', 'rero'] - data = entity_merge_data_values(entity_person_data) - assert data == { - '$schema': { - 'https://mef.test.rero.ch/schemas/gnd/' - 'gnd-contribution-v0.0.1.json': ['gnd'], - 'https://mef.test.rero.ch/schemas/idref/' - 'idref-contribution-v0.0.1.json': ['idref'], - 'https://mef.test.rero.ch/schemas/rero/' - 'rero-contribution-v0.0.1.json': ['rero'] - }, - 'authorized_access_point': { - 'Loy, Georg, 1885': ['gnd'], - 'Loy, Georg, 1885-19..': ['idref', 'rero'] - }, - 'bf:Agent': { - 'bf:Person': ['idref', 'gnd', 'rero'] - }, - 'biographical_information': { - 'Diss. philosophische Fakultät': ['gnd'] - }, - 'country_associated': { - 'gw': ['idref', 'rero'] - }, - 'date_of_birth': { - '1885': ['gnd'], - '1885-05-14': ['idref', 'rero'] - }, - 'date_of_death': { - '19..': ['idref', 'rero'] - }, - 'identifier': { - 'http://d-nb.info/gnd/13343771X': ['gnd'], - 'http://www.idref.fr/223977268': ['idref'], - 'http://data.rero.ch/02-A017671081': ['rero'] - }, - 'language': { - 'ger': ['idref', 'rero'] - }, - 'md5': { - '3dd3788c64af4200676a35a5ea35b180': ['idref'], - '3dd3788c64af4200676a35a5ea35b181': ['rero'], - '5dad1e77d5a47d39e87bb0ec37aaf51e': ['gnd'] - }, - 'pid': { - '13343771X': ['gnd'], - '223977268': ['idref'], - 'A017671081': ['rero'] - }, - 'preferred_name': { - 'Loy, Georg': ['idref', 'gnd', 'rero'] - }, - 'variant_name': { - 'Loy, George, di Madeiros': ['gnd'] - }, - 'gnd': { - 'identifier': 'http://d-nb.info/gnd/13343771X', - 'pid': '13343771X' - }, - 'idref': { - 'identifier': 'http://www.idref.fr/223977268', - 'pid': '223977268' - }, - 'rero': { - 'identifier': 'http://data.rero.ch/02-A017671081', - 'pid': 'A017671081' - } - - } diff --git a/tests/ui/entities/remote_entities/test_remote_entities_ui.py b/tests/ui/entities/remote_entities/test_remote_entities_ui.py index 3303ff713d..51a0a04a91 100644 --- a/tests/ui/entities/remote_entities/test_remote_entities_ui.py +++ b/tests/ui/entities/remote_entities/test_remote_entities_ui.py @@ -16,22 +16,19 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -"""Tests UI view for entities.""" +"""Jinja2 filters tests.""" -from flask import url_for +from rero_ils.modules.entities.views import entity_label -def test_remote_entity_person_detailed_view(client, entity_person): - """Test entity person detailed view.""" - res = client.get(url_for( - 'remote_entities.persons_proxy', - viewcode='global', pid=entity_person.pid)) - assert res.status_code == 200 - - -def test_remote_entity_organisation_detailed_view(client, entity_organisation): - """Test entity organisation detailed view.""" - res = client.get(url_for( - 'remote_entities.corporate_bodies_proxy', - viewcode='global', pid='ent_org')) - assert res.status_code == 200 +def test_remote_entity_label(app, entity_person_data): + """Test entity label.""" + app.config['RERO_ILS_AGENTS_LABEL_ORDER'] = { + 'fallback': 'fr', + 'fr': ['rero', 'idref', 'gnd'], + 'de': ['gnd', 'rero', 'idref'], + } + label = entity_label(entity_person_data, 'fr') + assert label == 'Loy, Georg, 1885-19..' + label = entity_label(entity_person_data, 'it') + assert label == 'Loy, Georg, 1885-19..' diff --git a/tests/ui/entities/test_entities_ui.py b/tests/ui/entities/test_entities_ui.py new file mode 100644 index 0000000000..1a040238f1 --- /dev/null +++ b/tests/ui/entities/test_entities_ui.py @@ -0,0 +1,137 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Tests UI view for entities.""" + +from flask import url_for +from invenio_i18n.ext import current_i18n + +from rero_ils.modules.entities.models import EntityType +from rero_ils.modules.entities.views import entity_icon, \ + extract_data_from_remote_entity, search_link, sources_link + + +def test_view(client, entity_person, local_entity_person): + """Entity detailed view test.""" + + # Check unauthorized type value in url + res = client.get(url_for( + 'entities.entity_detailed_view', + viewcode='global', + type='foo', + pid='foo' + )) + assert res.status_code == 404 + + # Check 404 error if entity does not exist + res = client.get(url_for( + 'entities.entity_detailed_view', + viewcode='global', + type='remote', + pid='foo' + )) + assert res.status_code == 404 + + # Remote entity + res = client.get(url_for( + 'entities.entity_detailed_view', + viewcode='global', + type='remote', + pid=entity_person.get('pid') + )) + assert res.status_code == 200 + + # Local entity + res = client.get(url_for( + 'entities.entity_detailed_view', + viewcode='global', + type='local', + pid=local_entity_person.get('pid') + )) + assert res.status_code == 200 + + +def test_entity_icon(): + """Entity icon test.""" + assert 'fa-building-o' == entity_icon(EntityType.ORGANISATION) + # Default icon if type not found + assert 'fa-question-circle-o' == entity_icon('foo') + + +def test_extract_data_from_record(app): + """Extract data from record test.""" + contrib_data = { + 'idref': {'data': 'idref'}, + 'rero': {'data': 'rero'}, + 'gnd': {'data': 'gnd'} + } + current_i18n.locale.language = 'fr' + source, data = extract_data_from_remote_entity(contrib_data) + assert source == 'idref' + assert contrib_data.get(source) == data + + current_i18n.locale.language = 'de' + source, data = extract_data_from_remote_entity(contrib_data) + assert source == 'gnd' + assert contrib_data.get(source) == data + + # Fallback test + current_i18n.locale.language = 'it' + source, data = extract_data_from_remote_entity(contrib_data) + assert source == 'idref' + assert contrib_data.get(source) == data + + # Control the selection cascade + contrib_data.pop('idref') + contrib_data.pop('gnd') + source, data = extract_data_from_remote_entity(contrib_data) + assert source == 'rero' + assert contrib_data.get(source) == data + + +def test_sources_link(app): + """Sources link test.""" + data = { + 'idref': {'identifier': 'http://www.idref.fr/066924502'}, + 'gnd': {'identifier': 'http://d-nb.info/gnd/118754688'}, + 'rero': {'identifier': 'http://data.rero.ch/02-A003795108'}, + 'sources': ['idref', 'gnd', 'rero'] + } + result = { + 'idref': 'http://www.idref.fr/066924502', + 'gnd': 'http://d-nb.info/gnd/118754688' + } + assert result == sources_link(data) + assert {} == sources_link({}) + + +def test_search_link(app, entity_organisation, local_entity_org): + """Search link test.""" + + # test remote link + link = search_link(entity_organisation) + assert link == 'contribution.entity.pids.rero:A027711299 ' \ + 'OR subjects.entity.pids.rero:A027711299 ' \ + 'OR genreForm.entity.pids.rero:A027711299' \ + '&simple=0' + # test local link + link = search_link(local_entity_org) + assert link == 'contribution.entity.pids.local:locent_org ' \ + 'OR subjects.entity.pids.local:locent_org ' \ + 'OR genreForm.entity.pids.local:locent_org' \ + '&simple=0' diff --git a/tests/ui/test_filters.py b/tests/ui/test_filters.py index b4cef0d6c8..7af95a3dd4 100644 --- a/tests/ui/test_filters.py +++ b/tests/ui/test_filters.py @@ -31,7 +31,7 @@ def test_get_record_by_ref(document_data, document): def test_date_filter_format_timestamp_en(app): """Test full english date and tile filter.""" datestring = format_date_filter('2018-06-06T09:29:55.947149+00:00') - assert 'Wednesday, June 6, 2018, 11:29:55 AM' in datestring + assert 'Wednesday, 6 June 2018, 11:29:55' in datestring datestring = format_date_filter( '2018-06-06T09:29:55.947149+00:00', locale='fr') @@ -46,28 +46,28 @@ def test_date_filter_format_default_en(app): """Test medium english date filter.""" datestring = format_date_filter( '1950-01-01', date_format='short', time_format=None) - assert '1/1/50' in datestring + assert '01/01/1950' in datestring def test_date_filter_timezone(app): """Test medium english date filter.""" datestring = format_date_filter( '2018-06-06T09:29:55.947149+00:00', timezone='Europe/Helsinki') - assert 'Wednesday, June 6, 2018, 12:29:55 PM' in datestring + assert 'Wednesday, 6 June 2018, 12:29:55' in datestring def test_date_filter_format_medium_date_en(app): """Test medium_date english date filter.""" datestring = format_date_filter( '1950-01-01', date_format='medium', time_format=None) - assert 'Jan 1, 1950' in datestring + assert '1 Jan 1950' in datestring def test_date_filter_format_full_en(app): """Test full english date filter.""" datestring = format_date_filter( '1950-01-01', date_format='full', time_format=None) - assert 'Sunday, January 1, 1950' in datestring + assert 'Sunday, 1 January 1950' in datestring def test_date_filter_format_full_fr(app): @@ -88,7 +88,7 @@ def test_time_filter_format_default(app): """Test default time.""" datestring = format_date_filter( '2018-06-06T09:29:55.947149+00:00', date_format=None) - assert datestring == '11:29:55 AM' + assert datestring == '11:29:55' def test_time_filter_format_fr(app): @@ -102,7 +102,7 @@ def test_time_filter_format_delimiter(app): """Test default time.""" datestring = format_date_filter( '2018-06-06T09:29:55.947149+00:00', delimiter=' - ') - assert datestring == 'Wednesday, June 6, 2018 - 11:29:55 AM' + assert datestring == 'Wednesday, 6 June 2018 - 11:29:55' def test_to_pretty(): diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 32b0f29d1f..c08f6d91a7 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -25,9 +25,9 @@ from pkg_resources import resource_string from utils import get_schema +from rero_ils.modules.entities.remote_entities.api import \ + RemoteEntitiesSearch, RemoteEntity from rero_ils.modules.patrons.api import Patron -from rero_ils.modules.entities.remote_entities.api import RemoteEntity, \ - RemoteEntitiesSearch @pytest.fixture(scope='module')