8000 viaf: fix update by rerowep · Pull Request #143 · rero/rero-mef · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

viaf: fix update #143

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Aug 22, 2023
Merged

viaf: fix update #143

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
923 changes: 510 additions & 413 deletions poetry.lock
View file
Open in desktop

Large diffs are not rendered by default.

117 changes: 85 additions & 32 deletions rero_mef/agents/api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""API for manipulating records."""

from flask import current_app

from .mef.api import build_ref_string
from ..api import Action, ReroIndexer, ReroMefRecord

Expand Down Expand Up @@ -50,56 +52,107 @@ def delete(self, force=False, dbcommit=False, delindex=False):
delindex=delindex
)

def create_or_update_mef(self, dbcommit=False, reindex=False):
def create_or_update_mef(self, dbcommit=False, reindex=False,
viaf_record=None):
"""Create or update MEF.

:param dbcommit: Commit changes to DB.
:param reindex: Reindex record.
:param viaf_record: VIAF record to use if we know it before.
:returns: MEF record, MEF action
"""
from rero_mef.agents import AgentMefRecord, AgentViafRecord

mef_data = {}
viaf_record = {}
update = False
# Try to get MEF by VIAF
if viaf_records := AgentViafRecord.get_viaf(self):
viaf_record = viaf_records[0]
if mef_records := AgentMefRecord.get_mef(
viaf_record.pid, viaf_record.name):
mef_data = mef_records[0]
update = True
# No MEF by VIAF found try to get MEF by agent
if not update:
if mef_records := AgentMefRecord.get_mef(self.pid, self.name):
mef_data = mef_records[0]
update = True

if self.deleted and not mef_data.get('deleted'):
mef_data['deleted'] = self.deleted
mef_records = []
mef_pids = set()
mef_actions = {}
viaf_records = [viaf_record] if viaf_record else []
viaf_pids = {viaf_record.pid} if viaf_record else set()
# get all VIAF records
for viaf in AgentViafRecord.get_viaf(self):
if viaf.pid not in viaf_pids:
viaf_pids.add(viaf.pid)
viaf_records.append(viaf)
if len(viaf_records) > 1:
current_app.logger.error(
f'MULTIPLE VIAF FOUND FOR: {self.name} {self.pid} | '
f'viaf: {", ".join([viaf.pid for viaf in viaf_records])}'
)
# get all VIAF associated MEF records.
for viaf in viaf_records:
for mef in AgentMefRecord.get_mef(viaf.pid, viaf.name):
if mef.pid not in mef_pids:
mef_pids.add(mef.pid)
mef_records.append(mef)
# get all MEF records by agent pid
for mef in AgentMefRecord.get_mef(self.pid, self.name):
if mef.pid not in mef_pids:
mef_pids.add(mef.pid)
mef_records.append(mef)
if len(mef_records) > 1:
current_app.logger.error(
f'MULTIPLE MEF FOUND FOR: {self.name} {self.pid} | '
f'mef: {", ".join([mef.pid for mef in mef_records])}'
)

ref_string = build_ref_string(agent=self.name, agent_pid=self.pid)
mef_data[self.name] = {'$ref': ref_string}

if viaf_record:
mef_data['viaf_pid'] = viaf_record['pid']
if update:
mef_action = Action.UPDATE
mef_record = mef_data.replace(
data=mef_data,
dbcommit=dbcommit,
reindex=reindex
)
old_pids = set()
if mef_records:
# We have MEF records change them.
for mef in mef_records[1:]:
# Delete ref in MEF records
if old_ref := mef.pop(self.name, None):
old_pid = old_ref['$ref'].split('/')[-1]
if old_pid != self.pid:
old_pids.add(old_pid)
mef_actions[old_pid] = Action.DELETE
mef.update(data=mef, dbcommit=dbcommit, reindex=reindex)
mef_actions[mef.pid] = Action.DISCARD
# Update first MEF record
mef_record = mef_records[0]
if old_ref := mef_record.get(self.name):
old_pid = old_ref['$ref'].split('/')[-1]
else:
old_pid = None
if old_pid != self.pid:
if old_pid:
old_pids.add(old_pid)
mef_actions[old_pid] = Action.DELETE
mef_record[self.name] = {'$ref': ref_string}
mef_record = mef_record.update(
data=mef_record,
dbcommit=dbcommit,
reindex=reindex
)
mef_actions[mef_record.pid] = Action.UPDATE
else:
if reindex:
mef_record.reindex()
mef_actions[mef_record.pid] = Action.UPTODATE
else:
mef_action = Action.CREATE
# No MEF record create one.
mef_data = {self.name: {'$ref': ref_string}}
if self.deleted:
mef_data['deleted'] = self.deleted
if viaf_records:
mef_data['viaf_pid'] = viaf_records[0].pid
mef_record = AgentMefRecord.create(
data=mef_data,
dbcommit=dbcommit,
reindex=reindex,
)
mef_actions[mef_record.pid] = Action.CREATE
if reindex:
AgentMefRecord.flush_indexes()
return mef_record, mef_action
# create all MEF records for old pids
for old_pid in old_pids:
old_rec = self.get_record_by_pid(old_pid)
mef, action = old_rec.create_or_update_mef(
dbcommit=True,
reindex=True
)
mef_actions[old_pid] = action
return mef_record, mef_actions

@classmethod
def get_online_record(cls, id, debug=False):
Expand Down
2 changes: 1 addition & 1 deletion rero_mef/agents/mef/api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ def add_information(self, resolve=False, sources=False):
:param sources: Add sources information to record
:returns: record
"""
replace_refs_data = deepcopy(self).replace_refs()
replace_refs_data = AgentMefRecord(deepcopy(self).replace_refs())
data = replace_refs_data if resolve else deepcopy(self)
my_sources = []
for agent in self.entities:
Expand Down
42 changes: 42 additions & 0 deletions rero_mef/agents/mef/mappings/v7/mef/mef-v0.0.1.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,20 @@
"identifier": {
"type": "keyword"
},
"identifiedBy": {
"type": "object",
"properties": {
"type": {
"type": "keyword"
},
"source": {
"type": "keyword"
},
"value": {
"type": "keyword"
}
}
},
"date_of_birth": {
"type": "text"
},
Expand Down Expand Up @@ -168,6 +182,20 @@
"identifier": {
"type": "keyword"
},
"identifiedBy": {
"type": "object",
"properties": {
"type": {
"type": "keyword"
},
"source": {
"type": "keyword"
},
"value": {
"type": "keyword"
}
}
},
"date_of_birth": {
"type": "text"
},
Expand Down Expand Up @@ -262,6 +290,20 @@
"identifier": {
"type": "keyword"
},
"identifiedBy": {
"type": "object",
"properties": {
"type": {
"type": "keyword"
},
"source": {
"type": "keyword"
},
"value": {
"type": "keyword"
}
}
},
"date_of_birth": {
"type": "text"
},
Expand Down
Loading
0