Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -427,15 +427,45 @@ def internal_notes(self, key, value):
@model.over("contributors", "^901__")
@for_each_value
def organisation(self, key, value):
"""Flag a 901__u value for manual curation."""
"""Translate a 901__u affiliation."""
personal_authors = [
person
for group in (self.get("creators", []), self.get("contributors", []))
for person in group
if isinstance(person, dict)
and (person.get("person_or_org") or {}).get("type") == "personal"
]
affiliations = []
for item in force_list(value.get("u", "")):
if item and StringValue(item).parse().strip():
raise UnexpectedValue(
"901__u present. Record requires manual curation.",
field=key,
subfield="u",
value=value,
)
item_parsed = StringValue(item).parse().strip()
if item and item_parsed and item_parsed not in affiliations:
affiliations.append(item_parsed)
if not affiliations:
raise IgnoreKey("contributors")

# One personal author takes every 901 value, ahead of the other cases.
# ``_input_901_count`` is every 901__u on the record, not just this field.
record_901_count = self.get("_input_901_count", len(affiliations))
if len(personal_authors) == 1:
targets = personal_authors
elif len(personal_authors) > 1 and record_901_count == 1:
targets = personal_authors
else:
contributors = self.setdefault("contributors", [])
for affiliation in affiliations:
contributor = {
"person_or_org": {"type": "organizational", "name": affiliation},
"role": {"id": "other"},
}
if contributor not in contributors:
contributors.append(contributor)
raise IgnoreKey("contributors")

for person in targets:
person_affiliations = person.setdefault("affiliations", [])
for affiliation in affiliations:
if affiliation not in person_affiliations:
person_affiliations.append(affiliation)
raise IgnoreKey("contributors")


Expand Down
18 changes: 18 additions & 0 deletions cds_migrator_kit/transform/overdo.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,19 @@
from dojson.utils import GroupableOrderedDict


def _count_901_values(items):
"""Count non-empty 901__u values across every 901 field."""
total = 0
for key, value in items:
if not str(key).startswith("901__") or not isinstance(value, dict):
continue
subfields = value.get("u") or ""
if not isinstance(subfields, (list, tuple, set)):
subfields = (subfields,)
total += sum(1 for subfield in subfields if str(subfield).strip())
return total


class CdsOverdo(Overdo):
"""Overwrite API of Overdo dojson class."""

Expand Down Expand Up @@ -80,6 +93,10 @@ def clean_missing(exc, output, key, value, rectype=None):
else:
items = iteritems(blob)
items = sorted(items, key=lambda item: item[0])
# 901 fields are applied one at a time, so a rule cannot tell a single
# 901 from several by looking at the current value. Count every 901__u
# on the record before the rules run.
output["_input_901_count"] = _count_901_values(items)
for key, value in items:
# Skip completely empty datafields that have no subfields
# Some records might have these for unknown reasons (e.g. https://cds.cern.ch/record/2964767/export/xm?ln=en)
Expand All @@ -104,4 +121,5 @@ def clean_missing(exc, output, key, value, rectype=None):
handler(exc, output, key, value)
else:
raise exc
output.pop("_input_901_count", None)
return output
89 changes: 89 additions & 0 deletions tests/cds-rdm/test_publications_rules.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,11 @@
issn,
journal,
meeting,
organisation,
resource_type,
udc,
)
from cds_migrator_kit.transform.overdo import _count_901_values


class TestIsbn:
Expand Down Expand Up @@ -1410,3 +1412,90 @@ def test_697C_lexi_adds_subject_instead_of_resource_type(self):
resource_type(record, "697C_", {"a": "lexisomething"})
assert record["subjects"] == [{"subject": "lexisomething"}]
assert "resource_type" not in record


def _person(name, affiliations=None):
person = {"person_or_org": {"type": "personal", "family_name": name, "name": name}}
if affiliations is not None:
person["affiliations"] = affiliations
return person


class TestOrganisation901:
"""901__u affiliation handling across every 901 field on the record."""

def test_one_personal_author_gets_every_901_value(self):
author = _person("Smith")
record = {"creators": [author], "_input_901_count": 2}
with pytest.raises(IgnoreKey):
organisation(record, "901__", {"u": ["ETH Zurich", "INFN"]})
assert author["affiliations"] == ["ETH Zurich", "INFN"]
assert "contributors" not in record

def test_one_personal_contributor_gets_the_901_value(self):
author = _person("Smith")
record = {"contributors": [author], "_input_901_count": 1}
with pytest.raises(IgnoreKey):
organisation(record, "901__", {"u": "CERN"})
assert author["affiliations"] == ["CERN"]

def test_several_authors_share_the_single_901_on_the_record(self):
smith = _person("Smith")
jones = _person("Jones")
record = {
"creators": [smith],
"contributors": [jones],
"_input_901_count": 1,
}
with pytest.raises(IgnoreKey):
organisation(record, "901__", {"u": "ETH Zurich"})
assert smith["affiliations"] == ["ETH Zurich"]
assert jones["affiliations"] == ["ETH Zurich"]

def test_several_authors_and_several_901_fields_become_contributors(self):
"""This field has one value, but another 901 exists on the record."""
smith = _person("Smith")
jones = _person("Jones")
record = {
"creators": [smith, jones],
"_input_901_count": 2,
}
with pytest.raises(IgnoreKey):
organisation(record, "901__", {"u": "ETH Zurich"})
assert "affiliations" not in smith
assert "affiliations" not in jones
assert record["contributors"] == [
{
"person_or_org": {"type": "organizational", "name": "ETH Zurich"},
"role": {"id": "other"},
}
]

def test_no_personal_author_becomes_contributor(self):
record = {"_input_901_count": 1}
with pytest.raises(IgnoreKey):
organisation(record, "901__", {"u": "ETH Zurich"})
assert record["contributors"] == [
{
"person_or_org": {"type": "organizational", "name": "ETH Zurich"},
"role": {"id": "other"},
}
]

def test_empty_901_is_ignored(self):
author = _person("Smith")
record = {"creators": [author]}
with pytest.raises(IgnoreKey):
organisation(record, "901__", {"u": " "})
assert "affiliations" not in author
assert "contributors" not in record

def test_count_includes_every_901_field(self):
items = [
("100__", {"a": "Smith, J"}),
("901__", {"u": "ETH Zurich"}),
("901__", {"u": ("CERN", "INFN")}),
("901__", {"u": " "}),
("906__", {"p": "Reviewer"}),
]
assert _count_901_values(items) == 3
Loading