From 3d61fb130c00bb39295e36128570a0eee88cb5df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Z=C3=BCbeyde=20Civelek?= Date: Thu, 5 Nov 2026 14:01:30 +0100 Subject: [PATCH] feat(research): fix rule 901__u --- .../xml_processing/rules/research.py | 46 ++++++++-- cds_migrator_kit/transform/overdo.py | 18 ++++ tests/cds-rdm/test_publications_rules.py | 89 +++++++++++++++++++ 3 files changed, 145 insertions(+), 8 deletions(-) diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py index f94ebc86..91dc48e6 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py @@ -427,15 +427,45 @@ def internal_notes(self, key, value): @model.over("contributors", "^901__") @for_each_value def organisation(self, key, value): - """Flag a 901__u value for manual curation.""" + """Translate a 901__u affiliation.""" + personal_authors = [ + person + for group in (self.get("creators", []), self.get("contributors", [])) + for person in group + if isinstance(person, dict) + and (person.get("person_or_org") or {}).get("type") == "personal" + ] + affiliations = [] for item in force_list(value.get("u", "")): - if item and StringValue(item).parse().strip(): - raise UnexpectedValue( - "901__u present. Record requires manual curation.", - field=key, - subfield="u", - value=value, - ) + item_parsed = StringValue(item).parse().strip() + if item and item_parsed and item_parsed not in affiliations: + affiliations.append(item_parsed) + if not affiliations: + raise IgnoreKey("contributors") + + # One personal author takes every 901 value, ahead of the other cases. + # ``_input_901_count`` is every 901__u on the record, not just this field. + record_901_count = self.get("_input_901_count", len(affiliations)) + if len(personal_authors) == 1: + targets = personal_authors + elif len(personal_authors) > 1 and record_901_count == 1: + targets = personal_authors + else: + contributors = self.setdefault("contributors", []) + for affiliation in affiliations: + contributor = { + "person_or_org": {"type": "organizational", "name": affiliation}, + "role": {"id": "other"}, + } + if contributor not in contributors: + contributors.append(contributor) + raise IgnoreKey("contributors") + + for person in targets: + person_affiliations = person.setdefault("affiliations", []) + for affiliation in affiliations: + if affiliation not in person_affiliations: + person_affiliations.append(affiliation) raise IgnoreKey("contributors") diff --git a/cds_migrator_kit/transform/overdo.py b/cds_migrator_kit/transform/overdo.py index 6aef1bce..da94d134 100644 --- a/cds_migrator_kit/transform/overdo.py +++ b/cds_migrator_kit/transform/overdo.py @@ -15,6 +15,19 @@ from dojson.utils import GroupableOrderedDict +def _count_901_values(items): + """Count non-empty 901__u values across every 901 field.""" + total = 0 + for key, value in items: + if not str(key).startswith("901__") or not isinstance(value, dict): + continue + subfields = value.get("u") or "" + if not isinstance(subfields, (list, tuple, set)): + subfields = (subfields,) + total += sum(1 for subfield in subfields if str(subfield).strip()) + return total + + class CdsOverdo(Overdo): """Overwrite API of Overdo dojson class.""" @@ -80,6 +93,10 @@ def clean_missing(exc, output, key, value, rectype=None): else: items = iteritems(blob) items = sorted(items, key=lambda item: item[0]) + # 901 fields are applied one at a time, so a rule cannot tell a single + # 901 from several by looking at the current value. Count every 901__u + # on the record before the rules run. + output["_input_901_count"] = _count_901_values(items) for key, value in items: # Skip completely empty datafields that have no subfields # Some records might have these for unknown reasons (e.g. https://cds.cern.ch/record/2964767/export/xm?ln=en) @@ -104,4 +121,5 @@ def clean_missing(exc, output, key, value, rectype=None): handler(exc, output, key, value) else: raise exc + output.pop("_input_901_count", None) return output diff --git a/tests/cds-rdm/test_publications_rules.py b/tests/cds-rdm/test_publications_rules.py index bb963e5f..37ea4cef 100644 --- a/tests/cds-rdm/test_publications_rules.py +++ b/tests/cds-rdm/test_publications_rules.py @@ -23,9 +23,11 @@ issn, journal, meeting, + organisation, resource_type, udc, ) +from cds_migrator_kit.transform.overdo import _count_901_values class TestIsbn: @@ -1410,3 +1412,90 @@ def test_697C_lexi_adds_subject_instead_of_resource_type(self): resource_type(record, "697C_", {"a": "lexisomething"}) assert record["subjects"] == [{"subject": "lexisomething"}] assert "resource_type" not in record + + +def _person(name, affiliations=None): + person = {"person_or_org": {"type": "personal", "family_name": name, "name": name}} + if affiliations is not None: + person["affiliations"] = affiliations + return person + + +class TestOrganisation901: + """901__u affiliation handling across every 901 field on the record.""" + + def test_one_personal_author_gets_every_901_value(self): + author = _person("Smith") + record = {"creators": [author], "_input_901_count": 2} + with pytest.raises(IgnoreKey): + organisation(record, "901__", {"u": ["ETH Zurich", "INFN"]}) + assert author["affiliations"] == ["ETH Zurich", "INFN"] + assert "contributors" not in record + + def test_one_personal_contributor_gets_the_901_value(self): + author = _person("Smith") + record = {"contributors": [author], "_input_901_count": 1} + with pytest.raises(IgnoreKey): + organisation(record, "901__", {"u": "CERN"}) + assert author["affiliations"] == ["CERN"] + + def test_several_authors_share_the_single_901_on_the_record(self): + smith = _person("Smith") + jones = _person("Jones") + record = { + "creators": [smith], + "contributors": [jones], + "_input_901_count": 1, + } + with pytest.raises(IgnoreKey): + organisation(record, "901__", {"u": "ETH Zurich"}) + assert smith["affiliations"] == ["ETH Zurich"] + assert jones["affiliations"] == ["ETH Zurich"] + + def test_several_authors_and_several_901_fields_become_contributors(self): + """This field has one value, but another 901 exists on the record.""" + smith = _person("Smith") + jones = _person("Jones") + record = { + "creators": [smith, jones], + "_input_901_count": 2, + } + with pytest.raises(IgnoreKey): + organisation(record, "901__", {"u": "ETH Zurich"}) + assert "affiliations" not in smith + assert "affiliations" not in jones + assert record["contributors"] == [ + { + "person_or_org": {"type": "organizational", "name": "ETH Zurich"}, + "role": {"id": "other"}, + } + ] + + def test_no_personal_author_becomes_contributor(self): + record = {"_input_901_count": 1} + with pytest.raises(IgnoreKey): + organisation(record, "901__", {"u": "ETH Zurich"}) + assert record["contributors"] == [ + { + "person_or_org": {"type": "organizational", "name": "ETH Zurich"}, + "role": {"id": "other"}, + } + ] + + def test_empty_901_is_ignored(self): + author = _person("Smith") + record = {"creators": [author]} + with pytest.raises(IgnoreKey): + organisation(record, "901__", {"u": " "}) + assert "affiliations" not in author + assert "contributors" not in record + + def test_count_includes_every_901_field(self): + items = [ + ("100__", {"a": "Smith, J"}), + ("901__", {"u": "ETH Zurich"}), + ("901__", {"u": ("CERN", "INFN")}), + ("901__", {"u": " "}), + ("906__", {"p": "Reviewer"}), + ] + assert _count_901_values(items) == 3