diff --git a/cds_migrator_kit/rdm/records/transform/config.py b/cds_migrator_kit/rdm/records/transform/config.py index cde82a34..6bb5b6f2 100644 --- a/cds_migrator_kit/rdm/records/transform/config.py +++ b/cds_migrator_kit/rdm/records/transform/config.py @@ -159,7 +159,7 @@ FILE_SUBFORMATS_TO_DROP = ["pdfa", "unstamped"] -CORRECTION_MATERIAL_TERMS = ("erratum", "addendum", "corrigendum") +CORRECTION_MATERIAL_TERMS = ("erratum", "erratum 2", "addendum", "corrigendum") # Known empty/corrupt legacy PDF stub (Ghostscript zero-page placeholder). EMPTY_PDF_CHECKSUM = "6f58aa0bfac8a909ac578c1595d55a60" @@ -233,6 +233,7 @@ "isoltrap": "IS394", "is485 (rex-isolde)": "IS485", "charm": "ea-irrad mixed-field", + "cern elena": "ELENA", } # 693__e values that are keywords rather than real experiments (curated as diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py index 3fffe6a5..f390bc9f 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py @@ -13,7 +13,7 @@ from urllib.parse import ParseResult, urlparse from dateutil.parser import ParserError, parse -from dojson.errors import IgnoreKey +from dojson.errors import IgnoreItem, IgnoreKey from dojson.utils import filter_values, flatten, force_list from idutils.validators import is_arxiv, is_doi, is_handle, is_urn from invenio_accounts.models import User @@ -731,6 +731,9 @@ def licenses(self, key, value): self["custom_fields"] = _custom_fields if not license_id: + # From SIS: "f" bronze won't have any 540_a/u/3 + if qualifier == "bronze": + raise IgnoreItem() # 540__f/b without a license id — side effects (funding model, copyright) already applied raise UnexpectedValue( "License title missing", field=key, subfield="a", value=value diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py index f94ebc86..158d254b 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py @@ -268,19 +268,24 @@ def journal(self, key, value): # entirely at the DOI level (0247_ $q -> related identifier with # "issupplementedby"), so this note carries no extra information and is # ignored. Any other value is surfaced for curation. + is_correction = False if "m" in value: allowed_m_values = {"publication", *CORRECTION_MATERIAL_TERMS} m_values = value.get("m") if not isinstance(m_values, (list, tuple)): m_values = (m_values,) + is_correction = True for m_value in m_values: - if StringValue(m_value).parse().lower() not in allowed_m_values: + parsed = StringValue(m_value).parse().lower() + if parsed not in allowed_m_values: raise UnexpectedValue( "Unexpected 773__m value. Record requires manual curation.", subfield="m", field=key, value=value, ) + if parsed not in CORRECTION_MATERIAL_TERMS: + is_correction = False # A repeated 773__c (dojson returns a tuple) is ambiguous page/session data # that can't be reduced to a single value - flag for manual curation instead @@ -310,6 +315,7 @@ def journal(self, key, value): # Some old records have slashes instead of hyphens in the INSPIRE conference cnum conference_cnum = conference_cnum.replace("/", "-") identifiers.append({"scheme": "inspire", "identifier": conference_cnum}) + if identifiers: new_meeting["identifiers"] = identifiers if conference_acronym: new_meeting["acronym"] = conference_acronym @@ -341,19 +347,27 @@ def journal(self, key, value): # Only populate journal fields from a journal 773 (has p/n/v). # A 773 with only 'c'+'w' is a conference proceedings reference and must # not overwrite journal data extracted from the sibling journal 773. - # Two journal-like 773s would otherwise silently overwrite title/volume/pages - # (e.g. main publication + erratum); flag for curation instead. + # Two journal-like 773s would otherwise silently overwrite title/volume/pages; + # flag those for curation. An erratum (addendum, corrigendum) 773 is not + # a second publication, so it must not raise or overwrite the journal. + # The subfields are still read: dojson reports any unread subfield as a + # missing rule (LossyConversion). if is_journal: - if journal_fields: - raise UnexpectedValue( - "Multiple journal 773 fields. Record requires manual curation.", - field=key, - value=value, - ) - journal_fields["title"] = StringValue(value.get("p", "")).parse() - journal_fields["issue"] = StringValue(value.get("n", "")).parse() - journal_fields["volume"] = StringValue(value.get("v", "")).parse() - journal_fields["pages"] = StringValue(value.get("c", "")).parse() + title = StringValue(value.get("p", "")).parse() + issue = StringValue(value.get("n", "")).parse() + volume = StringValue(value.get("v", "")).parse() + pages = StringValue(value.get("c", "")).parse() + if not (journal_fields and is_correction): + if journal_fields: + raise UnexpectedValue( + "Multiple journal 773 fields. Record requires manual curation.", + field=key, + value=value, + ) + journal_fields["title"] = title + journal_fields["issue"] = issue + journal_fields["volume"] = volume + journal_fields["pages"] = pages _custom_fields["journal:journal"] = journal_fields return _custom_fields diff --git a/migration_cli/collections/isolde.yaml b/migration_cli/collections/isolde.yaml index c2793a76..b35232f2 100644 --- a/migration_cli/collections/isolde.yaml +++ b/migration_cli/collections/isolde.yaml @@ -1,6 +1,6 @@ name: isolde queries: - - q: '(980:ARTICLE OR 980:PREPRINT OR 980:conferencepaper OR 980:NOTE OR 980:REPORT) AND 693__a:"CERN ISOLDE" -690C_:SCICOM -595:*ARCHIVE* -980:ANNOUNCEMENT -970__b:MERGE-* -980__:HIE-ISOLDE-Project-Notes' + - q: '980:ARTICLE OR 980:PREPRINT OR 980:ConferencePaper OR 980:NOTE OR 980:REPORT AND 693__:"CERN ISOLDE" -690C_:SCICOM -595:*ARCHIVE* -980__:HIE-ISOLDE-Project-Notes -980:ANNOUNCEMENT -970__b:MERGE-*' dump_file_prefix: isolde flags: include_plots: true diff --git a/tests/cds-rdm/test_publications_rules.py b/tests/cds-rdm/test_publications_rules.py index bb963e5f..8a935182 100644 --- a/tests/cds-rdm/test_publications_rules.py +++ b/tests/cds-rdm/test_publications_rules.py @@ -420,6 +420,19 @@ def test_773_acronym_only_creates_meeting_without_session(self): assert result["meeting:meeting"] == [{"acronym": "CHEP"}] assert result["journal:journal"] == {} + def test_773_url_saved_without_cnum(self): + """A 773__u is stored when q is present and w is absent.""" + result = journal({}, "773__", {"q": "CHEP", "u": "https://indico.cern.ch/e/1"}) + + assert result["meeting:meeting"] == [ + { + "acronym": "CHEP", + "identifiers": [ + {"scheme": "URL", "identifier": "https://indico.cern.ch/e/1"} + ], + } + ] + def test_773_sets_publication_date_if_missing(self): """A year in 773 adds a publication date if missing.""" record = {} @@ -747,7 +760,7 @@ def test_773_m_repeated_with_one_unknown_raises(self): journal({}, "773__", {"p": "Nature", "m": ("publication", "bogus")}) def test_multiple_journal_773_raises_for_curation(self): - """A second journal-like 773 must not overwrite journal:journal.""" + """A second publication 773 must not overwrite journal:journal.""" record = {} result = journal( record, @@ -760,9 +773,35 @@ def test_multiple_journal_773_raises_for_curation(self): journal( record, "773__", - {"p": "Journal-B", "v": "2", "c": "99", "m": "erratum"}, + {"p": "Journal-B", "v": "2", "c": "99"}, ) + def test_correction_773_keeps_publication_journal(self): + """An erratum 773 beside the publication is not a second journal. + + p/n/v must still be read. Unread subfields become a LossyConversion + ("Migration rule missing") for 773__p, 773__v and 773__n. + """ + from cds_dojson.utils import MementoDict + + record = {} + result = journal( + record, + "773__", + {"p": "Journal-A", "v": "1", "c": "10-12", "m": "publication"}, + ) + record["custom_fields"] = result + + erratum = MementoDict( + [("p", "Journal-B"), ("v", "2"), ("n", "4"), ("c", "99"), ("m", "erratum")] + ) + result = journal(record, "773__", erratum) + + assert result["journal:journal"]["title"] == "Journal-A" + assert result["journal:journal"]["volume"] == "1" + assert result["journal:journal"]["pages"] == "10-12" + assert erratum.not_accessed_keys.isdisjoint({"p", "n", "v", "c", "m"}) + def test_773_c_repeated_raises_for_curation(self): """A repeated 773__c (a tuple) is ambiguous and flagged for curation. @@ -1293,9 +1332,14 @@ def test_funding_model_other(self): assert self._cf(record)["cern:oa_funding_model"] == {"id": "other"} def test_bronze_does_not_set_funding_model(self): + """A 540 with only f=Bronze is ignored and sets no funding model. + + SIS: bronze has no 540__a/u/3, so the field is skipped instead of + failing the record for a missing license title. + """ record = {"custom_fields": {}} - with pytest.raises(UnexpectedValue): - oa_level_from_license(record, "540__", {"f": "Bronze"}) + result = oa_level_from_license(record, "540__", {"f": "Bronze"}) + assert result == [] assert "cern:oa_funding_model" not in self._cf(record) assert "cern:oa_level" not in self._cf(record)