Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion cds_migrator_kit/rdm/records/transform/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@

FILE_SUBFORMATS_TO_DROP = ["pdfa", "unstamped"]

CORRECTION_MATERIAL_TERMS = ("erratum", "addendum", "corrigendum")
CORRECTION_MATERIAL_TERMS = ("erratum", "erratum 2", "addendum", "corrigendum")

# Known empty/corrupt legacy PDF stub (Ghostscript zero-page placeholder).
EMPTY_PDF_CHECKSUM = "6f58aa0bfac8a909ac578c1595d55a60"
Expand Down Expand Up @@ -233,6 +233,7 @@
"isoltrap": "IS394",
"is485 (rex-isolde)": "IS485",
"charm": "ea-irrad mixed-field",
"cern elena": "ELENA",
}

# 693__e values that are keywords rather than real experiments (curated as
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
from urllib.parse import ParseResult, urlparse

from dateutil.parser import ParserError, parse
from dojson.errors import IgnoreKey
from dojson.errors import IgnoreItem, IgnoreKey
from dojson.utils import filter_values, flatten, force_list
from idutils.validators import is_arxiv, is_doi, is_handle, is_urn
from invenio_accounts.models import User
Expand Down Expand Up @@ -731,6 +731,9 @@ def licenses(self, key, value):
self["custom_fields"] = _custom_fields

if not license_id:
# From SIS: "f" bronze won't have any 540_a/u/3
if qualifier == "bronze":
raise IgnoreItem()
# 540__f/b without a license id — side effects (funding model, copyright) already applied
raise UnexpectedValue(
"License title missing", field=key, subfield="a", value=value
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -268,19 +268,24 @@ def journal(self, key, value):
# entirely at the DOI level (0247_ $q -> related identifier with
# "issupplementedby"), so this note carries no extra information and is
# ignored. Any other value is surfaced for curation.
is_correction = False
if "m" in value:
allowed_m_values = {"publication", *CORRECTION_MATERIAL_TERMS}
m_values = value.get("m")
if not isinstance(m_values, (list, tuple)):
m_values = (m_values,)
is_correction = True
for m_value in m_values:
if StringValue(m_value).parse().lower() not in allowed_m_values:
parsed = StringValue(m_value).parse().lower()
if parsed not in allowed_m_values:
raise UnexpectedValue(
"Unexpected 773__m value. Record requires manual curation.",
subfield="m",
field=key,
value=value,
)
if parsed not in CORRECTION_MATERIAL_TERMS:
is_correction = False

# A repeated 773__c (dojson returns a tuple) is ambiguous page/session data
# that can't be reduced to a single value - flag for manual curation instead
Expand Down Expand Up @@ -310,6 +315,7 @@ def journal(self, key, value):
# Some old records have slashes instead of hyphens in the INSPIRE conference cnum
conference_cnum = conference_cnum.replace("/", "-")
identifiers.append({"scheme": "inspire", "identifier": conference_cnum})
if identifiers:
new_meeting["identifiers"] = identifiers
if conference_acronym:
new_meeting["acronym"] = conference_acronym
Expand Down Expand Up @@ -341,19 +347,27 @@ def journal(self, key, value):
# Only populate journal fields from a journal 773 (has p/n/v).
# A 773 with only 'c'+'w' is a conference proceedings reference and must
# not overwrite journal data extracted from the sibling journal 773.
# Two journal-like 773s would otherwise silently overwrite title/volume/pages
# (e.g. main publication + erratum); flag for curation instead.
# Two journal-like 773s would otherwise silently overwrite title/volume/pages;
# flag those for curation. An erratum (addendum, corrigendum) 773 is not
# a second publication, so it must not raise or overwrite the journal.
# The subfields are still read: dojson reports any unread subfield as a
# missing rule (LossyConversion).
if is_journal:
if journal_fields:
raise UnexpectedValue(
"Multiple journal 773 fields. Record requires manual curation.",
field=key,
value=value,
)
journal_fields["title"] = StringValue(value.get("p", "")).parse()
journal_fields["issue"] = StringValue(value.get("n", "")).parse()
journal_fields["volume"] = StringValue(value.get("v", "")).parse()
journal_fields["pages"] = StringValue(value.get("c", "")).parse()
title = StringValue(value.get("p", "")).parse()
issue = StringValue(value.get("n", "")).parse()
volume = StringValue(value.get("v", "")).parse()
pages = StringValue(value.get("c", "")).parse()
if not (journal_fields and is_correction):

@zubeydecivelek zubeydecivelek Nov 10, 2026 •

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should not raise if there is erratum and multiple 773 journal field

if journal_fields:
raise UnexpectedValue(
"Multiple journal 773 fields. Record requires manual curation.",
field=key,
value=value,
)
journal_fields["title"] = title
journal_fields["issue"] = issue
journal_fields["volume"] = volume
journal_fields["pages"] = pages
_custom_fields["journal:journal"] = journal_fields
return _custom_fields

Expand Down
2 changes: 1 addition & 1 deletion migration_cli/collections/isolde.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
name: isolde
queries:
- q: '(980:ARTICLE OR 980:PREPRINT OR 980:conferencepaper OR 980:NOTE OR 980:REPORT) AND 693__a:"CERN ISOLDE" -690C_:SCICOM -595:*ARCHIVE* -980:ANNOUNCEMENT -970__b:MERGE-* -980__:HIE-ISOLDE-Project-Notes'

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

with "(" it's dumping 15 records

- q: '980:ARTICLE OR 980:PREPRINT OR 980:ConferencePaper OR 980:NOTE OR 980:REPORT AND 693__:"CERN ISOLDE" -690C_:SCICOM -595:*ARCHIVE* -980__:HIE-ISOLDE-Project-Notes -980:ANNOUNCEMENT -970__b:MERGE-*'
dump_file_prefix: isolde
flags:
include_plots: true
Expand Down
52 changes: 48 additions & 4 deletions tests/cds-rdm/test_publications_rules.py
Original file line number Diff line number Diff line change
Expand Up @@ -420,6 +420,19 @@ def test_773_acronym_only_creates_meeting_without_session(self):
assert result["meeting:meeting"] == [{"acronym": "CHEP"}]
assert result["journal:journal"] == {}

def test_773_url_saved_without_cnum(self):
"""A 773__u is stored when q is present and w is absent."""
result = journal({}, "773__", {"q": "CHEP", "u": "https://indico.cern.ch/e/1"})

assert result["meeting:meeting"] == [
{
"acronym": "CHEP",
"identifiers": [
{"scheme": "URL", "identifier": "https://indico.cern.ch/e/1"}
],
}
]

def test_773_sets_publication_date_if_missing(self):
"""A year in 773 adds a publication date if missing."""
record = {}
Expand Down Expand Up @@ -747,7 +760,7 @@ def test_773_m_repeated_with_one_unknown_raises(self):
journal({}, "773__", {"p": "Nature", "m": ("publication", "bogus")})

def test_multiple_journal_773_raises_for_curation(self):
"""A second journal-like 773 must not overwrite journal:journal."""
"""A second publication 773 must not overwrite journal:journal."""
record = {}
result = journal(
record,
Expand All @@ -760,9 +773,35 @@ def test_multiple_journal_773_raises_for_curation(self):
journal(
record,
"773__",
{"p": "Journal-B", "v": "2", "c": "99", "m": "erratum"},
{"p": "Journal-B", "v": "2", "c": "99"},
)

def test_correction_773_keeps_publication_journal(self):
"""An erratum 773 beside the publication is not a second journal.

p/n/v must still be read. Unread subfields become a LossyConversion
("Migration rule missing") for 773__p, 773__v and 773__n.
"""
from cds_dojson.utils import MementoDict

record = {}
result = journal(
record,
"773__",
{"p": "Journal-A", "v": "1", "c": "10-12", "m": "publication"},
)
record["custom_fields"] = result

erratum = MementoDict(
[("p", "Journal-B"), ("v", "2"), ("n", "4"), ("c", "99"), ("m", "erratum")]
)
result = journal(record, "773__", erratum)

assert result["journal:journal"]["title"] == "Journal-A"
assert result["journal:journal"]["volume"] == "1"
assert result["journal:journal"]["pages"] == "10-12"
assert erratum.not_accessed_keys.isdisjoint({"p", "n", "v", "c", "m"})

def test_773_c_repeated_raises_for_curation(self):
"""A repeated 773__c (a tuple) is ambiguous and flagged for curation.

Expand Down Expand Up @@ -1293,9 +1332,14 @@ def test_funding_model_other(self):
assert self._cf(record)["cern:oa_funding_model"] == {"id": "other"}

def test_bronze_does_not_set_funding_model(self):
"""A 540 with only f=Bronze is ignored and sets no funding model.

SIS: bronze has no 540__a/u/3, so the field is skipped instead of
failing the record for a missing license title.
"""
record = {"custom_fields": {}}
with pytest.raises(UnexpectedValue):
oa_level_from_license(record, "540__", {"f": "Bronze"})
result = oa_level_from_license(record, "540__", {"f": "Bronze"})
assert result == []
assert "cern:oa_funding_model" not in self._cf(record)
assert "cern:oa_level" not in self._cf(record)

Expand Down
Loading