diff --git a/cds_migrator_kit/rdm/collection_conf.py b/cds_migrator_kit/rdm/collection_conf.py index 1f5418a3..9bde612c 100644 --- a/cds_migrator_kit/rdm/collection_conf.py +++ b/cds_migrator_kit/rdm/collection_conf.py @@ -32,6 +32,8 @@ ``migration_cli/migrating-a-collection.md`` describes the whole flow. """ +from __future__ import annotations + import os import re import warnings @@ -293,7 +295,7 @@ def legacy_to_model_query(query): return "".join(out) -def conf_query(collection, exclusions="", confs_dir=None): +def conf_query(collections: str | list[str], exclusions="", confs_dir=None): """The ``__query__`` of a model, built from the queries of the collection conf. The queries of the conf (``queries``, each with its ``q``) are translated @@ -313,16 +315,25 @@ def conf_query(collection, exclusions="", confs_dir=None): or default_confs_dir() ) queries = [] - for item in read_conf(collection, confs_dir).get("queries") or []: - text = item.get("q") if isinstance(item, dict) else item - if not text: - raise ValueError( - "A query of `{}` has no text (`q`): {!r}".format(collection, item) - ) - queries.append("({})".format(legacy_to_model_query(text.strip()))) + + if isinstance(collections, str): + collections = [collections] + + for collection in collections: + for item in read_conf(collection, confs_dir).get("queries") or []: + text = item.get("q") if isinstance(item, dict) else item + if not text: + raise ValueError( + "A query of `{}` has no text (`q`): {!r}".format(collection, item) + ) + queries.append("({})".format(legacy_to_model_query(text.strip()))) + if not queries: raise ValueError( - "`queries` is not set for `{}` in {}".format(collection, confs_dir) + "`queries` is not set for any of {} in {}".format( + ", ".join(collections), confs_dir + ) ) + exclusions = " ".join((MODEL_EXCLUSIONS, exclusions.strip())).strip() return "({}) {}".format(" OR ".join(queries), legacy_to_model_query(exclusions)) diff --git a/cds_migrator_kit/rdm/migration_config.py b/cds_migrator_kit/rdm/migration_config.py index 19341209..0ebdc48a 100644 --- a/cds_migrator_kit/rdm/migration_config.py +++ b/cds_migrator_kit/rdm/migration_config.py @@ -656,4 +656,44 @@ def resolve_record_pid(pid): "counter_digits": 3, # zero-padding width, e.g. 3 → "001" }, }, + "742fea6c-0685-4da2-8000-c1f19635ba86": { + # NA58 + "label": "EP approval", # shown in UI buttons/headings + "referee_group": "cds-ph-ep-publications-referee-non-lhc", # CERN e-group slug + "report_number": { + "prefix": "CERN-EP", # literal prefix, e.g. "CERN-EP" + "include_year": True, # append the current year after prefix + "counter_digits": 3, # zero-padding width, e.g. 3 → "001" + }, + }, + "cf52a005-cc57-4a69-a135-771bf9f28d8a": { + # NA61 + "label": "EP approval", # shown in UI buttons/headings + "referee_group": "cds-ph-ep-publications-referee-non-lhc", # CERN e-group slug + "report_number": { + "prefix": "CERN-EP", # literal prefix, e.g. "CERN-EP" + "include_year": True, # append the current year after prefix + "counter_digits": 3, # zero-padding width, e.g. 3 → "001" + }, + }, + "71045612-b90c-4dba-83a5-c5e55d5b0622": { + # NA62 + "label": "EP approval", # shown in UI buttons/headings + "referee_group": "cds-ph-ep-publications-referee-non-lhc", # CERN e-group slug + "report_number": { + "prefix": "CERN-EP", # literal prefix, e.g. "CERN-EP" + "include_year": True, # append the current year after prefix + "counter_digits": 3, # zero-padding width, e.g. 3 → "001" + }, + }, + "25a0fa11-6e27-419a-a00e-3b4b30ef0182": { + # NA64 + "label": "EP approval", # shown in UI buttons/headings + "referee_group": "cds-ph-ep-publications-referee-non-lhc", # CERN e-group slug + "report_number": { + "prefix": "CERN-EP", # literal prefix, e.g. "CERN-EP" + "include_year": True, # append the current year after prefix + "counter_digits": 3, # zero-padding width, e.g. 3 → "001" + }, + }, } diff --git a/cds_migrator_kit/rdm/records/transform/models/_config.py b/cds_migrator_kit/rdm/records/transform/models/_config.py index c63ab62b..14ce6edb 100644 --- a/cds_migrator_kit/rdm/records/transform/models/_config.py +++ b/cds_migrator_kit/rdm/records/transform/models/_config.py @@ -37,6 +37,10 @@ "540__3", # Material of the license "540__9", # Also material of the license "540__g", # From SIS (see decision log) + "541__a", # See decision log + "541__c", # See decision log + "541__d", # See decision log + "541__e", # See decision log "542__3", # Also material of the license "595__i", # From SIS: see decision log "695__9", # From SIS: see decision log diff --git a/cds_migrator_kit/rdm/records/transform/models/clic.py b/cds_migrator_kit/rdm/records/transform/models/clic.py index f505f7f4..f74cd7e4 100644 --- a/cds_migrator_kit/rdm/records/transform/models/clic.py +++ b/cds_migrator_kit/rdm/records/transform/models/clic.py @@ -21,9 +21,7 @@ class ClicResearchModel(CdsOverdo): # the queries of the collection configuration (migration_cli/collections/clic.yaml) # (deleted, hidden and dummy records are always excluded), translated to the # notation of the models - __query__ = conf_query( - "clic", - ) + __query__ = conf_query("clic") # the keys of the research model (the source of truth, see research.py); the fields # ignored only here would be added with `| {...}` diff --git a/cds_migrator_kit/rdm/records/transform/models/clic_restricted.py b/cds_migrator_kit/rdm/records/transform/models/clic_restricted.py index b2c43507..29d3f19f 100644 --- a/cds_migrator_kit/rdm/records/transform/models/clic_restricted.py +++ b/cds_migrator_kit/rdm/records/transform/models/clic_restricted.py @@ -22,9 +22,7 @@ class ClicRestrictedModel(CdsOverdo): # (migration_cli/collections/clic_restr.yaml): the LCD notes that are NOT # approved. Disjoint from the public `clic` model (which requires # `591__b:approved`), so a record matches exactly one of the two. - __query__ = conf_query( - "clic_restr", - ) + __query__ = conf_query("clic_restr") __ignore_keys__ = ResearchModel.__ignore_keys__ diff --git a/cds_migrator_kit/rdm/records/transform/models/isolde.py b/cds_migrator_kit/rdm/records/transform/models/isolde.py index b342d6a3..bcc2078c 100644 --- a/cds_migrator_kit/rdm/records/transform/models/isolde.py +++ b/cds_migrator_kit/rdm/records/transform/models/isolde.py @@ -22,9 +22,7 @@ class ISOLDEModel(CdsOverdo): # the queries of the collection configuration (migration_cli/collections/isolde.yaml) # (deleted, hidden and dummy records are always excluded), translated to the # notation of the models - __query__ = conf_query( - "isolde", - ) + __query__ = conf_query("isolde") __ignore_keys__ = IGNORE_SYSTEM_KEYS | ResearchModel.__ignore_keys__ diff --git a/cds_migrator_kit/rdm/records/transform/models/north_area.py b/cds_migrator_kit/rdm/records/transform/models/north_area.py index 59c5e6e5..93954eff 100644 --- a/cds_migrator_kit/rdm/records/transform/models/north_area.py +++ b/cds_migrator_kit/rdm/records/transform/models/north_area.py @@ -7,6 +7,7 @@ """CDS-RDM North Area models (NA58 & NA61-66).""" +from cds_migrator_kit.rdm.collection_conf import conf_query from cds_migrator_kit.rdm.records.transform.models.base_publication_record import ( rdm_base_publication_model, ) @@ -17,11 +18,10 @@ class NorthAreaModel(CdsOverdo): """Translation model for North Area experiments.""" - __query__ = """ - 693__.e:"COMPASS" OR 693__.e:"COMPASS NA58" OR 693__.e:"NA61" OR 693__.e:"SHINE NA61" OR - 693__.e:"NA62" OR 693__.e:"NA63" OR 693__.e:"NA64" OR 693__.e:"DsTau NA65" OR 693__.e:"AMBER NA66" - -037__:CERN-STUDENTS-Note-* -690C_:SCICOM -980__:PHOTOLAB -980__:THESIS -980__:DELETED -980__:HIDDEN -980__:DUMMY - """ + __query__ = conf_query( + ["na58", "na61", "na62", "na63", "na64", "na65", "na66"], + exclusions="-037:CERN-STUDENTS-Note-* -980:THESIS -980:SCICOMMPUBLSPSC -980:PHOTOLAB", + ) # the keys of the research model (the source of truth, see research.py) plus the # fields ignored only here diff --git a/migration_cli/collections/na58.yaml b/migration_cli/collections/na58.yaml new file mode 100644 index 00000000..281b5ffb --- /dev/null +++ b/migration_cli/collections/na58.yaml @@ -0,0 +1,12 @@ +name: na58 +queries: + - q: '(980:PREPRINT OR 980:ARTICLE OR 980:CONFERENCEPAPER OR 980:SLIDES OR 980:NOTES) AND (693__e:"COMPASS" OR 693__e:"COMPASS NA58")' + dump_file_prefix: na58 +flags: + include_plots: true + create_inclusion_request: true + preferred_model: north_area +# slugs of the communities the records are added to, the same in every environment; +# they must exist in the environment +communities: + - na58 diff --git a/migration_cli/collections/na61.yaml b/migration_cli/collections/na61.yaml index a42041ee..953b2887 100644 --- a/migration_cli/collections/na61.yaml +++ b/migration_cli/collections/na61.yaml @@ -1,9 +1,12 @@ name: na61 -queries: [] # TODO legacy queries of the collection (used by the legacy script only) +queries: + - q: '(980:PREPRINT OR 980:ARTICLE OR 980:CONFERENCEPAPER OR 980:SLIDES OR 980:NOTES) AND (693__e:"NA61" OR 693__e:"SHINE NA61")' + dump_file_prefix: na61 flags: include_plots: true create_inclusion_request: true + preferred_model: north_area # slugs of the communities the records are added to, the same in every environment; # they must exist in the environment communities: - - TODO # slug of the community + - na61 diff --git a/migration_cli/collections/na62.yaml b/migration_cli/collections/na62.yaml index de64827c..3ab03a3f 100644 --- a/migration_cli/collections/na62.yaml +++ b/migration_cli/collections/na62.yaml @@ -1,9 +1,12 @@ name: na62 -queries: [] # TODO legacy queries of the collection (used by the legacy script only) +queries: + - q: "980__a:NA62_Papers OR 980__a:NA62_Reports OR 980__a:NA62_Proc" + dump_file_prefix: na62 flags: include_plots: true create_inclusion_request: true + preferred_model: north_area # slugs of the communities the records are added to, the same in every environment; # they must exist in the environment communities: - - TODO # slug of the community + - na62 diff --git a/migration_cli/collections/na63.yaml b/migration_cli/collections/na63.yaml index 4eabedbf..d5bb025a 100644 --- a/migration_cli/collections/na63.yaml +++ b/migration_cli/collections/na63.yaml @@ -1,9 +1,12 @@ name: na63 -queries: [] # TODO legacy queries of the collection (used by the legacy script only) +queries: + - q: '980:PREPRINT OR 980:ARTICLE OR 980:CONFERENCEPAPER 693__e:"NA63"' + dump_file_prefix: na63 flags: include_plots: true create_inclusion_request: true + preferred_model: north_area # slugs of the communities the records are added to, the same in every environment; # they must exist in the environment communities: - - TODO # slug of the community + - na63 diff --git a/migration_cli/collections/na64.yaml b/migration_cli/collections/na64.yaml index 523e52b3..3297f78a 100644 --- a/migration_cli/collections/na64.yaml +++ b/migration_cli/collections/na64.yaml @@ -1,9 +1,12 @@ name: na64 -queries: [] # TODO legacy queries of the collection (used by the legacy script only) +queries: + - q: '980:PREPRINT OR 980:ARTICLE OR 980:CONFERENCEPAPER OR 980:SLIDES OR 980:NOTES OR 980:POSTER 693__e:"NA64"' + dump_file_prefix: na64 flags: include_plots: true create_inclusion_request: true + preferred_model: north_area # slugs of the communities the records are added to, the same in every environment; # they must exist in the environment communities: - - TODO # slug of the community + - na64 diff --git a/migration_cli/collections/na65.yaml b/migration_cli/collections/na65.yaml index ceda51db..4d075dc4 100644 --- a/migration_cli/collections/na65.yaml +++ b/migration_cli/collections/na65.yaml @@ -1,9 +1,12 @@ name: na65 -queries: [] # TODO legacy queries of the collection (used by the legacy script only) +queries: + - q: '980:PREPRINT OR 980:ARTICLE OR 980:CONFERENCEPAPER OR 980:NOTE 693__e:"DsTau NA65"' + dump_file_prefix: na65 flags: include_plots: true create_inclusion_request: true + preferred_model: north_area # slugs of the communities the records are added to, the same in every environment; # they must exist in the environment communities: - - TODO # slug of the community + - na65 diff --git a/migration_cli/collections/na66.yaml b/migration_cli/collections/na66.yaml index 997d66c6..4e69f8c5 100644 --- a/migration_cli/collections/na66.yaml +++ b/migration_cli/collections/na66.yaml @@ -1,9 +1,12 @@ name: na66 -queries: [] # TODO legacy queries of the collection (used by the legacy script only) +queries: + - q: '980:PREPRINT OR 980:ARTICLE OR 980:CONFERENCEPAPER OR 980:NOTE OR 980:AMBERCERNTALK 693__e:"AMBER NA66"' + dump_file_prefix: na66 flags: include_plots: true create_inclusion_request: true + preferred_model: north_area # slugs of the communities the records are added to, the same in every environment; # they must exist in the environment communities: - - TODO # slug of the community + - na66