Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 20 additions & 9 deletions cds_migrator_kit/rdm/collection_conf.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@
``migration_cli/migrating-a-collection.md`` describes the whole flow.
"""

from __future__ import annotations

import os
import re
import warnings
Expand Down Expand Up @@ -293,7 +295,7 @@ def legacy_to_model_query(query):
return "".join(out)


def conf_query(collection, exclusions="", confs_dir=None):
def conf_query(collections: str | list[str], exclusions="", confs_dir=None):
"""The ``__query__`` of a model, built from the queries of the collection conf.

The queries of the conf (``queries``, each with its ``q``) are translated
Expand All @@ -313,16 +315,25 @@ def conf_query(collection, exclusions="", confs_dir=None):
or default_confs_dir()
)
queries = []
for item in read_conf(collection, confs_dir).get("queries") or []:
text = item.get("q") if isinstance(item, dict) else item
if not text:
raise ValueError(
"A query of `{}` has no text (`q`): {!r}".format(collection, item)
)
queries.append("({})".format(legacy_to_model_query(text.strip())))

if isinstance(collections, str):
collections = [collections]

for collection in collections:
for item in read_conf(collection, confs_dir).get("queries") or []:
text = item.get("q") if isinstance(item, dict) else item
if not text:
raise ValueError(
"A query of `{}` has no text (`q`): {!r}".format(collection, item)
)
queries.append("({})".format(legacy_to_model_query(text.strip())))

if not queries:
raise ValueError(
"`queries` is not set for `{}` in {}".format(collection, confs_dir)
"`queries` is not set for any of {} in {}".format(
", ".join(collections), confs_dir
)
)

exclusions = " ".join((MODEL_EXCLUSIONS, exclusions.strip())).strip()
return "({}) {}".format(" OR ".join(queries), legacy_to_model_query(exclusions))
40 changes: 40 additions & 0 deletions cds_migrator_kit/rdm/migration_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -656,4 +656,44 @@ def resolve_record_pid(pid):
"counter_digits": 3, # zero-padding width, e.g. 3 → "001"
},
},
"742fea6c-0685-4da2-8000-c1f19635ba86": {
# NA58
"label": "EP approval", # shown in UI buttons/headings
"referee_group": "cds-ph-ep-publications-referee-non-lhc", # CERN e-group slug
"report_number": {
"prefix": "CERN-EP", # literal prefix, e.g. "CERN-EP"
"include_year": True, # append the current year after prefix
"counter_digits": 3, # zero-padding width, e.g. 3 → "001"
},
},
"cf52a005-cc57-4a69-a135-771bf9f28d8a": {
# NA61
"label": "EP approval", # shown in UI buttons/headings
"referee_group": "cds-ph-ep-publications-referee-non-lhc", # CERN e-group slug
"report_number": {
"prefix": "CERN-EP", # literal prefix, e.g. "CERN-EP"
"include_year": True, # append the current year after prefix
"counter_digits": 3, # zero-padding width, e.g. 3 → "001"
},
},
"71045612-b90c-4dba-83a5-c5e55d5b0622": {
# NA62
"label": "EP approval", # shown in UI buttons/headings
"referee_group": "cds-ph-ep-publications-referee-non-lhc", # CERN e-group slug
"report_number": {
"prefix": "CERN-EP", # literal prefix, e.g. "CERN-EP"
"include_year": True, # append the current year after prefix
"counter_digits": 3, # zero-padding width, e.g. 3 → "001"
},
},
"25a0fa11-6e27-419a-a00e-3b4b30ef0182": {
# NA64
"label": "EP approval", # shown in UI buttons/headings
"referee_group": "cds-ph-ep-publications-referee-non-lhc", # CERN e-group slug
"report_number": {
"prefix": "CERN-EP", # literal prefix, e.g. "CERN-EP"
"include_year": True, # append the current year after prefix
"counter_digits": 3, # zero-padding width, e.g. 3 → "001"
},
},
}
4 changes: 4 additions & 0 deletions cds_migrator_kit/rdm/records/transform/models/_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,10 @@
"540__3", # Material of the license
"540__9", # Also material of the license
"540__g", # From SIS (see decision log)
"541__a", # See decision log
"541__c", # See decision log
"541__d", # See decision log
"541__e", # See decision log
"542__3", # Also material of the license
"595__i", # From SIS: see decision log
"695__9", # From SIS: see decision log
Expand Down
4 changes: 1 addition & 3 deletions cds_migrator_kit/rdm/records/transform/models/clic.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,7 @@ class ClicResearchModel(CdsOverdo):
# the queries of the collection configuration (migration_cli/collections/clic.yaml)
# (deleted, hidden and dummy records are always excluded), translated to the
# notation of the models
__query__ = conf_query(
"clic",
)
__query__ = conf_query("clic")

# the keys of the research model (the source of truth, see research.py); the fields
# ignored only here would be added with `| {...}`
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,7 @@ class ClicRestrictedModel(CdsOverdo):
# (migration_cli/collections/clic_restr.yaml): the LCD notes that are NOT
# approved. Disjoint from the public `clic` model (which requires
# `591__b:approved`), so a record matches exactly one of the two.
__query__ = conf_query(
"clic_restr",
)
__query__ = conf_query("clic_restr")

__ignore_keys__ = ResearchModel.__ignore_keys__

Expand Down
4 changes: 1 addition & 3 deletions cds_migrator_kit/rdm/records/transform/models/isolde.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,7 @@ class ISOLDEModel(CdsOverdo):
# the queries of the collection configuration (migration_cli/collections/isolde.yaml)
# (deleted, hidden and dummy records are always excluded), translated to the
# notation of the models
__query__ = conf_query(
"isolde",
)
__query__ = conf_query("isolde")

__ignore_keys__ = IGNORE_SYSTEM_KEYS | ResearchModel.__ignore_keys__

Expand Down
10 changes: 5 additions & 5 deletions cds_migrator_kit/rdm/records/transform/models/north_area.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

"""CDS-RDM North Area models (NA58 & NA61-66)."""

from cds_migrator_kit.rdm.collection_conf import conf_query
from cds_migrator_kit.rdm.records.transform.models.base_publication_record import (
rdm_base_publication_model,
)
Expand All @@ -17,11 +18,10 @@
class NorthAreaModel(CdsOverdo):
"""Translation model for North Area experiments."""

__query__ = """
693__.e:"COMPASS" OR 693__.e:"COMPASS NA58" OR 693__.e:"NA61" OR 693__.e:"SHINE NA61" OR
693__.e:"NA62" OR 693__.e:"NA63" OR 693__.e:"NA64" OR 693__.e:"DsTau NA65" OR 693__.e:"AMBER NA66"
-037__:CERN-STUDENTS-Note-* -690C_:SCICOM -980__:PHOTOLAB -980__:THESIS -980__:DELETED -980__:HIDDEN -980__:DUMMY
"""
__query__ = conf_query(
["na58", "na61", "na62", "na63", "na64", "na65", "na66"],
exclusions="-037:CERN-STUDENTS-Note-* -980:THESIS -980:SCICOMMPUBLSPSC -980:PHOTOLAB",
)

# the keys of the research model (the source of truth, see research.py) plus the
# fields ignored only here
Expand Down
12 changes: 12 additions & 0 deletions migration_cli/collections/na58.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
name: na58
queries:
- q: '(980:PREPRINT OR 980:ARTICLE OR 980:CONFERENCEPAPER OR 980:SLIDES OR 980:NOTES) AND (693__e:"COMPASS" OR 693__e:"COMPASS NA58")'
dump_file_prefix: na58
flags:
include_plots: true
create_inclusion_request: true
preferred_model: north_area
# slugs of the communities the records are added to, the same in every environment;
# they must exist in the environment
communities:
- na58
7 changes: 5 additions & 2 deletions migration_cli/collections/na61.yaml
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
name: na61
queries: [] # TODO legacy queries of the collection (used by the legacy script only)
queries:
- q: '(980:PREPRINT OR 980:ARTICLE OR 980:CONFERENCEPAPER OR 980:SLIDES OR 980:NOTES) AND (693__e:"NA61" OR 693__e:"SHINE NA61")'
dump_file_prefix: na61
flags:
include_plots: true
create_inclusion_request: true
preferred_model: north_area
# slugs of the communities the records are added to, the same in every environment;
# they must exist in the environment
communities:
- TODO # slug of the community
- na61
7 changes: 5 additions & 2 deletions migration_cli/collections/na62.yaml
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
name: na62
queries: [] # TODO legacy queries of the collection (used by the legacy script only)
queries:
- q: "980__a:NA62_Papers OR 980__a:NA62_Reports OR 980__a:NA62_Proc"
dump_file_prefix: na62
flags:
include_plots: true
create_inclusion_request: true
preferred_model: north_area
# slugs of the communities the records are added to, the same in every environment;
# they must exist in the environment
communities:
- TODO # slug of the community
- na62
7 changes: 5 additions & 2 deletions migration_cli/collections/na63.yaml
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
name: na63
queries: [] # TODO legacy queries of the collection (used by the legacy script only)
queries:
- q: '980:PREPRINT OR 980:ARTICLE OR 980:CONFERENCEPAPER 693__e:"NA63"'
dump_file_prefix: na63
flags:
include_plots: true
create_inclusion_request: true
preferred_model: north_area
# slugs of the communities the records are added to, the same in every environment;
# they must exist in the environment
communities:
- TODO # slug of the community
- na63
7 changes: 5 additions & 2 deletions migration_cli/collections/na64.yaml
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
name: na64
queries: [] # TODO legacy queries of the collection (used by the legacy script only)
queries:
- q: '980:PREPRINT OR 980:ARTICLE OR 980:CONFERENCEPAPER OR 980:SLIDES OR 980:NOTES OR 980:POSTER 693__e:"NA64"'
dump_file_prefix: na64
flags:
include_plots: true
create_inclusion_request: true
preferred_model: north_area
# slugs of the communities the records are added to, the same in every environment;
# they must exist in the environment
communities:
- TODO # slug of the community
- na64
7 changes: 5 additions & 2 deletions migration_cli/collections/na65.yaml
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
name: na65
queries: [] # TODO legacy queries of the collection (used by the legacy script only)
queries:
- q: '980:PREPRINT OR 980:ARTICLE OR 980:CONFERENCEPAPER OR 980:NOTE 693__e:"DsTau NA65"'
dump_file_prefix: na65
flags:
include_plots: true
create_inclusion_request: true
preferred_model: north_area
# slugs of the communities the records are added to, the same in every environment;
# they must exist in the environment
communities:
- TODO # slug of the community
- na65
7 changes: 5 additions & 2 deletions migration_cli/collections/na66.yaml
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
name: na66
queries: [] # TODO legacy queries of the collection (used by the legacy script only)
queries:
- q: '980:PREPRINT OR 980:ARTICLE OR 980:CONFERENCEPAPER OR 980:NOTE OR 980:AMBERCERNTALK 693__e:"AMBER NA66"'
dump_file_prefix: na66
flags:
include_plots: true
create_inclusion_request: true
preferred_model: north_area
# slugs of the communities the records are added to, the same in every environment;
# they must exist in the environment
communities:
- TODO # slug of the community
- na66
Loading