diff --git a/changelog.d/+hardware-schema-text.bugfix.md b/changelog.d/+hardware-schema-text.bugfix.md new file mode 100644 index 0000000..98490ac --- /dev/null +++ b/changelog.d/+hardware-schema-text.bugfix.md @@ -0,0 +1 @@ +`eb-pipeline schema` says which commands get the hardware class of `[targets.cpu_hardware]`: the source prefetch and build commands of each line, which build for one class of node, as they always have. It said that every command did, and did not say that `[targets.hardware]` is exported only when `cpu_hardware` is set. diff --git a/changelog.d/+module-names-as-written.bugfix.md b/changelog.d/+module-names-as-written.bugfix.md new file mode 100644 index 0000000..1c664ff --- /dev/null +++ b/changelog.d/+module-names-as-written.bugfix.md @@ -0,0 +1 @@ +The module wrapper gives each name of `modules.init` to `module load` as written. It split `EB_PIPELINE_MODULES` with pathname expansion on, so a name with a `*`, `?` or `[` that matched files, in the directory the command started in or under an absolute path, loaded the names of those files instead. `modules.shell` is documented as a POSIX shell, such as `bash` or `sh`, which the wrapper has always needed. diff --git a/changelog.d/+shares-placeholders-text.bugfix.md b/changelog.d/+shares-placeholders-text.bugfix.md new file mode 100644 index 0000000..ce62543 --- /dev/null +++ b/changelog.d/+shares-placeholders-text.bugfix.md @@ -0,0 +1 @@ +`eb-pipeline schema` lists all five placeholders that `targets.route.shares_ro` and `shares_rw` take: `{phase_dir}`, `{log_dir}`, `{work_dir}`, `{target}` and `{phase}`. It left out `{target}` and `{phase}`, which `check` accepts and a run replaces in the shares as it does in the route's `args`. diff --git a/changelog.d/+site-file-variable.removal.md b/changelog.d/+site-file-variable.removal.md new file mode 100644 index 0000000..5730dce --- /dev/null +++ b/changelog.d/+site-file-variable.removal.md @@ -0,0 +1 @@ +The commands of a run get the site file in `EB_PIPELINE_SITE`, as an absolute path, and the site name in the new `EB_PIPELINE_SITE_NAME`; `explain` and the plan show both. `EB_PIPELINE_SITE` held the name, so an `eb-pipeline` that a command started without `--site` tried to read a file of that name and exited 2. A command that reads the site name from `EB_PIPELINE_SITE` must read `EB_PIPELINE_SITE_NAME` instead. In the library, `resolve::explain` takes the site file as its second argument and `Scope` has a `site_file` field. diff --git a/docs/orgmode/howto/write-a-site-file.org b/docs/orgmode/howto/write-a-site-file.org index 54c63ca..ce897fb 100644 --- a/docs/orgmode/howto/write-a-site-file.org +++ b/docs/orgmode/howto/write-a-site-file.org @@ -88,7 +88,8 @@ For a site on top of the European Environment for Scientific Software Installati EB_PIPELINE_LOG_STAMP = "" EB_PIPELINE_MODULES = "EasyBuild" EB_PIPELINE_RUN_ID = "" - EB_PIPELINE_SITE = "example" + EB_PIPELINE_SITE = "$WORK/site.toml" + EB_PIPELINE_SITE_NAME = "example" EB_PIPELINE_TARGET = "zen4" [targets.phases.build] @@ -99,6 +100,8 @@ For a site on top of the European Environment for Scientific Software Installati ... #+end_example + =EB_PIPELINE_SITE= is the site file as an absolute path, here with =$WORK= for the directory that holds it, so that a command can run eb-pipeline on the same file. + 5. To change a value for one run without editing the file, set it in a variable named =EB_PIPELINE__SECTION__KEY=, here a node-local build directory: #+begin_src bash @@ -129,8 +132,10 @@ eb-pipeline run --site site.toml --list lists/zen4.list --target zen4 --build-ty set -e if [ -n "${EB_PIPELINE_UNUSE_PATH:-}" ]; then MODULEPATH=; export MODULEPATH; fi module purge -for m in ${EB_PIPELINE_MODULES:-}; do module load "$m"; done -exec "$@"' eb-pipeline eb Example-1.0-GCCcore-15.2.0.eb --tmp-logdir=/opt/site/logs/20261117_032712/zen4/easybuild-logs/Example-1.0-GCCcore-15.2.0 +set -f +for m in ${EB_PIPELINE_MODULES:-}; do set +f; module load "$m"; done +set +f +exec "$@"' eb-pipeline eb Example-1.0-GCCcore-15.2.0.eb --tmp-logdir=/opt/site/logs/20261117_043420/zen4/easybuild-logs/Example-1.0-GCCcore-15.2.0 ... # [caches] not configured [logs] TZ=UTC srun --job-name=example-production-logs -p cpu -N 1 --ntasks-per-node=1 -c 1 -t 15:00 eb-keep-logs rotate /opt/site/logs --compress-after 30 --delete-after 365 --lock-timeout 0 diff --git a/docs/orgmode/reference/environment.org b/docs/orgmode/reference/environment.org index 2b00fda..b9126b8 100644 --- a/docs/orgmode/reference/environment.org +++ b/docs/orgmode/reference/environment.org @@ -72,7 +72,8 @@ The keys of =[scheduler.env]= come first, and a variable of the table below with | =EASYBUILD_CUDA_COMPUTE_CAPABILITIES= | the target's =cuda_compute_capabilities= | it is set | | =EASYBUILD_READ_ONLY_INSTALLDIR= | =filesystems.read_only_installdir_value=, =1= by default | =filesystems.read_only_installdir= is =true= | | =EASYBUILD_CONFIGS_REPOPATH= | =filesystems.configs_repo= | it is set | -| =EB_PIPELINE_SITE= | =site.name= | always | +| =EB_PIPELINE_SITE= | the site file, as an absolute path | always | +| =EB_PIPELINE_SITE_NAME= | =site.name= | always | | =EB_PIPELINE_TARGET= | the target's name | always | | =EB_PIPELINE_BUILD_TYPE= | the build type | always | | =EB_PIPELINE_RUN_ID= | the run id | always | @@ -89,6 +90,9 @@ The keys of =[scheduler.env]= come first, and a variable of the table below with | the one named by =modules.eessi.accel_override_var=, =EESSI_ACCELERATOR_TARGET_OVERRIDE= by default | the target's =accel_target= | =[modules.eessi]= is on and =accel_target= is set | | =EESSI_SITE_SOFTWARE_PREFIX= | the target's =site_prefix=, else =filesystems.install_root= | =[modules.eessi]= is on | +=EB_PIPELINE_SITE=, =EB_PIPELINE_TARGET=, =EB_PIPELINE_BUILD_TYPE=, =EB_PIPELINE_RUN_ID= and =EB_PIPELINE_LOG_STAMP= are also the variables of the options above, so a command that runs =eb-pipeline= without those options gets the site file, target, build type, run id and log stamp of the run. +=eb-pipeline= does not read =EB_PIPELINE_SITE_NAME=; =EB_PIPELINE__SITE__NAME=, with two underscores, is the variable that sets =name= in the site file. + =[modules.eessi]= is on when the site file has the table and its =enabled= is not =false=. =[licences]= is on when its =enabled= and =enforce= are not =false=. =EESSI= stands for the European Environment for Scientific Software Installations. @@ -106,7 +110,7 @@ Some commands get more variables, set over those above. | =EB_PIPELINE_HW_VENDOR_ID=, =EB_PIPELINE_HW_CPU_FAMILY=, =EB_PIPELINE_HW_MODEL= | the =vendor_id=, =cpu_family= and =model= of the target's =cpu_hardware= for a CPU-role unit, of its =hardware= otherwise | the prefetch and build commands, when the target has a =cpu_hardware= table | | the keys of =targets.route.env= | their values, with the placeholders of the route replaced | the commands that the route wraps | | =EB_PIPELINE_ROUTE_RUNTIME_ROOT= | =targets.route.runtime_root= | the same, when it is set | -| =EB_PIPELINE_ROUTE_SHARES_RO=, =EB_PIPELINE_ROUTE_SHARES_RW= | the entries of =targets.route.shares_ro= and =shares_rw=, joined with =:= | the same, when they are not empty | +| =EB_PIPELINE_ROUTE_SHARES_RO=, =EB_PIPELINE_ROUTE_SHARES_RW= | the entries of =targets.route.shares_ro= and =shares_rw=, with their placeholders replaced, joined with =:= | the same, when they are not empty | | =EB_PIPELINE_ROUTE_RECEIPT_DIGEST= | the value of the variable that =targets.route.receipt_digest_env= names | the same, when it is set | | =EASYBUILD_OUTPUT_STYLE= | =basic=, whatever =[scheduler.env]= sets, so that =eb --last-log= prints the log path on one line | =eb-keep-logs failed=, after a line fails | | =TZ= | =UTC=, so that =eb-keep-logs rotate= counts the age of a run from the midnight of its log stamp | =eb-keep-logs rotate= | @@ -121,7 +125,9 @@ When =modules.init= is not empty, =modules.unuse_path= is =true= or =modules.she 1. =EB_PIPELINE_SHELL_INIT=: the file it sources first, when set. 2. =EB_PIPELINE_UNUSE_PATH=: when set, it empties =MODULEPATH=. 3. =EB_PIPELINE_MODULES=: it runs =module purge=, then =module load= for each name in it, in order, then the command. + It splits the variable at white space and gives each name to =module load= as written: a name with a pattern in it, such as =GCC/*=, is not matched against file names. +The wrapper is a script for a POSIX shell, such as =bash= or =sh=. The admit and rotate commands start without it. ** See also diff --git a/docs/orgmode/reference/plan-and-result.org b/docs/orgmode/reference/plan-and-result.org index 50ce42c..9923786 100644 --- a/docs/orgmode/reference/plan-and-result.org +++ b/docs/orgmode/reference/plan-and-result.org @@ -76,7 +76,8 @@ cat plan.json "EB_PIPELINE_LOG_DIR": "$WORK/logs/20261117_090000/cpu", "EB_PIPELINE_LOG_STAMP": "20261117_090000", "EB_PIPELINE_RUN_ID": "nightly", - "EB_PIPELINE_SITE": "example", + "EB_PIPELINE_SITE": "$WORK/site.toml", + "EB_PIPELINE_SITE_NAME": "example", "EB_PIPELINE_TARGET": "cpu" }, "name": "build", diff --git a/docs/orgmode/reference/site-file.org b/docs/orgmode/reference/site-file.org index 60fe1d7..162e06a 100644 --- a/docs/orgmode/reference/site-file.org +++ b/docs/orgmode/reference/site-file.org @@ -77,14 +77,14 @@ It is one file in Tom's Obvious Minimal Language (TOML), named by =--site=. ** =[site]= -| Key | Type | Default | Meaning | -|--------------------+--------+---------------------------------------------------+-----------------------------------------------------------------------------------------------------| -| =name= | string | required | The name of the site. A run exports it as =EB_PIPELINE_SITE=, and it is the default of =project=. | -| =project= | string | =site.name= | The value of ={project}= in =scheduler.job_name_template=. | -| =build_account= | string | none | The account that owns the installs; exported as =EB_PIPELINE_BUILD_ACCOUNT= and not read otherwise. | -| =default_group= | string | none | The default Unix group of the installs; exported as =EB_PIPELINE_DEFAULT_GROUP=. | -| =stack_generation= | string | none | A label for the generation of the software stack, such as =2026.1=; not read. | -| =tools= | table | all tools looked up on =PATH=, no minimum version | The programs of the sibling tools and the versions they need. | +| Key | Type | Default | Meaning | +|--------------------+--------+---------------------------------------------------+--------------------------------------------------------------------------------------------------------| +| =name= | string | required | The name of the site. A run exports it as =EB_PIPELINE_SITE_NAME=, and it is the default of =project=. | +| =project= | string | =site.name= | The value of ={project}= in =scheduler.job_name_template=. | +| =build_account= | string | none | The account that owns the installs; exported as =EB_PIPELINE_BUILD_ACCOUNT= and not read otherwise. | +| =default_group= | string | none | The default Unix group of the installs; exported as =EB_PIPELINE_DEFAULT_GROUP=. | +| =stack_generation= | string | none | A label for the generation of the software stack, such as =2026.1=; not read. | +| =tools= | table | all tools looked up on =PATH=, no minimum version | The programs of the sibling tools and the versions they need. | *** =[site.tools]= @@ -151,8 +151,8 @@ A run builds for one target, named by =--target=. | =phases= | table | =phase_defaults= | The launch arguments of the target, per role; a role it leaves out takes the one in =[phase_defaults]=. | | =route= | table | none | A command that wraps the CPU-role commands of the target. | | =select= | table | none | Which lines of a list the target builds; without it, all of them. | -| =hardware= | table | none | The CPU of the target's build nodes. | -| =cpu_hardware= | table | none | The CPU of the nodes that build the CPU-role units. When it is set, the source prefetch and build commands export a CPU class as =EB_PIPELINE_HW_VENDOR_ID=, =EB_PIPELINE_HW_CPU_FAMILY= and =EB_PIPELINE_HW_MODEL=: this one for a CPU-role unit, =hardware= for the others. | +| =hardware= | table | none | The CPU of the target's build nodes; the commands get it only when =cpu_hardware= is set. | +| =cpu_hardware= | table | none | The CPU of the nodes that build the CPU-role units. When it is set, the source prefetch and build commands export a CPU class as =EB_PIPELINE_HW_VENDOR_ID=, =EB_PIPELINE_HW_CPU_FAMILY= and =EB_PIPELINE_HW_MODEL=: this one for a CPU-role unit, =hardware= for the others. No other command gets them. | | =fetch_profile= | table | none | A table with any keys, passed to the source prefetch commands as =EB_PIPELINE_FETCH_PROFILE=, in JavaScript Object Notation (JSON) with sorted keys. | | =package_env= | table of tables | ={}= | Variables per package name, =[targets.package_env.NAME]=, for the lines of that package; per key, they win over =[modules.package_env.NAME]=. | | =build_types= | table of tables | ={}= | Values per build type, =[targets.build_types.NAME]=. | @@ -182,8 +182,8 @@ A route wraps the commands of the CPU-role units, the CPU-only prerequisites tha | =args= | array of string | =[]= | Arguments after =command=, with placeholders; when there are any, =--= follows them. | | =env= | table of string | ={}= | Variables of the wrapped commands; the values take placeholders. | | =runtime_root= | string | none | The root of the route's runtime: ={runtime_root}=, and =EB_PIPELINE_ROUTE_RUNTIME_ROOT= for the wrapped commands. | -| =shares_ro= | array of string | =[]= | Paths the route shares read-only: ={shares_ro}=, and =EB_PIPELINE_ROUTE_SHARES_RO=, joined with =:=. | -| =shares_rw= | array of string | =[]= | Paths the route shares read-write: ={shares_rw}=, and =EB_PIPELINE_ROUTE_SHARES_RW=, joined with =:=. | +| =shares_ro= | array of string | =[]= | Paths the route shares read-only, with placeholders; joined with =:=, they are ={shares_ro}= and =EB_PIPELINE_ROUTE_SHARES_RO=. | +| =shares_rw= | array of string | =[]= | Paths the route shares read-write, with placeholders; joined with =:=, they are ={shares_rw}= and =EB_PIPELINE_ROUTE_SHARES_RW=. | | =receipt_digest_env= | string | none | The name of a variable, in letters, digits and =_=, that holds a receipt digest. =run= refuses with exit 78 when it is unset or empty, and passes its value as =EB_PIPELINE_ROUTE_RECEIPT_DIGEST=; =plan= shows =${NAME}=. | | =phases= | array of string | =["build"]= | The phases whose CPU-role commands the route wraps: =build=, =sources= or both. | @@ -312,7 +312,7 @@ With =--disable-robot= in =extra_args=, a line fetches its own sources only: the |---------------+-----------------+----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | =init= | array of string | required | The modules to load, in order, after =module purge=, before the shard, prefetch, build, =eb-keep-logs failed= and cache commands; may be empty. | | =unuse_path= | boolean | =false= | =true= empties =MODULEPATH= after =shell_init= and before =init= loads, so that the first module of =init= needs an absolute path; =check= warns when it has none. | -| =shell= | string | =bash= | The shell that runs the module wrapper. | +| =shell= | string | =bash= | The shell that runs the module wrapper, which is a script for a POSIX shell, such as =bash= or =sh=. | | =shell_init= | string | none | A file to source before =module= is called, for a shell that does not define it. It is sourced without =set -e= and =set -u=; a =module= call that fails still stops the command. | | =eessi= | table | none | The site layer of EESSI. | | =package_env= | table of tables | ={}= | Variables per package name, =[modules.package_env.NAME]=, for the prefetch and build commands of the lines of that package. | diff --git a/src/main.rs b/src/main.rs index 605d232..0c87da9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -188,6 +188,14 @@ fn load_valid(path: &Path, lists_dir: Option<&Path>) -> Result Option { + std::path::absolute(path) + .ok() + .and_then(|p| p.to_str().map(str::to_string)) +} + fn cmd_check(site_path: &Path, lists_dir: Option<&Path>, probe: bool) -> Result { let s = load_valid(site_path, lists_dir)?; let m = &s.modules; @@ -237,7 +245,7 @@ fn cmd_explain( format: Format, ) -> Result { let s = load_valid(site_path, None)?; - let v = explain(&s, target, build_type)?; + let v = explain(&s, site_file(site_path).as_deref(), target, build_type)?; match format { Format::Json => write_json(&v, None)?, Format::Toml => print!( @@ -380,6 +388,7 @@ fn cmd_plan_or_run(c: &Common, m: Mode) -> Result { }); let scope = Scope { site: &s, + site_file: site_file(&c.site), target, build_type: Some(&c.build_type), run_id, diff --git a/src/resolve.rs b/src/resolve.rs index 8dc46f3..1647625 100644 --- a/src/resolve.rs +++ b/src/resolve.rs @@ -58,6 +58,9 @@ pub fn role_args(site: &Site, target: &Target, role: Role) -> Option { pub struct Scope<'a> { /// The site file. pub site: &'a Site, + /// The path of the site file, absolute, exported as `EB_PIPELINE_SITE` + /// so that a command can run eb-pipeline on the same file. + pub site_file: Option, /// The target the phase runs for. pub target: &'a Target, /// The build type, which picks the target's `build_types` override. @@ -410,7 +413,10 @@ impl<'a> Scope<'a> { if let Some(c) = &fs.configs_repo { put("EASYBUILD_CONFIGS_REPOPATH", c.clone()); } - put("EB_PIPELINE_SITE", s.site.name.clone()); + if let Some(f) = &self.site_file { + put("EB_PIPELINE_SITE", f.clone()); + } + put("EB_PIPELINE_SITE_NAME", s.site.name.clone()); put("EB_PIPELINE_TARGET", self.target.name.clone()); put("EB_PIPELINE_BUILD_TYPE", self.build_name().to_string()); put("EB_PIPELINE_RUN_ID", self.run_id.clone()); @@ -492,12 +498,16 @@ impl<'a> Scope<'a> { /// environment, never through interpolation. The site's init is sourced /// without `-e` or `-u`, as a login shell sources it: EESSI's Lmod init /// reads variables it never sets. `module` and the command run under `-e`. +/// The names are split with pathname expansion off, so that `module load` +/// gets each one as written, and back on for `module load` itself. pub const MODULE_SCRIPT: &str = "if [ -n \"${EB_PIPELINE_SHELL_INIT:-}\" ]; then . \"$EB_PIPELINE_SHELL_INIT\"; fi set -e if [ -n \"${EB_PIPELINE_UNUSE_PATH:-}\" ]; then MODULEPATH=; export MODULEPATH; fi module purge -for m in ${EB_PIPELINE_MODULES:-}; do module load \"$m\"; done +set -f +for m in ${EB_PIPELINE_MODULES:-}; do set +f; module load \"$m\"; done +set +f exec \"$@\""; /// Group that applies to a package, if any. A version that does not parse @@ -535,7 +545,13 @@ fn strip_nulls(v: Value) -> Value { } /// Resolved view of one or all targets after defaults and overrides. -pub fn explain(site: &Site, target: Option<&str>, build_type: Option<&str>) -> Result { +/// `site_file` is the `EB_PIPELINE_SITE` of the environment it shows. +pub fn explain( + site: &Site, + site_file: Option<&str>, + target: Option<&str>, + build_type: Option<&str>, +) -> Result { let selected: Vec<&Target> = match target { Some(t) => vec![site.target(t)?], None => site.targets.iter().collect(), @@ -544,6 +560,7 @@ pub fn explain(site: &Site, target: Option<&str>, build_type: Option<&str>) -> R for t in selected { let scope = Scope { site, + site_file: site_file.map(str::to_string), target: t, build_type, run_id: "".into(), diff --git a/src/schema.rs b/src/schema.rs index 29fe847..4f1b5b2 100644 --- a/src/schema.rs +++ b/src/schema.rs @@ -201,7 +201,7 @@ doc_struct! { "Root of the route's runtime; exported as EB_PIPELINE_ROUTE_RUNTIME_ROOT and available as {runtime_root}."), #[serde(default)] shares_ro: Vec => ("array of string", "[]", false, - "Paths shared read-only into the route; placeholders {phase_dir}, {log_dir}, {work_dir} allowed. Exported joined with `:` as EB_PIPELINE_ROUTE_SHARES_RO."), + "Paths shared read-only into the route; placeholders {phase_dir}, {log_dir}, {work_dir}, {target}, {phase} allowed, as in `args`. Exported joined with `:` as EB_PIPELINE_ROUTE_SHARES_RO."), #[serde(default)] shares_rw: Vec => ("array of string", "[]", false, "Paths shared read-write into the route; same placeholders as `shares_ro`. Exported as EB_PIPELINE_ROUTE_SHARES_RW."), @@ -281,9 +281,10 @@ doc_struct! { route: Option => ("table", "none", false, "CPU phase wrapper."), select: Option