diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7140aa7..c9f7c5c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,6 +74,7 @@ jobs: bash docs/examples/templates.sh bash docs/examples/site-file.sh bash docs/examples/ci.sh + bash docs/examples/site-installs.sh bash docs/examples/rotation.sh bash docs/examples/staged.sh bash docs/examples/select.sh diff --git a/README.md b/README.md index 4730f05..3223561 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ looks site-specific. - How-to guides: [install eb-pipeline](docs/orgmode/howto/install.org), [write a site file for your cluster](docs/orgmode/howto/write-a-site-file.org), [build a list on every target from continuous integration](docs/orgmode/howto/build-from-ci.org), + [let users load what a target installs on top of EESSI](docs/orgmode/howto/site-installs.org), [rotate the log tree](docs/orgmode/howto/rotate-logs.org), [rebuild installed software through eb-staged-rebuild](docs/orgmode/howto/staged-rebuilds.org), [build some lines of a list on some targets only](docs/orgmode/howto/select-lines.org). diff --git a/changelog.d/+users-see-site-installs.doc.md b/changelog.d/+users-see-site-installs.doc.md new file mode 100644 index 0000000..2e8df46 --- /dev/null +++ b/changelog.d/+users-see-site-installs.doc.md @@ -0,0 +1 @@ +A how-to guide says how to let users load what a target installs on top of EESSI: with `EESSI_SITE_SOFTWARE_PREFIX` set to the target's `site_prefix` before EESSI loads, without which the EESSI module takes only its `host_injections` tree, and with a `cpu_target` that is the target their jobs on those nodes load. With eessi-arch-guard as the Slurm `TaskProlog`, that is the target the guard selects, which can differ from the one EESSI selects on the same CPU; a tree built for another target is on no user's `MODULEPATH`. The guide also says that a job keeps the prefix of the shell that submitted it. diff --git a/docs/examples/site-installs.sh b/docs/examples/site-installs.sh new file mode 100755 index 0000000..2299318 --- /dev/null +++ b/docs/examples/site-installs.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# What docs/orgmode/howto/site-installs.org shows: the prefix and CPU target +# that each target of site-installs/site.toml exports, and the tree that +# EESSI-extend installs its builds into. The other steps of the page need a +# cluster with EESSI. +# shellcheck disable=SC2016 # commands stay quoted as the pages show them +# shellcheck source-path=SCRIPTDIR source=lib.sh +. "$(dirname -- "$0")/lib.sh" + +export WORK=$HOME/site-installs +mkdir -p "$WORK" +cd "$WORK" +cp "$pages/howto/site-installs/site.toml" site.toml + +run 'eb-pipeline check --site site.toml' +expect_status 0 + +run 'for t in $(eb-pipeline targets --site site.toml); do + eb-pipeline explain --site site.toml --target "$t" | grep -E "install_note|EESSI_SITE_SOFTWARE_PREFIX|EESSI_SOFTWARE_SUBDIR_OVERRIDE" +done' +expect_status 0 +expect_line 'EESSI_SITE_SOFTWARE_PREFIX = "/opt/site/zen2"' +expect_line 'EESSI_SOFTWARE_SUBDIR_OVERRIDE = "x86_64/amd/zen2"' +expect_line 'install_note = "EESSI-extend replaces EASYBUILD_INSTALLPATH when it loads: /opt/site/zen4/versions//software/linux/x86_64/amd/zen4"' +expect_line 'EESSI_SITE_SOFTWARE_PREFIX = "/opt/site/zen4"' +expect_line 'EESSI_SOFTWARE_SUBDIR_OVERRIDE = "x86_64/amd/zen4"' diff --git a/docs/orgmode/howto/index.org b/docs/orgmode/howto/index.org index e112436..f1e8e2f 100644 --- a/docs/orgmode/howto/index.org +++ b/docs/orgmode/howto/index.org @@ -14,6 +14,7 @@ The title names the task. install write-a-site-file build-from-ci + site-installs rotate-logs staged-rebuilds select-lines diff --git a/docs/orgmode/howto/site-installs.org b/docs/orgmode/howto/site-installs.org new file mode 100644 index 0000000..ff380d8 --- /dev/null +++ b/docs/orgmode/howto/site-installs.org @@ -0,0 +1,89 @@ +#+TITLE: Let users load what a target installs on top of EESSI +#+OPTIONS: toc:nil todo:nil num:nil author:nil title:nil +#+STARTUP: content + +* Let users load what a target installs on top of EESSI + +Make what the targets of a site file install on top of the European Environment for Scientific Software Installations (EESSI) loadable by the users of each target's nodes, in their shells and in their jobs. +The EESSI module adds a site tree to =MODULEPATH= from =EESSI_SITE_SOFTWARE_PREFIX= as it loads: =PREFIX/versions/RELEASE/software/linux/CPU_TARGET/modules/all=, for the CPU target it loads for, when that directory exists. +Without the variable, it takes the =host_injections= tree of the EESSI repository instead, so a run that installed every line can leave users without any of it. + +** Before you start + +- A site file whose targets install on top of EESSI, with =[modules.eessi]= and =site_install = true=, as the =eessi-site= template of [[file:../reference/templates.org][The site file templates]] has. +- A way to set a variable in the environment of users on the nodes before EESSI loads there, such as a file in =/etc/profile.d=. +- For step 2, where eessi-arch-guard is the Slurm =TaskProlog=, a shell with EESSI loaded. + +The site file on this page builds for the =zen2= and =zen4= nodes of Example University, each with a site prefix of its own: + +#+INCLUDE: "site-installs/site.toml" src toml + +** Steps + +1. Read the prefix and the CPU target that each target's builds install for, and the tree that EESSI-extend installs them into: + + #+begin_src bash + for t in $(eb-pipeline targets --site site.toml); do + eb-pipeline explain --site site.toml --target "$t" | grep -E "install_note|EESSI_SITE_SOFTWARE_PREFIX|EESSI_SOFTWARE_SUBDIR_OVERRIDE" + done + #+end_src + + #+begin_example + install_note = "EESSI-extend replaces EASYBUILD_INSTALLPATH when it loads: /opt/site/zen2/versions//software/linux/x86_64/amd/zen2" + EESSI_SITE_SOFTWARE_PREFIX = "/opt/site/zen2" + EESSI_SOFTWARE_SUBDIR_OVERRIDE = "x86_64/amd/zen2" + install_note = "EESSI-extend replaces EASYBUILD_INSTALLPATH when it loads: /opt/site/zen4/versions//software/linux/x86_64/amd/zen4" + EESSI_SITE_SOFTWARE_PREFIX = "/opt/site/zen4" + EESSI_SOFTWARE_SUBDIR_OVERRIDE = "x86_64/amd/zen4" + #+end_example + + =run= exports both variables to every command of a target: =EESSI_SITE_SOFTWARE_PREFIX= is its =site_prefix=, or =filesystems.install_root= when it sets none, and =EESSI_SOFTWARE_SUBDIR_OVERRIDE= is its =cpu_target=. + Users see what a target installed when their EESSI module loads with the same prefix, for the same CPU target. +2. Check that the =cpu_target= of each target is the CPU target that jobs load on its nodes. + With eessi-arch-guard as the Slurm =TaskProlog=, that is the target the guard selects for a task, which can differ from the one EESSI selects on the same CPU; a task prints it, here on a =zen4= node: + + #+begin_src bash + srun -p cpu -C zen4 -N 1 printenv EESSI_SOFTWARE_SUBDIR + #+end_src + + It should print =x86_64/amd/zen4=. + Any other target means that =cpu_target= does not match the nodes: set it to the target that the task printed, and build the target's lists again, as a tree built for a CPU target that jobs do not load is on no user's =MODULEPATH=. +3. Set the target's prefix before EESSI loads, in the environment that users get on its nodes, for example from a file in =/etc/profile.d= that runs before the one that loads EESSI; on the =zen4= nodes: + + #+begin_src bash + export EESSI_SITE_SOFTWARE_PREFIX=/opt/site/zen4 + #+end_src + + A Slurm job starts, by default, with the environment of the shell that submitted it, so a job submitted with the prefix of one target keeps it on the nodes of another, where EESSI finds no tree under it. + Targets that share one =site_prefix= need one value on every node, as the tree under it is per CPU target already; with staged rebuilds, each of them then sets =software_root= and =modules_root= to its own tree, as =check= says. + +** Check that it worked + +In a job on a node of the target, the EESSI module names the site tree it took, and =module avail= lists its module directory: + +#+begin_src bash +srun -p cpu -C zen4 -N 1 bash -c 'echo "$EESSI_SITE_SOFTWARE_PATH"; module -t avail 2>&1 | grep "^$EESSI_SITE_SOFTWARE_PATH"' +#+end_src + +On a =zen4= node, the first line is the tree that the =install_note= of =zen4= names, with the EESSI release in place of ==. +The second line is the =modules/all= directory of that tree, with a colon after it, once the target has installed a module there. + +** If it fails + +| Message | Cause | What to do | +|-------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------| +| =EESSI_SITE_SOFTWARE_PATH= is under =/cvmfs/software.eessi.io/host_injections= | =EESSI_SITE_SOFTWARE_PREFIX= was not set when EESSI loaded. | Set it before EESSI loads, as in step 3, and start a new shell. | +| =EESSI_SITE_SOFTWARE_PATH= starts with the prefix of another target | The job kept the prefix of the shell that submitted it. | Submit from a shell with the prefix of the target, or give the targets one =site_prefix=. | +| =EESSI_SITE_SOFTWARE_PATH= ends in another CPU target than the =cpu_target= of the target | The task loaded EESSI for another CPU target than the one the target builds for. | Set =cpu_target= to the target that the task prints, as in step 2, and build again. | +| No second line | Nothing is installed in that tree: the target has not installed a module yet, or the prefix is not mounted on the node. | Compare the tree with the =install_note= of the target, and check the mount. | +| =Exited with exit code 1=, from =srun=, and no target | The shell that ran =srun= had no EESSI loaded, so the task has no =EESSI_SOFTWARE_SUBDIR=. | Load EESSI first, as users do. | + +The output of step 1 comes from =docs/examples/site-installs.sh=; the other commands need a cluster with EESSI. + +** Related + +- [[file:../reference/environment.org][Environment variables]], for what =[modules.eessi]= exports +- [[file:../reference/site-file.org][The site file]], for =site_prefix=, =cpu_target= and =[modules.eessi]= +- [[file:../reference/templates.org][The site file templates]], for =eessi-site= +- [[file:build-from-ci.org][Build a list on every target from continuous integration]] +- The [[https://stackhands.github.io/eessi-arch-guard/][eessi-arch-guard documentation]], for the CPU target a task gets diff --git a/docs/orgmode/howto/site-installs/site.toml b/docs/orgmode/howto/site-installs/site.toml new file mode 100644 index 0000000..016a168 --- /dev/null +++ b/docs/orgmode/howto/site-installs/site.toml @@ -0,0 +1,56 @@ +# Example University: the zen2 and zen4 nodes of the cpu partition, each +# with a site prefix of its own, installing on top of EESSI 2026.06. +schema_version = 1 +kind = "site" + +[site] +name = "example" +build_account = "alice" +default_group = "alice" + +[scheduler] +kind = "slurm" +launch = "srun" +job_name_template = "{project}-{build}-{phase}-{target}" + +[phase_defaults] +plan = "-p cpu -N 1 --ntasks-per-node=1 -c 1 -t 15:00" +short = "-p cpu -N 1 --ntasks-per-node=1 -c 1 -t 15:00" +build = "-p cpu -N 1 --ntasks-per-node=1 -c 16 -t 1-00:00:00" +cpu = "-p cpu -N 1 --ntasks-per-node=1 -c 16 -t 1-00:00:00" +cache = "-p cpu -N 1 --ntasks-per-node=1 -c 1 -t 15:00" + +[[targets]] +name = "zen2" +build_lists = ["zen2.list"] +cpu_target = "x86_64/amd/zen2" +site_prefix = "/opt/site/zen2" + +[targets.phases] +build = "-p cpu -C zen2 -N 1 --ntasks-per-node=1 -c 16 -t 1-00:00:00" + +[[targets]] +name = "zen4" +build_lists = ["zen4.list"] +cpu_target = "x86_64/amd/zen4" +site_prefix = "/opt/site/zen4" +default = true + +[targets.phases] +build = "-p cpu -C zen4 -N 1 --ntasks-per-node=1 -c 16 -t 1-00:00:00" + +[filesystems] +scratch = "/scratch/alice" +tmp = "/scratch/alice/tmp" +# A build directory and a download directory per target, so that the +# targets can build at once. +build_path = "/scratch/alice/build/{target}" +source_paths = ["/scratch/alice/sources/{target}", "/opt/site/sources"] +install_root = "/opt/site" +log_root = "/opt/site/logs" + +[modules] +init = ["EESSI/2026.06", "EESSI-extend/2026.06-easybuild"] + +[modules.eessi] +site_install = true diff --git a/tests/schema_and_plan.rs b/tests/schema_and_plan.rs index 770e7d0..b98bced 100644 --- a/tests/schema_and_plan.rs +++ b/tests/schema_and_plan.rs @@ -1747,6 +1747,48 @@ fn eessi_site_prefix_follows_the_install_prefix() { .is_none()); } +#[test] +fn the_site_installs_guide_gives_users_the_prefix_and_target_builds_install_for() { + let howto = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("docs/orgmode/howto"); + let o = bin() + .args(["explain", "--format", "json", "--site"]) + .arg(howto.join("site-installs/site.toml")) + .output() + .unwrap(); + assert!(o.status.success(), "{}", String::from_utf8_lossy(&o.stderr)); + let r: serde_json::Value = serde_json::from_slice(&o.stdout).unwrap(); + let targets = r["targets"].as_array().unwrap(); + assert_eq!(targets.len(), 2); + for t in targets { + assert_eq!( + t["env"]["EESSI_SITE_SOFTWARE_PREFIX"], t["site_prefix"], + "{t}" + ); + assert_eq!( + t["env"]["EESSI_SOFTWARE_SUBDIR_OVERRIDE"], t["cpu_target"], + "{t}" + ); + } + + let guide = howto.join("site-installs.org"); + let page = std::fs::read_to_string(&guide).unwrap(); + let text = page.split_whitespace().collect::>().join(" "); + for needed in [ + "Without the variable, it takes the =host_injections= tree", + "With eessi-arch-guard as the Slurm =TaskProlog=, that is the target the guard selects", + "printenv EESSI_SOFTWARE_SUBDIR", + "Set the target's prefix before EESSI loads", + "export EESSI_SITE_SOFTWARE_PREFIX=/opt/site/zen4", + "keeps it on the nodes of another", + ] { + assert!( + text.contains(needed), + "{} lacks `{needed}`", + guide.display() + ); + } +} + #[test] fn the_log_root_reference_names_the_stamp_and_the_target() { let text = eb_pipeline::schema::reference();