Operate the daemons with the shared instrumentctl CLI

.gitmodules

@@ -1,15 +1,15 @@
 [submodule "src/driver/sunpower"]
 	path = src/driver/sunpower
-	url = git@github.com:COO-Utilities/sunpower.git
+	url = https://github.com/COO-Utilities/sunpower.git
 [submodule "src/driver/coo_ethercat"]
 	path = src/driver/coo_ethercat
-	url = git@github.com:CaltechOpticalObservatories/coo-ethercat.git
+	url = https://github.com/CaltechOpticalObservatories/coo-ethercat.git
 [submodule "src/driver/lris2_csu"]
 	path = src/driver/lris2_csu
-	url = git@github.com:CaltechOpticalObservatories/lris2-csu.git
+	url = https://github.com/CaltechOpticalObservatories/lris2-csu.git
 [submodule "src/driver/inficon"]
 	path = src/driver/inficon
-	url = git@github.com:COO-Utilities/inficon.git
+	url = https://github.com/COO-Utilities/inficon.git
 [submodule "src/driver/pdu"]
 	path = src/driver/pdu
-	url = git@github.com:COO-Utilities/pdu.git
+	url = https://github.com/COO-Utilities/pdu.git

README.md

@@ -4,7 +4,9 @@ LRIS2 Instrument Control Software
 
 ## Structure
 
-- `daemons/` – mKTL service daemons
+- `daemons/` – libby hardware daemons, one script per device type
+- `config/` – one YAML per daemon instance, named for its `peer_id`
+- `systemd/` – per-instance env files and the host setup script
 - `gui/` – Graphical interfaces (`slitmaskgui`, `demo`)
 - `scripts/` – AIT and utility scripts
 - `src/driver/` – Hardware driver submodules
@@ -36,9 +38,12 @@ in-process rather than over a socket:
 ```bash
 pip install -e ".[camera]"
 pip install <lris2-camerad-instrument>   # builds the camera_interface module
-daemons/camera -c config/lris2_rcam.yaml
+daemons/camera -c config/r/rcam.yaml
 ```
 
+On a deployed host it runs as a service and both checkouts live under
+`/opt/lris2`; see [`systemd/README.md`](systemd/README.md).
+
 Reading does not need the detector powered. Opening the link is enough:
 
 ```bash
@@ -51,7 +56,7 @@ configuration camerad parsed; a write patches a single configuration line and
 the line numbers have to agree. The controller also refuses to apply a bias
 change while the detector is unpowered.
 
-`config/lris2_rcam.yaml` names every channel that gets a keyword, and carries
+`config/r/rcam.yaml` names every channel that gets a keyword, and carries
 the voltage limits for the four module 4 rails that have them.
 
 ## Submodules
@@ -84,6 +89,26 @@ git add .gitmodules .
 git commit -m "Update submodules to latest on main"
 ```
 
+## Running the daemons
+
+Each daemon takes its config with `-c` and serves keywords over libby
+(RabbitMQ by default), addressed as `lris2.<peer_id>.<keyword>`:
+
+```bash
+daemons/vacuum-gauge -c config/lris2_rpress.yaml
+libby show lris2.rpress.pressure1
+```
+
+The `libby` CLI reads `~/.libby/cli_config.yaml`; on the instrument host it
+needs only:
+
+```yaml
+transport: rabbitmq
+rabbitmq_url: amqp://localhost
+```
+
+For running daemons as services, see [`systemd/README.md`](systemd/README.md).
+
 ## Testing
 
 Tests mirror the `src/` layout under `tests/`.

config/common/keygrabber.yaml

@@ -0,0 +1,40 @@
+# Keyword archiver: polls the LRIS2 daemons and writes to InfluxDB for Grafana
+#
+# Usage:
+#   daemons/keygrabber -c config/lris2_keygrabber.yaml
+#
+# The InfluxDB token comes from the environment variable named in token_env.
+
+peer_id: keygrabber
+group_id: lris2
+
+sink:
+  type: influxdb
+  url: http://meridian.caltech.edu:8086
+  org: coo
+  bucket: lris2
+  token_env: LRIS2_INFLUX_TOKEN
+
+# Reads run concurrently across peers; a single daemon still answers one at a time
+workers: 4
+
+defaults:
+  interval_s: 30.0
+  timeout_s: 2.0
+  refresh_s: 300.0
+
+# "%" takes everything readable except uptime and lasterror, which libby excludes
+collections:
+
+  # Vacuum and cryo: slow-moving, the trend over a cooldown is the point
+  rpress:
+    peer: lris2.rpress
+    keywords: ["%"]
+    interval_s: 60.0
+  rcryo:
+    peer: lris2.rcryo
+    keywords: ["%"]
+    interval_s: 60.0
+
+logging:
+  level: INFO

config/lris2_rcam.yaml → config/r/rcam.yaml

@@ -1,7 +1,7 @@
 # RCam - Archon detector controller, Red channel
 #
 # Usage:
-#   daemons/camera -c config/lris2_rcam.yaml
+#   daemons/camera -c config/r/rcam.yaml
 #
 # Addressed as lris2.rcam (group_id.peer_id), e.g.
 #   libby show lris2.rcam.xvbiasbackbias
@@ -14,7 +14,7 @@ group_id: lris2
 description: LRIS2 Red channel Archon controller
 
 camera:
-  config_file: /usr/local/share/lris2/lris2.cfg
+  config_file: /opt/lris2/camerad/config/lris2.cfg
   nframes: 1
 
 # Every labelled channel the three bias boards carry, taken from "bias list"

config/lris2_cryocooler.yaml → config/r/rcryo.yaml

@@ -2,7 +2,7 @@
 # Usage:
 #   daemons/cryocooler -c config/lris2_cryocooler.yaml
 
-peer_id: lris2_cryocooler
+peer_id: rcryo
 group_id: lris2
 
 hardware:

config/lris2_vacuum.yaml → config/r/rpress.yaml

@@ -2,7 +2,7 @@
 # Usage:
 #   daemons/vacuum-gauge -c config/lris2_vacuum.yaml
 
-peer_id: lris2_vacuum
+peer_id: rpress
 group_id: lris2
 
 hardware:

daemons/keygrabber

@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+"""Keyword archiver: polls the LRIS2 daemons into InfluxDB for Grafana.
+
+The daemon lives in libby; this shim lets the systemd template launch it like
+any other LRIS2 daemon.
+"""
+
+from libby.keygrabber.main import main
+
+if __name__ == '__main__':
+    raise SystemExit(main())

pyproject.toml

@@ -13,10 +13,18 @@ authors = [
 license = {text = "MIT"}
 readme = "README.md"
 dependencies = [
+    "instrumentctl@git+https://github.com/CaltechOpticalObservatories/[email protected]",
+    "libby@git+https://github.com/CaltechOpticalObservatories/libby.git",
+    # The driver submodules are vendored rather than installed, so what they
+    # import has to be declared here. The pin is the one they ask for
+    "hardware_device_base@git+https://github.com/COO-Utilities/[email protected]",
     "pyserial",
     "pyyaml",
 ]
 
+[project.scripts]
+lris2 = "lris2.cli:main"
+
 [project.optional-dependencies]
 dev = [
     "pytest",
@@ -32,7 +40,10 @@ camera = [
 lris2 = "src"
 
 [tool.setuptools]
-packages = ["lris2", "lris2.driver"]
+packages = ["lris2", "lris2.cli", "lris2.driver"]
+
+[tool.setuptools.package-data]
+"lris2.cli" = ["instrument.toml"]
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]

src/cli/__init__.py

@@ -0,0 +1,20 @@
+"""``lris2``: deploy, run and inspect the LRIS2 daemons on this host.
+
+Every command comes from instrumentctl, which is shared with the other
+instruments. All that is specific to LRIS2 is ``instrument.toml`` beside this
+file.
+"""
+
+from importlib import resources
+from pathlib import Path
+from typing import List, Optional
+
+from instrumentctl import run
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    """Entry point for the ``lris2`` console script."""
+    config = resources.files(__package__).joinpath("instrument.toml")
+    # Locates the checkout under an editable install, which is what lets the
+    # deployed CLI follow a git pull
+    return run(config, repo_hint=Path(__file__).resolve().parents[2], argv=argv)

src/cli/instrument.toml

@@ -0,0 +1,9 @@
+[instrument]
+name = "lris2"
+display_name = "LRIS2"
+ops_group = "lris2-ops"
+env_prefix = "LRIS2"
+
+[help]
+example_instance = "rpress"
+example_subsystem = "r"

systemd/README.md

@@ -0,0 +1,96 @@
+# Running daemons under systemd
+
+Operated with the `lris2` command, which comes from
+[instrumentctl](https://github.com/CaltechOpticalObservatories/instrumentctl)
+and is shared with the other instruments. All that is specific to LRIS2 is
+`src/cli/instrument.toml`.
+
+One template unit, `[email protected]`, runs any `daemons/<script>` that is an
+`Lris2Daemon` subclass taking `-c <config.yaml>` and blocking in
+`daemon.serve()`. Each running daemon is an instance, `lris2@<name>`, backed by
+an env file in `systemd/instances/`:
+
+```sh
+LRIS2_DAEMON=<path relative to daemons/, e.g. vacuum-gauge>
+```
+
+Its config is `/etc/lris2/<name>.yaml`, deployed there from
+`config/<subsystem>/<name>.yaml`.
+
+## Host setup
+
+Once per host, as an admin:
+
+```bash
+sudo git clone https://github.com/CaltechOpticalObservatories/lris2 /opt/lris2/app
+sudo git -C /opt/lris2/app submodule update --init --recursive
+sudo /opt/lris2/app/systemd/install.sh alice bob     # operators to enrol
+```
+
+`install.sh` builds the venv and hands over to `lris2 install`, which creates
+the `lris2` account and the `lris2-ops` group, enrols the operators named,
+makes `/etc/lris2`, `/etc/lris2/instances` and `/var/log/lris2`, seeds a
+root-only `secrets.env`, and writes the template unit, the polkit rule, the
+sudoers drop-in and the operator helpers. It is safe to re-run and reports only
+what it changes.
+
+Override `LRIS2_REPO_DIR`, `LRIS2_VENV_DIR` and `LRIS2_ETC_DIR` for other
+paths.
+
+## Day to day
+
+Nothing below needs root, for anyone in `lris2-ops`:
+
+```bash
+lris2 deploy rpress        # copy its files here, enable, start
+lris2 deploy --new         # every instance not deployed yet
+lris2 status               # what is deployed, and is it running
+lris2 restart rpress       # after editing /etc/lris2/rpress.yaml
+lris2 logs rpress -f       # follow its log
+lris2 doctor               # why isn't it working
+```
+
+Reading a keyword is `libby`, which `install.sh` links beside `lris2` and
+wires for TAB completion through `/etc/profile.d/lris2.sh`, so it works in a
+fresh login shell with nothing to set up.
+
+Do not activate a virtualenv to use them. One holding its own editable `lris2`
+shadows the deployed package while the daemons keep using `/opt/lris2/app`,
+which is a confusing way to lose an afternoon.
+
+`doctor` checks the host and your account, and compares the installed unit and
+polkit rule against what `install` would write now, so a host whose unit
+predates a template change is reported rather than silently stale.
+
+## Updating
+
+Code is an admin's to change, which is why the repo and the venv are not
+writable by `lris2-ops`. An operator changes configuration and run state; the
+daemons execute what an admin installed.
+
+```bash
+sudo git -C /opt/lris2/app pull
+lris2 restart <name>
+```
+
+The venv installs the repo editable, so a pull alone moves the daemons and the
+CLI. Only a change to the dependency list needs
+`sudo /opt/lris2/venv/bin/pip install -e /opt/lris2/app` as well.
+
+## The camera daemon's module
+
+`rcam` is optional, like any other daemon a host may not run, so neither its
+dependencies nor its module are installed by default. On a host that runs it,
+`camera_interface` has to be in the venv; built per instrument, it cannot be a
+dependency of this repository:
+
+```bash
+sudo git clone https://github.com/CaltechOpticalObservatories/lris2-camerad-instrument \
+     /opt/lris2/camerad
+sudo /opt/lris2/venv/bin/pip install -e "/opt/lris2/app[camera]"
+sudo /opt/lris2/venv/bin/pip install /opt/lris2/camerad
+```
+
+`config/r/rcam.yaml` reads the camerad config from that checkout, so a pull
+there moves the ACF the controller is loaded with. Without the module the
+daemon still starts and serves its keywords, reporting `state: nocamera`.

systemd/install.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+# Host setup for running the LRIS2 daemons under systemd. For admins / IT.
+# Run as root. Safe to re-run: it converges the host onto the current repo.
+#
+#   sudo ./systemd/install.sh                    # set the host up
+#   sudo ./systemd/install.sh alice bob          # ... and enrol two operators
+#
+# This builds the virtualenv and hands over to `lris2 install`, which does the
+# rest: the ops group, the daemon account, /etc/lris2, the template unit, the
+# polkit rule and the operator commands. Everything after that needs no root.
+
+set -euo pipefail
+
+REPO_DIR="${LRIS2_REPO_DIR:-/opt/lris2/app}"
+VENV_DIR="${LRIS2_VENV_DIR:-/opt/lris2/venv}"
+
+if [[ $EUID -ne 0 ]]; then
+    echo "Run as root (sudo)." >&2
+    exit 1
+fi
+
+if [[ ! -d "$REPO_DIR" ]]; then
+    echo "error: REPO_DIR $REPO_DIR does not exist." >&2
+    echo "Clone the repo there first, e.g.:" >&2
+    echo "  git clone <repo-url> $REPO_DIR" >&2
+    echo "  cd $REPO_DIR && git submodule update --init --recursive" >&2
+    exit 1
+fi
+
+# Editable, so the deployed CLI and daemons follow a git pull
+if [[ ! -x "$VENV_DIR/bin/python3" ]]; then
+    python3 -m venv "$VENV_DIR"
+    echo "created venv at $VENV_DIR"
+fi
+"$VENV_DIR/bin/pip" install -U pip
+"$VENV_DIR/bin/pip" install -e "$REPO_DIR"
+
+# Reading a keyword is libby, which otherwise lives only inside the venv
+ln -sf "$VENV_DIR/bin/libby" /usr/local/bin/libby
+echo "linked /usr/local/bin/libby"
+
+# Every login shell rather than every operator's shell rc
+install -m 0644 /dev/stdin /etc/profile.d/lris2.sh <<'PROFILE'
+# TAB completion over libby keyword names
+if command -v libby >/dev/null 2>&1; then
+    eval "$(libby completion bash)"
+fi
+PROFILE
+echo "installed /etc/profile.d/lris2.sh"
+
+exec "$VENV_DIR/bin/lris2" install "$@"

systemd/instances/keygrabber.env

@@ -0,0 +1,10 @@
+# Keyword archiver: polls the LRIS2 daemons into InfluxDB for Grafana
+#
+# lris2 deploy keygrabber
+#
+# Deployed alongside this file by the CLI:
+#   config/<subsystem>/keygrabber.yaml -> /etc/lris2/keygrabber.yaml
+#
+# The InfluxDB token goes in root-only /etc/lris2/secrets.env, not here:
+#   LRIS2_INFLUX_TOKEN=<token>
+LRIS2_DAEMON=keygrabber