Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,10 @@ common keys with their defaults, ready to copy.
| `secrets.key_suffix` | `_env` | a config key ending in this names the variable holding a credential |
| `help.example_instance` | `<instance>` | used in `--help`, where a real name beats a placeholder |
| `help.example_subsystem` | `<subsystem>` | likewise |
| `git.remote` | `origin` | where `hotfix` pushes and `update` fetches from |
| `git.base` | `main` | the branch hosts run, and hotfixes are reviewed into |
| `git.branch_prefix` | `hotfix/` | marks a branch `hotfix` made |
| `git.email_domain` | the hostname | operators commit as `<user>@<this>` |
| `legacy.unit` | none | a template unit an older version installed, reported by `doctor` |
| `legacy.polkit_rule` | none | likewise for its polkit rule |

Expand All @@ -76,6 +80,39 @@ An instance is a name with three files behind it: `<repo>/systemd/instances/
<name>.yaml`, and their deployed copies under `/etc/<name>`. Deployed means the
`.env` is in `/etc/<name>/instances/`, which is what the template unit reads.

## Hotfixes

An operator who has to change the code at night should not need to know the
GitHub process. They edit the checkout, restart what uses it, and run:

```bash
<name> hotfix "ADC hangs after power cycle"
```

which shows what changed, asks, commits it in their name to a branch such as
`hotfix/2026-10-15-<host>-adc-hangs-after-power-cycle`, pushes it, and opens a
pull request against `main`. The host keeps running the fix. Once a developer
merges it, `<name> update` returns the host to `main`; until then it refuses,
so a fix is never dropped by accident. On `main`, `update` just pulls.

- `<name> hotfix`, with no description, lists unsaved edits.
- `<name> hotfix --undo` puts edited files back as committed.
- `<name> hotfix --push` retries a push or pull request that failed.
- `<name> update --abandon` leaves an unmerged hotfix a developer rejected;
the branch stays, here and on GitHub.
- `doctor` warns when a host is on a hotfix branch, has commits GitHub does
not, or has edits nobody saved.

Pushing uses one token, so operators need no GitHub account. Make a
fine-grained personal access token limited to the instrument's repository, with
Contents and Pull requests read and write, and put it alone in
`/etc/<name>/github-token`. `install` makes that file root-owned and readable
by the ops group, which means every operator, and the daemon account, can read
it: scope it to the one repository, and protect `main` so that it needs a
reviewed pull request. Fine-grained tokens expire, at most a year out, and
`doctor` cannot tell an expired one from a good one, so put the date in a
calendar.

## Tests

```bash
Expand Down
8 changes: 8 additions & 0 deletions instrument.toml.example
Original file line number Diff line number Diff line change
Expand Up @@ -22,3 +22,11 @@ name = "demo"
# venv = "/opt/<name>/venv" # the interpreter the daemons run under
# repo = "/opt/<name>/app" # the checkout holding daemons/ and config/
# logs = "/var/log/<name>" # daemons' own log files

# Where `<name> hotfix` sends fixes made on a host, and what `<name> update`
# brings it back to
[git]
# remote = "origin"
# base = "main" # the branch hosts run and hotfixes are reviewed into
# branch_prefix = "hotfix/"
# email_domain = "" # operators commit as <user>@<this>; default: the hostname
28 changes: 27 additions & 1 deletion src/instrumentctl/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from pathlib import Path
from typing import List, Optional

from . import commands
from . import commands, hotfix
from .doctor import doctor
from .installer import install
from .instances import Paths
Expand All @@ -35,6 +35,8 @@ def _epilog(product: Product) -> str:
(f"restart {instance}", f"e.g. after editing {etc}/{instance}.yaml"),
(f"logs {instance} -f", "follow its log"),
("doctor", "why isn't it working?"),
('hotfix "what you fixed"', "save an edit to the code here and send it for review"),
("update", "back to the reviewed code, once the hotfix is merged"),
]
width = max(len(command) for command, _ in lines) + len(name) + 3
body = "\n".join(f" {name + ' ' + command:<{width}}{note}" for command, note in lines)
Expand Down Expand Up @@ -112,6 +114,30 @@ def add(name, func, help_text, description=None):
p.add_argument("--since", help='e.g. "1 hour ago", today, "2026-09-21 09:00"')
p.add_argument("-p", "--priority", help="e.g. err, warning")

p = add("hotfix", hotfix.hotfix,
"save edits to the code on this host and send them for review",
"Commit what you edited in the checkout to a hotfix branch in your name,\n"
"push it to GitHub and open a pull request. This host keeps running the\n"
f"fix; once a developer merges it, run: {product.name} update\n\n"
"With no description, show what is edited and not yet saved.")
p.add_argument("message", nargs="?", metavar="description",
help='what you fixed, e.g. "ADC hangs after power cycle"')
mode = p.add_mutually_exclusive_group()
mode.add_argument("--push", action="store_true",
help="send a saved hotfix again, e.g. after a network failure")
mode.add_argument("--undo", action="store_true",
help="discard the edits, putting the files back as committed")
p.add_argument("-y", "--yes", action="store_true", help="do not ask before saving")

p = add("update", hotfix.update,
f"bring the code on this host up to date with {product.git_base}",
f"Fetch {product.git_base} from GitHub and move this host onto it. If the host is\n"
"running a hotfix, only once its pull request is merged. Running daemons\n"
"keep the old code until they are restarted.")
p.add_argument("--abandon", action="store_true",
help="leave an unmerged hotfix, e.g. one a developer rejected; "
"it stays on GitHub")

add("doctor", doctor, "check this host and your account, and say what to fix")

p = add("install", install, "set this host up (root)",
Expand Down
39 changes: 39 additions & 0 deletions src/instrumentctl/doctor.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
from pathlib import Path
from typing import Optional

from . import hotfix
from . import instances as inst
from . import units
from .installer import _render
Expand Down Expand Up @@ -307,6 +308,43 @@ def _placement(r: Report, paths: Paths) -> None:
f"{' '.join(unassigned)}")


def _hotfixes(r: Report, paths: Paths) -> None:
"""Whether hotfix can work, and whether this host has drifted from the base."""
product = paths.product
name, base = product.name, product.git_base
r.section("Hotfixes")
token = paths.github_token
if not _is_file(token):
r.warn(f"no {token}, so hotfix cannot push")
r.hint("admin: put a fine-grained GitHub token there (see the README)")
elif hotfix.read_token(paths) is None:
r.bad(f"cannot read {token}, so hotfix cannot push")
r.hint(_install_hint(paths))
else:
r.ok(f"{token} is readable")
problem = hotfix.checkout_problem(paths)
if problem:
r.warn(problem)
return
branch = hotfix.current_branch(paths)
if hotfix.is_hotfix(paths, branch):
assert branch is not None
r.warn(f"this host is running {branch}, not {base}")
r.hint(f"once it is merged: {name} update")
waiting = hotfix.unsent(paths, branch)
if waiting:
r.warn(f"{waiting} commit(s) on {branch} are not on GitHub")
r.hint(f"{name} hotfix --push")
elif branch == base:
r.ok(f"{paths.repo} is on {base}")
else:
r.warn(f"{paths.repo} is on {branch or 'a detached HEAD'}, not {base}")
edited = hotfix.changes(paths).edited
if edited:
r.warn(f"{len(edited)} edited file(s) in {paths.repo} not saved: {' '.join(edited)}")
r.hint(f'{name} hotfix "what you fixed" # or: {name} hotfix --undo')


def doctor(paths: Paths, _args: argparse.Namespace) -> int:
"""Run every check and exit 1 if any FAILed."""
me = pwd.getpwuid(os.geteuid()).pw_name
Expand All @@ -319,6 +357,7 @@ def doctor(paths: Paths, _args: argparse.Namespace) -> int:
_deployed(r, paths)
_placement(r, paths)
_secrets(r, paths)
_hotfixes(r, paths)
print()
if r.problems == 0:
print("No problems found.")
Expand Down
Loading
Loading