feat: let offloaded pods reach cluster endpoints over the ssh shadow

cmd/virtual-kubelet/main.go

@@ -61,6 +61,7 @@ import (
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/client-go/kubernetes/scheme"
 	lease "k8s.io/client-go/kubernetes/typed/coordination/v1"
+	typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/tools/record"
@@ -541,7 +542,10 @@ func main() {
 	}()
 
 	eb := record.NewBroadcaster()
+	eb.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: localClient.CoreV1().Events("")})
+	defer eb.Shutdown()
 	EventRecorder := eb.NewRecorder(scheme.Scheme, v1.EventSource{Component: path.Join(cfg.NodeName, "pod-controller")})
+	nodeProvider.SetEventRecorder(EventRecorder)
 
 	podInformerFactory, scmInformerFactory, stopper := setupInformers(ctx, localClient, cfg.NodeName)
 	defer close(stopper)

pkg/virtualkubelet/config.go

@@ -254,8 +254,7 @@ const (
 // ForwardTarget selects where the forwarding session ends (the login node, or the
 // compute node's own sshd) and ForwardMode whether sshd forwards or session
 // commands relay. It covers the same ground as the wstunnel shadow, in the opposite
-// direction; it does not give the offloaded pod access back into the cluster (see
-// Network.FullMesh).
+// direction; Reverse gives the offloaded pod access to listed cluster endpoints.
 type SSHTunnel struct {
 	// LoginHost is the SSH login node to forward through (required)
 	LoginHost string `yaml:"LoginHost,omitempty"`
@@ -300,7 +299,8 @@ type SSHTunnel struct {
 	// ExecConnectCommand is the relay command of the "exec" mode, run on the login
 	// node. It must relay a connection on its stdin and stdout, and defaults to
 	// "nc": `<command> <compute node> <port>` with ForwardTarget "login",
-	// `<command> <compute node> 22` with "node".
+	// `<command> <compute node> 22` with "node". Reverse.NodeExec "srun" also runs
+	// `<command> 127.0.0.1 22` inside the job's allocation.
 	ExecConnectCommand string `yaml:"ExecConnectCommand,omitempty"`
 	// NodeHostKeys selects how compute node host keys are checked on every session
 	// that ends on a compute node: "verify" (default) checks them against
@@ -347,4 +347,42 @@ type SSHTunnel struct {
 	// exception: GSSAPIDelegateCredentials=yes restores Kerberos delegation on the
 	// login hop.
 	ExtraOptions []string `yaml:"ExtraOptions,omitempty"`
+	// Reverse lets offloaded pods open connections back into the cluster through
+	// the same login node, see the interlink.eu/reverse-* annotations.
+	Reverse SSHReverse `yaml:"Reverse,omitempty"`
+}
+
+// SSHReverse configures connections opened by offloaded pods towards cluster
+// endpoints. Each forward is made by the compute node's own sshd, or by a
+// user-space sshd as a fallback, with `ssh -R` to a unix socket in the pod's socket
+// directory; a relay the job starts puts the pod's loopback address in front of
+// those sockets and accepts only the job's user. Nothing on the compute node has
+// to reach the cluster.
+type SSHReverse struct {
+	// Enabled lets pods ask for it (default false). A pod that asks while this is
+	// false is refused at creation rather than run without its tunnel.
+	Enabled bool `yaml:"Enabled,omitempty"`
+	// Sshd selects which sshd makes the forwards on the compute node: "node"
+	// (default) is the node's own sshd, reached like ForwardTarget "node" and
+	// checked against NodeHostKeys; "userspace" is an sshd the shadow starts in the
+	// job as the job's user, for sites whose node sshd refuses forwarding.
+	Sshd string `yaml:"Sshd,omitempty"`
+	// AllowSocks lets pods ask for a SOCKS5 proxy with
+	// interlink.eu/reverse-socks-port (default false). The proxy listens on
+	// 127.0.0.1 of the compute node, where every user of that node can reach
+	// through it any endpoint the shadow reaches in the cluster.
+	AllowSocks bool `yaml:"AllowSocks,omitempty"`
+	// NodeExec is how the shadow reaches the compute node: "ssh" (default) opens
+	// node:22 through the login node as ForwardMode says, which the site must
+	// allow into a node where the user has a job; "srun" runs `srun
+	// --jobid=<job> --overlap` on the login node, inside the job's own allocation.
+	// srun is a last resort: lower throughput, and with Kerberos it delegates the
+	// ticket to the login node. Set here only, never per pod.
+	NodeExec string `yaml:"NodeExec,omitempty"`
+	// SshdPath is the sshd binary on the compute node, for Sshd "userspace" only
+	// (default "/usr/sbin/sshd")
+	SshdPath string `yaml:"SshdPath,omitempty"`
+	// PythonPath is the python3 (3.6 or later) on the compute node that runs the
+	// job's owner-only relay, isolated with -I (default "python3")
+	PythonPath string `yaml:"PythonPath,omitempty"`
 }

pkg/virtualkubelet/shadow_ssh.go

@@ -62,7 +62,7 @@ func NormalizeShadowConfig(config *Config) error {
 	// Mesh needs the compute node to dial out to the cluster, which is exactly what
 	// an SSH-only site cannot do. Combining the two is tracked separately.
 	if n.FullMesh {
-		return fmt.Errorf("Network.FullMesh is not supported with ShadowMode %q: the SSH shadow only exposes the offloaded pod's ports to the cluster, it does not give the pod access back into it (see interlink-hq/interLink#548)", ShadowModeSSH)
+		return fmt.Errorf("Network.FullMesh is not supported with ShadowMode %q: the SSH shadow has no mesh; pods reach cluster endpoints with Network.SSH.Reverse (see interlink-hq/interLink#548)", ShadowModeSSH)
 	}
 
 	s := &n.SSH
@@ -90,6 +90,10 @@ func NormalizeShadowConfig(config *Config) error {
 		s.ReplicateCredentials = &replicate
 	}
 
+	if err := normalizeReverseConfig(s); err != nil {
+		return err
+	}
+
 	s.ForwardMode = strings.ToLower(strings.TrimSpace(s.ForwardMode))
 	switch s.ForwardMode {
 	case "":
@@ -160,10 +164,44 @@ func NormalizeShadowConfig(config *Config) error {
 	return nil
 }
 
+// normalizeReverseConfig defaults and validates Network.SSH.Reverse.
+func normalizeReverseConfig(s *SSHTunnel) error {
+	r := &s.Reverse
+	r.NodeExec = strings.ToLower(strings.TrimSpace(r.NodeExec))
+	switch r.NodeExec {
+	case "":
+		r.NodeExec = SSHReverseNodeExecSSH
+	case SSHReverseNodeExecSSH, SSHReverseNodeExecSrun:
+	default:
+		return fmt.Errorf("unknown Network.SSH.Reverse.NodeExec %q: expected %q or %q", r.NodeExec, SSHReverseNodeExecSSH, SSHReverseNodeExecSrun)
+	}
+	if r.NodeExec == SSHReverseNodeExecSrun && strings.TrimSpace(s.ExecConnectCommand) == "" {
+		s.ExecConnectCommand = DefaultSSHExecConnectCommand
+	}
+	r.Sshd = strings.ToLower(strings.TrimSpace(r.Sshd))
+	switch r.Sshd {
+	case "":
+		r.Sshd = SSHReverseSshdNode
+	case SSHReverseSshdNode, SSHReverseSshdUserspace:
+	default:
+		return fmt.Errorf("unknown Network.SSH.Reverse.Sshd %q: expected %q or %q", r.Sshd, SSHReverseSshdNode, SSHReverseSshdUserspace)
+	}
+	if r.SshdPath == "" {
+		r.SshdPath = DefaultSSHReverseSshdPath
+	}
+	if r.PythonPath == "" {
+		r.PythonPath = DefaultSSHReversePythonPath
+	}
+	return nil
+}
+
 // usesNodeHops reports whether the shadow opens ssh sessions on compute nodes,
-// which is what NodeHostKeys governs.
+// which is what NodeHostKeys governs. Only a user-space sshd started through srun
+// avoids them: its host key is one the shadow generated.
 func usesNodeHops(s SSHTunnel) bool {
-	return s.ForwardTarget == SSHForwardTargetNode
+	r := s.Reverse
+	return s.ForwardTarget == SSHForwardTargetNode ||
+		(r.Enabled && (r.Sshd == SSHReverseSshdNode || r.NodeExec == SSHReverseNodeExecSSH))
 }
 
 // sshNodeWaitSeconds converts the configured wait into whole seconds for the shell