feat(api): add GET /vm/kvm-stats for KVM binary statistics

CHANGELOG.md

@@ -41,6 +41,11 @@ and this project adheres to
   exceed the vCPU count. Custom seccomp filters must include the new
   `blk_worker` thread category. See
   [block documentation](docs/block.md#threaded-mode-and-multiqueue).
+- [#6288](https://github.com/firecracker-microvm/firecracker/issues/6288): Added
+  a `GET /vm/kvm-stats` API endpoint that returns KVM's binary statistics
+  (`KVM_GET_STATS_FD`, Linux 5.14 or later) for the VM and each vCPU, with the
+  kind and unit of each, including histograms such as `halt_wait_hist`. See the
+  [metrics documentation](docs/metrics.md#kvm-statistics).
 
 ### Changed
 

docs/metrics.md

@@ -216,3 +216,77 @@ of full_key for below logic is `"vcpu.exit_io_in_agg.min_us"`
     else
         Unit is "Count"
 ```
+
+## KVM statistics
+
+Firecracker's metrics only cover the vCPU exits that reach Firecracker itself.
+For where the vCPUs spent the rest of their time (running guest code, exiting to
+KVM, halted, or halt-polling), `GET /vm/kvm-stats` returns KVM's binary
+statistics (`KVM_GET_STATS_FD`, Linux 5.14 or later) for the VM and each vCPU.
+These are the counters behind `/sys/kernel/debug/kvm`, but per vCPU rather than
+summed over the host, and with histograms such as `halt_wait_hist` (how long
+each halt waited, in nanoseconds) in full. Firecracker owns the VM and vCPU file
+descriptors, so this needs no privilege.
+
+The request is supported after the microVM starts, including after a snapshot
+is loaded. Each request reads the current values, which KVM keeps as running
+totals from when the VM and its vCPUs were created; reading them resets
+nothing. KVM statistics are not part of a snapshot, so a microVM loaded from a
+snapshot starts counting from zero. On a host kernel without binary statistics,
+the request fails with `400 Bad Request`.
+
+```bash
+curl --unix-socket /tmp/firecracker.socket -i \
+    -X GET 'http://localhost/vm/kvm-stats' \
+    -H 'Accept: application/json'
+```
+
+Each statistic is reported by its kernel name, with the kind and unit KVM
+declares for it, using the names QEMU's `query-stats` uses. The available
+statistics depend on the host kernel version and architecture.
+
+- `type`: `cumulative` (only increases), `instant`, `peak` (the highest value
+  seen), `linear-histogram` (bucket `i` counts values in
+  `[i * bucket_size, (i + 1) * bucket_size)`), or `log2-histogram` (bucket 0
+  counts zeros and bucket `i` counts values in `[2^(i-1), 2^i)`).
+- `unit`: `bytes`, `seconds`, `cycles`, or `boolean` (0 or 1). Absent for a
+  plain count.
+- `base` and `exponent`: the value is in `unit` scaled by `base^exponent`; for
+  example, nanoseconds for `seconds` with base 10 and exponent -9. Absent when
+  the exponent is 0.
+- `bucket_size`: the width of each bucket of a `linear-histogram`.
+- `value`: an integer, or a histogram's bucket counts.
+
+An abridged response:
+
+```json
+{
+    "vm": {
+        "pages_2m": {"type": "instant", "value": 60},
+        "remote_tlb_flush": {"type": "cumulative", "value": 3}
+    },
+    "vcpus": [
+        {
+            "index": 0,
+            "stats": {
+                "exits": {"type": "cumulative", "value": 18211},
+                "guest_mode": {"type": "instant", "unit": "boolean", "value": 0},
+                "halt_poll_success_ns": {
+                    "type": "cumulative",
+                    "unit": "seconds",
+                    "base": 10,
+                    "exponent": -9,
+                    "value": 81234
+                },
+                "halt_wait_hist": {
+                    "type": "log2-histogram",
+                    "unit": "seconds",
+                    "base": 10,
+                    "exponent": -9,
+                    "value": [0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 18, 40]
+                }
+            }
+        }
+    ]
+}
+```

resources/seccomp/aarch64-unknown-linux-musl.json

@@ -29,6 +29,10 @@
             {
                 "syscall": "read"
             },
+            {
+                "syscall": "pread64",
+                "comment": "Used to read KVM's binary statistics for GET /vm/kvm-stats"
+            },
             {
                 "syscall": "write"
             },

resources/seccomp/x86_64-unknown-linux-musl.json

@@ -29,6 +29,10 @@
             {
                 "syscall": "read"
             },
+            {
+                "syscall": "pread64",
+                "comment": "Used to read KVM's binary statistics for GET /vm/kvm-stats"
+            },
             {
                 "syscall": "write"
             },

src/firecracker/src/api_server/parsed_request.rs

@@ -85,9 +85,14 @@ impl TryFrom<&Request> for ParsedRequest {
             (Method::Get, "", None) => parse_get_instance_info(),
             (Method::Get, "balloon", None) => parse_get_balloon(path_tokens),
             (Method::Get, "version", None) => parse_get_version(),
-            (Method::Get, "vm", None) if path_tokens.next() == Some("config") => {
-                Ok(ParsedRequest::new_sync(VmmAction::GetFullVmConfig))
-            }
+            (Method::Get, "vm", None) => match path_tokens.next() {
+                Some("config") => Ok(ParsedRequest::new_sync(VmmAction::GetFullVmConfig)),
+                Some("kvm-stats") => Ok(ParsedRequest::new_sync(VmmAction::GetKvmStats)),
+                _ => Err(RequestError::InvalidPathMethod(
+                    path.to_string(),
+                    Method::Get,
+                )),
+            },
             (Method::Get, "machine-config", None) => parse_get_machine_config(),
             (Method::Get, "mmds", None) => parse_get_mmds(),
             (Method::Get, "hotplug", None) if path_tokens.next() == Some("memory") => {
@@ -210,6 +215,7 @@ impl ParsedRequest {
                     &serde_json::json!({ "firecracker_version": version.as_str() }),
                 ),
                 VmmData::FullVmConfig(config) => Self::success_response_with_data(config),
+                VmmData::KvmStats(stats) => Self::success_response_with_data(stats),
             },
             Err(vmm_action_error) => {
                 let mut response = match vmm_action_error {
@@ -365,6 +371,7 @@ pub mod tests {
     use vmm::vmm_config::balloon::{BalloonDeviceConfig, BalloonStats};
     use vmm::vmm_config::instance_info::InstanceInfo;
     use vmm::vmm_config::machine_config::MachineConfig;
+    use vmm::vstate::kvm_stats::KvmStats;
 
     use super::*;
 
@@ -615,6 +622,9 @@ pub mod tests {
                 VmmData::FullVmConfig(cfg) => {
                     http_response(&serde_json::to_string(cfg).unwrap(), 200)
                 }
+                VmmData::KvmStats(stats) => {
+                    http_response(&serde_json::to_string(stats).unwrap(), 200)
+                }
                 VmmData::MachineConfiguration(cfg) => {
                     http_response(&serde_json::to_string(cfg).unwrap(), 200)
                 }
@@ -645,6 +655,10 @@ pub mod tests {
         }));
         verify_ok_response_with(VmmData::Empty);
         verify_ok_response_with(VmmData::FullVmConfig(VmmConfig::default()));
+        verify_ok_response_with(VmmData::KvmStats(KvmStats {
+            vm: Default::default(),
+            vcpus: vec![],
+        }));
         verify_ok_response_with(VmmData::MachineConfiguration(MachineConfig::default()));
         verify_ok_response_with(VmmData::MmdsValue(serde_json::from_str("{}").unwrap()));
         verify_ok_response_with(VmmData::InstanceInformation(InstanceInfo::default()));
@@ -697,6 +711,31 @@ pub mod tests {
         ParsedRequest::try_from(&req).unwrap();
     }
 
+    #[test]
+    fn test_try_from_get_vm() {
+        for (path, action) in [
+            ("/vm/config", Some(VmmAction::GetFullVmConfig)),
+            ("/vm/kvm-stats", Some(VmmAction::GetKvmStats)),
+            ("/vm", None),
+            ("/vm/unknown", None),
+        ] {
+            let (mut sender, receiver) = UnixStream::pair().unwrap();
+            let mut connection = HttpConnection::new(receiver);
+            sender
+                .write_all(http_request("GET", path, None).as_bytes())
+                .unwrap();
+            connection.try_read().unwrap();
+            let req = connection.pop_parsed_request().unwrap();
+            match action {
+                Some(action) => assert_eq!(
+                    vmm_action_from_request(ParsedRequest::try_from(&req).unwrap()),
+                    action
+                ),
+                None => assert!(ParsedRequest::try_from(&req).is_err(), "{path}"),
+            }
+        }
+    }
+
     #[test]
     fn test_try_from_get_balloon_hinting() {
         let (mut sender, receiver) = UnixStream::pair().unwrap();

src/firecracker/swagger/firecracker.yaml

@@ -972,6 +972,30 @@ paths:
           schema:
             $ref: "#/definitions/Error"
 
+  /vm/kvm-stats:
+    get:
+      summary: Returns KVM's binary statistics for the VM and each vCPU. Post-boot only.
+      description:
+        Reads KVM's binary statistics (KVM_GET_STATS_FD) for the VM and each vCPU at the time of
+        the request. Values are running totals since the VM and its vCPUs were created, so they
+        start from zero in a microVM loaded from a snapshot. Needs Linux 5.14 or later; the request
+        fails on older host kernels. The available statistics and their names depend on the host
+        kernel version and architecture.
+      operationId: describeKvmStats
+      responses:
+        200:
+          description: The KVM statistics
+          schema:
+            $ref: "#/definitions/KvmStats"
+        400:
+          description: KVM statistics are not available
+          schema:
+            $ref: "#/definitions/Error"
+        default:
+          description: Internal server error
+          schema:
+            $ref: "#/definitions/Error"
+
   /vsock:
     put:
       summary: Creates/updates a vsock device. Pre-boot only.
@@ -1528,6 +1552,88 @@ definitions:
         description: MicroVM hypervisor build version.
         type: string
 
+  KvmStats:
+    type: object
+    description:
+      KVM's binary statistics for the VM and each vCPU, by name. The names, kinds and units are
+      those KVM declares, so they depend on the host kernel version and architecture.
+    required:
+      - vm
+      - vcpus
+    properties:
+      vm:
+        type: object
+        description: The VM's statistics, by name.
+        additionalProperties:
+          $ref: "#/definitions/KvmStat"
+      vcpus:
+        type: array
+        items:
+          $ref: "#/definitions/KvmVcpuStats"
+
+  KvmVcpuStats:
+    type: object
+    description: A vCPU's statistics.
+    required:
+      - index
+      - stats
+    properties:
+      index:
+        type: integer
+        description: The vCPU index.
+      stats:
+        type: object
+        description: The vCPU's statistics, by name.
+        additionalProperties:
+          $ref: "#/definitions/KvmStat"
+
+  KvmStat:
+    type: object
+    description:
+      A statistic, with the kind and unit KVM declares for it. The value is in unit scaled by
+      base^exponent; for example, nanoseconds for unit seconds, base 10 and exponent -9.
+    required:
+      - type
+      - value
+    properties:
+      type:
+        type: string
+        description:
+          The kind of statistic. cumulative only increases; instant can increase or decrease; peak
+          is the highest value seen. A linear-histogram's bucket i counts values in
+          [i * bucket_size, (i + 1) * bucket_size); a log2-histogram's bucket 0 counts zeros and
+          bucket i counts values in [2^(i-1), 2^i).
+        enum:
+          - cumulative
+          - instant
+          - peak
+          - linear-histogram
+          - log2-histogram
+      unit:
+        type: string
+        description: The unit. Absent for a plain count. A boolean value is 0 or 1.
+        enum:
+          - bytes
+          - seconds
+          - cycles
+          - boolean
+      base:
+        type: integer
+        description: The base of the unit's multiple. Present when exponent is not 0.
+        enum:
+          - 2
+          - 10
+      exponent:
+        type: integer
+        description: The exponent of the unit's multiple. Absent when 0.
+      bucket_size:
+        type: integer
+        description: The width of each bucket. Present for linear-histogram statistics.
+      value:
+        description:
+          The value. An integer for cumulative, instant and peak statistics; an array of integer
+          bucket counts for histograms.
+
   Logger:
     type: object
     description:

src/vmm/src/rpc_interface.rs

@@ -46,6 +46,7 @@ use crate::vmm_config::serial::SerialConfig;
 use crate::vmm_config::snapshot::{CreateSnapshotParams, LoadSnapshotParams, SnapshotType};
 use crate::vmm_config::vsock::{VsockConfigError, VsockDeviceConfig};
 use crate::vmm_config::{self, RateLimiterUpdate};
+use crate::vstate::kvm_stats::{KvmStats, KvmStatsError};
 
 /// This enum represents the public interface of the VMM. Each action contains various
 /// bits of information (ids, paths, etc.).
@@ -71,6 +72,8 @@ pub enum VmmAction {
     GetBalloonStats,
     /// Get complete microVM configuration in JSON format.
     GetFullVmConfig,
+    /// Get KVM's binary statistics for the VM and each vCPU.
+    GetKvmStats,
     /// Get MMDS contents.
     GetMMDS,
     /// Get the machine configuration of the microVM.
@@ -184,6 +187,8 @@ pub enum VmmActionError {
     MemoryHotplugUpdate(VmmError),
     /// Internal VMM error: {0}
     InternalVmm(#[from] VmmError),
+    /// KVM statistics error: {0}
+    KvmStats(#[from] KvmStatsError),
     /// Load snapshot error: {0}
     LoadSnapshot(#[from] LoadSnapshotError),
     /// Logger error: {0}
@@ -239,6 +244,8 @@ pub enum VmmData {
     Empty,
     /// The complete microVM configuration in JSON format.
     FullVmConfig(VmmConfig),
+    /// KVM's binary statistics for the VM and each vCPU.
+    KvmStats(KvmStats),
     /// The microVM configuration represented by `VmConfig`.
     MachineConfiguration(MachineConfig),
     /// Mmds contents.
@@ -511,6 +518,7 @@ impl<'a> PrebootApiController<'a> {
             | Pause
             | Resume
             | GetBalloonStats
+            | GetKvmStats
             | GetMemoryHotplugStatus
             | UpdateBalloon(_)
             | UpdateBalloonStatistics(_)
@@ -729,6 +737,9 @@ impl RuntimeApiController {
             GetFullVmConfig => Ok(VmmData::FullVmConfig(
                 self.vmm.lock().expect("Poisoned lock").full_config(),
             )),
+            GetKvmStats => Ok(VmmData::KvmStats(
+                self.vmm.lock().expect("Poisoned lock").vm.kvm_stats()?,
+            )),
             GetMemoryHotplugStatus => self
                 .vmm
                 .lock()
@@ -1228,6 +1239,7 @@ mod tests {
         check_unsupported(preboot_request(VmmAction::Pause));
         check_unsupported(preboot_request(VmmAction::Resume));
         check_unsupported(preboot_request(VmmAction::GetBalloonStats));
+        check_unsupported(preboot_request(VmmAction::GetKvmStats));
         check_unsupported(preboot_request(VmmAction::UpdateBalloon(
             BalloonUpdateConfig { amount_mib: 0 },
         )));
@@ -1285,6 +1297,18 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_runtime_get_kvm_stats() {
+        match runtime_request(VmmAction::GetKvmStats) {
+            Ok(VmmData::KvmStats(stats)) => {
+                assert_eq!(stats.vcpus.len(), 1);
+                assert!(stats.vcpus[0].stats.contains_key("exits"));
+            }
+            Err(VmmActionError::KvmStats(KvmStatsError::Unsupported)) => {}
+            res => panic!("{:?}", res),
+        }
+    }
+
     #[test]
     fn test_runtime_disallowed() {
         fn check_unsupported(res: Result<VmmData, VmmActionError>) {