[sled-agent] Add support for Support Bundle command execution (#7011)

This PR lays the groundwork for executing system commands and returning
their output as raw data (both on success and failure) via the internal
sled-agent HTTP API. Command execution collects both stdout and stdout
in an interleaved fashion so that the command output is exactly how an
operator would have seen it if they were sitting at the terminal.

Author: papertigers

illumos-utils/src/lib.rs

@@ -34,6 +34,7 @@ pub mod zone;
 pub mod zpool;
 
 pub const PFEXEC: &str = "/usr/bin/pfexec";
+pub const ZONEADM: &str = "/usr/sbin/zoneadm";
 
 #[derive(Debug)]
 pub struct CommandFailureInfo {

openapi/sled-agent.json

@@ -673,6 +673,48 @@
         }
       }
     },
+    "/support/ipadm-info": {
+      "get": {
+        "operationId": "support_ipadm_info",
+        "responses": {
+          "200": {
+            "description": "",
+            "content": {
+              "*/*": {
+                "schema": {}
+              }
+            }
+          },
+          "4XX": {
+            "$ref": "#/components/responses/Error"
+          },
+          "5XX": {
+            "$ref": "#/components/responses/Error"
+          }
+        }
+      }
+    },
+    "/support/zoneadm-info": {
+      "get": {
+        "operationId": "support_zoneadm_info",
+        "responses": {
+          "200": {
+            "description": "",
+            "content": {
+              "*/*": {
+                "schema": {}
+              }
+            }
+          },
+          "4XX": {
+            "$ref": "#/components/responses/Error"
+          },
+          "5XX": {
+            "$ref": "#/components/responses/Error"
+          }
+        }
+      }
+    },
     "/switch-ports": {
       "post": {
         "operationId": "uplink_ensure",

sled-agent/api/src/lib.rs

@@ -529,6 +529,22 @@ pub trait SledAgentApi {
         request_context: RequestContext<Self::Context>,
         body: TypedBody<ExternalIpGatewayMap>,
     ) -> Result<HttpResponseUpdatedNoContent, HttpError>;
+
+    #[endpoint {
+        method = GET,
+        path = "/support/zoneadm-info",
+    }]
+    async fn support_zoneadm_info(
+        request_context: RequestContext<Self::Context>,
+    ) -> Result<HttpResponseOk<FreeformBody>, HttpError>;
+
+    #[endpoint {
+        method = GET,
+        path = "/support/ipadm-info",
+    }]
+    async fn support_ipadm_info(
+        request_context: RequestContext<Self::Context>,
+    ) -> Result<HttpResponseOk<FreeformBody>, HttpError>;
 }
 
 #[derive(Clone, Debug, Deserialize, JsonSchema, Serialize)]

sled-agent/src/http_entrypoints.rs

@@ -6,6 +6,7 @@
 
 use super::sled_agent::SledAgent;
 use crate::sled_agent::Error as SledAgentError;
+use crate::support_bundle::SupportBundleCommandHttpOutput;
 use crate::zone_bundle::BundleError;
 use bootstore::schemes::v0::NetworkConfig;
 use camino::Utf8PathBuf;
@@ -770,4 +771,28 @@ impl SledAgentApi for SledAgentImpl {
         sa.set_eip_gateways(body.into_inner()).await?;
         Ok(HttpResponseUpdatedNoContent())
     }
+
+    async fn support_zoneadm_info(
+        request_context: RequestContext<Self::Context>,
+    ) -> Result<HttpResponseOk<FreeformBody>, HttpError> {
+        let sa = request_context.context();
+        let res = sa.support_zoneadm_info().await;
+        Ok(HttpResponseOk(FreeformBody(res.get_output().into())))
+    }
+
+    async fn support_ipadm_info(
+        request_context: RequestContext<Self::Context>,
+    ) -> Result<HttpResponseOk<FreeformBody>, HttpError> {
+        let sa = request_context.context();
+        let output = sa
+            .support_ipadm_info()
+            .await
+            .into_iter()
+            .map(|cmd| cmd.get_output())
+            .collect::<Vec<_>>()
+            .as_slice()
+            .join("\n\n");
+
+        Ok(HttpResponseOk(FreeformBody(output.into())))
+    }
 }

sled-agent/src/lib.rs

@@ -36,6 +36,7 @@ pub mod server;
 pub mod services;
 mod sled_agent;
 mod storage_monitor;
+mod support_bundle;
 mod swap_device;
 mod updates;
 mod vmm_reservoir;

sled-agent/src/sim/http_entrypoints.rs

@@ -542,6 +542,18 @@ impl SledAgentApi for SledAgentSimImpl {
     ) -> Result<HttpResponseOk<BootstoreStatus>, HttpError> {
         method_unimplemented()
     }
+
+    async fn support_zoneadm_info(
+        _request_context: RequestContext<Self::Context>,
+    ) -> Result<HttpResponseOk<FreeformBody>, HttpError> {
+        method_unimplemented()
+    }
+
+    async fn support_ipadm_info(
+        _request_context: RequestContext<Self::Context>,
+    ) -> Result<HttpResponseOk<FreeformBody>, HttpError> {
+        method_unimplemented()
+    }
 }
 
 fn method_unimplemented<T>() -> Result<T, HttpError> {

sled-agent/src/sled_agent.rs

@@ -19,10 +19,11 @@ use crate::params::OmicronZoneTypeExt;
 use crate::probe_manager::ProbeManager;
 use crate::services::{self, ServiceManager};
 use crate::storage_monitor::StorageMonitorHandle;
+use crate::support_bundle::{SupportBundleCmdError, SupportBundleCmdOutput};
 use crate::updates::{ConfigUpdates, UpdateManager};
 use crate::vmm_reservoir::{ReservoirMode, VmmReservoirManager};
-use crate::zone_bundle;
 use crate::zone_bundle::BundleError;
+use crate::{support_bundle, zone_bundle};
 use bootstore::schemes::v0 as bootstore;
 use camino::Utf8PathBuf;
 use derive_more::From;
@@ -1350,6 +1351,18 @@ impl SledAgent {
             datasets,
         })
     }
+
+    pub(crate) async fn support_zoneadm_info(
+        &self,
+    ) -> Result<SupportBundleCmdOutput, SupportBundleCmdError> {
+        support_bundle::zoneadm_info().await
+    }
+
+    pub(crate) async fn support_ipadm_info(
+        &self,
+    ) -> Vec<Result<SupportBundleCmdOutput, SupportBundleCmdError>> {
+        support_bundle::ipadm_info().await
+    }
 }
 
 #[derive(From, thiserror::Error, Debug)]