Support for External Jumbo Frames (RFD 689)

Cargo.toml

@@ -611,8 +611,8 @@ ntp-admin-client = { path = "clients/ntp-admin-client" }
 ntp-admin-v1-client = { path = "clients/ntp-admin-v1-client" }
 ntp-admin-types = { path = "ntp-admin/types" }
 ntp-admin-types-versions = { path = "ntp-admin/types/versions" }
-mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "7696ee48d5ee29a917dea459e281fe2e8ff20513" }
-ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "7696ee48d5ee29a917dea459e281fe2e8ff20513" }
+mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "b27bea5344fb21b3b88af6717b969e31ee6a9ccb" }
+ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "b27bea5344fb21b3b88af6717b969e31ee6a9ccb" }
 multimap = "0.10.1"
 nexus-auth = { path = "nexus/auth" }
 nexus-background-task-interface = { path = "nexus/background-task-interface" }
@@ -674,16 +674,16 @@ omicron-workspace-hack = "0.1.0"
 omicron-zone-package = "0.12.2"
 oxide-client = { path = "clients/oxide-client" }
 oxide-tokio-rt = "0.1.4"
-oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "2c6efefe14321dafe7e9e80129d38316adb2d238", features = [ "api", "std" ] }
+oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "84e91b62621406f38e4d0870a92bafacad96536e", features = [ "api", "std" ] }
 oxlog = { path = "dev-tools/oxlog" }
-oxnet = "0.1.4"
+oxnet = "0.1.5"
 once_cell = "1.21.3"
 openapi-lint = { git = "https://github.com/oxidecomputer/openapi-lint", branch = "main" }
 openapiv3 = "2.2.0"
 # must match samael's crate!
 openssl = "0.10"
 openssl-sys = "0.9"
-opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "2c6efefe14321dafe7e9e80129d38316adb2d238" }
+opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "84e91b62621406f38e4d0870a92bafacad96536e" }
 oso = "0.27"
 owo-colors = "4.2.2"
 oximeter = { path = "oximeter/oximeter" }
@@ -749,7 +749,7 @@ rats-corim = { git = "https://github.com/oxidecomputer/rats-corim.git", rev = "f
 raw-cpuid = { git = "https://github.com/oxidecomputer/rust-cpuid.git", rev = "a4cf01df76f35430ff5d39dc2fe470bcb953503b" }
 rayon = "1.10"
 rcgen = "0.12.1"
-rdb-types = { git = "https://github.com/oxidecomputer/maghemite", rev = "7696ee48d5ee29a917dea459e281fe2e8ff20513" }
+mg-api-types = { git = "https://github.com/oxidecomputer/maghemite", rev = "b27bea5344fb21b3b88af6717b969e31ee6a9ccb" }
 reconfigurator-cli = { path = "dev-tools/reconfigurator-cli" }
 reedline = "0.40.0"
 ref-cast = "1.0"

common/src/address.rs

@@ -24,6 +24,16 @@ pub const AZ_PREFIX: u8 = 48;
 pub const RACK_PREFIX: u8 = 56;
 pub const SLED_PREFIX: u8 = 64;
 
+/// Default Ethernet MTU for external-facing OPTE ports, in bytes. Used when
+/// the per-instance jumbo-frames opt-in is unset or when the fleet-wide opt-in
+/// is disabled.
+pub const EXTERNAL_DEFAULT_MTU: u32 = 1500;
+
+/// Effective MTU for external-facing OPTE ports when jumbo frames have been
+/// opted into. 500 bytes of headroom under the 9000 byte underlay MTU leaves
+/// room for encapsulation overhead.
+pub const EXTERNAL_JUMBO_FRAMES_MTU: u32 = 8500;
+
 // Multicast constants
 
 /// IPv4 Source-Specific Multicast (SSM) subnet.

end-to-end-tests/src/instance_launch.rs

@@ -88,6 +88,7 @@ async fn instance_launch() -> Result<()> {
             anti_affinity_groups: Vec::new(),
             cpu_platform: None,
             multicast_groups: Vec::new(),
+            enable_jumbo_frames: false,
         })
         .send()
         .await?;

illumos-utils/src/opte/non_illumos.rs

@@ -209,7 +209,7 @@ impl Handle {
         &self,
         name: &str,
         cfg: VpcCfg,
-        _: bool,
+        _mtu: Option<u32>,
     ) -> Result<NoResp, OpteError> {
         let name = name.to_string();
         let IpCfg::Ipv4(ip_cfg) = cfg.ip_cfg else {

illumos-utils/src/opte/port_manager.rs

@@ -147,6 +147,9 @@ pub struct PortCreateParams<'a> {
     pub firewall_rules: &'a [ResolvedVpcFirewallRule],
     pub dhcp_config: DhcpCfg,
     pub attached_subnets: Vec<AttachedSubnet>,
+    /// MTU to set on the xde device, in bytes. If `None`, OPTE applies its
+    /// default (1500). Used by jumbo-frame opt-in.
+    pub mtu: Option<u32>,
 }
 
 impl<'a> TryFrom<&PortCreateParams<'a>> for IpCfg {
@@ -371,6 +374,7 @@ impl PortManager {
             firewall_rules,
             dhcp_config,
             attached_subnets: _,
+            mtu,
         } = params;
         let is_service =
             matches!(nic.kind, NetworkInterfaceKind::Service { .. });
@@ -410,7 +414,7 @@ impl PortManager {
         );
         let hdl = {
             let hdl = Handle::new()?;
-            hdl.create_xde(&port_name, vpc_cfg, /* passthru = */ false)?;
+            hdl.create_xde(&port_name, vpc_cfg, mtu)?;
             hdl
         };
         let (port, ticket) = {
@@ -1335,6 +1339,7 @@ mod tests {
                     dns6_servers: Vec::new(),
                 },
                 attached_subnets: vec![],
+                mtu: None,
             })
             .unwrap();
 
@@ -1514,6 +1519,7 @@ mod tests {
                     dns6_servers: Vec::new(),
                 },
                 attached_subnets: vec![],
+                mtu: None,
             })
             .unwrap();
 
@@ -1685,6 +1691,7 @@ mod tests {
                 dns6_servers: vec![],
             },
             attached_subnets: vec![],
+            mtu: None,
         };
         let IpCfg::Ipv4(oxide_vpc::api::Ipv4Cfg {
             vpc_subnet,
@@ -1758,6 +1765,7 @@ mod tests {
                 dns6_servers: vec![],
             },
             attached_subnets: vec![],
+            mtu: None,
         };
         let IpCfg::Ipv6(oxide_vpc::api::Ipv6Cfg {
             vpc_subnet,
@@ -1842,6 +1850,7 @@ mod tests {
                 dns6_servers: vec![],
             },
             attached_subnets: vec![],
+            mtu: None,
         };
         let IpCfg::DualStack { ipv4, ipv6 } = IpCfg::try_from(&prs).unwrap()
         else {
@@ -1932,6 +1941,7 @@ mod tests {
                 dns6_servers: vec![],
             },
             attached_subnets: vec![],
+            mtu: None,
         };
         let _ = IpCfg::try_from(&prs).expect_err(
             "Should fail to convert with public IPv6 and private IPv4",
@@ -1978,6 +1988,7 @@ mod tests {
                 dns6_servers: vec![],
             },
             attached_subnets: vec![],
+            mtu: None,
         };
         let _ = IpCfg::try_from(&prs).expect_err(
             "Should fail to convert with public IPv4 and private IPv6",

nexus/Cargo.toml

@@ -96,7 +96,7 @@ qorb.workspace = true
 rand.workspace = true
 range-requests.workspace = true
 ref-cast.workspace = true
-rdb-types.workspace = true
+mg-api-types.workspace = true
 regex.workspace = true
 reqwest = { workspace = true, features = ["http2", "json"] }
 reqwest012 = { workspace = true }

nexus/db-model/src/instance.rs

@@ -136,6 +136,13 @@ pub struct Instance {
     /// a sled by any other constraints the instance will be incarnated with the
     /// most general CPU platform supported by the selected sled.
     pub cpu_platform: Option<InstanceCpuPlatform>,
+
+    /// When true, the instance has opted in to jumbo frames (8500 byte MTU)
+    /// on its primary OPTE interface. The effective MTU also depends on the
+    /// fleet-wide setting in `system_networking_settings`; if that flag is off
+    /// the OPTE port is created with the default MTU regardless of this field.
+    /// Changes to this field only take effect on the next instance restart.
+    pub enable_jumbo_frames: bool,
 }
 
 impl Instance {
@@ -185,6 +192,7 @@ impl Instance {
             boot_disk_id: None,
             intended_state,
             cpu_platform: params.cpu_platform.map(Into::into),
+            enable_jumbo_frames: params.enable_jumbo_frames,
         }
     }
 

nexus/db-model/src/lib.rs

@@ -195,6 +195,7 @@ mod serde_time_delta;
 mod silo_auth_settings;
 mod switch_interface;
 mod switch_port;
+mod system_networking_settings;
 mod target_release;
 mod trust_quorum;
 mod v2p_mapping;
@@ -373,6 +374,7 @@ pub use support_bundle::*;
 pub use switch::*;
 pub use switch_interface::*;
 pub use switch_port::*;
+pub use system_networking_settings::*;
 pub use target_release::*;
 pub use trust_quorum::*;
 pub use tuf_repo::*;

nexus/db-model/src/schema_versions.rs

@@ -16,7 +16,7 @@ use std::{collections::BTreeMap, sync::LazyLock};
 ///
 /// This must be updated when you change the database schema.  Refer to
 /// schema/crdb/README.adoc in the root of this repository for details.
-pub const SCHEMA_VERSION: Version = Version::new(261, 0, 0);
+pub const SCHEMA_VERSION: Version = Version::new(262, 0, 0);
 
 /// List of all past database schema versions, in *reverse* order
 ///
@@ -28,6 +28,7 @@ pub static KNOWN_VERSIONS: LazyLock<Vec<KnownVersion>> = LazyLock::new(|| {
         // |  leaving the first copy as an example for the next person.
         // v
         // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"),
+        KnownVersion::new(262, "external-jumbo-frames"),
         KnownVersion::new(261, "remove-add-zones-with-mupdate-override"),
         KnownVersion::new(260, "ereport-trim-serial-trailing-nulls"),
         KnownVersion::new(259, "vmm-failure-reason"),

nexus/db-model/src/system_networking_settings.rs

@@ -0,0 +1,37 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+use chrono::{DateTime, Utc};
+use nexus_db_schema::schema::system_networking_settings;
+use serde::{Deserialize, Serialize};
+
+/// Singleton row holding fleet-wide networking settings.
+#[derive(
+    Queryable,
+    Insertable,
+    Debug,
+    Clone,
+    Selectable,
+    Serialize,
+    Deserialize,
+    AsChangeset,
+)]
+#[diesel(table_name = system_networking_settings)]
+pub struct SystemNetworkingSettings {
+    pub singleton: bool,
+    pub time_created: DateTime<Utc>,
+    pub time_modified: DateTime<Utc>,
+    /// When true, end users may opt in to jumbo frames on the primary
+    /// interface of an instance. When false, the per-instance opt-in is
+    /// ignored and OPTE ports are created with the default MTU.
+    pub external_jumbo_frames_opt_in_enabled: bool,
+}
+
+/// Updates to the [`SystemNetworkingSettings`] singleton.
+#[derive(AsChangeset)]
+#[diesel(table_name = system_networking_settings)]
+pub struct SystemNetworkingSettingsUpdate {
+    pub external_jumbo_frames_opt_in_enabled: Option<bool>,
+    pub time_modified: DateTime<Utc>,
+}

nexus/db-queries/src/db/datastore/instance.rs

@@ -280,6 +280,16 @@ impl From<InstanceAndActiveVmm> for external::Instance {
     }
 }
 
+impl From<InstanceAndActiveVmm>
+    for nexus_types::external_api::instance::Instance
+{
+    fn from(value: InstanceAndActiveVmm) -> Self {
+        let enable_jumbo_frames = value.instance.enable_jumbo_frames;
+        let inner: external::Instance = value.into();
+        Self { inner, enable_jumbo_frames }
+    }
+}
+
 /// The totality of database records describing the current state of
 /// an instance: the [`Instance`] record itself, along with its active [`Vmm`],
 /// target [`Vmm`], and current [`Migration`], if they exist.
@@ -1203,6 +1213,26 @@ impl DataStore {
         Ok(instance_and_vmm)
     }
 
+    /// Update the per-instance jumbo-frames opt-in. Changes take effect on the
+    /// next instance restart.
+    pub async fn instance_set_enable_jumbo_frames(
+        &self,
+        opctx: &OpContext,
+        authz_instance: &authz::Instance,
+        enable_jumbo_frames: bool,
+    ) -> Result<(), Error> {
+        opctx.authorize(authz::Action::Modify, authz_instance).await?;
+        use nexus_db_schema::schema::instance::dsl as instance_dsl;
+        diesel::update(instance_dsl::instance)
+            .filter(instance_dsl::id.eq(authz_instance.id()))
+            .filter(instance_dsl::time_deleted.is_null())
+            .set(instance_dsl::enable_jumbo_frames.eq(enable_jumbo_frames))
+            .execute_async(&*self.pool_connection_authorized(opctx).await?)
+            .await
+            .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?;
+        Ok(())
+    }
+
     /// Set the boot disk on an instance, bypassing the rest of an instance
     /// update. You probably don't need this; it's only used at the end of
     /// instance creation, since the boot disk can't be set until the new
@@ -2376,6 +2406,7 @@ mod tests {
                         auto_restart_policy: Default::default(),
                         anti_affinity_groups: Vec::new(),
                         multicast_groups: Vec::new(),
+                        enable_jumbo_frames: false,
                     },
                 ),
             )

nexus/db-queries/src/db/datastore/migration.rs

@@ -242,6 +242,7 @@ mod tests {
                         auto_restart_policy: Default::default(),
                         anti_affinity_groups: Vec::new(),
                         multicast_groups: Vec::new(),
+                        enable_jumbo_frames: false,
                     },
                 ),
             )

nexus/db-queries/src/db/datastore/mod.rs

@@ -137,6 +137,7 @@ mod support_bundle;
 mod switch;
 mod switch_interface;
 mod switch_port;
+mod system_networking_settings;
 mod target_release;
 #[cfg(test)]
 pub(crate) mod test_utils;

nexus/db-queries/src/db/datastore/sled.rs

@@ -4423,6 +4423,7 @@ pub(in crate::db::datastore) mod test {
                         auto_restart_policy: Default::default(),
                         anti_affinity_groups: Vec::new(),
                         multicast_groups: Vec::new(),
+                        enable_jumbo_frames: false,
                     },
                 ),
             )

nexus/db-queries/src/db/datastore/system_networking_settings.rs

@@ -0,0 +1,59 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Datastore access for fleet-wide networking settings.
+
+use super::DataStore;
+use crate::authz;
+use crate::context::OpContext;
+use async_bb8_diesel::AsyncRunQueryDsl;
+use chrono::Utc;
+use diesel::prelude::*;
+use nexus_db_errors::ErrorHandler;
+use nexus_db_errors::public_error_from_diesel;
+use nexus_db_model::SystemNetworkingSettings;
+use nexus_db_model::SystemNetworkingSettingsUpdate;
+use omicron_common::api::external::Error;
+use omicron_common::api::external::UpdateResult;
+
+impl DataStore {
+    /// Read the singleton fleet networking settings row.
+    pub async fn system_networking_settings_view(
+        &self,
+        opctx: &OpContext,
+    ) -> Result<SystemNetworkingSettings, Error> {
+        opctx.authorize(authz::Action::Read, &authz::FLEET).await?;
+
+        use nexus_db_schema::schema::system_networking_settings::dsl;
+        dsl::system_networking_settings
+            .filter(dsl::singleton.eq(true))
+            .first_async(&*self.pool_connection_authorized(opctx).await?)
+            .await
+            .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))
+    }
+
+    /// Update the singleton fleet networking settings row. Fields left as
+    /// `None` in the supplied update are not modified.
+    pub async fn system_networking_settings_update(
+        &self,
+        opctx: &OpContext,
+        external_jumbo_frames_opt_in_enabled: Option<bool>,
+    ) -> UpdateResult<SystemNetworkingSettings> {
+        opctx.authorize(authz::Action::Modify, &authz::FLEET).await?;
+
+        let updates = SystemNetworkingSettingsUpdate {
+            external_jumbo_frames_opt_in_enabled,
+            time_modified: Utc::now(),
+        };
+
+        use nexus_db_schema::schema::system_networking_settings::dsl;
+        diesel::update(dsl::system_networking_settings)
+            .filter(dsl::singleton.eq(true))
+            .set(updates)
+            .returning(SystemNetworkingSettings::as_returning())
+            .get_result_async(&*self.pool_connection_authorized(opctx).await?)
+            .await
+            .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))
+    }
+}