Skip to content

Commit 23fdf58

Browse files
rzezeskirzezeski
committed
ZSTs, DEBUG kernels, and copyin, oh my! (#267)
1 parent d681a29 commit 23fdf58

File tree

9 files changed

+124
-63
lines changed

9 files changed

+124
-63
lines changed

‎dtrace/lib/common.d‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,3 +34,15 @@ typedef struct ht_run_sdt_arg {
3434
flow_id_sdt_arg_t *flow_before;
3535
flow_id_sdt_arg_t *flow_after;
3636
} ht_run_sdt_arg_t;
37+
38+
typedef struct opte_cmd_ioctl {
39+
uint64_t api_version;
40+
int cmd;
41+
uint64_t flags;
42+
uint64_t reserved;
43+
char *req_bytes;
44+
size_t req_len;
45+
char *resp_bytes;
46+
size_t resp_len;
47+
size_t resp_len_actual;
48+
} opte_cmd_ioctl_t;

‎dtrace/opte-ioctl.d‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/*
2+
* Track the OPTE command ioctls as they come in.
3+
*
4+
* dtrace -L ./lib -I . -Cqs ./opte-ioctl.d
5+
*/
6+
xde_dld_ioc_opte_cmd:entry {
7+
this->opte_cmd_ioctl = (opte_cmd_ioctl_t *)arg0;
8+
print(*this->opte_cmd_ioctl);
9+
printf("\n");
10+
self->t = 1;
11+
}
12+
13+
ddi_copyin:entry /self->t/ {
14+
printf("ddi_copyin(%p, %p, %u, 0x%x) =>", arg0, arg1, arg2, arg3);
15+
}
16+
17+
ddi_copyin:return /self->t/ {
18+
printf(" %d\n", arg1);
19+
}
20+
21+
xde_dld_ioc_opte_cmd:return {
22+
self->t = 0;
23+
}

‎opte-api/src/cmd.rs‎

Lines changed: 32 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -140,27 +140,52 @@ impl OpteCmdIoctl {
140140

141141
#[derive(Clone, Debug, Deserialize, Serialize)]
142142
pub enum OpteError {
143-
BadApiVersion { user: u64, kernel: u64 },
144-
BadLayerPos { layer: String, pos: String },
143+
BadApiVersion {
144+
user: u64,
145+
kernel: u64,
146+
},
147+
BadLayerPos {
148+
layer: String,
149+
pos: String,
150+
},
145151
BadName,
146152
BadState(String),
147153
CopyinReq,
148154
CopyoutResp,
149155
DeserCmdErr(String),
150156
DeserCmdReq(String),
151157
FlowExists(String),
152-
InvalidRouterEntry { dest: IpCidr, target: String },
158+
InvalidRouterEntry {
159+
dest: IpCidr,
160+
target: String,
161+
},
153162
LayerNotFound(String),
154-
MacExists { port: String, vni: Vni, mac: MacAddr },
163+
MacExists {
164+
port: String,
165+
vni: Vni,
166+
mac: MacAddr,
167+
},
155168
MaxCapacity(u64),
169+
170+
/// The OpteCmdIoctl has `req_len == 0` but the specified `cmd`
171+
/// types expects a request body. This can happen either by
172+
/// developer error or a hand-rolled, negligent/malicious ioctl.
173+
NoRequestBody,
174+
156175
PortCreate(String),
157176
PortExists(String),
158177
PortNotFound(String),
159-
RespTooLarge { needed: usize, given: usize },
178+
RespTooLarge {
179+
needed: usize,
180+
given: usize,
181+
},
160182
RuleNotFound(u64),
161183
SerCmdErr(String),
162184
SerCmdResp(String),
163-
System { errno: c_int, msg: String },
185+
System {
186+
errno: c_int,
187+
msg: String,
188+
},
164189
}
165190

166191
impl OpteError {
@@ -188,6 +213,7 @@ impl OpteError {
188213
Self::LayerNotFound(_) => ENOENT,
189214
Self::MacExists { .. } => EEXIST,
190215
Self::MaxCapacity(_) => ENFILE,
216+
Self::NoRequestBody => EINVAL,
191217
Self::PortCreate(_) => EINVAL,
192218
Self::PortExists(_) => EEXIST,
193219
Self::PortNotFound(_) => ENOENT,

‎opte-api/src/lib.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ pub use ulp::*;
5454
///
5555
/// We rely on CI and the check-api-version.sh script to verify that
5656
/// this number is incremented anytime the oxide-api code changes.
57-
pub const API_VERSION: u64 = 14;
57+
pub const API_VERSION: u64 = 15;
5858

5959
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
6060
pub enum Direction {

‎opte-ioctl/src/lib.rs‎

Lines changed: 28 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ use opte::api::XDE_DLD_OPTE_CMD;
1515
use oxide_vpc::api::AddRouterEntryReq;
1616
use oxide_vpc::api::CreateXdeReq;
1717
use oxide_vpc::api::DeleteXdeReq;
18-
use oxide_vpc::api::ListPortsReq;
1918
use oxide_vpc::api::ListPortsResp;
2019
use oxide_vpc::api::SetFwRulesReq;
2120
use oxide_vpc::api::SetVirt2PhysReq;
@@ -107,7 +106,7 @@ impl OpteHdl {
107106
let cmd = OpteCmd::CreateXde;
108107
let req = CreateXdeReq { xde_devname, linkid, cfg, passthrough };
109108

110-
let res = run_cmd_ioctl(self.device.as_raw_fd(), cmd, &req);
109+
let res = run_cmd_ioctl(self.device.as_raw_fd(), cmd, Some(&req));
111110

112111
if res.is_err() {
113112
let _ = link::delete_link_id(linkid, libnet::LinkFlags::Active);
@@ -121,16 +120,14 @@ impl OpteHdl {
121120
let link_id = libnet::LinkHandle::Name(name.into()).id()?;
122121
let req = DeleteXdeReq { xde_devname: name.into() };
123122
let cmd = OpteCmd::DeleteXde;
124-
let resp = run_cmd_ioctl(self.device.as_raw_fd(), cmd, &req)?;
123+
let resp = run_cmd_ioctl(self.device.as_raw_fd(), cmd, Some(&req))?;
125124
libnet::link::delete_link_id(link_id, libnet::LinkFlags::Active)?;
126125
Ok(resp)
127126
}
128127

129128
/// List the extant OPTE ports.
130129
pub fn list_ports(&self) -> Result<ListPortsResp, Error> {
131-
let req = ListPortsReq { unused: () };
132-
let cmd = OpteCmd::ListPorts;
133-
run_cmd_ioctl(self.device.as_raw_fd(), cmd, &req)
130+
run_cmd_ioctl(self.device.as_raw_fd(), OpteCmd::ListPorts, None::<&()>)
134131
}
135132

136133
/// Create a new handle to the OPTE control node.
@@ -142,7 +139,7 @@ impl OpteHdl {
142139

143140
pub fn set_v2p(&self, req: &SetVirt2PhysReq) -> Result<NoResp, Error> {
144141
let cmd = OpteCmd::SetVirt2Phys;
145-
run_cmd_ioctl(self.device.as_raw_fd(), cmd, &req)
142+
run_cmd_ioctl(self.device.as_raw_fd(), cmd, Some(&req))
146143
}
147144

148145
/// Set xde underlay devices.
@@ -153,35 +150,51 @@ impl OpteHdl {
153150
) -> Result<NoResp, Error> {
154151
let req = SetXdeUnderlayReq { u1: u1.into(), u2: u2.into() };
155152
let cmd = OpteCmd::SetXdeUnderlay;
156-
run_cmd_ioctl(self.device.as_raw_fd(), cmd, &req)
153+
run_cmd_ioctl(self.device.as_raw_fd(), cmd, Some(&req))
157154
}
158155

159156
pub fn add_router_entry(
160157
&self,
161158
req: &AddRouterEntryReq,
162159
) -> Result<NoResp, Error> {
163160
let cmd = OpteCmd::AddRouterEntry;
164-
run_cmd_ioctl(self.device.as_raw_fd(), cmd, &req)
161+
run_cmd_ioctl(self.device.as_raw_fd(), cmd, Some(&req))
165162
}
166163

167164
pub fn set_fw_rules(&self, req: &SetFwRulesReq) -> Result<NoResp, Error> {
168165
let cmd = OpteCmd::SetFwRules;
169-
run_cmd_ioctl(self.device.as_raw_fd(), cmd, &req)
166+
run_cmd_ioctl(self.device.as_raw_fd(), cmd, Some(&req))
170167
}
171168
}
172169

173170
#[cfg(target_os = "illumos")]
174171
pub fn run_cmd_ioctl<T, R>(
175172
dev: libc::c_int,
176173
cmd: OpteCmd,
177-
req: &R,
174+
req: Option<&R>,
178175
) -> Result<T, Error>
179176
where
180177
T: CmdOk + DeserializeOwned,
181178
R: Serialize,
182179
{
183-
let req_bytes =
184-
postcard::to_allocvec(req).map_err(|e| Error::ReqSer(cmd, e))?;
180+
let (req_bytes_ptr, req_len) = match req {
181+
Some(req) => {
182+
let bytes = postcard::to_allocvec(req)
183+
.map_err(|e| Error::ReqSer(cmd, e))?;
184+
let len = bytes.len();
185+
// This is here to catch the case where you, the
186+
// developer, have accidentally used a ZST as a request
187+
// type. I would have added a compile-time check but as
188+
// far as I can tell there is no way to use size_of with a
189+
// generic type. This check is sufficient, and is just a
190+
// means to save you hours of heartache if you were to
191+
// accidentally create a ZST request type.
192+
assert!(len > 0, "cannot use ZST for request type");
193+
(bytes.as_ptr(), len)
194+
}
195+
196+
None => (core::ptr::null(), 0),
197+
};
185198

186199
// It would be a shame if the command failed and we didn't have
187200
// enough bytes to serialize the error response, so we set this to
@@ -192,8 +205,8 @@ where
192205
cmd,
193206
flags: 0,
194207
reserved1: 0,
195-
req_bytes: req_bytes.as_ptr(),
196-
req_len: req_bytes.len(),
208+
req_bytes: req_bytes_ptr,
209+
req_len,
197210
resp_bytes: resp_buf.as_mut_ptr(),
198211
resp_len: resp_buf.len(),
199212
resp_len_actual: 0,

‎opteadm/src/lib.rs‎

Lines changed: 16 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ use oxide_vpc::api::AddRouterEntryReq;
2222
use oxide_vpc::api::CreateXdeReq;
2323
use oxide_vpc::api::DeleteXdeReq;
2424
use oxide_vpc::api::FirewallRule;
25-
use oxide_vpc::api::ListPortsReq;
2625
use oxide_vpc::api::ListPortsResp;
2726
use oxide_vpc::api::RemFwRuleReq;
2827
use oxide_vpc::api::SetFwRulesReq;
@@ -58,8 +57,7 @@ impl OpteAdm {
5857
let xde_devname = name.into();
5958
let cmd = OpteCmd::CreateXde;
6059
let req = CreateXdeReq { xde_devname, linkid, cfg, passthrough };
61-
62-
let res = run_cmd_ioctl(self.device.as_raw_fd(), cmd, &req);
60+
let res = run_cmd_ioctl(self.device.as_raw_fd(), cmd, Some(&req));
6361

6462
if res.is_err() {
6563
let _ = link::delete_link_id(linkid, libnet::LinkFlags::Active);
@@ -73,7 +71,7 @@ impl OpteAdm {
7371
let link_id = libnet::LinkHandle::Name(name.into()).id()?;
7472
let req = DeleteXdeReq { xde_devname: name.into() };
7573
let cmd = OpteCmd::DeleteXde;
76-
let resp = run_cmd_ioctl(self.device.as_raw_fd(), cmd, &req)?;
74+
let resp = run_cmd_ioctl(self.device.as_raw_fd(), cmd, Some(&req))?;
7775
libnet::link::delete_link_id(link_id, libnet::LinkFlags::Active)?;
7876
Ok(resp)
7977
}
@@ -86,7 +84,7 @@ impl OpteAdm {
8684
) -> Result<NoResp, Error> {
8785
let req = SetXdeUnderlayReq { u1: u1.into(), u2: u2.into() };
8886
let cmd = OpteCmd::SetXdeUnderlay;
89-
run_cmd_ioctl(self.device.as_raw_fd(), cmd, &req)
87+
run_cmd_ioctl(self.device.as_raw_fd(), cmd, Some(&req))
9088
}
9189

9290
/// Add a firewall rule
@@ -100,7 +98,7 @@ impl OpteAdm {
10098
port_name: port_name.to_string(),
10199
rule: rule.clone(),
102100
};
103-
run_cmd_ioctl(self.device.as_raw_fd(), cmd, &req)
101+
run_cmd_ioctl(self.device.as_raw_fd(), cmd, Some(&req))
104102
}
105103

106104
pub fn set_firewall_rules(
@@ -111,7 +109,7 @@ impl OpteAdm {
111109
let cmd = OpteCmd::SetFwRules;
112110
let req =
113111
SetFwRulesReq { port_name: port_name.to_string(), rules: rules };
114-
run_cmd_ioctl(self.device.as_raw_fd(), cmd, &req)
112+
run_cmd_ioctl(self.device.as_raw_fd(), cmd, Some(&req))
115113
}
116114

117115
/// Return the contents of an OPTE layer.
@@ -125,21 +123,12 @@ impl OpteAdm {
125123
port_name: port_name.to_string(),
126124
name: name.to_string(),
127125
};
128-
run_cmd_ioctl::<api::DumpLayerResp, _>(
129-
self.device.as_raw_fd(),
130-
cmd,
131-
&req,
132-
)
126+
run_cmd_ioctl(self.device.as_raw_fd(), cmd, Some(&req))
133127
}
134128

135129
/// List all the ports.
136130
pub fn list_ports(&self) -> Result<ListPortsResp, Error> {
137-
let cmd = OpteCmd::ListPorts;
138-
run_cmd_ioctl::<ListPortsResp, _>(
139-
self.device.as_raw_fd(),
140-
cmd,
141-
&ListPortsReq { unused: () },
142-
)
131+
run_cmd_ioctl(self.device.as_raw_fd(), OpteCmd::ListPorts, None::<&()>)
143132
}
144133

145134
pub fn list_layers(
@@ -150,7 +139,7 @@ impl OpteAdm {
150139
run_cmd_ioctl::<api::ListLayersResp, _>(
151140
self.device.as_raw_fd(),
152141
cmd,
153-
&api::ListLayersReq { port_name: port.to_string() },
142+
Some(&api::ListLayersReq { port_name: port.to_string() }),
154143
)
155144
}
156145

@@ -167,7 +156,7 @@ impl OpteAdm {
167156
req: &RemFwRuleReq,
168157
) -> Result<NoResp, Error> {
169158
let cmd = OpteCmd::RemFwRule;
170-
run_cmd_ioctl(self.device.as_raw_fd(), cmd, req)
159+
run_cmd_ioctl(self.device.as_raw_fd(), cmd, Some(req))
171160
}
172161

173162
/// Return the TCP flows.
@@ -179,7 +168,7 @@ impl OpteAdm {
179168
run_cmd_ioctl::<api::DumpTcpFlowsResp, _>(
180169
self.device.as_raw_fd(),
181170
cmd,
182-
&api::DumpTcpFlowsReq { port_name: port_name.to_string() },
171+
Some(&api::DumpTcpFlowsReq { port_name: port_name.to_string() }),
183172
)
184173
}
185174

@@ -189,7 +178,7 @@ impl OpteAdm {
189178
run_cmd_ioctl(
190179
self.device.as_raw_fd(),
191180
cmd,
192-
&api::ClearUftReq { port_name: port_name.to_string() },
181+
Some(&api::ClearUftReq { port_name: port_name.to_string() }),
193182
)
194183
}
195184

@@ -199,22 +188,22 @@ impl OpteAdm {
199188
run_cmd_ioctl::<api::DumpUftResp, _>(
200189
self.device.as_raw_fd(),
201190
cmd,
202-
&api::DumpUftReq { port_name: port_name.to_string() },
191+
Some(&api::DumpUftReq { port_name: port_name.to_string() }),
203192
)
204193
}
205194

206195
pub fn set_v2p(&self, req: &SetVirt2PhysReq) -> Result<NoResp, Error> {
207196
let cmd = OpteCmd::SetVirt2Phys;
208-
run_cmd_ioctl(self.device.as_raw_fd(), cmd, &req)
197+
run_cmd_ioctl(self.device.as_raw_fd(), cmd, Some(&req))
209198
}
210199

211200
/// Dump the Virtual-to-Physical mappings.
212201
pub fn dump_v2p(&self) -> Result<overlay::DumpVirt2PhysResp, Error> {
213202
let cmd = OpteCmd::DumpVirt2Phys;
214-
run_cmd_ioctl::<overlay::DumpVirt2PhysResp, _>(
203+
run_cmd_ioctl(
215204
self.device.as_raw_fd(),
216205
cmd,
217-
&overlay::DumpVirt2PhysReq { unused: 99 },
206+
Some(&overlay::DumpVirt2PhysReq { unused: 99 }),
218207
)
219208
}
220209

@@ -223,6 +212,6 @@ impl OpteAdm {
223212
req: &AddRouterEntryReq,
224213
) -> Result<NoResp, Error> {
225214
let cmd = OpteCmd::AddRouterEntry;
226-
run_cmd_ioctl(self.device.as_raw_fd(), cmd, &req)
215+
run_cmd_ioctl(self.device.as_raw_fd(), cmd, Some(&req))
227216
}
228217
}

‎oxide-vpc/src/api.rs‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -355,12 +355,6 @@ pub struct DeleteXdeReq {
355355
pub xde_devname: String,
356356
}
357357

358-
/// List existing xde ports.
359-
#[derive(Debug, Deserialize, Serialize)]
360-
pub struct ListPortsReq {
361-
pub unused: (),
362-
}
363-
364358
/// Information about a single existing xde port
365359
#[derive(Debug, Deserialize, Serialize)]
366360
pub struct PortInfo {

0 commit comments

Comments
 (0)