bus-bench: raw CAN over MQTT, measured on the car Orin
SocketCAN to yukart/can/raw and back, per frame or batched, with loss and latency from the kernel timestamp. No loss at a saturated 500 kbit/s bus; batched costs the broker ~1 % of a core. Report in docs/impl/reports. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01DDwfd6Ki8UfG4hK6W5YpJz
This commit is contained in:
@@ -0,0 +1,342 @@
|
||||
//! bus-bench — can MQTT carry raw CAN at real rates? (project-yukart decision 4)
|
||||
//!
|
||||
//! `bridge` reads a SocketCAN interface and publishes frames to `yukart/can/raw`,
|
||||
//! one message per frame or batched by time. `sink` subscribes, checks every frame
|
||||
//! arrived in order, and measures latency from the kernel's reception timestamp to
|
||||
//! arrival at the subscriber. Both sides run on the same host, so one clock serves.
|
||||
//!
|
||||
//! Test tool, not a service: the wire format below is provisional and lives only
|
||||
//! here until contracts freezes `yukart/can/raw`.
|
||||
//!
|
||||
//! message = seq u64 (first frame's) · count u32 · count × frame (little-endian)
|
||||
//! frame = ts_us i64 · can_id u32 (with EFF/RTR/ERR flags) · len u8 · 3 pad · data 8 (24 bytes)
|
||||
|
||||
#[path = "../recorder/can.rs"]
|
||||
#[allow(dead_code)]
|
||||
mod can;
|
||||
#[path = "../recorder/clock.rs"]
|
||||
#[allow(dead_code)]
|
||||
mod clock;
|
||||
|
||||
use std::process::ExitCode;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::mpsc::{sync_channel, RecvTimeoutError};
|
||||
use std::sync::Arc;
|
||||
use std::thread;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use rumqttc::{Client, Event, MqttOptions, NetworkOptions, Packet, QoS};
|
||||
use serde_json::json;
|
||||
|
||||
const USAGE: &str = "\
|
||||
usage:
|
||||
bus-bench bridge [options] SocketCAN → yukart/can/raw
|
||||
--iface NAME CAN interface (vcan0)
|
||||
--batch-ms N batch window; 0 = one message per frame (10)
|
||||
--secs N stop after N seconds (30)
|
||||
bus-bench sink [options] yukart/can/raw → counts and latency
|
||||
--secs N stop N seconds after the first frame (30)
|
||||
both:
|
||||
--broker HOST:PORT (127.0.0.1:1883)
|
||||
--topic TOPIC (yukart/can/raw)
|
||||
--nagle leave Nagle's algorithm on (rumqttc's default)";
|
||||
|
||||
const HEADER: usize = 12;
|
||||
const FRAME: usize = 24;
|
||||
|
||||
/// CPU time this process has used, user + system, in seconds.
|
||||
fn cpu_secs() -> f64 {
|
||||
let mut ru: libc::rusage = unsafe { std::mem::zeroed() };
|
||||
unsafe { libc::getrusage(libc::RUSAGE_SELF, &mut ru) };
|
||||
let tv = |t: libc::timeval| t.tv_sec as f64 + t.tv_usec as f64 * 1e-6;
|
||||
tv(ru.ru_utime) + tv(ru.ru_stime)
|
||||
}
|
||||
|
||||
fn now_us() -> i64 {
|
||||
(clock::realtime() * 1e6) as i64
|
||||
}
|
||||
|
||||
struct Opts {
|
||||
iface: String,
|
||||
batch_ms: u64,
|
||||
secs: u64,
|
||||
broker: (String, u16),
|
||||
topic: String,
|
||||
nagle: bool,
|
||||
}
|
||||
|
||||
fn parse(mut args: impl Iterator<Item = String>) -> Result<Opts, String> {
|
||||
let mut o = Opts {
|
||||
iface: "vcan0".into(),
|
||||
batch_ms: 10,
|
||||
secs: 30,
|
||||
broker: ("127.0.0.1".into(), 1883),
|
||||
topic: "yukart/can/raw".into(),
|
||||
nagle: false,
|
||||
};
|
||||
while let Some(a) = args.next() {
|
||||
let mut val = || args.next().ok_or(format!("{a} needs a value"));
|
||||
match a.as_str() {
|
||||
"--iface" => o.iface = val()?,
|
||||
"--batch-ms" => o.batch_ms = val()?.parse().map_err(|_| "--batch-ms needs a number")?,
|
||||
"--secs" => o.secs = val()?.parse().map_err(|_| "--secs needs a number")?,
|
||||
"--topic" => o.topic = val()?,
|
||||
"--nagle" => o.nagle = true,
|
||||
"--broker" => {
|
||||
let v = val()?;
|
||||
let (h, p) = v.rsplit_once(':').ok_or("--broker must be HOST:PORT")?;
|
||||
o.broker = (h.into(), p.parse().map_err(|_| "--broker must be HOST:PORT")?);
|
||||
}
|
||||
_ => return Err(format!("unknown option {a}")),
|
||||
}
|
||||
}
|
||||
Ok(o)
|
||||
}
|
||||
|
||||
/// A client whose event loop runs on its own thread. `connected` follows ConnAck;
|
||||
/// incoming publishes go to `on_publish`. Returns once connected, or None after 5 s.
|
||||
fn connect(
|
||||
id: &str,
|
||||
o: &Opts,
|
||||
cap: usize,
|
||||
subscribe: Option<String>,
|
||||
mut on_publish: impl FnMut(&[u8]) + Send + 'static,
|
||||
) -> Option<Client> {
|
||||
let mut opts = MqttOptions::new(id, o.broker.0.clone(), o.broker.1);
|
||||
opts.set_keep_alive(Duration::from_secs(5));
|
||||
opts.set_max_packet_size(1 << 20, 1 << 20);
|
||||
let (client, mut connection) = Client::new(opts, cap);
|
||||
let mut net = NetworkOptions::new();
|
||||
net.set_tcp_nodelay(!o.nagle);
|
||||
connection.eventloop.set_network_options(net);
|
||||
let connected = Arc::new(AtomicBool::new(false));
|
||||
let (c, up) = (client.clone(), connected.clone());
|
||||
thread::spawn(move || {
|
||||
for ev in connection.iter() {
|
||||
match ev {
|
||||
Ok(Event::Incoming(Packet::ConnAck(_))) => {
|
||||
if let Some(t) = &subscribe {
|
||||
let _ = c.try_subscribe(t.clone(), QoS::AtMostOnce);
|
||||
} else {
|
||||
up.store(true, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
Ok(Event::Incoming(Packet::SubAck(_))) => up.store(true, Ordering::Relaxed),
|
||||
Ok(Event::Incoming(Packet::Publish(p))) => on_publish(&p.payload),
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
eprintln!("{e}");
|
||||
up.store(false, Ordering::Relaxed);
|
||||
thread::sleep(Duration::from_millis(200));
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
let t = Instant::now();
|
||||
while !connected.load(Ordering::Relaxed) {
|
||||
if t.elapsed() > Duration::from_secs(5) {
|
||||
return None;
|
||||
}
|
||||
thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
Some(client)
|
||||
}
|
||||
|
||||
fn encode(f: &can::Frame, out: &mut Vec<u8>) {
|
||||
out.extend_from_slice(&(f.sec * 1_000_000 + f.usec).to_le_bytes());
|
||||
out.extend_from_slice(&f.id.to_le_bytes());
|
||||
out.extend_from_slice(&[f.len, 0, 0, 0]);
|
||||
out.extend_from_slice(&f.data);
|
||||
}
|
||||
|
||||
fn bridge(o: Opts) -> ExitCode {
|
||||
let Some(client) = connect("bus-bench-bridge", &o, 4096, None, |_| {}) else {
|
||||
eprintln!("bus-bench: no broker at {}:{}", o.broker.0, o.broker.1);
|
||||
return ExitCode::FAILURE;
|
||||
};
|
||||
let mut sock = match can::CanSocket::open(&o.iface, Duration::from_millis(2)) {
|
||||
Ok(s) => s,
|
||||
Err(e) => {
|
||||
eprintln!("bus-bench: {}: {e}", o.iface);
|
||||
return ExitCode::FAILURE;
|
||||
}
|
||||
};
|
||||
let window = Duration::from_millis(o.batch_ms);
|
||||
let (mut seq, mut frames, mut msgs, mut refused, mut kdrops) = (0u64, 0u64, 0u64, 0u64, 0u64);
|
||||
let mut buf: Vec<u8> = Vec::with_capacity(HEADER + FRAME * 512);
|
||||
let mut count = 0u32;
|
||||
let mut opened = Instant::now();
|
||||
let start = Instant::now();
|
||||
let cpu0 = cpu_secs();
|
||||
let mut flush = |buf: &mut Vec<u8>, count: &mut u32, seq: &mut u64| {
|
||||
if *count == 0 {
|
||||
return;
|
||||
}
|
||||
buf[8..12].copy_from_slice(&count.to_le_bytes());
|
||||
match client.try_publish(o.topic.clone(), QoS::AtMostOnce, false, buf.clone()) {
|
||||
Ok(()) => msgs += 1,
|
||||
Err(_) => refused += *count as u64, // client queue full: these frames are lost
|
||||
}
|
||||
*seq += *count as u64;
|
||||
*count = 0;
|
||||
buf.clear();
|
||||
};
|
||||
while start.elapsed() < Duration::from_secs(o.secs) {
|
||||
match sock.recv() {
|
||||
Ok(can::Recv::Frame(f, d)) => {
|
||||
kdrops += d as u64;
|
||||
if count == 0 {
|
||||
buf.extend_from_slice(&seq.to_le_bytes());
|
||||
buf.extend_from_slice(&[0; 4]);
|
||||
opened = Instant::now();
|
||||
}
|
||||
encode(&f, &mut buf);
|
||||
count += 1;
|
||||
frames += 1;
|
||||
}
|
||||
Ok(can::Recv::Timeout) => {}
|
||||
Err(e) => {
|
||||
eprintln!("bus-bench: {}: {e}", o.iface);
|
||||
return ExitCode::FAILURE;
|
||||
}
|
||||
}
|
||||
if count > 0 && opened.elapsed() >= window {
|
||||
flush(&mut buf, &mut count, &mut seq);
|
||||
}
|
||||
}
|
||||
flush(&mut buf, &mut count, &mut seq);
|
||||
thread::sleep(Duration::from_millis(500)); // let the queue drain before the process exits
|
||||
let secs = start.elapsed().as_secs_f64();
|
||||
let cpu = (cpu_secs() - cpu0) / secs * 100.0;
|
||||
println!(
|
||||
"{}",
|
||||
json!({ "role": "bridge", "batch_ms": o.batch_ms, "secs": (secs * 10.0).round() / 10.0,
|
||||
"frames": frames, "messages": msgs, "refused_frames": refused, "kernel_drops": kdrops,
|
||||
"cpu_pct": (cpu * 10.0).round() / 10.0 })
|
||||
);
|
||||
ExitCode::SUCCESS
|
||||
}
|
||||
|
||||
/// What the sink learns from one message: (first seq, frames, per-frame latency µs).
|
||||
fn decode(p: &[u8], now: i64) -> Option<(u64, u32, Vec<i64>)> {
|
||||
let seq = u64::from_le_bytes(p.get(0..8)?.try_into().ok()?);
|
||||
let count = u32::from_le_bytes(p.get(8..12)?.try_into().ok()?);
|
||||
if p.len() != HEADER + FRAME * count as usize {
|
||||
return None;
|
||||
}
|
||||
let lat = p[HEADER..]
|
||||
.chunks_exact(FRAME)
|
||||
.map(|f| now - i64::from_le_bytes(f[0..8].try_into().unwrap()))
|
||||
.collect();
|
||||
Some((seq, count, lat))
|
||||
}
|
||||
|
||||
fn pct(sorted: &[i64], q: f64) -> f64 {
|
||||
if sorted.is_empty() {
|
||||
return 0.0;
|
||||
}
|
||||
let i = ((sorted.len() - 1) as f64 * q).round() as usize;
|
||||
sorted[i] as f64 / 1000.0
|
||||
}
|
||||
|
||||
fn sink(o: Opts) -> ExitCode {
|
||||
// Decoding happens on the event-loop thread; the main thread only collects.
|
||||
let (tx, rx) = sync_channel::<(u64, u32, Vec<i64>, usize)>(1 << 16);
|
||||
let topic = o.topic.clone();
|
||||
let on_publish = move |p: &[u8]| match decode(p, now_us()) {
|
||||
Some((s, c, l)) => {
|
||||
let _ = tx.try_send((s, c, l, p.len()));
|
||||
}
|
||||
None => eprintln!("bus-bench: malformed message ({} bytes)", p.len()),
|
||||
};
|
||||
let Some(_client) = connect("bus-bench-sink", &o, 16, Some(topic), on_publish) else {
|
||||
eprintln!("bus-bench: no broker at {}:{}", o.broker.0, o.broker.1);
|
||||
return ExitCode::FAILURE;
|
||||
};
|
||||
eprintln!("bus-bench: subscribed to {}; waiting for frames", o.topic);
|
||||
let (mut frames, mut msgs, mut bytes, mut lost, mut reordered) = (0u64, 0u64, 0u64, 0u64, 0u64);
|
||||
let mut next: Option<u64> = None;
|
||||
let mut lat: Vec<i64> = Vec::new();
|
||||
let mut first: Option<Instant> = None;
|
||||
let mut cpu0 = 0.0;
|
||||
loop {
|
||||
if let Some(t) = first {
|
||||
if t.elapsed() >= Duration::from_secs(o.secs) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
match rx.recv_timeout(Duration::from_millis(100)) {
|
||||
Ok((s, c, l, n)) => {
|
||||
if first.is_none() {
|
||||
first = Some(Instant::now());
|
||||
cpu0 = cpu_secs();
|
||||
}
|
||||
match next {
|
||||
Some(want) if s > want => lost += s - want,
|
||||
Some(want) if s < want => reordered += 1,
|
||||
_ => {}
|
||||
}
|
||||
next = Some(s + c as u64);
|
||||
frames += c as u64;
|
||||
msgs += 1;
|
||||
bytes += n as u64;
|
||||
lat.extend(l);
|
||||
}
|
||||
Err(RecvTimeoutError::Timeout) => {}
|
||||
Err(RecvTimeoutError::Disconnected) => break,
|
||||
}
|
||||
}
|
||||
let secs = first.map(|t| t.elapsed().as_secs_f64()).unwrap_or(0.0);
|
||||
let cpu = (cpu_secs() - cpu0) / secs.max(1e-9) * 100.0;
|
||||
lat.sort_unstable();
|
||||
let r = |v: f64| (v * 100.0).round() / 100.0;
|
||||
println!(
|
||||
"{}",
|
||||
json!({ "role": "sink", "secs": r(secs), "frames": frames, "messages": msgs,
|
||||
"frames_per_s": (frames as f64 / secs.max(1e-9)).round(),
|
||||
"messages_per_s": (msgs as f64 / secs.max(1e-9)).round(),
|
||||
"kbytes_per_s": r(bytes as f64 / 1000.0 / secs.max(1e-9)),
|
||||
"lost_frames": lost, "reordered_messages": reordered,
|
||||
"latency_ms": { "p50": r(pct(&lat, 0.5)), "p99": r(pct(&lat, 0.99)),
|
||||
"p999": r(pct(&lat, 0.999)), "max": r(pct(&lat, 1.0)) },
|
||||
"cpu_pct": (cpu * 10.0).round() / 10.0 })
|
||||
);
|
||||
ExitCode::SUCCESS
|
||||
}
|
||||
|
||||
fn main() -> ExitCode {
|
||||
let mut args = std::env::args().skip(1);
|
||||
let role = args.next().unwrap_or_default();
|
||||
let o = match parse(args) {
|
||||
Ok(o) => o,
|
||||
Err(e) => {
|
||||
eprintln!("bus-bench: {e}\n{USAGE}");
|
||||
return ExitCode::from(2);
|
||||
}
|
||||
};
|
||||
match role.as_str() {
|
||||
"bridge" => bridge(o),
|
||||
"sink" => sink(o),
|
||||
_ => {
|
||||
eprintln!("{USAGE}");
|
||||
ExitCode::from(2)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn frame_round_trips_through_the_wire_format() {
|
||||
let f = can::Frame { sec: 10, usec: 5, id: 0x449, len: 8, data: [1, 2, 3, 4, 5, 6, 7, 8] };
|
||||
let mut m = 7u64.to_le_bytes().to_vec();
|
||||
m.extend_from_slice(&1u32.to_le_bytes());
|
||||
encode(&f, &mut m);
|
||||
assert_eq!(m.len(), HEADER + FRAME);
|
||||
let (seq, count, lat) = decode(&m, 10_000_105).unwrap();
|
||||
assert_eq!((seq, count, lat), (7, 1, vec![100]));
|
||||
assert!(decode(&m[..m.len() - 1], 0).is_none());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user