在使用了kernel tracepoints，尝试一下用户态的USDT tracepoints。
从之前的篇章来看，跟踪虚拟机网络流量，既可以在内核也可以在用户态，而且使用kernel tracepoints明显有劣处，因为它需要重新编译内核，还得重启机器使用新内核。
在这里先从通用的USDT tracepoint出发，熟悉之后再回到QEMU的例子。
DTRACE_PROBEn宏可以在任何用户态的C语言程序使用，它是在sys/sdt.h里定义，使用它需要安装systemtap-sdt-devel。
创建一个例子,把它编译成usdt_example。
#include <sys/sdt.h>int main(int argc, char **argv) {
int x = 0;
while (x < 1000000) {
x++;
if (x % 100000 == 0) {
DTRACE_PROBE1(usdt-examples, usdt1, x);
}
}
DTRACE_PROBE3(usdt-examples, usdt2, x, "Hello!", 5);
return 0;
}
使用BCC工具trace.py。
python trace.py \
'u:/tmp/usdt_example:usdt1 "x=%d", arg1' \
'u:/tmp/usdt_example:usdt2 "x=%d, arg2=%s, arg3=%d", arg1, arg2, arg3'
可以看到结果，
PID TID COMM FUNC -
15543 15543 usdt_example usdt1 x=100000
15543 15543 usdt_example usdt1 x=200000
15543 15543 usdt_example usdt1 x=300000
15543 15543 usdt_example usdt1 x=400000
15543 15543 usdt_example usdt1 x=500000
15543 15543 usdt_example usdt1 x=600000
15543 15543 usdt_example usdt1 x=700000
15543 15543 usdt_example usdt1 x=800000
15543 15543 usdt_example usdt1 x=900000
15543 15543 usdt_example usdt1 x=1000000
15543 15543 usdt_example usdt2 x=1000000, arg2=Hello!, arg3=5
#include <sys/sdt.h>:头文件来自于systemtap-sdt-devel，提供DTRACE_PROBEn之类的宏
DTRACE_PROBEn(provider_name, probe_name, arg1, arg2, ..., argn):
provider_name: 跟踪点的命名空间probe_name：跟踪点名称arg1,...,argn：USDT tracepoint的参数
回到qemu的例子，回头看之前使用uprobes,uretprobes里，virtio-net的处理函数是virtio_net_receive_rcu,virtio_net_flush_tx。
在virtio_net_receive_rcu加上USDT tracepoint:
static ssize_t virtio_net_receive_rcu(NetClientState *nc,
const uint8_t *buf,
size_t size) {
...
while (offset < size) {
...
}
...
DTRACE_PROBE2(qemu, virtio-net-rx-end, i, size);
return size;
}
在virtio_net_flush_tx加上
static int32_t virtio_net_flush_tx(VirtIONetQueue *q) {
...
for (;;) {
...
DTRACE_PROBE1(qemu, virtio-net-tx-get-len, ret);
}
return num_packets;
}
记得在
virtio-net.c文件前面加上#include <sys/sdt.h>
重新编译qemu
创建eBPF跟踪程序
#!/usr/bin/pythonfrom bcc import BPF, USDT
from time import sleep
from subprocess import check_output
bpf_txt = """
#include <uapi/linux/ptrace.h>BPF_HISTOGRAM(rx_pkts_hist);
BPF_HISTOGRAM(rx_len_hist);
BPF_HISTOGRAM(tx_pkts_hist);
BPF_HISTOGRAM(tx_len_hist);
BPF_HASH(length_hash, u64, size_t);
int rx_handler(struct pt_regs *ctx) {
size_t recv_pkts, total_len;
bpf_usdt_readarg(1, ctx, &recv_pkts);
bpf_usdt_readarg(2, ctx, &total_len);
rx_pkts_hist.increment(bpf_log2l(recv_pkts));
rx_len_hist.increment(bpf_log2l(total_len));
return 0;
}
int tx_handler(struct pt_regs *ctx) {
u64 hash_key = 123;
size_t *len_ptr, len, total_len;
bpf_usdt_readarg(1, ctx, &len);
len_ptr = length_hash.lookup(&hash_key);
if (len_ptr != NULL) {
total_len = len + *len_ptr;
length_hash.delete(&hash_key);
length_hash.update(&hash_key, &total_len);
}
else {
length_hash.update(&hash_key, &len);
}
return 0;
}
int tx_handler_ret(struct pt_regs *ctx) {
u64 hash_key = 123;
int32_t sent_pkts;
size_t *len_ptr, total_len;
sent_pkts = PT_REGS_RC(ctx);
len_ptr = length_hash.lookup(&hash_key);
if (len_ptr != NULL) {
total_len = *len_ptr;
length_hash.delete(&hash_key);
}
else return 0;
tx_pkts_hist.increment(bpf_log2l(sent_pkts));
tx_len_hist.increment(bpf_log2l(total_len));
return 0;
}
"""
# Load BPF program
dev_qemu_path = \
"/mnt/repos/oracle/qemu/build/x86_64-softmmu/qemu-system-x86_64"
qemu_pid = 0
try:
qemu_pid = check_output(["pidof", "qemu-system-x86_64"])
except:
print("Error: process qemu-system-x86_64 not found")
exit()
usdt_ctx = USDT(pid=int(qemu_pid))
usdt_ctx.enable_probe(probe="virtio-net-rx-end",
fn_name="rx_handler")
usdt_ctx.enable_probe(probe="virtio-net-tx-get-len",
fn_name="tx_handler")
bpf_ctx = BPF(text=bpf_txt, usdt_contexts=[usdt_ctx])
bpf_ctx.attach_uretprobe(name=dev_qemu_path,
sym="virtio_net_flush_tx",
fn_name="tx_handler_ret")
# Print header
print("Aggregating data from virtio-net RX & TX virtqueues... "
"Hit Ctrl-C to end.")
# Format output
while 1:
try:
sleep(1)
except KeyboardInterrupt:
print("\n")
break
# Output
print("Virtio-net RX log2 received packets histogram")
print("---------------------------------------------")
bpf_ctx["rx_pkts_hist"].print_log2_hist("recv pkts")
print("\n")
print("Virtio-net RX log2 total length histogram")
print("-----------------------------------------")
bpf_ctx["rx_len_hist"].print_log2_hist("len")
print("\n")
print("--------------------------------------------------")
print("\n")
print("Virtio-net TX log2 sent packets histogram")
print("-----------------------------------------")
bpf_ctx["tx_pkts_hist"].print_log2_hist("sent pkts")
print("\n")
print("Virtio-net TX log2 total length histogram")
print("-----------------------------------------")
bpf_ctx["tx_len_hist"].print_log2_hist("len")
print("\n")
运行一下
Aggregating data from virtio-net RX & TX virtqueues... Hit Ctrl-C to end.
^CVirtio-net RX log2 received packets histogram
---------------------------------------------
recv pkts : count distribution
0 -> 1 : 38820 |*** |
2 -> 3 : 499564 |****************************************|
4 -> 7 : 2200 | |
8 -> 15 : 21 | |
Virtio-net RX log2 total length histogram
-----------------------------------------
len : count distribution
0 -> 1 : 0 | |
2 -> 3 : 0 | |
4 -> 7 : 0 | |
8 -> 15 : 0 | |
16 -> 31 : 0 | |
32 -> 63 : 4 | |
64 -> 127 : 6 | |
128 -> 255 : 1 | |
256 -> 511 : 0 | |
512 -> 1023 : 4 | |
1024 -> 2047 : 17564 |* |
2048 -> 4095 : 21277 |* |
4096 -> 8191 : 69310 |****** |
8192 -> 16383 : 432271 |****************************************|
16384 -> 32767 : 157 | |
32768 -> 65535 : 11 | |
--------------------------------------------------
Virtio-net TX log2 sent packets histogram
-----------------------------------------
sent pkts : count distribution
0 -> 1 : 376922 |****************************************|
2 -> 3 : 58685 |****** |
4 -> 7 : 455 | |
8 -> 15 : 0 | |
16 -> 31 : 2 | |
Virtio-net TX log2 total length histogram
-----------------------------------------
len : count distribution
0 -> 1 : 0 | |
2 -> 3 : 0 | |
4 -> 7 : 0 | |
8 -> 15 : 0 | |
16 -> 31 : 0 | |
32 -> 63 : 0 | |
64 -> 127 : 376921 |****************************************|
128 -> 255 : 58541 |****** |
256 -> 511 : 584 | |
512 -> 1023 : 16 | |
1024 -> 2047 : 2 | |
bpf_usdt_readarg(index, ctx, &addr): 从USDT tracepoint读取参数
usdt_ctx = USDT(pid=int(qemu_pid))：获取要跟踪的进程
USDT.enable_probe(probe='...', fn_name='...')：对跟踪进程安装USDT tracepoint钩子
BPF(text=bpf_txt, usdt_contexts=[usdt_ctx]): 创建USDT上下文的BPF对象
暗号：ea755
如有侵权请联系:admin#unsafe.sh