I have an eBPF program with the following map definitions:
struct bpf_map_def SEC("maps") servers = {
.type = BPF_MAP_TYPE_HASH,
.key_size = sizeof(struct ip_key),
.value_size = sizeof(struct dest_info),
.max_entries = MAX_SERVERS,
};
struct bpf_map_def SEC("maps") client_addrs = {
.type = BPF_MAP_TYPE_HASH,
.key_size = sizeof(struct port_key),
.value_size = sizeof(struct client_port_addr),
.max_entries = MAX_CLIENTS,
};
where the struct definitions are as below:
struct port_key {
__u16 port;
__u16 pad[3];
};
struct ip_key {
__u32 key;
__u32 pad;
};
struct dest_info {
__u32 saddr;
__u32 daddr;
__u64 bytes;
__u64 pkts;
__u8 dmac[6];
__u16 pad;
};
struct client_port_addr {
__u32 client_ip;
__u8 dmac[6];
__u16 pad[3];
};
The program itself, after the pointer verifications and initial checks, is shown below.
struct port_key key = {0};
struct client_port_addr val;
key.port = udp->source;
val.client_ip = iph->saddr;
memcpy (val.dmac, eth->h_source, 6 * sizeof(__u8));
bpf_map_update_elem(&client_addrs, &key, &val, BPF_ANY);
iph->saddr = IP_ADDRESS(BALANCER);
iph->daddr = dest_tnl->daddr;
memcpy (eth->h_source, eth->h_dest, 6 * sizeof(__u8));
memcpy (eth->h_dest, dest_tnl->dmac, 6 * sizeof(__u8));
So, the problem is that I use bpf_map_update()
in my code, but while using it, I get the invalid indirect read from the stack
error as shown below.
libbpf:
0: (bf) r6 = r1
1: (61) r9 = *(u32 *)(r6 +4)
2: (61) r7 = *(u32 *)(r6 +0)
3: (18) r1 = 0xffffa59ac00b6000
5: (b7) r2 = 24
6: (85) call bpf_trace_printk#6
R1_w=map_value(id=0,off=0,ks=4,vs=50,imm=0) R2_w=inv24 R6_w=ctx(id=0,off=0,imm=0) R7_w=pkt(id=0,off=0,r=0,imm=0) R9_w=pkt_end(id=0,off=0,imm=0) R10=fp0
last_idx 6 first_idx 0
regs=4 stack=0 before 5: (b7) r2 = 24
7: (b7) r8 = 1
8: (bf) r1 = r7
9: (07) r1 += 14
10: (2d) if r1 > r9 goto pc+130
R0_w=inv(id=0) R1_w=pkt(id=0,off=14,r=14,imm=0) R6_w=ctx(id=0,off=0,imm=0) R7_w=pkt(id=0,off=0,r=14,imm=0) R8_w=inv1 R9_w=pkt_end(id=0,off=0,imm=0) R10=fp0
11: (71) r1 = *(u8 *)(r7 +12)
12: (71) r2 = *(u8 *)(r7 +13)
13: (67) r2 <<= 8
14: (4f) r2 |= r1
15: (b7) r8 = 2
16: (55) if r2 != 0x8 goto pc+124
R0=inv(id=0) R1=inv(id=0,umax_value=255,var_off=(0x0; 0xff)) R2=inv8 R6=ctx(id=0,off=0,imm=0) R7=pkt(id=0,off=0,r=14,imm=0) R8=inv2 R9=pkt_end(id=0,off=0,imm=0) R10=fp0
17: (61) r7 = *(u32 *)(r6 +4)
18: (61) r9 = *(u32 *)(r6 +0)
19: (bf) r6 = r9
20: (07) r6 += 14
21: (b7) r8 = 1
22: (2d) if r6 > r7 goto pc+118
R0=inv(id=0) R1=inv(id=0,umax_value=255,var_off=(0x0; 0xff)) R2=inv8 R6_w=pkt(id=0,off=14,r=14,imm=0) R7_w=pkt_end(id=0,off=0,imm=0) R8_w=inv1 R9_w=pkt(id=0,off=0,r=14,imm=0) R10=fp0
23: (bf) r1 = r9
24: (07) r1 += 34
25: (b7) r8 = 1
26: (2d) if r1 > r7 goto pc+114
R0=inv(id=0) R1=pkt(id=0,off=34,r=34,imm=0) R2=inv8 R6=pkt(id=0,off=14,r=34,imm=0) R7=pkt_end(id=0,off=0,imm=0) R8=inv1 R9=pkt(id=0,off=0,r=34,imm=0) R10=fp0
27: (71) r1 = *(u8 *)(r6 +0)
28: (57) r1 &= 15
29: (b7) r8 = 1
30: (55) if r1 != 0x5 goto pc+110
R0=inv(id=0) R1_w=inv5 R2=inv8 R6=pkt(id=0,off=14,r=34,imm=0) R7=pkt_end(id=0,off=0,imm=0) R8_w=inv1 R9=pkt(id=0,off=0,r=34,imm=0) R10=fp0
31: (61) r3 = *(u32 *)(r9 +26)
32: (18) r1 = 0xffffa59ac00b6018
34: (b7) r2 = 26
35: (85) call bpf_trace_printk#6
R0=inv(id=0) R1_w=map_value(id=0,off=24,ks=4,vs=50,imm=0) R2_w=inv26 R3_w=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R6=pkt(id=0,off=14,r=34,imm=0) R7=pkt_end(id=0,off=0,imm=0) R8_w=inv1 R9=pkt(id=0,off=0,r=34,imm=0) R10=fp0
last_idx 35 first_idx 26
regs=4 stack=0 before 34: (b7) r2 = 26
36: (69) r1 = *(u16 *)(r9 +20)
37: (57) r1 &= 65343
38: (b7) r8 = 1
39: (55) if r1 != 0x0 goto pc+101
R0=inv(id=0) R1_w=inv0 R6=pkt(id=0,off=14,r=34,imm=0) R7=pkt_end(id=0,off=0,imm=0) R8_w=inv1 R9=pkt(id=0,off=0,r=34,imm=0) R10=fp0
40: (71) r1 = *(u8 *)(r9 +23)
41: (b7) r8 = 2
42: (55) if r1 != 0x11 goto pc+98
R0=inv(id=0) R1_w=inv17 R6=pkt(id=0,off=14,r=34,imm=0) R7=pkt_end(id=0,off=0,imm=0) R8_w=inv2 R9=pkt(id=0,off=0,r=34,imm=0) R10=fp0
43: (bf) r1 = r9
44: (07) r1 += 42
45: (b7) r8 = 1
46: (2d) if r1 > r7 goto pc+94
R0=inv(id=0) R1=pkt(id=0,off=42,r=42,imm=0) R6=pkt(id=0,off=14,r=42,imm=0) R7=pkt_end(id=0,off=0,imm=0) R8=inv1 R9=pkt(id=0,off=0,r=42,imm=0) R10=fp0
47: (b7) r8 = 0
48: (7b) *(u64 *)(r10 -8) = r8
last_idx 48 first_idx 46
regs=100 stack=0 before 47: (b7) r8 = 0
49: (bf) r2 = r10
50: (07) r2 += -8
51: (18) r1 = 0xffff9a7bed1bc000
53: (85) call bpf_map_lookup_elem#1
54: (bf) r7 = r0
55: (15) if r7 == 0x0 goto pc+85
R0=map_value(id=0,off=0,ks=8,vs=32,imm=0) R6=pkt(id=0,off=14,r=42,imm=0) R7=map_value(id=0,off=0,ks=8,vs=32,imm=0) R8=invP0 R9=pkt(id=0,off=0,r=42,imm=0) R10=fp0 fp-8=mmmmmmmm
56: (b7) r8 = 0
57: (7b) *(u64 *)(r10 -16) = r8
last_idx 57 first_idx 55
regs=100 stack=0 before 56: (b7) r8 = 0
58: (69) r1 = *(u16 *)(r9 +34)
59: (6b) *(u16 *)(r10 -16) = r1
60: (61) r1 = *(u32 *)(r9 +26)
61: (63) *(u32 *)(r10 -32) = r1
62: (71) r1 = *(u8 *)(r9 +11)
63: (73) *(u8 *)(r10 -23) = r1
64: (71) r1 = *(u8 *)(r9 +10)
65: (73) *(u8 *)(r10 -24) = r1
66: (71) r1 = *(u8 *)(r9 +7)
67: (67) r1 <<= 8
68: (71) r2 = *(u8 *)(r9 +6)
69: (4f) r1 |= r2
70: (71) r2 = *(u8 *)(r9 +9)
71: (67) r2 <<= 8
72: (71) r3 = *(u8 *)(r9 +8)
73: (4f) r2 |= r3
74: (67) r2 <<= 16
75: (4f) r2 |= r1
76: (63) *(u32 *)(r10 -28) = r2
77: (bf) r2 = r10
78: (07) r2 += -16
79: (bf) r3 = r10
80: (07) r3 += -32
81: (18) r1 = 0xffff9a7bed1bf400
83: (b7) r4 = 0
84: (85) call bpf_map_update_elem#2
invalid indirect read from stack R3 off -32+10 size 16
processed 81 insns (limit 1000000) max_states_per_insn 0 total_states 5 peak_states 5 mark_read 2
libbpf: -- END LOG --
libbpf: failed to load program 'loadbal'
All of the defined structs for keys and values are padded to their next multiple of 8 bytes. Since I could not find any useful and descriptive explanation on my issue, explanations of this topic and maybe even a bit of detail are much appreciated.
Please let me know if you need more information.
bpf_map_lookup_elem
is the issue.if ... else ...
structure, towards the bottom, which I have minimalized by commenting out the branches that did not cause an error. Everything beforeisFromServers = is_from_back_servers(iph->saddr);
is just in the verification and no functionality is included there. I also added the map and their struct definitions. Please see if this should help or something else is still required. Also, please note that the error is updated tobpf_map_update_elem
. There was a little struct misuse I fixed yesterday.bpf_map_update()
, which leaves us with some uninitialized bytes that the verifier does not feel comfortable with. Many thanks to @pchaigno. I also shared the full log of this minimal version for the record. I am only wondering, could there be any non-trivial reason for the misuse ofbpf_map_update()
other than the padding of the structs and uninitialized bytes?val
have been initialised before you tried to read from it when updating the map, because of the padding. Have you tried something like this before filling the struct?:memset(&val, 0, sizeof(val))
. Or just copying 0s intoval.pad