1 /* include/net/xdp.h
2 *
3 * Copyright (c) 2017 Jesper Dangaard Brouer, Red Hat Inc.
4 * Released under terms in GPL version 2. See COPYING.
5 */
6 #ifndef __LINUX_NET_XDP_H__
7 #define __LINUX_NET_XDP_H__
8
9 /**
10 * DOC: XDP RX-queue information
11 *
12 * The XDP RX-queue info (xdp_rxq_info) is associated with the driver
13 * level RX-ring queues. It is information that is specific to how
14 * the driver have configured a given RX-ring queue.
15 *
16 * Each xdp_buff frame received in the driver carry a (pointer)
17 * reference to this xdp_rxq_info structure. This provides the XDP
18 * data-path read-access to RX-info for both kernel and bpf-side
19 * (limited subset).
20 *
21 * For now, direct access is only safe while running in NAPI/softirq
22 * context. Contents is read-mostly and must not be updated during
23 * driver NAPI/softirq poll.
24 *
25 * The driver usage API is a register and unregister API.
26 *
27 * The struct is not directly tied to the XDP prog. A new XDP prog
28 * can be attached as long as it doesn't change the underlying
29 * RX-ring. If the RX-ring does change significantly, the NIC driver
30 * naturally need to stop the RX-ring before purging and reallocating
31 * memory. In that process the driver MUST call unregistor (which
32 * also apply for driver shutdown and unload). The register API is
33 * also mandatory during RX-ring setup.
34 */
35
36 enum xdp_mem_type {
37 MEM_TYPE_PAGE_SHARED = 0, /* Split-page refcnt based model */
38 MEM_TYPE_PAGE_ORDER0, /* Orig XDP full page model */
39 MEM_TYPE_PAGE_POOL,
40 MEM_TYPE_ZERO_COPY,
41 MEM_TYPE_MAX,
42 };
43
44 /* XDP flags for ndo_xdp_xmit */
45 #define XDP_XMIT_FLUSH (1U << 0) /* doorbell signal consumer */
46 #define XDP_XMIT_FLAGS_MASK XDP_XMIT_FLUSH
47
48 struct xdp_mem_info {
49 u32 type; /* enum xdp_mem_type, but known size type */
50 u32 id;
51 };
52
53 struct page_pool;
54
55 struct zero_copy_allocator {
56 void (*free)(struct zero_copy_allocator *zca, unsigned long handle);
57 };
58
59 struct xdp_rxq_info {
60 struct net_device *dev;
61 u32 queue_index;
62 u32 reg_state;
63 struct xdp_mem_info mem;
64 } ____cacheline_aligned; /* perf critical, avoid false-sharing */
65
66 struct xdp_buff {
67 void *data;
68 void *data_end;
69 void *data_meta;
70 void *data_hard_start;
71 unsigned long handle;
72 struct xdp_rxq_info *rxq;
73 };
74
75 struct xdp_frame {
76 void *data;
77 u16 len;
78 u16 headroom;
79 u16 metasize;
80 /* Lifetime of xdp_rxq_info is limited to NAPI/enqueue time,
81 * while mem info is valid on remote CPU.
82 */
83 struct xdp_mem_info mem;
84 struct net_device *dev_rx; /* used by cpumap */
85 };
86
87 /* Clear kernel pointers in xdp_frame */
xdp_scrub_frame(struct xdp_frame * frame)88 static inline void xdp_scrub_frame(struct xdp_frame *frame)
89 {
90 frame->data = NULL;
91 frame->dev_rx = NULL;
92 }
93
94 /* Convert xdp_buff to xdp_frame */
95 static inline
convert_to_xdp_frame(struct xdp_buff * xdp)96 struct xdp_frame *convert_to_xdp_frame(struct xdp_buff *xdp)
97 {
98 struct xdp_frame *xdp_frame;
99 int metasize;
100 int headroom;
101
102 /* TODO: implement clone, copy, use "native" MEM_TYPE */
103 if (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY)
104 return NULL;
105
106 /* Assure headroom is available for storing info */
107 headroom = xdp->data - xdp->data_hard_start;
108 metasize = xdp->data - xdp->data_meta;
109 metasize = metasize > 0 ? metasize : 0;
110 if (unlikely((headroom - metasize) < sizeof(*xdp_frame)))
111 return NULL;
112
113 /* Store info in top of packet */
114 xdp_frame = xdp->data_hard_start;
115
116 xdp_frame->data = xdp->data;
117 xdp_frame->len = xdp->data_end - xdp->data;
118 xdp_frame->headroom = headroom - sizeof(*xdp_frame);
119 xdp_frame->metasize = metasize;
120
121 /* rxq only valid until napi_schedule ends, convert to xdp_mem_info */
122 xdp_frame->mem = xdp->rxq->mem;
123
124 return xdp_frame;
125 }
126
127 void xdp_return_frame(struct xdp_frame *xdpf);
128 void xdp_return_frame_rx_napi(struct xdp_frame *xdpf);
129 void xdp_return_buff(struct xdp_buff *xdp);
130
131 int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq,
132 struct net_device *dev, u32 queue_index);
133 void xdp_rxq_info_unreg(struct xdp_rxq_info *xdp_rxq);
134 void xdp_rxq_info_unused(struct xdp_rxq_info *xdp_rxq);
135 bool xdp_rxq_info_is_reg(struct xdp_rxq_info *xdp_rxq);
136 int xdp_rxq_info_reg_mem_model(struct xdp_rxq_info *xdp_rxq,
137 enum xdp_mem_type type, void *allocator);
138
139 /* Drivers not supporting XDP metadata can use this helper, which
140 * rejects any room expansion for metadata as a result.
141 */
142 static __always_inline void
xdp_set_data_meta_invalid(struct xdp_buff * xdp)143 xdp_set_data_meta_invalid(struct xdp_buff *xdp)
144 {
145 xdp->data_meta = xdp->data + 1;
146 }
147
148 static __always_inline bool
xdp_data_meta_unsupported(const struct xdp_buff * xdp)149 xdp_data_meta_unsupported(const struct xdp_buff *xdp)
150 {
151 return unlikely(xdp->data_meta > xdp->data);
152 }
153
154 struct xdp_attachment_info {
155 struct bpf_prog *prog;
156 u32 flags;
157 };
158
159 struct netdev_bpf;
160 int xdp_attachment_query(struct xdp_attachment_info *info,
161 struct netdev_bpf *bpf);
162 bool xdp_attachment_flags_ok(struct xdp_attachment_info *info,
163 struct netdev_bpf *bpf);
164 void xdp_attachment_setup(struct xdp_attachment_info *info,
165 struct netdev_bpf *bpf);
166
167 #endif /* __LINUX_NET_XDP_H__ */
168