1 /* include/net/xdp.h
2  *
3  * Copyright (c) 2017 Jesper Dangaard Brouer, Red Hat Inc.
4  * Released under terms in GPL version 2.  See COPYING.
5  */
6 #ifndef __LINUX_NET_XDP_H__
7 #define __LINUX_NET_XDP_H__
8 
9 /**
10  * DOC: XDP RX-queue information
11  *
12  * The XDP RX-queue info (xdp_rxq_info) is associated with the driver
13  * level RX-ring queues.  It is information that is specific to how
14  * the driver have configured a given RX-ring queue.
15  *
16  * Each xdp_buff frame received in the driver carry a (pointer)
17  * reference to this xdp_rxq_info structure.  This provides the XDP
18  * data-path read-access to RX-info for both kernel and bpf-side
19  * (limited subset).
20  *
21  * For now, direct access is only safe while running in NAPI/softirq
22  * context.  Contents is read-mostly and must not be updated during
23  * driver NAPI/softirq poll.
24  *
25  * The driver usage API is a register and unregister API.
26  *
27  * The struct is not directly tied to the XDP prog.  A new XDP prog
28  * can be attached as long as it doesn't change the underlying
29  * RX-ring.  If the RX-ring does change significantly, the NIC driver
30  * naturally need to stop the RX-ring before purging and reallocating
31  * memory.  In that process the driver MUST call unregistor (which
32  * also apply for driver shutdown and unload).  The register API is
33  * also mandatory during RX-ring setup.
34  */
35 
36 enum xdp_mem_type {
37 	MEM_TYPE_PAGE_SHARED = 0, /* Split-page refcnt based model */
38 	MEM_TYPE_PAGE_ORDER0,     /* Orig XDP full page model */
39 	MEM_TYPE_PAGE_POOL,
40 	MEM_TYPE_ZERO_COPY,
41 	MEM_TYPE_MAX,
42 };
43 
44 /* XDP flags for ndo_xdp_xmit */
45 #define XDP_XMIT_FLUSH		(1U << 0)	/* doorbell signal consumer */
46 #define XDP_XMIT_FLAGS_MASK	XDP_XMIT_FLUSH
47 
48 struct xdp_mem_info {
49 	u32 type; /* enum xdp_mem_type, but known size type */
50 	u32 id;
51 };
52 
53 struct page_pool;
54 
55 struct zero_copy_allocator {
56 	void (*free)(struct zero_copy_allocator *zca, unsigned long handle);
57 };
58 
59 struct xdp_rxq_info {
60 	struct net_device *dev;
61 	u32 queue_index;
62 	u32 reg_state;
63 	struct xdp_mem_info mem;
64 } ____cacheline_aligned; /* perf critical, avoid false-sharing */
65 
66 struct xdp_buff {
67 	void *data;
68 	void *data_end;
69 	void *data_meta;
70 	void *data_hard_start;
71 	unsigned long handle;
72 	struct xdp_rxq_info *rxq;
73 };
74 
75 struct xdp_frame {
76 	void *data;
77 	u16 len;
78 	u16 headroom;
79 	u16 metasize;
80 	/* Lifetime of xdp_rxq_info is limited to NAPI/enqueue time,
81 	 * while mem info is valid on remote CPU.
82 	 */
83 	struct xdp_mem_info mem;
84 	struct net_device *dev_rx; /* used by cpumap */
85 };
86 
87 /* Clear kernel pointers in xdp_frame */
xdp_scrub_frame(struct xdp_frame * frame)88 static inline void xdp_scrub_frame(struct xdp_frame *frame)
89 {
90 	frame->data = NULL;
91 	frame->dev_rx = NULL;
92 }
93 
94 /* Convert xdp_buff to xdp_frame */
95 static inline
convert_to_xdp_frame(struct xdp_buff * xdp)96 struct xdp_frame *convert_to_xdp_frame(struct xdp_buff *xdp)
97 {
98 	struct xdp_frame *xdp_frame;
99 	int metasize;
100 	int headroom;
101 
102 	/* TODO: implement clone, copy, use "native" MEM_TYPE */
103 	if (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY)
104 		return NULL;
105 
106 	/* Assure headroom is available for storing info */
107 	headroom = xdp->data - xdp->data_hard_start;
108 	metasize = xdp->data - xdp->data_meta;
109 	metasize = metasize > 0 ? metasize : 0;
110 	if (unlikely((headroom - metasize) < sizeof(*xdp_frame)))
111 		return NULL;
112 
113 	/* Store info in top of packet */
114 	xdp_frame = xdp->data_hard_start;
115 
116 	xdp_frame->data = xdp->data;
117 	xdp_frame->len  = xdp->data_end - xdp->data;
118 	xdp_frame->headroom = headroom - sizeof(*xdp_frame);
119 	xdp_frame->metasize = metasize;
120 
121 	/* rxq only valid until napi_schedule ends, convert to xdp_mem_info */
122 	xdp_frame->mem = xdp->rxq->mem;
123 
124 	return xdp_frame;
125 }
126 
127 void xdp_return_frame(struct xdp_frame *xdpf);
128 void xdp_return_frame_rx_napi(struct xdp_frame *xdpf);
129 void xdp_return_buff(struct xdp_buff *xdp);
130 
131 int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq,
132 		     struct net_device *dev, u32 queue_index);
133 void xdp_rxq_info_unreg(struct xdp_rxq_info *xdp_rxq);
134 void xdp_rxq_info_unused(struct xdp_rxq_info *xdp_rxq);
135 bool xdp_rxq_info_is_reg(struct xdp_rxq_info *xdp_rxq);
136 int xdp_rxq_info_reg_mem_model(struct xdp_rxq_info *xdp_rxq,
137 			       enum xdp_mem_type type, void *allocator);
138 
139 /* Drivers not supporting XDP metadata can use this helper, which
140  * rejects any room expansion for metadata as a result.
141  */
142 static __always_inline void
xdp_set_data_meta_invalid(struct xdp_buff * xdp)143 xdp_set_data_meta_invalid(struct xdp_buff *xdp)
144 {
145 	xdp->data_meta = xdp->data + 1;
146 }
147 
148 static __always_inline bool
xdp_data_meta_unsupported(const struct xdp_buff * xdp)149 xdp_data_meta_unsupported(const struct xdp_buff *xdp)
150 {
151 	return unlikely(xdp->data_meta > xdp->data);
152 }
153 
154 struct xdp_attachment_info {
155 	struct bpf_prog *prog;
156 	u32 flags;
157 };
158 
159 struct netdev_bpf;
160 int xdp_attachment_query(struct xdp_attachment_info *info,
161 			 struct netdev_bpf *bpf);
162 bool xdp_attachment_flags_ok(struct xdp_attachment_info *info,
163 			     struct netdev_bpf *bpf);
164 void xdp_attachment_setup(struct xdp_attachment_info *info,
165 			  struct netdev_bpf *bpf);
166 
167 #endif /* __LINUX_NET_XDP_H__ */
168