1 /*
2  * Kernel Connection Multiplexor
3  *
4  * Copyright (c) 2016 Tom Herbert <tom@herbertland.com>
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License version 2
8  * as published by the Free Software Foundation.
9  */
10 
11 #ifndef __NET_KCM_H_
12 #define __NET_KCM_H_
13 
14 #include <linux/skbuff.h>
15 #include <net/sock.h>
16 #include <net/strparser.h>
17 #include <uapi/linux/kcm.h>
18 
19 extern unsigned int kcm_net_id;
20 
21 #define KCM_STATS_ADD(stat, count) ((stat) += (count))
22 #define KCM_STATS_INCR(stat) ((stat)++)
23 
24 struct kcm_psock_stats {
25 	unsigned long long tx_msgs;
26 	unsigned long long tx_bytes;
27 	unsigned long long reserved;
28 	unsigned long long unreserved;
29 	unsigned int tx_aborts;
30 };
31 
32 struct kcm_mux_stats {
33 	unsigned long long rx_msgs;
34 	unsigned long long rx_bytes;
35 	unsigned long long tx_msgs;
36 	unsigned long long tx_bytes;
37 	unsigned int rx_ready_drops;
38 	unsigned int tx_retries;
39 	unsigned int psock_attach;
40 	unsigned int psock_unattach_rsvd;
41 	unsigned int psock_unattach;
42 };
43 
44 struct kcm_stats {
45 	unsigned long long rx_msgs;
46 	unsigned long long rx_bytes;
47 	unsigned long long tx_msgs;
48 	unsigned long long tx_bytes;
49 };
50 
51 struct kcm_tx_msg {
52 	unsigned int sent;
53 	unsigned int fragidx;
54 	unsigned int frag_offset;
55 	unsigned int msg_flags;
56 	struct sk_buff *frag_skb;
57 	struct sk_buff *last_skb;
58 };
59 
60 /* Socket structure for KCM client sockets */
61 struct kcm_sock {
62 	struct sock sk;
63 	struct kcm_mux *mux;
64 	struct list_head kcm_sock_list;
65 	int index;
66 	u32 done : 1;
67 	struct work_struct done_work;
68 
69 	struct kcm_stats stats;
70 
71 	/* Transmit */
72 	struct kcm_psock *tx_psock;
73 	struct work_struct tx_work;
74 	struct list_head wait_psock_list;
75 	struct sk_buff *seq_skb;
76 	u32 tx_stopped : 1;
77 
78 	/* Don't use bit fields here, these are set under different locks */
79 	bool tx_wait;
80 	bool tx_wait_more;
81 
82 	/* Receive */
83 	struct kcm_psock *rx_psock;
84 	struct list_head wait_rx_list; /* KCMs waiting for receiving */
85 	bool rx_wait;
86 	u32 rx_disabled : 1;
87 };
88 
89 struct bpf_prog;
90 
91 /* Structure for an attached lower socket */
92 struct kcm_psock {
93 	struct sock *sk;
94 	struct strparser strp;
95 	struct kcm_mux *mux;
96 	int index;
97 
98 	u32 tx_stopped : 1;
99 	u32 done : 1;
100 	u32 unattaching : 1;
101 
102 	void (*save_state_change)(struct sock *sk);
103 	void (*save_data_ready)(struct sock *sk);
104 	void (*save_write_space)(struct sock *sk);
105 
106 	struct list_head psock_list;
107 
108 	struct kcm_psock_stats stats;
109 
110 	/* Receive */
111 	struct list_head psock_ready_list;
112 	struct bpf_prog *bpf_prog;
113 	struct kcm_sock *rx_kcm;
114 	unsigned long long saved_rx_bytes;
115 	unsigned long long saved_rx_msgs;
116 	struct sk_buff *ready_rx_msg;
117 
118 	/* Transmit */
119 	struct kcm_sock *tx_kcm;
120 	struct list_head psock_avail_list;
121 	unsigned long long saved_tx_bytes;
122 	unsigned long long saved_tx_msgs;
123 };
124 
125 /* Per net MUX list */
126 struct kcm_net {
127 	struct mutex mutex;
128 	struct kcm_psock_stats aggregate_psock_stats;
129 	struct kcm_mux_stats aggregate_mux_stats;
130 	struct strp_aggr_stats aggregate_strp_stats;
131 	struct list_head mux_list;
132 	int count;
133 };
134 
135 /* Structure for a MUX */
136 struct kcm_mux {
137 	struct list_head kcm_mux_list;
138 	struct rcu_head rcu;
139 	struct kcm_net *knet;
140 
141 	struct list_head kcm_socks;	/* All KCM sockets on MUX */
142 	int kcm_socks_cnt;		/* Total KCM socket count for MUX */
143 	struct list_head psocks;	/* List of all psocks on MUX */
144 	int psocks_cnt;		/* Total attached sockets */
145 
146 	struct kcm_mux_stats stats;
147 	struct kcm_psock_stats aggregate_psock_stats;
148 	struct strp_aggr_stats aggregate_strp_stats;
149 
150 	/* Receive */
151 	spinlock_t rx_lock ____cacheline_aligned_in_smp;
152 	struct list_head kcm_rx_waiters; /* KCMs waiting for receiving */
153 	struct list_head psocks_ready;	/* List of psocks with a msg ready */
154 	struct sk_buff_head rx_hold_queue;
155 
156 	/* Transmit */
157 	spinlock_t  lock ____cacheline_aligned_in_smp;	/* TX and mux locking */
158 	struct list_head psocks_avail;	/* List of available psocks */
159 	struct list_head kcm_tx_waiters; /* KCMs waiting for a TX psock */
160 };
161 
162 #ifdef CONFIG_PROC_FS
163 int kcm_proc_init(void);
164 void kcm_proc_exit(void);
165 #else
kcm_proc_init(void)166 static inline int kcm_proc_init(void) { return 0; }
kcm_proc_exit(void)167 static inline void kcm_proc_exit(void) { }
168 #endif
169 
aggregate_psock_stats(struct kcm_psock_stats * stats,struct kcm_psock_stats * agg_stats)170 static inline void aggregate_psock_stats(struct kcm_psock_stats *stats,
171 					 struct kcm_psock_stats *agg_stats)
172 {
173 	/* Save psock statistics in the mux when psock is being unattached. */
174 
175 #define SAVE_PSOCK_STATS(_stat) (agg_stats->_stat += stats->_stat)
176 	SAVE_PSOCK_STATS(tx_msgs);
177 	SAVE_PSOCK_STATS(tx_bytes);
178 	SAVE_PSOCK_STATS(reserved);
179 	SAVE_PSOCK_STATS(unreserved);
180 	SAVE_PSOCK_STATS(tx_aborts);
181 #undef SAVE_PSOCK_STATS
182 }
183 
aggregate_mux_stats(struct kcm_mux_stats * stats,struct kcm_mux_stats * agg_stats)184 static inline void aggregate_mux_stats(struct kcm_mux_stats *stats,
185 				       struct kcm_mux_stats *agg_stats)
186 {
187 	/* Save psock statistics in the mux when psock is being unattached. */
188 
189 #define SAVE_MUX_STATS(_stat) (agg_stats->_stat += stats->_stat)
190 	SAVE_MUX_STATS(rx_msgs);
191 	SAVE_MUX_STATS(rx_bytes);
192 	SAVE_MUX_STATS(tx_msgs);
193 	SAVE_MUX_STATS(tx_bytes);
194 	SAVE_MUX_STATS(rx_ready_drops);
195 	SAVE_MUX_STATS(psock_attach);
196 	SAVE_MUX_STATS(psock_unattach_rsvd);
197 	SAVE_MUX_STATS(psock_unattach);
198 #undef SAVE_MUX_STATS
199 }
200 
201 #endif /* __NET_KCM_H_ */
202