42 static void __scm_free_rq(struct scm_request *scmrq)  in __scm_free_rq()  argument
44 	struct aob_rq_header *aobrq = to_aobrq(scmrq);  in __scm_free_rq()
46 	free_page((unsigned long) scmrq->aob);  in __scm_free_rq()
47 	kfree(scmrq->request);  in __scm_free_rq()
54 	struct scm_request *scmrq;  in scm_free_rqs()  local
58 		scmrq = list_entry(iter, struct scm_request, list);  in scm_free_rqs()
59 		list_del(&scmrq->list);  in scm_free_rqs()
60 		__scm_free_rq(scmrq);  in scm_free_rqs()
70 	struct scm_request *scmrq;  in __scm_alloc_rq()  local
72 	aobrq = kzalloc(sizeof(*aobrq) + sizeof(*scmrq), GFP_KERNEL);  in __scm_alloc_rq()
76 	scmrq = (void *) aobrq->data;  in __scm_alloc_rq()
77 	scmrq->aob = (void *) get_zeroed_page(GFP_DMA);  in __scm_alloc_rq()
78 	if (!scmrq->aob)  in __scm_alloc_rq()
81 	scmrq->request = kcalloc(nr_requests_per_io, sizeof(scmrq->request[0]),  in __scm_alloc_rq()
83 	if (!scmrq->request)  in __scm_alloc_rq()
86 	INIT_LIST_HEAD(&scmrq->list);  in __scm_alloc_rq()
88 	list_add(&scmrq->list, &inactive_requests);  in __scm_alloc_rq()
93 	__scm_free_rq(scmrq);  in __scm_alloc_rq()
113 	struct scm_request *scmrq = NULL;  in scm_request_fetch()  local
118 	scmrq = list_first_entry(&inactive_requests, struct scm_request, list);  in scm_request_fetch()
119 	list_del(&scmrq->list);  in scm_request_fetch()
122 	return scmrq;  in scm_request_fetch()
125 static void scm_request_done(struct scm_request *scmrq)  in scm_request_done()  argument
132 	for (i = 0; i < nr_requests_per_io && scmrq->request[i]; i++) {  in scm_request_done()
133 		msb = &scmrq->aob->msb[i];  in scm_request_done()
142 	list_add(&scmrq->list, &inactive_requests);  in scm_request_done()
166 struct aidaw *scm_aidaw_fetch(struct scm_request *scmrq, unsigned int bytes)  in scm_aidaw_fetch()  argument
170 	if (scm_aidaw_bytes(scmrq->next_aidaw) >= bytes)  in scm_aidaw_fetch()
171 		return scmrq->next_aidaw;  in scm_aidaw_fetch()
179 static int scm_request_prepare(struct scm_request *scmrq)  in scm_request_prepare()  argument
181 	struct scm_blk_dev *bdev = scmrq->bdev;  in scm_request_prepare()
183 	int pos = scmrq->aob->request.msb_count;  in scm_request_prepare()
184 	struct msb *msb = &scmrq->aob->msb[pos];  in scm_request_prepare()
185 	struct request *req = scmrq->request[pos];  in scm_request_prepare()
190 	aidaw = scm_aidaw_fetch(scmrq, blk_rq_bytes(req));  in scm_request_prepare()
195 	scmrq->aob->request.msb_count++;  in scm_request_prepare()
208 	scmrq->next_aidaw = aidaw;  in scm_request_prepare()
212 static inline void scm_request_set(struct scm_request *scmrq,  in scm_request_set()  argument
215 	scmrq->request[scmrq->aob->request.msb_count] = req;  in scm_request_set()
219 				    struct scm_request *scmrq)  in scm_request_init()  argument
221 	struct aob_rq_header *aobrq = to_aobrq(scmrq);  in scm_request_init()
222 	struct aob *aob = scmrq->aob;  in scm_request_init()
224 	memset(scmrq->request, 0,  in scm_request_init()
225 	       nr_requests_per_io * sizeof(scmrq->request[0]));  in scm_request_init()
230 	scmrq->bdev = bdev;  in scm_request_init()
231 	scmrq->retries = 4;  in scm_request_init()
232 	scmrq->error = BLK_STS_OK;  in scm_request_init()
234 	scmrq->next_aidaw = (void *) &aob->msb[nr_requests_per_io];  in scm_request_init()
237 static void scm_request_requeue(struct scm_request *scmrq)  in scm_request_requeue()  argument
239 	struct scm_blk_dev *bdev = scmrq->bdev;  in scm_request_requeue()
242 	for (i = 0; i < nr_requests_per_io && scmrq->request[i]; i++)  in scm_request_requeue()
243 		blk_mq_requeue_request(scmrq->request[i], false);  in scm_request_requeue()
246 	scm_request_done(scmrq);  in scm_request_requeue()
250 static void scm_request_finish(struct scm_request *scmrq)  in scm_request_finish()  argument
252 	struct scm_blk_dev *bdev = scmrq->bdev;  in scm_request_finish()
256 	for (i = 0; i < nr_requests_per_io && scmrq->request[i]; i++) {  in scm_request_finish()
257 		error = blk_mq_rq_to_pdu(scmrq->request[i]);  in scm_request_finish()
258 		*error = scmrq->error;  in scm_request_finish()
259 		blk_mq_complete_request(scmrq->request[i]);  in scm_request_finish()
263 	scm_request_done(scmrq);  in scm_request_finish()
266 static void scm_request_start(struct scm_request *scmrq)  in scm_request_start()  argument
268 	struct scm_blk_dev *bdev = scmrq->bdev;  in scm_request_start()
271 	if (eadm_start_aob(scmrq->aob)) {  in scm_request_start()
273 		scm_request_requeue(scmrq);  in scm_request_start()
278 	struct scm_request *scmrq;  member
289 	struct scm_request *scmrq;  in scm_blk_request()  local
297 	scmrq = sq->scmrq;  in scm_blk_request()
298 	if (!scmrq) {  in scm_blk_request()
299 		scmrq = scm_request_fetch();  in scm_blk_request()
300 		if (!scmrq) {  in scm_blk_request()
305 		scm_request_init(bdev, scmrq);  in scm_blk_request()
306 		sq->scmrq = scmrq;  in scm_blk_request()
308 	scm_request_set(scmrq, req);  in scm_blk_request()
310 	if (scm_request_prepare(scmrq)) {  in scm_blk_request()
312 		scm_request_set(scmrq, NULL);  in scm_blk_request()
314 		if (scmrq->aob->request.msb_count)  in scm_blk_request()
315 			scm_request_start(scmrq);  in scm_blk_request()
317 		sq->scmrq = NULL;  in scm_blk_request()
323 	if (qd->last || scmrq->aob->request.msb_count == nr_requests_per_io) {  in scm_blk_request()
324 		scm_request_start(scmrq);  in scm_blk_request()
325 		sq->scmrq = NULL;  in scm_blk_request()
349 	WARN_ON(qd->scmrq);  in scm_blk_exit_hctx()
354 static void __scmrq_log_error(struct scm_request *scmrq)  in __scmrq_log_error()  argument
356 	struct aob *aob = scmrq->aob;  in __scmrq_log_error()
358 	if (scmrq->error == BLK_STS_TIMEOUT)  in __scmrq_log_error()
364 	if (scmrq->retries)  in __scmrq_log_error()
368 		       scmrq->error);  in __scmrq_log_error()
371 static void scm_blk_handle_error(struct scm_request *scmrq)  in scm_blk_handle_error()  argument
373 	struct scm_blk_dev *bdev = scmrq->bdev;  in scm_blk_handle_error()
376 	if (scmrq->error != BLK_STS_IOERR)  in scm_blk_handle_error()
380 	switch (scmrq->aob->response.eqc) {  in scm_blk_handle_error()
394 	if (!eadm_start_aob(scmrq->aob))  in scm_blk_handle_error()
398 	scm_request_requeue(scmrq);  in scm_blk_handle_error()
403 	struct scm_request *scmrq = data;  in scm_blk_irq()  local
405 	scmrq->error = error;  in scm_blk_irq()
407 		__scmrq_log_error(scmrq);  in scm_blk_irq()
408 		if (scmrq->retries-- > 0) {  in scm_blk_irq()
409 			scm_blk_handle_error(scmrq);  in scm_blk_irq()
414 	scm_request_finish(scmrq);  in scm_blk_irq()