203 	int (*cmd_send)(struct perf_peer *peer, enum perf_cmd cmd, u64 data);
248 static inline bool perf_link_is_up(struct perf_peer *peer)  in perf_link_is_up()  argument
252 	link = ntb_link_is_up(peer->perf->ntb, NULL, NULL);  in perf_link_is_up()
253 	return !!(link & BIT_ULL_MASK(peer->pidx));  in perf_link_is_up()
256 static int perf_spad_cmd_send(struct perf_peer *peer, enum perf_cmd cmd,  in perf_spad_cmd_send()  argument
259 	struct perf_ctx *perf = peer->perf;  in perf_spad_cmd_send()
273 		if (!perf_link_is_up(peer))  in perf_spad_cmd_send()
276 		sts = ntb_peer_spad_read(perf->ntb, peer->pidx,  in perf_spad_cmd_send()
283 		ntb_peer_spad_write(perf->ntb, peer->pidx,  in perf_spad_cmd_send()
286 		ntb_peer_spad_write(perf->ntb, peer->pidx,  in perf_spad_cmd_send()
290 		ntb_peer_spad_write(perf->ntb, peer->pidx,  in perf_spad_cmd_send()
294 		ntb_peer_db_set(perf->ntb, PERF_SPAD_NOTIFY(peer->gidx));  in perf_spad_cmd_send()
297 			PERF_SPAD_NOTIFY(peer->gidx));  in perf_spad_cmd_send()
308 	struct perf_peer *peer;  in perf_spad_cmd_recv()  local
320 		peer = &perf->peers[*pidx];  in perf_spad_cmd_recv()
322 		if (!perf_link_is_up(peer))  in perf_spad_cmd_recv()
325 		val = ntb_spad_read(perf->ntb, PERF_SPAD_CMD(peer->gidx));  in perf_spad_cmd_recv()
331 		val = ntb_spad_read(perf->ntb, PERF_SPAD_LDATA(peer->gidx));  in perf_spad_cmd_recv()
334 		val = ntb_spad_read(perf->ntb, PERF_SPAD_HDATA(peer->gidx));  in perf_spad_cmd_recv()
338 		ntb_spad_write(perf->ntb, PERF_SPAD_CMD(peer->gidx),  in perf_spad_cmd_recv()
349 static int perf_msg_cmd_send(struct perf_peer *peer, enum perf_cmd cmd,  in perf_msg_cmd_send()  argument
352 	struct perf_ctx *perf = peer->perf;  in perf_msg_cmd_send()
367 		if (!perf_link_is_up(peer))  in perf_msg_cmd_send()
374 		ntb_peer_msg_write(perf->ntb, peer->pidx, PERF_MSG_LDATA,  in perf_msg_cmd_send()
382 		ntb_peer_msg_write(perf->ntb, peer->pidx, PERF_MSG_HDATA,  in perf_msg_cmd_send()
387 		ntb_peer_msg_write(perf->ntb, peer->pidx, PERF_MSG_CMD, cmd);  in perf_msg_cmd_send()
423 static int perf_cmd_send(struct perf_peer *peer, enum perf_cmd cmd, u64 data)  in perf_cmd_send()  argument
425 	struct perf_ctx *perf = peer->perf;  in perf_cmd_send()
428 		return perf->cmd_send(peer, cmd, data);  in perf_cmd_send()
434 static int perf_cmd_exec(struct perf_peer *peer, enum perf_cmd cmd)  in perf_cmd_exec()  argument
444 		dev_err(&peer->perf->ntb->dev, "Exec invalid command\n");  in perf_cmd_exec()
449 	set_bit(cmd, &peer->sts);  in perf_cmd_exec()
451 	dev_dbg(&peer->perf->ntb->dev, "CMD exec: %d\n", cmd);  in perf_cmd_exec()
453 	(void)queue_work(system_highpri_wq, &peer->service);  in perf_cmd_exec()
460 	struct perf_peer *peer;  in perf_cmd_recv()  local
465 		peer = &perf->peers[pidx];  in perf_cmd_recv()
469 			peer->inbuf_size = data;  in perf_cmd_recv()
470 			return perf_cmd_exec(peer, PERF_CMD_RSIZE);  in perf_cmd_recv()
472 			peer->outbuf_xlat = data;  in perf_cmd_recv()
473 			return perf_cmd_exec(peer, PERF_CMD_RXLAT);  in perf_cmd_recv()
487 	struct perf_peer *peer;  in perf_link_event()  local
492 		peer = &perf->peers[pidx];  in perf_link_event()
494 		lnk_up = perf_link_is_up(peer);  in perf_link_event()
497 		    !test_and_set_bit(PERF_STS_LNKUP, &peer->sts)) {  in perf_link_event()
498 			perf_cmd_exec(peer, PERF_CMD_SSIZE);  in perf_link_event()
500 			   test_and_clear_bit(PERF_STS_LNKUP, &peer->sts)) {  in perf_link_event()
501 			perf_cmd_exec(peer, PERF_CMD_CLEAR);  in perf_link_event()
534 static void perf_free_outbuf(struct perf_peer *peer)  in perf_free_outbuf()  argument
536 	(void)ntb_peer_mw_clear_trans(peer->perf->ntb, peer->pidx, peer->gidx);  in perf_free_outbuf()
539 static int perf_setup_outbuf(struct perf_peer *peer)  in perf_setup_outbuf()  argument
541 	struct perf_ctx *perf = peer->perf;  in perf_setup_outbuf()
545 	ret = ntb_peer_mw_set_trans(perf->ntb, peer->pidx, peer->gidx,  in perf_setup_outbuf()
546 				    peer->outbuf_xlat, peer->outbuf_size);  in perf_setup_outbuf()
553 	set_bit(PERF_STS_DONE, &peer->sts);  in perf_setup_outbuf()
554 	complete_all(&peer->init_comp);  in perf_setup_outbuf()
559 static void perf_free_inbuf(struct perf_peer *peer)  in perf_free_inbuf()  argument
561 	if (!peer->inbuf)  in perf_free_inbuf()
564 	(void)ntb_mw_clear_trans(peer->perf->ntb, peer->pidx, peer->gidx);  in perf_free_inbuf()
565 	dma_free_coherent(&peer->perf->ntb->pdev->dev, peer->inbuf_size,  in perf_free_inbuf()
566 			  peer->inbuf, peer->inbuf_xlat);  in perf_free_inbuf()
567 	peer->inbuf = NULL;  in perf_free_inbuf()
570 static int perf_setup_inbuf(struct perf_peer *peer)  in perf_setup_inbuf()  argument
573 	struct perf_ctx *perf = peer->perf;  in perf_setup_inbuf()
577 	ret = ntb_mw_get_align(perf->ntb, peer->pidx, perf->gidx,  in perf_setup_inbuf()
584 	if (peer->inbuf_size > size_max) {  in perf_setup_inbuf()
586 			&peer->inbuf_size, &size_max);  in perf_setup_inbuf()
590 	peer->inbuf_size = round_up(peer->inbuf_size, size_align);  in perf_setup_inbuf()
592 	perf_free_inbuf(peer);  in perf_setup_inbuf()
594 	peer->inbuf = dma_alloc_coherent(&perf->ntb->pdev->dev,  in perf_setup_inbuf()
595 					 peer->inbuf_size, &peer->inbuf_xlat,  in perf_setup_inbuf()
597 	if (!peer->inbuf) {  in perf_setup_inbuf()
599 			&peer->inbuf_size);  in perf_setup_inbuf()
602 	if (!IS_ALIGNED(peer->inbuf_xlat, xlat_align)) {  in perf_setup_inbuf()
608 	ret = ntb_mw_set_trans(perf->ntb, peer->pidx, peer->gidx,  in perf_setup_inbuf()
609 			       peer->inbuf_xlat, peer->inbuf_size);  in perf_setup_inbuf()
620 	(void)perf_cmd_exec(peer, PERF_CMD_SXLAT);  in perf_setup_inbuf()
625 	perf_free_inbuf(peer);  in perf_setup_inbuf()
632 	struct perf_peer *peer = to_peer_service(work);  in perf_service_work()  local
634 	if (test_and_clear_bit(PERF_CMD_SSIZE, &peer->sts))  in perf_service_work()
635 		perf_cmd_send(peer, PERF_CMD_SSIZE, peer->outbuf_size);  in perf_service_work()
637 	if (test_and_clear_bit(PERF_CMD_RSIZE, &peer->sts))  in perf_service_work()
638 		perf_setup_inbuf(peer);  in perf_service_work()
640 	if (test_and_clear_bit(PERF_CMD_SXLAT, &peer->sts))  in perf_service_work()
641 		perf_cmd_send(peer, PERF_CMD_SXLAT, peer->inbuf_xlat);  in perf_service_work()
643 	if (test_and_clear_bit(PERF_CMD_RXLAT, &peer->sts))  in perf_service_work()
644 		perf_setup_outbuf(peer);  in perf_service_work()
646 	if (test_and_clear_bit(PERF_CMD_CLEAR, &peer->sts)) {  in perf_service_work()
647 		init_completion(&peer->init_comp);  in perf_service_work()
648 		clear_bit(PERF_STS_DONE, &peer->sts);  in perf_service_work()
649 		if (test_bit(0, &peer->perf->busy_flag) &&  in perf_service_work()
650 		    peer == peer->perf->test_peer) {  in perf_service_work()
651 			dev_warn(&peer->perf->ntb->dev,  in perf_service_work()
653 			perf_terminate_test(peer->perf);  in perf_service_work()
655 		perf_free_outbuf(peer);  in perf_service_work()
656 		perf_free_inbuf(peer);  in perf_service_work()
899 	struct perf_peer *peer = pthr->perf->test_peer;  in perf_run_test()  local
908 	chunk_size = min_t(u64, peer->outbuf_size, chunk_size);  in perf_run_test()
911 	bnd_dst = peer->outbuf + peer->outbuf_size;  in perf_run_test()
912 	flt_dst = peer->outbuf;  in perf_run_test()
929 		if (flt_dst >= bnd_dst || flt_dst < peer->outbuf) {  in perf_run_test()
930 			flt_dst = peer->outbuf;  in perf_run_test()
1049 static int perf_submit_test(struct perf_peer *peer)  in perf_submit_test()  argument
1051 	struct perf_ctx *perf = peer->perf;  in perf_submit_test()
1055 	ret = wait_for_completion_interruptible(&peer->init_comp);  in perf_submit_test()
1062 	perf->test_peer = peer;  in perf_submit_test()
1156 	struct perf_peer *peer;  in perf_dbgfs_read_info()  local
1185 		peer = &perf->peers[pidx];  in perf_dbgfs_read_info()
1189 			ntb_peer_port_number(perf->ntb, peer->pidx), peer->pidx,  in perf_dbgfs_read_info()
1190 			peer->gidx);  in perf_dbgfs_read_info()
1194 			test_bit(PERF_STS_LNKUP, &peer->sts) ? "up" : "down");  in perf_dbgfs_read_info()
1197 			"\tOut buffer addr 0x%pK\n", peer->outbuf);  in perf_dbgfs_read_info()
1200 			"\tOut buffer size %pa\n", &peer->outbuf_size);  in perf_dbgfs_read_info()
1203 			"\tOut buffer xlat 0x%016llx[p]\n", peer->outbuf_xlat);  in perf_dbgfs_read_info()
1205 		if (!peer->inbuf) {  in perf_dbgfs_read_info()
1212 			"\tIn buffer addr 0x%pK\n", peer->inbuf);  in perf_dbgfs_read_info()
1215 			"\tIn buffer size %pa\n", &peer->inbuf_size);  in perf_dbgfs_read_info()
1218 			"\tIn buffer xlat %pad[p]\n", &peer->inbuf_xlat);  in perf_dbgfs_read_info()
1258 	struct perf_peer *peer;  in perf_dbgfs_write_run()  local
1268 	peer = &perf->peers[pidx];  in perf_dbgfs_write_run()
1270 	ret = perf_submit_test(peer);  in perf_dbgfs_write_run()
1376 static int perf_setup_peer_mw(struct perf_peer *peer)  in perf_setup_peer_mw()  argument
1378 	struct perf_ctx *perf = peer->perf;  in perf_setup_peer_mw()
1384 				   &peer->outbuf_size);  in perf_setup_peer_mw()
1388 	peer->outbuf = devm_ioremap_wc(&perf->ntb->dev, phys_addr,  in perf_setup_peer_mw()
1389 					peer->outbuf_size);  in perf_setup_peer_mw()
1390 	if (!peer->outbuf)  in perf_setup_peer_mw()
1393 	if (max_mw_size && peer->outbuf_size > max_mw_size) {  in perf_setup_peer_mw()
1394 		peer->outbuf_size = max_mw_size;  in perf_setup_peer_mw()
1395 		dev_warn(&peer->perf->ntb->dev,  in perf_setup_peer_mw()
1396 			"Peer %d outbuf reduced to %pa\n", peer->pidx,  in perf_setup_peer_mw()
1397 			&peer->outbuf_size);  in perf_setup_peer_mw()
1405 	struct perf_peer *peer;  in perf_init_peers()  local
1411 		peer = &perf->peers[pidx];  in perf_init_peers()
1413 		peer->perf = perf;  in perf_init_peers()
1414 		peer->pidx = pidx;  in perf_init_peers()
1418 			peer->gidx = pidx + 1;  in perf_init_peers()
1420 			peer->gidx = pidx;  in perf_init_peers()
1422 		INIT_WORK(&peer->service, perf_service_work);  in perf_init_peers()
1423 		init_completion(&peer->init_comp);  in perf_init_peers()