xref: /illumos-gate/usr/src/lib/udapl/udapl_tavor/tavor/dapl_tavor_ibtf_dto.c (revision 9e39c5ba00a55fa05777cc94b148296af305e135)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #include "dapl.h"
28 #include "dapl_adapter_util.h"
29 #include "dapl_lmr_util.h"
30 #include "dapl_rmr_util.h"
31 #include "dapl_cookie.h"
32 
33 #include "dapl_tavor_ibtf_impl.h"
34 
35 /*
36  *
37  *
38  * MODULE: dapl_tavor_ibtf_dto.c
39  *
40  * PURPOSE: Utility routines for data transfer operations
41  *
42  */
43 
44 
45 /*
46  * dapls_ib_post_recv
47  *
48  * Provider specific Post RECV function
49  */
50 DAT_RETURN
dapls_ib_post_recv(IN DAPL_EP * ep_ptr,IN DAPL_COOKIE * cookie,IN DAT_COUNT num_segments,IN DAT_LMR_TRIPLET * local_iov,IN DAT_COMPLETION_FLAGS completion_flags)51 dapls_ib_post_recv(
52 	IN DAPL_EP		*ep_ptr,
53 	IN DAPL_COOKIE		*cookie,
54 	IN DAT_COUNT		num_segments,
55 	IN DAT_LMR_TRIPLET	*local_iov,
56 	IN DAT_COMPLETION_FLAGS completion_flags)
57 {
58 	ibt_recv_wr_t		pr_wr;
59 	ibt_wr_ds_t		pr_sgl_arr[DAPL_MAX_IOV];
60 	ibt_wr_ds_t		*pr_sgl;
61 	boolean_t		suppress_notification;
62 	DAT_COUNT		total_len;
63 	int			retval;
64 	int			i;
65 
66 	total_len = 0;
67 
68 	if (ep_ptr->qp_handle == NULL) {
69 		dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_recv: "
70 		    "qp_handle == NULL\n");
71 		return (DAT_INVALID_PARAMETER);
72 	}
73 
74 	/* allocate scatter-gather list on the heap if its large */
75 	if (num_segments > DAPL_MAX_IOV) {
76 		pr_sgl = dapl_os_alloc(num_segments * sizeof (ibt_wr_ds_t));
77 		if (NULL == pr_sgl) {
78 			dapl_dbg_log(DAPL_DBG_TYPE_ERR,
79 			    "dapls_ib_post_recv: pr_sgl alloc failed");
80 			return (DAT_INSUFFICIENT_RESOURCES);
81 		}
82 	} else {
83 		pr_sgl = pr_sgl_arr;
84 	}
85 
86 	for (i = 0; i < num_segments; i++) {
87 		pr_sgl[i].ds_va = (ib_vaddr_t)local_iov[i].virtual_address;
88 		pr_sgl[i].ds_key = (ibt_lkey_t)local_iov[i].lmr_context;
89 		pr_sgl[i].ds_len = (ib_msglen_t)local_iov[i].segment_length;
90 
91 		total_len += pr_sgl[i].ds_len;
92 		dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_recv: "
93 		    "i(%d) va(%p), lmrctxt(0x%x), len(%llu)\n", i,
94 		    pr_sgl[i].ds_va, pr_sgl[i].ds_key, pr_sgl[i].ds_len);
95 	}
96 
97 	if (cookie != NULL) {
98 		cookie->val.dto.size =  total_len;
99 		dapl_dbg_log(DAPL_DBG_TYPE_EP,
100 		    "dapls_ib_post_recv: dto_cookie(%p), num_seg(%d), "
101 		    "size(%d) hkey(%016llx)\n", cookie, num_segments,
102 		    cookie->val.dto.size, ep_ptr->qp_handle->ep_hkey);
103 	}
104 
105 	pr_wr.wr_id = (ibt_wrid_t)(uintptr_t)cookie;
106 	pr_wr.wr_nds = (uint32_t)num_segments;
107 	if (num_segments > 0) {
108 		pr_wr.wr_sgl = &pr_sgl[0];
109 	} else {
110 		pr_wr.wr_sgl = NULL;
111 	}
112 
113 	if (ep_ptr->param.ep_attr.recv_completion_flags &
114 	    DAT_COMPLETION_UNSIGNALLED_FLAG) {
115 		/* This flag is used to control notification of completions */
116 		suppress_notification = (completion_flags &
117 		    DAT_COMPLETION_UNSIGNALLED_FLAG) ? B_TRUE : B_FALSE;
118 	} else {
119 		/*
120 		 * The evd waiter will use threshold to control wakeups
121 		 * Hence the event notification will be done via arming the
122 		 * CQ so we do not need special notification generation
123 		 * hence set suppression to true
124 		 */
125 		suppress_notification = B_TRUE;
126 	}
127 
128 	retval = DAPL_RECV(ep_ptr)(ep_ptr, &pr_wr, suppress_notification);
129 
130 	if (retval != 0) {
131 		dapl_dbg_log(DAPL_DBG_TYPE_EP,
132 		    "dapls_ib_post_recv: post_recv failed %s\n",
133 		    strerror(errno));
134 	}
135 
136 	/* free the pr_sgl if we had allocated it */
137 	if (num_segments > DAPL_MAX_IOV) {
138 		dapl_os_free(pr_sgl, num_segments*sizeof (ibt_wr_ds_t));
139 	}
140 
141 	return (retval);
142 }
143 
144 /*
145  * dapls_ib_post_recv_one
146  *
147  * Provider specific Post RECV function
148  */
149 DAT_RETURN
dapls_ib_post_recv_one(IN DAPL_EP * ep_ptr,IN DAPL_COOKIE * cookie,IN DAT_LMR_TRIPLET * local_iov)150 dapls_ib_post_recv_one(
151 	IN DAPL_EP		*ep_ptr,
152 	IN DAPL_COOKIE		*cookie,
153 	IN DAT_LMR_TRIPLET	*local_iov)
154 {
155 	ibt_recv_wr_t		pr_wr;
156 	ibt_wr_ds_t		pr_sgl;
157 	boolean_t		suppress_notification;
158 	DAT_COUNT		total_len;
159 	int			retval;
160 
161 	if (ep_ptr->qp_handle == NULL) {
162 		dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_recv_one: "
163 		    "qp_handle == NULL\n");
164 		return (DAT_INVALID_PARAMETER);
165 	}
166 
167 	pr_sgl.ds_va = (ib_vaddr_t)local_iov->virtual_address;
168 	pr_sgl.ds_key = (ibt_lkey_t)local_iov->lmr_context;
169 	pr_sgl.ds_len = (ib_msglen_t)local_iov->segment_length;
170 
171 	total_len = pr_sgl.ds_len;
172 	dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_recv_one: "
173 	    "va(%p), lmrctxt(0x%x), len(%llu)\n",
174 	    pr_sgl.ds_va, pr_sgl.ds_key, pr_sgl.ds_len);
175 
176 	if (cookie != NULL) {
177 		cookie->val.dto.size =  total_len;
178 		dapl_dbg_log(DAPL_DBG_TYPE_EP,
179 		    "dapls_ib_post_recv_one: dto_cookie(%p), num_seg(1), "
180 		    "size(%d) hkey(%016llx)\n", cookie,
181 		    cookie->val.dto.size, ep_ptr->qp_handle->ep_hkey);
182 	}
183 
184 	pr_wr.wr_id = (ibt_wrid_t)(uintptr_t)cookie;
185 	pr_wr.wr_nds = 1;
186 	pr_wr.wr_sgl = &pr_sgl;
187 
188 	if (ep_ptr->param.ep_attr.recv_completion_flags &
189 	    DAT_COMPLETION_UNSIGNALLED_FLAG) {
190 		/* This flag is used to control notification of completions */
191 		suppress_notification = B_FALSE;
192 	} else {
193 		/*
194 		 * The evd waiter will use threshold to control wakeups
195 		 * Hence the event notification will be done via arming the
196 		 * CQ so we do not need special notification generation
197 		 * hence set suppression to true
198 		 */
199 		suppress_notification = B_TRUE;
200 	}
201 
202 	retval = DAPL_RECV(ep_ptr)(ep_ptr, &pr_wr, suppress_notification);
203 
204 	if (retval != 0) {
205 		dapl_dbg_log(DAPL_DBG_TYPE_EP,
206 		    "dapls_ib_post_recv_one: post_recv failed %s\n",
207 		    strerror(errno));
208 	}
209 
210 	return (retval);
211 }
212 
213 /*
214  * dapls_ib_srq_post_recv
215  *
216  * Provider specific SRQ Post RECV function
217  */
218 DAT_RETURN
dapls_ib_post_srq(IN DAPL_SRQ * srq_ptr,IN DAPL_COOKIE * cookie,IN DAT_COUNT num_segments,IN DAT_LMR_TRIPLET * local_iov)219 dapls_ib_post_srq(
220 	IN DAPL_SRQ		*srq_ptr,
221 	IN DAPL_COOKIE		*cookie,
222 	IN DAT_COUNT		num_segments,
223 	IN DAT_LMR_TRIPLET	*local_iov)
224 {
225 	ibt_recv_wr_t		pr_wr;
226 	ibt_wr_ds_t		pr_sgl_arr[DAPL_MAX_IOV];
227 	ibt_wr_ds_t		*pr_sgl;
228 	DAT_COUNT		total_len;
229 	int			retval;
230 	int			i;
231 
232 	total_len = 0;
233 
234 	if (srq_ptr->srq_handle == NULL) {
235 		dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_srq: "
236 		    "srq_handle == NULL\n");
237 		return (DAT_INVALID_PARAMETER);
238 	}
239 
240 	/* allocate scatter-gather list on the heap if its large */
241 	if (num_segments > DAPL_MAX_IOV) {
242 		pr_sgl = dapl_os_alloc(num_segments * sizeof (ibt_wr_ds_t));
243 		if (NULL == pr_sgl) {
244 			dapl_dbg_log(DAPL_DBG_TYPE_ERR,
245 			    "dapls_ib_post_srq: pr_sgl alloc failed");
246 			return (DAT_INSUFFICIENT_RESOURCES);
247 		}
248 	} else {
249 		pr_sgl = pr_sgl_arr;
250 	}
251 
252 	for (i = 0; i < num_segments; i++) {
253 		pr_sgl[i].ds_va = (ib_vaddr_t)local_iov[i].virtual_address;
254 		pr_sgl[i].ds_key = (ibt_lkey_t)local_iov[i].lmr_context;
255 		pr_sgl[i].ds_len = (ib_msglen_t)local_iov[i].segment_length;
256 
257 		total_len += pr_sgl[i].ds_len;
258 		dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_srq: "
259 		    "i(%d) va(%p), lmrctxt(0x%x), len(%u)\n", i,
260 		    pr_sgl[i].ds_va, pr_sgl[i].ds_key, pr_sgl[i].ds_len);
261 	}
262 
263 	if (cookie != NULL) {
264 		cookie->val.dto.size =  total_len;
265 		dapl_dbg_log(DAPL_DBG_TYPE_EP,
266 		    "dapls_ib_post_srq: dto_cookie(%p), num_seg(%d), "
267 		    "size(%d) hkey(%016llx)\n", cookie, num_segments,
268 		    cookie->val.dto.size, srq_ptr->srq_handle->srq_hkey);
269 	}
270 
271 	pr_wr.wr_id = (ibt_wrid_t)(uintptr_t)cookie;
272 	pr_wr.wr_nds = (uint32_t)num_segments;
273 	if (num_segments > 0) {
274 		pr_wr.wr_sgl = &pr_sgl[0];
275 	} else {
276 		pr_wr.wr_sgl = NULL;
277 	}
278 
279 	retval = DAPL_SRECV(srq_ptr)(srq_ptr, &pr_wr, B_TRUE);
280 
281 	if (retval != 0) {
282 		dapl_dbg_log(DAPL_DBG_TYPE_EP,
283 		    "dapls_ib_post_srq: post_recv failed %s\n",
284 		    strerror(errno));
285 	}
286 
287 	/* free the pr_sgl if we had allocated it */
288 	if (num_segments > DAPL_MAX_IOV) {
289 		dapl_os_free(pr_sgl, num_segments*sizeof (ibt_wr_ds_t));
290 	}
291 
292 	return (retval);
293 }
294 
295 /*
296  * dapls_ib_post_send
297  *
298  * Provider specific Post SEND function
299  */
300 DAT_RETURN
dapls_ib_post_send(IN DAPL_EP * ep_ptr,IN ib_send_op_type_t op_type,IN DAPL_COOKIE * cookie,IN DAT_COUNT num_segments,IN DAT_LMR_TRIPLET * local_iov,IN const DAT_RMR_TRIPLET * remote_iov,IN DAT_COMPLETION_FLAGS completion_flags)301 dapls_ib_post_send(IN DAPL_EP *ep_ptr,
302     IN ib_send_op_type_t op_type,
303     IN DAPL_COOKIE *cookie,
304     IN DAT_COUNT num_segments,
305     IN DAT_LMR_TRIPLET *local_iov,
306     IN const DAT_RMR_TRIPLET *remote_iov,
307     IN DAT_COMPLETION_FLAGS completion_flags)
308 {
309 	ibt_send_wr_t		ps_wr;
310 	ibt_wr_ds_t		ps_sgl_arr[DAPL_MAX_IOV];
311 	ibt_wr_ds_t		*ps_sgl;
312 	DAT_COUNT		total_len;
313 	boolean_t		suppress_notification;
314 	int			retval;
315 	int			i;
316 
317 	total_len = 0;
318 	retval = DAT_SUCCESS;
319 
320 	if (ep_ptr->qp_handle == NULL) {
321 		dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_send: "
322 		    "qp_handle == NULL\n");
323 		return (DAT_INVALID_PARAMETER);
324 	}
325 
326 	/* allocate scatter-gather list on the heap if its large */
327 	if (num_segments > DAPL_MAX_IOV) {
328 		ps_sgl = dapl_os_alloc(num_segments * sizeof (ibt_wr_ds_t));
329 		if (NULL == ps_sgl) {
330 			dapl_dbg_log(DAPL_DBG_TYPE_ERR,
331 			    "dapls_ib_post_send: pr_sgl alloc failed");
332 			return (DAT_INSUFFICIENT_RESOURCES);
333 		}
334 	} else {
335 		ps_sgl = ps_sgl_arr;
336 	}
337 
338 	for (i = 0; i < num_segments; i++) {
339 		ps_sgl[i].ds_va = (ib_vaddr_t)local_iov[i].virtual_address;
340 		ps_sgl[i].ds_key = (ibt_lkey_t)local_iov[i].lmr_context;
341 		ps_sgl[i].ds_len = (ib_msglen_t)local_iov[i].segment_length;
342 		total_len += ps_sgl[i].ds_len;
343 
344 		dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_send: "
345 		    "i(%d), va(0x%llx), lmrctxt(0x%x), len(%u)\n",
346 		    i, ps_sgl[i].ds_va, ps_sgl[i].ds_key, ps_sgl[i].ds_len);
347 	}
348 
349 	if (cookie != NULL)	{
350 		cookie->val.dto.size =  total_len;
351 		dapl_dbg_log(DAPL_DBG_TYPE_EVD,
352 		    "dapls_ib_post_send: op_type(%d), cookie(%p) "
353 		    "num_seg(%d) size(%d) hkey(%016llx)\n", op_type,
354 		    cookie, num_segments, cookie->val.dto.size,
355 		    ep_ptr->qp_handle->ep_hkey);
356 	}
357 
358 	ps_wr.wr_id = (ibt_wrid_t)(uintptr_t)cookie;
359 	/* Translate dapl flags */
360 	ps_wr.wr_flags = (DAT_COMPLETION_BARRIER_FENCE_FLAG &
361 	    completion_flags) ? IBT_WR_SEND_FENCE : 0;
362 	/* suppress completions */
363 	ps_wr.wr_flags |= (DAT_COMPLETION_SUPPRESS_FLAG &
364 	    completion_flags) ? 0 : IBT_WR_SEND_SIGNAL;
365 
366 	/* Solicited wait flag is valid only for post_send */
367 	if (op_type == OP_SEND) {
368 		ps_wr.wr_flags |= (DAT_COMPLETION_SOLICITED_WAIT_FLAG &
369 		    completion_flags) ? IBT_WR_SEND_SOLICIT : 0;
370 	}
371 
372 	ps_wr.wr_opcode = (ibt_wrc_opcode_t)op_type;
373 	ps_wr.wr_nds = (uint32_t)num_segments;
374 	if (num_segments > 0) {
375 		ps_wr.wr_sgl = &ps_sgl[0];
376 		if (op_type == OP_RDMA_READ || op_type == OP_RDMA_WRITE) {
377 			if (remote_iov == NULL) {
378 				/* free the ps_sgl if we had allocated it */
379 				if (num_segments > DAPL_MAX_IOV) {
380 					dapl_os_free(ps_sgl,
381 					    num_segments*sizeof (ibt_wr_ds_t));
382 				}
383 				dapl_dbg_log(DAPL_DBG_TYPE_EP,
384 				    "dapls_ib_post_send: "
385 				    "remote_iov == NULL\n");
386 				return (DAT_INVALID_PARAMETER);
387 			}
388 
389 			if (remote_iov->segment_length != (DAT_VLEN)total_len) {
390 				/* free the ps_sgl if we had allocated it */
391 				if (num_segments > DAPL_MAX_IOV) {
392 					dapl_os_free(ps_sgl,
393 					    num_segments*sizeof (ibt_wr_ds_t));
394 				}
395 				dapl_dbg_log(DAPL_DBG_TYPE_EP,
396 				    "dapls_ib_post_send: "
397 				    "remote_iov length(%llu != %llu)\n",
398 				    (DAT_VLEN)total_len,
399 				    remote_iov->segment_length);
400 				return (DAT_LENGTH_ERROR);
401 			}
402 
403 			ps_wr.wr.rc.rcwr.rdma.rdma_raddr =
404 			    (ib_vaddr_t)remote_iov->target_address;
405 			ps_wr.wr.rc.rcwr.rdma.rdma_rkey =
406 			    (ibt_rkey_t)remote_iov->rmr_context;
407 
408 			dapl_dbg_log(DAPL_DBG_TYPE_EP,
409 			    "dapls_ib_post_send: remote_iov taddr(0x%llx), "
410 			    "rmr(0x%x)\n", remote_iov->target_address,
411 			    remote_iov->rmr_context);
412 		}
413 	} else {
414 		ps_wr.wr_sgl = NULL;
415 	}
416 
417 	if (ep_ptr->param.ep_attr.recv_completion_flags &
418 	    DAT_COMPLETION_UNSIGNALLED_FLAG) {
419 		/* This flag is used to control notification of completions */
420 		suppress_notification = (completion_flags &
421 		    DAT_COMPLETION_UNSIGNALLED_FLAG) ? B_TRUE : B_FALSE;
422 	} else {
423 		/*
424 		 * The evd waiter will use threshold to control wakeups
425 		 * Hence the event notification will be done via arming the
426 		 * CQ so we do not need special notification generation
427 		 * hence set suppression to true
428 		 */
429 		suppress_notification = B_TRUE;
430 	}
431 
432 	retval = DAPL_SEND(ep_ptr)(ep_ptr, &ps_wr, suppress_notification);
433 
434 	if (retval != 0) {
435 		dapl_dbg_log(DAPL_DBG_TYPE_EP,
436 		    "dapls_ib_post_send: post_send failed %d\n", retval);
437 	}
438 
439 	/* free the pr_sgl if we had allocated it */
440 	if (num_segments > DAPL_MAX_IOV) {
441 		dapl_os_free(ps_sgl, num_segments*sizeof (ibt_wr_ds_t));
442 	}
443 
444 	return (retval);
445 }
446 
447 /*
448  * dapls_ib_post_send_one
449  *
450  * Provider specific Post SEND function - special case for the common case of
451  * sgl num_segments == 1 and completion_flags == DAT_COMPLETION_DEFAULT_FLAG.
452  */
453 DAT_RETURN
dapls_ib_post_send_one(IN DAPL_EP * ep_ptr,IN ib_send_op_type_t op_type,IN DAPL_COOKIE * cookie,IN DAT_LMR_TRIPLET * local_iov,IN const DAT_RMR_TRIPLET * remote_iov)454 dapls_ib_post_send_one(IN DAPL_EP *ep_ptr,
455     IN ib_send_op_type_t op_type,
456     IN DAPL_COOKIE *cookie,
457     IN DAT_LMR_TRIPLET *local_iov,
458     IN const DAT_RMR_TRIPLET *remote_iov)
459 {
460 	ibt_send_wr_t		ps_wr;
461 	ibt_wr_ds_t		ps_sgl;
462 	boolean_t		suppress_notification;
463 	int			retval;
464 
465 	if (ep_ptr->qp_handle == NULL) {
466 		dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_send_one: "
467 		    "qp_handle == NULL\n");
468 		return (DAT_INVALID_PARAMETER);
469 	}
470 
471 	ps_sgl.ds_va = (ib_vaddr_t)local_iov[0].virtual_address;
472 	ps_sgl.ds_key = (ibt_lkey_t)local_iov[0].lmr_context;
473 	ps_sgl.ds_len = (ib_msglen_t)local_iov[0].segment_length;
474 
475 	dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_send_one: "
476 	    "i(%d), va(0x%llx), lmrctxt(0x%x), len(%u)\n",
477 	    0, ps_sgl.ds_va, ps_sgl.ds_key, ps_sgl.ds_len);
478 
479 	cookie->val.dto.size =  ps_sgl.ds_len;
480 	dapl_dbg_log(DAPL_DBG_TYPE_EVD,
481 	    "dapls_ib_post_send_one: op_type(%d), cookie(%p) "
482 	    "num_seg(%d) size(%d) hkey(%016llx)\n", op_type,
483 	    cookie, 1, cookie->val.dto.size,
484 	    ep_ptr->qp_handle->ep_hkey);
485 
486 	ps_wr.wr_id = (ibt_wrid_t)(uintptr_t)cookie;
487 	/* suppress completions */
488 	ps_wr.wr_flags = IBT_WR_SEND_SIGNAL;
489 
490 	ps_wr.wr_opcode = (ibt_wrc_opcode_t)op_type;
491 	ps_wr.wr_nds = 1;
492 
493 	ps_wr.wr_sgl = &ps_sgl;
494 	if (op_type == OP_RDMA_READ || op_type == OP_RDMA_WRITE) {
495 		if (remote_iov == NULL) {
496 			/* free the ps_sgl if we had allocated it */
497 			dapl_dbg_log(DAPL_DBG_TYPE_EP,
498 			    "dapls_ib_post_send_one: "
499 			    "remote_iov == NULL\n");
500 			return (DAT_INVALID_PARAMETER);
501 		}
502 
503 		if (remote_iov->segment_length != (DAT_VLEN)ps_sgl.ds_len) {
504 			dapl_dbg_log(DAPL_DBG_TYPE_EP,
505 			    "dapls_ib_post_send_one: "
506 			    "remote_iov length(%llu != %llu)\n",
507 			    (DAT_VLEN)ps_sgl.ds_len,
508 			    remote_iov->segment_length);
509 			return (DAT_LENGTH_ERROR);
510 		}
511 
512 		ps_wr.wr.rc.rcwr.rdma.rdma_raddr =
513 		    (ib_vaddr_t)remote_iov->target_address;
514 		ps_wr.wr.rc.rcwr.rdma.rdma_rkey =
515 		    (ibt_rkey_t)remote_iov->rmr_context;
516 
517 		dapl_dbg_log(DAPL_DBG_TYPE_EP,
518 		    "dapls_ib_post_send_one: remote_iov taddr(0x%llx), "
519 		    "rmr(0x%x)\n", remote_iov->target_address,
520 		    remote_iov->rmr_context);
521 	}
522 
523 	if (ep_ptr->param.ep_attr.recv_completion_flags &
524 	    DAT_COMPLETION_UNSIGNALLED_FLAG) {
525 		/* This flag is used to control notification of completions */
526 		suppress_notification = B_FALSE;
527 	} else {
528 		/*
529 		 * The evd waiter will use threshold to control wakeups
530 		 * Hence the event notification will be done via arming the
531 		 * CQ so we do not need special notification generation
532 		 * hence set suppression to true
533 		 */
534 		suppress_notification = B_TRUE;
535 	}
536 
537 	retval = DAPL_SEND(ep_ptr)(ep_ptr, &ps_wr, suppress_notification);
538 
539 	if (retval != 0) {
540 		dapl_dbg_log(DAPL_DBG_TYPE_EP,
541 		    "dapls_ib_post_send_one: post_send failed %d\n", retval);
542 	}
543 
544 	return (retval);
545 }
546