xref: /illumos-gate/usr/src/uts/common/sys/ib/clients/rds/rdsib_ib.h (revision 00a3eaf3896a33935e11fd5c5fb5c1714225c067)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 /*
26  * Copyright (c) 2005 SilverStorm Technologies, Inc. All rights reserved.
27  *
28  * This software is available to you under a choice of one of two
29  * licenses.  You may choose to be licensed under the terms of the GNU
30  * General Public License (GPL) Version 2, available from the file
31  * COPYING in the main directory of this source tree, or the
32  * OpenIB.org BSD license below:
33  *
34  *     Redistribution and use in source and binary forms, with or
35  *     without modification, are permitted provided that the following
36  *     conditions are met:
37  *
38  *	- Redistributions of source code must retain the above
39  *	  copyright notice, this list of conditions and the following
40  *	  disclaimer.
41  *
42  *	- Redistributions in binary form must reproduce the above
43  *	  copyright notice, this list of conditions and the following
44  *	  disclaimer in the documentation and/or other materials
45  *	  provided with the distribution.
46  *
47  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
48  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
49  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
50  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
51  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
52  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
53  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
54  * SOFTWARE.
55  *
56  */
57 /*
58  * Sun elects to include this software in Sun product
59  * under the OpenIB BSD license.
60  *
61  *
62  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
63  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
64  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
65  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
66  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
67  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
68  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
69  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
70  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
71  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
72  * POSSIBILITY OF SUCH DAMAGE.
73  */
74 
75 #ifndef _RDSIB_IB_H
76 #define	_RDSIB_IB_H
77 
78 #ifdef __cplusplus
79 extern "C" {
80 #endif
81 
82 #include <sys/types.h>
83 #include <sys/ib/ibtl/ibti.h>
84 #include "rdsib_debug.h"
85 #include "rdsib_protocol.h"
86 
87 /*
88  * Global Configuration Variables
89  * As defined in RDS proposal
90  */
91 extern uint_t		MaxNodes;
92 extern uint_t		UserBufferSize;
93 extern uint_t		RdsPktSize;
94 extern uint_t		NDataRX;
95 extern uint_t		MaxDataSendBuffers;
96 extern uint_t		MaxDataRecvBuffers;
97 extern uint_t		MaxCtrlSendBuffers;
98 extern uint_t		MaxCtrlRecvBuffers;
99 extern uint_t		DataRecvBufferLWM;
100 extern uint_t		CtrlRecvBufferLWM;
101 extern uint_t		PendingRxPktsHWM;
102 extern uint_t		MinRnrRetry;
103 extern uint8_t		IBPathRetryCount;
104 extern uint8_t		IBPktLifeTime;
105 
106 #ifdef DEBUG
107 extern uint32_t		rdsdbglvl;
108 #else
109 extern uint32_t		rdsdbglvl;
110 #endif
111 
112 /* performance tunables */
113 extern uint_t		rds_no_interrupts;
114 extern uint_t		rds_poll_percent_full;
115 extern uint_t		rds_wc_signal;
116 extern uint_t		rds_waittime_ms;
117 
118 /* loopback port map */
119 #define			RDS_PORT_MAP_SIZE	8192
120 extern krwlock_t	rds_loopback_portmap_lock;
121 extern uint8_t		rds_loopback_portmap[RDS_PORT_MAP_SIZE];
122 
123 extern ddi_taskq_t	*rds_taskq;
124 extern uint_t		rds_rx_pkts_pending_hwm; /* readonly */
125 
126 /* Number of WCs to poll in a single call */
127 #define	RDS_NUM_DATA_SEND_WCS	10
128 #define	RDS_RDMAW_WRID	0xdabadaba
129 #define	RDS_NUM_ACKS	4 /* only 1 is used */
130 
131 typedef enum rds_hca_state_s {
132 	RDS_HCA_STATE_ADDED		= 0,
133 	RDS_HCA_STATE_OPEN		= 1,
134 	RDS_HCA_STATE_MEM_REGISTERED	= 2,
135 	RDS_HCA_STATE_STOPPING		= 3,
136 	RDS_HCA_STATE_REMOVED		= 4
137 } rds_hca_state_t;
138 
139 /*
140  * There is one of this structure for each HCA in the system.
141  * This holds all the information about the HCA.
142  *
143  * hca_nextp - Points to the next hca in the system.
144  * hca_state - State of the hca (only modified on HCA attach/detach)
145  * hca_guid - HCA Guid
146  * hca_nports - Number of ports on the HCA
147  * hca_hdl - HCA hdl obtained after opening the HCA
148  * hca_pdhdl - PD hdl
149  * hca_lkey - LKey for the registered global receive buffer pool memory
150  * hca_rkey - Rkey for the registered global receive buffer pool memory
151  * hca_attrp - HCA attributes
152  * hca_pinfop - ptr to portinfo data, allocated by ibtf
153  * hca_pinfo_sz - Sizeof of portinfo data
154  */
155 typedef struct rds_hca_s {
156 	struct rds_hca_s	*hca_nextp;
157 	rds_hca_state_t		hca_state;
158 	ib_guid_t		hca_guid;
159 	uint_t			hca_nports;
160 	ibt_hca_hdl_t		hca_hdl;
161 	ibt_pd_hdl_t		hca_pdhdl;
162 	ibt_mr_hdl_t		hca_mrhdl;
163 	ibt_lkey_t		hca_lkey;
164 	ibt_rkey_t		hca_rkey;
165 	ibt_sbind_hdl_t		hca_bindhdl[4];
166 	ibt_hca_attr_t		hca_attr;
167 	ibt_hca_portinfo_t	*hca_pinfop;
168 	uint_t			hca_pinfo_sz;
169 } rds_hca_t;
170 
171 /*
172  * RDS Soft State
173  * NOTE: Only one soft state per driver and NOT per instance.
174  *
175  * sessionlock - protects the rds_session_t:session_nextp, this lock has
176  *      to be taken for read/write acess of the sessions list.
177  * nsessions - Number of sessions in sessionlist
178  * sessionlistp - Pointer to the first session.
179  * ibhdl - Clnt handle acquired after registering with IBTF
180  * nhcas - Number of HCAs initialized. This is also the number of rds_hca_t
181  * 	structures in the rds_hcalistp.
182  * hcalistp - list of rds_hca_t.
183  * srvhdl - RDS service handle
184  */
185 typedef struct rds_state_s {
186 	krwlock_t		rds_sessionlock;
187 	uint_t			rds_nsessions;
188 	struct rds_session_s	*rds_sessionlistp;
189 	ibt_clnt_hdl_t		rds_ibhdl;
190 	krwlock_t		rds_hca_lock;
191 	uint_t			rds_nhcas;
192 	rds_hca_t		*rds_hcalistp;
193 	ibt_srv_hdl_t		rds_srvhdl;
194 	ib_svc_id_t		rds_service_id;
195 } rds_state_t;
196 
197 extern rds_state_t	*rdsib_statep; /* global */
198 
199 /* defined in rds_cm.c */
200 ibt_srv_hdl_t rds_register_service(ibt_clnt_hdl_t rds_ibhdl);
201 int rds_bind_service(struct rds_state_s *statep);
202 
203 /* defined in rds_ib.c */
204 void rds_recvcq_handler(ibt_cq_hdl_t cq, void *);
205 rds_hca_t *rds_gid_to_hcap(rds_state_t *statep, ib_gid_t gid);
206 rds_hca_t *rds_get_hcap(rds_state_t *statep, ib_guid_t guid);
207 int rdsib_initialize_ib();
208 void rdsib_deinitialize_ib();
209 
210 /* defined in rds_debug.c */
211 void rds_logging_initialization();
212 void rds_logging_destroy();
213 
214 #ifdef __cplusplus
215 }
216 #endif
217 
218 #endif	/* _RDSIB_IB_H */
219