1 #ifndef _HFI1_IOWAIT_H 2 #define _HFI1_IOWAIT_H 3 /* 4 * Copyright(c) 2015, 2016 Intel Corporation. 5 * 6 * This file is provided under a dual BSD/GPLv2 license. When using or 7 * redistributing this file, you may do so under either license. 8 * 9 * GPL LICENSE SUMMARY 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of version 2 of the GNU General Public License as 13 * published by the Free Software Foundation. 14 * 15 * This program is distributed in the hope that it will be useful, but 16 * WITHOUT ANY WARRANTY; without even the implied warranty of 17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 18 * General Public License for more details. 19 * 20 * BSD LICENSE 21 * 22 * Redistribution and use in source and binary forms, with or without 23 * modification, are permitted provided that the following conditions 24 * are met: 25 * 26 * - Redistributions of source code must retain the above copyright 27 * notice, this list of conditions and the following disclaimer. 28 * - Redistributions in binary form must reproduce the above copyright 29 * notice, this list of conditions and the following disclaimer in 30 * the documentation and/or other materials provided with the 31 * distribution. 32 * - Neither the name of Intel Corporation nor the names of its 33 * contributors may be used to endorse or promote products derived 34 * from this software without specific prior written permission. 35 * 36 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 37 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 38 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 39 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 40 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 41 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 42 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 43 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 44 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 45 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 46 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 47 * 48 */ 49 50 #include <linux/list.h> 51 #include <linux/workqueue.h> 52 #include <linux/sched.h> 53 54 #include "sdma_txreq.h" 55 56 /* 57 * typedef (*restart_t)() - restart callback 58 * @work: pointer to work structure 59 */ 60 typedef void (*restart_t)(struct work_struct *work); 61 62 struct sdma_txreq; 63 struct sdma_engine; 64 /** 65 * struct iowait - linkage for delayed progress/waiting 66 * @list: used to add/insert into QP/PQ wait lists 67 * @lock: uses to record the list head lock 68 * @tx_head: overflow list of sdma_txreq's 69 * @sleep: no space callback 70 * @wakeup: space callback wakeup 71 * @sdma_drained: sdma count drained 72 * @iowork: workqueue overhead 73 * @wait_dma: wait for sdma_busy == 0 74 * @wait_pio: wait for pio_busy == 0 75 * @sdma_busy: # of packets in flight 76 * @count: total number of descriptors in tx_head'ed list 77 * @tx_limit: limit for overflow queuing 78 * @tx_count: number of tx entry's in tx_head'ed list 79 * 80 * This is to be embedded in user's state structure 81 * (QP or PQ). 82 * 83 * The sleep and wakeup members are a 84 * bit misnamed. They do not strictly 85 * speaking sleep or wake up, but they 86 * are callbacks for the ULP to implement 87 * what ever queuing/dequeuing of 88 * the embedded iowait and its containing struct 89 * when a resource shortage like SDMA ring space is seen. 90 * 91 * Both potentially have locks help 92 * so sleeping is not allowed. 93 * 94 * The wait_dma member along with the iow 95 * 96 * The lock field is used by waiters to record 97 * the seqlock_t that guards the list head. 98 * Waiters explicity know that, but the destroy 99 * code that unwaits QPs does not. 100 */ 101 102 struct iowait { 103 struct list_head list; 104 struct list_head tx_head; 105 int (*sleep)( 106 struct sdma_engine *sde, 107 struct iowait *wait, 108 struct sdma_txreq *tx, 109 uint seq, 110 bool pkts_sent 111 ); 112 void (*wakeup)(struct iowait *wait, int reason); 113 void (*sdma_drained)(struct iowait *wait); 114 seqlock_t *lock; 115 struct work_struct iowork; 116 wait_queue_head_t wait_dma; 117 wait_queue_head_t wait_pio; 118 atomic_t sdma_busy; 119 atomic_t pio_busy; 120 u32 count; 121 u32 tx_limit; 122 u32 tx_count; 123 u8 starved_cnt; 124 }; 125 126 #define SDMA_AVAIL_REASON 0 127 128 /** 129 * iowait_init() - initialize wait structure 130 * @wait: wait struct to initialize 131 * @tx_limit: limit for overflow queuing 132 * @func: restart function for workqueue 133 * @sleep: sleep function for no space 134 * @resume: wakeup function for no space 135 * 136 * This function initializes the iowait 137 * structure embedded in the QP or PQ. 138 * 139 */ 140 141 static inline void iowait_init( 142 struct iowait *wait, 143 u32 tx_limit, 144 void (*func)(struct work_struct *work), 145 int (*sleep)( 146 struct sdma_engine *sde, 147 struct iowait *wait, 148 struct sdma_txreq *tx, 149 uint seq, 150 bool pkts_sent), 151 void (*wakeup)(struct iowait *wait, int reason), 152 void (*sdma_drained)(struct iowait *wait)) 153 { 154 wait->count = 0; 155 wait->lock = NULL; 156 INIT_LIST_HEAD(&wait->list); 157 INIT_LIST_HEAD(&wait->tx_head); 158 INIT_WORK(&wait->iowork, func); 159 init_waitqueue_head(&wait->wait_dma); 160 init_waitqueue_head(&wait->wait_pio); 161 atomic_set(&wait->sdma_busy, 0); 162 atomic_set(&wait->pio_busy, 0); 163 wait->tx_limit = tx_limit; 164 wait->sleep = sleep; 165 wait->wakeup = wakeup; 166 wait->sdma_drained = sdma_drained; 167 } 168 169 /** 170 * iowait_schedule() - initialize wait structure 171 * @wait: wait struct to schedule 172 * @wq: workqueue for schedule 173 * @cpu: cpu 174 */ 175 static inline void iowait_schedule( 176 struct iowait *wait, 177 struct workqueue_struct *wq, 178 int cpu) 179 { 180 queue_work_on(cpu, wq, &wait->iowork); 181 } 182 183 /** 184 * iowait_sdma_drain() - wait for DMAs to drain 185 * 186 * @wait: iowait structure 187 * 188 * This will delay until the iowait sdmas have 189 * completed. 190 */ 191 static inline void iowait_sdma_drain(struct iowait *wait) 192 { 193 wait_event(wait->wait_dma, !atomic_read(&wait->sdma_busy)); 194 } 195 196 /** 197 * iowait_sdma_pending() - return sdma pending count 198 * 199 * @wait: iowait structure 200 * 201 */ 202 static inline int iowait_sdma_pending(struct iowait *wait) 203 { 204 return atomic_read(&wait->sdma_busy); 205 } 206 207 /** 208 * iowait_sdma_inc - note sdma io pending 209 * @wait: iowait structure 210 */ 211 static inline void iowait_sdma_inc(struct iowait *wait) 212 { 213 atomic_inc(&wait->sdma_busy); 214 } 215 216 /** 217 * iowait_sdma_add - add count to pending 218 * @wait: iowait structure 219 */ 220 static inline void iowait_sdma_add(struct iowait *wait, int count) 221 { 222 atomic_add(count, &wait->sdma_busy); 223 } 224 225 /** 226 * iowait_sdma_dec - note sdma complete 227 * @wait: iowait structure 228 */ 229 static inline int iowait_sdma_dec(struct iowait *wait) 230 { 231 return atomic_dec_and_test(&wait->sdma_busy); 232 } 233 234 /** 235 * iowait_pio_drain() - wait for pios to drain 236 * 237 * @wait: iowait structure 238 * 239 * This will delay until the iowait pios have 240 * completed. 241 */ 242 static inline void iowait_pio_drain(struct iowait *wait) 243 { 244 wait_event_timeout(wait->wait_pio, 245 !atomic_read(&wait->pio_busy), 246 HZ); 247 } 248 249 /** 250 * iowait_pio_pending() - return pio pending count 251 * 252 * @wait: iowait structure 253 * 254 */ 255 static inline int iowait_pio_pending(struct iowait *wait) 256 { 257 return atomic_read(&wait->pio_busy); 258 } 259 260 /** 261 * iowait_pio_inc - note pio pending 262 * @wait: iowait structure 263 */ 264 static inline void iowait_pio_inc(struct iowait *wait) 265 { 266 atomic_inc(&wait->pio_busy); 267 } 268 269 /** 270 * iowait_sdma_dec - note pio complete 271 * @wait: iowait structure 272 */ 273 static inline int iowait_pio_dec(struct iowait *wait) 274 { 275 return atomic_dec_and_test(&wait->pio_busy); 276 } 277 278 /** 279 * iowait_drain_wakeup() - trigger iowait_drain() waiter 280 * 281 * @wait: iowait structure 282 * 283 * This will trigger any waiters. 284 */ 285 static inline void iowait_drain_wakeup(struct iowait *wait) 286 { 287 wake_up(&wait->wait_dma); 288 wake_up(&wait->wait_pio); 289 if (wait->sdma_drained) 290 wait->sdma_drained(wait); 291 } 292 293 /** 294 * iowait_get_txhead() - get packet off of iowait list 295 * 296 * @wait wait struture 297 */ 298 static inline struct sdma_txreq *iowait_get_txhead(struct iowait *wait) 299 { 300 struct sdma_txreq *tx = NULL; 301 302 if (!list_empty(&wait->tx_head)) { 303 tx = list_first_entry( 304 &wait->tx_head, 305 struct sdma_txreq, 306 list); 307 list_del_init(&tx->list); 308 } 309 return tx; 310 } 311 312 /** 313 * iowait_queue - Put the iowait on a wait queue 314 * @pkts_sent: have some packets been sent before queuing? 315 * @w: the iowait struct 316 * @wait_head: the wait queue 317 * 318 * This function is called to insert an iowait struct into a 319 * wait queue after a resource (eg, sdma decriptor or pio 320 * buffer) is run out. 321 */ 322 static inline void iowait_queue(bool pkts_sent, struct iowait *w, 323 struct list_head *wait_head) 324 { 325 /* 326 * To play fair, insert the iowait at the tail of the wait queue if it 327 * has already sent some packets; Otherwise, put it at the head. 328 */ 329 if (pkts_sent) { 330 list_add_tail(&w->list, wait_head); 331 w->starved_cnt = 0; 332 } else { 333 list_add(&w->list, wait_head); 334 w->starved_cnt++; 335 } 336 } 337 338 /** 339 * iowait_starve_clear - clear the wait queue's starve count 340 * @pkts_sent: have some packets been sent? 341 * @w: the iowait struct 342 * 343 * This function is called to clear the starve count. If no 344 * packets have been sent, the starve count will not be cleared. 345 */ 346 static inline void iowait_starve_clear(bool pkts_sent, struct iowait *w) 347 { 348 if (pkts_sent) 349 w->starved_cnt = 0; 350 } 351 352 /** 353 * iowait_starve_find_max - Find the maximum of the starve count 354 * @w: the iowait struct 355 * @max: a variable containing the max starve count 356 * @idx: the index of the current iowait in an array 357 * @max_idx: a variable containing the array index for the 358 * iowait entry that has the max starve count 359 * 360 * This function is called to compare the starve count of a 361 * given iowait with the given max starve count. The max starve 362 * count and the index will be updated if the iowait's start 363 * count is larger. 364 */ 365 static inline void iowait_starve_find_max(struct iowait *w, u8 *max, 366 uint idx, uint *max_idx) 367 { 368 if (w->starved_cnt > *max) { 369 *max = w->starved_cnt; 370 *max_idx = idx; 371 } 372 } 373 374 #endif 375