1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2018 Intel Corporation 5 */ 6 7 #include <linux/kref.h> 8 #include <linux/string_helpers.h> 9 10 #include "gem/i915_gem_pm.h" 11 #include "gt/intel_gt.h" 12 13 #include "i915_selftest.h" 14 15 #include "igt_flush_test.h" 16 #include "lib_sw_fence.h" 17 18 struct live_active { 19 struct i915_active base; 20 struct kref ref; 21 bool retired; 22 }; 23 24 static void __live_get(struct live_active *active) 25 { 26 kref_get(&active->ref); 27 } 28 29 static void __live_free(struct live_active *active) 30 { 31 i915_active_fini(&active->base); 32 kfree(active); 33 } 34 35 static void __live_release(struct kref *ref) 36 { 37 struct live_active *active = container_of(ref, typeof(*active), ref); 38 39 __live_free(active); 40 } 41 42 static void __live_put(struct live_active *active) 43 { 44 kref_put(&active->ref, __live_release); 45 } 46 47 static int __live_active(struct i915_active *base) 48 { 49 struct live_active *active = container_of(base, typeof(*active), base); 50 51 __live_get(active); 52 return 0; 53 } 54 55 static void __live_retire(struct i915_active *base) 56 { 57 struct live_active *active = container_of(base, typeof(*active), base); 58 59 active->retired = true; 60 __live_put(active); 61 } 62 63 static struct live_active *__live_alloc(struct drm_i915_private *i915) 64 { 65 struct live_active *active; 66 67 active = kzalloc(sizeof(*active), GFP_KERNEL); 68 if (!active) 69 return NULL; 70 71 kref_init(&active->ref); 72 i915_active_init(&active->base, __live_active, __live_retire, 0); 73 74 return active; 75 } 76 77 static struct live_active * 78 __live_active_setup(struct drm_i915_private *i915) 79 { 80 struct intel_engine_cs *engine; 81 struct i915_sw_fence *submit; 82 struct live_active *active; 83 unsigned int count = 0; 84 int err = 0; 85 86 active = __live_alloc(i915); 87 if (!active) 88 return ERR_PTR(-ENOMEM); 89 90 submit = heap_fence_create(GFP_KERNEL); 91 if (!submit) { 92 kfree(active); 93 return ERR_PTR(-ENOMEM); 94 } 95 96 err = i915_active_acquire(&active->base); 97 if (err) 98 goto out; 99 100 for_each_uabi_engine(engine, i915) { 101 struct i915_request *rq; 102 103 rq = intel_engine_create_kernel_request(engine); 104 if (IS_ERR(rq)) { 105 err = PTR_ERR(rq); 106 break; 107 } 108 109 err = i915_sw_fence_await_sw_fence_gfp(&rq->submit, 110 submit, 111 GFP_KERNEL); 112 if (err >= 0) 113 err = i915_active_add_request(&active->base, rq); 114 i915_request_add(rq); 115 if (err) { 116 pr_err("Failed to track active ref!\n"); 117 break; 118 } 119 120 count++; 121 } 122 123 i915_active_release(&active->base); 124 if (READ_ONCE(active->retired) && count) { 125 pr_err("i915_active retired before submission!\n"); 126 err = -EINVAL; 127 } 128 if (atomic_read(&active->base.count) != count) { 129 pr_err("i915_active not tracking all requests, found %d, expected %d\n", 130 atomic_read(&active->base.count), count); 131 err = -EINVAL; 132 } 133 134 out: 135 i915_sw_fence_commit(submit); 136 heap_fence_put(submit); 137 if (err) { 138 __live_put(active); 139 active = ERR_PTR(err); 140 } 141 142 return active; 143 } 144 145 static int live_active_wait(void *arg) 146 { 147 struct drm_i915_private *i915 = arg; 148 struct live_active *active; 149 int err = 0; 150 151 /* Check that we get a callback when requests retire upon waiting */ 152 153 active = __live_active_setup(i915); 154 if (IS_ERR(active)) 155 return PTR_ERR(active); 156 157 __i915_active_wait(&active->base, TASK_UNINTERRUPTIBLE); 158 if (!READ_ONCE(active->retired)) { 159 struct drm_printer p = drm_err_printer(__func__); 160 161 pr_err("i915_active not retired after waiting!\n"); 162 i915_active_print(&active->base, &p); 163 164 err = -EINVAL; 165 } 166 167 __live_put(active); 168 169 if (igt_flush_test(i915)) 170 err = -EIO; 171 172 return err; 173 } 174 175 static int live_active_retire(void *arg) 176 { 177 struct drm_i915_private *i915 = arg; 178 struct live_active *active; 179 int err = 0; 180 181 /* Check that we get a callback when requests are indirectly retired */ 182 183 active = __live_active_setup(i915); 184 if (IS_ERR(active)) 185 return PTR_ERR(active); 186 187 /* waits for & retires all requests */ 188 if (igt_flush_test(i915)) 189 err = -EIO; 190 191 if (!READ_ONCE(active->retired)) { 192 struct drm_printer p = drm_err_printer(__func__); 193 194 pr_err("i915_active not retired after flushing!\n"); 195 i915_active_print(&active->base, &p); 196 197 err = -EINVAL; 198 } 199 200 __live_put(active); 201 202 return err; 203 } 204 205 static int live_active_barrier(void *arg) 206 { 207 struct drm_i915_private *i915 = arg; 208 struct intel_engine_cs *engine; 209 struct live_active *active; 210 int err = 0; 211 212 /* Check that we get a callback when requests retire upon waiting */ 213 214 active = __live_alloc(i915); 215 if (!active) 216 return -ENOMEM; 217 218 err = i915_active_acquire(&active->base); 219 if (err) 220 goto out; 221 222 for_each_uabi_engine(engine, i915) { 223 err = i915_active_acquire_preallocate_barrier(&active->base, 224 engine); 225 if (err) 226 break; 227 228 i915_active_acquire_barrier(&active->base); 229 } 230 231 i915_active_release(&active->base); 232 if (err) 233 goto out; 234 235 __i915_active_wait(&active->base, TASK_UNINTERRUPTIBLE); 236 if (!READ_ONCE(active->retired)) { 237 pr_err("i915_active not retired after flushing barriers!\n"); 238 err = -EINVAL; 239 } 240 241 out: 242 __live_put(active); 243 244 if (igt_flush_test(i915)) 245 err = -EIO; 246 247 return err; 248 } 249 250 int i915_active_live_selftests(struct drm_i915_private *i915) 251 { 252 static const struct i915_subtest tests[] = { 253 SUBTEST(live_active_wait), 254 SUBTEST(live_active_retire), 255 SUBTEST(live_active_barrier), 256 }; 257 258 if (intel_gt_is_wedged(to_gt(i915))) 259 return 0; 260 261 return i915_subtests(tests, i915); 262 } 263 264 static struct intel_engine_cs *node_to_barrier(struct active_node *it) 265 { 266 struct intel_engine_cs *engine; 267 268 if (!is_barrier(&it->base)) 269 return NULL; 270 271 engine = __barrier_to_engine(it); 272 smp_rmb(); /* serialise with add_active_barriers */ 273 if (!is_barrier(&it->base)) 274 return NULL; 275 276 return engine; 277 } 278 279 void i915_active_print(struct i915_active *ref, struct drm_printer *m) 280 { 281 drm_printf(m, "active %ps:%ps\n", ref->active, ref->retire); 282 drm_printf(m, "\tcount: %d\n", atomic_read(&ref->count)); 283 drm_printf(m, "\tpreallocated barriers? %s\n", 284 str_yes_no(!llist_empty(&ref->preallocated_barriers))); 285 286 if (i915_active_acquire_if_busy(ref)) { 287 struct active_node *it, *n; 288 289 rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) { 290 struct intel_engine_cs *engine; 291 292 engine = node_to_barrier(it); 293 if (engine) { 294 drm_printf(m, "\tbarrier: %s\n", engine->name); 295 continue; 296 } 297 298 if (i915_active_fence_isset(&it->base)) { 299 drm_printf(m, 300 "\ttimeline: %llx\n", it->timeline); 301 continue; 302 } 303 } 304 305 i915_active_release(ref); 306 } 307 } 308 309 static void spin_unlock_wait(spinlock_t *lock) 310 { 311 spin_lock_irq(lock); 312 spin_unlock_irq(lock); 313 } 314 315 static void active_flush(struct i915_active *ref, 316 struct i915_active_fence *active) 317 { 318 struct dma_fence *fence; 319 320 fence = xchg(__active_fence_slot(active), NULL); 321 if (!fence) 322 return; 323 324 spin_lock_irq(fence->lock); 325 __list_del_entry(&active->cb.node); 326 spin_unlock_irq(fence->lock); /* serialise with fence->cb_list */ 327 atomic_dec(&ref->count); 328 329 GEM_BUG_ON(!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)); 330 } 331 332 void i915_active_unlock_wait(struct i915_active *ref) 333 { 334 if (i915_active_acquire_if_busy(ref)) { 335 struct active_node *it, *n; 336 337 /* Wait for all active callbacks */ 338 rcu_read_lock(); 339 active_flush(ref, &ref->excl); 340 rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) 341 active_flush(ref, &it->base); 342 rcu_read_unlock(); 343 344 i915_active_release(ref); 345 } 346 347 /* And wait for the retire callback */ 348 spin_unlock_wait(&ref->tree_lock); 349 350 /* ... which may have been on a thread instead */ 351 flush_work(&ref->work); 352 } 353