xref: /linux/drivers/accel/qaic/qaic_drv.c (revision 8dd765a5d769c521d73931850d1c8708fbc490cb) 
1 // SPDX-License-Identifier: GPL-2.0-only
2 
3 /* Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. */
4 /* Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */
5 
6 #include <linux/delay.h>
7 #include <linux/dma-mapping.h>
8 #include <linux/idr.h>
9 #include <linux/interrupt.h>
10 #include <linux/list.h>
11 #include <linux/kref.h>
12 #include <linux/mhi.h>
13 #include <linux/module.h>
14 #include <linux/msi.h>
15 #include <linux/mutex.h>
16 #include <linux/pci.h>
17 #include <linux/spinlock.h>
18 #include <linux/workqueue.h>
19 #include <linux/wait.h>
20 #include <drm/drm_accel.h>
21 #include <drm/drm_drv.h>
22 #include <drm/drm_file.h>
23 #include <drm/drm_gem.h>
24 #include <drm/drm_ioctl.h>
25 #include <uapi/drm/qaic_accel.h>
26 
27 #include "mhi_controller.h"
28 #include "qaic.h"
29 
30 MODULE_IMPORT_NS(DMA_BUF);
31 
32 #define PCI_DEV_AIC100			0xa100
33 #define QAIC_NAME			"qaic"
34 #define QAIC_DESC			"Qualcomm Cloud AI Accelerators"
35 #define CNTL_MAJOR			5
36 #define CNTL_MINOR			0
37 
38 bool datapath_polling;
39 module_param(datapath_polling, bool, 0400);
40 MODULE_PARM_DESC(datapath_polling, "Operate the datapath in polling mode");
41 static bool link_up;
42 static DEFINE_IDA(qaic_usrs);
43 
44 static int qaic_create_drm_device(struct qaic_device *qdev, s32 partition_id);
45 static void qaic_destroy_drm_device(struct qaic_device *qdev, s32 partition_id);
46 
47 static void free_usr(struct kref *kref)
48 {
49 	struct qaic_user *usr = container_of(kref, struct qaic_user, ref_count);
50 
51 	cleanup_srcu_struct(&usr->qddev_lock);
52 	ida_free(&qaic_usrs, usr->handle);
53 	kfree(usr);
54 }
55 
56 static int qaic_open(struct drm_device *dev, struct drm_file *file)
57 {
58 	struct qaic_drm_device *qddev = dev->dev_private;
59 	struct qaic_device *qdev = qddev->qdev;
60 	struct qaic_user *usr;
61 	int rcu_id;
62 	int ret;
63 
64 	rcu_id = srcu_read_lock(&qdev->dev_lock);
65 	if (qdev->in_reset) {
66 		ret = -ENODEV;
67 		goto dev_unlock;
68 	}
69 
70 	usr = kmalloc(sizeof(*usr), GFP_KERNEL);
71 	if (!usr) {
72 		ret = -ENOMEM;
73 		goto dev_unlock;
74 	}
75 
76 	usr->handle = ida_alloc(&qaic_usrs, GFP_KERNEL);
77 	if (usr->handle < 0) {
78 		ret = usr->handle;
79 		goto free_usr;
80 	}
81 	usr->qddev = qddev;
82 	atomic_set(&usr->chunk_id, 0);
83 	init_srcu_struct(&usr->qddev_lock);
84 	kref_init(&usr->ref_count);
85 
86 	ret = mutex_lock_interruptible(&qddev->users_mutex);
87 	if (ret)
88 		goto cleanup_usr;
89 
90 	list_add(&usr->node, &qddev->users);
91 	mutex_unlock(&qddev->users_mutex);
92 
93 	file->driver_priv = usr;
94 
95 	srcu_read_unlock(&qdev->dev_lock, rcu_id);
96 	return 0;
97 
98 cleanup_usr:
99 	cleanup_srcu_struct(&usr->qddev_lock);
100 	ida_free(&qaic_usrs, usr->handle);
101 free_usr:
102 	kfree(usr);
103 dev_unlock:
104 	srcu_read_unlock(&qdev->dev_lock, rcu_id);
105 	return ret;
106 }
107 
108 static void qaic_postclose(struct drm_device *dev, struct drm_file *file)
109 {
110 	struct qaic_user *usr = file->driver_priv;
111 	struct qaic_drm_device *qddev;
112 	struct qaic_device *qdev;
113 	int qdev_rcu_id;
114 	int usr_rcu_id;
115 	int i;
116 
117 	qddev = usr->qddev;
118 	usr_rcu_id = srcu_read_lock(&usr->qddev_lock);
119 	if (qddev) {
120 		qdev = qddev->qdev;
121 		qdev_rcu_id = srcu_read_lock(&qdev->dev_lock);
122 		if (!qdev->in_reset) {
123 			qaic_release_usr(qdev, usr);
124 			for (i = 0; i < qdev->num_dbc; ++i)
125 				if (qdev->dbc[i].usr && qdev->dbc[i].usr->handle == usr->handle)
126 					release_dbc(qdev, i);
127 		}
128 		srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);
129 
130 		mutex_lock(&qddev->users_mutex);
131 		if (!list_empty(&usr->node))
132 			list_del_init(&usr->node);
133 		mutex_unlock(&qddev->users_mutex);
134 	}
135 
136 	srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);
137 	kref_put(&usr->ref_count, free_usr);
138 
139 	file->driver_priv = NULL;
140 }
141 
142 DEFINE_DRM_ACCEL_FOPS(qaic_accel_fops);
143 
144 static const struct drm_ioctl_desc qaic_drm_ioctls[] = {
145 	DRM_IOCTL_DEF_DRV(QAIC_MANAGE, qaic_manage_ioctl, 0),
146 	DRM_IOCTL_DEF_DRV(QAIC_CREATE_BO, qaic_create_bo_ioctl, 0),
147 	DRM_IOCTL_DEF_DRV(QAIC_MMAP_BO, qaic_mmap_bo_ioctl, 0),
148 	DRM_IOCTL_DEF_DRV(QAIC_ATTACH_SLICE_BO, qaic_attach_slice_bo_ioctl, 0),
149 	DRM_IOCTL_DEF_DRV(QAIC_EXECUTE_BO, qaic_execute_bo_ioctl, 0),
150 	DRM_IOCTL_DEF_DRV(QAIC_PARTIAL_EXECUTE_BO, qaic_partial_execute_bo_ioctl, 0),
151 	DRM_IOCTL_DEF_DRV(QAIC_WAIT_BO, qaic_wait_bo_ioctl, 0),
152 	DRM_IOCTL_DEF_DRV(QAIC_PERF_STATS_BO, qaic_perf_stats_bo_ioctl, 0),
153 };
154 
155 static const struct drm_driver qaic_accel_driver = {
156 	.driver_features	= DRIVER_GEM | DRIVER_COMPUTE_ACCEL,
157 
158 	.name			= QAIC_NAME,
159 	.desc			= QAIC_DESC,
160 	.date			= "20190618",
161 
162 	.fops			= &qaic_accel_fops,
163 	.open			= qaic_open,
164 	.postclose		= qaic_postclose,
165 
166 	.ioctls			= qaic_drm_ioctls,
167 	.num_ioctls		= ARRAY_SIZE(qaic_drm_ioctls),
168 	.gem_prime_import	= qaic_gem_prime_import,
169 };
170 
171 static int qaic_create_drm_device(struct qaic_device *qdev, s32 partition_id)
172 {
173 	struct qaic_drm_device *qddev;
174 	struct drm_device *ddev;
175 	struct device *pdev;
176 	int ret;
177 
178 	/* Hold off implementing partitions until the uapi is determined */
179 	if (partition_id != QAIC_NO_PARTITION)
180 		return -EINVAL;
181 
182 	pdev = &qdev->pdev->dev;
183 
184 	qddev = kzalloc(sizeof(*qddev), GFP_KERNEL);
185 	if (!qddev)
186 		return -ENOMEM;
187 
188 	ddev = drm_dev_alloc(&qaic_accel_driver, pdev);
189 	if (IS_ERR(ddev)) {
190 		ret = PTR_ERR(ddev);
191 		goto ddev_fail;
192 	}
193 
194 	ddev->dev_private = qddev;
195 	qddev->ddev = ddev;
196 
197 	qddev->qdev = qdev;
198 	qddev->partition_id = partition_id;
199 	INIT_LIST_HEAD(&qddev->users);
200 	mutex_init(&qddev->users_mutex);
201 
202 	qdev->qddev = qddev;
203 
204 	ret = drm_dev_register(ddev, 0);
205 	if (ret) {
206 		pci_dbg(qdev->pdev, "%s: drm_dev_register failed %d\n", __func__, ret);
207 		goto drm_reg_fail;
208 	}
209 
210 	return 0;
211 
212 drm_reg_fail:
213 	mutex_destroy(&qddev->users_mutex);
214 	qdev->qddev = NULL;
215 	drm_dev_put(ddev);
216 ddev_fail:
217 	kfree(qddev);
218 	return ret;
219 }
220 
221 static void qaic_destroy_drm_device(struct qaic_device *qdev, s32 partition_id)
222 {
223 	struct qaic_drm_device *qddev;
224 	struct qaic_user *usr;
225 
226 	qddev = qdev->qddev;
227 	qdev->qddev = NULL;
228 	if (!qddev)
229 		return;
230 
231 	/*
232 	 * Existing users get unresolvable errors till they close FDs.
233 	 * Need to sync carefully with users calling close(). The
234 	 * list of users can be modified elsewhere when the lock isn't
235 	 * held here, but the sync'ing the srcu with the mutex held
236 	 * could deadlock. Grab the mutex so that the list will be
237 	 * unmodified. The user we get will exist as long as the
238 	 * lock is held. Signal that the qcdev is going away, and
239 	 * grab a reference to the user so they don't go away for
240 	 * synchronize_srcu(). Then release the mutex to avoid
241 	 * deadlock and make sure the user has observed the signal.
242 	 * With the lock released, we cannot maintain any state of the
243 	 * user list.
244 	 */
245 	mutex_lock(&qddev->users_mutex);
246 	while (!list_empty(&qddev->users)) {
247 		usr = list_first_entry(&qddev->users, struct qaic_user, node);
248 		list_del_init(&usr->node);
249 		kref_get(&usr->ref_count);
250 		usr->qddev = NULL;
251 		mutex_unlock(&qddev->users_mutex);
252 		synchronize_srcu(&usr->qddev_lock);
253 		kref_put(&usr->ref_count, free_usr);
254 		mutex_lock(&qddev->users_mutex);
255 	}
256 	mutex_unlock(&qddev->users_mutex);
257 
258 	if (qddev->ddev) {
259 		drm_dev_unregister(qddev->ddev);
260 		drm_dev_put(qddev->ddev);
261 	}
262 
263 	kfree(qddev);
264 }
265 
266 static int qaic_mhi_probe(struct mhi_device *mhi_dev, const struct mhi_device_id *id)
267 {
268 	u16 major = -1, minor = -1;
269 	struct qaic_device *qdev;
270 	int ret;
271 
272 	/*
273 	 * Invoking this function indicates that the control channel to the
274 	 * device is available. We use that as a signal to indicate that
275 	 * the device side firmware has booted. The device side firmware
276 	 * manages the device resources, so we need to communicate with it
277 	 * via the control channel in order to utilize the device. Therefore
278 	 * we wait until this signal to create the drm dev that userspace will
279 	 * use to control the device, because without the device side firmware,
280 	 * userspace can't do anything useful.
281 	 */
282 
283 	qdev = pci_get_drvdata(to_pci_dev(mhi_dev->mhi_cntrl->cntrl_dev));
284 
285 	qdev->in_reset = false;
286 
287 	dev_set_drvdata(&mhi_dev->dev, qdev);
288 	qdev->cntl_ch = mhi_dev;
289 
290 	ret = qaic_control_open(qdev);
291 	if (ret) {
292 		pci_dbg(qdev->pdev, "%s: control_open failed %d\n", __func__, ret);
293 		return ret;
294 	}
295 
296 	ret = get_cntl_version(qdev, NULL, &major, &minor);
297 	if (ret || major != CNTL_MAJOR || minor > CNTL_MINOR) {
298 		pci_err(qdev->pdev, "%s: Control protocol version (%d.%d) not supported. Supported version is (%d.%d). Ret: %d\n",
299 			__func__, major, minor, CNTL_MAJOR, CNTL_MINOR, ret);
300 		ret = -EINVAL;
301 		goto close_control;
302 	}
303 
304 	ret = qaic_create_drm_device(qdev, QAIC_NO_PARTITION);
305 
306 	return ret;
307 
308 close_control:
309 	qaic_control_close(qdev);
310 	return ret;
311 }
312 
313 static void qaic_mhi_remove(struct mhi_device *mhi_dev)
314 {
315 /* This is redundant since we have already observed the device crash */
316 }
317 
318 static void qaic_notify_reset(struct qaic_device *qdev)
319 {
320 	int i;
321 
322 	qdev->in_reset = true;
323 	/* wake up any waiters to avoid waiting for timeouts at sync */
324 	wake_all_cntl(qdev);
325 	for (i = 0; i < qdev->num_dbc; ++i)
326 		wakeup_dbc(qdev, i);
327 	synchronize_srcu(&qdev->dev_lock);
328 }
329 
330 void qaic_dev_reset_clean_local_state(struct qaic_device *qdev, bool exit_reset)
331 {
332 	int i;
333 
334 	qaic_notify_reset(qdev);
335 
336 	/* remove drmdevs to prevent new users from coming in */
337 	qaic_destroy_drm_device(qdev, QAIC_NO_PARTITION);
338 
339 	/* start tearing things down */
340 	for (i = 0; i < qdev->num_dbc; ++i)
341 		release_dbc(qdev, i);
342 
343 	if (exit_reset)
344 		qdev->in_reset = false;
345 }
346 
347 static struct qaic_device *create_qdev(struct pci_dev *pdev, const struct pci_device_id *id)
348 {
349 	struct qaic_device *qdev;
350 	int i;
351 
352 	qdev = devm_kzalloc(&pdev->dev, sizeof(*qdev), GFP_KERNEL);
353 	if (!qdev)
354 		return NULL;
355 
356 	if (id->device == PCI_DEV_AIC100) {
357 		qdev->num_dbc = 16;
358 		qdev->dbc = devm_kcalloc(&pdev->dev, qdev->num_dbc, sizeof(*qdev->dbc), GFP_KERNEL);
359 		if (!qdev->dbc)
360 			return NULL;
361 	}
362 
363 	qdev->cntl_wq = alloc_workqueue("qaic_cntl", WQ_UNBOUND, 0);
364 	if (!qdev->cntl_wq)
365 		return NULL;
366 
367 	pci_set_drvdata(pdev, qdev);
368 	qdev->pdev = pdev;
369 
370 	mutex_init(&qdev->cntl_mutex);
371 	INIT_LIST_HEAD(&qdev->cntl_xfer_list);
372 	init_srcu_struct(&qdev->dev_lock);
373 
374 	for (i = 0; i < qdev->num_dbc; ++i) {
375 		spin_lock_init(&qdev->dbc[i].xfer_lock);
376 		qdev->dbc[i].qdev = qdev;
377 		qdev->dbc[i].id = i;
378 		INIT_LIST_HEAD(&qdev->dbc[i].xfer_list);
379 		init_srcu_struct(&qdev->dbc[i].ch_lock);
380 		init_waitqueue_head(&qdev->dbc[i].dbc_release);
381 		INIT_LIST_HEAD(&qdev->dbc[i].bo_lists);
382 	}
383 
384 	return qdev;
385 }
386 
387 static void cleanup_qdev(struct qaic_device *qdev)
388 {
389 	int i;
390 
391 	for (i = 0; i < qdev->num_dbc; ++i)
392 		cleanup_srcu_struct(&qdev->dbc[i].ch_lock);
393 	cleanup_srcu_struct(&qdev->dev_lock);
394 	pci_set_drvdata(qdev->pdev, NULL);
395 	destroy_workqueue(qdev->cntl_wq);
396 }
397 
398 static int init_pci(struct qaic_device *qdev, struct pci_dev *pdev)
399 {
400 	int bars;
401 	int ret;
402 
403 	bars = pci_select_bars(pdev, IORESOURCE_MEM);
404 
405 	/* make sure the device has the expected BARs */
406 	if (bars != (BIT(0) | BIT(2) | BIT(4))) {
407 		pci_dbg(pdev, "%s: expected BARs 0, 2, and 4 not found in device. Found 0x%x\n",
408 			__func__, bars);
409 		return -EINVAL;
410 	}
411 
412 	ret = pcim_enable_device(pdev);
413 	if (ret)
414 		return ret;
415 
416 	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
417 	if (ret)
418 		return ret;
419 	ret = dma_set_max_seg_size(&pdev->dev, UINT_MAX);
420 	if (ret)
421 		return ret;
422 
423 	qdev->bar_0 = devm_ioremap_resource(&pdev->dev, &pdev->resource[0]);
424 	if (IS_ERR(qdev->bar_0))
425 		return PTR_ERR(qdev->bar_0);
426 
427 	qdev->bar_2 = devm_ioremap_resource(&pdev->dev, &pdev->resource[2]);
428 	if (IS_ERR(qdev->bar_2))
429 		return PTR_ERR(qdev->bar_2);
430 
431 	/* Managed release since we use pcim_enable_device above */
432 	pci_set_master(pdev);
433 
434 	return 0;
435 }
436 
437 static int init_msi(struct qaic_device *qdev, struct pci_dev *pdev)
438 {
439 	int mhi_irq;
440 	int ret;
441 	int i;
442 
443 	/* Managed release since we use pcim_enable_device */
444 	ret = pci_alloc_irq_vectors(pdev, 1, 32, PCI_IRQ_MSI);
445 	if (ret < 0)
446 		return ret;
447 
448 	if (ret < 32) {
449 		pci_err(pdev, "%s: Requested 32 MSIs. Obtained %d MSIs which is less than the 32 required.\n",
450 			__func__, ret);
451 		return -ENODEV;
452 	}
453 
454 	mhi_irq = pci_irq_vector(pdev, 0);
455 	if (mhi_irq < 0)
456 		return mhi_irq;
457 
458 	for (i = 0; i < qdev->num_dbc; ++i) {
459 		ret = devm_request_threaded_irq(&pdev->dev, pci_irq_vector(pdev, i + 1),
460 						dbc_irq_handler, dbc_irq_threaded_fn, IRQF_SHARED,
461 						"qaic_dbc", &qdev->dbc[i]);
462 		if (ret)
463 			return ret;
464 
465 		if (datapath_polling) {
466 			qdev->dbc[i].irq = pci_irq_vector(pdev, i + 1);
467 			disable_irq_nosync(qdev->dbc[i].irq);
468 			INIT_WORK(&qdev->dbc[i].poll_work, irq_polling_work);
469 		}
470 	}
471 
472 	return mhi_irq;
473 }
474 
475 static int qaic_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
476 {
477 	struct qaic_device *qdev;
478 	int mhi_irq;
479 	int ret;
480 	int i;
481 
482 	qdev = create_qdev(pdev, id);
483 	if (!qdev)
484 		return -ENOMEM;
485 
486 	ret = init_pci(qdev, pdev);
487 	if (ret)
488 		goto cleanup_qdev;
489 
490 	for (i = 0; i < qdev->num_dbc; ++i)
491 		qdev->dbc[i].dbc_base = qdev->bar_2 + QAIC_DBC_OFF(i);
492 
493 	mhi_irq = init_msi(qdev, pdev);
494 	if (mhi_irq < 0) {
495 		ret = mhi_irq;
496 		goto cleanup_qdev;
497 	}
498 
499 	qdev->mhi_cntrl = qaic_mhi_register_controller(pdev, qdev->bar_0, mhi_irq);
500 	if (IS_ERR(qdev->mhi_cntrl)) {
501 		ret = PTR_ERR(qdev->mhi_cntrl);
502 		goto cleanup_qdev;
503 	}
504 
505 	return 0;
506 
507 cleanup_qdev:
508 	cleanup_qdev(qdev);
509 	return ret;
510 }
511 
512 static void qaic_pci_remove(struct pci_dev *pdev)
513 {
514 	struct qaic_device *qdev = pci_get_drvdata(pdev);
515 
516 	if (!qdev)
517 		return;
518 
519 	qaic_dev_reset_clean_local_state(qdev, false);
520 	qaic_mhi_free_controller(qdev->mhi_cntrl, link_up);
521 	cleanup_qdev(qdev);
522 }
523 
524 static void qaic_pci_shutdown(struct pci_dev *pdev)
525 {
526 	/* see qaic_exit for what link_up is doing */
527 	link_up = true;
528 	qaic_pci_remove(pdev);
529 }
530 
531 static pci_ers_result_t qaic_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t error)
532 {
533 	return PCI_ERS_RESULT_NEED_RESET;
534 }
535 
536 static void qaic_pci_reset_prepare(struct pci_dev *pdev)
537 {
538 	struct qaic_device *qdev = pci_get_drvdata(pdev);
539 
540 	qaic_notify_reset(qdev);
541 	qaic_mhi_start_reset(qdev->mhi_cntrl);
542 	qaic_dev_reset_clean_local_state(qdev, false);
543 }
544 
545 static void qaic_pci_reset_done(struct pci_dev *pdev)
546 {
547 	struct qaic_device *qdev = pci_get_drvdata(pdev);
548 
549 	qdev->in_reset = false;
550 	qaic_mhi_reset_done(qdev->mhi_cntrl);
551 }
552 
553 static const struct mhi_device_id qaic_mhi_match_table[] = {
554 	{ .chan = "QAIC_CONTROL", },
555 	{},
556 };
557 
558 static struct mhi_driver qaic_mhi_driver = {
559 	.id_table = qaic_mhi_match_table,
560 	.remove = qaic_mhi_remove,
561 	.probe = qaic_mhi_probe,
562 	.ul_xfer_cb = qaic_mhi_ul_xfer_cb,
563 	.dl_xfer_cb = qaic_mhi_dl_xfer_cb,
564 	.driver = {
565 		.name = "qaic_mhi",
566 	},
567 };
568 
569 static const struct pci_device_id qaic_ids[] = {
570 	{ PCI_DEVICE(PCI_VENDOR_ID_QCOM, PCI_DEV_AIC100), },
571 	{ }
572 };
573 MODULE_DEVICE_TABLE(pci, qaic_ids);
574 
575 static const struct pci_error_handlers qaic_pci_err_handler = {
576 	.error_detected = qaic_pci_error_detected,
577 	.reset_prepare = qaic_pci_reset_prepare,
578 	.reset_done = qaic_pci_reset_done,
579 };
580 
581 static struct pci_driver qaic_pci_driver = {
582 	.name = QAIC_NAME,
583 	.id_table = qaic_ids,
584 	.probe = qaic_pci_probe,
585 	.remove = qaic_pci_remove,
586 	.shutdown = qaic_pci_shutdown,
587 	.err_handler = &qaic_pci_err_handler,
588 };
589 
590 static int __init qaic_init(void)
591 {
592 	int ret;
593 
594 	ret = mhi_driver_register(&qaic_mhi_driver);
595 	if (ret) {
596 		pr_debug("qaic: mhi_driver_register failed %d\n", ret);
597 		return ret;
598 	}
599 
600 	ret = pci_register_driver(&qaic_pci_driver);
601 	if (ret) {
602 		pr_debug("qaic: pci_register_driver failed %d\n", ret);
603 		goto free_mhi;
604 	}
605 
606 	return 0;
607 
608 free_mhi:
609 	mhi_driver_unregister(&qaic_mhi_driver);
610 	return ret;
611 }
612 
613 static void __exit qaic_exit(void)
614 {
615 	/*
616 	 * We assume that qaic_pci_remove() is called due to a hotplug event
617 	 * which would mean that the link is down, and thus
618 	 * qaic_mhi_free_controller() should not try to access the device during
619 	 * cleanup.
620 	 * We call pci_unregister_driver() below, which also triggers
621 	 * qaic_pci_remove(), but since this is module exit, we expect the link
622 	 * to the device to be up, in which case qaic_mhi_free_controller()
623 	 * should try to access the device during cleanup to put the device in
624 	 * a sane state.
625 	 * For that reason, we set link_up here to let qaic_mhi_free_controller
626 	 * know the expected link state. Since the module is going to be
627 	 * removed at the end of this, we don't need to worry about
628 	 * reinitializing the link_up state after the cleanup is done.
629 	 */
630 	link_up = true;
631 	pci_unregister_driver(&qaic_pci_driver);
632 	mhi_driver_unregister(&qaic_mhi_driver);
633 }
634 
635 module_init(qaic_init);
636 module_exit(qaic_exit);
637 
638 MODULE_AUTHOR(QAIC_DESC " Kernel Driver Team");
639 MODULE_DESCRIPTION(QAIC_DESC " Accel Driver");
640 MODULE_LICENSE("GPL");
641