1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Support KVM gust page tracking 4 * 5 * This feature allows us to track page access in guest. Currently, only 6 * write access is tracked. 7 * 8 * Copyright(C) 2015 Intel Corporation. 9 * 10 * Author: 11 * Xiao Guangrong <guangrong.xiao@linux.intel.com> 12 */ 13 14 #include <linux/kvm_host.h> 15 #include <linux/rculist.h> 16 17 #include <asm/kvm_page_track.h> 18 19 #include "mmu_internal.h" 20 21 void kvm_page_track_free_memslot(struct kvm_memory_slot *slot) 22 { 23 int i; 24 25 for (i = 0; i < KVM_PAGE_TRACK_MAX; i++) { 26 kvfree(slot->arch.gfn_track[i]); 27 slot->arch.gfn_track[i] = NULL; 28 } 29 } 30 31 int kvm_page_track_create_memslot(struct kvm_memory_slot *slot, 32 unsigned long npages) 33 { 34 int i; 35 36 for (i = 0; i < KVM_PAGE_TRACK_MAX; i++) { 37 slot->arch.gfn_track[i] = 38 kvcalloc(npages, sizeof(*slot->arch.gfn_track[i]), 39 GFP_KERNEL_ACCOUNT); 40 if (!slot->arch.gfn_track[i]) 41 goto track_free; 42 } 43 44 return 0; 45 46 track_free: 47 kvm_page_track_free_memslot(slot); 48 return -ENOMEM; 49 } 50 51 static inline bool page_track_mode_is_valid(enum kvm_page_track_mode mode) 52 { 53 if (mode < 0 || mode >= KVM_PAGE_TRACK_MAX) 54 return false; 55 56 return true; 57 } 58 59 static void update_gfn_track(struct kvm_memory_slot *slot, gfn_t gfn, 60 enum kvm_page_track_mode mode, short count) 61 { 62 int index, val; 63 64 index = gfn_to_index(gfn, slot->base_gfn, PG_LEVEL_4K); 65 66 val = slot->arch.gfn_track[mode][index]; 67 68 if (WARN_ON(val + count < 0 || val + count > USHRT_MAX)) 69 return; 70 71 slot->arch.gfn_track[mode][index] += count; 72 } 73 74 /* 75 * add guest page to the tracking pool so that corresponding access on that 76 * page will be intercepted. 77 * 78 * It should be called under the protection both of mmu-lock and kvm->srcu 79 * or kvm->slots_lock. 80 * 81 * @kvm: the guest instance we are interested in. 82 * @slot: the @gfn belongs to. 83 * @gfn: the guest page. 84 * @mode: tracking mode, currently only write track is supported. 85 */ 86 void kvm_slot_page_track_add_page(struct kvm *kvm, 87 struct kvm_memory_slot *slot, gfn_t gfn, 88 enum kvm_page_track_mode mode) 89 { 90 91 if (WARN_ON(!page_track_mode_is_valid(mode))) 92 return; 93 94 update_gfn_track(slot, gfn, mode, 1); 95 96 /* 97 * new track stops large page mapping for the 98 * tracked page. 99 */ 100 kvm_mmu_gfn_disallow_lpage(slot, gfn); 101 102 if (mode == KVM_PAGE_TRACK_WRITE) 103 if (kvm_mmu_slot_gfn_write_protect(kvm, slot, gfn)) 104 kvm_flush_remote_tlbs(kvm); 105 } 106 EXPORT_SYMBOL_GPL(kvm_slot_page_track_add_page); 107 108 /* 109 * remove the guest page from the tracking pool which stops the interception 110 * of corresponding access on that page. It is the opposed operation of 111 * kvm_slot_page_track_add_page(). 112 * 113 * It should be called under the protection both of mmu-lock and kvm->srcu 114 * or kvm->slots_lock. 115 * 116 * @kvm: the guest instance we are interested in. 117 * @slot: the @gfn belongs to. 118 * @gfn: the guest page. 119 * @mode: tracking mode, currently only write track is supported. 120 */ 121 void kvm_slot_page_track_remove_page(struct kvm *kvm, 122 struct kvm_memory_slot *slot, gfn_t gfn, 123 enum kvm_page_track_mode mode) 124 { 125 if (WARN_ON(!page_track_mode_is_valid(mode))) 126 return; 127 128 update_gfn_track(slot, gfn, mode, -1); 129 130 /* 131 * allow large page mapping for the tracked page 132 * after the tracker is gone. 133 */ 134 kvm_mmu_gfn_allow_lpage(slot, gfn); 135 } 136 EXPORT_SYMBOL_GPL(kvm_slot_page_track_remove_page); 137 138 /* 139 * check if the corresponding access on the specified guest page is tracked. 140 */ 141 bool kvm_page_track_is_active(struct kvm_vcpu *vcpu, gfn_t gfn, 142 enum kvm_page_track_mode mode) 143 { 144 struct kvm_memory_slot *slot; 145 int index; 146 147 if (WARN_ON(!page_track_mode_is_valid(mode))) 148 return false; 149 150 slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); 151 if (!slot) 152 return false; 153 154 index = gfn_to_index(gfn, slot->base_gfn, PG_LEVEL_4K); 155 return !!READ_ONCE(slot->arch.gfn_track[mode][index]); 156 } 157 158 void kvm_page_track_cleanup(struct kvm *kvm) 159 { 160 struct kvm_page_track_notifier_head *head; 161 162 head = &kvm->arch.track_notifier_head; 163 cleanup_srcu_struct(&head->track_srcu); 164 } 165 166 void kvm_page_track_init(struct kvm *kvm) 167 { 168 struct kvm_page_track_notifier_head *head; 169 170 head = &kvm->arch.track_notifier_head; 171 init_srcu_struct(&head->track_srcu); 172 INIT_HLIST_HEAD(&head->track_notifier_list); 173 } 174 175 /* 176 * register the notifier so that event interception for the tracked guest 177 * pages can be received. 178 */ 179 void 180 kvm_page_track_register_notifier(struct kvm *kvm, 181 struct kvm_page_track_notifier_node *n) 182 { 183 struct kvm_page_track_notifier_head *head; 184 185 head = &kvm->arch.track_notifier_head; 186 187 write_lock(&kvm->mmu_lock); 188 hlist_add_head_rcu(&n->node, &head->track_notifier_list); 189 write_unlock(&kvm->mmu_lock); 190 } 191 EXPORT_SYMBOL_GPL(kvm_page_track_register_notifier); 192 193 /* 194 * stop receiving the event interception. It is the opposed operation of 195 * kvm_page_track_register_notifier(). 196 */ 197 void 198 kvm_page_track_unregister_notifier(struct kvm *kvm, 199 struct kvm_page_track_notifier_node *n) 200 { 201 struct kvm_page_track_notifier_head *head; 202 203 head = &kvm->arch.track_notifier_head; 204 205 write_lock(&kvm->mmu_lock); 206 hlist_del_rcu(&n->node); 207 write_unlock(&kvm->mmu_lock); 208 synchronize_srcu(&head->track_srcu); 209 } 210 EXPORT_SYMBOL_GPL(kvm_page_track_unregister_notifier); 211 212 /* 213 * Notify the node that write access is intercepted and write emulation is 214 * finished at this time. 215 * 216 * The node should figure out if the written page is the one that node is 217 * interested in by itself. 218 */ 219 void kvm_page_track_write(struct kvm_vcpu *vcpu, gpa_t gpa, const u8 *new, 220 int bytes) 221 { 222 struct kvm_page_track_notifier_head *head; 223 struct kvm_page_track_notifier_node *n; 224 int idx; 225 226 head = &vcpu->kvm->arch.track_notifier_head; 227 228 if (hlist_empty(&head->track_notifier_list)) 229 return; 230 231 idx = srcu_read_lock(&head->track_srcu); 232 hlist_for_each_entry_srcu(n, &head->track_notifier_list, node, 233 srcu_read_lock_held(&head->track_srcu)) 234 if (n->track_write) 235 n->track_write(vcpu, gpa, new, bytes, n); 236 srcu_read_unlock(&head->track_srcu, idx); 237 } 238 239 /* 240 * Notify the node that memory slot is being removed or moved so that it can 241 * drop write-protection for the pages in the memory slot. 242 * 243 * The node should figure out it has any write-protected pages in this slot 244 * by itself. 245 */ 246 void kvm_page_track_flush_slot(struct kvm *kvm, struct kvm_memory_slot *slot) 247 { 248 struct kvm_page_track_notifier_head *head; 249 struct kvm_page_track_notifier_node *n; 250 int idx; 251 252 head = &kvm->arch.track_notifier_head; 253 254 if (hlist_empty(&head->track_notifier_list)) 255 return; 256 257 idx = srcu_read_lock(&head->track_srcu); 258 hlist_for_each_entry_srcu(n, &head->track_notifier_list, node, 259 srcu_read_lock_held(&head->track_srcu)) 260 if (n->track_flush_slot) 261 n->track_flush_slot(kvm, slot, n); 262 srcu_read_unlock(&head->track_srcu, idx); 263 } 264