1 #include <linux/swap_cgroup.h> 2 #include <linux/vmalloc.h> 3 #include <linux/mm.h> 4 5 #include <linux/swapops.h> /* depends on mm.h include */ 6 7 static DEFINE_MUTEX(swap_cgroup_mutex); 8 struct swap_cgroup_ctrl { 9 struct page **map; 10 unsigned long length; 11 spinlock_t lock; 12 }; 13 14 static struct swap_cgroup_ctrl swap_cgroup_ctrl[MAX_SWAPFILES]; 15 16 struct swap_cgroup { 17 unsigned short id; 18 }; 19 #define SC_PER_PAGE (PAGE_SIZE/sizeof(struct swap_cgroup)) 20 21 /* 22 * SwapCgroup implements "lookup" and "exchange" operations. 23 * In typical usage, this swap_cgroup is accessed via memcg's charge/uncharge 24 * against SwapCache. At swap_free(), this is accessed directly from swap. 25 * 26 * This means, 27 * - we have no race in "exchange" when we're accessed via SwapCache because 28 * SwapCache(and its swp_entry) is under lock. 29 * - When called via swap_free(), there is no user of this entry and no race. 30 * Then, we don't need lock around "exchange". 31 * 32 * TODO: we can push these buffers out to HIGHMEM. 33 */ 34 35 /* 36 * allocate buffer for swap_cgroup. 37 */ 38 static int swap_cgroup_prepare(int type) 39 { 40 struct page *page; 41 struct swap_cgroup_ctrl *ctrl; 42 unsigned long idx, max; 43 44 ctrl = &swap_cgroup_ctrl[type]; 45 46 for (idx = 0; idx < ctrl->length; idx++) { 47 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 48 if (!page) 49 goto not_enough_page; 50 ctrl->map[idx] = page; 51 } 52 return 0; 53 not_enough_page: 54 max = idx; 55 for (idx = 0; idx < max; idx++) 56 __free_page(ctrl->map[idx]); 57 58 return -ENOMEM; 59 } 60 61 static struct swap_cgroup *lookup_swap_cgroup(swp_entry_t ent, 62 struct swap_cgroup_ctrl **ctrlp) 63 { 64 pgoff_t offset = swp_offset(ent); 65 struct swap_cgroup_ctrl *ctrl; 66 struct page *mappage; 67 struct swap_cgroup *sc; 68 69 ctrl = &swap_cgroup_ctrl[swp_type(ent)]; 70 if (ctrlp) 71 *ctrlp = ctrl; 72 73 mappage = ctrl->map[offset / SC_PER_PAGE]; 74 sc = page_address(mappage); 75 return sc + offset % SC_PER_PAGE; 76 } 77 78 /** 79 * swap_cgroup_cmpxchg - cmpxchg mem_cgroup's id for this swp_entry. 80 * @ent: swap entry to be cmpxchged 81 * @old: old id 82 * @new: new id 83 * 84 * Returns old id at success, 0 at failure. 85 * (There is no mem_cgroup using 0 as its id) 86 */ 87 unsigned short swap_cgroup_cmpxchg(swp_entry_t ent, 88 unsigned short old, unsigned short new) 89 { 90 struct swap_cgroup_ctrl *ctrl; 91 struct swap_cgroup *sc; 92 unsigned long flags; 93 unsigned short retval; 94 95 sc = lookup_swap_cgroup(ent, &ctrl); 96 97 spin_lock_irqsave(&ctrl->lock, flags); 98 retval = sc->id; 99 if (retval == old) 100 sc->id = new; 101 else 102 retval = 0; 103 spin_unlock_irqrestore(&ctrl->lock, flags); 104 return retval; 105 } 106 107 /** 108 * swap_cgroup_record - record mem_cgroup for this swp_entry. 109 * @ent: swap entry to be recorded into 110 * @id: mem_cgroup to be recorded 111 * 112 * Returns old value at success, 0 at failure. 113 * (Of course, old value can be 0.) 114 */ 115 unsigned short swap_cgroup_record(swp_entry_t ent, unsigned short id) 116 { 117 struct swap_cgroup_ctrl *ctrl; 118 struct swap_cgroup *sc; 119 unsigned short old; 120 unsigned long flags; 121 122 sc = lookup_swap_cgroup(ent, &ctrl); 123 124 spin_lock_irqsave(&ctrl->lock, flags); 125 old = sc->id; 126 sc->id = id; 127 spin_unlock_irqrestore(&ctrl->lock, flags); 128 129 return old; 130 } 131 132 /** 133 * lookup_swap_cgroup_id - lookup mem_cgroup id tied to swap entry 134 * @ent: swap entry to be looked up. 135 * 136 * Returns ID of mem_cgroup at success. 0 at failure. (0 is invalid ID) 137 */ 138 unsigned short lookup_swap_cgroup_id(swp_entry_t ent) 139 { 140 return lookup_swap_cgroup(ent, NULL)->id; 141 } 142 143 int swap_cgroup_swapon(int type, unsigned long max_pages) 144 { 145 void *array; 146 unsigned long array_size; 147 unsigned long length; 148 struct swap_cgroup_ctrl *ctrl; 149 150 if (!do_swap_account) 151 return 0; 152 153 length = DIV_ROUND_UP(max_pages, SC_PER_PAGE); 154 array_size = length * sizeof(void *); 155 156 array = vzalloc(array_size); 157 if (!array) 158 goto nomem; 159 160 ctrl = &swap_cgroup_ctrl[type]; 161 mutex_lock(&swap_cgroup_mutex); 162 ctrl->length = length; 163 ctrl->map = array; 164 spin_lock_init(&ctrl->lock); 165 if (swap_cgroup_prepare(type)) { 166 /* memory shortage */ 167 ctrl->map = NULL; 168 ctrl->length = 0; 169 mutex_unlock(&swap_cgroup_mutex); 170 vfree(array); 171 goto nomem; 172 } 173 mutex_unlock(&swap_cgroup_mutex); 174 175 return 0; 176 nomem: 177 pr_info("couldn't allocate enough memory for swap_cgroup\n"); 178 pr_info("swap_cgroup can be disabled by swapaccount=0 boot option\n"); 179 return -ENOMEM; 180 } 181 182 void swap_cgroup_swapoff(int type) 183 { 184 struct page **map; 185 unsigned long i, length; 186 struct swap_cgroup_ctrl *ctrl; 187 188 if (!do_swap_account) 189 return; 190 191 mutex_lock(&swap_cgroup_mutex); 192 ctrl = &swap_cgroup_ctrl[type]; 193 map = ctrl->map; 194 length = ctrl->length; 195 ctrl->map = NULL; 196 ctrl->length = 0; 197 mutex_unlock(&swap_cgroup_mutex); 198 199 if (map) { 200 for (i = 0; i < length; i++) { 201 struct page *page = map[i]; 202 if (page) 203 __free_page(page); 204 if (!(i % SWAP_CLUSTER_MAX)) 205 cond_resched(); 206 } 207 vfree(map); 208 } 209 } 210