1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright 2013, Michael (Ellerman|Neuling), IBM Corporation. 4 */ 5 6 #define pr_fmt(fmt) "powernv: " fmt 7 8 #include <linux/kernel.h> 9 #include <linux/cpu.h> 10 #include <linux/cpumask.h> 11 #include <linux/device.h> 12 #include <linux/gfp.h> 13 #include <linux/smp.h> 14 #include <linux/stop_machine.h> 15 16 #include <asm/cputhreads.h> 17 #include <asm/cpuidle.h> 18 #include <asm/kvm_ppc.h> 19 #include <asm/machdep.h> 20 #include <asm/opal.h> 21 #include <asm/smp.h> 22 23 #include "subcore.h" 24 #include "powernv.h" 25 26 27 /* 28 * Split/unsplit procedure: 29 * 30 * A core can be in one of three states, unsplit, 2-way split, and 4-way split. 31 * 32 * The mapping to subcores_per_core is simple: 33 * 34 * State | subcores_per_core 35 * ------------|------------------ 36 * Unsplit | 1 37 * 2-way split | 2 38 * 4-way split | 4 39 * 40 * The core is split along thread boundaries, the mapping between subcores and 41 * threads is as follows: 42 * 43 * Unsplit: 44 * ---------------------------- 45 * Subcore | 0 | 46 * ---------------------------- 47 * Thread | 0 1 2 3 4 5 6 7 | 48 * ---------------------------- 49 * 50 * 2-way split: 51 * ------------------------------------- 52 * Subcore | 0 | 1 | 53 * ------------------------------------- 54 * Thread | 0 1 2 3 | 4 5 6 7 | 55 * ------------------------------------- 56 * 57 * 4-way split: 58 * ----------------------------------------- 59 * Subcore | 0 | 1 | 2 | 3 | 60 * ----------------------------------------- 61 * Thread | 0 1 | 2 3 | 4 5 | 6 7 | 62 * ----------------------------------------- 63 * 64 * 65 * Transitions 66 * ----------- 67 * 68 * It is not possible to transition between either of the split states, the 69 * core must first be unsplit. The legal transitions are: 70 * 71 * ----------- --------------- 72 * | | <----> | 2-way split | 73 * | | --------------- 74 * | Unsplit | 75 * | | --------------- 76 * | | <----> | 4-way split | 77 * ----------- --------------- 78 * 79 * Unsplitting 80 * ----------- 81 * 82 * Unsplitting is the simpler procedure. It requires thread 0 to request the 83 * unsplit while all other threads NAP. 84 * 85 * Thread 0 clears HID0_POWER8_DYNLPARDIS (Dynamic LPAR Disable). This tells 86 * the hardware that if all threads except 0 are napping, the hardware should 87 * unsplit the core. 88 * 89 * Non-zero threads are sent to a NAP loop, they don't exit the loop until they 90 * see the core unsplit. 91 * 92 * Core 0 spins waiting for the hardware to see all the other threads napping 93 * and perform the unsplit. 94 * 95 * Once thread 0 sees the unsplit, it IPIs the secondary threads to wake them 96 * out of NAP. They will then see the core unsplit and exit the NAP loop. 97 * 98 * Splitting 99 * --------- 100 * 101 * The basic splitting procedure is fairly straight forward. However it is 102 * complicated by the fact that after the split occurs, the newly created 103 * subcores are not in a fully initialised state. 104 * 105 * Most notably the subcores do not have the correct value for SDR1, which 106 * means they must not be running in virtual mode when the split occurs. The 107 * subcores have separate timebases SPRs but these are pre-synchronised by 108 * opal. 109 * 110 * To begin with secondary threads are sent to an assembly routine. There they 111 * switch to real mode, so they are immune to the uninitialised SDR1 value. 112 * Once in real mode they indicate that they are in real mode, and spin waiting 113 * to see the core split. 114 * 115 * Thread 0 waits to see that all secondaries are in real mode, and then begins 116 * the splitting procedure. It firstly sets HID0_POWER8_DYNLPARDIS, which 117 * prevents the hardware from unsplitting. Then it sets the appropriate HID bit 118 * to request the split, and spins waiting to see that the split has happened. 119 * 120 * Concurrently the secondaries will notice the split. When they do they set up 121 * their SPRs, notably SDR1, and then they can return to virtual mode and exit 122 * the procedure. 123 */ 124 125 /* Initialised at boot by subcore_init() */ 126 static int subcores_per_core; 127 128 /* 129 * Used to communicate to offline cpus that we want them to pop out of the 130 * offline loop and do a split or unsplit. 131 * 132 * 0 - no split happening 133 * 1 - unsplit in progress 134 * 2 - split to 2 in progress 135 * 4 - split to 4 in progress 136 */ 137 static int new_split_mode; 138 139 static cpumask_var_t cpu_offline_mask; 140 141 struct split_state { 142 u8 step; 143 u8 master; 144 }; 145 146 static DEFINE_PER_CPU(struct split_state, split_state); 147 148 static void wait_for_sync_step(int step) 149 { 150 int i, cpu = smp_processor_id(); 151 152 for (i = cpu + 1; i < cpu + threads_per_core; i++) 153 while(per_cpu(split_state, i).step < step) 154 barrier(); 155 156 /* Order the wait loop vs any subsequent loads/stores. */ 157 mb(); 158 } 159 160 static void update_hid_in_slw(u64 hid0) 161 { 162 u64 idle_states = pnv_get_supported_cpuidle_states(); 163 164 if (idle_states & OPAL_PM_WINKLE_ENABLED) { 165 /* OPAL call to patch slw with the new HID0 value */ 166 u64 cpu_pir = hard_smp_processor_id(); 167 168 opal_slw_set_reg(cpu_pir, SPRN_HID0, hid0); 169 } 170 } 171 172 static inline void update_power8_hid0(unsigned long hid0) 173 { 174 /* 175 * The HID0 update on Power8 should at the very least be 176 * preceded by a SYNC instruction followed by an ISYNC 177 * instruction 178 */ 179 asm volatile("sync; mtspr %0,%1; isync":: "i"(SPRN_HID0), "r"(hid0)); 180 } 181 182 static void unsplit_core(void) 183 { 184 u64 hid0, mask; 185 int i, cpu; 186 187 mask = HID0_POWER8_2LPARMODE | HID0_POWER8_4LPARMODE; 188 189 cpu = smp_processor_id(); 190 if (cpu_thread_in_core(cpu) != 0) { 191 while (mfspr(SPRN_HID0) & mask) 192 power7_idle_type(PNV_THREAD_NAP); 193 194 per_cpu(split_state, cpu).step = SYNC_STEP_UNSPLIT; 195 return; 196 } 197 198 hid0 = mfspr(SPRN_HID0); 199 hid0 &= ~HID0_POWER8_DYNLPARDIS; 200 update_power8_hid0(hid0); 201 update_hid_in_slw(hid0); 202 203 while (mfspr(SPRN_HID0) & mask) 204 cpu_relax(); 205 206 /* Wake secondaries out of NAP */ 207 for (i = cpu + 1; i < cpu + threads_per_core; i++) 208 smp_send_reschedule(i); 209 210 wait_for_sync_step(SYNC_STEP_UNSPLIT); 211 } 212 213 static void split_core(int new_mode) 214 { 215 struct { u64 value; u64 mask; } split_parms[2] = { 216 { HID0_POWER8_1TO2LPAR, HID0_POWER8_2LPARMODE }, 217 { HID0_POWER8_1TO4LPAR, HID0_POWER8_4LPARMODE } 218 }; 219 int i, cpu; 220 u64 hid0; 221 222 /* Convert new_mode (2 or 4) into an index into our parms array */ 223 i = (new_mode >> 1) - 1; 224 BUG_ON(i < 0 || i > 1); 225 226 cpu = smp_processor_id(); 227 if (cpu_thread_in_core(cpu) != 0) { 228 split_core_secondary_loop(&per_cpu(split_state, cpu).step); 229 return; 230 } 231 232 wait_for_sync_step(SYNC_STEP_REAL_MODE); 233 234 /* Write new mode */ 235 hid0 = mfspr(SPRN_HID0); 236 hid0 |= HID0_POWER8_DYNLPARDIS | split_parms[i].value; 237 update_power8_hid0(hid0); 238 update_hid_in_slw(hid0); 239 240 /* Wait for it to happen */ 241 while (!(mfspr(SPRN_HID0) & split_parms[i].mask)) 242 cpu_relax(); 243 } 244 245 static void cpu_do_split(int new_mode) 246 { 247 /* 248 * At boot subcores_per_core will be 0, so we will always unsplit at 249 * boot. In the usual case where the core is already unsplit it's a 250 * nop, and this just ensures the kernel's notion of the mode is 251 * consistent with the hardware. 252 */ 253 if (subcores_per_core != 1) 254 unsplit_core(); 255 256 if (new_mode != 1) 257 split_core(new_mode); 258 259 mb(); 260 per_cpu(split_state, smp_processor_id()).step = SYNC_STEP_FINISHED; 261 } 262 263 bool cpu_core_split_required(void) 264 { 265 smp_rmb(); 266 267 if (!new_split_mode) 268 return false; 269 270 cpu_do_split(new_split_mode); 271 272 return true; 273 } 274 275 void update_subcore_sibling_mask(void) 276 { 277 int cpu; 278 /* 279 * sibling mask for the first cpu. Left shift this by required bits 280 * to get sibling mask for the rest of the cpus. 281 */ 282 int sibling_mask_first_cpu = (1 << threads_per_subcore) - 1; 283 284 for_each_possible_cpu(cpu) { 285 int tid = cpu_thread_in_core(cpu); 286 int offset = (tid / threads_per_subcore) * threads_per_subcore; 287 int mask = sibling_mask_first_cpu << offset; 288 289 paca_ptrs[cpu]->subcore_sibling_mask = mask; 290 291 } 292 } 293 294 static int cpu_update_split_mode(void *data) 295 { 296 int cpu, new_mode = *(int *)data; 297 298 if (this_cpu_ptr(&split_state)->master) { 299 new_split_mode = new_mode; 300 smp_wmb(); 301 302 cpumask_andnot(cpu_offline_mask, cpu_present_mask, 303 cpu_online_mask); 304 305 /* This should work even though the cpu is offline */ 306 for_each_cpu(cpu, cpu_offline_mask) 307 smp_send_reschedule(cpu); 308 } 309 310 cpu_do_split(new_mode); 311 312 if (this_cpu_ptr(&split_state)->master) { 313 /* Wait for all cpus to finish before we touch subcores_per_core */ 314 for_each_present_cpu(cpu) { 315 if (cpu >= setup_max_cpus) 316 break; 317 318 while(per_cpu(split_state, cpu).step < SYNC_STEP_FINISHED) 319 barrier(); 320 } 321 322 new_split_mode = 0; 323 324 /* Make the new mode public */ 325 subcores_per_core = new_mode; 326 threads_per_subcore = threads_per_core / subcores_per_core; 327 update_subcore_sibling_mask(); 328 329 /* Make sure the new mode is written before we exit */ 330 mb(); 331 } 332 333 return 0; 334 } 335 336 static int set_subcores_per_core(int new_mode) 337 { 338 struct split_state *state; 339 int cpu; 340 341 if (kvm_hv_mode_active()) { 342 pr_err("Unable to change split core mode while KVM active.\n"); 343 return -EBUSY; 344 } 345 346 /* 347 * We are only called at boot, or from the sysfs write. If that ever 348 * changes we'll need a lock here. 349 */ 350 BUG_ON(new_mode < 1 || new_mode > 4 || new_mode == 3); 351 352 for_each_present_cpu(cpu) { 353 state = &per_cpu(split_state, cpu); 354 state->step = SYNC_STEP_INITIAL; 355 state->master = 0; 356 } 357 358 cpus_read_lock(); 359 360 /* This cpu will update the globals before exiting stop machine */ 361 this_cpu_ptr(&split_state)->master = 1; 362 363 /* Ensure state is consistent before we call the other cpus */ 364 mb(); 365 366 stop_machine_cpuslocked(cpu_update_split_mode, &new_mode, 367 cpu_online_mask); 368 369 cpus_read_unlock(); 370 371 return 0; 372 } 373 374 static ssize_t __used store_subcores_per_core(struct device *dev, 375 struct device_attribute *attr, const char *buf, 376 size_t count) 377 { 378 unsigned long val; 379 int rc; 380 381 /* We are serialised by the attribute lock */ 382 383 rc = sscanf(buf, "%lx", &val); 384 if (rc != 1) 385 return -EINVAL; 386 387 switch (val) { 388 case 1: 389 case 2: 390 case 4: 391 if (subcores_per_core == val) 392 /* Nothing to do */ 393 goto out; 394 break; 395 default: 396 return -EINVAL; 397 } 398 399 rc = set_subcores_per_core(val); 400 if (rc) 401 return rc; 402 403 out: 404 return count; 405 } 406 407 static ssize_t show_subcores_per_core(struct device *dev, 408 struct device_attribute *attr, char *buf) 409 { 410 return sprintf(buf, "%x\n", subcores_per_core); 411 } 412 413 static DEVICE_ATTR(subcores_per_core, 0644, 414 show_subcores_per_core, store_subcores_per_core); 415 416 static int subcore_init(void) 417 { 418 unsigned pvr_ver; 419 420 pvr_ver = PVR_VER(mfspr(SPRN_PVR)); 421 422 if (pvr_ver != PVR_POWER8 && 423 pvr_ver != PVR_POWER8E && 424 pvr_ver != PVR_POWER8NVL) 425 return 0; 426 427 /* 428 * We need all threads in a core to be present to split/unsplit so 429 * continue only if max_cpus are aligned to threads_per_core. 430 */ 431 if (setup_max_cpus % threads_per_core) 432 return 0; 433 434 BUG_ON(!alloc_cpumask_var(&cpu_offline_mask, GFP_KERNEL)); 435 436 set_subcores_per_core(1); 437 438 return device_create_file(cpu_subsys.dev_root, 439 &dev_attr_subcores_per_core); 440 } 441 machine_device_initcall(powernv, subcore_init); 442