1 /* leon_smp.c: Sparc-Leon SMP support. 2 * 3 * based on sun4m_smp.c 4 * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) 5 * Copyright (C) 2009 Daniel Hellstrom (daniel@gaisler.com) Aeroflex Gaisler AB 6 * Copyright (C) 2009 Konrad Eisele (konrad@gaisler.com) Aeroflex Gaisler AB 7 */ 8 9 #include <asm/head.h> 10 11 #include <linux/kernel.h> 12 #include <linux/sched.h> 13 #include <linux/threads.h> 14 #include <linux/smp.h> 15 #include <linux/interrupt.h> 16 #include <linux/kernel_stat.h> 17 #include <linux/init.h> 18 #include <linux/spinlock.h> 19 #include <linux/mm.h> 20 #include <linux/swap.h> 21 #include <linux/profile.h> 22 #include <linux/pm.h> 23 #include <linux/delay.h> 24 #include <linux/gfp.h> 25 26 #include <asm/cacheflush.h> 27 #include <asm/tlbflush.h> 28 29 #include <asm/ptrace.h> 30 #include <asm/atomic.h> 31 #include <asm/irq_regs.h> 32 33 #include <asm/delay.h> 34 #include <asm/irq.h> 35 #include <asm/page.h> 36 #include <asm/pgalloc.h> 37 #include <asm/pgtable.h> 38 #include <asm/oplib.h> 39 #include <asm/cpudata.h> 40 #include <asm/asi.h> 41 #include <asm/leon.h> 42 #include <asm/leon_amba.h> 43 44 #include "kernel.h" 45 46 #ifdef CONFIG_SPARC_LEON 47 48 #include "irq.h" 49 50 extern ctxd_t *srmmu_ctx_table_phys; 51 static int smp_processors_ready; 52 extern volatile unsigned long cpu_callin_map[NR_CPUS]; 53 extern unsigned char boot_cpu_id; 54 extern cpumask_t smp_commenced_mask; 55 void __init leon_configure_cache_smp(void); 56 57 static inline unsigned long do_swap(volatile unsigned long *ptr, 58 unsigned long val) 59 { 60 __asm__ __volatile__("swapa [%2] %3, %0\n\t" : "=&r"(val) 61 : "0"(val), "r"(ptr), "i"(ASI_LEON_DCACHE_MISS) 62 : "memory"); 63 return val; 64 } 65 66 static void smp_setup_percpu_timer(void); 67 68 void __cpuinit leon_callin(void) 69 { 70 int cpuid = hard_smpleon_processor_id(); 71 72 local_flush_cache_all(); 73 local_flush_tlb_all(); 74 leon_configure_cache_smp(); 75 76 /* Get our local ticker going. */ 77 smp_setup_percpu_timer(); 78 79 calibrate_delay(); 80 smp_store_cpu_info(cpuid); 81 82 local_flush_cache_all(); 83 local_flush_tlb_all(); 84 85 /* 86 * Unblock the master CPU _only_ when the scheduler state 87 * of all secondary CPUs will be up-to-date, so after 88 * the SMP initialization the master will be just allowed 89 * to call the scheduler code. 90 * Allow master to continue. 91 */ 92 do_swap(&cpu_callin_map[cpuid], 1); 93 94 local_flush_cache_all(); 95 local_flush_tlb_all(); 96 97 cpu_probe(); 98 99 /* Fix idle thread fields. */ 100 __asm__ __volatile__("ld [%0], %%g6\n\t" : : "r"(¤t_set[cpuid]) 101 : "memory" /* paranoid */); 102 103 /* Attach to the address space of init_task. */ 104 atomic_inc(&init_mm.mm_count); 105 current->active_mm = &init_mm; 106 107 while (!cpu_isset(cpuid, smp_commenced_mask)) 108 mb(); 109 110 local_irq_enable(); 111 cpu_set(cpuid, cpu_online_map); 112 } 113 114 /* 115 * Cycle through the processors asking the PROM to start each one. 116 */ 117 118 extern struct linux_prom_registers smp_penguin_ctable; 119 120 void __init leon_configure_cache_smp(void) 121 { 122 unsigned long cfg = sparc_leon3_get_dcachecfg(); 123 int me = smp_processor_id(); 124 125 if (ASI_LEON3_SYSCTRL_CFG_SSIZE(cfg) > 4) { 126 printk(KERN_INFO "Note: SMP with snooping only works on 4k cache, found %dk(0x%x) on cpu %d, disabling caches\n", 127 (unsigned int)ASI_LEON3_SYSCTRL_CFG_SSIZE(cfg), 128 (unsigned int)cfg, (unsigned int)me); 129 sparc_leon3_disable_cache(); 130 } else { 131 if (cfg & ASI_LEON3_SYSCTRL_CFG_SNOOPING) { 132 sparc_leon3_enable_snooping(); 133 } else { 134 printk(KERN_INFO "Note: You have to enable snooping in the vhdl model cpu %d, disabling caches\n", 135 me); 136 sparc_leon3_disable_cache(); 137 } 138 } 139 140 local_flush_cache_all(); 141 local_flush_tlb_all(); 142 } 143 144 void leon_smp_setbroadcast(unsigned int mask) 145 { 146 int broadcast = 147 ((LEON3_BYPASS_LOAD_PA(&(leon3_irqctrl_regs->mpstatus)) >> 148 LEON3_IRQMPSTATUS_BROADCAST) & 1); 149 if (!broadcast) { 150 prom_printf("######## !!!! The irqmp-ctrl must have broadcast enabled, smp wont work !!!!! ####### nr cpus: %d\n", 151 leon_smp_nrcpus()); 152 if (leon_smp_nrcpus() > 1) { 153 BUG(); 154 } else { 155 prom_printf("continue anyway\n"); 156 return; 157 } 158 } 159 LEON_BYPASS_STORE_PA(&(leon3_irqctrl_regs->mpbroadcast), mask); 160 } 161 162 unsigned int leon_smp_getbroadcast(void) 163 { 164 unsigned int mask; 165 mask = LEON_BYPASS_LOAD_PA(&(leon3_irqctrl_regs->mpbroadcast)); 166 return mask; 167 } 168 169 int leon_smp_nrcpus(void) 170 { 171 int nrcpu = 172 ((LEON3_BYPASS_LOAD_PA(&(leon3_irqctrl_regs->mpstatus)) >> 173 LEON3_IRQMPSTATUS_CPUNR) & 0xf) + 1; 174 return nrcpu; 175 } 176 177 void __init leon_boot_cpus(void) 178 { 179 int nrcpu = leon_smp_nrcpus(); 180 int me = smp_processor_id(); 181 182 printk(KERN_INFO "%d:(%d:%d) cpus mpirq at 0x%x\n", (unsigned int)me, 183 (unsigned int)nrcpu, (unsigned int)NR_CPUS, 184 (unsigned int)&(leon3_irqctrl_regs->mpstatus)); 185 186 leon_enable_irq_cpu(LEON3_IRQ_CROSS_CALL, me); 187 leon_enable_irq_cpu(LEON3_IRQ_TICKER, me); 188 leon_enable_irq_cpu(LEON3_IRQ_RESCHEDULE, me); 189 190 leon_smp_setbroadcast(1 << LEON3_IRQ_TICKER); 191 192 leon_configure_cache_smp(); 193 smp_setup_percpu_timer(); 194 local_flush_cache_all(); 195 196 } 197 198 int __cpuinit leon_boot_one_cpu(int i) 199 { 200 201 struct task_struct *p; 202 int timeout; 203 204 /* Cook up an idler for this guy. */ 205 p = fork_idle(i); 206 207 current_set[i] = task_thread_info(p); 208 209 /* See trampoline.S:leon_smp_cpu_startup for details... 210 * Initialize the contexts table 211 * Since the call to prom_startcpu() trashes the structure, 212 * we need to re-initialize it for each cpu 213 */ 214 smp_penguin_ctable.which_io = 0; 215 smp_penguin_ctable.phys_addr = (unsigned int)srmmu_ctx_table_phys; 216 smp_penguin_ctable.reg_size = 0; 217 218 /* whirrr, whirrr, whirrrrrrrrr... */ 219 printk(KERN_INFO "Starting CPU %d : (irqmp: 0x%x)\n", (unsigned int)i, 220 (unsigned int)&leon3_irqctrl_regs->mpstatus); 221 local_flush_cache_all(); 222 223 LEON_BYPASS_STORE_PA(&(leon3_irqctrl_regs->mpstatus), 1 << i); 224 225 /* wheee... it's going... */ 226 for (timeout = 0; timeout < 10000; timeout++) { 227 if (cpu_callin_map[i]) 228 break; 229 udelay(200); 230 } 231 printk(KERN_INFO "Started CPU %d\n", (unsigned int)i); 232 233 if (!(cpu_callin_map[i])) { 234 printk(KERN_ERR "Processor %d is stuck.\n", i); 235 return -ENODEV; 236 } else { 237 leon_enable_irq_cpu(LEON3_IRQ_CROSS_CALL, i); 238 leon_enable_irq_cpu(LEON3_IRQ_TICKER, i); 239 leon_enable_irq_cpu(LEON3_IRQ_RESCHEDULE, i); 240 } 241 242 local_flush_cache_all(); 243 return 0; 244 } 245 246 void __init leon_smp_done(void) 247 { 248 249 int i, first; 250 int *prev; 251 252 /* setup cpu list for irq rotation */ 253 first = 0; 254 prev = &first; 255 for (i = 0; i < NR_CPUS; i++) { 256 if (cpu_online(i)) { 257 *prev = i; 258 prev = &cpu_data(i).next; 259 } 260 } 261 *prev = first; 262 local_flush_cache_all(); 263 264 /* Free unneeded trap tables */ 265 if (!cpu_isset(1, cpu_present_map)) { 266 ClearPageReserved(virt_to_page(&trapbase_cpu1)); 267 init_page_count(virt_to_page(&trapbase_cpu1)); 268 free_page((unsigned long)&trapbase_cpu1); 269 totalram_pages++; 270 num_physpages++; 271 } 272 if (!cpu_isset(2, cpu_present_map)) { 273 ClearPageReserved(virt_to_page(&trapbase_cpu2)); 274 init_page_count(virt_to_page(&trapbase_cpu2)); 275 free_page((unsigned long)&trapbase_cpu2); 276 totalram_pages++; 277 num_physpages++; 278 } 279 if (!cpu_isset(3, cpu_present_map)) { 280 ClearPageReserved(virt_to_page(&trapbase_cpu3)); 281 init_page_count(virt_to_page(&trapbase_cpu3)); 282 free_page((unsigned long)&trapbase_cpu3); 283 totalram_pages++; 284 num_physpages++; 285 } 286 /* Ok, they are spinning and ready to go. */ 287 smp_processors_ready = 1; 288 289 } 290 291 void leon_irq_rotate(int cpu) 292 { 293 } 294 295 static struct smp_funcall { 296 smpfunc_t func; 297 unsigned long arg1; 298 unsigned long arg2; 299 unsigned long arg3; 300 unsigned long arg4; 301 unsigned long arg5; 302 unsigned long processors_in[NR_CPUS]; /* Set when ipi entered. */ 303 unsigned long processors_out[NR_CPUS]; /* Set when ipi exited. */ 304 } ccall_info; 305 306 static DEFINE_SPINLOCK(cross_call_lock); 307 308 /* Cross calls must be serialized, at least currently. */ 309 static void leon_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1, 310 unsigned long arg2, unsigned long arg3, 311 unsigned long arg4) 312 { 313 if (smp_processors_ready) { 314 register int high = NR_CPUS - 1; 315 unsigned long flags; 316 317 spin_lock_irqsave(&cross_call_lock, flags); 318 319 { 320 /* If you make changes here, make sure gcc generates proper code... */ 321 register smpfunc_t f asm("i0") = func; 322 register unsigned long a1 asm("i1") = arg1; 323 register unsigned long a2 asm("i2") = arg2; 324 register unsigned long a3 asm("i3") = arg3; 325 register unsigned long a4 asm("i4") = arg4; 326 register unsigned long a5 asm("i5") = 0; 327 328 __asm__ __volatile__("std %0, [%6]\n\t" 329 "std %2, [%6 + 8]\n\t" 330 "std %4, [%6 + 16]\n\t" : : 331 "r"(f), "r"(a1), "r"(a2), "r"(a3), 332 "r"(a4), "r"(a5), 333 "r"(&ccall_info.func)); 334 } 335 336 /* Init receive/complete mapping, plus fire the IPI's off. */ 337 { 338 register int i; 339 340 cpu_clear(smp_processor_id(), mask); 341 cpus_and(mask, cpu_online_map, mask); 342 for (i = 0; i <= high; i++) { 343 if (cpu_isset(i, mask)) { 344 ccall_info.processors_in[i] = 0; 345 ccall_info.processors_out[i] = 0; 346 set_cpu_int(i, LEON3_IRQ_CROSS_CALL); 347 348 } 349 } 350 } 351 352 { 353 register int i; 354 355 i = 0; 356 do { 357 if (!cpu_isset(i, mask)) 358 continue; 359 360 while (!ccall_info.processors_in[i]) 361 barrier(); 362 } while (++i <= high); 363 364 i = 0; 365 do { 366 if (!cpu_isset(i, mask)) 367 continue; 368 369 while (!ccall_info.processors_out[i]) 370 barrier(); 371 } while (++i <= high); 372 } 373 374 spin_unlock_irqrestore(&cross_call_lock, flags); 375 } 376 } 377 378 /* Running cross calls. */ 379 void leon_cross_call_irq(void) 380 { 381 int i = smp_processor_id(); 382 383 ccall_info.processors_in[i] = 1; 384 ccall_info.func(ccall_info.arg1, ccall_info.arg2, ccall_info.arg3, 385 ccall_info.arg4, ccall_info.arg5); 386 ccall_info.processors_out[i] = 1; 387 } 388 389 void leon_percpu_timer_interrupt(struct pt_regs *regs) 390 { 391 struct pt_regs *old_regs; 392 int cpu = smp_processor_id(); 393 394 old_regs = set_irq_regs(regs); 395 396 leon_clear_profile_irq(cpu); 397 398 profile_tick(CPU_PROFILING); 399 400 if (!--prof_counter(cpu)) { 401 int user = user_mode(regs); 402 403 irq_enter(); 404 update_process_times(user); 405 irq_exit(); 406 407 prof_counter(cpu) = prof_multiplier(cpu); 408 } 409 set_irq_regs(old_regs); 410 } 411 412 static void __init smp_setup_percpu_timer(void) 413 { 414 int cpu = smp_processor_id(); 415 416 prof_counter(cpu) = prof_multiplier(cpu) = 1; 417 } 418 419 void __init leon_blackbox_id(unsigned *addr) 420 { 421 int rd = *addr & 0x3e000000; 422 int rs1 = rd >> 11; 423 424 /* patch places where ___b_hard_smp_processor_id appears */ 425 addr[0] = 0x81444000 | rd; /* rd %asr17, reg */ 426 addr[1] = 0x8130201c | rd | rs1; /* srl reg, 0x1c, reg */ 427 addr[2] = 0x01000000; /* nop */ 428 } 429 430 void __init leon_blackbox_current(unsigned *addr) 431 { 432 int rd = *addr & 0x3e000000; 433 int rs1 = rd >> 11; 434 435 /* patch LOAD_CURRENT macro where ___b_load_current appears */ 436 addr[0] = 0x81444000 | rd; /* rd %asr17, reg */ 437 addr[2] = 0x8130201c | rd | rs1; /* srl reg, 0x1c, reg */ 438 addr[4] = 0x81282002 | rd | rs1; /* sll reg, 0x2, reg */ 439 440 } 441 442 void __init leon_init_smp(void) 443 { 444 /* Patch ipi15 trap table */ 445 t_nmi[1] = t_nmi[1] + (linux_trap_ipi15_leon - linux_trap_ipi15_sun4m); 446 447 BTFIXUPSET_BLACKBOX(hard_smp_processor_id, leon_blackbox_id); 448 BTFIXUPSET_BLACKBOX(load_current, leon_blackbox_current); 449 BTFIXUPSET_CALL(smp_cross_call, leon_cross_call, BTFIXUPCALL_NORM); 450 BTFIXUPSET_CALL(__hard_smp_processor_id, __leon_processor_id, 451 BTFIXUPCALL_NORM); 452 } 453 454 #endif /* CONFIG_SPARC_LEON */ 455