1 /* 2 * Firmware replacement code. 3 * 4 * Work around broken BIOSes that don't set an aperture, only set the 5 * aperture in the AGP bridge, or set too small aperture. 6 * 7 * If all fails map the aperture over some low memory. This is cheaper than 8 * doing bounce buffering. The memory is lost. This is done at early boot 9 * because only the bootmem allocator can allocate 32+MB. 10 * 11 * Copyright 2002 Andi Kleen, SuSE Labs. 12 */ 13 #include <linux/kernel.h> 14 #include <linux/types.h> 15 #include <linux/init.h> 16 #include <linux/memblock.h> 17 #include <linux/mmzone.h> 18 #include <linux/pci_ids.h> 19 #include <linux/pci.h> 20 #include <linux/bitops.h> 21 #include <linux/ioport.h> 22 #include <linux/suspend.h> 23 #include <asm/e820.h> 24 #include <asm/io.h> 25 #include <asm/iommu.h> 26 #include <asm/gart.h> 27 #include <asm/pci-direct.h> 28 #include <asm/dma.h> 29 #include <asm/amd_nb.h> 30 #include <asm/x86_init.h> 31 32 /* 33 * Using 512M as goal, in case kexec will load kernel_big 34 * that will do the on-position decompress, and could overlap with 35 * with the gart aperture that is used. 36 * Sequence: 37 * kernel_small 38 * ==> kexec (with kdump trigger path or gart still enabled) 39 * ==> kernel_small (gart area become e820_reserved) 40 * ==> kexec (with kdump trigger path or gart still enabled) 41 * ==> kerne_big (uncompressed size will be big than 64M or 128M) 42 * So don't use 512M below as gart iommu, leave the space for kernel 43 * code for safe. 44 */ 45 #define GART_MIN_ADDR (512ULL << 20) 46 #define GART_MAX_ADDR (1ULL << 32) 47 48 int gart_iommu_aperture; 49 int gart_iommu_aperture_disabled __initdata; 50 int gart_iommu_aperture_allowed __initdata; 51 52 int fallback_aper_order __initdata = 1; /* 64MB */ 53 int fallback_aper_force __initdata; 54 55 int fix_aperture __initdata = 1; 56 57 static struct resource gart_resource = { 58 .name = "GART", 59 .flags = IORESOURCE_MEM, 60 }; 61 62 static void __init insert_aperture_resource(u32 aper_base, u32 aper_size) 63 { 64 gart_resource.start = aper_base; 65 gart_resource.end = aper_base + aper_size - 1; 66 insert_resource(&iomem_resource, &gart_resource); 67 } 68 69 /* This code runs before the PCI subsystem is initialized, so just 70 access the northbridge directly. */ 71 72 static u32 __init allocate_aperture(void) 73 { 74 u32 aper_size; 75 unsigned long addr; 76 77 /* aper_size should <= 1G */ 78 if (fallback_aper_order > 5) 79 fallback_aper_order = 5; 80 aper_size = (32 * 1024 * 1024) << fallback_aper_order; 81 82 /* 83 * Aperture has to be naturally aligned. This means a 2GB aperture 84 * won't have much chance of finding a place in the lower 4GB of 85 * memory. Unfortunately we cannot move it up because that would 86 * make the IOMMU useless. 87 */ 88 addr = memblock_find_in_range(GART_MIN_ADDR, GART_MAX_ADDR, 89 aper_size, aper_size); 90 if (!addr || addr + aper_size > GART_MAX_ADDR) { 91 printk(KERN_ERR 92 "Cannot allocate aperture memory hole (%lx,%uK)\n", 93 addr, aper_size>>10); 94 return 0; 95 } 96 memblock_reserve(addr, aper_size); 97 printk(KERN_INFO "Mapping aperture over %d KB of RAM @ %lx\n", 98 aper_size >> 10, addr); 99 insert_aperture_resource((u32)addr, aper_size); 100 register_nosave_region(addr >> PAGE_SHIFT, 101 (addr+aper_size) >> PAGE_SHIFT); 102 103 return (u32)addr; 104 } 105 106 107 /* Find a PCI capability */ 108 static u32 __init find_cap(int bus, int slot, int func, int cap) 109 { 110 int bytes; 111 u8 pos; 112 113 if (!(read_pci_config_16(bus, slot, func, PCI_STATUS) & 114 PCI_STATUS_CAP_LIST)) 115 return 0; 116 117 pos = read_pci_config_byte(bus, slot, func, PCI_CAPABILITY_LIST); 118 for (bytes = 0; bytes < 48 && pos >= 0x40; bytes++) { 119 u8 id; 120 121 pos &= ~3; 122 id = read_pci_config_byte(bus, slot, func, pos+PCI_CAP_LIST_ID); 123 if (id == 0xff) 124 break; 125 if (id == cap) 126 return pos; 127 pos = read_pci_config_byte(bus, slot, func, 128 pos+PCI_CAP_LIST_NEXT); 129 } 130 return 0; 131 } 132 133 /* Read a standard AGPv3 bridge header */ 134 static u32 __init read_agp(int bus, int slot, int func, int cap, u32 *order) 135 { 136 u32 apsize; 137 u32 apsizereg; 138 int nbits; 139 u32 aper_low, aper_hi; 140 u64 aper; 141 u32 old_order; 142 143 printk(KERN_INFO "AGP bridge at %02x:%02x:%02x\n", bus, slot, func); 144 apsizereg = read_pci_config_16(bus, slot, func, cap + 0x14); 145 if (apsizereg == 0xffffffff) { 146 printk(KERN_ERR "APSIZE in AGP bridge unreadable\n"); 147 return 0; 148 } 149 150 /* old_order could be the value from NB gart setting */ 151 old_order = *order; 152 153 apsize = apsizereg & 0xfff; 154 /* Some BIOS use weird encodings not in the AGPv3 table. */ 155 if (apsize & 0xff) 156 apsize |= 0xf00; 157 nbits = hweight16(apsize); 158 *order = 7 - nbits; 159 if ((int)*order < 0) /* < 32MB */ 160 *order = 0; 161 162 aper_low = read_pci_config(bus, slot, func, 0x10); 163 aper_hi = read_pci_config(bus, slot, func, 0x14); 164 aper = (aper_low & ~((1<<22)-1)) | ((u64)aper_hi << 32); 165 166 /* 167 * On some sick chips, APSIZE is 0. It means it wants 4G 168 * so let double check that order, and lets trust AMD NB settings: 169 */ 170 printk(KERN_INFO "Aperture from AGP @ %Lx old size %u MB\n", 171 aper, 32 << old_order); 172 if (aper + (32ULL<<(20 + *order)) > 0x100000000ULL) { 173 printk(KERN_INFO "Aperture size %u MB (APSIZE %x) is not right, using settings from NB\n", 174 32 << *order, apsizereg); 175 *order = old_order; 176 } 177 178 printk(KERN_INFO "Aperture from AGP @ %Lx size %u MB (APSIZE %x)\n", 179 aper, 32 << *order, apsizereg); 180 181 if (!aperture_valid(aper, (32*1024*1024) << *order, 32<<20)) 182 return 0; 183 return (u32)aper; 184 } 185 186 /* 187 * Look for an AGP bridge. Windows only expects the aperture in the 188 * AGP bridge and some BIOS forget to initialize the Northbridge too. 189 * Work around this here. 190 * 191 * Do an PCI bus scan by hand because we're running before the PCI 192 * subsystem. 193 * 194 * All AMD AGP bridges are AGPv3 compliant, so we can do this scan 195 * generically. It's probably overkill to always scan all slots because 196 * the AGP bridges should be always an own bus on the HT hierarchy, 197 * but do it here for future safety. 198 */ 199 static u32 __init search_agp_bridge(u32 *order, int *valid_agp) 200 { 201 int bus, slot, func; 202 203 /* Poor man's PCI discovery */ 204 for (bus = 0; bus < 256; bus++) { 205 for (slot = 0; slot < 32; slot++) { 206 for (func = 0; func < 8; func++) { 207 u32 class, cap; 208 u8 type; 209 class = read_pci_config(bus, slot, func, 210 PCI_CLASS_REVISION); 211 if (class == 0xffffffff) 212 break; 213 214 switch (class >> 16) { 215 case PCI_CLASS_BRIDGE_HOST: 216 case PCI_CLASS_BRIDGE_OTHER: /* needed? */ 217 /* AGP bridge? */ 218 cap = find_cap(bus, slot, func, 219 PCI_CAP_ID_AGP); 220 if (!cap) 221 break; 222 *valid_agp = 1; 223 return read_agp(bus, slot, func, cap, 224 order); 225 } 226 227 /* No multi-function device? */ 228 type = read_pci_config_byte(bus, slot, func, 229 PCI_HEADER_TYPE); 230 if (!(type & 0x80)) 231 break; 232 } 233 } 234 } 235 printk(KERN_INFO "No AGP bridge found\n"); 236 237 return 0; 238 } 239 240 static int gart_fix_e820 __initdata = 1; 241 242 static int __init parse_gart_mem(char *p) 243 { 244 if (!p) 245 return -EINVAL; 246 247 if (!strncmp(p, "off", 3)) 248 gart_fix_e820 = 0; 249 else if (!strncmp(p, "on", 2)) 250 gart_fix_e820 = 1; 251 252 return 0; 253 } 254 early_param("gart_fix_e820", parse_gart_mem); 255 256 void __init early_gart_iommu_check(void) 257 { 258 /* 259 * in case it is enabled before, esp for kexec/kdump, 260 * previous kernel already enable that. memset called 261 * by allocate_aperture/__alloc_bootmem_nopanic cause restart. 262 * or second kernel have different position for GART hole. and new 263 * kernel could use hole as RAM that is still used by GART set by 264 * first kernel 265 * or BIOS forget to put that in reserved. 266 * try to update e820 to make that region as reserved. 267 */ 268 u32 agp_aper_order = 0; 269 int i, fix, slot, valid_agp = 0; 270 u32 ctl; 271 u32 aper_size = 0, aper_order = 0, last_aper_order = 0; 272 u64 aper_base = 0, last_aper_base = 0; 273 int aper_enabled = 0, last_aper_enabled = 0, last_valid = 0; 274 275 if (!early_pci_allowed()) 276 return; 277 278 /* This is mostly duplicate of iommu_hole_init */ 279 search_agp_bridge(&agp_aper_order, &valid_agp); 280 281 fix = 0; 282 for (i = 0; amd_nb_bus_dev_ranges[i].dev_limit; i++) { 283 int bus; 284 int dev_base, dev_limit; 285 286 bus = amd_nb_bus_dev_ranges[i].bus; 287 dev_base = amd_nb_bus_dev_ranges[i].dev_base; 288 dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; 289 290 for (slot = dev_base; slot < dev_limit; slot++) { 291 if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) 292 continue; 293 294 ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL); 295 aper_enabled = ctl & GARTEN; 296 aper_order = (ctl >> 1) & 7; 297 aper_size = (32 * 1024 * 1024) << aper_order; 298 aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff; 299 aper_base <<= 25; 300 301 if (last_valid) { 302 if ((aper_order != last_aper_order) || 303 (aper_base != last_aper_base) || 304 (aper_enabled != last_aper_enabled)) { 305 fix = 1; 306 break; 307 } 308 } 309 310 last_aper_order = aper_order; 311 last_aper_base = aper_base; 312 last_aper_enabled = aper_enabled; 313 last_valid = 1; 314 } 315 } 316 317 if (!fix && !aper_enabled) 318 return; 319 320 if (!aper_base || !aper_size || aper_base + aper_size > 0x100000000UL) 321 fix = 1; 322 323 if (gart_fix_e820 && !fix && aper_enabled) { 324 if (e820_any_mapped(aper_base, aper_base + aper_size, 325 E820_RAM)) { 326 /* reserve it, so we can reuse it in second kernel */ 327 printk(KERN_INFO "update e820 for GART\n"); 328 e820_add_region(aper_base, aper_size, E820_RESERVED); 329 update_e820(); 330 } 331 } 332 333 if (valid_agp) 334 return; 335 336 /* disable them all at first */ 337 for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) { 338 int bus; 339 int dev_base, dev_limit; 340 341 bus = amd_nb_bus_dev_ranges[i].bus; 342 dev_base = amd_nb_bus_dev_ranges[i].dev_base; 343 dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; 344 345 for (slot = dev_base; slot < dev_limit; slot++) { 346 if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) 347 continue; 348 349 ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL); 350 ctl &= ~GARTEN; 351 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl); 352 } 353 } 354 355 } 356 357 static int __initdata printed_gart_size_msg; 358 359 int __init gart_iommu_hole_init(void) 360 { 361 u32 agp_aper_base = 0, agp_aper_order = 0; 362 u32 aper_size, aper_alloc = 0, aper_order = 0, last_aper_order = 0; 363 u64 aper_base, last_aper_base = 0; 364 int fix, slot, valid_agp = 0; 365 int i, node; 366 367 if (gart_iommu_aperture_disabled || !fix_aperture || 368 !early_pci_allowed()) 369 return -ENODEV; 370 371 printk(KERN_INFO "Checking aperture...\n"); 372 373 if (!fallback_aper_force) 374 agp_aper_base = search_agp_bridge(&agp_aper_order, &valid_agp); 375 376 fix = 0; 377 node = 0; 378 for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) { 379 int bus; 380 int dev_base, dev_limit; 381 u32 ctl; 382 383 bus = amd_nb_bus_dev_ranges[i].bus; 384 dev_base = amd_nb_bus_dev_ranges[i].dev_base; 385 dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; 386 387 for (slot = dev_base; slot < dev_limit; slot++) { 388 if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) 389 continue; 390 391 iommu_detected = 1; 392 gart_iommu_aperture = 1; 393 x86_init.iommu.iommu_init = gart_iommu_init; 394 395 ctl = read_pci_config(bus, slot, 3, 396 AMD64_GARTAPERTURECTL); 397 398 /* 399 * Before we do anything else disable the GART. It may 400 * still be enabled if we boot into a crash-kernel here. 401 * Reconfiguring the GART while it is enabled could have 402 * unknown side-effects. 403 */ 404 ctl &= ~GARTEN; 405 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl); 406 407 aper_order = (ctl >> 1) & 7; 408 aper_size = (32 * 1024 * 1024) << aper_order; 409 aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff; 410 aper_base <<= 25; 411 412 printk(KERN_INFO "Node %d: aperture @ %Lx size %u MB\n", 413 node, aper_base, aper_size >> 20); 414 node++; 415 416 if (!aperture_valid(aper_base, aper_size, 64<<20)) { 417 if (valid_agp && agp_aper_base && 418 agp_aper_base == aper_base && 419 agp_aper_order == aper_order) { 420 /* the same between two setting from NB and agp */ 421 if (!no_iommu && 422 max_pfn > MAX_DMA32_PFN && 423 !printed_gart_size_msg) { 424 printk(KERN_ERR "you are using iommu with agp, but GART size is less than 64M\n"); 425 printk(KERN_ERR "please increase GART size in your BIOS setup\n"); 426 printk(KERN_ERR "if BIOS doesn't have that option, contact your HW vendor!\n"); 427 printed_gart_size_msg = 1; 428 } 429 } else { 430 fix = 1; 431 goto out; 432 } 433 } 434 435 if ((last_aper_order && aper_order != last_aper_order) || 436 (last_aper_base && aper_base != last_aper_base)) { 437 fix = 1; 438 goto out; 439 } 440 last_aper_order = aper_order; 441 last_aper_base = aper_base; 442 } 443 } 444 445 out: 446 if (!fix && !fallback_aper_force) { 447 if (last_aper_base) { 448 unsigned long n = (32 * 1024 * 1024) << last_aper_order; 449 450 insert_aperture_resource((u32)last_aper_base, n); 451 return 1; 452 } 453 return 0; 454 } 455 456 if (!fallback_aper_force) { 457 aper_alloc = agp_aper_base; 458 aper_order = agp_aper_order; 459 } 460 461 if (aper_alloc) { 462 /* Got the aperture from the AGP bridge */ 463 } else if ((!no_iommu && max_pfn > MAX_DMA32_PFN) || 464 force_iommu || 465 valid_agp || 466 fallback_aper_force) { 467 printk(KERN_INFO 468 "Your BIOS doesn't leave a aperture memory hole\n"); 469 printk(KERN_INFO 470 "Please enable the IOMMU option in the BIOS setup\n"); 471 printk(KERN_INFO 472 "This costs you %d MB of RAM\n", 473 32 << fallback_aper_order); 474 475 aper_order = fallback_aper_order; 476 aper_alloc = allocate_aperture(); 477 if (!aper_alloc) { 478 /* 479 * Could disable AGP and IOMMU here, but it's 480 * probably not worth it. But the later users 481 * cannot deal with bad apertures and turning 482 * on the aperture over memory causes very 483 * strange problems, so it's better to panic 484 * early. 485 */ 486 panic("Not enough memory for aperture"); 487 } 488 } else { 489 return 0; 490 } 491 492 /* Fix up the north bridges */ 493 for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) { 494 int bus, dev_base, dev_limit; 495 496 /* 497 * Don't enable translation yet but enable GART IO and CPU 498 * accesses and set DISTLBWALKPRB since GART table memory is UC. 499 */ 500 u32 ctl = aper_order << 1; 501 502 bus = amd_nb_bus_dev_ranges[i].bus; 503 dev_base = amd_nb_bus_dev_ranges[i].dev_base; 504 dev_limit = amd_nb_bus_dev_ranges[i].dev_limit; 505 for (slot = dev_base; slot < dev_limit; slot++) { 506 if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00))) 507 continue; 508 509 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl); 510 write_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE, aper_alloc >> 25); 511 } 512 } 513 514 set_up_gart_resume(aper_order, aper_alloc); 515 516 return 1; 517 } 518