1 /* 2 * Hypervisor supplied "24x7" performance counter support 3 * 4 * Author: Cody P Schafer <cody@linux.vnet.ibm.com> 5 * Copyright 2014 IBM Corporation. 6 * 7 * This program is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU General Public License 9 * as published by the Free Software Foundation; either version 10 * 2 of the License, or (at your option) any later version. 11 */ 12 13 #define pr_fmt(fmt) "hv-24x7: " fmt 14 15 #include <linux/perf_event.h> 16 #include <linux/rbtree.h> 17 #include <linux/module.h> 18 #include <linux/slab.h> 19 #include <linux/vmalloc.h> 20 21 #include <asm/firmware.h> 22 #include <asm/hvcall.h> 23 #include <asm/io.h> 24 #include <linux/byteorder/generic.h> 25 26 #include "hv-24x7.h" 27 #include "hv-24x7-catalog.h" 28 #include "hv-common.h" 29 30 static const char *event_domain_suffix(unsigned domain) 31 { 32 switch (domain) { 33 #define DOMAIN(n, v, x, c) \ 34 case HV_PERF_DOMAIN_##n: \ 35 return "__" #n; 36 #include "hv-24x7-domains.h" 37 #undef DOMAIN 38 default: 39 WARN(1, "unknown domain %d\n", domain); 40 return "__UNKNOWN_DOMAIN_SUFFIX"; 41 } 42 } 43 44 static bool domain_is_valid(unsigned domain) 45 { 46 switch (domain) { 47 #define DOMAIN(n, v, x, c) \ 48 case HV_PERF_DOMAIN_##n: \ 49 /* fall through */ 50 #include "hv-24x7-domains.h" 51 #undef DOMAIN 52 return true; 53 default: 54 return false; 55 } 56 } 57 58 static bool is_physical_domain(unsigned domain) 59 { 60 switch (domain) { 61 #define DOMAIN(n, v, x, c) \ 62 case HV_PERF_DOMAIN_##n: \ 63 return c; 64 #include "hv-24x7-domains.h" 65 #undef DOMAIN 66 default: 67 return false; 68 } 69 } 70 71 static bool catalog_entry_domain_is_valid(unsigned domain) 72 { 73 return is_physical_domain(domain); 74 } 75 76 /* 77 * TODO: Merging events: 78 * - Think of the hcall as an interface to a 4d array of counters: 79 * - x = domains 80 * - y = indexes in the domain (core, chip, vcpu, node, etc) 81 * - z = offset into the counter space 82 * - w = lpars (guest vms, "logical partitions") 83 * - A single request is: x,y,y_last,z,z_last,w,w_last 84 * - this means we can retrieve a rectangle of counters in y,z for a single x. 85 * 86 * - Things to consider (ignoring w): 87 * - input cost_per_request = 16 88 * - output cost_per_result(ys,zs) = 8 + 8 * ys + ys * zs 89 * - limited number of requests per hcall (must fit into 4K bytes) 90 * - 4k = 16 [buffer header] - 16 [request size] * request_count 91 * - 255 requests per hcall 92 * - sometimes it will be more efficient to read extra data and discard 93 */ 94 95 /* 96 * Example usage: 97 * perf stat -e 'hv_24x7/domain=2,offset=8,vcpu=0,lpar=0xffffffff/' 98 */ 99 100 /* u3 0-6, one of HV_24X7_PERF_DOMAIN */ 101 EVENT_DEFINE_RANGE_FORMAT(domain, config, 0, 3); 102 /* u16 */ 103 EVENT_DEFINE_RANGE_FORMAT(core, config, 16, 31); 104 EVENT_DEFINE_RANGE_FORMAT(vcpu, config, 16, 31); 105 /* u32, see "data_offset" */ 106 EVENT_DEFINE_RANGE_FORMAT(offset, config, 32, 63); 107 /* u16 */ 108 EVENT_DEFINE_RANGE_FORMAT(lpar, config1, 0, 15); 109 110 EVENT_DEFINE_RANGE(reserved1, config, 4, 15); 111 EVENT_DEFINE_RANGE(reserved2, config1, 16, 63); 112 EVENT_DEFINE_RANGE(reserved3, config2, 0, 63); 113 114 static struct attribute *format_attrs[] = { 115 &format_attr_domain.attr, 116 &format_attr_offset.attr, 117 &format_attr_core.attr, 118 &format_attr_vcpu.attr, 119 &format_attr_lpar.attr, 120 NULL, 121 }; 122 123 static struct attribute_group format_group = { 124 .name = "format", 125 .attrs = format_attrs, 126 }; 127 128 static struct attribute_group event_group = { 129 .name = "events", 130 /* .attrs is set in init */ 131 }; 132 133 static struct attribute_group event_desc_group = { 134 .name = "event_descs", 135 /* .attrs is set in init */ 136 }; 137 138 static struct attribute_group event_long_desc_group = { 139 .name = "event_long_descs", 140 /* .attrs is set in init */ 141 }; 142 143 static struct kmem_cache *hv_page_cache; 144 145 /* 146 * request_buffer and result_buffer are not required to be 4k aligned, 147 * but are not allowed to cross any 4k boundary. Aligning them to 4k is 148 * the simplest way to ensure that. 149 */ 150 #define H24x7_DATA_BUFFER_SIZE 4096 151 DEFINE_PER_CPU(char, hv_24x7_reqb[H24x7_DATA_BUFFER_SIZE]) __aligned(4096); 152 DEFINE_PER_CPU(char, hv_24x7_resb[H24x7_DATA_BUFFER_SIZE]) __aligned(4096); 153 154 static char *event_name(struct hv_24x7_event_data *ev, int *len) 155 { 156 *len = be16_to_cpu(ev->event_name_len) - 2; 157 return (char *)ev->remainder; 158 } 159 160 static char *event_desc(struct hv_24x7_event_data *ev, int *len) 161 { 162 unsigned nl = be16_to_cpu(ev->event_name_len); 163 __be16 *desc_len = (__be16 *)(ev->remainder + nl - 2); 164 165 *len = be16_to_cpu(*desc_len) - 2; 166 return (char *)ev->remainder + nl; 167 } 168 169 static char *event_long_desc(struct hv_24x7_event_data *ev, int *len) 170 { 171 unsigned nl = be16_to_cpu(ev->event_name_len); 172 __be16 *desc_len_ = (__be16 *)(ev->remainder + nl - 2); 173 unsigned desc_len = be16_to_cpu(*desc_len_); 174 __be16 *long_desc_len = (__be16 *)(ev->remainder + nl + desc_len - 2); 175 176 *len = be16_to_cpu(*long_desc_len) - 2; 177 return (char *)ev->remainder + nl + desc_len; 178 } 179 180 static bool event_fixed_portion_is_within(struct hv_24x7_event_data *ev, 181 void *end) 182 { 183 void *start = ev; 184 185 return (start + offsetof(struct hv_24x7_event_data, remainder)) < end; 186 } 187 188 /* 189 * Things we don't check: 190 * - padding for desc, name, and long/detailed desc is required to be '\0' 191 * bytes. 192 * 193 * Return NULL if we pass end, 194 * Otherwise return the address of the byte just following the event. 195 */ 196 static void *event_end(struct hv_24x7_event_data *ev, void *end) 197 { 198 void *start = ev; 199 __be16 *dl_, *ldl_; 200 unsigned dl, ldl; 201 unsigned nl = be16_to_cpu(ev->event_name_len); 202 203 if (nl < 2) { 204 pr_debug("%s: name length too short: %d", __func__, nl); 205 return NULL; 206 } 207 208 if (start + nl > end) { 209 pr_debug("%s: start=%p + nl=%u > end=%p", 210 __func__, start, nl, end); 211 return NULL; 212 } 213 214 dl_ = (__be16 *)(ev->remainder + nl - 2); 215 if (!IS_ALIGNED((uintptr_t)dl_, 2)) 216 pr_warn("desc len not aligned %p", dl_); 217 dl = be16_to_cpu(*dl_); 218 if (dl < 2) { 219 pr_debug("%s: desc len too short: %d", __func__, dl); 220 return NULL; 221 } 222 223 if (start + nl + dl > end) { 224 pr_debug("%s: (start=%p + nl=%u + dl=%u)=%p > end=%p", 225 __func__, start, nl, dl, start + nl + dl, end); 226 return NULL; 227 } 228 229 ldl_ = (__be16 *)(ev->remainder + nl + dl - 2); 230 if (!IS_ALIGNED((uintptr_t)ldl_, 2)) 231 pr_warn("long desc len not aligned %p", ldl_); 232 ldl = be16_to_cpu(*ldl_); 233 if (ldl < 2) { 234 pr_debug("%s: long desc len too short (ldl=%u)", 235 __func__, ldl); 236 return NULL; 237 } 238 239 if (start + nl + dl + ldl > end) { 240 pr_debug("%s: start=%p + nl=%u + dl=%u + ldl=%u > end=%p", 241 __func__, start, nl, dl, ldl, end); 242 return NULL; 243 } 244 245 return start + nl + dl + ldl; 246 } 247 248 static unsigned long h_get_24x7_catalog_page_(unsigned long phys_4096, 249 unsigned long version, 250 unsigned long index) 251 { 252 pr_devel("h_get_24x7_catalog_page(0x%lx, %lu, %lu)", 253 phys_4096, version, index); 254 255 WARN_ON(!IS_ALIGNED(phys_4096, 4096)); 256 257 return plpar_hcall_norets(H_GET_24X7_CATALOG_PAGE, 258 phys_4096, version, index); 259 } 260 261 static unsigned long h_get_24x7_catalog_page(char page[], 262 u64 version, u32 index) 263 { 264 return h_get_24x7_catalog_page_(virt_to_phys(page), 265 version, index); 266 } 267 268 static unsigned core_domains[] = { 269 HV_PERF_DOMAIN_PHYS_CORE, 270 HV_PERF_DOMAIN_VCPU_HOME_CORE, 271 HV_PERF_DOMAIN_VCPU_HOME_CHIP, 272 HV_PERF_DOMAIN_VCPU_HOME_NODE, 273 HV_PERF_DOMAIN_VCPU_REMOTE_NODE, 274 }; 275 /* chip event data always yeilds a single event, core yeilds multiple */ 276 #define MAX_EVENTS_PER_EVENT_DATA ARRAY_SIZE(core_domains) 277 278 static char *event_fmt(struct hv_24x7_event_data *event, unsigned domain) 279 { 280 const char *sindex; 281 const char *lpar; 282 283 if (is_physical_domain(domain)) { 284 lpar = "0x0"; 285 sindex = "core"; 286 } else { 287 lpar = "?"; 288 sindex = "vcpu"; 289 } 290 291 return kasprintf(GFP_KERNEL, 292 "domain=0x%x,offset=0x%x,%s=?,lpar=%s", 293 domain, 294 be16_to_cpu(event->event_counter_offs) + 295 be16_to_cpu(event->event_group_record_offs), 296 sindex, 297 lpar); 298 } 299 300 /* Avoid trusting fw to NUL terminate strings */ 301 static char *memdup_to_str(char *maybe_str, int max_len, gfp_t gfp) 302 { 303 return kasprintf(gfp, "%.*s", max_len, maybe_str); 304 } 305 306 static ssize_t device_show_string(struct device *dev, 307 struct device_attribute *attr, char *buf) 308 { 309 struct dev_ext_attribute *d; 310 311 d = container_of(attr, struct dev_ext_attribute, attr); 312 313 return sprintf(buf, "%s\n", (char *)d->var); 314 } 315 316 static struct attribute *device_str_attr_create_(char *name, char *str) 317 { 318 struct dev_ext_attribute *attr = kzalloc(sizeof(*attr), GFP_KERNEL); 319 320 if (!attr) 321 return NULL; 322 323 sysfs_attr_init(&attr->attr.attr); 324 325 attr->var = str; 326 attr->attr.attr.name = name; 327 attr->attr.attr.mode = 0444; 328 attr->attr.show = device_show_string; 329 330 return &attr->attr.attr; 331 } 332 333 static struct attribute *device_str_attr_create(char *name, int name_max, 334 int name_nonce, 335 char *str, size_t str_max) 336 { 337 char *n; 338 char *s = memdup_to_str(str, str_max, GFP_KERNEL); 339 struct attribute *a; 340 341 if (!s) 342 return NULL; 343 344 if (!name_nonce) 345 n = kasprintf(GFP_KERNEL, "%.*s", name_max, name); 346 else 347 n = kasprintf(GFP_KERNEL, "%.*s__%d", name_max, name, 348 name_nonce); 349 if (!n) 350 goto out_s; 351 352 a = device_str_attr_create_(n, s); 353 if (!a) 354 goto out_n; 355 356 return a; 357 out_n: 358 kfree(n); 359 out_s: 360 kfree(s); 361 return NULL; 362 } 363 364 static void device_str_attr_destroy(struct attribute *attr) 365 { 366 struct dev_ext_attribute *d; 367 368 d = container_of(attr, struct dev_ext_attribute, attr.attr); 369 kfree(d->var); 370 kfree(d->attr.attr.name); 371 kfree(d); 372 } 373 374 static struct attribute *event_to_attr(unsigned ix, 375 struct hv_24x7_event_data *event, 376 unsigned domain, 377 int nonce) 378 { 379 int event_name_len; 380 char *ev_name, *a_ev_name, *val; 381 const char *ev_suffix; 382 struct attribute *attr; 383 384 if (!domain_is_valid(domain)) { 385 pr_warn("catalog event %u has invalid domain %u\n", 386 ix, domain); 387 return NULL; 388 } 389 390 val = event_fmt(event, domain); 391 if (!val) 392 return NULL; 393 394 ev_suffix = event_domain_suffix(domain); 395 ev_name = event_name(event, &event_name_len); 396 if (!nonce) 397 a_ev_name = kasprintf(GFP_KERNEL, "%.*s%s", 398 (int)event_name_len, ev_name, ev_suffix); 399 else 400 a_ev_name = kasprintf(GFP_KERNEL, "%.*s%s__%d", 401 (int)event_name_len, ev_name, ev_suffix, nonce); 402 403 if (!a_ev_name) 404 goto out_val; 405 406 attr = device_str_attr_create_(a_ev_name, val); 407 if (!attr) 408 goto out_name; 409 410 return attr; 411 out_name: 412 kfree(a_ev_name); 413 out_val: 414 kfree(val); 415 return NULL; 416 } 417 418 static struct attribute *event_to_desc_attr(struct hv_24x7_event_data *event, 419 int nonce) 420 { 421 int nl, dl; 422 char *name = event_name(event, &nl); 423 char *desc = event_desc(event, &dl); 424 425 /* If there isn't a description, don't create the sysfs file */ 426 if (!dl) 427 return NULL; 428 429 return device_str_attr_create(name, nl, nonce, desc, dl); 430 } 431 432 static struct attribute * 433 event_to_long_desc_attr(struct hv_24x7_event_data *event, int nonce) 434 { 435 int nl, dl; 436 char *name = event_name(event, &nl); 437 char *desc = event_long_desc(event, &dl); 438 439 /* If there isn't a description, don't create the sysfs file */ 440 if (!dl) 441 return NULL; 442 443 return device_str_attr_create(name, nl, nonce, desc, dl); 444 } 445 446 static ssize_t event_data_to_attrs(unsigned ix, struct attribute **attrs, 447 struct hv_24x7_event_data *event, int nonce) 448 { 449 unsigned i; 450 451 switch (event->domain) { 452 case HV_PERF_DOMAIN_PHYS_CHIP: 453 *attrs = event_to_attr(ix, event, event->domain, nonce); 454 return 1; 455 case HV_PERF_DOMAIN_PHYS_CORE: 456 for (i = 0; i < ARRAY_SIZE(core_domains); i++) { 457 attrs[i] = event_to_attr(ix, event, core_domains[i], 458 nonce); 459 if (!attrs[i]) { 460 pr_warn("catalog event %u: individual attr %u " 461 "creation failure\n", ix, i); 462 for (; i; i--) 463 device_str_attr_destroy(attrs[i - 1]); 464 return -1; 465 } 466 } 467 return i; 468 default: 469 pr_warn("catalog event %u: domain %u is not allowed in the " 470 "catalog\n", ix, event->domain); 471 return -1; 472 } 473 } 474 475 static size_t event_to_attr_ct(struct hv_24x7_event_data *event) 476 { 477 switch (event->domain) { 478 case HV_PERF_DOMAIN_PHYS_CHIP: 479 return 1; 480 case HV_PERF_DOMAIN_PHYS_CORE: 481 return ARRAY_SIZE(core_domains); 482 default: 483 return 0; 484 } 485 } 486 487 static unsigned long vmalloc_to_phys(void *v) 488 { 489 struct page *p = vmalloc_to_page(v); 490 491 BUG_ON(!p); 492 return page_to_phys(p) + offset_in_page(v); 493 } 494 495 /* */ 496 struct event_uniq { 497 struct rb_node node; 498 const char *name; 499 int nl; 500 unsigned ct; 501 unsigned domain; 502 }; 503 504 static int memord(const void *d1, size_t s1, const void *d2, size_t s2) 505 { 506 if (s1 < s2) 507 return 1; 508 if (s2 > s1) 509 return -1; 510 511 return memcmp(d1, d2, s1); 512 } 513 514 static int ev_uniq_ord(const void *v1, size_t s1, unsigned d1, const void *v2, 515 size_t s2, unsigned d2) 516 { 517 int r = memord(v1, s1, v2, s2); 518 519 if (r) 520 return r; 521 if (d1 > d2) 522 return 1; 523 if (d2 > d1) 524 return -1; 525 return 0; 526 } 527 528 static int event_uniq_add(struct rb_root *root, const char *name, int nl, 529 unsigned domain) 530 { 531 struct rb_node **new = &(root->rb_node), *parent = NULL; 532 struct event_uniq *data; 533 534 /* Figure out where to put new node */ 535 while (*new) { 536 struct event_uniq *it; 537 int result; 538 539 it = container_of(*new, struct event_uniq, node); 540 result = ev_uniq_ord(name, nl, domain, it->name, it->nl, 541 it->domain); 542 543 parent = *new; 544 if (result < 0) 545 new = &((*new)->rb_left); 546 else if (result > 0) 547 new = &((*new)->rb_right); 548 else { 549 it->ct++; 550 pr_info("found a duplicate event %.*s, ct=%u\n", nl, 551 name, it->ct); 552 return it->ct; 553 } 554 } 555 556 data = kmalloc(sizeof(*data), GFP_KERNEL); 557 if (!data) 558 return -ENOMEM; 559 560 *data = (struct event_uniq) { 561 .name = name, 562 .nl = nl, 563 .ct = 0, 564 .domain = domain, 565 }; 566 567 /* Add new node and rebalance tree. */ 568 rb_link_node(&data->node, parent, new); 569 rb_insert_color(&data->node, root); 570 571 /* data->ct */ 572 return 0; 573 } 574 575 static void event_uniq_destroy(struct rb_root *root) 576 { 577 /* 578 * the strings we point to are in the giant block of memory filled by 579 * the catalog, and are freed separately. 580 */ 581 struct event_uniq *pos, *n; 582 583 rbtree_postorder_for_each_entry_safe(pos, n, root, node) 584 kfree(pos); 585 } 586 587 588 /* 589 * ensure the event structure's sizes are self consistent and don't cause us to 590 * read outside of the event 591 * 592 * On success, return the event length in bytes. 593 * Otherwise, return -1 (and print as appropriate). 594 */ 595 static ssize_t catalog_event_len_validate(struct hv_24x7_event_data *event, 596 size_t event_idx, 597 size_t event_data_bytes, 598 size_t event_entry_count, 599 size_t offset, void *end) 600 { 601 ssize_t ev_len; 602 void *ev_end, *calc_ev_end; 603 604 if (offset >= event_data_bytes) 605 return -1; 606 607 if (event_idx >= event_entry_count) { 608 pr_devel("catalog event data has %zu bytes of padding after last event\n", 609 event_data_bytes - offset); 610 return -1; 611 } 612 613 if (!event_fixed_portion_is_within(event, end)) { 614 pr_warn("event %zu fixed portion is not within range\n", 615 event_idx); 616 return -1; 617 } 618 619 ev_len = be16_to_cpu(event->length); 620 621 if (ev_len % 16) 622 pr_info("event %zu has length %zu not divisible by 16: event=%pK\n", 623 event_idx, ev_len, event); 624 625 ev_end = (__u8 *)event + ev_len; 626 if (ev_end > end) { 627 pr_warn("event %zu has .length=%zu, ends after buffer end: ev_end=%pK > end=%pK, offset=%zu\n", 628 event_idx, ev_len, ev_end, end, 629 offset); 630 return -1; 631 } 632 633 calc_ev_end = event_end(event, end); 634 if (!calc_ev_end) { 635 pr_warn("event %zu has a calculated length which exceeds buffer length %zu: event=%pK end=%pK, offset=%zu\n", 636 event_idx, event_data_bytes, event, end, 637 offset); 638 return -1; 639 } 640 641 if (calc_ev_end > ev_end) { 642 pr_warn("event %zu exceeds it's own length: event=%pK, end=%pK, offset=%zu, calc_ev_end=%pK\n", 643 event_idx, event, ev_end, offset, calc_ev_end); 644 return -1; 645 } 646 647 return ev_len; 648 } 649 650 #define MAX_4K (SIZE_MAX / 4096) 651 652 static int create_events_from_catalog(struct attribute ***events_, 653 struct attribute ***event_descs_, 654 struct attribute ***event_long_descs_) 655 { 656 unsigned long hret; 657 size_t catalog_len, catalog_page_len, event_entry_count, 658 event_data_len, event_data_offs, 659 event_data_bytes, junk_events, event_idx, event_attr_ct, i, 660 attr_max, event_idx_last, desc_ct, long_desc_ct; 661 ssize_t ct, ev_len; 662 uint32_t catalog_version_num; 663 struct attribute **events, **event_descs, **event_long_descs; 664 struct hv_24x7_catalog_page_0 *page_0 = 665 kmem_cache_alloc(hv_page_cache, GFP_KERNEL); 666 void *page = page_0; 667 void *event_data, *end; 668 struct hv_24x7_event_data *event; 669 struct rb_root ev_uniq = RB_ROOT; 670 int ret = 0; 671 672 if (!page) { 673 ret = -ENOMEM; 674 goto e_out; 675 } 676 677 hret = h_get_24x7_catalog_page(page, 0, 0); 678 if (hret) { 679 ret = -EIO; 680 goto e_free; 681 } 682 683 catalog_version_num = be64_to_cpu(page_0->version); 684 catalog_page_len = be32_to_cpu(page_0->length); 685 686 if (MAX_4K < catalog_page_len) { 687 pr_err("invalid page count: %zu\n", catalog_page_len); 688 ret = -EIO; 689 goto e_free; 690 } 691 692 catalog_len = catalog_page_len * 4096; 693 694 event_entry_count = be16_to_cpu(page_0->event_entry_count); 695 event_data_offs = be16_to_cpu(page_0->event_data_offs); 696 event_data_len = be16_to_cpu(page_0->event_data_len); 697 698 pr_devel("cv %zu cl %zu eec %zu edo %zu edl %zu\n", 699 (size_t)catalog_version_num, catalog_len, 700 event_entry_count, event_data_offs, event_data_len); 701 702 if ((MAX_4K < event_data_len) 703 || (MAX_4K < event_data_offs) 704 || (MAX_4K - event_data_offs < event_data_len)) { 705 pr_err("invalid event data offs %zu and/or len %zu\n", 706 event_data_offs, event_data_len); 707 ret = -EIO; 708 goto e_free; 709 } 710 711 if ((event_data_offs + event_data_len) > catalog_page_len) { 712 pr_err("event data %zu-%zu does not fit inside catalog 0-%zu\n", 713 event_data_offs, 714 event_data_offs + event_data_len, 715 catalog_page_len); 716 ret = -EIO; 717 goto e_free; 718 } 719 720 if (SIZE_MAX / MAX_EVENTS_PER_EVENT_DATA - 1 < event_entry_count) { 721 pr_err("event_entry_count %zu is invalid\n", 722 event_entry_count); 723 ret = -EIO; 724 goto e_free; 725 } 726 727 event_data_bytes = event_data_len * 4096; 728 729 /* 730 * event data can span several pages, events can cross between these 731 * pages. Use vmalloc to make this easier. 732 */ 733 event_data = vmalloc(event_data_bytes); 734 if (!event_data) { 735 pr_err("could not allocate event data\n"); 736 ret = -ENOMEM; 737 goto e_free; 738 } 739 740 end = event_data + event_data_bytes; 741 742 /* 743 * using vmalloc_to_phys() like this only works if PAGE_SIZE is 744 * divisible by 4096 745 */ 746 BUILD_BUG_ON(PAGE_SIZE % 4096); 747 748 for (i = 0; i < event_data_len; i++) { 749 hret = h_get_24x7_catalog_page_( 750 vmalloc_to_phys(event_data + i * 4096), 751 catalog_version_num, 752 i + event_data_offs); 753 if (hret) { 754 pr_err("failed to get event data in page %zu\n", 755 i + event_data_offs); 756 ret = -EIO; 757 goto e_event_data; 758 } 759 } 760 761 /* 762 * scan the catalog to determine the number of attributes we need, and 763 * verify it at the same time. 764 */ 765 for (junk_events = 0, event = event_data, event_idx = 0, attr_max = 0; 766 ; 767 event_idx++, event = (void *)event + ev_len) { 768 size_t offset = (void *)event - (void *)event_data; 769 char *name; 770 int nl; 771 772 ev_len = catalog_event_len_validate(event, event_idx, 773 event_data_bytes, 774 event_entry_count, 775 offset, end); 776 if (ev_len < 0) 777 break; 778 779 name = event_name(event, &nl); 780 781 if (event->event_group_record_len == 0) { 782 pr_devel("invalid event %zu (%.*s): group_record_len == 0, skipping\n", 783 event_idx, nl, name); 784 junk_events++; 785 continue; 786 } 787 788 if (!catalog_entry_domain_is_valid(event->domain)) { 789 pr_info("event %zu (%.*s) has invalid domain %d\n", 790 event_idx, nl, name, event->domain); 791 junk_events++; 792 continue; 793 } 794 795 attr_max += event_to_attr_ct(event); 796 } 797 798 event_idx_last = event_idx; 799 if (event_idx_last != event_entry_count) 800 pr_warn("event buffer ended before listed # of events were parsed (got %zu, wanted %zu, junk %zu)\n", 801 event_idx_last, event_entry_count, junk_events); 802 803 events = kmalloc_array(attr_max + 1, sizeof(*events), GFP_KERNEL); 804 if (!events) { 805 ret = -ENOMEM; 806 goto e_event_data; 807 } 808 809 event_descs = kmalloc_array(event_idx + 1, sizeof(*event_descs), 810 GFP_KERNEL); 811 if (!event_descs) { 812 ret = -ENOMEM; 813 goto e_event_attrs; 814 } 815 816 event_long_descs = kmalloc_array(event_idx + 1, 817 sizeof(*event_long_descs), GFP_KERNEL); 818 if (!event_long_descs) { 819 ret = -ENOMEM; 820 goto e_event_descs; 821 } 822 823 /* Iterate over the catalog filling in the attribute vector */ 824 for (junk_events = 0, event_attr_ct = 0, desc_ct = 0, long_desc_ct = 0, 825 event = event_data, event_idx = 0; 826 event_idx < event_idx_last; 827 event_idx++, ev_len = be16_to_cpu(event->length), 828 event = (void *)event + ev_len) { 829 char *name; 830 int nl; 831 int nonce; 832 /* 833 * these are the only "bad" events that are intermixed and that 834 * we can ignore without issue. make sure to skip them here 835 */ 836 if (event->event_group_record_len == 0) 837 continue; 838 if (!catalog_entry_domain_is_valid(event->domain)) 839 continue; 840 841 name = event_name(event, &nl); 842 nonce = event_uniq_add(&ev_uniq, name, nl, event->domain); 843 ct = event_data_to_attrs(event_idx, events + event_attr_ct, 844 event, nonce); 845 if (ct <= 0) { 846 pr_warn("event %zu (%.*s) creation failure, skipping\n", 847 event_idx, nl, name); 848 junk_events++; 849 } else { 850 event_attr_ct += ct; 851 event_descs[desc_ct] = event_to_desc_attr(event, nonce); 852 if (event_descs[desc_ct]) 853 desc_ct++; 854 event_long_descs[long_desc_ct] = 855 event_to_long_desc_attr(event, nonce); 856 if (event_long_descs[long_desc_ct]) 857 long_desc_ct++; 858 } 859 } 860 861 pr_info("read %zu catalog entries, created %zu event attrs (%zu failures), %zu descs\n", 862 event_idx, event_attr_ct, junk_events, desc_ct); 863 864 events[event_attr_ct] = NULL; 865 event_descs[desc_ct] = NULL; 866 event_long_descs[long_desc_ct] = NULL; 867 868 event_uniq_destroy(&ev_uniq); 869 vfree(event_data); 870 kmem_cache_free(hv_page_cache, page); 871 872 *events_ = events; 873 *event_descs_ = event_descs; 874 *event_long_descs_ = event_long_descs; 875 return 0; 876 877 e_event_descs: 878 kfree(event_descs); 879 e_event_attrs: 880 kfree(events); 881 e_event_data: 882 vfree(event_data); 883 e_free: 884 kmem_cache_free(hv_page_cache, page); 885 e_out: 886 *events_ = NULL; 887 *event_descs_ = NULL; 888 *event_long_descs_ = NULL; 889 return ret; 890 } 891 892 static ssize_t catalog_read(struct file *filp, struct kobject *kobj, 893 struct bin_attribute *bin_attr, char *buf, 894 loff_t offset, size_t count) 895 { 896 unsigned long hret; 897 ssize_t ret = 0; 898 size_t catalog_len = 0, catalog_page_len = 0; 899 loff_t page_offset = 0; 900 loff_t offset_in_page; 901 size_t copy_len; 902 uint64_t catalog_version_num = 0; 903 void *page = kmem_cache_alloc(hv_page_cache, GFP_USER); 904 struct hv_24x7_catalog_page_0 *page_0 = page; 905 906 if (!page) 907 return -ENOMEM; 908 909 hret = h_get_24x7_catalog_page(page, 0, 0); 910 if (hret) { 911 ret = -EIO; 912 goto e_free; 913 } 914 915 catalog_version_num = be64_to_cpu(page_0->version); 916 catalog_page_len = be32_to_cpu(page_0->length); 917 catalog_len = catalog_page_len * 4096; 918 919 page_offset = offset / 4096; 920 offset_in_page = offset % 4096; 921 922 if (page_offset >= catalog_page_len) 923 goto e_free; 924 925 if (page_offset != 0) { 926 hret = h_get_24x7_catalog_page(page, catalog_version_num, 927 page_offset); 928 if (hret) { 929 ret = -EIO; 930 goto e_free; 931 } 932 } 933 934 copy_len = 4096 - offset_in_page; 935 if (copy_len > count) 936 copy_len = count; 937 938 memcpy(buf, page+offset_in_page, copy_len); 939 ret = copy_len; 940 941 e_free: 942 if (hret) 943 pr_err("h_get_24x7_catalog_page(ver=%lld, page=%lld) failed:" 944 " rc=%ld\n", 945 catalog_version_num, page_offset, hret); 946 kmem_cache_free(hv_page_cache, page); 947 948 pr_devel("catalog_read: offset=%lld(%lld) count=%zu " 949 "catalog_len=%zu(%zu) => %zd\n", offset, page_offset, 950 count, catalog_len, catalog_page_len, ret); 951 952 return ret; 953 } 954 955 #define PAGE_0_ATTR(_name, _fmt, _expr) \ 956 static ssize_t _name##_show(struct device *dev, \ 957 struct device_attribute *dev_attr, \ 958 char *buf) \ 959 { \ 960 unsigned long hret; \ 961 ssize_t ret = 0; \ 962 void *page = kmem_cache_alloc(hv_page_cache, GFP_USER); \ 963 struct hv_24x7_catalog_page_0 *page_0 = page; \ 964 if (!page) \ 965 return -ENOMEM; \ 966 hret = h_get_24x7_catalog_page(page, 0, 0); \ 967 if (hret) { \ 968 ret = -EIO; \ 969 goto e_free; \ 970 } \ 971 ret = sprintf(buf, _fmt, _expr); \ 972 e_free: \ 973 kmem_cache_free(hv_page_cache, page); \ 974 return ret; \ 975 } \ 976 static DEVICE_ATTR_RO(_name) 977 978 PAGE_0_ATTR(catalog_version, "%lld\n", 979 (unsigned long long)be64_to_cpu(page_0->version)); 980 PAGE_0_ATTR(catalog_len, "%lld\n", 981 (unsigned long long)be32_to_cpu(page_0->length) * 4096); 982 static BIN_ATTR_RO(catalog, 0/* real length varies */); 983 984 static struct bin_attribute *if_bin_attrs[] = { 985 &bin_attr_catalog, 986 NULL, 987 }; 988 989 static struct attribute *if_attrs[] = { 990 &dev_attr_catalog_len.attr, 991 &dev_attr_catalog_version.attr, 992 NULL, 993 }; 994 995 static struct attribute_group if_group = { 996 .name = "interface", 997 .bin_attrs = if_bin_attrs, 998 .attrs = if_attrs, 999 }; 1000 1001 static const struct attribute_group *attr_groups[] = { 1002 &format_group, 1003 &event_group, 1004 &event_desc_group, 1005 &event_long_desc_group, 1006 &if_group, 1007 NULL, 1008 }; 1009 1010 static void log_24x7_hcall(struct hv_24x7_request_buffer *request_buffer, 1011 struct hv_24x7_data_result_buffer *result_buffer, 1012 unsigned long ret) 1013 { 1014 struct hv_24x7_request *req; 1015 1016 req = &request_buffer->requests[0]; 1017 pr_notice_ratelimited("hcall failed: [%d %#x %#x %d] => " 1018 "ret 0x%lx (%ld) detail=0x%x failing ix=%x\n", 1019 req->performance_domain, req->data_offset, 1020 req->starting_ix, req->starting_lpar_ix, ret, ret, 1021 result_buffer->detailed_rc, 1022 result_buffer->failing_request_ix); 1023 } 1024 1025 /* 1026 * Start the process for a new H_GET_24x7_DATA hcall. 1027 */ 1028 static void init_24x7_request(struct hv_24x7_request_buffer *request_buffer, 1029 struct hv_24x7_data_result_buffer *result_buffer) 1030 { 1031 1032 memset(request_buffer, 0, 4096); 1033 memset(result_buffer, 0, 4096); 1034 1035 request_buffer->interface_version = HV_24X7_IF_VERSION_CURRENT; 1036 /* memset above set request_buffer->num_requests to 0 */ 1037 } 1038 1039 /* 1040 * Commit (i.e perform) the H_GET_24x7_DATA hcall using the data collected 1041 * by 'init_24x7_request()' and 'add_event_to_24x7_request()'. 1042 */ 1043 static int make_24x7_request(struct hv_24x7_request_buffer *request_buffer, 1044 struct hv_24x7_data_result_buffer *result_buffer) 1045 { 1046 unsigned long ret; 1047 1048 /* 1049 * NOTE: Due to variable number of array elements in request and 1050 * result buffer(s), sizeof() is not reliable. Use the actual 1051 * allocated buffer size, H24x7_DATA_BUFFER_SIZE. 1052 */ 1053 ret = plpar_hcall_norets(H_GET_24X7_DATA, 1054 virt_to_phys(request_buffer), H24x7_DATA_BUFFER_SIZE, 1055 virt_to_phys(result_buffer), H24x7_DATA_BUFFER_SIZE); 1056 1057 if (ret) 1058 log_24x7_hcall(request_buffer, result_buffer, ret); 1059 1060 return ret; 1061 } 1062 1063 /* 1064 * Add the given @event to the next slot in the 24x7 request_buffer. 1065 * 1066 * Note that H_GET_24X7_DATA hcall allows reading several counters' 1067 * values in a single HCALL. We expect the caller to add events to the 1068 * request buffer one by one, make the HCALL and process the results. 1069 */ 1070 static int add_event_to_24x7_request(struct perf_event *event, 1071 struct hv_24x7_request_buffer *request_buffer) 1072 { 1073 u16 idx; 1074 int i; 1075 struct hv_24x7_request *req; 1076 1077 if (request_buffer->num_requests > 254) { 1078 pr_devel("Too many requests for 24x7 HCALL %d\n", 1079 request_buffer->num_requests); 1080 return -EINVAL; 1081 } 1082 1083 if (is_physical_domain(event_get_domain(event))) 1084 idx = event_get_core(event); 1085 else 1086 idx = event_get_vcpu(event); 1087 1088 i = request_buffer->num_requests++; 1089 req = &request_buffer->requests[i]; 1090 1091 req->performance_domain = event_get_domain(event); 1092 req->data_size = cpu_to_be16(8); 1093 req->data_offset = cpu_to_be32(event_get_offset(event)); 1094 req->starting_lpar_ix = cpu_to_be16(event_get_lpar(event)), 1095 req->max_num_lpars = cpu_to_be16(1); 1096 req->starting_ix = cpu_to_be16(idx); 1097 req->max_ix = cpu_to_be16(1); 1098 1099 return 0; 1100 } 1101 1102 static unsigned long single_24x7_request(struct perf_event *event, u64 *count) 1103 { 1104 unsigned long ret; 1105 struct hv_24x7_request_buffer *request_buffer; 1106 struct hv_24x7_data_result_buffer *result_buffer; 1107 struct hv_24x7_result *resb; 1108 1109 BUILD_BUG_ON(sizeof(*request_buffer) > 4096); 1110 BUILD_BUG_ON(sizeof(*result_buffer) > 4096); 1111 1112 request_buffer = (void *)get_cpu_var(hv_24x7_reqb); 1113 result_buffer = (void *)get_cpu_var(hv_24x7_resb); 1114 1115 init_24x7_request(request_buffer, result_buffer); 1116 1117 ret = add_event_to_24x7_request(event, request_buffer); 1118 if (ret) 1119 goto out; 1120 1121 ret = make_24x7_request(request_buffer, result_buffer); 1122 if (ret) { 1123 log_24x7_hcall(request_buffer, result_buffer, ret); 1124 goto out; 1125 } 1126 1127 /* process result from hcall */ 1128 resb = &result_buffer->results[0]; 1129 *count = be64_to_cpu(resb->elements[0].element_data[0]); 1130 1131 out: 1132 put_cpu_var(hv_24x7_reqb); 1133 put_cpu_var(hv_24x7_resb); 1134 return ret; 1135 } 1136 1137 1138 static int h_24x7_event_init(struct perf_event *event) 1139 { 1140 struct hv_perf_caps caps; 1141 unsigned domain; 1142 unsigned long hret; 1143 u64 ct; 1144 1145 /* Not our event */ 1146 if (event->attr.type != event->pmu->type) 1147 return -ENOENT; 1148 1149 /* Unused areas must be 0 */ 1150 if (event_get_reserved1(event) || 1151 event_get_reserved2(event) || 1152 event_get_reserved3(event)) { 1153 pr_devel("reserved set when forbidden 0x%llx(0x%llx) 0x%llx(0x%llx) 0x%llx(0x%llx)\n", 1154 event->attr.config, 1155 event_get_reserved1(event), 1156 event->attr.config1, 1157 event_get_reserved2(event), 1158 event->attr.config2, 1159 event_get_reserved3(event)); 1160 return -EINVAL; 1161 } 1162 1163 /* unsupported modes and filters */ 1164 if (event->attr.exclude_user || 1165 event->attr.exclude_kernel || 1166 event->attr.exclude_hv || 1167 event->attr.exclude_idle || 1168 event->attr.exclude_host || 1169 event->attr.exclude_guest) 1170 return -EINVAL; 1171 1172 /* no branch sampling */ 1173 if (has_branch_stack(event)) 1174 return -EOPNOTSUPP; 1175 1176 /* offset must be 8 byte aligned */ 1177 if (event_get_offset(event) % 8) { 1178 pr_devel("bad alignment\n"); 1179 return -EINVAL; 1180 } 1181 1182 /* Domains above 6 are invalid */ 1183 domain = event_get_domain(event); 1184 if (domain > 6) { 1185 pr_devel("invalid domain %d\n", domain); 1186 return -EINVAL; 1187 } 1188 1189 hret = hv_perf_caps_get(&caps); 1190 if (hret) { 1191 pr_devel("could not get capabilities: rc=%ld\n", hret); 1192 return -EIO; 1193 } 1194 1195 /* Physical domains & other lpars require extra capabilities */ 1196 if (!caps.collect_privileged && (is_physical_domain(domain) || 1197 (event_get_lpar(event) != event_get_lpar_max()))) { 1198 pr_devel("hv permissions disallow: is_physical_domain:%d, lpar=0x%llx\n", 1199 is_physical_domain(domain), 1200 event_get_lpar(event)); 1201 return -EACCES; 1202 } 1203 1204 /* see if the event complains */ 1205 if (single_24x7_request(event, &ct)) { 1206 pr_devel("test hcall failed\n"); 1207 return -EIO; 1208 } 1209 1210 return 0; 1211 } 1212 1213 static u64 h_24x7_get_value(struct perf_event *event) 1214 { 1215 unsigned long ret; 1216 u64 ct; 1217 ret = single_24x7_request(event, &ct); 1218 if (ret) 1219 /* We checked this in event init, shouldn't fail here... */ 1220 return 0; 1221 1222 return ct; 1223 } 1224 1225 static void update_event_count(struct perf_event *event, u64 now) 1226 { 1227 s64 prev; 1228 1229 prev = local64_xchg(&event->hw.prev_count, now); 1230 local64_add(now - prev, &event->count); 1231 } 1232 1233 static void h_24x7_event_read(struct perf_event *event) 1234 { 1235 u64 now; 1236 1237 now = h_24x7_get_value(event); 1238 update_event_count(event, now); 1239 } 1240 1241 static void h_24x7_event_start(struct perf_event *event, int flags) 1242 { 1243 if (flags & PERF_EF_RELOAD) 1244 local64_set(&event->hw.prev_count, h_24x7_get_value(event)); 1245 } 1246 1247 static void h_24x7_event_stop(struct perf_event *event, int flags) 1248 { 1249 h_24x7_event_read(event); 1250 } 1251 1252 static int h_24x7_event_add(struct perf_event *event, int flags) 1253 { 1254 if (flags & PERF_EF_START) 1255 h_24x7_event_start(event, flags); 1256 1257 return 0; 1258 } 1259 1260 static struct pmu h_24x7_pmu = { 1261 .task_ctx_nr = perf_invalid_context, 1262 1263 .name = "hv_24x7", 1264 .attr_groups = attr_groups, 1265 .event_init = h_24x7_event_init, 1266 .add = h_24x7_event_add, 1267 .del = h_24x7_event_stop, 1268 .start = h_24x7_event_start, 1269 .stop = h_24x7_event_stop, 1270 .read = h_24x7_event_read, 1271 }; 1272 1273 static int hv_24x7_init(void) 1274 { 1275 int r; 1276 unsigned long hret; 1277 struct hv_perf_caps caps; 1278 1279 if (!firmware_has_feature(FW_FEATURE_LPAR)) { 1280 pr_debug("not a virtualized system, not enabling\n"); 1281 return -ENODEV; 1282 } 1283 1284 hret = hv_perf_caps_get(&caps); 1285 if (hret) { 1286 pr_debug("could not obtain capabilities, not enabling, rc=%ld\n", 1287 hret); 1288 return -ENODEV; 1289 } 1290 1291 hv_page_cache = kmem_cache_create("hv-page-4096", 4096, 4096, 0, NULL); 1292 if (!hv_page_cache) 1293 return -ENOMEM; 1294 1295 /* sampling not supported */ 1296 h_24x7_pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT; 1297 1298 r = create_events_from_catalog(&event_group.attrs, 1299 &event_desc_group.attrs, 1300 &event_long_desc_group.attrs); 1301 1302 if (r) 1303 return r; 1304 1305 r = perf_pmu_register(&h_24x7_pmu, h_24x7_pmu.name, -1); 1306 if (r) 1307 return r; 1308 1309 return 0; 1310 } 1311 1312 device_initcall(hv_24x7_init); 1313