1 /* 2 drbd_state.c 3 4 This file is part of DRBD by Philipp Reisner and Lars Ellenberg. 5 6 Copyright (C) 2001-2008, LINBIT Information Technologies GmbH. 7 Copyright (C) 1999-2008, Philipp Reisner <philipp.reisner@linbit.com>. 8 Copyright (C) 2002-2008, Lars Ellenberg <lars.ellenberg@linbit.com>. 9 10 Thanks to Carter Burden, Bart Grantham and Gennadiy Nerubayev 11 from Logicworks, Inc. for making SDP replication support possible. 12 13 drbd is free software; you can redistribute it and/or modify 14 it under the terms of the GNU General Public License as published by 15 the Free Software Foundation; either version 2, or (at your option) 16 any later version. 17 18 drbd is distributed in the hope that it will be useful, 19 but WITHOUT ANY WARRANTY; without even the implied warranty of 20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 21 GNU General Public License for more details. 22 23 You should have received a copy of the GNU General Public License 24 along with drbd; see the file COPYING. If not, write to 25 the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA. 26 */ 27 28 #include <linux/drbd_limits.h> 29 #include "drbd_int.h" 30 #include "drbd_protocol.h" 31 #include "drbd_req.h" 32 #include "drbd_state_change.h" 33 34 struct after_state_chg_work { 35 struct drbd_work w; 36 struct drbd_device *device; 37 union drbd_state os; 38 union drbd_state ns; 39 enum chg_state_flags flags; 40 struct completion *done; 41 struct drbd_state_change *state_change; 42 }; 43 44 enum sanitize_state_warnings { 45 NO_WARNING, 46 ABORTED_ONLINE_VERIFY, 47 ABORTED_RESYNC, 48 CONNECTION_LOST_NEGOTIATING, 49 IMPLICITLY_UPGRADED_DISK, 50 IMPLICITLY_UPGRADED_PDSK, 51 }; 52 53 static void count_objects(struct drbd_resource *resource, 54 unsigned int *n_devices, 55 unsigned int *n_connections) 56 { 57 struct drbd_device *device; 58 struct drbd_connection *connection; 59 int vnr; 60 61 *n_devices = 0; 62 *n_connections = 0; 63 64 idr_for_each_entry(&resource->devices, device, vnr) 65 (*n_devices)++; 66 for_each_connection(connection, resource) 67 (*n_connections)++; 68 } 69 70 static struct drbd_state_change *alloc_state_change(unsigned int n_devices, unsigned int n_connections, gfp_t gfp) 71 { 72 struct drbd_state_change *state_change; 73 unsigned int size, n; 74 75 size = sizeof(struct drbd_state_change) + 76 n_devices * sizeof(struct drbd_device_state_change) + 77 n_connections * sizeof(struct drbd_connection_state_change) + 78 n_devices * n_connections * sizeof(struct drbd_peer_device_state_change); 79 state_change = kmalloc(size, gfp); 80 if (!state_change) 81 return NULL; 82 state_change->n_devices = n_devices; 83 state_change->n_connections = n_connections; 84 state_change->devices = (void *)(state_change + 1); 85 state_change->connections = (void *)&state_change->devices[n_devices]; 86 state_change->peer_devices = (void *)&state_change->connections[n_connections]; 87 state_change->resource->resource = NULL; 88 for (n = 0; n < n_devices; n++) 89 state_change->devices[n].device = NULL; 90 for (n = 0; n < n_connections; n++) 91 state_change->connections[n].connection = NULL; 92 return state_change; 93 } 94 95 struct drbd_state_change *remember_old_state(struct drbd_resource *resource, gfp_t gfp) 96 { 97 struct drbd_state_change *state_change; 98 struct drbd_device *device; 99 unsigned int n_devices; 100 struct drbd_connection *connection; 101 unsigned int n_connections; 102 int vnr; 103 104 struct drbd_device_state_change *device_state_change; 105 struct drbd_peer_device_state_change *peer_device_state_change; 106 struct drbd_connection_state_change *connection_state_change; 107 108 /* Caller holds req_lock spinlock. 109 * No state, no device IDR, no connections lists can change. */ 110 count_objects(resource, &n_devices, &n_connections); 111 state_change = alloc_state_change(n_devices, n_connections, gfp); 112 if (!state_change) 113 return NULL; 114 115 kref_get(&resource->kref); 116 state_change->resource->resource = resource; 117 state_change->resource->role[OLD] = 118 conn_highest_role(first_connection(resource)); 119 state_change->resource->susp[OLD] = resource->susp; 120 state_change->resource->susp_nod[OLD] = resource->susp_nod; 121 state_change->resource->susp_fen[OLD] = resource->susp_fen; 122 123 connection_state_change = state_change->connections; 124 for_each_connection(connection, resource) { 125 kref_get(&connection->kref); 126 connection_state_change->connection = connection; 127 connection_state_change->cstate[OLD] = 128 connection->cstate; 129 connection_state_change->peer_role[OLD] = 130 conn_highest_peer(connection); 131 connection_state_change++; 132 } 133 134 device_state_change = state_change->devices; 135 peer_device_state_change = state_change->peer_devices; 136 idr_for_each_entry(&resource->devices, device, vnr) { 137 kref_get(&device->kref); 138 device_state_change->device = device; 139 device_state_change->disk_state[OLD] = device->state.disk; 140 141 /* The peer_devices for each device have to be enumerated in 142 the order of the connections. We may not use for_each_peer_device() here. */ 143 for_each_connection(connection, resource) { 144 struct drbd_peer_device *peer_device; 145 146 peer_device = conn_peer_device(connection, device->vnr); 147 peer_device_state_change->peer_device = peer_device; 148 peer_device_state_change->disk_state[OLD] = 149 device->state.pdsk; 150 peer_device_state_change->repl_state[OLD] = 151 max_t(enum drbd_conns, 152 C_WF_REPORT_PARAMS, device->state.conn); 153 peer_device_state_change->resync_susp_user[OLD] = 154 device->state.user_isp; 155 peer_device_state_change->resync_susp_peer[OLD] = 156 device->state.peer_isp; 157 peer_device_state_change->resync_susp_dependency[OLD] = 158 device->state.aftr_isp; 159 peer_device_state_change++; 160 } 161 device_state_change++; 162 } 163 164 return state_change; 165 } 166 167 static void remember_new_state(struct drbd_state_change *state_change) 168 { 169 struct drbd_resource_state_change *resource_state_change; 170 struct drbd_resource *resource; 171 unsigned int n; 172 173 if (!state_change) 174 return; 175 176 resource_state_change = &state_change->resource[0]; 177 resource = resource_state_change->resource; 178 179 resource_state_change->role[NEW] = 180 conn_highest_role(first_connection(resource)); 181 resource_state_change->susp[NEW] = resource->susp; 182 resource_state_change->susp_nod[NEW] = resource->susp_nod; 183 resource_state_change->susp_fen[NEW] = resource->susp_fen; 184 185 for (n = 0; n < state_change->n_devices; n++) { 186 struct drbd_device_state_change *device_state_change = 187 &state_change->devices[n]; 188 struct drbd_device *device = device_state_change->device; 189 190 device_state_change->disk_state[NEW] = device->state.disk; 191 } 192 193 for (n = 0; n < state_change->n_connections; n++) { 194 struct drbd_connection_state_change *connection_state_change = 195 &state_change->connections[n]; 196 struct drbd_connection *connection = 197 connection_state_change->connection; 198 199 connection_state_change->cstate[NEW] = connection->cstate; 200 connection_state_change->peer_role[NEW] = 201 conn_highest_peer(connection); 202 } 203 204 for (n = 0; n < state_change->n_devices * state_change->n_connections; n++) { 205 struct drbd_peer_device_state_change *peer_device_state_change = 206 &state_change->peer_devices[n]; 207 struct drbd_device *device = 208 peer_device_state_change->peer_device->device; 209 union drbd_dev_state state = device->state; 210 211 peer_device_state_change->disk_state[NEW] = state.pdsk; 212 peer_device_state_change->repl_state[NEW] = 213 max_t(enum drbd_conns, C_WF_REPORT_PARAMS, state.conn); 214 peer_device_state_change->resync_susp_user[NEW] = 215 state.user_isp; 216 peer_device_state_change->resync_susp_peer[NEW] = 217 state.peer_isp; 218 peer_device_state_change->resync_susp_dependency[NEW] = 219 state.aftr_isp; 220 } 221 } 222 223 void copy_old_to_new_state_change(struct drbd_state_change *state_change) 224 { 225 struct drbd_resource_state_change *resource_state_change = &state_change->resource[0]; 226 unsigned int n_device, n_connection, n_peer_device, n_peer_devices; 227 228 #define OLD_TO_NEW(x) \ 229 (x[NEW] = x[OLD]) 230 231 OLD_TO_NEW(resource_state_change->role); 232 OLD_TO_NEW(resource_state_change->susp); 233 OLD_TO_NEW(resource_state_change->susp_nod); 234 OLD_TO_NEW(resource_state_change->susp_fen); 235 236 for (n_connection = 0; n_connection < state_change->n_connections; n_connection++) { 237 struct drbd_connection_state_change *connection_state_change = 238 &state_change->connections[n_connection]; 239 240 OLD_TO_NEW(connection_state_change->peer_role); 241 OLD_TO_NEW(connection_state_change->cstate); 242 } 243 244 for (n_device = 0; n_device < state_change->n_devices; n_device++) { 245 struct drbd_device_state_change *device_state_change = 246 &state_change->devices[n_device]; 247 248 OLD_TO_NEW(device_state_change->disk_state); 249 } 250 251 n_peer_devices = state_change->n_devices * state_change->n_connections; 252 for (n_peer_device = 0; n_peer_device < n_peer_devices; n_peer_device++) { 253 struct drbd_peer_device_state_change *p = 254 &state_change->peer_devices[n_peer_device]; 255 256 OLD_TO_NEW(p->disk_state); 257 OLD_TO_NEW(p->repl_state); 258 OLD_TO_NEW(p->resync_susp_user); 259 OLD_TO_NEW(p->resync_susp_peer); 260 OLD_TO_NEW(p->resync_susp_dependency); 261 } 262 263 #undef OLD_TO_NEW 264 } 265 266 void forget_state_change(struct drbd_state_change *state_change) 267 { 268 unsigned int n; 269 270 if (!state_change) 271 return; 272 273 if (state_change->resource->resource) 274 kref_put(&state_change->resource->resource->kref, drbd_destroy_resource); 275 for (n = 0; n < state_change->n_devices; n++) { 276 struct drbd_device *device = state_change->devices[n].device; 277 278 if (device) 279 kref_put(&device->kref, drbd_destroy_device); 280 } 281 for (n = 0; n < state_change->n_connections; n++) { 282 struct drbd_connection *connection = 283 state_change->connections[n].connection; 284 285 if (connection) 286 kref_put(&connection->kref, drbd_destroy_connection); 287 } 288 kfree(state_change); 289 } 290 291 static int w_after_state_ch(struct drbd_work *w, int unused); 292 static void after_state_ch(struct drbd_device *device, union drbd_state os, 293 union drbd_state ns, enum chg_state_flags flags, 294 struct drbd_state_change *); 295 static enum drbd_state_rv is_valid_state(struct drbd_device *, union drbd_state); 296 static enum drbd_state_rv is_valid_soft_transition(union drbd_state, union drbd_state, struct drbd_connection *); 297 static enum drbd_state_rv is_valid_transition(union drbd_state os, union drbd_state ns); 298 static union drbd_state sanitize_state(struct drbd_device *device, union drbd_state os, 299 union drbd_state ns, enum sanitize_state_warnings *warn); 300 301 static inline bool is_susp(union drbd_state s) 302 { 303 return s.susp || s.susp_nod || s.susp_fen; 304 } 305 306 bool conn_all_vols_unconf(struct drbd_connection *connection) 307 { 308 struct drbd_peer_device *peer_device; 309 bool rv = true; 310 int vnr; 311 312 rcu_read_lock(); 313 idr_for_each_entry(&connection->peer_devices, peer_device, vnr) { 314 struct drbd_device *device = peer_device->device; 315 if (device->state.disk != D_DISKLESS || 316 device->state.conn != C_STANDALONE || 317 device->state.role != R_SECONDARY) { 318 rv = false; 319 break; 320 } 321 } 322 rcu_read_unlock(); 323 324 return rv; 325 } 326 327 /* Unfortunately the states where not correctly ordered, when 328 they where defined. therefore can not use max_t() here. */ 329 static enum drbd_role max_role(enum drbd_role role1, enum drbd_role role2) 330 { 331 if (role1 == R_PRIMARY || role2 == R_PRIMARY) 332 return R_PRIMARY; 333 if (role1 == R_SECONDARY || role2 == R_SECONDARY) 334 return R_SECONDARY; 335 return R_UNKNOWN; 336 } 337 338 static enum drbd_role min_role(enum drbd_role role1, enum drbd_role role2) 339 { 340 if (role1 == R_UNKNOWN || role2 == R_UNKNOWN) 341 return R_UNKNOWN; 342 if (role1 == R_SECONDARY || role2 == R_SECONDARY) 343 return R_SECONDARY; 344 return R_PRIMARY; 345 } 346 347 enum drbd_role conn_highest_role(struct drbd_connection *connection) 348 { 349 enum drbd_role role = R_SECONDARY; 350 struct drbd_peer_device *peer_device; 351 int vnr; 352 353 rcu_read_lock(); 354 idr_for_each_entry(&connection->peer_devices, peer_device, vnr) { 355 struct drbd_device *device = peer_device->device; 356 role = max_role(role, device->state.role); 357 } 358 rcu_read_unlock(); 359 360 return role; 361 } 362 363 enum drbd_role conn_highest_peer(struct drbd_connection *connection) 364 { 365 enum drbd_role peer = R_UNKNOWN; 366 struct drbd_peer_device *peer_device; 367 int vnr; 368 369 rcu_read_lock(); 370 idr_for_each_entry(&connection->peer_devices, peer_device, vnr) { 371 struct drbd_device *device = peer_device->device; 372 peer = max_role(peer, device->state.peer); 373 } 374 rcu_read_unlock(); 375 376 return peer; 377 } 378 379 enum drbd_disk_state conn_highest_disk(struct drbd_connection *connection) 380 { 381 enum drbd_disk_state disk_state = D_DISKLESS; 382 struct drbd_peer_device *peer_device; 383 int vnr; 384 385 rcu_read_lock(); 386 idr_for_each_entry(&connection->peer_devices, peer_device, vnr) { 387 struct drbd_device *device = peer_device->device; 388 disk_state = max_t(enum drbd_disk_state, disk_state, device->state.disk); 389 } 390 rcu_read_unlock(); 391 392 return disk_state; 393 } 394 395 enum drbd_disk_state conn_lowest_disk(struct drbd_connection *connection) 396 { 397 enum drbd_disk_state disk_state = D_MASK; 398 struct drbd_peer_device *peer_device; 399 int vnr; 400 401 rcu_read_lock(); 402 idr_for_each_entry(&connection->peer_devices, peer_device, vnr) { 403 struct drbd_device *device = peer_device->device; 404 disk_state = min_t(enum drbd_disk_state, disk_state, device->state.disk); 405 } 406 rcu_read_unlock(); 407 408 return disk_state; 409 } 410 411 enum drbd_disk_state conn_highest_pdsk(struct drbd_connection *connection) 412 { 413 enum drbd_disk_state disk_state = D_DISKLESS; 414 struct drbd_peer_device *peer_device; 415 int vnr; 416 417 rcu_read_lock(); 418 idr_for_each_entry(&connection->peer_devices, peer_device, vnr) { 419 struct drbd_device *device = peer_device->device; 420 disk_state = max_t(enum drbd_disk_state, disk_state, device->state.pdsk); 421 } 422 rcu_read_unlock(); 423 424 return disk_state; 425 } 426 427 enum drbd_conns conn_lowest_conn(struct drbd_connection *connection) 428 { 429 enum drbd_conns conn = C_MASK; 430 struct drbd_peer_device *peer_device; 431 int vnr; 432 433 rcu_read_lock(); 434 idr_for_each_entry(&connection->peer_devices, peer_device, vnr) { 435 struct drbd_device *device = peer_device->device; 436 conn = min_t(enum drbd_conns, conn, device->state.conn); 437 } 438 rcu_read_unlock(); 439 440 return conn; 441 } 442 443 static bool no_peer_wf_report_params(struct drbd_connection *connection) 444 { 445 struct drbd_peer_device *peer_device; 446 int vnr; 447 bool rv = true; 448 449 rcu_read_lock(); 450 idr_for_each_entry(&connection->peer_devices, peer_device, vnr) 451 if (peer_device->device->state.conn == C_WF_REPORT_PARAMS) { 452 rv = false; 453 break; 454 } 455 rcu_read_unlock(); 456 457 return rv; 458 } 459 460 static void wake_up_all_devices(struct drbd_connection *connection) 461 { 462 struct drbd_peer_device *peer_device; 463 int vnr; 464 465 rcu_read_lock(); 466 idr_for_each_entry(&connection->peer_devices, peer_device, vnr) 467 wake_up(&peer_device->device->state_wait); 468 rcu_read_unlock(); 469 470 } 471 472 473 /** 474 * cl_wide_st_chg() - true if the state change is a cluster wide one 475 * @device: DRBD device. 476 * @os: old (current) state. 477 * @ns: new (wanted) state. 478 */ 479 static int cl_wide_st_chg(struct drbd_device *device, 480 union drbd_state os, union drbd_state ns) 481 { 482 return (os.conn >= C_CONNECTED && ns.conn >= C_CONNECTED && 483 ((os.role != R_PRIMARY && ns.role == R_PRIMARY) || 484 (os.conn != C_STARTING_SYNC_T && ns.conn == C_STARTING_SYNC_T) || 485 (os.conn != C_STARTING_SYNC_S && ns.conn == C_STARTING_SYNC_S) || 486 (os.disk != D_FAILED && ns.disk == D_FAILED))) || 487 (os.conn >= C_CONNECTED && ns.conn == C_DISCONNECTING) || 488 (os.conn == C_CONNECTED && ns.conn == C_VERIFY_S) || 489 (os.conn == C_CONNECTED && ns.conn == C_WF_REPORT_PARAMS); 490 } 491 492 static union drbd_state 493 apply_mask_val(union drbd_state os, union drbd_state mask, union drbd_state val) 494 { 495 union drbd_state ns; 496 ns.i = (os.i & ~mask.i) | val.i; 497 return ns; 498 } 499 500 enum drbd_state_rv 501 drbd_change_state(struct drbd_device *device, enum chg_state_flags f, 502 union drbd_state mask, union drbd_state val) 503 { 504 unsigned long flags; 505 union drbd_state ns; 506 enum drbd_state_rv rv; 507 508 spin_lock_irqsave(&device->resource->req_lock, flags); 509 ns = apply_mask_val(drbd_read_state(device), mask, val); 510 rv = _drbd_set_state(device, ns, f, NULL); 511 spin_unlock_irqrestore(&device->resource->req_lock, flags); 512 513 return rv; 514 } 515 516 /** 517 * drbd_force_state() - Impose a change which happens outside our control on our state 518 * @device: DRBD device. 519 * @mask: mask of state bits to change. 520 * @val: value of new state bits. 521 */ 522 void drbd_force_state(struct drbd_device *device, 523 union drbd_state mask, union drbd_state val) 524 { 525 drbd_change_state(device, CS_HARD, mask, val); 526 } 527 528 static enum drbd_state_rv 529 _req_st_cond(struct drbd_device *device, union drbd_state mask, 530 union drbd_state val) 531 { 532 union drbd_state os, ns; 533 unsigned long flags; 534 enum drbd_state_rv rv; 535 536 if (test_and_clear_bit(CL_ST_CHG_SUCCESS, &device->flags)) 537 return SS_CW_SUCCESS; 538 539 if (test_and_clear_bit(CL_ST_CHG_FAIL, &device->flags)) 540 return SS_CW_FAILED_BY_PEER; 541 542 spin_lock_irqsave(&device->resource->req_lock, flags); 543 os = drbd_read_state(device); 544 ns = sanitize_state(device, os, apply_mask_val(os, mask, val), NULL); 545 rv = is_valid_transition(os, ns); 546 if (rv >= SS_SUCCESS) 547 rv = SS_UNKNOWN_ERROR; /* cont waiting, otherwise fail. */ 548 549 if (!cl_wide_st_chg(device, os, ns)) 550 rv = SS_CW_NO_NEED; 551 if (rv == SS_UNKNOWN_ERROR) { 552 rv = is_valid_state(device, ns); 553 if (rv >= SS_SUCCESS) { 554 rv = is_valid_soft_transition(os, ns, first_peer_device(device)->connection); 555 if (rv >= SS_SUCCESS) 556 rv = SS_UNKNOWN_ERROR; /* cont waiting, otherwise fail. */ 557 } 558 } 559 spin_unlock_irqrestore(&device->resource->req_lock, flags); 560 561 return rv; 562 } 563 564 /** 565 * drbd_req_state() - Perform an eventually cluster wide state change 566 * @device: DRBD device. 567 * @mask: mask of state bits to change. 568 * @val: value of new state bits. 569 * @f: flags 570 * 571 * Should not be called directly, use drbd_request_state() or 572 * _drbd_request_state(). 573 */ 574 static enum drbd_state_rv 575 drbd_req_state(struct drbd_device *device, union drbd_state mask, 576 union drbd_state val, enum chg_state_flags f) 577 { 578 struct completion done; 579 unsigned long flags; 580 union drbd_state os, ns; 581 enum drbd_state_rv rv; 582 void *buffer = NULL; 583 584 init_completion(&done); 585 586 if (f & CS_SERIALIZE) 587 mutex_lock(device->state_mutex); 588 if (f & CS_INHIBIT_MD_IO) 589 buffer = drbd_md_get_buffer(device, __func__); 590 591 spin_lock_irqsave(&device->resource->req_lock, flags); 592 os = drbd_read_state(device); 593 ns = sanitize_state(device, os, apply_mask_val(os, mask, val), NULL); 594 rv = is_valid_transition(os, ns); 595 if (rv < SS_SUCCESS) { 596 spin_unlock_irqrestore(&device->resource->req_lock, flags); 597 goto abort; 598 } 599 600 if (cl_wide_st_chg(device, os, ns)) { 601 rv = is_valid_state(device, ns); 602 if (rv == SS_SUCCESS) 603 rv = is_valid_soft_transition(os, ns, first_peer_device(device)->connection); 604 spin_unlock_irqrestore(&device->resource->req_lock, flags); 605 606 if (rv < SS_SUCCESS) { 607 if (f & CS_VERBOSE) 608 print_st_err(device, os, ns, rv); 609 goto abort; 610 } 611 612 if (drbd_send_state_req(first_peer_device(device), mask, val)) { 613 rv = SS_CW_FAILED_BY_PEER; 614 if (f & CS_VERBOSE) 615 print_st_err(device, os, ns, rv); 616 goto abort; 617 } 618 619 wait_event(device->state_wait, 620 (rv = _req_st_cond(device, mask, val))); 621 622 if (rv < SS_SUCCESS) { 623 if (f & CS_VERBOSE) 624 print_st_err(device, os, ns, rv); 625 goto abort; 626 } 627 spin_lock_irqsave(&device->resource->req_lock, flags); 628 ns = apply_mask_val(drbd_read_state(device), mask, val); 629 rv = _drbd_set_state(device, ns, f, &done); 630 } else { 631 rv = _drbd_set_state(device, ns, f, &done); 632 } 633 634 spin_unlock_irqrestore(&device->resource->req_lock, flags); 635 636 if (f & CS_WAIT_COMPLETE && rv == SS_SUCCESS) { 637 D_ASSERT(device, current != first_peer_device(device)->connection->worker.task); 638 wait_for_completion(&done); 639 } 640 641 abort: 642 if (buffer) 643 drbd_md_put_buffer(device); 644 if (f & CS_SERIALIZE) 645 mutex_unlock(device->state_mutex); 646 647 return rv; 648 } 649 650 /** 651 * _drbd_request_state() - Request a state change (with flags) 652 * @device: DRBD device. 653 * @mask: mask of state bits to change. 654 * @val: value of new state bits. 655 * @f: flags 656 * 657 * Cousin of drbd_request_state(), useful with the CS_WAIT_COMPLETE 658 * flag, or when logging of failed state change requests is not desired. 659 */ 660 enum drbd_state_rv 661 _drbd_request_state(struct drbd_device *device, union drbd_state mask, 662 union drbd_state val, enum chg_state_flags f) 663 { 664 enum drbd_state_rv rv; 665 666 wait_event(device->state_wait, 667 (rv = drbd_req_state(device, mask, val, f)) != SS_IN_TRANSIENT_STATE); 668 669 return rv; 670 } 671 672 /* 673 * We grab drbd_md_get_buffer(), because we don't want to "fail" the disk while 674 * there is IO in-flight: the transition into D_FAILED for detach purposes 675 * may get misinterpreted as actual IO error in a confused endio function. 676 * 677 * We wrap it all into wait_event(), to retry in case the drbd_req_state() 678 * returns SS_IN_TRANSIENT_STATE. 679 * 680 * To avoid potential deadlock with e.g. the receiver thread trying to grab 681 * drbd_md_get_buffer() while trying to get out of the "transient state", we 682 * need to grab and release the meta data buffer inside of that wait_event loop. 683 */ 684 static enum drbd_state_rv 685 request_detach(struct drbd_device *device) 686 { 687 return drbd_req_state(device, NS(disk, D_FAILED), 688 CS_VERBOSE | CS_ORDERED | CS_INHIBIT_MD_IO); 689 } 690 691 enum drbd_state_rv 692 drbd_request_detach_interruptible(struct drbd_device *device) 693 { 694 enum drbd_state_rv rv; 695 int ret; 696 697 drbd_suspend_io(device); /* so no-one is stuck in drbd_al_begin_io */ 698 wait_event_interruptible(device->state_wait, 699 (rv = request_detach(device)) != SS_IN_TRANSIENT_STATE); 700 drbd_resume_io(device); 701 702 ret = wait_event_interruptible(device->misc_wait, 703 device->state.disk != D_FAILED); 704 705 if (rv == SS_IS_DISKLESS) 706 rv = SS_NOTHING_TO_DO; 707 if (ret) 708 rv = ERR_INTR; 709 710 return rv; 711 } 712 713 enum drbd_state_rv 714 _drbd_request_state_holding_state_mutex(struct drbd_device *device, union drbd_state mask, 715 union drbd_state val, enum chg_state_flags f) 716 { 717 enum drbd_state_rv rv; 718 719 BUG_ON(f & CS_SERIALIZE); 720 721 wait_event_cmd(device->state_wait, 722 (rv = drbd_req_state(device, mask, val, f)) != SS_IN_TRANSIENT_STATE, 723 mutex_unlock(device->state_mutex), 724 mutex_lock(device->state_mutex)); 725 726 return rv; 727 } 728 729 static void print_st(struct drbd_device *device, const char *name, union drbd_state ns) 730 { 731 drbd_err(device, " %s = { cs:%s ro:%s/%s ds:%s/%s %c%c%c%c%c%c }\n", 732 name, 733 drbd_conn_str(ns.conn), 734 drbd_role_str(ns.role), 735 drbd_role_str(ns.peer), 736 drbd_disk_str(ns.disk), 737 drbd_disk_str(ns.pdsk), 738 is_susp(ns) ? 's' : 'r', 739 ns.aftr_isp ? 'a' : '-', 740 ns.peer_isp ? 'p' : '-', 741 ns.user_isp ? 'u' : '-', 742 ns.susp_fen ? 'F' : '-', 743 ns.susp_nod ? 'N' : '-' 744 ); 745 } 746 747 void print_st_err(struct drbd_device *device, union drbd_state os, 748 union drbd_state ns, enum drbd_state_rv err) 749 { 750 if (err == SS_IN_TRANSIENT_STATE) 751 return; 752 drbd_err(device, "State change failed: %s\n", drbd_set_st_err_str(err)); 753 print_st(device, " state", os); 754 print_st(device, "wanted", ns); 755 } 756 757 static long print_state_change(char *pb, union drbd_state os, union drbd_state ns, 758 enum chg_state_flags flags) 759 { 760 char *pbp; 761 pbp = pb; 762 *pbp = 0; 763 764 if (ns.role != os.role && flags & CS_DC_ROLE) 765 pbp += sprintf(pbp, "role( %s -> %s ) ", 766 drbd_role_str(os.role), 767 drbd_role_str(ns.role)); 768 if (ns.peer != os.peer && flags & CS_DC_PEER) 769 pbp += sprintf(pbp, "peer( %s -> %s ) ", 770 drbd_role_str(os.peer), 771 drbd_role_str(ns.peer)); 772 if (ns.conn != os.conn && flags & CS_DC_CONN) 773 pbp += sprintf(pbp, "conn( %s -> %s ) ", 774 drbd_conn_str(os.conn), 775 drbd_conn_str(ns.conn)); 776 if (ns.disk != os.disk && flags & CS_DC_DISK) 777 pbp += sprintf(pbp, "disk( %s -> %s ) ", 778 drbd_disk_str(os.disk), 779 drbd_disk_str(ns.disk)); 780 if (ns.pdsk != os.pdsk && flags & CS_DC_PDSK) 781 pbp += sprintf(pbp, "pdsk( %s -> %s ) ", 782 drbd_disk_str(os.pdsk), 783 drbd_disk_str(ns.pdsk)); 784 785 return pbp - pb; 786 } 787 788 static void drbd_pr_state_change(struct drbd_device *device, union drbd_state os, union drbd_state ns, 789 enum chg_state_flags flags) 790 { 791 char pb[300]; 792 char *pbp = pb; 793 794 pbp += print_state_change(pbp, os, ns, flags ^ CS_DC_MASK); 795 796 if (ns.aftr_isp != os.aftr_isp) 797 pbp += sprintf(pbp, "aftr_isp( %d -> %d ) ", 798 os.aftr_isp, 799 ns.aftr_isp); 800 if (ns.peer_isp != os.peer_isp) 801 pbp += sprintf(pbp, "peer_isp( %d -> %d ) ", 802 os.peer_isp, 803 ns.peer_isp); 804 if (ns.user_isp != os.user_isp) 805 pbp += sprintf(pbp, "user_isp( %d -> %d ) ", 806 os.user_isp, 807 ns.user_isp); 808 809 if (pbp != pb) 810 drbd_info(device, "%s\n", pb); 811 } 812 813 static void conn_pr_state_change(struct drbd_connection *connection, union drbd_state os, union drbd_state ns, 814 enum chg_state_flags flags) 815 { 816 char pb[300]; 817 char *pbp = pb; 818 819 pbp += print_state_change(pbp, os, ns, flags); 820 821 if (is_susp(ns) != is_susp(os) && flags & CS_DC_SUSP) 822 pbp += sprintf(pbp, "susp( %d -> %d ) ", 823 is_susp(os), 824 is_susp(ns)); 825 826 if (pbp != pb) 827 drbd_info(connection, "%s\n", pb); 828 } 829 830 831 /** 832 * is_valid_state() - Returns an SS_ error code if ns is not valid 833 * @device: DRBD device. 834 * @ns: State to consider. 835 */ 836 static enum drbd_state_rv 837 is_valid_state(struct drbd_device *device, union drbd_state ns) 838 { 839 /* See drbd_state_sw_errors in drbd_strings.c */ 840 841 enum drbd_fencing_p fp; 842 enum drbd_state_rv rv = SS_SUCCESS; 843 struct net_conf *nc; 844 845 rcu_read_lock(); 846 fp = FP_DONT_CARE; 847 if (get_ldev(device)) { 848 fp = rcu_dereference(device->ldev->disk_conf)->fencing; 849 put_ldev(device); 850 } 851 852 nc = rcu_dereference(first_peer_device(device)->connection->net_conf); 853 if (nc) { 854 if (!nc->two_primaries && ns.role == R_PRIMARY) { 855 if (ns.peer == R_PRIMARY) 856 rv = SS_TWO_PRIMARIES; 857 else if (conn_highest_peer(first_peer_device(device)->connection) == R_PRIMARY) 858 rv = SS_O_VOL_PEER_PRI; 859 } 860 } 861 862 if (rv <= 0) 863 goto out; /* already found a reason to abort */ 864 else if (ns.role == R_SECONDARY && device->open_cnt) 865 rv = SS_DEVICE_IN_USE; 866 867 else if (ns.role == R_PRIMARY && ns.conn < C_CONNECTED && ns.disk < D_UP_TO_DATE) 868 rv = SS_NO_UP_TO_DATE_DISK; 869 870 else if (fp >= FP_RESOURCE && 871 ns.role == R_PRIMARY && ns.conn < C_CONNECTED && ns.pdsk >= D_UNKNOWN) 872 rv = SS_PRIMARY_NOP; 873 874 else if (ns.role == R_PRIMARY && ns.disk <= D_INCONSISTENT && ns.pdsk <= D_INCONSISTENT) 875 rv = SS_NO_UP_TO_DATE_DISK; 876 877 else if (ns.conn > C_CONNECTED && ns.disk < D_INCONSISTENT) 878 rv = SS_NO_LOCAL_DISK; 879 880 else if (ns.conn > C_CONNECTED && ns.pdsk < D_INCONSISTENT) 881 rv = SS_NO_REMOTE_DISK; 882 883 else if (ns.conn > C_CONNECTED && ns.disk < D_UP_TO_DATE && ns.pdsk < D_UP_TO_DATE) 884 rv = SS_NO_UP_TO_DATE_DISK; 885 886 else if ((ns.conn == C_CONNECTED || 887 ns.conn == C_WF_BITMAP_S || 888 ns.conn == C_SYNC_SOURCE || 889 ns.conn == C_PAUSED_SYNC_S) && 890 ns.disk == D_OUTDATED) 891 rv = SS_CONNECTED_OUTDATES; 892 893 else if ((ns.conn == C_VERIFY_S || ns.conn == C_VERIFY_T) && 894 (nc->verify_alg[0] == 0)) 895 rv = SS_NO_VERIFY_ALG; 896 897 else if ((ns.conn == C_VERIFY_S || ns.conn == C_VERIFY_T) && 898 first_peer_device(device)->connection->agreed_pro_version < 88) 899 rv = SS_NOT_SUPPORTED; 900 901 else if (ns.role == R_PRIMARY && ns.disk < D_UP_TO_DATE && ns.pdsk < D_UP_TO_DATE) 902 rv = SS_NO_UP_TO_DATE_DISK; 903 904 else if ((ns.conn == C_STARTING_SYNC_S || ns.conn == C_STARTING_SYNC_T) && 905 ns.pdsk == D_UNKNOWN) 906 rv = SS_NEED_CONNECTION; 907 908 else if (ns.conn >= C_CONNECTED && ns.pdsk == D_UNKNOWN) 909 rv = SS_CONNECTED_OUTDATES; 910 911 out: 912 rcu_read_unlock(); 913 914 return rv; 915 } 916 917 /** 918 * is_valid_soft_transition() - Returns an SS_ error code if the state transition is not possible 919 * This function limits state transitions that may be declined by DRBD. I.e. 920 * user requests (aka soft transitions). 921 * @device: DRBD device. 922 * @ns: new state. 923 * @os: old state. 924 */ 925 static enum drbd_state_rv 926 is_valid_soft_transition(union drbd_state os, union drbd_state ns, struct drbd_connection *connection) 927 { 928 enum drbd_state_rv rv = SS_SUCCESS; 929 930 if ((ns.conn == C_STARTING_SYNC_T || ns.conn == C_STARTING_SYNC_S) && 931 os.conn > C_CONNECTED) 932 rv = SS_RESYNC_RUNNING; 933 934 if (ns.conn == C_DISCONNECTING && os.conn == C_STANDALONE) 935 rv = SS_ALREADY_STANDALONE; 936 937 if (ns.disk > D_ATTACHING && os.disk == D_DISKLESS) 938 rv = SS_IS_DISKLESS; 939 940 if (ns.conn == C_WF_CONNECTION && os.conn < C_UNCONNECTED) 941 rv = SS_NO_NET_CONFIG; 942 943 if (ns.disk == D_OUTDATED && os.disk < D_OUTDATED && os.disk != D_ATTACHING) 944 rv = SS_LOWER_THAN_OUTDATED; 945 946 if (ns.conn == C_DISCONNECTING && os.conn == C_UNCONNECTED) 947 rv = SS_IN_TRANSIENT_STATE; 948 949 /* While establishing a connection only allow cstate to change. 950 Delay/refuse role changes, detach attach etc... (they do not touch cstate) */ 951 if (test_bit(STATE_SENT, &connection->flags) && 952 !((ns.conn == C_WF_REPORT_PARAMS && os.conn == C_WF_CONNECTION) || 953 (ns.conn >= C_CONNECTED && os.conn == C_WF_REPORT_PARAMS))) 954 rv = SS_IN_TRANSIENT_STATE; 955 956 /* Do not promote during resync handshake triggered by "force primary". 957 * This is a hack. It should really be rejected by the peer during the 958 * cluster wide state change request. */ 959 if (os.role != R_PRIMARY && ns.role == R_PRIMARY 960 && ns.pdsk == D_UP_TO_DATE 961 && ns.disk != D_UP_TO_DATE && ns.disk != D_DISKLESS 962 && (ns.conn <= C_WF_SYNC_UUID || ns.conn != os.conn)) 963 rv = SS_IN_TRANSIENT_STATE; 964 965 if ((ns.conn == C_VERIFY_S || ns.conn == C_VERIFY_T) && os.conn < C_CONNECTED) 966 rv = SS_NEED_CONNECTION; 967 968 if ((ns.conn == C_VERIFY_S || ns.conn == C_VERIFY_T) && 969 ns.conn != os.conn && os.conn > C_CONNECTED) 970 rv = SS_RESYNC_RUNNING; 971 972 if ((ns.conn == C_STARTING_SYNC_S || ns.conn == C_STARTING_SYNC_T) && 973 os.conn < C_CONNECTED) 974 rv = SS_NEED_CONNECTION; 975 976 if ((ns.conn == C_SYNC_TARGET || ns.conn == C_SYNC_SOURCE) 977 && os.conn < C_WF_REPORT_PARAMS) 978 rv = SS_NEED_CONNECTION; /* No NetworkFailure -> SyncTarget etc... */ 979 980 if (ns.conn == C_DISCONNECTING && ns.pdsk == D_OUTDATED && 981 os.conn < C_CONNECTED && os.pdsk > D_OUTDATED) 982 rv = SS_OUTDATE_WO_CONN; 983 984 return rv; 985 } 986 987 static enum drbd_state_rv 988 is_valid_conn_transition(enum drbd_conns oc, enum drbd_conns nc) 989 { 990 /* no change -> nothing to do, at least for the connection part */ 991 if (oc == nc) 992 return SS_NOTHING_TO_DO; 993 994 /* disconnect of an unconfigured connection does not make sense */ 995 if (oc == C_STANDALONE && nc == C_DISCONNECTING) 996 return SS_ALREADY_STANDALONE; 997 998 /* from C_STANDALONE, we start with C_UNCONNECTED */ 999 if (oc == C_STANDALONE && nc != C_UNCONNECTED) 1000 return SS_NEED_CONNECTION; 1001 1002 /* When establishing a connection we need to go through WF_REPORT_PARAMS! 1003 Necessary to do the right thing upon invalidate-remote on a disconnected resource */ 1004 if (oc < C_WF_REPORT_PARAMS && nc >= C_CONNECTED) 1005 return SS_NEED_CONNECTION; 1006 1007 /* After a network error only C_UNCONNECTED or C_DISCONNECTING may follow. */ 1008 if (oc >= C_TIMEOUT && oc <= C_TEAR_DOWN && nc != C_UNCONNECTED && nc != C_DISCONNECTING) 1009 return SS_IN_TRANSIENT_STATE; 1010 1011 /* After C_DISCONNECTING only C_STANDALONE may follow */ 1012 if (oc == C_DISCONNECTING && nc != C_STANDALONE) 1013 return SS_IN_TRANSIENT_STATE; 1014 1015 return SS_SUCCESS; 1016 } 1017 1018 1019 /** 1020 * is_valid_transition() - Returns an SS_ error code if the state transition is not possible 1021 * This limits hard state transitions. Hard state transitions are facts there are 1022 * imposed on DRBD by the environment. E.g. disk broke or network broke down. 1023 * But those hard state transitions are still not allowed to do everything. 1024 * @ns: new state. 1025 * @os: old state. 1026 */ 1027 static enum drbd_state_rv 1028 is_valid_transition(union drbd_state os, union drbd_state ns) 1029 { 1030 enum drbd_state_rv rv; 1031 1032 rv = is_valid_conn_transition(os.conn, ns.conn); 1033 1034 /* we cannot fail (again) if we already detached */ 1035 if (ns.disk == D_FAILED && os.disk == D_DISKLESS) 1036 rv = SS_IS_DISKLESS; 1037 1038 return rv; 1039 } 1040 1041 static void print_sanitize_warnings(struct drbd_device *device, enum sanitize_state_warnings warn) 1042 { 1043 static const char *msg_table[] = { 1044 [NO_WARNING] = "", 1045 [ABORTED_ONLINE_VERIFY] = "Online-verify aborted.", 1046 [ABORTED_RESYNC] = "Resync aborted.", 1047 [CONNECTION_LOST_NEGOTIATING] = "Connection lost while negotiating, no data!", 1048 [IMPLICITLY_UPGRADED_DISK] = "Implicitly upgraded disk", 1049 [IMPLICITLY_UPGRADED_PDSK] = "Implicitly upgraded pdsk", 1050 }; 1051 1052 if (warn != NO_WARNING) 1053 drbd_warn(device, "%s\n", msg_table[warn]); 1054 } 1055 1056 /** 1057 * sanitize_state() - Resolves implicitly necessary additional changes to a state transition 1058 * @device: DRBD device. 1059 * @os: old state. 1060 * @ns: new state. 1061 * @warn_sync_abort: 1062 * 1063 * When we loose connection, we have to set the state of the peers disk (pdsk) 1064 * to D_UNKNOWN. This rule and many more along those lines are in this function. 1065 */ 1066 static union drbd_state sanitize_state(struct drbd_device *device, union drbd_state os, 1067 union drbd_state ns, enum sanitize_state_warnings *warn) 1068 { 1069 enum drbd_fencing_p fp; 1070 enum drbd_disk_state disk_min, disk_max, pdsk_min, pdsk_max; 1071 1072 if (warn) 1073 *warn = NO_WARNING; 1074 1075 fp = FP_DONT_CARE; 1076 if (get_ldev(device)) { 1077 rcu_read_lock(); 1078 fp = rcu_dereference(device->ldev->disk_conf)->fencing; 1079 rcu_read_unlock(); 1080 put_ldev(device); 1081 } 1082 1083 /* Implications from connection to peer and peer_isp */ 1084 if (ns.conn < C_CONNECTED) { 1085 ns.peer_isp = 0; 1086 ns.peer = R_UNKNOWN; 1087 if (ns.pdsk > D_UNKNOWN || ns.pdsk < D_INCONSISTENT) 1088 ns.pdsk = D_UNKNOWN; 1089 } 1090 1091 /* Clear the aftr_isp when becoming unconfigured */ 1092 if (ns.conn == C_STANDALONE && ns.disk == D_DISKLESS && ns.role == R_SECONDARY) 1093 ns.aftr_isp = 0; 1094 1095 /* An implication of the disk states onto the connection state */ 1096 /* Abort resync if a disk fails/detaches */ 1097 if (ns.conn > C_CONNECTED && (ns.disk <= D_FAILED || ns.pdsk <= D_FAILED)) { 1098 if (warn) 1099 *warn = ns.conn == C_VERIFY_S || ns.conn == C_VERIFY_T ? 1100 ABORTED_ONLINE_VERIFY : ABORTED_RESYNC; 1101 ns.conn = C_CONNECTED; 1102 } 1103 1104 /* Connection breaks down before we finished "Negotiating" */ 1105 if (ns.conn < C_CONNECTED && ns.disk == D_NEGOTIATING && 1106 get_ldev_if_state(device, D_NEGOTIATING)) { 1107 if (device->ed_uuid == device->ldev->md.uuid[UI_CURRENT]) { 1108 ns.disk = device->new_state_tmp.disk; 1109 ns.pdsk = device->new_state_tmp.pdsk; 1110 } else { 1111 if (warn) 1112 *warn = CONNECTION_LOST_NEGOTIATING; 1113 ns.disk = D_DISKLESS; 1114 ns.pdsk = D_UNKNOWN; 1115 } 1116 put_ldev(device); 1117 } 1118 1119 /* D_CONSISTENT and D_OUTDATED vanish when we get connected */ 1120 if (ns.conn >= C_CONNECTED && ns.conn < C_AHEAD) { 1121 if (ns.disk == D_CONSISTENT || ns.disk == D_OUTDATED) 1122 ns.disk = D_UP_TO_DATE; 1123 if (ns.pdsk == D_CONSISTENT || ns.pdsk == D_OUTDATED) 1124 ns.pdsk = D_UP_TO_DATE; 1125 } 1126 1127 /* Implications of the connection stat on the disk states */ 1128 disk_min = D_DISKLESS; 1129 disk_max = D_UP_TO_DATE; 1130 pdsk_min = D_INCONSISTENT; 1131 pdsk_max = D_UNKNOWN; 1132 switch ((enum drbd_conns)ns.conn) { 1133 case C_WF_BITMAP_T: 1134 case C_PAUSED_SYNC_T: 1135 case C_STARTING_SYNC_T: 1136 case C_WF_SYNC_UUID: 1137 case C_BEHIND: 1138 disk_min = D_INCONSISTENT; 1139 disk_max = D_OUTDATED; 1140 pdsk_min = D_UP_TO_DATE; 1141 pdsk_max = D_UP_TO_DATE; 1142 break; 1143 case C_VERIFY_S: 1144 case C_VERIFY_T: 1145 disk_min = D_UP_TO_DATE; 1146 disk_max = D_UP_TO_DATE; 1147 pdsk_min = D_UP_TO_DATE; 1148 pdsk_max = D_UP_TO_DATE; 1149 break; 1150 case C_CONNECTED: 1151 disk_min = D_DISKLESS; 1152 disk_max = D_UP_TO_DATE; 1153 pdsk_min = D_DISKLESS; 1154 pdsk_max = D_UP_TO_DATE; 1155 break; 1156 case C_WF_BITMAP_S: 1157 case C_PAUSED_SYNC_S: 1158 case C_STARTING_SYNC_S: 1159 case C_AHEAD: 1160 disk_min = D_UP_TO_DATE; 1161 disk_max = D_UP_TO_DATE; 1162 pdsk_min = D_INCONSISTENT; 1163 pdsk_max = D_CONSISTENT; /* D_OUTDATED would be nice. But explicit outdate necessary*/ 1164 break; 1165 case C_SYNC_TARGET: 1166 disk_min = D_INCONSISTENT; 1167 disk_max = D_INCONSISTENT; 1168 pdsk_min = D_UP_TO_DATE; 1169 pdsk_max = D_UP_TO_DATE; 1170 break; 1171 case C_SYNC_SOURCE: 1172 disk_min = D_UP_TO_DATE; 1173 disk_max = D_UP_TO_DATE; 1174 pdsk_min = D_INCONSISTENT; 1175 pdsk_max = D_INCONSISTENT; 1176 break; 1177 case C_STANDALONE: 1178 case C_DISCONNECTING: 1179 case C_UNCONNECTED: 1180 case C_TIMEOUT: 1181 case C_BROKEN_PIPE: 1182 case C_NETWORK_FAILURE: 1183 case C_PROTOCOL_ERROR: 1184 case C_TEAR_DOWN: 1185 case C_WF_CONNECTION: 1186 case C_WF_REPORT_PARAMS: 1187 case C_MASK: 1188 break; 1189 } 1190 if (ns.disk > disk_max) 1191 ns.disk = disk_max; 1192 1193 if (ns.disk < disk_min) { 1194 if (warn) 1195 *warn = IMPLICITLY_UPGRADED_DISK; 1196 ns.disk = disk_min; 1197 } 1198 if (ns.pdsk > pdsk_max) 1199 ns.pdsk = pdsk_max; 1200 1201 if (ns.pdsk < pdsk_min) { 1202 if (warn) 1203 *warn = IMPLICITLY_UPGRADED_PDSK; 1204 ns.pdsk = pdsk_min; 1205 } 1206 1207 if (fp == FP_STONITH && 1208 (ns.role == R_PRIMARY && ns.conn < C_CONNECTED && ns.pdsk > D_OUTDATED) && 1209 !(os.role == R_PRIMARY && os.conn < C_CONNECTED && os.pdsk > D_OUTDATED)) 1210 ns.susp_fen = 1; /* Suspend IO while fence-peer handler runs (peer lost) */ 1211 1212 if (device->resource->res_opts.on_no_data == OND_SUSPEND_IO && 1213 (ns.role == R_PRIMARY && ns.disk < D_UP_TO_DATE && ns.pdsk < D_UP_TO_DATE) && 1214 !(os.role == R_PRIMARY && os.disk < D_UP_TO_DATE && os.pdsk < D_UP_TO_DATE)) 1215 ns.susp_nod = 1; /* Suspend IO while no data available (no accessible data available) */ 1216 1217 if (ns.aftr_isp || ns.peer_isp || ns.user_isp) { 1218 if (ns.conn == C_SYNC_SOURCE) 1219 ns.conn = C_PAUSED_SYNC_S; 1220 if (ns.conn == C_SYNC_TARGET) 1221 ns.conn = C_PAUSED_SYNC_T; 1222 } else { 1223 if (ns.conn == C_PAUSED_SYNC_S) 1224 ns.conn = C_SYNC_SOURCE; 1225 if (ns.conn == C_PAUSED_SYNC_T) 1226 ns.conn = C_SYNC_TARGET; 1227 } 1228 1229 return ns; 1230 } 1231 1232 void drbd_resume_al(struct drbd_device *device) 1233 { 1234 if (test_and_clear_bit(AL_SUSPENDED, &device->flags)) 1235 drbd_info(device, "Resumed AL updates\n"); 1236 } 1237 1238 /* helper for _drbd_set_state */ 1239 static void set_ov_position(struct drbd_device *device, enum drbd_conns cs) 1240 { 1241 if (first_peer_device(device)->connection->agreed_pro_version < 90) 1242 device->ov_start_sector = 0; 1243 device->rs_total = drbd_bm_bits(device); 1244 device->ov_position = 0; 1245 if (cs == C_VERIFY_T) { 1246 /* starting online verify from an arbitrary position 1247 * does not fit well into the existing protocol. 1248 * on C_VERIFY_T, we initialize ov_left and friends 1249 * implicitly in receive_DataRequest once the 1250 * first P_OV_REQUEST is received */ 1251 device->ov_start_sector = ~(sector_t)0; 1252 } else { 1253 unsigned long bit = BM_SECT_TO_BIT(device->ov_start_sector); 1254 if (bit >= device->rs_total) { 1255 device->ov_start_sector = 1256 BM_BIT_TO_SECT(device->rs_total - 1); 1257 device->rs_total = 1; 1258 } else 1259 device->rs_total -= bit; 1260 device->ov_position = device->ov_start_sector; 1261 } 1262 device->ov_left = device->rs_total; 1263 } 1264 1265 /** 1266 * _drbd_set_state() - Set a new DRBD state 1267 * @device: DRBD device. 1268 * @ns: new state. 1269 * @flags: Flags 1270 * @done: Optional completion, that will get completed after the after_state_ch() finished 1271 * 1272 * Caller needs to hold req_lock. Do not call directly. 1273 */ 1274 enum drbd_state_rv 1275 _drbd_set_state(struct drbd_device *device, union drbd_state ns, 1276 enum chg_state_flags flags, struct completion *done) 1277 { 1278 struct drbd_peer_device *peer_device = first_peer_device(device); 1279 struct drbd_connection *connection = peer_device ? peer_device->connection : NULL; 1280 union drbd_state os; 1281 enum drbd_state_rv rv = SS_SUCCESS; 1282 enum sanitize_state_warnings ssw; 1283 struct after_state_chg_work *ascw; 1284 struct drbd_state_change *state_change; 1285 1286 os = drbd_read_state(device); 1287 1288 ns = sanitize_state(device, os, ns, &ssw); 1289 if (ns.i == os.i) 1290 return SS_NOTHING_TO_DO; 1291 1292 rv = is_valid_transition(os, ns); 1293 if (rv < SS_SUCCESS) 1294 return rv; 1295 1296 if (!(flags & CS_HARD)) { 1297 /* pre-state-change checks ; only look at ns */ 1298 /* See drbd_state_sw_errors in drbd_strings.c */ 1299 1300 rv = is_valid_state(device, ns); 1301 if (rv < SS_SUCCESS) { 1302 /* If the old state was illegal as well, then let 1303 this happen...*/ 1304 1305 if (is_valid_state(device, os) == rv) 1306 rv = is_valid_soft_transition(os, ns, connection); 1307 } else 1308 rv = is_valid_soft_transition(os, ns, connection); 1309 } 1310 1311 if (rv < SS_SUCCESS) { 1312 if (flags & CS_VERBOSE) 1313 print_st_err(device, os, ns, rv); 1314 return rv; 1315 } 1316 1317 print_sanitize_warnings(device, ssw); 1318 1319 drbd_pr_state_change(device, os, ns, flags); 1320 1321 /* Display changes to the susp* flags that where caused by the call to 1322 sanitize_state(). Only display it here if we where not called from 1323 _conn_request_state() */ 1324 if (!(flags & CS_DC_SUSP)) 1325 conn_pr_state_change(connection, os, ns, 1326 (flags & ~CS_DC_MASK) | CS_DC_SUSP); 1327 1328 /* if we are going -> D_FAILED or D_DISKLESS, grab one extra reference 1329 * on the ldev here, to be sure the transition -> D_DISKLESS resp. 1330 * drbd_ldev_destroy() won't happen before our corresponding 1331 * after_state_ch works run, where we put_ldev again. */ 1332 if ((os.disk != D_FAILED && ns.disk == D_FAILED) || 1333 (os.disk != D_DISKLESS && ns.disk == D_DISKLESS)) 1334 atomic_inc(&device->local_cnt); 1335 1336 if (!is_sync_state(os.conn) && is_sync_state(ns.conn)) 1337 clear_bit(RS_DONE, &device->flags); 1338 1339 /* FIXME: Have any flags been set earlier in this function already? */ 1340 state_change = remember_old_state(device->resource, GFP_ATOMIC); 1341 1342 /* changes to local_cnt and device flags should be visible before 1343 * changes to state, which again should be visible before anything else 1344 * depending on that change happens. */ 1345 smp_wmb(); 1346 device->state.i = ns.i; 1347 device->resource->susp = ns.susp; 1348 device->resource->susp_nod = ns.susp_nod; 1349 device->resource->susp_fen = ns.susp_fen; 1350 smp_wmb(); 1351 1352 remember_new_state(state_change); 1353 1354 /* put replicated vs not-replicated requests in seperate epochs */ 1355 if (drbd_should_do_remote((union drbd_dev_state)os.i) != 1356 drbd_should_do_remote((union drbd_dev_state)ns.i)) 1357 start_new_tl_epoch(connection); 1358 1359 if (os.disk == D_ATTACHING && ns.disk >= D_NEGOTIATING) 1360 drbd_print_uuids(device, "attached to UUIDs"); 1361 1362 /* Wake up role changes, that were delayed because of connection establishing */ 1363 if (os.conn == C_WF_REPORT_PARAMS && ns.conn != C_WF_REPORT_PARAMS && 1364 no_peer_wf_report_params(connection)) { 1365 clear_bit(STATE_SENT, &connection->flags); 1366 wake_up_all_devices(connection); 1367 } 1368 1369 wake_up(&device->misc_wait); 1370 wake_up(&device->state_wait); 1371 wake_up(&connection->ping_wait); 1372 1373 /* Aborted verify run, or we reached the stop sector. 1374 * Log the last position, unless end-of-device. */ 1375 if ((os.conn == C_VERIFY_S || os.conn == C_VERIFY_T) && 1376 ns.conn <= C_CONNECTED) { 1377 device->ov_start_sector = 1378 BM_BIT_TO_SECT(drbd_bm_bits(device) - device->ov_left); 1379 if (device->ov_left) 1380 drbd_info(device, "Online Verify reached sector %llu\n", 1381 (unsigned long long)device->ov_start_sector); 1382 } 1383 1384 if ((os.conn == C_PAUSED_SYNC_T || os.conn == C_PAUSED_SYNC_S) && 1385 (ns.conn == C_SYNC_TARGET || ns.conn == C_SYNC_SOURCE)) { 1386 drbd_info(device, "Syncer continues.\n"); 1387 device->rs_paused += (long)jiffies 1388 -(long)device->rs_mark_time[device->rs_last_mark]; 1389 if (ns.conn == C_SYNC_TARGET) 1390 mod_timer(&device->resync_timer, jiffies); 1391 } 1392 1393 if ((os.conn == C_SYNC_TARGET || os.conn == C_SYNC_SOURCE) && 1394 (ns.conn == C_PAUSED_SYNC_T || ns.conn == C_PAUSED_SYNC_S)) { 1395 drbd_info(device, "Resync suspended\n"); 1396 device->rs_mark_time[device->rs_last_mark] = jiffies; 1397 } 1398 1399 if (os.conn == C_CONNECTED && 1400 (ns.conn == C_VERIFY_S || ns.conn == C_VERIFY_T)) { 1401 unsigned long now = jiffies; 1402 int i; 1403 1404 set_ov_position(device, ns.conn); 1405 device->rs_start = now; 1406 device->rs_last_sect_ev = 0; 1407 device->ov_last_oos_size = 0; 1408 device->ov_last_oos_start = 0; 1409 1410 for (i = 0; i < DRBD_SYNC_MARKS; i++) { 1411 device->rs_mark_left[i] = device->ov_left; 1412 device->rs_mark_time[i] = now; 1413 } 1414 1415 drbd_rs_controller_reset(device); 1416 1417 if (ns.conn == C_VERIFY_S) { 1418 drbd_info(device, "Starting Online Verify from sector %llu\n", 1419 (unsigned long long)device->ov_position); 1420 mod_timer(&device->resync_timer, jiffies); 1421 } 1422 } 1423 1424 if (get_ldev(device)) { 1425 u32 mdf = device->ldev->md.flags & ~(MDF_CONSISTENT|MDF_PRIMARY_IND| 1426 MDF_CONNECTED_IND|MDF_WAS_UP_TO_DATE| 1427 MDF_PEER_OUT_DATED|MDF_CRASHED_PRIMARY); 1428 1429 mdf &= ~MDF_AL_CLEAN; 1430 if (test_bit(CRASHED_PRIMARY, &device->flags)) 1431 mdf |= MDF_CRASHED_PRIMARY; 1432 if (device->state.role == R_PRIMARY || 1433 (device->state.pdsk < D_INCONSISTENT && device->state.peer == R_PRIMARY)) 1434 mdf |= MDF_PRIMARY_IND; 1435 if (device->state.conn > C_WF_REPORT_PARAMS) 1436 mdf |= MDF_CONNECTED_IND; 1437 if (device->state.disk > D_INCONSISTENT) 1438 mdf |= MDF_CONSISTENT; 1439 if (device->state.disk > D_OUTDATED) 1440 mdf |= MDF_WAS_UP_TO_DATE; 1441 if (device->state.pdsk <= D_OUTDATED && device->state.pdsk >= D_INCONSISTENT) 1442 mdf |= MDF_PEER_OUT_DATED; 1443 if (mdf != device->ldev->md.flags) { 1444 device->ldev->md.flags = mdf; 1445 drbd_md_mark_dirty(device); 1446 } 1447 if (os.disk < D_CONSISTENT && ns.disk >= D_CONSISTENT) 1448 drbd_set_ed_uuid(device, device->ldev->md.uuid[UI_CURRENT]); 1449 put_ldev(device); 1450 } 1451 1452 /* Peer was forced D_UP_TO_DATE & R_PRIMARY, consider to resync */ 1453 if (os.disk == D_INCONSISTENT && os.pdsk == D_INCONSISTENT && 1454 os.peer == R_SECONDARY && ns.peer == R_PRIMARY) 1455 set_bit(CONSIDER_RESYNC, &device->flags); 1456 1457 /* Receiver should clean up itself */ 1458 if (os.conn != C_DISCONNECTING && ns.conn == C_DISCONNECTING) 1459 drbd_thread_stop_nowait(&connection->receiver); 1460 1461 /* Now the receiver finished cleaning up itself, it should die */ 1462 if (os.conn != C_STANDALONE && ns.conn == C_STANDALONE) 1463 drbd_thread_stop_nowait(&connection->receiver); 1464 1465 /* Upon network failure, we need to restart the receiver. */ 1466 if (os.conn > C_WF_CONNECTION && 1467 ns.conn <= C_TEAR_DOWN && ns.conn >= C_TIMEOUT) 1468 drbd_thread_restart_nowait(&connection->receiver); 1469 1470 /* Resume AL writing if we get a connection */ 1471 if (os.conn < C_CONNECTED && ns.conn >= C_CONNECTED) { 1472 drbd_resume_al(device); 1473 connection->connect_cnt++; 1474 } 1475 1476 /* remember last attach time so request_timer_fn() won't 1477 * kill newly established sessions while we are still trying to thaw 1478 * previously frozen IO */ 1479 if ((os.disk == D_ATTACHING || os.disk == D_NEGOTIATING) && 1480 ns.disk > D_NEGOTIATING) 1481 device->last_reattach_jif = jiffies; 1482 1483 ascw = kmalloc(sizeof(*ascw), GFP_ATOMIC); 1484 if (ascw) { 1485 ascw->os = os; 1486 ascw->ns = ns; 1487 ascw->flags = flags; 1488 ascw->w.cb = w_after_state_ch; 1489 ascw->device = device; 1490 ascw->done = done; 1491 ascw->state_change = state_change; 1492 drbd_queue_work(&connection->sender_work, 1493 &ascw->w); 1494 } else { 1495 drbd_err(device, "Could not kmalloc an ascw\n"); 1496 } 1497 1498 return rv; 1499 } 1500 1501 static int w_after_state_ch(struct drbd_work *w, int unused) 1502 { 1503 struct after_state_chg_work *ascw = 1504 container_of(w, struct after_state_chg_work, w); 1505 struct drbd_device *device = ascw->device; 1506 1507 after_state_ch(device, ascw->os, ascw->ns, ascw->flags, ascw->state_change); 1508 forget_state_change(ascw->state_change); 1509 if (ascw->flags & CS_WAIT_COMPLETE) 1510 complete(ascw->done); 1511 kfree(ascw); 1512 1513 return 0; 1514 } 1515 1516 static void abw_start_sync(struct drbd_device *device, int rv) 1517 { 1518 if (rv) { 1519 drbd_err(device, "Writing the bitmap failed not starting resync.\n"); 1520 _drbd_request_state(device, NS(conn, C_CONNECTED), CS_VERBOSE); 1521 return; 1522 } 1523 1524 switch (device->state.conn) { 1525 case C_STARTING_SYNC_T: 1526 _drbd_request_state(device, NS(conn, C_WF_SYNC_UUID), CS_VERBOSE); 1527 break; 1528 case C_STARTING_SYNC_S: 1529 drbd_start_resync(device, C_SYNC_SOURCE); 1530 break; 1531 } 1532 } 1533 1534 int drbd_bitmap_io_from_worker(struct drbd_device *device, 1535 int (*io_fn)(struct drbd_device *), 1536 char *why, enum bm_flag flags) 1537 { 1538 int rv; 1539 1540 D_ASSERT(device, current == first_peer_device(device)->connection->worker.task); 1541 1542 /* open coded non-blocking drbd_suspend_io(device); */ 1543 atomic_inc(&device->suspend_cnt); 1544 1545 drbd_bm_lock(device, why, flags); 1546 rv = io_fn(device); 1547 drbd_bm_unlock(device); 1548 1549 drbd_resume_io(device); 1550 1551 return rv; 1552 } 1553 1554 void notify_resource_state_change(struct sk_buff *skb, 1555 unsigned int seq, 1556 struct drbd_resource_state_change *resource_state_change, 1557 enum drbd_notification_type type) 1558 { 1559 struct drbd_resource *resource = resource_state_change->resource; 1560 struct resource_info resource_info = { 1561 .res_role = resource_state_change->role[NEW], 1562 .res_susp = resource_state_change->susp[NEW], 1563 .res_susp_nod = resource_state_change->susp_nod[NEW], 1564 .res_susp_fen = resource_state_change->susp_fen[NEW], 1565 }; 1566 1567 notify_resource_state(skb, seq, resource, &resource_info, type); 1568 } 1569 1570 void notify_connection_state_change(struct sk_buff *skb, 1571 unsigned int seq, 1572 struct drbd_connection_state_change *connection_state_change, 1573 enum drbd_notification_type type) 1574 { 1575 struct drbd_connection *connection = connection_state_change->connection; 1576 struct connection_info connection_info = { 1577 .conn_connection_state = connection_state_change->cstate[NEW], 1578 .conn_role = connection_state_change->peer_role[NEW], 1579 }; 1580 1581 notify_connection_state(skb, seq, connection, &connection_info, type); 1582 } 1583 1584 void notify_device_state_change(struct sk_buff *skb, 1585 unsigned int seq, 1586 struct drbd_device_state_change *device_state_change, 1587 enum drbd_notification_type type) 1588 { 1589 struct drbd_device *device = device_state_change->device; 1590 struct device_info device_info = { 1591 .dev_disk_state = device_state_change->disk_state[NEW], 1592 }; 1593 1594 notify_device_state(skb, seq, device, &device_info, type); 1595 } 1596 1597 void notify_peer_device_state_change(struct sk_buff *skb, 1598 unsigned int seq, 1599 struct drbd_peer_device_state_change *p, 1600 enum drbd_notification_type type) 1601 { 1602 struct drbd_peer_device *peer_device = p->peer_device; 1603 struct peer_device_info peer_device_info = { 1604 .peer_repl_state = p->repl_state[NEW], 1605 .peer_disk_state = p->disk_state[NEW], 1606 .peer_resync_susp_user = p->resync_susp_user[NEW], 1607 .peer_resync_susp_peer = p->resync_susp_peer[NEW], 1608 .peer_resync_susp_dependency = p->resync_susp_dependency[NEW], 1609 }; 1610 1611 notify_peer_device_state(skb, seq, peer_device, &peer_device_info, type); 1612 } 1613 1614 static void broadcast_state_change(struct drbd_state_change *state_change) 1615 { 1616 struct drbd_resource_state_change *resource_state_change = &state_change->resource[0]; 1617 bool resource_state_has_changed; 1618 unsigned int n_device, n_connection, n_peer_device, n_peer_devices; 1619 void (*last_func)(struct sk_buff *, unsigned int, void *, 1620 enum drbd_notification_type) = NULL; 1621 void *uninitialized_var(last_arg); 1622 1623 #define HAS_CHANGED(state) ((state)[OLD] != (state)[NEW]) 1624 #define FINAL_STATE_CHANGE(type) \ 1625 ({ if (last_func) \ 1626 last_func(NULL, 0, last_arg, type); \ 1627 }) 1628 #define REMEMBER_STATE_CHANGE(func, arg, type) \ 1629 ({ FINAL_STATE_CHANGE(type | NOTIFY_CONTINUES); \ 1630 last_func = (typeof(last_func))func; \ 1631 last_arg = arg; \ 1632 }) 1633 1634 mutex_lock(¬ification_mutex); 1635 1636 resource_state_has_changed = 1637 HAS_CHANGED(resource_state_change->role) || 1638 HAS_CHANGED(resource_state_change->susp) || 1639 HAS_CHANGED(resource_state_change->susp_nod) || 1640 HAS_CHANGED(resource_state_change->susp_fen); 1641 1642 if (resource_state_has_changed) 1643 REMEMBER_STATE_CHANGE(notify_resource_state_change, 1644 resource_state_change, NOTIFY_CHANGE); 1645 1646 for (n_connection = 0; n_connection < state_change->n_connections; n_connection++) { 1647 struct drbd_connection_state_change *connection_state_change = 1648 &state_change->connections[n_connection]; 1649 1650 if (HAS_CHANGED(connection_state_change->peer_role) || 1651 HAS_CHANGED(connection_state_change->cstate)) 1652 REMEMBER_STATE_CHANGE(notify_connection_state_change, 1653 connection_state_change, NOTIFY_CHANGE); 1654 } 1655 1656 for (n_device = 0; n_device < state_change->n_devices; n_device++) { 1657 struct drbd_device_state_change *device_state_change = 1658 &state_change->devices[n_device]; 1659 1660 if (HAS_CHANGED(device_state_change->disk_state)) 1661 REMEMBER_STATE_CHANGE(notify_device_state_change, 1662 device_state_change, NOTIFY_CHANGE); 1663 } 1664 1665 n_peer_devices = state_change->n_devices * state_change->n_connections; 1666 for (n_peer_device = 0; n_peer_device < n_peer_devices; n_peer_device++) { 1667 struct drbd_peer_device_state_change *p = 1668 &state_change->peer_devices[n_peer_device]; 1669 1670 if (HAS_CHANGED(p->disk_state) || 1671 HAS_CHANGED(p->repl_state) || 1672 HAS_CHANGED(p->resync_susp_user) || 1673 HAS_CHANGED(p->resync_susp_peer) || 1674 HAS_CHANGED(p->resync_susp_dependency)) 1675 REMEMBER_STATE_CHANGE(notify_peer_device_state_change, 1676 p, NOTIFY_CHANGE); 1677 } 1678 1679 FINAL_STATE_CHANGE(NOTIFY_CHANGE); 1680 mutex_unlock(¬ification_mutex); 1681 1682 #undef HAS_CHANGED 1683 #undef FINAL_STATE_CHANGE 1684 #undef REMEMBER_STATE_CHANGE 1685 } 1686 1687 /* takes old and new peer disk state */ 1688 static bool lost_contact_to_peer_data(enum drbd_disk_state os, enum drbd_disk_state ns) 1689 { 1690 if ((os >= D_INCONSISTENT && os != D_UNKNOWN && os != D_OUTDATED) 1691 && (ns < D_INCONSISTENT || ns == D_UNKNOWN || ns == D_OUTDATED)) 1692 return true; 1693 1694 /* Scenario, starting with normal operation 1695 * Connected Primary/Secondary UpToDate/UpToDate 1696 * NetworkFailure Primary/Unknown UpToDate/DUnknown (frozen) 1697 * ... 1698 * Connected Primary/Secondary UpToDate/Diskless (resumed; needs to bump uuid!) 1699 */ 1700 if (os == D_UNKNOWN 1701 && (ns == D_DISKLESS || ns == D_FAILED || ns == D_OUTDATED)) 1702 return true; 1703 1704 return false; 1705 } 1706 1707 /** 1708 * after_state_ch() - Perform after state change actions that may sleep 1709 * @device: DRBD device. 1710 * @os: old state. 1711 * @ns: new state. 1712 * @flags: Flags 1713 */ 1714 static void after_state_ch(struct drbd_device *device, union drbd_state os, 1715 union drbd_state ns, enum chg_state_flags flags, 1716 struct drbd_state_change *state_change) 1717 { 1718 struct drbd_resource *resource = device->resource; 1719 struct drbd_peer_device *peer_device = first_peer_device(device); 1720 struct drbd_connection *connection = peer_device ? peer_device->connection : NULL; 1721 struct sib_info sib; 1722 1723 broadcast_state_change(state_change); 1724 1725 sib.sib_reason = SIB_STATE_CHANGE; 1726 sib.os = os; 1727 sib.ns = ns; 1728 1729 if ((os.disk != D_UP_TO_DATE || os.pdsk != D_UP_TO_DATE) 1730 && (ns.disk == D_UP_TO_DATE && ns.pdsk == D_UP_TO_DATE)) { 1731 clear_bit(CRASHED_PRIMARY, &device->flags); 1732 if (device->p_uuid) 1733 device->p_uuid[UI_FLAGS] &= ~((u64)2); 1734 } 1735 1736 /* Inform userspace about the change... */ 1737 drbd_bcast_event(device, &sib); 1738 1739 if (!(os.role == R_PRIMARY && os.disk < D_UP_TO_DATE && os.pdsk < D_UP_TO_DATE) && 1740 (ns.role == R_PRIMARY && ns.disk < D_UP_TO_DATE && ns.pdsk < D_UP_TO_DATE)) 1741 drbd_khelper(device, "pri-on-incon-degr"); 1742 1743 /* Here we have the actions that are performed after a 1744 state change. This function might sleep */ 1745 1746 if (ns.susp_nod) { 1747 enum drbd_req_event what = NOTHING; 1748 1749 spin_lock_irq(&device->resource->req_lock); 1750 if (os.conn < C_CONNECTED && conn_lowest_conn(connection) >= C_CONNECTED) 1751 what = RESEND; 1752 1753 if ((os.disk == D_ATTACHING || os.disk == D_NEGOTIATING) && 1754 conn_lowest_disk(connection) == D_UP_TO_DATE) 1755 what = RESTART_FROZEN_DISK_IO; 1756 1757 if (resource->susp_nod && what != NOTHING) { 1758 _tl_restart(connection, what); 1759 _conn_request_state(connection, 1760 (union drbd_state) { { .susp_nod = 1 } }, 1761 (union drbd_state) { { .susp_nod = 0 } }, 1762 CS_VERBOSE); 1763 } 1764 spin_unlock_irq(&device->resource->req_lock); 1765 } 1766 1767 if (ns.susp_fen) { 1768 spin_lock_irq(&device->resource->req_lock); 1769 if (resource->susp_fen && conn_lowest_conn(connection) >= C_CONNECTED) { 1770 /* case2: The connection was established again: */ 1771 struct drbd_peer_device *peer_device; 1772 int vnr; 1773 1774 rcu_read_lock(); 1775 idr_for_each_entry(&connection->peer_devices, peer_device, vnr) 1776 clear_bit(NEW_CUR_UUID, &peer_device->device->flags); 1777 rcu_read_unlock(); 1778 1779 /* We should actively create a new uuid, _before_ 1780 * we resume/resent, if the peer is diskless 1781 * (recovery from a multiple error scenario). 1782 * Currently, this happens with a slight delay 1783 * below when checking lost_contact_to_peer_data() ... 1784 */ 1785 _tl_restart(connection, RESEND); 1786 _conn_request_state(connection, 1787 (union drbd_state) { { .susp_fen = 1 } }, 1788 (union drbd_state) { { .susp_fen = 0 } }, 1789 CS_VERBOSE); 1790 } 1791 spin_unlock_irq(&device->resource->req_lock); 1792 } 1793 1794 /* Became sync source. With protocol >= 96, we still need to send out 1795 * the sync uuid now. Need to do that before any drbd_send_state, or 1796 * the other side may go "paused sync" before receiving the sync uuids, 1797 * which is unexpected. */ 1798 if ((os.conn != C_SYNC_SOURCE && os.conn != C_PAUSED_SYNC_S) && 1799 (ns.conn == C_SYNC_SOURCE || ns.conn == C_PAUSED_SYNC_S) && 1800 connection->agreed_pro_version >= 96 && get_ldev(device)) { 1801 drbd_gen_and_send_sync_uuid(peer_device); 1802 put_ldev(device); 1803 } 1804 1805 /* Do not change the order of the if above and the two below... */ 1806 if (os.pdsk == D_DISKLESS && 1807 ns.pdsk > D_DISKLESS && ns.pdsk != D_UNKNOWN) { /* attach on the peer */ 1808 /* we probably will start a resync soon. 1809 * make sure those things are properly reset. */ 1810 device->rs_total = 0; 1811 device->rs_failed = 0; 1812 atomic_set(&device->rs_pending_cnt, 0); 1813 drbd_rs_cancel_all(device); 1814 1815 drbd_send_uuids(peer_device); 1816 drbd_send_state(peer_device, ns); 1817 } 1818 /* No point in queuing send_bitmap if we don't have a connection 1819 * anymore, so check also the _current_ state, not only the new state 1820 * at the time this work was queued. */ 1821 if (os.conn != C_WF_BITMAP_S && ns.conn == C_WF_BITMAP_S && 1822 device->state.conn == C_WF_BITMAP_S) 1823 drbd_queue_bitmap_io(device, &drbd_send_bitmap, NULL, 1824 "send_bitmap (WFBitMapS)", 1825 BM_LOCKED_TEST_ALLOWED); 1826 1827 /* Lost contact to peer's copy of the data */ 1828 if (lost_contact_to_peer_data(os.pdsk, ns.pdsk)) { 1829 if (get_ldev(device)) { 1830 if ((ns.role == R_PRIMARY || ns.peer == R_PRIMARY) && 1831 device->ldev->md.uuid[UI_BITMAP] == 0 && ns.disk >= D_UP_TO_DATE) { 1832 if (drbd_suspended(device)) { 1833 set_bit(NEW_CUR_UUID, &device->flags); 1834 } else { 1835 drbd_uuid_new_current(device); 1836 drbd_send_uuids(peer_device); 1837 } 1838 } 1839 put_ldev(device); 1840 } 1841 } 1842 1843 if (ns.pdsk < D_INCONSISTENT && get_ldev(device)) { 1844 if (os.peer != R_PRIMARY && ns.peer == R_PRIMARY && 1845 device->ldev->md.uuid[UI_BITMAP] == 0 && ns.disk >= D_UP_TO_DATE) { 1846 drbd_uuid_new_current(device); 1847 drbd_send_uuids(peer_device); 1848 } 1849 /* D_DISKLESS Peer becomes secondary */ 1850 if (os.peer == R_PRIMARY && ns.peer == R_SECONDARY) 1851 /* We may still be Primary ourselves. 1852 * No harm done if the bitmap still changes, 1853 * redirtied pages will follow later. */ 1854 drbd_bitmap_io_from_worker(device, &drbd_bm_write, 1855 "demote diskless peer", BM_LOCKED_SET_ALLOWED); 1856 put_ldev(device); 1857 } 1858 1859 /* Write out all changed bits on demote. 1860 * Though, no need to da that just yet 1861 * if there is a resync going on still */ 1862 if (os.role == R_PRIMARY && ns.role == R_SECONDARY && 1863 device->state.conn <= C_CONNECTED && get_ldev(device)) { 1864 /* No changes to the bitmap expected this time, so assert that, 1865 * even though no harm was done if it did change. */ 1866 drbd_bitmap_io_from_worker(device, &drbd_bm_write, 1867 "demote", BM_LOCKED_TEST_ALLOWED); 1868 put_ldev(device); 1869 } 1870 1871 /* Last part of the attaching process ... */ 1872 if (ns.conn >= C_CONNECTED && 1873 os.disk == D_ATTACHING && ns.disk == D_NEGOTIATING) { 1874 drbd_send_sizes(peer_device, 0, 0); /* to start sync... */ 1875 drbd_send_uuids(peer_device); 1876 drbd_send_state(peer_device, ns); 1877 } 1878 1879 /* We want to pause/continue resync, tell peer. */ 1880 if (ns.conn >= C_CONNECTED && 1881 ((os.aftr_isp != ns.aftr_isp) || 1882 (os.user_isp != ns.user_isp))) 1883 drbd_send_state(peer_device, ns); 1884 1885 /* In case one of the isp bits got set, suspend other devices. */ 1886 if ((!os.aftr_isp && !os.peer_isp && !os.user_isp) && 1887 (ns.aftr_isp || ns.peer_isp || ns.user_isp)) 1888 suspend_other_sg(device); 1889 1890 /* Make sure the peer gets informed about eventual state 1891 changes (ISP bits) while we were in WFReportParams. */ 1892 if (os.conn == C_WF_REPORT_PARAMS && ns.conn >= C_CONNECTED) 1893 drbd_send_state(peer_device, ns); 1894 1895 if (os.conn != C_AHEAD && ns.conn == C_AHEAD) 1896 drbd_send_state(peer_device, ns); 1897 1898 /* We are in the progress to start a full sync... */ 1899 if ((os.conn != C_STARTING_SYNC_T && ns.conn == C_STARTING_SYNC_T) || 1900 (os.conn != C_STARTING_SYNC_S && ns.conn == C_STARTING_SYNC_S)) 1901 /* no other bitmap changes expected during this phase */ 1902 drbd_queue_bitmap_io(device, 1903 &drbd_bmio_set_n_write, &abw_start_sync, 1904 "set_n_write from StartingSync", BM_LOCKED_TEST_ALLOWED); 1905 1906 /* first half of local IO error, failure to attach, 1907 * or administrative detach */ 1908 if (os.disk != D_FAILED && ns.disk == D_FAILED) { 1909 enum drbd_io_error_p eh = EP_PASS_ON; 1910 int was_io_error = 0; 1911 /* corresponding get_ldev was in _drbd_set_state, to serialize 1912 * our cleanup here with the transition to D_DISKLESS. 1913 * But is is still not save to dreference ldev here, since 1914 * we might come from an failed Attach before ldev was set. */ 1915 if (device->ldev) { 1916 rcu_read_lock(); 1917 eh = rcu_dereference(device->ldev->disk_conf)->on_io_error; 1918 rcu_read_unlock(); 1919 1920 was_io_error = test_and_clear_bit(WAS_IO_ERROR, &device->flags); 1921 1922 /* Intentionally call this handler first, before drbd_send_state(). 1923 * See: 2932204 drbd: call local-io-error handler early 1924 * People may chose to hard-reset the box from this handler. 1925 * It is useful if this looks like a "regular node crash". */ 1926 if (was_io_error && eh == EP_CALL_HELPER) 1927 drbd_khelper(device, "local-io-error"); 1928 1929 /* Immediately allow completion of all application IO, 1930 * that waits for completion from the local disk, 1931 * if this was a force-detach due to disk_timeout 1932 * or administrator request (drbdsetup detach --force). 1933 * Do NOT abort otherwise. 1934 * Aborting local requests may cause serious problems, 1935 * if requests are completed to upper layers already, 1936 * and then later the already submitted local bio completes. 1937 * This can cause DMA into former bio pages that meanwhile 1938 * have been re-used for other things. 1939 * So aborting local requests may cause crashes, 1940 * or even worse, silent data corruption. 1941 */ 1942 if (test_and_clear_bit(FORCE_DETACH, &device->flags)) 1943 tl_abort_disk_io(device); 1944 1945 /* current state still has to be D_FAILED, 1946 * there is only one way out: to D_DISKLESS, 1947 * and that may only happen after our put_ldev below. */ 1948 if (device->state.disk != D_FAILED) 1949 drbd_err(device, 1950 "ASSERT FAILED: disk is %s during detach\n", 1951 drbd_disk_str(device->state.disk)); 1952 1953 if (ns.conn >= C_CONNECTED) 1954 drbd_send_state(peer_device, ns); 1955 1956 drbd_rs_cancel_all(device); 1957 1958 /* In case we want to get something to stable storage still, 1959 * this may be the last chance. 1960 * Following put_ldev may transition to D_DISKLESS. */ 1961 drbd_md_sync(device); 1962 } 1963 put_ldev(device); 1964 } 1965 1966 /* second half of local IO error, failure to attach, 1967 * or administrative detach, 1968 * after local_cnt references have reached zero again */ 1969 if (os.disk != D_DISKLESS && ns.disk == D_DISKLESS) { 1970 /* We must still be diskless, 1971 * re-attach has to be serialized with this! */ 1972 if (device->state.disk != D_DISKLESS) 1973 drbd_err(device, 1974 "ASSERT FAILED: disk is %s while going diskless\n", 1975 drbd_disk_str(device->state.disk)); 1976 1977 if (ns.conn >= C_CONNECTED) 1978 drbd_send_state(peer_device, ns); 1979 /* corresponding get_ldev in __drbd_set_state 1980 * this may finally trigger drbd_ldev_destroy. */ 1981 put_ldev(device); 1982 } 1983 1984 /* Notify peer that I had a local IO error, and did not detached.. */ 1985 if (os.disk == D_UP_TO_DATE && ns.disk == D_INCONSISTENT && ns.conn >= C_CONNECTED) 1986 drbd_send_state(peer_device, ns); 1987 1988 /* Disks got bigger while they were detached */ 1989 if (ns.disk > D_NEGOTIATING && ns.pdsk > D_NEGOTIATING && 1990 test_and_clear_bit(RESYNC_AFTER_NEG, &device->flags)) { 1991 if (ns.conn == C_CONNECTED) 1992 resync_after_online_grow(device); 1993 } 1994 1995 /* A resync finished or aborted, wake paused devices... */ 1996 if ((os.conn > C_CONNECTED && ns.conn <= C_CONNECTED) || 1997 (os.peer_isp && !ns.peer_isp) || 1998 (os.user_isp && !ns.user_isp)) 1999 resume_next_sg(device); 2000 2001 /* sync target done with resync. Explicitly notify peer, even though 2002 * it should (at least for non-empty resyncs) already know itself. */ 2003 if (os.disk < D_UP_TO_DATE && os.conn >= C_SYNC_SOURCE && ns.conn == C_CONNECTED) 2004 drbd_send_state(peer_device, ns); 2005 2006 /* Verify finished, or reached stop sector. Peer did not know about 2007 * the stop sector, and we may even have changed the stop sector during 2008 * verify to interrupt/stop early. Send the new state. */ 2009 if (os.conn == C_VERIFY_S && ns.conn == C_CONNECTED 2010 && verify_can_do_stop_sector(device)) 2011 drbd_send_state(peer_device, ns); 2012 2013 /* This triggers bitmap writeout of potentially still unwritten pages 2014 * if the resync finished cleanly, or aborted because of peer disk 2015 * failure, or on transition from resync back to AHEAD/BEHIND. 2016 * 2017 * Connection loss is handled in drbd_disconnected() by the receiver. 2018 * 2019 * For resync aborted because of local disk failure, we cannot do 2020 * any bitmap writeout anymore. 2021 * 2022 * No harm done if some bits change during this phase. 2023 */ 2024 if ((os.conn > C_CONNECTED && os.conn < C_AHEAD) && 2025 (ns.conn == C_CONNECTED || ns.conn >= C_AHEAD) && get_ldev(device)) { 2026 drbd_queue_bitmap_io(device, &drbd_bm_write_copy_pages, NULL, 2027 "write from resync_finished", BM_LOCKED_CHANGE_ALLOWED); 2028 put_ldev(device); 2029 } 2030 2031 if (ns.disk == D_DISKLESS && 2032 ns.conn == C_STANDALONE && 2033 ns.role == R_SECONDARY) { 2034 if (os.aftr_isp != ns.aftr_isp) 2035 resume_next_sg(device); 2036 } 2037 2038 drbd_md_sync(device); 2039 } 2040 2041 struct after_conn_state_chg_work { 2042 struct drbd_work w; 2043 enum drbd_conns oc; 2044 union drbd_state ns_min; 2045 union drbd_state ns_max; /* new, max state, over all devices */ 2046 enum chg_state_flags flags; 2047 struct drbd_connection *connection; 2048 struct drbd_state_change *state_change; 2049 }; 2050 2051 static int w_after_conn_state_ch(struct drbd_work *w, int unused) 2052 { 2053 struct after_conn_state_chg_work *acscw = 2054 container_of(w, struct after_conn_state_chg_work, w); 2055 struct drbd_connection *connection = acscw->connection; 2056 enum drbd_conns oc = acscw->oc; 2057 union drbd_state ns_max = acscw->ns_max; 2058 struct drbd_peer_device *peer_device; 2059 int vnr; 2060 2061 broadcast_state_change(acscw->state_change); 2062 forget_state_change(acscw->state_change); 2063 kfree(acscw); 2064 2065 /* Upon network configuration, we need to start the receiver */ 2066 if (oc == C_STANDALONE && ns_max.conn == C_UNCONNECTED) 2067 drbd_thread_start(&connection->receiver); 2068 2069 if (oc == C_DISCONNECTING && ns_max.conn == C_STANDALONE) { 2070 struct net_conf *old_conf; 2071 2072 mutex_lock(¬ification_mutex); 2073 idr_for_each_entry(&connection->peer_devices, peer_device, vnr) 2074 notify_peer_device_state(NULL, 0, peer_device, NULL, 2075 NOTIFY_DESTROY | NOTIFY_CONTINUES); 2076 notify_connection_state(NULL, 0, connection, NULL, NOTIFY_DESTROY); 2077 mutex_unlock(¬ification_mutex); 2078 2079 mutex_lock(&connection->resource->conf_update); 2080 old_conf = connection->net_conf; 2081 connection->my_addr_len = 0; 2082 connection->peer_addr_len = 0; 2083 RCU_INIT_POINTER(connection->net_conf, NULL); 2084 conn_free_crypto(connection); 2085 mutex_unlock(&connection->resource->conf_update); 2086 2087 synchronize_rcu(); 2088 kfree(old_conf); 2089 } 2090 2091 if (ns_max.susp_fen) { 2092 /* case1: The outdate peer handler is successful: */ 2093 if (ns_max.pdsk <= D_OUTDATED) { 2094 rcu_read_lock(); 2095 idr_for_each_entry(&connection->peer_devices, peer_device, vnr) { 2096 struct drbd_device *device = peer_device->device; 2097 if (test_bit(NEW_CUR_UUID, &device->flags)) { 2098 drbd_uuid_new_current(device); 2099 clear_bit(NEW_CUR_UUID, &device->flags); 2100 } 2101 } 2102 rcu_read_unlock(); 2103 spin_lock_irq(&connection->resource->req_lock); 2104 _tl_restart(connection, CONNECTION_LOST_WHILE_PENDING); 2105 _conn_request_state(connection, 2106 (union drbd_state) { { .susp_fen = 1 } }, 2107 (union drbd_state) { { .susp_fen = 0 } }, 2108 CS_VERBOSE); 2109 spin_unlock_irq(&connection->resource->req_lock); 2110 } 2111 } 2112 kref_put(&connection->kref, drbd_destroy_connection); 2113 2114 conn_md_sync(connection); 2115 2116 return 0; 2117 } 2118 2119 static void conn_old_common_state(struct drbd_connection *connection, union drbd_state *pcs, enum chg_state_flags *pf) 2120 { 2121 enum chg_state_flags flags = ~0; 2122 struct drbd_peer_device *peer_device; 2123 int vnr, first_vol = 1; 2124 union drbd_dev_state os, cs = { 2125 { .role = R_SECONDARY, 2126 .peer = R_UNKNOWN, 2127 .conn = connection->cstate, 2128 .disk = D_DISKLESS, 2129 .pdsk = D_UNKNOWN, 2130 } }; 2131 2132 rcu_read_lock(); 2133 idr_for_each_entry(&connection->peer_devices, peer_device, vnr) { 2134 struct drbd_device *device = peer_device->device; 2135 os = device->state; 2136 2137 if (first_vol) { 2138 cs = os; 2139 first_vol = 0; 2140 continue; 2141 } 2142 2143 if (cs.role != os.role) 2144 flags &= ~CS_DC_ROLE; 2145 2146 if (cs.peer != os.peer) 2147 flags &= ~CS_DC_PEER; 2148 2149 if (cs.conn != os.conn) 2150 flags &= ~CS_DC_CONN; 2151 2152 if (cs.disk != os.disk) 2153 flags &= ~CS_DC_DISK; 2154 2155 if (cs.pdsk != os.pdsk) 2156 flags &= ~CS_DC_PDSK; 2157 } 2158 rcu_read_unlock(); 2159 2160 *pf |= CS_DC_MASK; 2161 *pf &= flags; 2162 (*pcs).i = cs.i; 2163 } 2164 2165 static enum drbd_state_rv 2166 conn_is_valid_transition(struct drbd_connection *connection, union drbd_state mask, union drbd_state val, 2167 enum chg_state_flags flags) 2168 { 2169 enum drbd_state_rv rv = SS_SUCCESS; 2170 union drbd_state ns, os; 2171 struct drbd_peer_device *peer_device; 2172 int vnr; 2173 2174 rcu_read_lock(); 2175 idr_for_each_entry(&connection->peer_devices, peer_device, vnr) { 2176 struct drbd_device *device = peer_device->device; 2177 os = drbd_read_state(device); 2178 ns = sanitize_state(device, os, apply_mask_val(os, mask, val), NULL); 2179 2180 if (flags & CS_IGN_OUTD_FAIL && ns.disk == D_OUTDATED && os.disk < D_OUTDATED) 2181 ns.disk = os.disk; 2182 2183 if (ns.i == os.i) 2184 continue; 2185 2186 rv = is_valid_transition(os, ns); 2187 2188 if (rv >= SS_SUCCESS && !(flags & CS_HARD)) { 2189 rv = is_valid_state(device, ns); 2190 if (rv < SS_SUCCESS) { 2191 if (is_valid_state(device, os) == rv) 2192 rv = is_valid_soft_transition(os, ns, connection); 2193 } else 2194 rv = is_valid_soft_transition(os, ns, connection); 2195 } 2196 2197 if (rv < SS_SUCCESS) { 2198 if (flags & CS_VERBOSE) 2199 print_st_err(device, os, ns, rv); 2200 break; 2201 } 2202 } 2203 rcu_read_unlock(); 2204 2205 return rv; 2206 } 2207 2208 static void 2209 conn_set_state(struct drbd_connection *connection, union drbd_state mask, union drbd_state val, 2210 union drbd_state *pns_min, union drbd_state *pns_max, enum chg_state_flags flags) 2211 { 2212 union drbd_state ns, os, ns_max = { }; 2213 union drbd_state ns_min = { 2214 { .role = R_MASK, 2215 .peer = R_MASK, 2216 .conn = val.conn, 2217 .disk = D_MASK, 2218 .pdsk = D_MASK 2219 } }; 2220 struct drbd_peer_device *peer_device; 2221 enum drbd_state_rv rv; 2222 int vnr, number_of_volumes = 0; 2223 2224 if (mask.conn == C_MASK) { 2225 /* remember last connect time so request_timer_fn() won't 2226 * kill newly established sessions while we are still trying to thaw 2227 * previously frozen IO */ 2228 if (connection->cstate != C_WF_REPORT_PARAMS && val.conn == C_WF_REPORT_PARAMS) 2229 connection->last_reconnect_jif = jiffies; 2230 2231 connection->cstate = val.conn; 2232 } 2233 2234 rcu_read_lock(); 2235 idr_for_each_entry(&connection->peer_devices, peer_device, vnr) { 2236 struct drbd_device *device = peer_device->device; 2237 number_of_volumes++; 2238 os = drbd_read_state(device); 2239 ns = apply_mask_val(os, mask, val); 2240 ns = sanitize_state(device, os, ns, NULL); 2241 2242 if (flags & CS_IGN_OUTD_FAIL && ns.disk == D_OUTDATED && os.disk < D_OUTDATED) 2243 ns.disk = os.disk; 2244 2245 rv = _drbd_set_state(device, ns, flags, NULL); 2246 BUG_ON(rv < SS_SUCCESS); 2247 ns.i = device->state.i; 2248 ns_max.role = max_role(ns.role, ns_max.role); 2249 ns_max.peer = max_role(ns.peer, ns_max.peer); 2250 ns_max.conn = max_t(enum drbd_conns, ns.conn, ns_max.conn); 2251 ns_max.disk = max_t(enum drbd_disk_state, ns.disk, ns_max.disk); 2252 ns_max.pdsk = max_t(enum drbd_disk_state, ns.pdsk, ns_max.pdsk); 2253 2254 ns_min.role = min_role(ns.role, ns_min.role); 2255 ns_min.peer = min_role(ns.peer, ns_min.peer); 2256 ns_min.conn = min_t(enum drbd_conns, ns.conn, ns_min.conn); 2257 ns_min.disk = min_t(enum drbd_disk_state, ns.disk, ns_min.disk); 2258 ns_min.pdsk = min_t(enum drbd_disk_state, ns.pdsk, ns_min.pdsk); 2259 } 2260 rcu_read_unlock(); 2261 2262 if (number_of_volumes == 0) { 2263 ns_min = ns_max = (union drbd_state) { { 2264 .role = R_SECONDARY, 2265 .peer = R_UNKNOWN, 2266 .conn = val.conn, 2267 .disk = D_DISKLESS, 2268 .pdsk = D_UNKNOWN 2269 } }; 2270 } 2271 2272 ns_min.susp = ns_max.susp = connection->resource->susp; 2273 ns_min.susp_nod = ns_max.susp_nod = connection->resource->susp_nod; 2274 ns_min.susp_fen = ns_max.susp_fen = connection->resource->susp_fen; 2275 2276 *pns_min = ns_min; 2277 *pns_max = ns_max; 2278 } 2279 2280 static enum drbd_state_rv 2281 _conn_rq_cond(struct drbd_connection *connection, union drbd_state mask, union drbd_state val) 2282 { 2283 enum drbd_state_rv err, rv = SS_UNKNOWN_ERROR; /* continue waiting */; 2284 2285 if (test_and_clear_bit(CONN_WD_ST_CHG_OKAY, &connection->flags)) 2286 rv = SS_CW_SUCCESS; 2287 2288 if (test_and_clear_bit(CONN_WD_ST_CHG_FAIL, &connection->flags)) 2289 rv = SS_CW_FAILED_BY_PEER; 2290 2291 err = conn_is_valid_transition(connection, mask, val, 0); 2292 if (err == SS_SUCCESS && connection->cstate == C_WF_REPORT_PARAMS) 2293 return rv; 2294 2295 return err; 2296 } 2297 2298 enum drbd_state_rv 2299 _conn_request_state(struct drbd_connection *connection, union drbd_state mask, union drbd_state val, 2300 enum chg_state_flags flags) 2301 { 2302 enum drbd_state_rv rv = SS_SUCCESS; 2303 struct after_conn_state_chg_work *acscw; 2304 enum drbd_conns oc = connection->cstate; 2305 union drbd_state ns_max, ns_min, os; 2306 bool have_mutex = false; 2307 struct drbd_state_change *state_change; 2308 2309 if (mask.conn) { 2310 rv = is_valid_conn_transition(oc, val.conn); 2311 if (rv < SS_SUCCESS) 2312 goto abort; 2313 } 2314 2315 rv = conn_is_valid_transition(connection, mask, val, flags); 2316 if (rv < SS_SUCCESS) 2317 goto abort; 2318 2319 if (oc == C_WF_REPORT_PARAMS && val.conn == C_DISCONNECTING && 2320 !(flags & (CS_LOCAL_ONLY | CS_HARD))) { 2321 2322 /* This will be a cluster-wide state change. 2323 * Need to give up the spinlock, grab the mutex, 2324 * then send the state change request, ... */ 2325 spin_unlock_irq(&connection->resource->req_lock); 2326 mutex_lock(&connection->cstate_mutex); 2327 have_mutex = true; 2328 2329 set_bit(CONN_WD_ST_CHG_REQ, &connection->flags); 2330 if (conn_send_state_req(connection, mask, val)) { 2331 /* sending failed. */ 2332 clear_bit(CONN_WD_ST_CHG_REQ, &connection->flags); 2333 rv = SS_CW_FAILED_BY_PEER; 2334 /* need to re-aquire the spin lock, though */ 2335 goto abort_unlocked; 2336 } 2337 2338 if (val.conn == C_DISCONNECTING) 2339 set_bit(DISCONNECT_SENT, &connection->flags); 2340 2341 /* ... and re-aquire the spinlock. 2342 * If _conn_rq_cond() returned >= SS_SUCCESS, we must call 2343 * conn_set_state() within the same spinlock. */ 2344 spin_lock_irq(&connection->resource->req_lock); 2345 wait_event_lock_irq(connection->ping_wait, 2346 (rv = _conn_rq_cond(connection, mask, val)), 2347 connection->resource->req_lock); 2348 clear_bit(CONN_WD_ST_CHG_REQ, &connection->flags); 2349 if (rv < SS_SUCCESS) 2350 goto abort; 2351 } 2352 2353 state_change = remember_old_state(connection->resource, GFP_ATOMIC); 2354 conn_old_common_state(connection, &os, &flags); 2355 flags |= CS_DC_SUSP; 2356 conn_set_state(connection, mask, val, &ns_min, &ns_max, flags); 2357 conn_pr_state_change(connection, os, ns_max, flags); 2358 remember_new_state(state_change); 2359 2360 acscw = kmalloc(sizeof(*acscw), GFP_ATOMIC); 2361 if (acscw) { 2362 acscw->oc = os.conn; 2363 acscw->ns_min = ns_min; 2364 acscw->ns_max = ns_max; 2365 acscw->flags = flags; 2366 acscw->w.cb = w_after_conn_state_ch; 2367 kref_get(&connection->kref); 2368 acscw->connection = connection; 2369 acscw->state_change = state_change; 2370 drbd_queue_work(&connection->sender_work, &acscw->w); 2371 } else { 2372 drbd_err(connection, "Could not kmalloc an acscw\n"); 2373 } 2374 2375 abort: 2376 if (have_mutex) { 2377 /* mutex_unlock() "... must not be used in interrupt context.", 2378 * so give up the spinlock, then re-aquire it */ 2379 spin_unlock_irq(&connection->resource->req_lock); 2380 abort_unlocked: 2381 mutex_unlock(&connection->cstate_mutex); 2382 spin_lock_irq(&connection->resource->req_lock); 2383 } 2384 if (rv < SS_SUCCESS && flags & CS_VERBOSE) { 2385 drbd_err(connection, "State change failed: %s\n", drbd_set_st_err_str(rv)); 2386 drbd_err(connection, " mask = 0x%x val = 0x%x\n", mask.i, val.i); 2387 drbd_err(connection, " old_conn:%s wanted_conn:%s\n", drbd_conn_str(oc), drbd_conn_str(val.conn)); 2388 } 2389 return rv; 2390 } 2391 2392 enum drbd_state_rv 2393 conn_request_state(struct drbd_connection *connection, union drbd_state mask, union drbd_state val, 2394 enum chg_state_flags flags) 2395 { 2396 enum drbd_state_rv rv; 2397 2398 spin_lock_irq(&connection->resource->req_lock); 2399 rv = _conn_request_state(connection, mask, val, flags); 2400 spin_unlock_irq(&connection->resource->req_lock); 2401 2402 return rv; 2403 } 2404