1 /* 2 * fs/f2fs/data.c 3 * 4 * Copyright (c) 2012 Samsung Electronics Co., Ltd. 5 * http://www.samsung.com/ 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 */ 11 #include <linux/fs.h> 12 #include <linux/f2fs_fs.h> 13 #include <linux/buffer_head.h> 14 #include <linux/mpage.h> 15 #include <linux/writeback.h> 16 #include <linux/backing-dev.h> 17 #include <linux/pagevec.h> 18 #include <linux/blkdev.h> 19 #include <linux/bio.h> 20 #include <linux/prefetch.h> 21 #include <linux/uio.h> 22 #include <linux/cleancache.h> 23 24 #include "f2fs.h" 25 #include "node.h" 26 #include "segment.h" 27 #include "trace.h" 28 #include <trace/events/f2fs.h> 29 30 static void f2fs_read_end_io(struct bio *bio) 31 { 32 struct bio_vec *bvec; 33 int i; 34 35 if (f2fs_bio_encrypted(bio)) { 36 if (bio->bi_error) { 37 f2fs_release_crypto_ctx(bio->bi_private); 38 } else { 39 f2fs_end_io_crypto_work(bio->bi_private, bio); 40 return; 41 } 42 } 43 44 bio_for_each_segment_all(bvec, bio, i) { 45 struct page *page = bvec->bv_page; 46 47 if (!bio->bi_error) { 48 SetPageUptodate(page); 49 } else { 50 ClearPageUptodate(page); 51 SetPageError(page); 52 } 53 unlock_page(page); 54 } 55 bio_put(bio); 56 } 57 58 static void f2fs_write_end_io(struct bio *bio) 59 { 60 struct f2fs_sb_info *sbi = bio->bi_private; 61 struct bio_vec *bvec; 62 int i; 63 64 bio_for_each_segment_all(bvec, bio, i) { 65 struct page *page = bvec->bv_page; 66 67 f2fs_restore_and_release_control_page(&page); 68 69 if (unlikely(bio->bi_error)) { 70 set_page_dirty(page); 71 set_bit(AS_EIO, &page->mapping->flags); 72 f2fs_stop_checkpoint(sbi); 73 } 74 end_page_writeback(page); 75 dec_page_count(sbi, F2FS_WRITEBACK); 76 } 77 78 if (!get_pages(sbi, F2FS_WRITEBACK) && 79 !list_empty(&sbi->cp_wait.task_list)) 80 wake_up(&sbi->cp_wait); 81 82 bio_put(bio); 83 } 84 85 /* 86 * Low-level block read/write IO operations. 87 */ 88 static struct bio *__bio_alloc(struct f2fs_sb_info *sbi, block_t blk_addr, 89 int npages, bool is_read) 90 { 91 struct bio *bio; 92 93 bio = f2fs_bio_alloc(npages); 94 95 bio->bi_bdev = sbi->sb->s_bdev; 96 bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(blk_addr); 97 bio->bi_end_io = is_read ? f2fs_read_end_io : f2fs_write_end_io; 98 bio->bi_private = is_read ? NULL : sbi; 99 100 return bio; 101 } 102 103 static void __submit_merged_bio(struct f2fs_bio_info *io) 104 { 105 struct f2fs_io_info *fio = &io->fio; 106 107 if (!io->bio) 108 return; 109 110 if (is_read_io(fio->rw)) 111 trace_f2fs_submit_read_bio(io->sbi->sb, fio, io->bio); 112 else 113 trace_f2fs_submit_write_bio(io->sbi->sb, fio, io->bio); 114 115 submit_bio(fio->rw, io->bio); 116 io->bio = NULL; 117 } 118 119 void f2fs_submit_merged_bio(struct f2fs_sb_info *sbi, 120 enum page_type type, int rw) 121 { 122 enum page_type btype = PAGE_TYPE_OF_BIO(type); 123 struct f2fs_bio_info *io; 124 125 io = is_read_io(rw) ? &sbi->read_io : &sbi->write_io[btype]; 126 127 down_write(&io->io_rwsem); 128 129 /* change META to META_FLUSH in the checkpoint procedure */ 130 if (type >= META_FLUSH) { 131 io->fio.type = META_FLUSH; 132 if (test_opt(sbi, NOBARRIER)) 133 io->fio.rw = WRITE_FLUSH | REQ_META | REQ_PRIO; 134 else 135 io->fio.rw = WRITE_FLUSH_FUA | REQ_META | REQ_PRIO; 136 } 137 __submit_merged_bio(io); 138 up_write(&io->io_rwsem); 139 } 140 141 /* 142 * Fill the locked page with data located in the block address. 143 * Return unlocked page. 144 */ 145 int f2fs_submit_page_bio(struct f2fs_io_info *fio) 146 { 147 struct bio *bio; 148 struct page *page = fio->encrypted_page ? fio->encrypted_page : fio->page; 149 150 trace_f2fs_submit_page_bio(page, fio); 151 f2fs_trace_ios(fio, 0); 152 153 /* Allocate a new bio */ 154 bio = __bio_alloc(fio->sbi, fio->blk_addr, 1, is_read_io(fio->rw)); 155 156 if (bio_add_page(bio, page, PAGE_CACHE_SIZE, 0) < PAGE_CACHE_SIZE) { 157 bio_put(bio); 158 return -EFAULT; 159 } 160 161 submit_bio(fio->rw, bio); 162 return 0; 163 } 164 165 void f2fs_submit_page_mbio(struct f2fs_io_info *fio) 166 { 167 struct f2fs_sb_info *sbi = fio->sbi; 168 enum page_type btype = PAGE_TYPE_OF_BIO(fio->type); 169 struct f2fs_bio_info *io; 170 bool is_read = is_read_io(fio->rw); 171 struct page *bio_page; 172 173 io = is_read ? &sbi->read_io : &sbi->write_io[btype]; 174 175 verify_block_addr(sbi, fio->blk_addr); 176 177 down_write(&io->io_rwsem); 178 179 if (!is_read) 180 inc_page_count(sbi, F2FS_WRITEBACK); 181 182 if (io->bio && (io->last_block_in_bio != fio->blk_addr - 1 || 183 io->fio.rw != fio->rw)) 184 __submit_merged_bio(io); 185 alloc_new: 186 if (io->bio == NULL) { 187 int bio_blocks = MAX_BIO_BLOCKS(sbi); 188 189 io->bio = __bio_alloc(sbi, fio->blk_addr, bio_blocks, is_read); 190 io->fio = *fio; 191 } 192 193 bio_page = fio->encrypted_page ? fio->encrypted_page : fio->page; 194 195 if (bio_add_page(io->bio, bio_page, PAGE_CACHE_SIZE, 0) < 196 PAGE_CACHE_SIZE) { 197 __submit_merged_bio(io); 198 goto alloc_new; 199 } 200 201 io->last_block_in_bio = fio->blk_addr; 202 f2fs_trace_ios(fio, 0); 203 204 up_write(&io->io_rwsem); 205 trace_f2fs_submit_page_mbio(fio->page, fio); 206 } 207 208 /* 209 * Lock ordering for the change of data block address: 210 * ->data_page 211 * ->node_page 212 * update block addresses in the node page 213 */ 214 void set_data_blkaddr(struct dnode_of_data *dn) 215 { 216 struct f2fs_node *rn; 217 __le32 *addr_array; 218 struct page *node_page = dn->node_page; 219 unsigned int ofs_in_node = dn->ofs_in_node; 220 221 f2fs_wait_on_page_writeback(node_page, NODE); 222 223 rn = F2FS_NODE(node_page); 224 225 /* Get physical address of data block */ 226 addr_array = blkaddr_in_node(rn); 227 addr_array[ofs_in_node] = cpu_to_le32(dn->data_blkaddr); 228 set_page_dirty(node_page); 229 } 230 231 int reserve_new_block(struct dnode_of_data *dn) 232 { 233 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); 234 235 if (unlikely(is_inode_flag_set(F2FS_I(dn->inode), FI_NO_ALLOC))) 236 return -EPERM; 237 if (unlikely(!inc_valid_block_count(sbi, dn->inode, 1))) 238 return -ENOSPC; 239 240 trace_f2fs_reserve_new_block(dn->inode, dn->nid, dn->ofs_in_node); 241 242 dn->data_blkaddr = NEW_ADDR; 243 set_data_blkaddr(dn); 244 mark_inode_dirty(dn->inode); 245 sync_inode_page(dn); 246 return 0; 247 } 248 249 int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index) 250 { 251 bool need_put = dn->inode_page ? false : true; 252 int err; 253 254 err = get_dnode_of_data(dn, index, ALLOC_NODE); 255 if (err) 256 return err; 257 258 if (dn->data_blkaddr == NULL_ADDR) 259 err = reserve_new_block(dn); 260 if (err || need_put) 261 f2fs_put_dnode(dn); 262 return err; 263 } 264 265 int f2fs_get_block(struct dnode_of_data *dn, pgoff_t index) 266 { 267 struct extent_info ei; 268 struct inode *inode = dn->inode; 269 270 if (f2fs_lookup_extent_cache(inode, index, &ei)) { 271 dn->data_blkaddr = ei.blk + index - ei.fofs; 272 return 0; 273 } 274 275 return f2fs_reserve_block(dn, index); 276 } 277 278 struct page *get_read_data_page(struct inode *inode, pgoff_t index, 279 int rw, bool for_write) 280 { 281 struct address_space *mapping = inode->i_mapping; 282 struct dnode_of_data dn; 283 struct page *page; 284 struct extent_info ei; 285 int err; 286 struct f2fs_io_info fio = { 287 .sbi = F2FS_I_SB(inode), 288 .type = DATA, 289 .rw = rw, 290 .encrypted_page = NULL, 291 }; 292 293 if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) 294 return read_mapping_page(mapping, index, NULL); 295 296 page = f2fs_grab_cache_page(mapping, index, for_write); 297 if (!page) 298 return ERR_PTR(-ENOMEM); 299 300 if (f2fs_lookup_extent_cache(inode, index, &ei)) { 301 dn.data_blkaddr = ei.blk + index - ei.fofs; 302 goto got_it; 303 } 304 305 set_new_dnode(&dn, inode, NULL, NULL, 0); 306 err = get_dnode_of_data(&dn, index, LOOKUP_NODE); 307 if (err) 308 goto put_err; 309 f2fs_put_dnode(&dn); 310 311 if (unlikely(dn.data_blkaddr == NULL_ADDR)) { 312 err = -ENOENT; 313 goto put_err; 314 } 315 got_it: 316 if (PageUptodate(page)) { 317 unlock_page(page); 318 return page; 319 } 320 321 /* 322 * A new dentry page is allocated but not able to be written, since its 323 * new inode page couldn't be allocated due to -ENOSPC. 324 * In such the case, its blkaddr can be remained as NEW_ADDR. 325 * see, f2fs_add_link -> get_new_data_page -> init_inode_metadata. 326 */ 327 if (dn.data_blkaddr == NEW_ADDR) { 328 zero_user_segment(page, 0, PAGE_CACHE_SIZE); 329 SetPageUptodate(page); 330 unlock_page(page); 331 return page; 332 } 333 334 fio.blk_addr = dn.data_blkaddr; 335 fio.page = page; 336 err = f2fs_submit_page_bio(&fio); 337 if (err) 338 goto put_err; 339 return page; 340 341 put_err: 342 f2fs_put_page(page, 1); 343 return ERR_PTR(err); 344 } 345 346 struct page *find_data_page(struct inode *inode, pgoff_t index) 347 { 348 struct address_space *mapping = inode->i_mapping; 349 struct page *page; 350 351 page = find_get_page(mapping, index); 352 if (page && PageUptodate(page)) 353 return page; 354 f2fs_put_page(page, 0); 355 356 page = get_read_data_page(inode, index, READ_SYNC, false); 357 if (IS_ERR(page)) 358 return page; 359 360 if (PageUptodate(page)) 361 return page; 362 363 wait_on_page_locked(page); 364 if (unlikely(!PageUptodate(page))) { 365 f2fs_put_page(page, 0); 366 return ERR_PTR(-EIO); 367 } 368 return page; 369 } 370 371 /* 372 * If it tries to access a hole, return an error. 373 * Because, the callers, functions in dir.c and GC, should be able to know 374 * whether this page exists or not. 375 */ 376 struct page *get_lock_data_page(struct inode *inode, pgoff_t index, 377 bool for_write) 378 { 379 struct address_space *mapping = inode->i_mapping; 380 struct page *page; 381 repeat: 382 page = get_read_data_page(inode, index, READ_SYNC, for_write); 383 if (IS_ERR(page)) 384 return page; 385 386 /* wait for read completion */ 387 lock_page(page); 388 if (unlikely(!PageUptodate(page))) { 389 f2fs_put_page(page, 1); 390 return ERR_PTR(-EIO); 391 } 392 if (unlikely(page->mapping != mapping)) { 393 f2fs_put_page(page, 1); 394 goto repeat; 395 } 396 return page; 397 } 398 399 /* 400 * Caller ensures that this data page is never allocated. 401 * A new zero-filled data page is allocated in the page cache. 402 * 403 * Also, caller should grab and release a rwsem by calling f2fs_lock_op() and 404 * f2fs_unlock_op(). 405 * Note that, ipage is set only by make_empty_dir, and if any error occur, 406 * ipage should be released by this function. 407 */ 408 struct page *get_new_data_page(struct inode *inode, 409 struct page *ipage, pgoff_t index, bool new_i_size) 410 { 411 struct address_space *mapping = inode->i_mapping; 412 struct page *page; 413 struct dnode_of_data dn; 414 int err; 415 repeat: 416 page = f2fs_grab_cache_page(mapping, index, true); 417 if (!page) { 418 /* 419 * before exiting, we should make sure ipage will be released 420 * if any error occur. 421 */ 422 f2fs_put_page(ipage, 1); 423 return ERR_PTR(-ENOMEM); 424 } 425 426 set_new_dnode(&dn, inode, ipage, NULL, 0); 427 err = f2fs_reserve_block(&dn, index); 428 if (err) { 429 f2fs_put_page(page, 1); 430 return ERR_PTR(err); 431 } 432 if (!ipage) 433 f2fs_put_dnode(&dn); 434 435 if (PageUptodate(page)) 436 goto got_it; 437 438 if (dn.data_blkaddr == NEW_ADDR) { 439 zero_user_segment(page, 0, PAGE_CACHE_SIZE); 440 SetPageUptodate(page); 441 } else { 442 f2fs_put_page(page, 1); 443 444 page = get_read_data_page(inode, index, READ_SYNC, true); 445 if (IS_ERR(page)) 446 goto repeat; 447 448 /* wait for read completion */ 449 lock_page(page); 450 } 451 got_it: 452 if (new_i_size && i_size_read(inode) < 453 ((loff_t)(index + 1) << PAGE_CACHE_SHIFT)) { 454 i_size_write(inode, ((loff_t)(index + 1) << PAGE_CACHE_SHIFT)); 455 /* Only the directory inode sets new_i_size */ 456 set_inode_flag(F2FS_I(inode), FI_UPDATE_DIR); 457 } 458 return page; 459 } 460 461 static int __allocate_data_block(struct dnode_of_data *dn) 462 { 463 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); 464 struct f2fs_inode_info *fi = F2FS_I(dn->inode); 465 struct f2fs_summary sum; 466 struct node_info ni; 467 int seg = CURSEG_WARM_DATA; 468 pgoff_t fofs; 469 470 if (unlikely(is_inode_flag_set(F2FS_I(dn->inode), FI_NO_ALLOC))) 471 return -EPERM; 472 473 dn->data_blkaddr = datablock_addr(dn->node_page, dn->ofs_in_node); 474 if (dn->data_blkaddr == NEW_ADDR) 475 goto alloc; 476 477 if (unlikely(!inc_valid_block_count(sbi, dn->inode, 1))) 478 return -ENOSPC; 479 480 alloc: 481 get_node_info(sbi, dn->nid, &ni); 482 set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version); 483 484 if (dn->ofs_in_node == 0 && dn->inode_page == dn->node_page) 485 seg = CURSEG_DIRECT_IO; 486 487 allocate_data_block(sbi, NULL, dn->data_blkaddr, &dn->data_blkaddr, 488 &sum, seg); 489 set_data_blkaddr(dn); 490 491 /* update i_size */ 492 fofs = start_bidx_of_node(ofs_of_node(dn->node_page), fi) + 493 dn->ofs_in_node; 494 if (i_size_read(dn->inode) < ((loff_t)(fofs + 1) << PAGE_CACHE_SHIFT)) 495 i_size_write(dn->inode, 496 ((loff_t)(fofs + 1) << PAGE_CACHE_SHIFT)); 497 498 /* direct IO doesn't use extent cache to maximize the performance */ 499 f2fs_drop_largest_extent(dn->inode, fofs); 500 501 return 0; 502 } 503 504 static void __allocate_data_blocks(struct inode *inode, loff_t offset, 505 size_t count) 506 { 507 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 508 struct dnode_of_data dn; 509 u64 start = F2FS_BYTES_TO_BLK(offset); 510 u64 len = F2FS_BYTES_TO_BLK(count); 511 bool allocated; 512 u64 end_offset; 513 514 while (len) { 515 f2fs_balance_fs(sbi); 516 f2fs_lock_op(sbi); 517 518 /* When reading holes, we need its node page */ 519 set_new_dnode(&dn, inode, NULL, NULL, 0); 520 if (get_dnode_of_data(&dn, start, ALLOC_NODE)) 521 goto out; 522 523 allocated = false; 524 end_offset = ADDRS_PER_PAGE(dn.node_page, F2FS_I(inode)); 525 526 while (dn.ofs_in_node < end_offset && len) { 527 block_t blkaddr; 528 529 if (unlikely(f2fs_cp_error(sbi))) 530 goto sync_out; 531 532 blkaddr = datablock_addr(dn.node_page, dn.ofs_in_node); 533 if (blkaddr == NULL_ADDR || blkaddr == NEW_ADDR) { 534 if (__allocate_data_block(&dn)) 535 goto sync_out; 536 allocated = true; 537 } 538 len--; 539 start++; 540 dn.ofs_in_node++; 541 } 542 543 if (allocated) 544 sync_inode_page(&dn); 545 546 f2fs_put_dnode(&dn); 547 f2fs_unlock_op(sbi); 548 } 549 return; 550 551 sync_out: 552 if (allocated) 553 sync_inode_page(&dn); 554 f2fs_put_dnode(&dn); 555 out: 556 f2fs_unlock_op(sbi); 557 return; 558 } 559 560 /* 561 * f2fs_map_blocks() now supported readahead/bmap/rw direct_IO with 562 * f2fs_map_blocks structure. 563 * If original data blocks are allocated, then give them to blockdev. 564 * Otherwise, 565 * a. preallocate requested block addresses 566 * b. do not use extent cache for better performance 567 * c. give the block addresses to blockdev 568 */ 569 static int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, 570 int create, int flag) 571 { 572 unsigned int maxblocks = map->m_len; 573 struct dnode_of_data dn; 574 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 575 int mode = create ? ALLOC_NODE : LOOKUP_NODE_RA; 576 pgoff_t pgofs, end_offset; 577 int err = 0, ofs = 1; 578 struct extent_info ei; 579 bool allocated = false; 580 581 map->m_len = 0; 582 map->m_flags = 0; 583 584 /* it only supports block size == page size */ 585 pgofs = (pgoff_t)map->m_lblk; 586 587 if (f2fs_lookup_extent_cache(inode, pgofs, &ei)) { 588 map->m_pblk = ei.blk + pgofs - ei.fofs; 589 map->m_len = min((pgoff_t)maxblocks, ei.fofs + ei.len - pgofs); 590 map->m_flags = F2FS_MAP_MAPPED; 591 goto out; 592 } 593 594 if (create) 595 f2fs_lock_op(F2FS_I_SB(inode)); 596 597 /* When reading holes, we need its node page */ 598 set_new_dnode(&dn, inode, NULL, NULL, 0); 599 err = get_dnode_of_data(&dn, pgofs, mode); 600 if (err) { 601 if (err == -ENOENT) 602 err = 0; 603 goto unlock_out; 604 } 605 606 if (dn.data_blkaddr == NEW_ADDR || dn.data_blkaddr == NULL_ADDR) { 607 if (create) { 608 if (unlikely(f2fs_cp_error(sbi))) { 609 err = -EIO; 610 goto put_out; 611 } 612 err = __allocate_data_block(&dn); 613 if (err) 614 goto put_out; 615 allocated = true; 616 map->m_flags = F2FS_MAP_NEW; 617 } else { 618 if (flag != F2FS_GET_BLOCK_FIEMAP || 619 dn.data_blkaddr != NEW_ADDR) { 620 if (flag == F2FS_GET_BLOCK_BMAP) 621 err = -ENOENT; 622 goto put_out; 623 } 624 625 /* 626 * preallocated unwritten block should be mapped 627 * for fiemap. 628 */ 629 if (dn.data_blkaddr == NEW_ADDR) 630 map->m_flags = F2FS_MAP_UNWRITTEN; 631 } 632 } 633 634 map->m_flags |= F2FS_MAP_MAPPED; 635 map->m_pblk = dn.data_blkaddr; 636 map->m_len = 1; 637 638 end_offset = ADDRS_PER_PAGE(dn.node_page, F2FS_I(inode)); 639 dn.ofs_in_node++; 640 pgofs++; 641 642 get_next: 643 if (dn.ofs_in_node >= end_offset) { 644 if (allocated) 645 sync_inode_page(&dn); 646 allocated = false; 647 f2fs_put_dnode(&dn); 648 649 set_new_dnode(&dn, inode, NULL, NULL, 0); 650 err = get_dnode_of_data(&dn, pgofs, mode); 651 if (err) { 652 if (err == -ENOENT) 653 err = 0; 654 goto unlock_out; 655 } 656 657 end_offset = ADDRS_PER_PAGE(dn.node_page, F2FS_I(inode)); 658 } 659 660 if (maxblocks > map->m_len) { 661 block_t blkaddr = datablock_addr(dn.node_page, dn.ofs_in_node); 662 663 if (blkaddr == NEW_ADDR || blkaddr == NULL_ADDR) { 664 if (create) { 665 if (unlikely(f2fs_cp_error(sbi))) { 666 err = -EIO; 667 goto sync_out; 668 } 669 err = __allocate_data_block(&dn); 670 if (err) 671 goto sync_out; 672 allocated = true; 673 map->m_flags |= F2FS_MAP_NEW; 674 blkaddr = dn.data_blkaddr; 675 } else { 676 /* 677 * we only merge preallocated unwritten blocks 678 * for fiemap. 679 */ 680 if (flag != F2FS_GET_BLOCK_FIEMAP || 681 blkaddr != NEW_ADDR) 682 goto sync_out; 683 } 684 } 685 686 /* Give more consecutive addresses for the readahead */ 687 if ((map->m_pblk != NEW_ADDR && 688 blkaddr == (map->m_pblk + ofs)) || 689 (map->m_pblk == NEW_ADDR && 690 blkaddr == NEW_ADDR)) { 691 ofs++; 692 dn.ofs_in_node++; 693 pgofs++; 694 map->m_len++; 695 goto get_next; 696 } 697 } 698 sync_out: 699 if (allocated) 700 sync_inode_page(&dn); 701 put_out: 702 f2fs_put_dnode(&dn); 703 unlock_out: 704 if (create) 705 f2fs_unlock_op(F2FS_I_SB(inode)); 706 out: 707 trace_f2fs_map_blocks(inode, map, err); 708 return err; 709 } 710 711 static int __get_data_block(struct inode *inode, sector_t iblock, 712 struct buffer_head *bh, int create, int flag) 713 { 714 struct f2fs_map_blocks map; 715 int ret; 716 717 map.m_lblk = iblock; 718 map.m_len = bh->b_size >> inode->i_blkbits; 719 720 ret = f2fs_map_blocks(inode, &map, create, flag); 721 if (!ret) { 722 map_bh(bh, inode->i_sb, map.m_pblk); 723 bh->b_state = (bh->b_state & ~F2FS_MAP_FLAGS) | map.m_flags; 724 bh->b_size = map.m_len << inode->i_blkbits; 725 } 726 return ret; 727 } 728 729 static int get_data_block(struct inode *inode, sector_t iblock, 730 struct buffer_head *bh_result, int create, int flag) 731 { 732 return __get_data_block(inode, iblock, bh_result, create, flag); 733 } 734 735 static int get_data_block_dio(struct inode *inode, sector_t iblock, 736 struct buffer_head *bh_result, int create) 737 { 738 return __get_data_block(inode, iblock, bh_result, create, 739 F2FS_GET_BLOCK_DIO); 740 } 741 742 static int get_data_block_bmap(struct inode *inode, sector_t iblock, 743 struct buffer_head *bh_result, int create) 744 { 745 return __get_data_block(inode, iblock, bh_result, create, 746 F2FS_GET_BLOCK_BMAP); 747 } 748 749 static inline sector_t logical_to_blk(struct inode *inode, loff_t offset) 750 { 751 return (offset >> inode->i_blkbits); 752 } 753 754 static inline loff_t blk_to_logical(struct inode *inode, sector_t blk) 755 { 756 return (blk << inode->i_blkbits); 757 } 758 759 int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, 760 u64 start, u64 len) 761 { 762 struct buffer_head map_bh; 763 sector_t start_blk, last_blk; 764 loff_t isize = i_size_read(inode); 765 u64 logical = 0, phys = 0, size = 0; 766 u32 flags = 0; 767 bool past_eof = false, whole_file = false; 768 int ret = 0; 769 770 ret = fiemap_check_flags(fieinfo, FIEMAP_FLAG_SYNC); 771 if (ret) 772 return ret; 773 774 if (f2fs_has_inline_data(inode)) { 775 ret = f2fs_inline_data_fiemap(inode, fieinfo, start, len); 776 if (ret != -EAGAIN) 777 return ret; 778 } 779 780 mutex_lock(&inode->i_mutex); 781 782 if (len >= isize) { 783 whole_file = true; 784 len = isize; 785 } 786 787 if (logical_to_blk(inode, len) == 0) 788 len = blk_to_logical(inode, 1); 789 790 start_blk = logical_to_blk(inode, start); 791 last_blk = logical_to_blk(inode, start + len - 1); 792 next: 793 memset(&map_bh, 0, sizeof(struct buffer_head)); 794 map_bh.b_size = len; 795 796 ret = get_data_block(inode, start_blk, &map_bh, 0, 797 F2FS_GET_BLOCK_FIEMAP); 798 if (ret) 799 goto out; 800 801 /* HOLE */ 802 if (!buffer_mapped(&map_bh)) { 803 start_blk++; 804 805 if (!past_eof && blk_to_logical(inode, start_blk) >= isize) 806 past_eof = 1; 807 808 if (past_eof && size) { 809 flags |= FIEMAP_EXTENT_LAST; 810 ret = fiemap_fill_next_extent(fieinfo, logical, 811 phys, size, flags); 812 } else if (size) { 813 ret = fiemap_fill_next_extent(fieinfo, logical, 814 phys, size, flags); 815 size = 0; 816 } 817 818 /* if we have holes up to/past EOF then we're done */ 819 if (start_blk > last_blk || past_eof || ret) 820 goto out; 821 } else { 822 if (start_blk > last_blk && !whole_file) { 823 ret = fiemap_fill_next_extent(fieinfo, logical, 824 phys, size, flags); 825 goto out; 826 } 827 828 /* 829 * if size != 0 then we know we already have an extent 830 * to add, so add it. 831 */ 832 if (size) { 833 ret = fiemap_fill_next_extent(fieinfo, logical, 834 phys, size, flags); 835 if (ret) 836 goto out; 837 } 838 839 logical = blk_to_logical(inode, start_blk); 840 phys = blk_to_logical(inode, map_bh.b_blocknr); 841 size = map_bh.b_size; 842 flags = 0; 843 if (buffer_unwritten(&map_bh)) 844 flags = FIEMAP_EXTENT_UNWRITTEN; 845 846 start_blk += logical_to_blk(inode, size); 847 848 /* 849 * If we are past the EOF, then we need to make sure as 850 * soon as we find a hole that the last extent we found 851 * is marked with FIEMAP_EXTENT_LAST 852 */ 853 if (!past_eof && logical + size >= isize) 854 past_eof = true; 855 } 856 cond_resched(); 857 if (fatal_signal_pending(current)) 858 ret = -EINTR; 859 else 860 goto next; 861 out: 862 if (ret == 1) 863 ret = 0; 864 865 mutex_unlock(&inode->i_mutex); 866 return ret; 867 } 868 869 /* 870 * This function was originally taken from fs/mpage.c, and customized for f2fs. 871 * Major change was from block_size == page_size in f2fs by default. 872 */ 873 static int f2fs_mpage_readpages(struct address_space *mapping, 874 struct list_head *pages, struct page *page, 875 unsigned nr_pages) 876 { 877 struct bio *bio = NULL; 878 unsigned page_idx; 879 sector_t last_block_in_bio = 0; 880 struct inode *inode = mapping->host; 881 const unsigned blkbits = inode->i_blkbits; 882 const unsigned blocksize = 1 << blkbits; 883 sector_t block_in_file; 884 sector_t last_block; 885 sector_t last_block_in_file; 886 sector_t block_nr; 887 struct block_device *bdev = inode->i_sb->s_bdev; 888 struct f2fs_map_blocks map; 889 890 map.m_pblk = 0; 891 map.m_lblk = 0; 892 map.m_len = 0; 893 map.m_flags = 0; 894 895 for (page_idx = 0; nr_pages; page_idx++, nr_pages--) { 896 897 prefetchw(&page->flags); 898 if (pages) { 899 page = list_entry(pages->prev, struct page, lru); 900 list_del(&page->lru); 901 if (add_to_page_cache_lru(page, mapping, 902 page->index, GFP_KERNEL)) 903 goto next_page; 904 } 905 906 block_in_file = (sector_t)page->index; 907 last_block = block_in_file + nr_pages; 908 last_block_in_file = (i_size_read(inode) + blocksize - 1) >> 909 blkbits; 910 if (last_block > last_block_in_file) 911 last_block = last_block_in_file; 912 913 /* 914 * Map blocks using the previous result first. 915 */ 916 if ((map.m_flags & F2FS_MAP_MAPPED) && 917 block_in_file > map.m_lblk && 918 block_in_file < (map.m_lblk + map.m_len)) 919 goto got_it; 920 921 /* 922 * Then do more f2fs_map_blocks() calls until we are 923 * done with this page. 924 */ 925 map.m_flags = 0; 926 927 if (block_in_file < last_block) { 928 map.m_lblk = block_in_file; 929 map.m_len = last_block - block_in_file; 930 931 if (f2fs_map_blocks(inode, &map, 0, 932 F2FS_GET_BLOCK_READ)) 933 goto set_error_page; 934 } 935 got_it: 936 if ((map.m_flags & F2FS_MAP_MAPPED)) { 937 block_nr = map.m_pblk + block_in_file - map.m_lblk; 938 SetPageMappedToDisk(page); 939 940 if (!PageUptodate(page) && !cleancache_get_page(page)) { 941 SetPageUptodate(page); 942 goto confused; 943 } 944 } else { 945 zero_user_segment(page, 0, PAGE_CACHE_SIZE); 946 SetPageUptodate(page); 947 unlock_page(page); 948 goto next_page; 949 } 950 951 /* 952 * This page will go to BIO. Do we need to send this 953 * BIO off first? 954 */ 955 if (bio && (last_block_in_bio != block_nr - 1)) { 956 submit_and_realloc: 957 submit_bio(READ, bio); 958 bio = NULL; 959 } 960 if (bio == NULL) { 961 struct f2fs_crypto_ctx *ctx = NULL; 962 963 if (f2fs_encrypted_inode(inode) && 964 S_ISREG(inode->i_mode)) { 965 966 ctx = f2fs_get_crypto_ctx(inode); 967 if (IS_ERR(ctx)) 968 goto set_error_page; 969 970 /* wait the page to be moved by cleaning */ 971 f2fs_wait_on_encrypted_page_writeback( 972 F2FS_I_SB(inode), block_nr); 973 } 974 975 bio = bio_alloc(GFP_KERNEL, 976 min_t(int, nr_pages, BIO_MAX_PAGES)); 977 if (!bio) { 978 if (ctx) 979 f2fs_release_crypto_ctx(ctx); 980 goto set_error_page; 981 } 982 bio->bi_bdev = bdev; 983 bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(block_nr); 984 bio->bi_end_io = f2fs_read_end_io; 985 bio->bi_private = ctx; 986 } 987 988 if (bio_add_page(bio, page, blocksize, 0) < blocksize) 989 goto submit_and_realloc; 990 991 last_block_in_bio = block_nr; 992 goto next_page; 993 set_error_page: 994 SetPageError(page); 995 zero_user_segment(page, 0, PAGE_CACHE_SIZE); 996 unlock_page(page); 997 goto next_page; 998 confused: 999 if (bio) { 1000 submit_bio(READ, bio); 1001 bio = NULL; 1002 } 1003 unlock_page(page); 1004 next_page: 1005 if (pages) 1006 page_cache_release(page); 1007 } 1008 BUG_ON(pages && !list_empty(pages)); 1009 if (bio) 1010 submit_bio(READ, bio); 1011 return 0; 1012 } 1013 1014 static int f2fs_read_data_page(struct file *file, struct page *page) 1015 { 1016 struct inode *inode = page->mapping->host; 1017 int ret = -EAGAIN; 1018 1019 trace_f2fs_readpage(page, DATA); 1020 1021 /* If the file has inline data, try to read it directly */ 1022 if (f2fs_has_inline_data(inode)) 1023 ret = f2fs_read_inline_data(inode, page); 1024 if (ret == -EAGAIN) 1025 ret = f2fs_mpage_readpages(page->mapping, NULL, page, 1); 1026 return ret; 1027 } 1028 1029 static int f2fs_read_data_pages(struct file *file, 1030 struct address_space *mapping, 1031 struct list_head *pages, unsigned nr_pages) 1032 { 1033 struct inode *inode = file->f_mapping->host; 1034 struct page *page = list_entry(pages->prev, struct page, lru); 1035 1036 trace_f2fs_readpages(inode, page, nr_pages); 1037 1038 /* If the file has inline data, skip readpages */ 1039 if (f2fs_has_inline_data(inode)) 1040 return 0; 1041 1042 return f2fs_mpage_readpages(mapping, pages, NULL, nr_pages); 1043 } 1044 1045 int do_write_data_page(struct f2fs_io_info *fio) 1046 { 1047 struct page *page = fio->page; 1048 struct inode *inode = page->mapping->host; 1049 struct dnode_of_data dn; 1050 int err = 0; 1051 1052 set_new_dnode(&dn, inode, NULL, NULL, 0); 1053 err = get_dnode_of_data(&dn, page->index, LOOKUP_NODE); 1054 if (err) 1055 return err; 1056 1057 fio->blk_addr = dn.data_blkaddr; 1058 1059 /* This page is already truncated */ 1060 if (fio->blk_addr == NULL_ADDR) { 1061 ClearPageUptodate(page); 1062 goto out_writepage; 1063 } 1064 1065 if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) { 1066 1067 /* wait for GCed encrypted page writeback */ 1068 f2fs_wait_on_encrypted_page_writeback(F2FS_I_SB(inode), 1069 fio->blk_addr); 1070 1071 fio->encrypted_page = f2fs_encrypt(inode, fio->page); 1072 if (IS_ERR(fio->encrypted_page)) { 1073 err = PTR_ERR(fio->encrypted_page); 1074 goto out_writepage; 1075 } 1076 } 1077 1078 set_page_writeback(page); 1079 1080 /* 1081 * If current allocation needs SSR, 1082 * it had better in-place writes for updated data. 1083 */ 1084 if (unlikely(fio->blk_addr != NEW_ADDR && 1085 !is_cold_data(page) && 1086 need_inplace_update(inode))) { 1087 rewrite_data_page(fio); 1088 set_inode_flag(F2FS_I(inode), FI_UPDATE_WRITE); 1089 trace_f2fs_do_write_data_page(page, IPU); 1090 } else { 1091 write_data_page(&dn, fio); 1092 set_data_blkaddr(&dn); 1093 f2fs_update_extent_cache(&dn); 1094 trace_f2fs_do_write_data_page(page, OPU); 1095 set_inode_flag(F2FS_I(inode), FI_APPEND_WRITE); 1096 if (page->index == 0) 1097 set_inode_flag(F2FS_I(inode), FI_FIRST_BLOCK_WRITTEN); 1098 } 1099 out_writepage: 1100 f2fs_put_dnode(&dn); 1101 return err; 1102 } 1103 1104 static int f2fs_write_data_page(struct page *page, 1105 struct writeback_control *wbc) 1106 { 1107 struct inode *inode = page->mapping->host; 1108 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1109 loff_t i_size = i_size_read(inode); 1110 const pgoff_t end_index = ((unsigned long long) i_size) 1111 >> PAGE_CACHE_SHIFT; 1112 unsigned offset = 0; 1113 bool need_balance_fs = false; 1114 int err = 0; 1115 struct f2fs_io_info fio = { 1116 .sbi = sbi, 1117 .type = DATA, 1118 .rw = (wbc->sync_mode == WB_SYNC_ALL) ? WRITE_SYNC : WRITE, 1119 .page = page, 1120 .encrypted_page = NULL, 1121 }; 1122 1123 trace_f2fs_writepage(page, DATA); 1124 1125 if (page->index < end_index) 1126 goto write; 1127 1128 /* 1129 * If the offset is out-of-range of file size, 1130 * this page does not have to be written to disk. 1131 */ 1132 offset = i_size & (PAGE_CACHE_SIZE - 1); 1133 if ((page->index >= end_index + 1) || !offset) 1134 goto out; 1135 1136 zero_user_segment(page, offset, PAGE_CACHE_SIZE); 1137 write: 1138 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) 1139 goto redirty_out; 1140 if (f2fs_is_drop_cache(inode)) 1141 goto out; 1142 if (f2fs_is_volatile_file(inode) && !wbc->for_reclaim && 1143 available_free_memory(sbi, BASE_CHECK)) 1144 goto redirty_out; 1145 1146 /* Dentry blocks are controlled by checkpoint */ 1147 if (S_ISDIR(inode->i_mode)) { 1148 if (unlikely(f2fs_cp_error(sbi))) 1149 goto redirty_out; 1150 err = do_write_data_page(&fio); 1151 goto done; 1152 } 1153 1154 /* we should bypass data pages to proceed the kworkder jobs */ 1155 if (unlikely(f2fs_cp_error(sbi))) { 1156 SetPageError(page); 1157 goto out; 1158 } 1159 1160 if (!wbc->for_reclaim) 1161 need_balance_fs = true; 1162 else if (has_not_enough_free_secs(sbi, 0)) 1163 goto redirty_out; 1164 1165 err = -EAGAIN; 1166 f2fs_lock_op(sbi); 1167 if (f2fs_has_inline_data(inode)) 1168 err = f2fs_write_inline_data(inode, page); 1169 if (err == -EAGAIN) 1170 err = do_write_data_page(&fio); 1171 f2fs_unlock_op(sbi); 1172 done: 1173 if (err && err != -ENOENT) 1174 goto redirty_out; 1175 1176 clear_cold_data(page); 1177 out: 1178 inode_dec_dirty_pages(inode); 1179 if (err) 1180 ClearPageUptodate(page); 1181 unlock_page(page); 1182 if (need_balance_fs) 1183 f2fs_balance_fs(sbi); 1184 if (wbc->for_reclaim) 1185 f2fs_submit_merged_bio(sbi, DATA, WRITE); 1186 return 0; 1187 1188 redirty_out: 1189 redirty_page_for_writepage(wbc, page); 1190 return AOP_WRITEPAGE_ACTIVATE; 1191 } 1192 1193 static int __f2fs_writepage(struct page *page, struct writeback_control *wbc, 1194 void *data) 1195 { 1196 struct address_space *mapping = data; 1197 int ret = mapping->a_ops->writepage(page, wbc); 1198 mapping_set_error(mapping, ret); 1199 return ret; 1200 } 1201 1202 /* 1203 * This function was copied from write_cche_pages from mm/page-writeback.c. 1204 * The major change is making write step of cold data page separately from 1205 * warm/hot data page. 1206 */ 1207 static int f2fs_write_cache_pages(struct address_space *mapping, 1208 struct writeback_control *wbc, writepage_t writepage, 1209 void *data) 1210 { 1211 int ret = 0; 1212 int done = 0; 1213 struct pagevec pvec; 1214 int nr_pages; 1215 pgoff_t uninitialized_var(writeback_index); 1216 pgoff_t index; 1217 pgoff_t end; /* Inclusive */ 1218 pgoff_t done_index; 1219 int cycled; 1220 int range_whole = 0; 1221 int tag; 1222 int step = 0; 1223 1224 pagevec_init(&pvec, 0); 1225 next: 1226 if (wbc->range_cyclic) { 1227 writeback_index = mapping->writeback_index; /* prev offset */ 1228 index = writeback_index; 1229 if (index == 0) 1230 cycled = 1; 1231 else 1232 cycled = 0; 1233 end = -1; 1234 } else { 1235 index = wbc->range_start >> PAGE_CACHE_SHIFT; 1236 end = wbc->range_end >> PAGE_CACHE_SHIFT; 1237 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) 1238 range_whole = 1; 1239 cycled = 1; /* ignore range_cyclic tests */ 1240 } 1241 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) 1242 tag = PAGECACHE_TAG_TOWRITE; 1243 else 1244 tag = PAGECACHE_TAG_DIRTY; 1245 retry: 1246 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) 1247 tag_pages_for_writeback(mapping, index, end); 1248 done_index = index; 1249 while (!done && (index <= end)) { 1250 int i; 1251 1252 nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag, 1253 min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1); 1254 if (nr_pages == 0) 1255 break; 1256 1257 for (i = 0; i < nr_pages; i++) { 1258 struct page *page = pvec.pages[i]; 1259 1260 if (page->index > end) { 1261 done = 1; 1262 break; 1263 } 1264 1265 done_index = page->index; 1266 1267 lock_page(page); 1268 1269 if (unlikely(page->mapping != mapping)) { 1270 continue_unlock: 1271 unlock_page(page); 1272 continue; 1273 } 1274 1275 if (!PageDirty(page)) { 1276 /* someone wrote it for us */ 1277 goto continue_unlock; 1278 } 1279 1280 if (step == is_cold_data(page)) 1281 goto continue_unlock; 1282 1283 if (PageWriteback(page)) { 1284 if (wbc->sync_mode != WB_SYNC_NONE) 1285 f2fs_wait_on_page_writeback(page, DATA); 1286 else 1287 goto continue_unlock; 1288 } 1289 1290 BUG_ON(PageWriteback(page)); 1291 if (!clear_page_dirty_for_io(page)) 1292 goto continue_unlock; 1293 1294 ret = (*writepage)(page, wbc, data); 1295 if (unlikely(ret)) { 1296 if (ret == AOP_WRITEPAGE_ACTIVATE) { 1297 unlock_page(page); 1298 ret = 0; 1299 } else { 1300 done_index = page->index + 1; 1301 done = 1; 1302 break; 1303 } 1304 } 1305 1306 if (--wbc->nr_to_write <= 0 && 1307 wbc->sync_mode == WB_SYNC_NONE) { 1308 done = 1; 1309 break; 1310 } 1311 } 1312 pagevec_release(&pvec); 1313 cond_resched(); 1314 } 1315 1316 if (step < 1) { 1317 step++; 1318 goto next; 1319 } 1320 1321 if (!cycled && !done) { 1322 cycled = 1; 1323 index = 0; 1324 end = writeback_index - 1; 1325 goto retry; 1326 } 1327 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0)) 1328 mapping->writeback_index = done_index; 1329 1330 return ret; 1331 } 1332 1333 static int f2fs_write_data_pages(struct address_space *mapping, 1334 struct writeback_control *wbc) 1335 { 1336 struct inode *inode = mapping->host; 1337 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1338 bool locked = false; 1339 int ret; 1340 long diff; 1341 1342 trace_f2fs_writepages(mapping->host, wbc, DATA); 1343 1344 /* deal with chardevs and other special file */ 1345 if (!mapping->a_ops->writepage) 1346 return 0; 1347 1348 /* skip writing if there is no dirty page in this inode */ 1349 if (!get_dirty_pages(inode) && wbc->sync_mode == WB_SYNC_NONE) 1350 return 0; 1351 1352 if (S_ISDIR(inode->i_mode) && wbc->sync_mode == WB_SYNC_NONE && 1353 get_dirty_pages(inode) < nr_pages_to_skip(sbi, DATA) && 1354 available_free_memory(sbi, DIRTY_DENTS)) 1355 goto skip_write; 1356 1357 /* during POR, we don't need to trigger writepage at all. */ 1358 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) 1359 goto skip_write; 1360 1361 diff = nr_pages_to_write(sbi, DATA, wbc); 1362 1363 if (!S_ISDIR(inode->i_mode)) { 1364 mutex_lock(&sbi->writepages); 1365 locked = true; 1366 } 1367 ret = f2fs_write_cache_pages(mapping, wbc, __f2fs_writepage, mapping); 1368 f2fs_submit_merged_bio(sbi, DATA, WRITE); 1369 if (locked) 1370 mutex_unlock(&sbi->writepages); 1371 1372 remove_dirty_dir_inode(inode); 1373 1374 wbc->nr_to_write = max((long)0, wbc->nr_to_write - diff); 1375 return ret; 1376 1377 skip_write: 1378 wbc->pages_skipped += get_dirty_pages(inode); 1379 return 0; 1380 } 1381 1382 static void f2fs_write_failed(struct address_space *mapping, loff_t to) 1383 { 1384 struct inode *inode = mapping->host; 1385 1386 if (to > inode->i_size) { 1387 truncate_pagecache(inode, inode->i_size); 1388 truncate_blocks(inode, inode->i_size, true); 1389 } 1390 } 1391 1392 static int f2fs_write_begin(struct file *file, struct address_space *mapping, 1393 loff_t pos, unsigned len, unsigned flags, 1394 struct page **pagep, void **fsdata) 1395 { 1396 struct inode *inode = mapping->host; 1397 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1398 struct page *page = NULL; 1399 struct page *ipage; 1400 pgoff_t index = ((unsigned long long) pos) >> PAGE_CACHE_SHIFT; 1401 struct dnode_of_data dn; 1402 int err = 0; 1403 1404 trace_f2fs_write_begin(inode, pos, len, flags); 1405 1406 f2fs_balance_fs(sbi); 1407 1408 /* 1409 * We should check this at this moment to avoid deadlock on inode page 1410 * and #0 page. The locking rule for inline_data conversion should be: 1411 * lock_page(page #0) -> lock_page(inode_page) 1412 */ 1413 if (index != 0) { 1414 err = f2fs_convert_inline_inode(inode); 1415 if (err) 1416 goto fail; 1417 } 1418 repeat: 1419 page = grab_cache_page_write_begin(mapping, index, flags); 1420 if (!page) { 1421 err = -ENOMEM; 1422 goto fail; 1423 } 1424 1425 *pagep = page; 1426 1427 f2fs_lock_op(sbi); 1428 1429 /* check inline_data */ 1430 ipage = get_node_page(sbi, inode->i_ino); 1431 if (IS_ERR(ipage)) { 1432 err = PTR_ERR(ipage); 1433 goto unlock_fail; 1434 } 1435 1436 set_new_dnode(&dn, inode, ipage, ipage, 0); 1437 1438 if (f2fs_has_inline_data(inode)) { 1439 if (pos + len <= MAX_INLINE_DATA) { 1440 read_inline_data(page, ipage); 1441 set_inode_flag(F2FS_I(inode), FI_DATA_EXIST); 1442 sync_inode_page(&dn); 1443 goto put_next; 1444 } 1445 err = f2fs_convert_inline_page(&dn, page); 1446 if (err) 1447 goto put_fail; 1448 } 1449 1450 err = f2fs_get_block(&dn, index); 1451 if (err) 1452 goto put_fail; 1453 put_next: 1454 f2fs_put_dnode(&dn); 1455 f2fs_unlock_op(sbi); 1456 1457 f2fs_wait_on_page_writeback(page, DATA); 1458 1459 /* wait for GCed encrypted page writeback */ 1460 if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) 1461 f2fs_wait_on_encrypted_page_writeback(sbi, dn.data_blkaddr); 1462 1463 if (len == PAGE_CACHE_SIZE) 1464 goto out_update; 1465 if (PageUptodate(page)) 1466 goto out_clear; 1467 1468 if ((pos & PAGE_CACHE_MASK) >= i_size_read(inode)) { 1469 unsigned start = pos & (PAGE_CACHE_SIZE - 1); 1470 unsigned end = start + len; 1471 1472 /* Reading beyond i_size is simple: memset to zero */ 1473 zero_user_segments(page, 0, start, end, PAGE_CACHE_SIZE); 1474 goto out_update; 1475 } 1476 1477 if (dn.data_blkaddr == NEW_ADDR) { 1478 zero_user_segment(page, 0, PAGE_CACHE_SIZE); 1479 } else { 1480 struct f2fs_io_info fio = { 1481 .sbi = sbi, 1482 .type = DATA, 1483 .rw = READ_SYNC, 1484 .blk_addr = dn.data_blkaddr, 1485 .page = page, 1486 .encrypted_page = NULL, 1487 }; 1488 err = f2fs_submit_page_bio(&fio); 1489 if (err) 1490 goto fail; 1491 1492 lock_page(page); 1493 if (unlikely(!PageUptodate(page))) { 1494 err = -EIO; 1495 goto fail; 1496 } 1497 if (unlikely(page->mapping != mapping)) { 1498 f2fs_put_page(page, 1); 1499 goto repeat; 1500 } 1501 1502 /* avoid symlink page */ 1503 if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) { 1504 err = f2fs_decrypt_one(inode, page); 1505 if (err) 1506 goto fail; 1507 } 1508 } 1509 out_update: 1510 SetPageUptodate(page); 1511 out_clear: 1512 clear_cold_data(page); 1513 return 0; 1514 1515 put_fail: 1516 f2fs_put_dnode(&dn); 1517 unlock_fail: 1518 f2fs_unlock_op(sbi); 1519 fail: 1520 f2fs_put_page(page, 1); 1521 f2fs_write_failed(mapping, pos + len); 1522 return err; 1523 } 1524 1525 static int f2fs_write_end(struct file *file, 1526 struct address_space *mapping, 1527 loff_t pos, unsigned len, unsigned copied, 1528 struct page *page, void *fsdata) 1529 { 1530 struct inode *inode = page->mapping->host; 1531 1532 trace_f2fs_write_end(inode, pos, len, copied); 1533 1534 set_page_dirty(page); 1535 1536 if (pos + copied > i_size_read(inode)) { 1537 i_size_write(inode, pos + copied); 1538 mark_inode_dirty(inode); 1539 update_inode_page(inode); 1540 } 1541 1542 f2fs_put_page(page, 1); 1543 return copied; 1544 } 1545 1546 static int check_direct_IO(struct inode *inode, struct iov_iter *iter, 1547 loff_t offset) 1548 { 1549 unsigned blocksize_mask = inode->i_sb->s_blocksize - 1; 1550 1551 if (offset & blocksize_mask) 1552 return -EINVAL; 1553 1554 if (iov_iter_alignment(iter) & blocksize_mask) 1555 return -EINVAL; 1556 1557 return 0; 1558 } 1559 1560 static ssize_t f2fs_direct_IO(struct kiocb *iocb, struct iov_iter *iter, 1561 loff_t offset) 1562 { 1563 struct file *file = iocb->ki_filp; 1564 struct address_space *mapping = file->f_mapping; 1565 struct inode *inode = mapping->host; 1566 size_t count = iov_iter_count(iter); 1567 int err; 1568 1569 /* we don't need to use inline_data strictly */ 1570 if (f2fs_has_inline_data(inode)) { 1571 err = f2fs_convert_inline_inode(inode); 1572 if (err) 1573 return err; 1574 } 1575 1576 if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) 1577 return 0; 1578 1579 err = check_direct_IO(inode, iter, offset); 1580 if (err) 1581 return err; 1582 1583 trace_f2fs_direct_IO_enter(inode, offset, count, iov_iter_rw(iter)); 1584 1585 if (iov_iter_rw(iter) == WRITE) { 1586 __allocate_data_blocks(inode, offset, count); 1587 if (unlikely(f2fs_cp_error(F2FS_I_SB(inode)))) { 1588 err = -EIO; 1589 goto out; 1590 } 1591 } 1592 1593 err = blockdev_direct_IO(iocb, inode, iter, offset, get_data_block_dio); 1594 out: 1595 if (err < 0 && iov_iter_rw(iter) == WRITE) 1596 f2fs_write_failed(mapping, offset + count); 1597 1598 trace_f2fs_direct_IO_exit(inode, offset, count, iov_iter_rw(iter), err); 1599 1600 return err; 1601 } 1602 1603 void f2fs_invalidate_page(struct page *page, unsigned int offset, 1604 unsigned int length) 1605 { 1606 struct inode *inode = page->mapping->host; 1607 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1608 1609 if (inode->i_ino >= F2FS_ROOT_INO(sbi) && 1610 (offset % PAGE_CACHE_SIZE || length != PAGE_CACHE_SIZE)) 1611 return; 1612 1613 if (PageDirty(page)) { 1614 if (inode->i_ino == F2FS_META_INO(sbi)) 1615 dec_page_count(sbi, F2FS_DIRTY_META); 1616 else if (inode->i_ino == F2FS_NODE_INO(sbi)) 1617 dec_page_count(sbi, F2FS_DIRTY_NODES); 1618 else 1619 inode_dec_dirty_pages(inode); 1620 } 1621 1622 /* This is atomic written page, keep Private */ 1623 if (IS_ATOMIC_WRITTEN_PAGE(page)) 1624 return; 1625 1626 ClearPagePrivate(page); 1627 } 1628 1629 int f2fs_release_page(struct page *page, gfp_t wait) 1630 { 1631 /* If this is dirty page, keep PagePrivate */ 1632 if (PageDirty(page)) 1633 return 0; 1634 1635 /* This is atomic written page, keep Private */ 1636 if (IS_ATOMIC_WRITTEN_PAGE(page)) 1637 return 0; 1638 1639 ClearPagePrivate(page); 1640 return 1; 1641 } 1642 1643 static int f2fs_set_data_page_dirty(struct page *page) 1644 { 1645 struct address_space *mapping = page->mapping; 1646 struct inode *inode = mapping->host; 1647 1648 trace_f2fs_set_page_dirty(page, DATA); 1649 1650 SetPageUptodate(page); 1651 1652 if (f2fs_is_atomic_file(inode)) { 1653 if (!IS_ATOMIC_WRITTEN_PAGE(page)) { 1654 register_inmem_page(inode, page); 1655 return 1; 1656 } 1657 /* 1658 * Previously, this page has been registered, we just 1659 * return here. 1660 */ 1661 return 0; 1662 } 1663 1664 if (!PageDirty(page)) { 1665 __set_page_dirty_nobuffers(page); 1666 update_dirty_page(inode, page); 1667 return 1; 1668 } 1669 return 0; 1670 } 1671 1672 static sector_t f2fs_bmap(struct address_space *mapping, sector_t block) 1673 { 1674 struct inode *inode = mapping->host; 1675 1676 if (f2fs_has_inline_data(inode)) 1677 return 0; 1678 1679 /* make sure allocating whole blocks */ 1680 if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) 1681 filemap_write_and_wait(mapping); 1682 1683 return generic_block_bmap(mapping, block, get_data_block_bmap); 1684 } 1685 1686 const struct address_space_operations f2fs_dblock_aops = { 1687 .readpage = f2fs_read_data_page, 1688 .readpages = f2fs_read_data_pages, 1689 .writepage = f2fs_write_data_page, 1690 .writepages = f2fs_write_data_pages, 1691 .write_begin = f2fs_write_begin, 1692 .write_end = f2fs_write_end, 1693 .set_page_dirty = f2fs_set_data_page_dirty, 1694 .invalidatepage = f2fs_invalidate_page, 1695 .releasepage = f2fs_release_page, 1696 .direct_IO = f2fs_direct_IO, 1697 .bmap = f2fs_bmap, 1698 }; 1699