diff options
Diffstat (limited to 'block/blk.h')
| -rw-r--r-- | block/blk.h | 94 |
1 files changed, 71 insertions, 23 deletions
diff --git a/block/blk.h b/block/blk.h index e4c433f62dfc..b998a7761faf 100644 --- a/block/blk.h +++ b/block/blk.h @@ -55,7 +55,7 @@ bool __blk_freeze_queue_start(struct request_queue *q, struct task_struct *owner); int __bio_queue_enter(struct request_queue *q, struct bio *bio); void submit_bio_noacct_nocheck(struct bio *bio, bool split); -void bio_await_chain(struct bio *bio); +int bio_submit_or_kill(struct bio *bio, unsigned int flags); static inline bool blk_try_enter_queue(struct request_queue *q, bool pm) { @@ -108,11 +108,6 @@ static inline void blk_wait_io(struct completion *done) struct block_device *blkdev_get_no_open(dev_t dev, bool autoload); void blkdev_put_no_open(struct block_device *bdev); -#define BIO_INLINE_VECS 4 -struct bio_vec *bvec_alloc(mempool_t *pool, unsigned short *nr_vecs, - gfp_t gfp_mask); -void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned short nr_vecs); - bool bvec_try_merge_hw_page(struct request_queue *q, struct bio_vec *bv, struct page *page, unsigned len, unsigned offset); @@ -132,6 +127,8 @@ static inline bool biovec_phys_mergeable(struct request_queue *q, if (addr1 + vec1->bv_len != addr2) return false; + if (!zone_device_pages_have_same_pgmap(vec1->bv_page, vec2->bv_page)) + return false; if (xen_domain() && !xen_biovec_phys_mergeable(vec1, vec2->bv_page)) return false; if ((addr1 | mask) != ((addr2 + vec2->bv_len - 1) | mask)) @@ -139,6 +136,25 @@ static inline bool biovec_phys_mergeable(struct request_queue *q, return true; } +/* + * Check if two pages from potentially different zone device pgmaps can + * coexist as separate bvec entries in the same bio. + * + * The block DMA iterator (blk_dma_map_iter_start) caches the P2PDMA mapping + * state from the first segment and applies it to all subsequent segments, so + * P2PDMA pages from different pgmaps must not be mixed in the same bio. + * + * Other zone device types (FS_DAX, GENERIC) use the same dma_map_phys() path + * as normal RAM. PRIVATE and COHERENT pages never appear in bios. + */ +static inline bool zone_device_pages_compatible(const struct page *a, + const struct page *b) +{ + if (is_pci_p2pdma_page(a) || is_pci_p2pdma_page(b)) + return zone_device_pages_have_same_pgmap(a, b); + return true; +} + static inline bool __bvec_gap_to_prev(const struct queue_limits *lim, struct bio_vec *bprv, unsigned int offset) { @@ -208,10 +224,14 @@ static inline unsigned int blk_queue_get_max_sectors(struct request *rq) struct request_queue *q = rq->q; enum req_op op = req_op(rq); - if (unlikely(op == REQ_OP_DISCARD || op == REQ_OP_SECURE_ERASE)) + if (unlikely(op == REQ_OP_DISCARD)) return min(q->limits.max_discard_sectors, UINT_MAX >> SECTOR_SHIFT); + if (unlikely(op == REQ_OP_SECURE_ERASE)) + return min(q->limits.max_secure_erase_sectors, + UINT_MAX >> SECTOR_SHIFT); + if (unlikely(op == REQ_OP_WRITE_ZEROES)) return q->limits.max_write_zeroes_sectors; @@ -371,12 +391,18 @@ struct bio *bio_split_zone_append(struct bio *bio, static inline bool bio_may_need_split(struct bio *bio, const struct queue_limits *lim) { + const struct bio_vec *bv; + if (lim->chunk_sectors) return true; - if (bio->bi_vcnt != 1) + + if (!bio->bi_io_vec) return true; - return bio->bi_io_vec->bv_len + bio->bi_io_vec->bv_offset > - lim->max_fast_segment_size; + + bv = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter); + if (bio->bi_iter.bi_size > bv->bv_len - bio->bi_iter.bi_bvec_done) + return true; + return bv->bv_len + bv->bv_offset > lim->max_fast_segment_size; } /** @@ -589,17 +615,6 @@ void bdev_set_nr_sectors(struct block_device *bdev, sector_t sectors); struct gendisk *__alloc_disk_node(struct request_queue *q, int node_id, struct lock_class_key *lkclass); - -/* - * Clean up a page appropriately, where the page may be pinned, may have a - * ref taken on it or neither. - */ -static inline void bio_release_page(struct bio *bio, struct page *page) -{ - if (bio_flagged(bio, BIO_PAGE_PINNED)) - unpin_user_page(page); -} - struct request_queue *blk_alloc_queue(struct queue_limits *lim, int node_id); int disk_scan_partitions(struct gendisk *disk, blk_mode_t mode); @@ -700,8 +715,10 @@ int bdev_open(struct block_device *bdev, blk_mode_t mode, void *holder, const struct blk_holder_ops *hops, struct file *bdev_file); int bdev_permission(dev_t dev, blk_mode_t mode, void *holder); -void blk_integrity_generate(struct bio *bio); -void blk_integrity_verify_iter(struct bio *bio, struct bvec_iter *saved_iter); +void bio_integrity_generate(struct bio *bio); +blk_status_t bio_integrity_verify(struct bio *bio, + struct bvec_iter *saved_iter); + void blk_integrity_prepare(struct request *rq); void blk_integrity_complete(struct request *rq, unsigned int nr_bytes); @@ -730,4 +747,35 @@ static inline void blk_unfreeze_release_lock(struct request_queue *q) } #endif +/* + * debugfs directory and file creation can trigger fs reclaim, which can enter + * back into the block layer request_queue. This can cause deadlock if the + * queue is frozen. Use NOIO context together with debugfs_mutex to prevent fs + * reclaim from triggering block I/O. + */ +static inline void blk_debugfs_lock_nomemsave(struct request_queue *q) +{ + mutex_lock(&q->debugfs_mutex); +} + +static inline void blk_debugfs_unlock_nomemrestore(struct request_queue *q) +{ + mutex_unlock(&q->debugfs_mutex); +} + +static inline unsigned int __must_check blk_debugfs_lock(struct request_queue *q) +{ + unsigned int memflags = memalloc_noio_save(); + + blk_debugfs_lock_nomemsave(q); + return memflags; +} + +static inline void blk_debugfs_unlock(struct request_queue *q, + unsigned int memflags) +{ + blk_debugfs_unlock_nomemrestore(q); + memalloc_noio_restore(memflags); +} + #endif /* BLK_INTERNAL_H */ |