1 /* SPDX-License-Identifier: GPL-2.0 */ 2 3 #ifndef _NET_PAGE_POOL_TYPES_H 4 #define _NET_PAGE_POOL_TYPES_H 5 6 #include <linux/dma-direction.h> 7 #include <linux/ptr_ring.h> 8 #include <linux/types.h> 9 10 #define PP_FLAG_DMA_MAP BIT(0) /* Should page_pool do the DMA 11 * map/unmap 12 */ 13 #define PP_FLAG_DMA_SYNC_DEV BIT(1) /* If set all pages that the driver gets 14 * from page_pool will be 15 * DMA-synced-for-device according to 16 * the length provided by the device 17 * driver. 18 * Please note DMA-sync-for-CPU is still 19 * device driver responsibility 20 */ 21 #define PP_FLAG_SYSTEM_POOL BIT(2) /* Global system page_pool */ 22 #define PP_FLAG_ALL (PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV | \ 23 PP_FLAG_SYSTEM_POOL) 24 25 /* 26 * Fast allocation side cache array/stack 27 * 28 * The cache size and refill watermark is related to the network 29 * use-case. The NAPI budget is 64 packets. After a NAPI poll the RX 30 * ring is usually refilled and the max consumed elements will be 64, 31 * thus a natural max size of objects needed in the cache. 32 * 33 * Keeping room for more objects, is due to XDP_DROP use-case. As 34 * XDP_DROP allows the opportunity to recycle objects directly into 35 * this array, as it shares the same softirq/NAPI protection. If 36 * cache is already full (or partly full) then the XDP_DROP recycles 37 * would have to take a slower code path. 38 */ 39 #define PP_ALLOC_CACHE_SIZE 128 40 #define PP_ALLOC_CACHE_REFILL 64 41 struct pp_alloc_cache { 42 u32 count; 43 struct page *cache[PP_ALLOC_CACHE_SIZE]; 44 }; 45 46 /** 47 * struct page_pool_params - page pool parameters 48 * @flags: PP_FLAG_DMA_MAP, PP_FLAG_DMA_SYNC_DEV 49 * @order: 2^order pages on allocation 50 * @pool_size: size of the ptr_ring 51 * @nid: NUMA node id to allocate from pages from 52 * @dev: device, for DMA pre-mapping purposes 53 * @netdev: netdev this pool will serve (leave as NULL if none or multiple) 54 * @napi: NAPI which is the sole consumer of pages, otherwise NULL 55 * @dma_dir: DMA mapping direction 56 * @max_len: max DMA sync memory size for PP_FLAG_DMA_SYNC_DEV 57 * @offset: DMA sync address offset for PP_FLAG_DMA_SYNC_DEV 58 */ 59 struct page_pool_params { 60 struct_group_tagged(page_pool_params_fast, fast, 61 unsigned int flags; 62 unsigned int order; 63 unsigned int pool_size; 64 int nid; 65 struct device *dev; 66 struct napi_struct *napi; 67 enum dma_data_direction dma_dir; 68 unsigned int max_len; 69 unsigned int offset; 70 ); 71 struct_group_tagged(page_pool_params_slow, slow, 72 struct net_device *netdev; 73 /* private: used by test code only */ 74 void (*init_callback)(struct page *page, void *arg); 75 void *init_arg; 76 ); 77 }; 78 79 #ifdef CONFIG_PAGE_POOL_STATS 80 /** 81 * struct page_pool_alloc_stats - allocation statistics 82 * @fast: successful fast path allocations 83 * @slow: slow path order-0 allocations 84 * @slow_high_order: slow path high order allocations 85 * @empty: ptr ring is empty, so a slow path allocation was forced 86 * @refill: an allocation which triggered a refill of the cache 87 * @waive: pages obtained from the ptr ring that cannot be added to 88 * the cache due to a NUMA mismatch 89 */ 90 struct page_pool_alloc_stats { 91 u64 fast; 92 u64 slow; 93 u64 slow_high_order; 94 u64 empty; 95 u64 refill; 96 u64 waive; 97 }; 98 99 /** 100 * struct page_pool_recycle_stats - recycling (freeing) statistics 101 * @cached: recycling placed page in the page pool cache 102 * @cache_full: page pool cache was full 103 * @ring: page placed into the ptr ring 104 * @ring_full: page released from page pool because the ptr ring was full 105 * @released_refcnt: page released (and not recycled) because refcnt > 1 106 */ 107 struct page_pool_recycle_stats { 108 u64 cached; 109 u64 cache_full; 110 u64 ring; 111 u64 ring_full; 112 u64 released_refcnt; 113 }; 114 115 /** 116 * struct page_pool_stats - combined page pool use statistics 117 * @alloc_stats: see struct page_pool_alloc_stats 118 * @recycle_stats: see struct page_pool_recycle_stats 119 * 120 * Wrapper struct for combining page pool stats with different storage 121 * requirements. 122 */ 123 struct page_pool_stats { 124 struct page_pool_alloc_stats alloc_stats; 125 struct page_pool_recycle_stats recycle_stats; 126 }; 127 #endif 128 129 struct page_pool { 130 struct page_pool_params_fast p; 131 132 int cpuid; 133 bool has_init_callback; 134 135 long frag_users; 136 struct page *frag_page; 137 unsigned int frag_offset; 138 u32 pages_state_hold_cnt; 139 140 struct delayed_work release_dw; 141 void (*disconnect)(void *pool); 142 unsigned long defer_start; 143 unsigned long defer_warn; 144 145 #ifdef CONFIG_PAGE_POOL_STATS 146 /* these stats are incremented while in softirq context */ 147 struct page_pool_alloc_stats alloc_stats; 148 #endif 149 u32 xdp_mem_id; 150 151 /* 152 * Data structure for allocation side 153 * 154 * Drivers allocation side usually already perform some kind 155 * of resource protection. Piggyback on this protection, and 156 * require driver to protect allocation side. 157 * 158 * For NIC drivers this means, allocate a page_pool per 159 * RX-queue. As the RX-queue is already protected by 160 * Softirq/BH scheduling and napi_schedule. NAPI schedule 161 * guarantee that a single napi_struct will only be scheduled 162 * on a single CPU (see napi_schedule). 163 */ 164 struct pp_alloc_cache alloc ____cacheline_aligned_in_smp; 165 166 /* Data structure for storing recycled pages. 167 * 168 * Returning/freeing pages is more complicated synchronization 169 * wise, because free's can happen on remote CPUs, with no 170 * association with allocation resource. 171 * 172 * Use ptr_ring, as it separates consumer and producer 173 * efficiently, it a way that doesn't bounce cache-lines. 174 * 175 * TODO: Implement bulk return pages into this structure. 176 */ 177 struct ptr_ring ring; 178 179 #ifdef CONFIG_PAGE_POOL_STATS 180 /* recycle stats are per-cpu to avoid locking */ 181 struct page_pool_recycle_stats __percpu *recycle_stats; 182 #endif 183 atomic_t pages_state_release_cnt; 184 185 /* A page_pool is strictly tied to a single RX-queue being 186 * protected by NAPI, due to above pp_alloc_cache. This 187 * refcnt serves purpose is to simplify drivers error handling. 188 */ 189 refcount_t user_cnt; 190 191 u64 destroy_cnt; 192 193 /* Slow/Control-path information follows */ 194 struct page_pool_params_slow slow; 195 /* User-facing fields, protected by page_pools_lock */ 196 struct { 197 struct hlist_node list; 198 u64 detach_time; 199 u32 napi_id; 200 u32 id; 201 } user; 202 }; 203 204 struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp); 205 struct page *page_pool_alloc_frag(struct page_pool *pool, unsigned int *offset, 206 unsigned int size, gfp_t gfp); 207 struct page_pool *page_pool_create(const struct page_pool_params *params); 208 struct page_pool *page_pool_create_percpu(const struct page_pool_params *params, 209 int cpuid); 210 211 struct xdp_mem_info; 212 213 #ifdef CONFIG_PAGE_POOL 214 void page_pool_destroy(struct page_pool *pool); 215 void page_pool_use_xdp_mem(struct page_pool *pool, void (*disconnect)(void *), 216 struct xdp_mem_info *mem); 217 void page_pool_put_page_bulk(struct page_pool *pool, void **data, 218 int count); 219 #else 220 static inline void page_pool_destroy(struct page_pool *pool) 221 { 222 } 223 224 static inline void page_pool_use_xdp_mem(struct page_pool *pool, 225 void (*disconnect)(void *), 226 struct xdp_mem_info *mem) 227 { 228 } 229 230 static inline void page_pool_put_page_bulk(struct page_pool *pool, void **data, 231 int count) 232 { 233 } 234 #endif 235 236 void page_pool_put_unrefed_page(struct page_pool *pool, struct page *page, 237 unsigned int dma_sync_size, 238 bool allow_direct); 239 240 static inline bool is_page_pool_compiled_in(void) 241 { 242 #ifdef CONFIG_PAGE_POOL 243 return true; 244 #else 245 return false; 246 #endif 247 } 248 249 /* Caller must provide appropriate safe context, e.g. NAPI. */ 250 void page_pool_update_nid(struct page_pool *pool, int new_nid); 251 252 #endif /* _NET_PAGE_POOL_H */ 253