1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 /* Copyright (C) 2015-2018 Netronome Systems, Inc. */ 3 4 #include <linux/delay.h> 5 #include <linux/device.h> 6 #include <linux/jiffies.h> 7 #include <linux/types.h> 8 #include <linux/slab.h> 9 #include <linux/wait.h> 10 11 #include "nfp_cpp.h" 12 #include "nfp6000/nfp6000.h" 13 14 struct nfp_cpp_mutex { 15 struct nfp_cpp *cpp; 16 int target; 17 u16 depth; 18 unsigned long long address; 19 u32 key; 20 }; 21 22 static u32 nfp_mutex_locked(u16 interface) 23 { 24 return (u32)interface << 16 | 0x000f; 25 } 26 27 static u32 nfp_mutex_unlocked(u16 interface) 28 { 29 return (u32)interface << 16 | 0x0000; 30 } 31 32 static u32 nfp_mutex_owner(u32 val) 33 { 34 return val >> 16; 35 } 36 37 static bool nfp_mutex_is_locked(u32 val) 38 { 39 return (val & 0xffff) == 0x000f; 40 } 41 42 static bool nfp_mutex_is_unlocked(u32 val) 43 { 44 return (val & 0xffff) == 0000; 45 } 46 47 /* If you need more than 65536 recursive locks, please rethink your code. */ 48 #define NFP_MUTEX_DEPTH_MAX 0xffff 49 50 static int 51 nfp_cpp_mutex_validate(u16 interface, int *target, unsigned long long address) 52 { 53 /* Not permitted on invalid interfaces */ 54 if (NFP_CPP_INTERFACE_TYPE_of(interface) == 55 NFP_CPP_INTERFACE_TYPE_INVALID) 56 return -EINVAL; 57 58 /* Address must be 64-bit aligned */ 59 if (address & 7) 60 return -EINVAL; 61 62 if (*target != NFP_CPP_TARGET_MU) 63 return -EINVAL; 64 65 return 0; 66 } 67 68 /** 69 * nfp_cpp_mutex_init() - Initialize a mutex location 70 * @cpp: NFP CPP handle 71 * @target: NFP CPP target ID (ie NFP_CPP_TARGET_CLS or NFP_CPP_TARGET_MU) 72 * @address: Offset into the address space of the NFP CPP target ID 73 * @key: Unique 32-bit value for this mutex 74 * 75 * The CPP target:address must point to a 64-bit aligned location, and 76 * will initialize 64 bits of data at the location. 77 * 78 * This creates the initial mutex state, as locked by this 79 * nfp_cpp_interface(). 80 * 81 * This function should only be called when setting up 82 * the initial lock state upon boot-up of the system. 83 * 84 * Return: 0 on success, or -errno on failure 85 */ 86 int nfp_cpp_mutex_init(struct nfp_cpp *cpp, 87 int target, unsigned long long address, u32 key) 88 { 89 const u32 muw = NFP_CPP_ID(target, 4, 0); /* atomic_write */ 90 u16 interface = nfp_cpp_interface(cpp); 91 int err; 92 93 err = nfp_cpp_mutex_validate(interface, &target, address); 94 if (err) 95 return err; 96 97 err = nfp_cpp_writel(cpp, muw, address + 4, key); 98 if (err) 99 return err; 100 101 err = nfp_cpp_writel(cpp, muw, address, nfp_mutex_locked(interface)); 102 if (err) 103 return err; 104 105 return 0; 106 } 107 108 /** 109 * nfp_cpp_mutex_alloc() - Create a mutex handle 110 * @cpp: NFP CPP handle 111 * @target: NFP CPP target ID (ie NFP_CPP_TARGET_CLS or NFP_CPP_TARGET_MU) 112 * @address: Offset into the address space of the NFP CPP target ID 113 * @key: 32-bit unique key (must match the key at this location) 114 * 115 * The CPP target:address must point to a 64-bit aligned location, and 116 * reserve 64 bits of data at the location for use by the handle. 117 * 118 * Only target/address pairs that point to entities that support the 119 * MU Atomic Engine's CmpAndSwap32 command are supported. 120 * 121 * Return: A non-NULL struct nfp_cpp_mutex * on success, NULL on failure. 122 */ 123 struct nfp_cpp_mutex *nfp_cpp_mutex_alloc(struct nfp_cpp *cpp, int target, 124 unsigned long long address, u32 key) 125 { 126 const u32 mur = NFP_CPP_ID(target, 3, 0); /* atomic_read */ 127 u16 interface = nfp_cpp_interface(cpp); 128 struct nfp_cpp_mutex *mutex; 129 int err; 130 u32 tmp; 131 132 err = nfp_cpp_mutex_validate(interface, &target, address); 133 if (err) 134 return NULL; 135 136 err = nfp_cpp_readl(cpp, mur, address + 4, &tmp); 137 if (err < 0) 138 return NULL; 139 140 if (tmp != key) 141 return NULL; 142 143 mutex = kzalloc(sizeof(*mutex), GFP_KERNEL); 144 if (!mutex) 145 return NULL; 146 147 mutex->cpp = cpp; 148 mutex->target = target; 149 mutex->address = address; 150 mutex->key = key; 151 mutex->depth = 0; 152 153 return mutex; 154 } 155 156 /** 157 * nfp_cpp_mutex_free() - Free a mutex handle - does not alter the lock state 158 * @mutex: NFP CPP Mutex handle 159 */ 160 void nfp_cpp_mutex_free(struct nfp_cpp_mutex *mutex) 161 { 162 kfree(mutex); 163 } 164 165 /** 166 * nfp_cpp_mutex_lock() - Lock a mutex handle, using the NFP MU Atomic Engine 167 * @mutex: NFP CPP Mutex handle 168 * 169 * Return: 0 on success, or -errno on failure 170 */ 171 int nfp_cpp_mutex_lock(struct nfp_cpp_mutex *mutex) 172 { 173 unsigned long warn_at = jiffies + NFP_MUTEX_WAIT_FIRST_WARN * HZ; 174 unsigned long err_at = jiffies + NFP_MUTEX_WAIT_ERROR * HZ; 175 unsigned int timeout_ms = 1; 176 int err; 177 178 /* We can't use a waitqueue here, because the unlocker 179 * might be on a separate CPU. 180 * 181 * So just wait for now. 182 */ 183 for (;;) { 184 err = nfp_cpp_mutex_trylock(mutex); 185 if (err != -EBUSY) 186 break; 187 188 err = msleep_interruptible(timeout_ms); 189 if (err != 0) { 190 nfp_info(mutex->cpp, 191 "interrupted waiting for NFP mutex\n"); 192 return -ERESTARTSYS; 193 } 194 195 if (time_is_before_eq_jiffies(warn_at)) { 196 warn_at = jiffies + NFP_MUTEX_WAIT_NEXT_WARN * HZ; 197 nfp_warn(mutex->cpp, 198 "Warning: waiting for NFP mutex [depth:%hd target:%d addr:%llx key:%08x]\n", 199 mutex->depth, 200 mutex->target, mutex->address, mutex->key); 201 } 202 if (time_is_before_eq_jiffies(err_at)) { 203 nfp_err(mutex->cpp, "Error: mutex wait timed out\n"); 204 return -EBUSY; 205 } 206 } 207 208 return err; 209 } 210 211 /** 212 * nfp_cpp_mutex_unlock() - Unlock a mutex handle, using the MU Atomic Engine 213 * @mutex: NFP CPP Mutex handle 214 * 215 * Return: 0 on success, or -errno on failure 216 */ 217 int nfp_cpp_mutex_unlock(struct nfp_cpp_mutex *mutex) 218 { 219 const u32 muw = NFP_CPP_ID(mutex->target, 4, 0); /* atomic_write */ 220 const u32 mur = NFP_CPP_ID(mutex->target, 3, 0); /* atomic_read */ 221 struct nfp_cpp *cpp = mutex->cpp; 222 u32 key, value; 223 u16 interface; 224 int err; 225 226 interface = nfp_cpp_interface(cpp); 227 228 if (mutex->depth > 1) { 229 mutex->depth--; 230 return 0; 231 } 232 233 err = nfp_cpp_readl(mutex->cpp, mur, mutex->address + 4, &key); 234 if (err < 0) 235 return err; 236 237 if (key != mutex->key) 238 return -EPERM; 239 240 err = nfp_cpp_readl(mutex->cpp, mur, mutex->address, &value); 241 if (err < 0) 242 return err; 243 244 if (value != nfp_mutex_locked(interface)) 245 return -EACCES; 246 247 err = nfp_cpp_writel(cpp, muw, mutex->address, 248 nfp_mutex_unlocked(interface)); 249 if (err < 0) 250 return err; 251 252 mutex->depth = 0; 253 return 0; 254 } 255 256 /** 257 * nfp_cpp_mutex_trylock() - Attempt to lock a mutex handle 258 * @mutex: NFP CPP Mutex handle 259 * 260 * Return: 0 if the lock succeeded, -errno on failure 261 */ 262 int nfp_cpp_mutex_trylock(struct nfp_cpp_mutex *mutex) 263 { 264 const u32 muw = NFP_CPP_ID(mutex->target, 4, 0); /* atomic_write */ 265 const u32 mus = NFP_CPP_ID(mutex->target, 5, 3); /* test_set_imm */ 266 const u32 mur = NFP_CPP_ID(mutex->target, 3, 0); /* atomic_read */ 267 struct nfp_cpp *cpp = mutex->cpp; 268 u32 key, value, tmp; 269 int err; 270 271 if (mutex->depth > 0) { 272 if (mutex->depth == NFP_MUTEX_DEPTH_MAX) 273 return -E2BIG; 274 mutex->depth++; 275 return 0; 276 } 277 278 /* Verify that the lock marker is not damaged */ 279 err = nfp_cpp_readl(cpp, mur, mutex->address + 4, &key); 280 if (err < 0) 281 return err; 282 283 if (key != mutex->key) 284 return -EPERM; 285 286 /* Compare against the unlocked state, and if true, 287 * write the interface id into the top 16 bits, and 288 * mark as locked. 289 */ 290 value = nfp_mutex_locked(nfp_cpp_interface(cpp)); 291 292 /* We use test_set_imm here, as it implies a read 293 * of the current state, and sets the bits in the 294 * bytemask of the command to 1s. Since the mutex 295 * is guaranteed to be 64-bit aligned, the bytemask 296 * of this 32-bit command is ensured to be 8'b00001111, 297 * which implies that the lower 4 bits will be set to 298 * ones regardless of the initial state. 299 * 300 * Since this is a 'Readback' operation, with no Pull 301 * data, we can treat this as a normal Push (read) 302 * atomic, which returns the original value. 303 */ 304 err = nfp_cpp_readl(cpp, mus, mutex->address, &tmp); 305 if (err < 0) 306 return err; 307 308 /* Was it unlocked? */ 309 if (nfp_mutex_is_unlocked(tmp)) { 310 /* The read value can only be 0x....0000 in the unlocked state. 311 * If there was another contending for this lock, then 312 * the lock state would be 0x....000f 313 */ 314 315 /* Write our owner ID into the lock 316 * While not strictly necessary, this helps with 317 * debug and bookkeeping. 318 */ 319 err = nfp_cpp_writel(cpp, muw, mutex->address, value); 320 if (err < 0) 321 return err; 322 323 mutex->depth = 1; 324 return 0; 325 } 326 327 return nfp_mutex_is_locked(tmp) ? -EBUSY : -EINVAL; 328 } 329 330 /** 331 * nfp_cpp_mutex_reclaim() - Unlock mutex if held by local endpoint 332 * @cpp: NFP CPP handle 333 * @target: NFP CPP target ID (ie NFP_CPP_TARGET_CLS or NFP_CPP_TARGET_MU) 334 * @address: Offset into the address space of the NFP CPP target ID 335 * 336 * Release lock if held by local system. Extreme care is advised, call only 337 * when no local lock users can exist. 338 * 339 * Return: 0 if the lock was OK, 1 if locked by us, -errno on invalid mutex 340 */ 341 int nfp_cpp_mutex_reclaim(struct nfp_cpp *cpp, int target, 342 unsigned long long address) 343 { 344 const u32 mur = NFP_CPP_ID(target, 3, 0); /* atomic_read */ 345 const u32 muw = NFP_CPP_ID(target, 4, 0); /* atomic_write */ 346 u16 interface = nfp_cpp_interface(cpp); 347 int err; 348 u32 tmp; 349 350 err = nfp_cpp_mutex_validate(interface, &target, address); 351 if (err) 352 return err; 353 354 /* Check lock */ 355 err = nfp_cpp_readl(cpp, mur, address, &tmp); 356 if (err < 0) 357 return err; 358 359 if (nfp_mutex_is_unlocked(tmp) || nfp_mutex_owner(tmp) != interface) 360 return 0; 361 362 /* Bust the lock */ 363 err = nfp_cpp_writel(cpp, muw, address, nfp_mutex_unlocked(interface)); 364 if (err < 0) 365 return err; 366 367 return 1; 368 } 369