spin_lock.h

/*
 * Copyright (c) 2024 Raspberry Pi (Trading) Ltd.
 *
 * SPDX-License-Identifier: BSD-3-Clause
 */
 
#ifndef _HARDWARE_SYNC_SPIN_LOCK_H
#define _HARDWARE_SYNC_SPIN_LOCK_H
 
#include "pico.h"
#include "hardware/sync.h"
 
// PICO_CONFIG: PICO_USE_SW_SPIN_LOCKS, Use software implementation for spin locks, type=bool, default=1 on RP2350 due to errata E2, group=hardware_sync
#ifndef PICO_USE_SW_SPIN_LOCKS
#if PICO_RP2350
#define PICO_USE_SW_SPIN_LOCKS (!PICO_XIP_RAM)
#endif
#endif
 
// Default spinlock assignments
//
// On RP2350, writes to SIO register offsets +0x180..0x1fc spuriously release the spinlock addressed
// 0x80 bytes lower (erratum RP2350-E2). By default we use software spinlocks to work around this.
// Otherwise (PICO_USE_SW_SPIN_LOCKS=0) we change the default numbering to avoid locks which alias
// with writable registers: 5-7, 10-11 and 26-31 have no register at that offset, and 18-25 alias
// the read-only TMDS PEEK/POP registers. The rest alias a writable register: 0-4
// DOORBELL_*/PERI_NONSEC, 8-9 RISCV_SOFTIRQ/MTIME_CTRL, 12-15 MTIME*, 16-17 TMDS_CTRL/WDATA.
//
// On RP2040 we preserve the original default lock numbers existing since SDK v1.0, since they are a
// compatibility detail that out-of-tree code might rely on.
 
// PICO_CONFIG: PICO_SPINLOCK_ID_IRQ, Spinlock ID for IRQ protection, min=0, max=31, default=9 (5 on RP2350 when PICO_USE_SW_SPIN_LOCKS is 0), group=hardware_sync
#ifndef PICO_SPINLOCK_ID_IRQ
#if PICO_RP2350 && !PICO_USE_SW_SPIN_LOCKS
#define PICO_SPINLOCK_ID_IRQ 5
#else
#define PICO_SPINLOCK_ID_IRQ 9
#endif
#endif
 
// PICO_CONFIG: PICO_SPINLOCK_ID_TIMER, Spinlock ID for Timer protection, min=0, max=31, default=10 (6 on RP2350 when PICO_USE_SW_SPIN_LOCKS is 0), group=hardware_sync
#ifndef PICO_SPINLOCK_ID_TIMER
#if PICO_RP2350 && !PICO_USE_SW_SPIN_LOCKS
#define PICO_SPINLOCK_ID_TIMER 6
#else
#define PICO_SPINLOCK_ID_TIMER 10
#endif
#endif
 
// PICO_CONFIG: PICO_SPINLOCK_ID_HARDWARE_CLAIM, Spinlock ID for Hardware claim protection, min=0, max=31, default=11 (7 on RP2350 when PICO_USE_SW_SPIN_LOCKS is 0), group=hardware_sync
#ifndef PICO_SPINLOCK_ID_HARDWARE_CLAIM
#if PICO_RP2350 && !PICO_USE_SW_SPIN_LOCKS
#define PICO_SPINLOCK_ID_HARDWARE_CLAIM 7
#else
#define PICO_SPINLOCK_ID_HARDWARE_CLAIM 11
#endif
#endif
 
// PICO_CONFIG: PICO_SPINLOCK_ID_RAND, Spinlock ID for Random Number Generator, min=0, max=31, default=12 (10 on RP2350 when PICO_USE_SW_SPIN_LOCKS is 0), group=hardware_sync
#ifndef PICO_SPINLOCK_ID_RAND
#if PICO_RP2350 && !PICO_USE_SW_SPIN_LOCKS
#define PICO_SPINLOCK_ID_RAND 10
#else
#define PICO_SPINLOCK_ID_RAND 12
#endif
#endif
 
// PICO_CONFIG: PICO_SPINLOCK_ID_ATOMIC, Spinlock ID for atomics, min=0, max=31, default=13 (11 on RP2350 when PICO_USE_SW_SPIN_LOCKS is 0), group=hardware_sync
#ifndef PICO_SPINLOCK_ID_ATOMIC
#if PICO_RP2350 && !PICO_USE_SW_SPIN_LOCKS
#define PICO_SPINLOCK_ID_ATOMIC 11
#else
#define PICO_SPINLOCK_ID_ATOMIC 13
#endif
#endif
 
// PICO_CONFIG: PICO_SPINLOCK_ID_OS1, First Spinlock ID reserved for use by low level OS style software, min=0, max=31, default=14 (18 on RP2350 when PICO_USE_SW_SPIN_LOCKS is 0), group=hardware_sync
#ifndef PICO_SPINLOCK_ID_OS1
#if PICO_RP2350 && !PICO_USE_SW_SPIN_LOCKS
#define PICO_SPINLOCK_ID_OS1 18
#else
#define PICO_SPINLOCK_ID_OS1 14
#endif
#endif
 
// PICO_CONFIG: PICO_SPINLOCK_ID_OS2, Second Spinlock ID reserved for use by low level OS style software, min=0, max=31, default=15 (19 on RP2350 when PICO_USE_SW_SPIN_LOCKS is 0), group=hardware_sync
#ifndef PICO_SPINLOCK_ID_OS2
#if PICO_RP2350 && !PICO_USE_SW_SPIN_LOCKS
#define PICO_SPINLOCK_ID_OS2 19
#else
#define PICO_SPINLOCK_ID_OS2 15
#endif
#endif
 
// PICO_CONFIG: PICO_SPINLOCK_ID_STRIPED_FIRST, Lowest Spinlock ID in the 'striped' range, min=0, max=31, default=16 (20 on RP2350 when PICO_USE_SW_SPIN_LOCKS is 0), group=hardware_sync
#ifndef PICO_SPINLOCK_ID_STRIPED_FIRST
#if PICO_RP2350 && !PICO_USE_SW_SPIN_LOCKS
#define PICO_SPINLOCK_ID_STRIPED_FIRST 20
#else
#define PICO_SPINLOCK_ID_STRIPED_FIRST 16
#endif
#endif
 
// PICO_CONFIG: PICO_SPINLOCK_ID_STRIPED_LAST, Highest Spinlock ID in the 'striped' range, min=0, max=31, default=23 (25 on RP2350 when PICO_USE_SW_SPIN_LOCKS is 0), group=hardware_sync
#ifndef PICO_SPINLOCK_ID_STRIPED_LAST
#if PICO_RP2350 && !PICO_USE_SW_SPIN_LOCKS
#define PICO_SPINLOCK_ID_STRIPED_LAST 25
#else
#define PICO_SPINLOCK_ID_STRIPED_LAST 23
#endif
#endif
 
// PICO_CONFIG: PICO_SPINLOCK_ID_CLAIM_FREE_FIRST, Lowest Spinlock ID in the 'claim free' range, min=0, max=31, default=24 (26 on RP2350 when PICO_USE_SW_SPIN_LOCKS is 0), group=hardware_sync
#ifndef PICO_SPINLOCK_ID_CLAIM_FREE_FIRST
#if PICO_RP2350 && !PICO_USE_SW_SPIN_LOCKS
#define PICO_SPINLOCK_ID_CLAIM_FREE_FIRST 26
#else
#define PICO_SPINLOCK_ID_CLAIM_FREE_FIRST 24
#endif
#endif
 
#ifdef PICO_SPINLOCK_ID_CLAIM_FREE_END
#warning PICO_SPINLOCK_ID_CLAIM_FREE_END has been renamed to PICO_SPINLOCK_ID_CLAIM_FREE_LAST
#endif
 
// PICO_CONFIG: PICO_SPINLOCK_ID_CLAIM_FREE_LAST, Highest Spinlock ID in the 'claim free' range, min=0, max=31, default=31, group=hardware_sync
#ifndef PICO_SPINLOCK_ID_CLAIM_FREE_LAST
#define PICO_SPINLOCK_ID_CLAIM_FREE_LAST 31
#endif

#if !PICO_USE_SW_SPIN_LOCKS
// Hardware lock flag in SIO:
typedef io_rw_32 spin_lock_t;
#else
#ifndef SW_SPIN_LOCK_TYPE
// Byte flag in memory:
#define SW_SPIN_LOCK_TYPE volatile uint8_t
#endif
typedef SW_SPIN_LOCK_TYPE spin_lock_t;
#endif
 
#if PICO_USE_SW_SPIN_LOCKS
 
#ifndef SW_SPIN_LOCK_INSTANCE
#define SW_SPIN_LOCK_INSTANCE(lock_num) ({             \
    extern spin_lock_t _sw_spin_locks[NUM_SPIN_LOCKS]; \
    &_sw_spin_locks[lock_num];                         \
    })
#endif
 
#ifndef SW_SPIN_LOCK_NUM
#define SW_SPIN_LOCK_NUM(lock) ({                          \
        extern spin_lock_t _sw_spin_locks[NUM_SPIN_LOCKS]; \
        (lock) - _sw_spin_locks;                           \
        })
#endif
 
#ifndef SW_SPIN_LOCK_IS_LOCKED
#define SW_SPIN_LOCK_IS_LOCKED(lock) ((bool) *(lock))
#endif
 
#ifndef SW_SPIN_LOCK_LOCK
#if __ARM_ARCH_8M_MAIN__
#define SW_SPIN_LOCK_LOCK(lock) ({                             \
    uint32_t _tmp0, _tmp1;                                     \
    pico_default_asm_volatile (                                \
    "1:\n"                                                     \
    "ldaexb %1, [%2]\n"                                        \
    "movs %0, #1\n" /* fill dependency slot */                 \
    "cmp %1, #0\n"                                             \
    /* Immediately retry if lock is seen to be taken */        \
    "bne 1b\n"                                                 \
    /* Attempt to claim */                                     \
    "strexb %1, %0, [%2]\n"                                    \
    "cmp %1, #0\n"                                             \
    /* Claim failed due to intervening write, so retry */      \
    "bne 1b\n"                                                 \
    : "=&r" (_tmp0), "=&r" (_tmp1) : "r" (lock)                \
    );                                                         \
    __mem_fence_acquire();                                     \
    })
#elif __riscv && (defined(__riscv_a) || defined(__riscv_zaamo))
#define SW_SPIN_LOCK_LOCK(lock) ({                                              \
    uint32_t _tmp0, _tmp1;                                                      \
    pico_default_asm_volatile (                                                 \
        /* Get word address, and bit mask for LSB of the */                     \
        /* correct byte within that word -- note shamt is modulo xlen: */       \
        "slli %1, %0, 3\n"                                                      \
        "bset %1, zero, %1\n"                                                   \
        "andi %0, %0, -4\n"                                                     \
        /* Repeatedly set the bit until we see that it was clear at the */      \
        /* point we set it. A set from 0 -> 1 is a successful lock take. */     \
    "1:"                                                                        \
        "amoor.w.aq %2, %1, (%0)\n"                                             \
        "and %2, %2, %1\n"                                                      \
        "bnez %2, 1b\n"                                                         \
        : "+r" (lock), "=r" (_tmp0), "=r" (_tmp1)                               \
    );                                                                          \
    __mem_fence_acquire();                                                      \
    })
#else
#error no SW_SPIN_LOCK_LOCK available for PICO_USE_SW_SPIN_LOCK on this platform
#endif
#endif
 
#ifndef SW_SPIN_TRY_LOCK
#if __ARM_ARCH_8M_MAIN__
#define SW_SPIN_TRY_LOCK(lock) ({                              \
    uint32_t _tmp0, _tmp1;                                     \
    pico_default_asm_volatile (                                \
    "ldaexb %1, [%2]\n"                                        \
    "movs %0, #1\n" /* fill dependency slot */                 \
    "cmp %1, #0\n"                                             \
    /* Immediately give up if lock is seen to be taken */      \
    "bne 1f\n"                                                 \
    /* Otherwise attempt to claim, once. */                    \
    "strexb %1, %0, [%2]\n"                                    \
    "1:\n"                                                     \
    : "=&r" (_tmp0), "=&r" (_tmp1) : "r" (lock)                \
    );                                                         \
    __mem_fence_acquire();                                     \
    !_tmp1;                                                    \
    })
#elif __riscv && (defined(__riscv_a) || defined(__riscv_zaamo))
#define SW_SPIN_TRY_LOCK(lock) ({                                               \
    uint32_t _tmp0;                                                             \
    pico_default_asm_volatile (                                                 \
        /* Get word address, and bit mask for LSB of the */                     \
        /* correct byte within that word -- note shamt is modulo xlen: */       \
        "slli %1, %0, 3\n"                                                      \
        "bset %1, zero, %1\n"                                                   \
        "andi %0, %0, -4\n"                                                     \
        /* Set the bit. If it was clear at the point we set it, then we took */ \
        /* the lock. Otherwise the lock was already held, and we give up. */    \
        "amoor.w.aq %0, %1, (%0)\n"                                             \
        "and %1, %1, %0\n"                                                      \
        : "+r" (lock), "=r" (_tmp0)                                             \
    );                                                                          \
    __mem_fence_acquire();                                                      \
    !_tmp0;                                                                     \
    })
#else
#error no SW_SPIN_TRY_LOCK available for PICO_USE_SW_SPIN_LOCK on this platform
#endif
#endif
 
#ifndef SW_SPIN_LOCK_UNLOCK
#if __ARM_ARCH_8M_MAIN__
#define SW_SPIN_LOCK_UNLOCK(lock) ({                                        \
    /* Release-ordered store is available: use instead of separate fence */ \
    uint32_t zero = 0;                                                      \
    pico_default_asm_volatile(                                              \
        "stlb %0, [%1]\n"                                                   \
        : : "r" (zero), "r" (lock)                                          \
    );                                                                      \
    })
#elif __riscv
#define SW_SPIN_LOCK_UNLOCK(lock) ({                              \
    __mem_fence_release();                                        \
    *(lock) = 0; /* write to spinlock register (release lock) */  \
    })
#else
#error no SW_SPIN_LOCK_UNLOCK available for PICO_USE_SW_SPIN_LOCK on this platform
#endif
#endif
 
#endif

__force_inline static spin_lock_t *spin_lock_instance(uint lock_num) {
    invalid_params_if(HARDWARE_SYNC, lock_num >= NUM_SPIN_LOCKS);
#if PICO_USE_SW_SPIN_LOCKS
    return SW_SPIN_LOCK_INSTANCE(lock_num);
#else
    return (spin_lock_t *) (SIO_BASE + SIO_SPINLOCK0_OFFSET + lock_num * 4);
#endif
}

__force_inline static uint spin_lock_get_num(spin_lock_t *lock) {
#if PICO_USE_SW_SPIN_LOCKS
    uint lock_num = SW_SPIN_LOCK_NUM(lock);
    invalid_params_if(HARDWARE_SYNC, lock_num >= (uint)NUM_SPIN_LOCKS);
    return lock_num;
#else
    invalid_params_if(HARDWARE_SYNC, (uint) lock < SIO_BASE + SIO_SPINLOCK0_OFFSET ||
                            (uint) lock >= NUM_SPIN_LOCKS * sizeof(spin_lock_t) + SIO_BASE + SIO_SPINLOCK0_OFFSET ||
                            ((uint) lock - SIO_BASE + SIO_SPINLOCK0_OFFSET) % sizeof(spin_lock_t) != 0);
    return (uint) (lock - (spin_lock_t *) (SIO_BASE + SIO_SPINLOCK0_OFFSET));
#endif
}

__force_inline static void spin_lock_unsafe_blocking(spin_lock_t *lock) {
    // Note we don't do a wfe or anything, because by convention these spin_locks are VERY SHORT LIVED and NEVER BLOCK and run
    // with INTERRUPTS disabled (to ensure that)... therefore nothing on our core could be blocking us, so we just need to wait on another core
    // anyway which should be finished soon
#if PICO_USE_SW_SPIN_LOCKS
    SW_SPIN_LOCK_LOCK(lock);
#else
    while (__builtin_expect(!*lock, 0)) { // read from spinlock register (tries to acquire the lock)
        tight_loop_contents();
    }
    __mem_fence_acquire();
#endif
}
 
__force_inline static bool spin_try_lock_unsafe(spin_lock_t *lock) {
#if PICO_USE_SW_SPIN_LOCKS
    return SW_SPIN_TRY_LOCK(lock);
#else
    return *lock;
#endif
}
__force_inline static void spin_unlock_unsafe(spin_lock_t *lock) {
#if PICO_USE_SW_SPIN_LOCKS
    SW_SPIN_LOCK_UNLOCK(lock);
#else
    __mem_fence_release();
    *lock = 0; // write to spinlock register (release lock)
#endif
}

__force_inline static uint32_t spin_lock_blocking(spin_lock_t *lock) {
    uint32_t save = save_and_disable_interrupts();
    spin_lock_unsafe_blocking(lock);
    return save;
}

inline static bool is_spin_locked(spin_lock_t *lock) {
#if PICO_USE_SW_SPIN_LOCKS
    return SW_SPIN_LOCK_IS_LOCKED(lock);
#else
    check_hw_size(spin_lock_t, 4);
    uint lock_num = spin_lock_get_num(lock);
    return 0 != (*(io_ro_32 *) (SIO_BASE + SIO_SPINLOCK_ST_OFFSET) & (1u << lock_num));
#endif
}

__force_inline static void spin_unlock(spin_lock_t *lock, uint32_t saved_irq) {
    spin_unlock_unsafe(lock);
    restore_interrupts_from_disabled(saved_irq);
}

spin_lock_t *spin_lock_init(uint lock_num);

void spin_locks_reset(void);
 
#endif