diff options
Diffstat (limited to 'riscv/riscv-probe/libfemto/include/arch')
| -rw-r--r-- | riscv/riscv-probe/libfemto/include/arch/riscv/csr.h | 81 | ||||
| -rw-r--r-- | riscv/riscv-probe/libfemto/include/arch/riscv/encoding.h | 197 | ||||
| -rw-r--r-- | riscv/riscv-probe/libfemto/include/arch/riscv/machine.h | 183 | ||||
| -rw-r--r-- | riscv/riscv-probe/libfemto/include/arch/riscv/pte.h | 121 | ||||
| -rw-r--r-- | riscv/riscv-probe/libfemto/include/arch/riscv/spinlock.h | 192 | ||||
| -rw-r--r-- | riscv/riscv-probe/libfemto/include/arch/riscv/stdatomic.h | 592 | ||||
| -rw-r--r-- | riscv/riscv-probe/libfemto/include/arch/riscv/trap.h | 51 |
7 files changed, 1417 insertions, 0 deletions
diff --git a/riscv/riscv-probe/libfemto/include/arch/riscv/csr.h b/riscv/riscv-probe/libfemto/include/arch/riscv/csr.h new file mode 100644 index 0000000..8bbc5f4 --- /dev/null +++ b/riscv/riscv-probe/libfemto/include/arch/riscv/csr.h @@ -0,0 +1,81 @@ +// See LICENSE for license details. + +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +int* csr_enum_array(); +const char** csr_name_array(); +long read_csr_enum(int csrenum); +void write_csr_enum(int csrenum, long value); + +enum { + csr_none, + csr_fflags, + csr_frm, + csr_fcsr, + csr_mcycle, + csr_minstret, + csr_mcycleh, + csr_minstreth, + csr_cycle, + csr_time, + csr_instret, + csr_cycleh, + csr_timeh, + csr_instreth, + csr_mvendorid, + csr_marchid, + csr_mimpid, + csr_mhartid, + csr_mstatus, + csr_misa, + csr_medeleg, + csr_mideleg, + csr_mie, + csr_mtvec, + csr_mcounteren, + csr_mscratch, + csr_mepc, + csr_mcause, + csr_mtval, + csr_mip, + csr_sstatus, + csr_sedeleg, + csr_sideleg, + csr_sie, + csr_stvec, + csr_scounteren, + csr_sscratch, + csr_sepc, + csr_scause, + csr_stval, + csr_sip, + csr_satp, + csr_pmpcfg0, + csr_pmpcfg1, + csr_pmpcfg2, + csr_pmpcfg3, + csr_pmpaddr0, + csr_pmpaddr1, + csr_pmpaddr2, + csr_pmpaddr3, + csr_pmpaddr4, + csr_pmpaddr5, + csr_pmpaddr6, + csr_pmpaddr7, + csr_pmpaddr8, + csr_pmpaddr9, + csr_pmpaddr10, + csr_pmpaddr11, + csr_pmpaddr12, + csr_pmpaddr13, + csr_pmpaddr14, + csr_pmpaddr15 +}; + +#ifdef __cplusplus +} +#endif diff --git a/riscv/riscv-probe/libfemto/include/arch/riscv/encoding.h b/riscv/riscv-probe/libfemto/include/arch/riscv/encoding.h new file mode 100644 index 0000000..d592bbc --- /dev/null +++ b/riscv/riscv-probe/libfemto/include/arch/riscv/encoding.h @@ -0,0 +1,197 @@ +// See LICENSE for license details. + +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +#define MSTATUS_UIE 0x00000001 +#define MSTATUS_SIE 0x00000002 +#define MSTATUS_HIE 0x00000004 +#define MSTATUS_MIE 0x00000008 +#define MSTATUS_UPIE 0x00000010 +#define MSTATUS_SPIE 0x00000020 +#define MSTATUS_HPIE 0x00000040 +#define MSTATUS_MPIE 0x00000080 +#define MSTATUS_SPP 0x00000100 +#define MSTATUS_HPP 0x00000600 +#define MSTATUS_MPP 0x00001800 +#define MSTATUS_FS 0x00006000 +#define MSTATUS_XS 0x00018000 +#define MSTATUS_MPRV 0x00020000 +#define MSTATUS_SUM 0x00040000 +#define MSTATUS_MXR 0x00080000 +#define MSTATUS_TVM 0x00100000 +#define MSTATUS_TW 0x00200000 +#define MSTATUS_TSR 0x00400000 +#define MSTATUS32_SD 0x80000000 +#define MSTATUS64_SD 0x8000000000000000 + +#define SSTATUS_UIE 0x00000001 +#define SSTATUS_SIE 0x00000002 +#define SSTATUS_UPIE 0x00000010 +#define SSTATUS_SPIE 0x00000020 +#define SSTATUS_SPP 0x00000100 +#define SSTATUS_FS 0x00006000 +#define SSTATUS_XS 0x00018000 +#define SSTATUS_SUM 0x00040000 +#define SSTATUS_MXR 0x00080000 +#define SSTATUS32_SD 0x80000000 +#define SSTATUS64_SD 0x8000000000000000 + +#define DCSR_XDEBUGVER (3U<<30) +#define DCSR_NDRESET (1<<29) +#define DCSR_FULLRESET (1<<28) +#define DCSR_EBREAKM (1<<15) +#define DCSR_EBREAKH (1<<14) +#define DCSR_EBREAKS (1<<13) +#define DCSR_EBREAKU (1<<12) +#define DCSR_STOPCYCLE (1<<10) +#define DCSR_STOPTIME (1<<9) +#define DCSR_CAUSE (7<<6) +#define DCSR_DEBUGINT (1<<5) +#define DCSR_HALT (1<<3) +#define DCSR_STEP (1<<2) +#define DCSR_PRV (3<<0) + +#define DCSR_CAUSE_NONE 0 +#define DCSR_CAUSE_SWBP 1 +#define DCSR_CAUSE_HWBP 2 +#define DCSR_CAUSE_DEBUGINT 3 +#define DCSR_CAUSE_STEP 4 +#define DCSR_CAUSE_HALT 5 + +#define MCONTROL_TYPE(xlen) (0xfULL<<((xlen)-4)) +#define MCONTROL_DMODE(xlen) (1ULL<<((xlen)-5)) +#define MCONTROL_MASKMAX(xlen) (0x3fULL<<((xlen)-11)) + +#define MCONTROL_SELECT (1<<19) +#define MCONTROL_TIMING (1<<18) +#define MCONTROL_ACTION (0x3f<<12) +#define MCONTROL_CHAIN (1<<11) +#define MCONTROL_MATCH (0xf<<7) +#define MCONTROL_M (1<<6) +#define MCONTROL_H (1<<5) +#define MCONTROL_S (1<<4) +#define MCONTROL_U (1<<3) +#define MCONTROL_EXECUTE (1<<2) +#define MCONTROL_STORE (1<<1) +#define MCONTROL_LOAD (1<<0) + +#define MCONTROL_TYPE_NONE 0 +#define MCONTROL_TYPE_MATCH 2 + +#define MCONTROL_ACTION_DEBUG_EXCEPTION 0 +#define MCONTROL_ACTION_DEBUG_MODE 1 +#define MCONTROL_ACTION_TRACE_START 2 +#define MCONTROL_ACTION_TRACE_STOP 3 +#define MCONTROL_ACTION_TRACE_EMIT 4 + +#define MCONTROL_MATCH_EQUAL 0 +#define MCONTROL_MATCH_NAPOT 1 +#define MCONTROL_MATCH_GE 2 +#define MCONTROL_MATCH_LT 3 +#define MCONTROL_MATCH_MASK_LOW 4 +#define MCONTROL_MATCH_MASK_HIGH 5 + +#define MIP_SSIP (1 << IRQ_S_SOFT) +#define MIP_HSIP (1 << IRQ_H_SOFT) +#define MIP_MSIP (1 << IRQ_M_SOFT) +#define MIP_STIP (1 << IRQ_S_TIMER) +#define MIP_HTIP (1 << IRQ_H_TIMER) +#define MIP_MTIP (1 << IRQ_M_TIMER) +#define MIP_SEIP (1 << IRQ_S_EXT) +#define MIP_HEIP (1 << IRQ_H_EXT) +#define MIP_MEIP (1 << IRQ_M_EXT) + +#define SIP_SSIP MIP_SSIP +#define SIP_STIP MIP_STIP + +#define PRV_U 0 +#define PRV_S 1 +#define PRV_H 2 +#define PRV_M 3 + +#define SPTBR32_MODE 0x80000000 +#define SPTBR32_ASID 0x7FC00000 +#define SPTBR32_PPN 0x003FFFFF +#define SPTBR64_MODE 0xF000000000000000 +#define SPTBR64_ASID 0x0FFFF00000000000 +#define SPTBR64_PPN 0x00000FFFFFFFFFFF + +#define SPTBR_MODE_OFF 0 +#define SPTBR_MODE_SV32 1 +#define SPTBR_MODE_SV39 8 +#define SPTBR_MODE_SV48 9 +#define SPTBR_MODE_SV57 10 +#define SPTBR_MODE_SV64 11 + +#define PMP_R 0x01 +#define PMP_W 0x02 +#define PMP_X 0x04 +#define PMP_A 0x18 +#define PMP_L 0x80 +#define PMP_SHIFT 2 +#define PMPCFG_COUNT 4 +#define PMPADDR_COUNT 16 + +#define PMP_OFF 0x00 +#define PMP_TOR 0x08 +#define PMP_NA4 0x10 +#define PMP_NAPOT 0x18 + +#define IRQ_S_SOFT 1 +#define IRQ_H_SOFT 2 +#define IRQ_M_SOFT 3 +#define IRQ_S_TIMER 5 +#define IRQ_H_TIMER 6 +#define IRQ_M_TIMER 7 +#define IRQ_S_EXT 9 +#define IRQ_H_EXT 10 +#define IRQ_M_EXT 11 +#define IRQ_COP 12 +#define IRQ_HOST 13 + +#define DEFAULT_RSTVEC 0x00001000 +#define CLINT_BASE 0x02000000 +#define CLINT_SIZE 0x000c0000 +#define EXT_IO_BASE 0x40000000 +#define DRAM_BASE 0x80000000 + +// page table entry (PTE) fields +#define PTE_V 0x001 // Valid +#define PTE_R 0x002 // Read +#define PTE_W 0x004 // Write +#define PTE_X 0x008 // Execute +#define PTE_U 0x010 // User +#define PTE_G 0x020 // Global +#define PTE_A 0x040 // Accessed +#define PTE_D 0x080 // Dirty +#define PTE_SOFT 0x300 // Reserved for Software + +#define PTE_PPN_SHIFT 10 + +#define PTE_TABLE(PTE) (((PTE) & (PTE_V | PTE_R | PTE_W | PTE_X)) == PTE_V) + +#ifdef __riscv + +#if __riscv_xlen == 64 +# define MSTATUS_SD MSTATUS64_SD +# define SSTATUS_SD SSTATUS64_SD +# define RISCV_PGLEVEL_BITS 9 +# define SPTBR_MODE SPTBR64_MODE +#else +# define MSTATUS_SD MSTATUS32_SD +# define SSTATUS_SD SSTATUS32_SD +# define RISCV_PGLEVEL_BITS 10 +# define SPTBR_MODE SPTBR32_MODE +#endif +#define RISCV_PGSHIFT 12 +#define RISCV_PGSIZE (1 << RISCV_PGSHIFT) + +#endif + +#ifdef __cplusplus +} +#endif diff --git a/riscv/riscv-probe/libfemto/include/arch/riscv/machine.h b/riscv/riscv-probe/libfemto/include/arch/riscv/machine.h new file mode 100644 index 0000000..077a6c1 --- /dev/null +++ b/riscv/riscv-probe/libfemto/include/arch/riscv/machine.h @@ -0,0 +1,183 @@ +// See LICENSE for license details. + +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +#include <stdint.h> +#include <stddef.h> + +void arch_setup(); +void exit(int status) __attribute__((noreturn)); + +#define die(str, ...) ({ \ + printf("%s:%d: " str "\n", __FILE__, __LINE__, ##__VA_ARGS__); exit(-1); }) + +#define assert(x) ({ if (!(x)) die("assertion failed: %s", #x); }) + +#define read_const_csr(reg) ({ unsigned long __tmp; \ + asm ("csrr %0, " #reg : "=r"(__tmp)); __tmp; }) + +#define read_csr(reg) ({ unsigned long __tmp; \ + asm volatile ("csrr %0, " #reg : "=r"(__tmp)); __tmp; }) + +#define write_csr(reg, val) ({ \ + asm volatile ("csrw " #reg ", %0" :: "rK"(val)); }) + +#define swap_csr(reg, val) ({ unsigned long __tmp; \ + asm volatile ("csrrw %0, " #reg ", %1" : "=r"(__tmp) : "rK"(val)); __tmp; }) + +#define set_csr(reg, bit) ({ unsigned long __tmp; \ + asm volatile ("csrrs %0, " #reg ", %1" : "=r"(__tmp) : "rK"(bit)); __tmp; }) + +#define clear_csr(reg, bit) ({ unsigned long __tmp; \ + asm volatile ("csrrc %0, " #reg ", %1" : "=r"(__tmp) : "rK"(bit)); __tmp; }) + +static inline uintptr_t get_field(uintptr_t reg, uintptr_t mask) +{ + return ((reg & mask) / (mask & ~(mask << 1))); +} + +static inline uintptr_t set_field(uintptr_t reg, uintptr_t mask, uintptr_t val) +{ + return ((reg & ~mask) | ((val * (mask & ~(mask << 1))) & mask)); +} + +static inline unsigned long rdtime() { return read_csr(time); } +static inline unsigned long rdcycle() { return read_csr(cycle); } +static inline unsigned long rdinstret() { return read_csr(instret); } +static inline int64_t misa() { return read_const_csr(misa); } +static inline int has_ext(char ext) { return misa() & (1 << (ext - 'a')); } +static inline int xlen() { return misa() < 0 ? 64 : 32; } +static inline void wfi() { asm volatile ("wfi" ::: "memory"); } + +__attribute__((noreturn)) static inline void mret() +{ + asm volatile ("mret"); + __builtin_unreachable(); +} + + +/* + * Memory + * + * TODO - improve this API to return a list of memory segments + */ + +typedef struct memory_info +{ + uintptr_t start; + uintptr_t end; +} memory_info_t; + + +/* + * memory_probe - return memory_info + */ +memory_info_t memory_probe(); + +/* + * memory_probe_range - probe a memory address range + */ +uintptr_t memory_probe_range(uintptr_t start, uintptr_t end); + + +/* + * Physical Memory Protection + * + * PMP is optional but if implememented, enforcement must be enabled by + * default, if no PMP entries are set. This means loads, stores or fetches + * from any mode besides M mode, will fail unless explicitly allowed. + * PMP must be configured irregardless of whether it is implemented. + */ + +typedef struct pmp_info +{ + int width; + int granularity; + int count; +} pmp_info_t; + +/* + * pmp_probe - return pmp_info + */ +pmp_info_t pmp_probe(); + +/* + * pmp_entry_granularity - return PMP entry width (physical memory width) + */ +int pmp_entry_width(); + +/* + * pmp_entry_granularity - return PMP entry granularity (smallest entry size) + */ +int pmp_entry_granularity(); + +/* + * pmp_entry_count - return number of PMP entries + */ +int pmp_entry_count(); + +/* + * pmp_clear_all - set PMP to disallow mode != PRV_M physical memory accesses + */ +void pmp_clear_all(); + +/* + * pmp_allow_all - set PMP to allow mode != PRV_M physical memory accesses + */ +void pmp_allow_all(); + +/* + * pmp_entry_set - set one PMP entry + * + * - n : pmp entry number + * - prot : protection (PMP_R | PMP_W | PMP_X) + * - addr : start address + * - len : power of two length + */ +int pmp_entry_set(unsigned n, uint8_t prot, uint64_t addr, uint64_t len); + + +/* + * Privileged modes + */ + +/* + * mode_set_and_jump + * + * Set mstatus.mpp, sets mepc to passed function pointer and then issues mret + * Note: the hart will continue running on the same stack + */ +static inline void mode_set_and_jump(unsigned mode, void (*fn)(void)) +{ + assert(mode <= PRV_U); + write_csr(mstatus, set_field(read_csr(mstatus), MSTATUS_MPP, mode)); + write_csr(mepc, fn); + mret(); +} + +/* + * mode_set_and_continue + * + * Set mstatus.mpp, sets mepc to instruction after mret and then issues mret + * Note: the hart will continue running on the same stack + */ +static inline void mode_set_and_continue(unsigned mode) +{ + assert(mode <= PRV_U); + write_csr(mstatus, set_field(read_csr(mstatus), MSTATUS_MPP, mode)); + asm volatile ( + "lla t0, 1f\n" + "csrw mepc, t0\n" + "mret\n" + "1:" + ::: "t0" + ); +} + +#ifdef __cplusplus +} +#endif diff --git a/riscv/riscv-probe/libfemto/include/arch/riscv/pte.h b/riscv/riscv-probe/libfemto/include/arch/riscv/pte.h new file mode 100644 index 0000000..38d7b8d --- /dev/null +++ b/riscv/riscv-probe/libfemto/include/arch/riscv/pte.h @@ -0,0 +1,121 @@ +#pragma once + +enum { + PTE_SHIFT_V = 0, + PTE_SHIFT_R = 1, + PTE_SHIFT_W = 2, + PTE_SHIFT_X = 3, + PTE_SHIFT_U = 4, + PTE_SHIFT_G = 5, + PTE_SHIFT_A = 6, + PTE_SHIFT_D = 7, + PTE_SHIFT_SW1 = 8, + PTE_SHIFT_SW2 = 9, + PTE_V = 1 << PTE_SHIFT_V, + PTE_R = 1 << PTE_SHIFT_R, + PTE_W = 1 << PTE_SHIFT_W, + PTE_X = 1 << PTE_SHIFT_X, + PTE_U = 1 << PTE_SHIFT_U, + PTE_G = 1 << PTE_SHIFT_G, + PTE_A = 1 << PTE_SHIFT_A, + PTE_D = 1 << PTE_SHIFT_D, + PTE_SW1 = 1 << PTE_SHIFT_SW1, + PTE_SW2 = 1 << PTE_SHIFT_SW2 +}; + +enum { + SV32_LEVELS = 2, + SV32_LEVEL_BITS = 10, + SV32_PTE_SIZE = 4 +}; + +union sv32_va { + uint32_t val; + struct { + uint32_t pg_off : 12; + uint32_t vpn : 20; + } va; +}; + +union sv32_pa { + uint64_t val; + struct { + uint64_t pg_off : 12; + uint64_t ppn : 22; + uint64_t rsrv : 30; + } pa; +}; + +union sv32_pte { + uint32_t val; + struct { + uint32_t flags : 10; + uint32_t ppn : 22; + } pte; +}; + +enum { + SV39_LEVELS = 3, + SV39_LEVEL_BITS = 9, + SV39_PTE_SIZE = 8 +}; + +union sv39_va { + uint64_t val; + struct { + uint64_t pg_off : 12; + uint64_t vpn : 27; + uint64_t rsrv : 25; + } va; +}; + +union sv39_pa { + uint64_t val; + struct { + uint64_t pg_off : 12; + uint64_t ppn : 44; + uint64_t rsrv : 8; + } pa; +}; + +union sv39_pte { + uint64_t val; + struct { + uint64_t flags : 10; + uint64_t ppn : 44; + uint64_t rsrv : 10; + } pte; +}; + +enum { + SV48_LEVELS = 4, + SV48_LEVEL_BITS = 9, + SV48_PTE_SIZE = 8 +}; + +union sv48_va { + uint64_t val; + struct { + uint64_t pg_off : 12; + uint64_t vpn : 36; + uint64_t rsrv : 16; + } va; +}; + +union sv48_pa { + uint64_t val; + struct { + uint64_t pg_off : 12; + uint64_t ppn : 44; + uint64_t rsrv : 8; + } pa; +}; + +union sv48_pte { + uint64_t val; + struct { + uint64_t flags : 10; + uint64_t ppn : 44; + uint64_t rsrv : 10; + } pte; +}; diff --git a/riscv/riscv-probe/libfemto/include/arch/riscv/spinlock.h b/riscv/riscv-probe/libfemto/include/arch/riscv/spinlock.h new file mode 100644 index 0000000..d747cef --- /dev/null +++ b/riscv/riscv-probe/libfemto/include/arch/riscv/spinlock.h @@ -0,0 +1,192 @@ +// See LICENSE for license details. + +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Ticket Lock + * + * Author: Michael Clark <[email protected]> + * + * Ticket locks are fair spinlocks that order acccess on a first-in, + * first-out basis. The lock is composed of a head counter and a tail + * counter. The head counter indicates the ticket number of the current + * lock owner. The tail counter indicates the last issued ticket number. + * To acquire the lock, the acquiring thread atomically increments the + * tail counter to assign itself a ticket number. It then waits until + * its assigned ticket number is seen in the head counter. If the lock + * is not contended, then head will equal the ticket number just assigned. + * To release the lock, the lock owner atomically increments head counter + * thereby allowing access to the thread holding the next ticket number. + * + * Initialializtion - the lock is initialized unlocked, so that the + * next tail increment returns a ticket that will acquire the lock. + * + * tail | head + * -------|------- + * 0x0000 | 0x0001 + * + * Lock - the tail is incremented, and the thread waits unti the tail + * is equal to the head before returning control, which is the typical + * case if the lock is uncontended. + * + * tail | head + * -------|------- + * 0x0001 | 0x0001 + * + * Unlock - the head is incremented, so that it will now equal the value + * of the ticket of the next waiter, if any. + * + * tail | head + * -------|------- + * 0x0001 | 0x0002 + * + * Waiters - the number of waiters can be calculated by subtracting + * head (current ticket) from the tail plus one (next ticket) + * + * num_waiters = tail + 1 - head + */ + +/** + * Ticket lock types + */ + +typedef unsigned short ticket_t; +typedef unsigned int ticketdata_t; + +typedef struct { + volatile ticketdata_t data; +} spinlock_t; + +#define TICKET_SHIFT ((sizeof(ticketdata_t) / sizeof(ticket_t)) * 8) + +/** + * Ticket lock encode + */ +static inline ticketdata_t ticketlock_data(ticket_t tail, ticket_t head) +{ + return (((ticketdata_t)tail) << TICKET_SHIFT) | (ticketdata_t)head; +} + +/** + * Ticket lock tail + */ +static inline ticket_t ticketlock_tail(ticketdata_t data) { return data >> 16; } + +/** + * Ticket lock head + */ +static inline ticket_t ticketlock_head(ticketdata_t data) { return data; } + +/** + * Ticket lock initializer + */ +#define SPINLOCK_INIT (spinlock_t){1}; + +/** + * load reserved acquire + * + * \param pvalue pointer to word to load from + * \return loaded value + */ +static inline unsigned int _lr_aq_w(volatile unsigned int *pvalue) +{ + unsigned int value; + asm volatile ( + "lr.w.aq %0, %1" + : "=&r"(value), "+A"(*pvalue)); + return value; +} + +/** + * store conditional release + * + * \param pvalue pointer to word to store to + * \param value word to write + * \return true for failure + */ +static inline int _sc_rl_w(volatile unsigned int *pvalue, unsigned int value) +{ + int result; + asm volatile ( + "sc.w.rl %0, %2, %1" + : "=&r"(result), "+A"(*pvalue) + : "r"(value) + : "memory" + ); + return result; +} + +/** + * load acquire + * + * \param pvalue pointer to word to load from + * \return loaded value + */ +static inline unsigned int _l_aq_w(volatile unsigned int *pvalue) +{ + unsigned int result; + asm volatile ( + "lw %0, %1\n" + "fence r,rw" + : "=&r"(result), "+A"(*pvalue) + ); + return result; +} + +/** + * barrier acquire + * + * order subsequent modifications after prior lock reads + * + * <read lock> + * _barrier_acquire(); + */ +static inline void _barrier_acquire() { asm volatile ("fence r,rw"); } + +/** + * barrier release + * + * order prior modifications before subsequent lock write + * + * _barrier_release(); + * <write lock> + */ +static inline void _barrier_release() { asm volatile ("fence rw,w"); } + +/** + * spinlock lock + * + * \param lock pointer to lock + */ +void spinlock_lock(spinlock_t *lock); + +/** + * spinlock trylock + * + * \param lock pointer to lock + * \return non zero if lock was successfully taken + */ +int spinlock_trylock(spinlock_t *lock); + +/** + * spinlock unlock + * + * \param lock pointer to lock + */ +void spinlock_unlock(spinlock_t *lock); + +/** + * spinlock waiters + * + * \param lock pointer to lock + * \return number of waiters + */ +int spinlock_waiters(spinlock_t *lock); + +#ifdef __cplusplus +} +#endif diff --git a/riscv/riscv-probe/libfemto/include/arch/riscv/stdatomic.h b/riscv/riscv-probe/libfemto/include/arch/riscv/stdatomic.h new file mode 100644 index 0000000..8672636 --- /dev/null +++ b/riscv/riscv-probe/libfemto/include/arch/riscv/stdatomic.h @@ -0,0 +1,592 @@ +// See LICENSE for license details. + +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * We use the GCC implementation of __atomic_is_lock_free + */ + +#define atomic_is_lock_free(obj) \ + __atomic_is_lock_free(sizeof(obj), obj) + +/* + * GCC Built-in Atomics (not enabled by default, because they are buggy) + */ + +#if defined(LIBFEMTO_USE_GCC_BUILTINS) + +#define atomic_flag_test_and_set(obj) \ + __atomic_test_and_set(obj, __ATOMIC_SEQ_CST) +#define atomic_flag_test_and_set_explicit(obj, order) \ + __atomic_test_and_set(obj, order) + +#define atomic_flag_clear(obj) \ + __atomic_test_and_set(obj, __ATOMIC_SEQ_CST) +#define atomic_flag_clear_explicit(obj, order) \ + __atomic_test_and_set(obj, order) + +#define atomic_thread_fence(order) \ + __atomic_thread_fence(order) + +#define atomic_signal_fence(order) \ + __atomic_signal_fence(order) + +#define atomic_store(obj, desired) \ + __atomic_store_n(obj, desired, __ATOMIC_SEQ_CST) +#define atomic_store_explicit(obj, desired, order) \ + __atomic_store_n(obj, desired, order) + +#define atomic_load(obj) \ + __atomic_load_n(obj, __ATOMIC_SEQ_CST) +#define atomic_load_explicit(obj,order) \ + __atomic_load_n(obj,order) + +#define atomic_exchange(obj, desired) \ + __atomic_exchange_n(obj, desired, __ATOMIC_SEQ_CST) +#define atomic_exchange_explicit(obj, desired, order) \ + __atomic_exchange_n(obj, desired, order) + +#define atomic_compare_exchange_strong(obj, expected, desired) \ + __atomic_compare_exchange_n(obj, expected, desired, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) +#define atomic_compare_exchange_strong_explicit(obj, expected, desired, succ, fail) \ + __atomic_compare_exchange_n(obj, expected, desired, 0, succ, fail) + +#define atomic_compare_exchange_weak(obj, expected, desired) \ + __atomic_compare_exchange_n(obj, expected, desired, 1, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) +#define atomic_compare_exchange_weak_explicit(obj, expected, desired, succ, fail) \ + __atomic_compare_exchange_n(obj, expected, desired, 1, succ, fail) + +#define atomic_fetch_add(obj, arg) \ + __atomic_fetch_add(obj, arg, __ATOMIC_SEQ_CST) +#define atomic_fetch_add_explicit(obj, arg, order) \ + __atomic_fetch_add(obj, arg, order) + +#define atomic_fetch_sub(obj, arg) \ + __atomic_fetch_sub(obj, arg, __ATOMIC_SEQ_CST) +#define atomic_fetch_sub_explicit(obj, arg, order) \ + __atomic_fetch_sub(obj, arg, order) + +#define atomic_fetch_or(obj, arg) \ + __atomic_fetch_or(obj, arg, __ATOMIC_SEQ_CST) +#define atomic_fetch_or_explicit(obj, arg, order) \ + __atomic_fetch_or(obj, arg, order) + +#define atomic_fetch_xor(obj, arg) \ + __atomic_fetch_xor(obj, arg, __ATOMIC_SEQ_CST) +#define atomic_fetch_xor_explicit(obj, arg, order) \ + __atomic_fetch_xor(obj, arg, order) + +#define atomic_fetch_and(obj, arg) \ + __atomic_fetch_and(obj, arg, __ATOMIC_SEQ_CST) +#define atomic_fetch_and_explicit(obj, arg, order) \ + __atomic_fetch_and(obj, arg, order) +#endif + +#elif defined(__riscv) + +/* + * atomic_load + */ + +#define __atomic_load_asm(obj, ASM_BARRIER, ASM_ACQUIRE) \ +__extension__ ({ \ + __typeof__(obj) __obj = (obj); \ + __typeof__(*(obj)) __result; \ + switch (sizeof(__typeof__(*obj))) { \ + case 1: \ + __asm__ volatile ( \ + ASM_BARRIER "\n" "lb %0, %1\n" ASM_ACQUIRE \ + : "=&r"(__result), "+A"(*__obj) \ + ); \ + break; \ + case 2: \ + __asm__ volatile ( \ + ASM_BARRIER "\n" "lh %0, %1\n" ASM_ACQUIRE \ + : "=&r"(__result), "+A"(*__obj) \ + ); \ + break; \ + case 4: \ + __asm__ volatile ( \ + ASM_BARRIER "\n" "lw %0, %1\n" ASM_ACQUIRE \ + : "=&r"(__result), "+A"(*__obj) \ + ); \ + break; \ + case 8: \ + if (__riscv_xlen < 64) __builtin_unreachable(); \ + __asm__ volatile ( \ + ASM_BARRIER "\n" "ld %0, %1\n" ASM_ACQUIRE \ + : "=&r"(__result), "+A"(*__obj) \ + ); \ + break; \ + } \ + __result; \ +}) + +#define __atomic_load_relaxed(obj) \ + __atomic_load_asm(obj, "", "") + +#define __atomic_load_acquire(obj) \ + __atomic_load_asm(obj, "", "fence r,rw") + +#define __atomic_load_seq_cst(obj) \ + __atomic_load_asm(obj, "fence rw,rw", "fence r,rw") + +#define atomic_load_explicit(obj, order) \ +__extension__ ({ \ + __typeof__(*(obj)) __result; \ + switch (order) { \ + case __ATOMIC_ACQUIRE: \ + __result = __atomic_load_acquire(obj) ; break; \ + case __ATOMIC_SEQ_CST: \ + __result = __atomic_load_seq_cst(obj) ; break; \ + case __ATOMIC_RELAXED: \ + default: \ + __result = __atomic_load_relaxed(obj) ; break; \ + } \ + __result; \ +}) + +#define atomic_load(obj) \ + atomic_load_explicit(obj, __ATOMIC_SEQ_CST) + +/* + * atomic_store + */ + +#define __atomic_store_asm(obj, value, ASM_RELEASE) \ +__extension__ ({ \ + __typeof__(obj) __obj = (obj); \ + __typeof__(*(obj)) __value = (value); \ + switch (sizeof(__typeof__(*obj))) { \ + case 1: \ + __asm__ volatile ( \ + ASM_RELEASE "\n" "sb %1, %0\n" \ + : "+A"(*__obj) : "r"(__value) : "memory" \ + ); \ + break; \ + case 2: \ + __asm__ volatile ( \ + ASM_RELEASE "\n" "sh %1, %0\n" \ + : "+A"(*__obj) : "r"(__value) : "memory" \ + ); \ + break; \ + case 4: \ + __asm__ volatile ( \ + ASM_RELEASE "\n" "sw %1, %0\n" \ + : "+A"(*__obj) : "r"(__value) : "memory" \ + ); \ + break; \ + case 8: \ + if (__riscv_xlen < 64) __builtin_unreachable(); \ + __asm__ volatile ( \ + ASM_RELEASE "\n" "sd %1, %0\n" \ + : "+A"(*__obj) : "r"(__value) : "memory" \ + ); \ + break; \ + } \ +}) + +#define __atomic_store_relaxed(obj, value) \ + __atomic_store_asm(obj, value, "") + +#define __atomic_store_release(obj, value) \ + __atomic_store_asm(obj, value, "fence rw,w") + +#define __atomic_store_seq_cst(obj, value) \ + __atomic_store_asm(obj, value, "fence rw,w") + + +#define atomic_store_explicit(obj, value, order) \ +__extension__ ({ \ + switch (order) { \ + case __ATOMIC_RELEASE: \ + __atomic_store_release(obj, value) ; break; \ + case __ATOMIC_SEQ_CST: \ + __atomic_store_seq_cst(obj, value) ; break; \ + case __ATOMIC_RELAXED: \ + default: \ + __atomic_store_relaxed(obj, value) ; break; \ + } \ +}) + +#define atomic_store(obj, value) \ + atomic_store_explicit(obj, value, __ATOMIC_SEQ_CST) + +/* + * atomic_compare_exchange + */ + +#define __atomic_cmpxchg_asm(obj, exp, val, ASM_AQ, ASM_RL) \ +__extension__ ({ \ + __typeof__(obj) __obj = (obj); \ + __typeof__(*(obj)) __exp = (exp); \ + __typeof__(*(obj)) __val = (val); \ + __typeof__(*(obj)) __result; \ + register unsigned int __ret; \ + switch (sizeof(__typeof__(*obj))) { \ + case 1: \ + case 2: \ + __builtin_unreachable(); \ + break; \ + case 4: \ + __asm__ volatile ( \ + "0: lr.w" ASM_AQ " %0, %2\n" \ + " bne %0, %z3, 1f\n" \ + " sc.w" ASM_RL " %1, %z4, %2\n" \ + " bnez %1, 0b \n" /* always strong */ \ + "1:\n" \ + : "=&r"(__result), "=&r" (__ret), "+A"(*__obj) \ + : "r"(__exp), "r"(__val) \ + : "memory" \ + ); \ + break; \ + case 8: \ + if (__riscv_xlen < 64) __builtin_unreachable(); \ + __asm__ volatile ( \ + "0: lr.d" ASM_AQ " %0, %2\n" \ + " bne %0, %z3, 1f\n" \ + " sc.d" ASM_RL " %1, %z4, %2\n" \ + " bnez %1, 0b \n" /* always strong */ \ + "1:\n" \ + : "=&r"(__result), "=&r" (__ret), "+A"(*__obj) \ + : "r"(__exp), "r"(__val) \ + : "memory" \ + ); \ + break; \ + } \ + __result; \ +}) + +#define __atomic_cmpxchg_relaxed(obj, exp, val) \ + __atomic_cmpxchg_asm(obj, exp, val, "", "") + +#define __atomic_cmpxchg_acquire(obj, exp, val) \ + __atomic_cmpxchg_asm(obj, exp, val, ".aq", "") + +#define __atomic_cmpxchg_release(obj, exp, val) \ + __atomic_cmpxchg_asm(obj, exp, val, "", ".rl") + +#define __atomic_cmpxchg_acq_rel(obj, exp, val) \ + __atomic_cmpxchg_asm(obj, exp, val, ".aq", ".rl") + +#define __atomic_cmpxchg_seq_cst(obj, exp, val) \ + __atomic_cmpxchg_asm(obj, exp, val, ".aqrl", ".rl") + +#define __atomic_cmpxchg_strong(obj, exp, val, succ, fail) \ +__extension__ ({ \ + __typeof__(*(obj)) __result; \ + switch (succ) { \ + case __ATOMIC_ACQUIRE: \ + case __ATOMIC_CONSUME: /* promote to acquire for now */ \ + __result = __atomic_cmpxchg_acquire(obj, exp, val); break; \ + case __ATOMIC_RELEASE: \ + __result = __atomic_cmpxchg_release(obj, exp, val); break; \ + case __ATOMIC_ACQ_REL: \ + __result = __atomic_cmpxchg_acq_rel(obj, exp, val); break; \ + case __ATOMIC_SEQ_CST: \ + __result = __atomic_cmpxchg_seq_cst(obj, exp, val); break; \ + case __ATOMIC_RELAXED: \ + default: \ + __result = __atomic_cmpxchg_relaxed(obj, exp, val); break; \ + } \ + __result; \ +}) + +#define atomic_compare_exchange_strong(obj, exp, val) \ + __atomic_cmpxchg_strong(obj, exp, val, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) +#define atomic_compare_exchange_strong_explicit(obj, exp, val, succ, fail) \ + __atomic_cmpxchg_strong(obj, exp, val, succ, fail) + +#define atomic_compare_exchange_weak(obj, exp, val) \ + __atomic_cmpxchg_strong(obj, exp, val, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) +#define atomic_compare_exchange_weak_explicit(obj, exp, val, succ, fail) \ + __atomic_cmpxchg_strong(obj, exp, val, succ, fail) + +/* + * atomic_op template + */ + +#define __atomic_op_asm(AMO_OP, obj, arg, ASM_AQRL) \ +__extension__ ({ \ + __typeof__(obj) __obj = (obj); \ + __typeof__(*(obj)) __arg = (arg); \ + __typeof__(*(obj)) __result; \ + switch (sizeof(__typeof__(*obj))) { \ + case 1: \ + case 2: \ + __builtin_unreachable(); \ + break; \ + case 4: \ + __asm__ volatile ( \ + AMO_OP ".w" ASM_AQRL " %0, %2, %1\n" \ + : "=&r"(__result), "+A"(*__obj) \ + : "r"(__arg) \ + : "memory" \ + ); \ + break; \ + case 8: \ + if (__riscv_xlen < 64) __builtin_unreachable(); \ + __asm__ volatile ( \ + AMO_OP ".d" ASM_AQRL " %0, %2, %1\n" \ + : "=&r"(__result), "+A"(*__obj) \ + : "r"(__arg) \ + : "memory" \ + ); \ + break; \ + } \ + __result; \ +}) + +#define __atomic_op_relaxed(op, obj, arg) \ + __atomic_op_asm(op, obj, arg, "") + +#define __atomic_op_acquire(op, obj, arg) \ + __atomic_op_asm(op, obj, arg, ".aq") + +#define __atomic_op_release(op, obj, arg) \ + __atomic_op_asm(op, obj, arg, ".rl") + +#define __atomic_op_acq_rel(op, obj, arg) \ + __atomic_op_asm(op, obj, arg, ".aqrl") + +#define __atomic_op_seq_cst(op, obj, arg) \ + __atomic_op_asm(op, obj, arg, ".aqrl") + +#define __atomic_op(op, obj, arg, order) \ +__extension__ ({ \ + __typeof__(*(obj)) __result; \ + switch (order) { \ + case __ATOMIC_ACQUIRE: \ + case __ATOMIC_CONSUME: /* promote to acquire for now */ \ + __result = __atomic_op_acquire(op, obj, arg); break; \ + case __ATOMIC_RELEASE: \ + __result = __atomic_op_release(op, obj, arg); break; \ + case __ATOMIC_ACQ_REL: \ + __result = __atomic_op_acq_rel(op, obj, arg); break; \ + case __ATOMIC_SEQ_CST: \ + __result = __atomic_op_seq_cst(op, obj, arg); break; \ + case __ATOMIC_RELAXED: \ + default: \ + __result = __atomic_op_relaxed(op, obj, arg); break; \ + } \ + __result; \ +}) + +/* + * atomic_exchange + */ + +#define atomic_exchange(obj, arg) \ + __atomic_op("amoswap", obj, arg, __ATOMIC_SEQ_CST) +#define atomic_exchange_explicit(obj, arg, order) \ + __atomic_op("amoswap", obj, arg, order) + +/* + * atomic_fetch_add + */ + +#define atomic_fetch_add(obj, arg) \ + __atomic_op("amoadd", obj, arg, __ATOMIC_SEQ_CST) +#define atomic_fetch_add_explicit(obj, arg, order) \ + __atomic_op("amoadd", obj, arg, order) + +/* + * atomic_fetch_sub + */ + +#define atomic_fetch_sub(obj, arg) \ + __atomic_op("amoadd", obj, -arg, __ATOMIC_SEQ_CST) +#define atomic_fetch_sub_explicit(obj, arg, order) \ + __atomic_op("amoadd", obj, -arg, order) + +/* + * atomic_fetch_or + */ + +#define atomic_fetch_or(obj, arg) \ + __atomic_op("amoor", obj, -arg, __ATOMIC_SEQ_CST) +#define atomic_fetch_or_explicit(obj, arg, order) \ + __atomic_op("amoor", obj, -arg, order) + +/* + * atomic_fetch_xor + */ + +#define atomic_fetch_xor(obj, arg) \ + __atomic_op("amoxor", obj, -arg, __ATOMIC_SEQ_CST) +#define atomic_fetch_xor_explicit(obj, arg, order) \ + __atomic_op("amoxor", obj, -arg, order) + +/* + * atomic_fetch_and + */ + +#define atomic_fetch_and(obj, arg) \ + __atomic_op("amoand", obj, -arg, __ATOMIC_SEQ_CST) +#define atomic_fetch_and_explicit(obj, arg, order) \ + __atomic_op("amoand", obj, -arg, order) + + +/* + * atomic_flag_test_and_set + */ + +#define __atomic_flag_test_and_set_asm(obj, ASM_AQRL) \ +__extension__ ({ \ + struct atomic_flag* __obj = (obj); \ + size_t __shift = (((unsigned long)__obj) & 3) << 3; \ + uint32_t *__word = (uint32_t *)(((unsigned long)__obj) & ~3); \ + uint32_t __mask = 1 << __shift; \ + uint32_t __result; \ + __asm__ volatile ( \ + "amoor.w" ASM_AQRL " %0, %2, %1\n" \ + : "=&r"(__result), "+A"(*__word) \ + : "r"(__mask) \ + : "memory" \ + ); \ + ((__result >> __shift) & 0xff); \ +}) + +#define __atomic_flag_test_and_set_relaxed(obj) \ + __atomic_flag_test_and_set_asm(obj, "") + +#define __atomic_flag_test_and_set_acquire(obj) \ + __atomic_flag_test_and_set_asm(obj, ".aq") + +#define __atomic_flag_test_and_set_release(obj) \ + __atomic_flag_test_and_set_asm(obj, ".rl") + +#define __atomic_flag_test_and_set_acq_rel(obj) \ + __atomic_flag_test_and_set_asm(obj, ".aqrl") + +#define __atomic_flag_test_and_set_seq_cst(obj) \ + __atomic_flag_test_and_set_asm(obj, ".aqrl") + + +#define __atomic_flag_test_and_set(obj, order) \ +__extension__ ({ \ + _Bool __result; \ + switch (order) { \ + case __ATOMIC_ACQUIRE: \ + case __ATOMIC_CONSUME: /* promote to acquire for now */ \ + __result = __atomic_flag_test_and_set_acquire(obj); break; \ + case __ATOMIC_RELEASE: \ + __result = __atomic_flag_test_and_set_release(obj); break; \ + case __ATOMIC_ACQ_REL: \ + __result = __atomic_flag_test_and_set_acq_rel(obj); break; \ + case __ATOMIC_SEQ_CST: \ + __result = __atomic_flag_test_and_set_seq_cst(obj); break; \ + case __ATOMIC_RELAXED: \ + default: \ + __result = __atomic_flag_test_and_set_relaxed(obj); break; \ + } \ + __result; \ +}) + +#define atomic_flag_test_and_set_explicit(obj, order) \ + __atomic_flag_test_and_set(obj, order) +#define atomic_flag_test_and_set(obj, order) \ + __atomic_flag_test_and_set(obj, __ATOMIC_SEQ_CST) + +/* + * atomic_flag_clear + */ + +#define __atomic_flag_clear_asm(obj, ASM_AQRL) \ +__extension__ ({ \ + struct atomic_flag* __obj = (obj); \ + size_t __shift = (((unsigned long)__obj) & 3) << 3; \ + uint32_t *__word = (uint32_t *)(((unsigned long)__obj) & ~3); \ + uint32_t __mask = ~(0xff << __shift); \ + uint32_t __result; \ + __asm__ volatile ( \ + "amoand.w" ASM_AQRL " %0, %2, %1\n" \ + : "=&r"(__result), "+A"(*__word) \ + : "r"(__mask) \ + : "memory" \ + ); \ + ((__result >> __shift) & 0xff); \ +}) + +#define __atomic_flag_clear_relaxed(obj) \ + __atomic_flag_clear_asm(obj, "") + +#define __atomic_flag_clear_acquire(obj) \ + __atomic_flag_clear_asm(obj, ".aq") + +#define __atomic_flag_clear_release(obj) \ + __atomic_flag_clear_asm(obj, ".rl") + +#define __atomic_flag_clear_acq_rel(obj) \ + __atomic_flag_clear_asm(obj, ".aqrl") + +#define __atomic_flag_clear_seq_cst(obj) \ + __atomic_flag_clear_asm(obj, ".aqrl") + + +#define __atomic_flag_clear(obj, order) \ +__extension__ ({ \ + _Bool __result; \ + switch (order) { \ + case __ATOMIC_ACQUIRE: \ + case __ATOMIC_CONSUME: /* promote to acquire for now */ \ + __result = __atomic_flag_clear_acquire(obj); break; \ + case __ATOMIC_RELEASE: \ + __result = __atomic_flag_clear_release(obj); break; \ + case __ATOMIC_ACQ_REL: \ + __result = __atomic_flag_clear_acq_rel(obj); break; \ + case __ATOMIC_SEQ_CST: \ + __result = __atomic_flag_clear_seq_cst(obj); break; \ + case __ATOMIC_RELAXED: \ + default: \ + __result = __atomic_flag_clear_relaxed(obj); break; \ + } \ + __result; \ +}) + +#define atomic_flag_clear_explicit(obj, order) \ + __atomic_flag_clear(obj, order) +#define atomic_flag_clear(obj, order) \ + __atomic_flag_clear(obj, __ATOMIC_SEQ_CST) + +/* + * atomic_thread_fence + */ + +#define __atomic_thread_fence_asm(order) \ +__extension__ ({ \ + switch (order) { \ + case __ATOMIC_ACQUIRE: \ + case __ATOMIC_CONSUME: /* promote to acquire for now */ \ + __asm__ volatile ("fence r,rw" ::: "memory"); break; \ + case __ATOMIC_RELEASE: \ + __asm__ volatile ("fence rw,w" ::: "memory"); break; \ + case __ATOMIC_ACQ_REL: /* should be fence.tso */ \ + __asm__ volatile ("fence rw,rw" ::: "memory"); break; \ + case __ATOMIC_SEQ_CST: \ + __asm__ volatile ("fence rw,rw" ::: "memory"); break; \ + case __ATOMIC_RELAXED: \ + default: \ + __asm__ volatile ("" ::: "memory"); break; \ + } \ +}) + +#define atomic_thread_fence(order) \ + __atomic_thread_fence_asm(order) + +/* + * atomic_signal_fence + */ + +#define atomic_signal_fence(order) \ + __asm__ volatile ("" ::: "memory") + +#endif /* __riscv */ + +#ifdef __cplusplus +} +#endif diff --git a/riscv/riscv-probe/libfemto/include/arch/riscv/trap.h b/riscv/riscv-probe/libfemto/include/arch/riscv/trap.h new file mode 100644 index 0000000..0b33ec5 --- /dev/null +++ b/riscv/riscv-probe/libfemto/include/arch/riscv/trap.h @@ -0,0 +1,51 @@ +// See LICENSE for license details. + +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void (*trap_fn)(uintptr_t* regs, uintptr_t mcause, uintptr_t mepc); +trap_fn get_trap_fn(); +void set_trap_fn(trap_fn fn); + +const char * riscv_excp_names[16]; +const char * riscv_intr_names[16]; + +enum { + cause_misaligned_fetch = 0, + cause_fault_fetch = 1, + cause_illegal_instruction = 2, + cause_breakpoint = 3, + cause_misaligned_load = 4, + cause_fault_load = 5, + cause_misaligned_store = 6, + cause_fault_store = 7, + cause_user_ecall = 8, + cause_supervisor_ecall = 9, + cause_hypervisor_ecall = 10, + cause_machine_ecall = 11, + cause_exec_page_fault = 12, + cause_load_page_fault = 13, + cause_store_page_fault = 15 +}; + +enum { + intr_u_software = 0, + intr_s_software = 1, + intr_h_software = 2, + intr_m_software = 3, + intr_u_timer = 4, + intr_s_timer = 5, + intr_h_timer = 6, + intr_m_timer = 7, + intr_u_external = 8, + intr_s_external = 9, + intr_h_external = 10, + intr_m_external = 11, +}; + +#ifdef __cplusplus +} +#endif |