diff --git a/arch/risc-v/src/mpfs/Make.defs b/arch/risc-v/src/mpfs/Make.defs index d4765ba0be8..a57c6164c36 100644 --- a/arch/risc-v/src/mpfs/Make.defs +++ b/arch/risc-v/src/mpfs/Make.defs @@ -84,6 +84,7 @@ endif ifeq (${CONFIG_MPFS_OPENSBI},y) CHIP_ASRCS += mpfs_opensbi_utils.S +CHIP_ASRCS += mpfs_opensbi_trap.S CHIP_CSRCS += mpfs_opensbi.c endif diff --git a/arch/risc-v/src/mpfs/hardware/mpfs_ihc.h b/arch/risc-v/src/mpfs/hardware/mpfs_ihc.h index af7a081dd94..9b8bf00571e 100644 --- a/arch/risc-v/src/mpfs/hardware/mpfs_ihc.h +++ b/arch/risc-v/src/mpfs/hardware/mpfs_ihc.h @@ -27,8 +27,8 @@ enum mpfs_irq_type_e { - MP_IRQ = 0x0, - ACK_IRQ = 0x1, + MP_IRQ = 0x1, + ACK_IRQ = 0x2, }; #define IHC_MAX_MESSAGE_SIZE 2 diff --git a/arch/risc-v/src/mpfs/mpfs_ihc.c b/arch/risc-v/src/mpfs/mpfs_ihc.c index a9b0e71ec4e..a8c878d622c 100644 --- a/arch/risc-v/src/mpfs/mpfs_ihc.c +++ b/arch/risc-v/src/mpfs/mpfs_ihc.c @@ -39,6 +39,7 @@ #include #include #include +#include #include #include @@ -138,6 +139,12 @@ struct mpfs_queue_table_s void *data; }; +struct mpfs_ihc_work_arg_s +{ + uint32_t mhartid; + uint32_t rhartid; +}; + /**************************************************************************** * Private Function Prototypes ****************************************************************************/ @@ -155,6 +162,7 @@ static int mpfs_rptun_notify(struct rptun_dev_s *dev, uint32_t notifyid); static int mpfs_rptun_register_callback(struct rptun_dev_s *dev, rptun_callback_t callback, void *arg); +static void mpfs_rptun_worker(void *arg); /**************************************************************************** * Private Data @@ -179,8 +187,11 @@ static struct mpfs_rptun_shmem_s g_shmem; static struct rpmsg_device *g_mpfs_rpmsg_device; static struct rpmsg_virtio_device *g_mpfs_virtio_device; -static sem_t g_mpfs_ack_sig = SEM_INITIALIZER(0); +#ifdef MPFS_RPTUN_USE_THREAD static sem_t g_mpfs_rx_sig = SEM_INITIALIZER(0); +#else +static struct work_s g_rptun_work; +#endif static struct list_node g_dev_list = LIST_INITIAL_VALUE(g_dev_list); static uint32_t g_connected_hart_ints; @@ -189,6 +200,11 @@ static uint16_t g_vq_idx; static int g_plic_irq; static bool g_rptun_initialized; +#ifdef IHC_AVOID_ACK_AND_MP +static struct mpfs_ihc_work_arg_s g_work_arg; +static struct work_s g_ihc_work; +#endif + const uint32_t ihcia_remote_harts[MPFS_NUM_HARTS] = { IHCIA_H0_REMOTE_HARTS, @@ -237,6 +253,7 @@ static const struct rptun_ops_s g_mpfs_rptun_ops = * mhartid base on the context, not necessarily the actual * mhartid. * is_ack - Boolean that is set true if an ack has been found + * is_msg - Boolean that is set true if a message is present * * Returned Value: * Remote hart id @@ -244,7 +261,8 @@ static const struct rptun_ops_s g_mpfs_rptun_ops = ****************************************************************************/ static uint32_t mpfs_ihc_parse_incoming_hartid(uint32_t mhartid, - bool *is_ack) + bool *is_ack, + bool *is_msg) { uint32_t hart_id = 0; uint32_t return_hart_id = UNDEFINED_HART_ID; @@ -262,6 +280,13 @@ static uint32_t mpfs_ihc_parse_incoming_hartid(uint32_t mhartid, { return_hart_id = hart_id; *is_ack = true; + + test_int = (1 << (hart_id * 2)); + + if ((g_connected_hart_ints & test_int) == test_int) + { + *is_msg = true; + } break; } } @@ -273,7 +298,7 @@ static uint32_t mpfs_ihc_parse_incoming_hartid(uint32_t mhartid, if (((g_connected_hart_ints & test_int) == test_int)) { return_hart_id = hart_id; - *is_ack = false; + *is_msg = true; break; } } @@ -415,39 +440,68 @@ static uint32_t mpfs_ihc_context_to_local_hart_id(ihc_channel_t channel) * Name: mpfs_ihc_rx_handler * * Description: - * This handles the received information and either lets the vq to proceed - * via posting g_mpfs_ack_sig, or lets the mpfs_rptun_thread() run as it - * waits for the g_mpfs_rx_sig. virtqueue_notification() cannot be called - * from the interrupt context, thus the thread that will perform it. + * This handles the received information and lets the vq to proceed. + * virtqueue_notification() cannot be called from the interrupt context, + * thus the thread or work queue that will perform it. * * Input Parameters: * message - Pointer to the incoming message - * is_ack - Boolean indicating whether an ack is received * * Returned Value: * None * ****************************************************************************/ -static void mpfs_ihc_rx_handler(uint32_t *message, bool is_ack) +static void mpfs_ihc_rx_handler(uint32_t *message) { - if (is_ack) - { - /* Received the ack */ + g_vq_idx = message[0]; - nxsem_post(&g_mpfs_ack_sig); - } - else - { - g_vq_idx = message[0]; + DEBUGASSERT((g_vq_idx == VRING0_NOTIFYID) || + (g_vq_idx == VRING1_NOTIFYID)); - DEBUGASSERT((g_vq_idx == VRING0_NOTIFYID) || - (g_vq_idx == VRING1_NOTIFYID)); - - nxsem_post(&g_mpfs_rx_sig); - } +#ifdef MPFS_RPTUN_USE_THREAD + nxsem_post(&g_mpfs_rx_sig); +#else + work_queue(HPWORK, &g_rptun_work, mpfs_rptun_worker, NULL, 0); +#endif } +/**************************************************************************** + * Name: mpfs_ihc_worker + * + * Description: + * This function is used to wait for the remote message present condition, + * after which the ACK is sent. ACK wasn't sent before, as the remote end + * has no way of knowing which one came first: the ACK or RMP. + * + * Input Parameters: + * arg - Pointer to the arguments struct + * + * Returned Value: + * None + * + ****************************************************************************/ + +#ifdef IHC_AVOID_ACK_AND_MP +static void mpfs_ihc_worker(void *arg) +{ + uint32_t ctrl_reg; + uint32_t retries = 5000; + + do + { + ctrl_reg = getreg32(MPFS_IHC_CTRL(g_work_arg.mhartid, + g_work_arg.rhartid)); + } + while ((ctrl_reg & RMP_MESSAGE_PRESENT) && --retries); + + DEBUGASSERT(retries != 0); + + modifyreg32(MPFS_IHC_CTRL(g_work_arg.mhartid, + g_work_arg.rhartid), 0, ACK_INT); +} +#endif + /**************************************************************************** * Name: mpfs_ihc_rx_message * @@ -459,7 +513,6 @@ static void mpfs_ihc_rx_handler(uint32_t *message, bool is_ack) * channel - Enum that describes the channel used. * mhartid - Context hart id, not necessarily the absolute mhartid but * rather, the primary hartid of the set of harts. - * is_ack - Boolean indicating an ack message * msg - For storing data, could be NULL * * Returned Value: @@ -468,58 +521,54 @@ static void mpfs_ihc_rx_handler(uint32_t *message, bool is_ack) ****************************************************************************/ static void mpfs_ihc_rx_message(ihc_channel_t channel, uint32_t mhartid, - bool is_ack, uint32_t *msg) + uint32_t *msg) { uint32_t rhartid = mpfs_ihc_context_to_remote_hart_id(channel); uint32_t ctrl_reg = getreg32(MPFS_IHC_CTRL(mhartid, rhartid)); - if (is_ack) - { - if (mhartid == CONTEXTB_HARTID) - { - uintptr_t msg_in = MPFS_IHC_MSG_IN(mhartid, rhartid); - DEBUGASSERT(msg == NULL); - mpfs_ihc_rx_handler((uint32_t *)msg_in, is_ack); - } - else - { - /* This path is meant for the OpenSBI vendor extension only */ + /* Check if we have a message */ - DEBUGPANIC(); - } + if (mhartid == CONTEXTB_HARTID) + { + uintptr_t msg_in = MPFS_IHC_MSG_IN(mhartid, rhartid); + DEBUGASSERT(msg == NULL); + mpfs_ihc_rx_handler((uint32_t *)msg_in); } - else if (MP_MESSAGE_PRESENT == (ctrl_reg & MP_MASK)) + else { - /* Check if we have a message */ + /* This path is meant for the OpenSBI vendor extension only */ - if (mhartid == CONTEXTB_HARTID) - { - uintptr_t msg_in = MPFS_IHC_MSG_IN(mhartid, rhartid); - DEBUGASSERT(msg == NULL); - mpfs_ihc_rx_handler((uint32_t *)msg_in, is_ack); - } - else - { - /* This path is meant for the OpenSBI vendor extension only */ + DEBUGPANIC(); + } - DEBUGPANIC(); - } + /* Set MP to 0. Note this generates an interrupt on the other hart + * if it has RMPIE bit set in the control register + */ - /* Set MP to 0. Note this generates an interrupt on the other hart - * if it has RMPIE bit set in the control register + ctrl_reg = getreg32(MPFS_IHC_CTRL(mhartid, rhartid)); + if (ctrl_reg & RMP_MESSAGE_PRESENT) + { + /* If we send the ACK here, Linux will have the ACK and the + * MP flags sets. IHC_AVOID_ACK_AND_MP assures only one + * is present at once. */ - volatile uint32_t temp = getreg32(MPFS_IHC_CTRL(mhartid, rhartid)) & - ~MP_MASK; +#ifdef IHC_AVOID_ACK_AND_MP + g_work_arg.mhartid = mhartid; + g_work_arg.rhartid = rhartid; + modifyreg32(MPFS_IHC_CTRL(mhartid, rhartid), MP_MASK, 0); + work_queue(HPWORK, &g_ihc_work, mpfs_ihc_worker, NULL, 0); +#else + modifyreg32(MPFS_IHC_CTRL(mhartid, rhartid), MP_MASK, 0); + modifyreg32(MPFS_IHC_CTRL(mhartid, rhartid), 0, ACK_INT); +#endif + } + else + { + /* We can send the ACK now and clear the MP */ - /* Check if ACKIE_EN is set */ - - if (temp & ACKIE_EN) - { - temp |= ACK_INT; - } - - putreg32(temp, MPFS_IHC_CTRL(mhartid, rhartid)); + modifyreg32(MPFS_IHC_CTRL(mhartid, rhartid), MP_MASK, 0); + modifyreg32(MPFS_IHC_CTRL(mhartid, rhartid), 0, ACK_INT); } } @@ -541,11 +590,13 @@ static void mpfs_ihc_rx_message(ihc_channel_t channel, uint32_t mhartid, static void mpfs_ihc_message_present_isr(void) { uint64_t mhartid = riscv_mhartid(); - bool is_ack; + bool is_ack = false; + bool is_msg = false; /* Check all our channels */ - uint32_t origin_hart = mpfs_ihc_parse_incoming_hartid(mhartid, &is_ack); + uint32_t origin_hart = mpfs_ihc_parse_incoming_hartid(mhartid, &is_ack, + &is_msg); if (origin_hart != UNDEFINED_HART_ID) { @@ -564,7 +615,10 @@ static void mpfs_ihc_message_present_isr(void) /* Process incoming packet */ - mpfs_ihc_rx_message(origin_hart, mhartid, is_ack, NULL); + if (is_msg) + { + mpfs_ihc_rx_message(origin_hart, mhartid, NULL); + } if (is_ack) { @@ -698,7 +752,7 @@ static int mpfs_ihc_tx_message(ihc_channel_t channel, uint32_t *message) { ctrl_reg = getreg32(MPFS_IHC_CTRL(mhartid, rhartid)); } - while ((ctrl_reg & (RMP_MESSAGE_PRESENT | ACK_INT)) && --retries); + while ((ctrl_reg & (RMP_MESSAGE_PRESENT)) && --retries); /* Return if RMP bit 1 indicating busy */ @@ -706,10 +760,6 @@ static int mpfs_ihc_tx_message(ihc_channel_t channel, uint32_t *message) { return -EBUSY; } - else if (ACK_INT == (ctrl_reg & ACK_INT_MASK)) - { - return -EBUSY; - } else { /* Fill the buffer */ @@ -719,18 +769,18 @@ static int mpfs_ihc_tx_message(ihc_channel_t channel, uint32_t *message) putreg32(message[i], MPFS_IHC_MSG_OUT(mhartid, rhartid) + i * 4); } + ctrl_reg = getreg32(MPFS_IHC_CTRL(mhartid, rhartid)); + + /* If we're unlucky, we cannot send MP yet.. come back later */ + + if (ctrl_reg & MP_MESSAGE_PRESENT) + { + return -EBUSY; + } + /* Set the MP bit. This will notify other of incoming hart message */ modifyreg32(MPFS_IHC_CTRL(mhartid, rhartid), 0, RMP_MESSAGE_PRESENT); - - /* Wait for the ACK to arrive to maintain the logic */ - - if (mhartid == CONTEXTB_HARTID) - { - /* Only applicable for the CONTEXTB_HART */ - - nxsem_wait_uninterruptible(&g_mpfs_ack_sig); - } } return OK; @@ -846,12 +896,10 @@ mpfs_rptun_get_resource(struct rptun_dev_s *dev) rsc->rpmsg_vdev.id = VIRTIO_ID_RPMSG; rsc->rpmsg_vdev.notifyid = VDEV_NOTIFYID; rsc->rpmsg_vdev.dfeatures = 1 << VIRTIO_RPMSG_F_NS | - 1 << VIRTIO_RPMSG_F_ACK | - VIRTIO_RING_F_EVENT_IDX; + 1 << VIRTIO_RPMSG_F_ACK; rsc->rpmsg_vdev.gfeatures = 1 << VIRTIO_RPMSG_F_NS | - 1 << VIRTIO_RPMSG_F_ACK | - VIRTIO_RING_F_EVENT_IDX; + 1 << VIRTIO_RPMSG_F_ACK; /* Set to VIRTIO_CONFIG_STATUS_DRIVER_OK when master is up */ @@ -980,6 +1028,8 @@ static int mpfs_rptun_stop(struct rptun_dev_s *dev) static int mpfs_rptun_notify(struct rptun_dev_s *dev, uint32_t notifyid) { uint32_t tx_msg[IHC_MAX_MESSAGE_SIZE]; + uint32_t retries = 5; + int ret = OK; /* We only care about the queue with notifyid VRING0 */ @@ -988,10 +1038,18 @@ static int mpfs_rptun_notify(struct rptun_dev_s *dev, uint32_t notifyid) tx_msg[0] = notifyid; tx_msg[1] = 0; - return mpfs_ihc_tx_message(CONTEXTA_HARTID, tx_msg); + /* This failure should happen very rarely */ + + do + { + ret = mpfs_ihc_tx_message(CONTEXTA_HARTID, tx_msg); + } + while ((ret != OK) && --retries); + + DEBUGASSERT(ret == OK); } - return OK; + return ret; } /**************************************************************************** @@ -1153,6 +1211,33 @@ static void mpfs_rpmsg_device_created(struct rpmsg_device *rdev, void *priv_) mpfs_echo_ping_init(rdev, &g_mpgs_echo_ping_ept); } +/**************************************************************************** + * Name: mpfs_rptun_worker + * + * Description: + * This is used to notify the associated virtqueue via the scheduled work. + * This doesn't use a separate thread, but a HPWORK instead, which is a + * way to avoid deadlocks with net_lock() that also originate from HPWORK. + * + * Input Parameters: + * arg - Argument + + * Returned Value: + * None + * + ****************************************************************************/ + +#ifndef MPFS_RPTUN_USE_THREAD +static void mpfs_rptun_worker(void *arg) +{ + struct mpfs_queue_table_s *info; + + DEBUGASSERT((g_vq_idx - VRING0_NOTIFYID) < VRINGS); + info = &g_mpfs_virtqueue_table[g_vq_idx - VRING0_NOTIFYID]; + virtqueue_notification((struct virtqueue *)info->data); +} +#endif + /**************************************************************************** * Name: mpfs_rptun_thread * @@ -1170,6 +1255,7 @@ static void mpfs_rpmsg_device_created(struct rpmsg_device *rdev, void *priv_) * ****************************************************************************/ +#ifdef MPFS_RPTUN_USE_THREAD static int mpfs_rptun_thread(int argc, char *argv[]) { struct mpfs_queue_table_s *info; @@ -1185,6 +1271,7 @@ static int mpfs_rptun_thread(int argc, char *argv[]) return 0; } +#endif /**************************************************************************** * Public Functions @@ -1210,8 +1297,10 @@ static int mpfs_rptun_thread(int argc, char *argv[]) int mpfs_ihc_init(void) { uint32_t mhartid = (uint32_t)riscv_mhartid(); +#ifdef MPFS_RPTUN_USE_THREAD char *argv[3]; char arg1[19]; +#endif uint32_t rhartid; int ret; @@ -1275,6 +1364,8 @@ int mpfs_ihc_init(void) goto init_error; } +#ifdef MPFS_RPTUN_USE_THREAD + /* Thread initialization */ snprintf(arg1, sizeof(arg1), "%p", @@ -1292,6 +1383,7 @@ int mpfs_ihc_init(void) NULL, NULL, NULL); goto init_error; } +#endif return OK; diff --git a/arch/risc-v/src/mpfs/mpfs_ihc_sbi.c b/arch/risc-v/src/mpfs/mpfs_ihc_sbi.c index bf90abc20f9..a52f548e328 100644 --- a/arch/risc-v/src/mpfs/mpfs_ihc_sbi.c +++ b/arch/risc-v/src/mpfs/mpfs_ihc_sbi.c @@ -67,6 +67,35 @@ static uint32_t g_connected_harts_c; * Private Functions ****************************************************************************/ +/**************************************************************************** + * Name: mpfs_modifyreg32 + * + * Description: + * This is a copy of modifyreg32() without spinlock. That function is a + * real danger here as it is likely located in eNVM, thus being a real + * bottleneck. All functions called from this file should be located in + * the zero device. + * + * Input Parameters: + * addr - Address to perform the operation + * clearbits - Bits to clear + * setbits - Bits to set + * + * Returned Value: + * Remote hart id + * + ****************************************************************************/ + +void mpfs_modifyreg32(uintptr_t addr, uint32_t clearbits, uint32_t setbits) +{ + uint32_t regval; + + regval = getreg32(addr); + regval &= ~clearbits; + regval |= setbits; + putreg32(regval, addr); +} + /**************************************************************************** * Name: mpfs_ihc_sbi_parse_incoming_hartid * @@ -79,6 +108,7 @@ static uint32_t g_connected_harts_c; * mhartid base on the context, not necessarily the actual * mhartid. * is_ack - Boolean that is set true if an ack has been found + * is_mp - Boolean that is set true if a msg is present also * * Returned Value: * Remote hart id @@ -86,7 +116,8 @@ static uint32_t g_connected_harts_c; ****************************************************************************/ static uint32_t mpfs_ihc_sbi_parse_incoming_hartid(uint32_t mhartid, - bool *is_ack) + bool *is_ack, + bool *is_mp) { uint32_t hart_id = 0; uint32_t return_hart_id = UNDEFINED_HART_ID; @@ -112,6 +143,16 @@ static uint32_t mpfs_ihc_sbi_parse_incoming_hartid(uint32_t mhartid, { return_hart_id = hart_id; *is_ack = true; + + /* We might also have a msg present */ + + test_int = (1 << (hart_id * 2)); + + if (msg_avail & test_int) + { + *is_mp = true; + } + break; } } @@ -124,6 +165,7 @@ static uint32_t mpfs_ihc_sbi_parse_incoming_hartid(uint32_t mhartid, { return_hart_id = hart_id; *is_ack = false; + *is_mp = true; break; } } @@ -231,7 +273,6 @@ static uint32_t mpfs_ihc_sbi_context_to_local_hart_id(ihc_channel_t channel) } DEBUGASSERT(hart < MPFS_NUM_HARTS); - return hart; } @@ -259,7 +300,8 @@ static uint32_t mpfs_ihc_sbi_context_to_local_hart_id(ihc_channel_t channel) static void mpfs_ihc_sbi_message_present_handler(uint32_t *message, uint32_t mhartid, uint32_t rhartid, - bool is_ack) + bool is_ack, + bool is_mp) { struct ihc_sbi_rx_msg_s *msg; uintptr_t message_ihc = (uintptr_t)MPFS_IHC_MSG_IN(mhartid, rhartid); @@ -267,17 +309,22 @@ static void mpfs_ihc_sbi_message_present_handler(uint32_t *message, msg = (struct ihc_sbi_rx_msg_s *)message; - if (is_ack) + if (is_ack && !is_mp) { msg->irq_type = ACK_IRQ; /* msg->ihc_msg content doesn't matter here */ } - else + else if (is_mp && !is_ack) { msg->irq_type = MP_IRQ; msg->ihc_msg = *(struct mpfs_ihc_msg_s *)message_ihc; } + else + { + msg->irq_type = ACK_IRQ | MP_IRQ; + msg->ihc_msg = *(struct mpfs_ihc_msg_s *)message_ihc; + } DEBUGASSERT(sizeof(msg->ihc_msg) >= message_size_ihc); } @@ -302,11 +349,9 @@ static void mpfs_ihc_sbi_message_present_handler(uint32_t *message, ****************************************************************************/ static void mpfs_ihc_sbi_rx_message(uint32_t rhartid, uint32_t mhartid, - bool is_ack, uint32_t *msg) + bool is_ack, bool is_mp, uint32_t *msg) { - uint32_t ctrl_reg = getreg32(MPFS_IHC_CTRL(mhartid, rhartid)); - - if (is_ack) + if (is_ack && !is_mp) { if (mhartid == CONTEXTB_HARTID) { @@ -318,10 +363,14 @@ static void mpfs_ihc_sbi_rx_message(uint32_t rhartid, uint32_t mhartid, DEBUGASSERT(msg != NULL); mpfs_ihc_sbi_message_present_handler(msg, mhartid, rhartid, - is_ack); + is_ack, is_mp); + + /* Clear the ack */ + + mpfs_modifyreg32(MPFS_IHC_CTRL(mhartid, rhartid), ACK_CLR, 0); } } - else if (MP_MESSAGE_PRESENT == (ctrl_reg & MP_MASK)) + else if (is_mp && !is_ack) { /* Check if we have a message */ @@ -335,24 +384,25 @@ static void mpfs_ihc_sbi_rx_message(uint32_t rhartid, uint32_t mhartid, DEBUGASSERT(msg != NULL); mpfs_ihc_sbi_message_present_handler(msg, mhartid, rhartid, - is_ack); + is_ack, is_mp); } /* Set MP to 0. Note this generates an interrupt on the other hart * if it has RMPIE bit set in the control register */ - volatile uint32_t temp = getreg32(MPFS_IHC_CTRL(mhartid, rhartid)) & - ~MP_MASK; + mpfs_modifyreg32(MPFS_IHC_CTRL(mhartid, rhartid), MP_MASK, ACK_INT); + } + else if (is_ack && is_mp) + { + DEBUGASSERT(msg != NULL); + mpfs_ihc_sbi_message_present_handler(msg, mhartid, rhartid, + is_ack, is_mp); - /* Check if ACKIE_EN is set */ + /* Clear the ack and mp */ - if (temp & ACKIE_EN) - { - temp |= ACK_INT; - } - - putreg32(temp, MPFS_IHC_CTRL(mhartid, rhartid)); + mpfs_modifyreg32(MPFS_IHC_CTRL(mhartid, rhartid), ACK_CLR | MP_MASK, + ACK_INT); } } @@ -377,22 +427,16 @@ void mpfs_ihc_sbi_message_present_indirect_isr(ihc_channel_t channel, uint32_t *msg) { bool is_ack; + bool is_mp = false; uint32_t mhartid = mpfs_ihc_sbi_context_to_local_hart_id(channel); uint32_t origin_hart = mpfs_ihc_sbi_parse_incoming_hartid(mhartid, - &is_ack); - + &is_ack, + &is_mp); if (origin_hart != UNDEFINED_HART_ID) { /* Process incoming packet */ - mpfs_ihc_sbi_rx_message(origin_hart, mhartid, is_ack, msg); - - if (is_ack) - { - /* Clear the ack */ - - modifyreg32(MPFS_IHC_CTRL(mhartid, origin_hart), ACK_CLR, 0); - } + mpfs_ihc_sbi_rx_message(origin_hart, mhartid, is_ack, is_mp, msg); } } @@ -470,16 +514,16 @@ static void mpfs_ihc_sbi_local_remote_config(uint32_t hart_to_configure, } #endif - modifyreg32(MPFS_IHC_CTRL(hart_to_configure, rhartid), 0, MPIE_EN | - ACKIE_EN); + mpfs_modifyreg32(MPFS_IHC_CTRL(hart_to_configure, rhartid), 0, MPIE_EN | + ACKIE_EN); /* OpenSBI extension may configure 2x consecutive harts */ #ifdef CONFIG_MPFS_IHC_TWO_RPMSG_CHANNELS if ((hart_to_configure + 1) < MPFS_NUM_HARTS) { - modifyreg32(MPFS_IHC_CTRL(hart_to_configure + 1, rhartid + 1), 0, - MPIE_EN | ACKIE_EN); + mpfs_modifyreg32(MPFS_IHC_CTRL(hart_to_configure + 1, rhartid + 1), 0, + MPIE_EN | ACKIE_EN); } #endif } @@ -539,7 +583,8 @@ static int mpfs_ihc_sbi_tx_message(ihc_channel_t channel, uint32_t *message) /* Set the MP bit. This will notify other of incoming hart message */ - modifyreg32(MPFS_IHC_CTRL(mhartid, rhartid), 0, RMP_MESSAGE_PRESENT); + mpfs_modifyreg32(MPFS_IHC_CTRL(mhartid, rhartid), 0, + RMP_MESSAGE_PRESENT); } return OK; diff --git a/arch/risc-v/src/mpfs/mpfs_opensbi_trap.S b/arch/risc-v/src/mpfs/mpfs_opensbi_trap.S new file mode 100644 index 00000000000..517cc20dc98 --- /dev/null +++ b/arch/risc-v/src/mpfs/mpfs_opensbi_trap.S @@ -0,0 +1,225 @@ +/**************************************************************************** + * arch/risc-v/src/mpfs/mpfs_opensbi_trap.S + * + * mpfs_exception_opensbi function is based on OpenSBI fw_base.S + * + * The 2-Clause BSD License + * SPDX short identifier: BSD-2-Clause + * + * Copyright (c) 2019 Western Digital Corporation or its affiliates and other + * contributors. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/**************************************************************************** + * Included Files + ****************************************************************************/ + +#include + +#include +#include +#include +#include +#include + +/**************************************************************************** + * Pre-processor Definitions + ****************************************************************************/ + +/**************************************************************************** + * Public Symbols + ****************************************************************************/ + + .global mpfs_exception_opensbi + +/**************************************************************************** + * Private Data + ****************************************************************************/ + +mpfs_global_pointer: + .dword __global_pointer$ + +/**************************************************************************** + * Name: mpfs_exception_opensbi: + * + * Description: + * This is the trap entry into OpenSBI. + * + * Input Parameters: + * None + * + * Returned Value: + * None + * + ****************************************************************************/ + + .align 3 + mpfs_exception_opensbi: + + /* Swap TP and MSCRATCH */ + + csrrw tp, CSR_MSCRATCH, tp + + /* Save T0 in scratch space */ + + REG_S t0, SBI_SCRATCH_TMP0_OFFSET(tp) + + /* + * Set T0 to appropriate exception stack + * + * Came_From_M_Mode = ((MSTATUS.MPP < PRV_M) ? 1 : 0) - 1; + * Exception_Stack = TP ^ (Came_From_M_Mode & (SP ^ TP)) + * + * Came_From_M_Mode = 0 ==> Exception_Stack = TP + * Came_From_M_Mode = -1 ==> Exception_Stack = SP + */ + + csrr t0, CSR_MSTATUS + srl t0, t0, MSTATUS_MPP_SHIFT + and t0, t0, PRV_M + slti t0, t0, PRV_M + add t0, t0, -1 + xor sp, sp, tp + and t0, t0, sp + xor sp, sp, tp + xor t0, tp, t0 + + /* Save original SP on exception stack */ + + REG_S sp, (SBI_TRAP_REGS_OFFSET(sp) - SBI_TRAP_REGS_SIZE)(t0) + + /* Set SP to exception stack and make room for trap registers */ + + add sp, t0, -(SBI_TRAP_REGS_SIZE) + + /* Restore T0 from scratch space */ + + REG_L t0, SBI_SCRATCH_TMP0_OFFSET(tp) + + /* Save T0 on stack */ + + REG_S t0, SBI_TRAP_REGS_OFFSET(t0)(sp) + + /* Swap TP and MSCRATCH */ + + csrrw tp, CSR_MSCRATCH, tp + + /* Save MEPC and MSTATUS CSRs */ + + csrr t0, CSR_MEPC + REG_S t0, SBI_TRAP_REGS_OFFSET(mepc)(sp) + csrr t0, CSR_MSTATUS + REG_S t0, SBI_TRAP_REGS_OFFSET(mstatus)(sp) + REG_S zero, SBI_TRAP_REGS_OFFSET(mstatusH)(sp) + + /* Save all general regisers except SP and T0 */ + + REG_S zero, SBI_TRAP_REGS_OFFSET(zero)(sp) + REG_S ra, SBI_TRAP_REGS_OFFSET(ra)(sp) + REG_S gp, SBI_TRAP_REGS_OFFSET(gp)(sp) + REG_S tp, SBI_TRAP_REGS_OFFSET(tp)(sp) + REG_S t1, SBI_TRAP_REGS_OFFSET(t1)(sp) + REG_S t2, SBI_TRAP_REGS_OFFSET(t2)(sp) + REG_S s0, SBI_TRAP_REGS_OFFSET(s0)(sp) + REG_S s1, SBI_TRAP_REGS_OFFSET(s1)(sp) + REG_S a0, SBI_TRAP_REGS_OFFSET(a0)(sp) + REG_S a1, SBI_TRAP_REGS_OFFSET(a1)(sp) + REG_S a2, SBI_TRAP_REGS_OFFSET(a2)(sp) + REG_S a3, SBI_TRAP_REGS_OFFSET(a3)(sp) + REG_S a4, SBI_TRAP_REGS_OFFSET(a4)(sp) + REG_S a5, SBI_TRAP_REGS_OFFSET(a5)(sp) + REG_S a6, SBI_TRAP_REGS_OFFSET(a6)(sp) + REG_S a7, SBI_TRAP_REGS_OFFSET(a7)(sp) + REG_S s2, SBI_TRAP_REGS_OFFSET(s2)(sp) + REG_S s3, SBI_TRAP_REGS_OFFSET(s3)(sp) + REG_S s4, SBI_TRAP_REGS_OFFSET(s4)(sp) + REG_S s5, SBI_TRAP_REGS_OFFSET(s5)(sp) + REG_S s6, SBI_TRAP_REGS_OFFSET(s6)(sp) + REG_S s7, SBI_TRAP_REGS_OFFSET(s7)(sp) + REG_S s8, SBI_TRAP_REGS_OFFSET(s8)(sp) + REG_S s9, SBI_TRAP_REGS_OFFSET(s9)(sp) + REG_S s10, SBI_TRAP_REGS_OFFSET(s10)(sp) + REG_S s11, SBI_TRAP_REGS_OFFSET(s11)(sp) + REG_S t3, SBI_TRAP_REGS_OFFSET(t3)(sp) + REG_S t4, SBI_TRAP_REGS_OFFSET(t4)(sp) + REG_S t5, SBI_TRAP_REGS_OFFSET(t5)(sp) + REG_S t6, SBI_TRAP_REGS_OFFSET(t6)(sp) + + /* Restore GP */ + + la a0, mpfs_global_pointer + ld gp, 0(a0) + + /* Call C routine */ + + add a0, sp, zero + call sbi_trap_handler + + /* Restore all general regisers except A0 and T0 */ + + REG_L ra, SBI_TRAP_REGS_OFFSET(ra)(a0) + REG_L sp, SBI_TRAP_REGS_OFFSET(sp)(a0) + REG_L gp, SBI_TRAP_REGS_OFFSET(gp)(a0) + REG_L tp, SBI_TRAP_REGS_OFFSET(tp)(a0) + REG_L t1, SBI_TRAP_REGS_OFFSET(t1)(a0) + REG_L t2, SBI_TRAP_REGS_OFFSET(t2)(a0) + REG_L s0, SBI_TRAP_REGS_OFFSET(s0)(a0) + REG_L s1, SBI_TRAP_REGS_OFFSET(s1)(a0) + REG_L a1, SBI_TRAP_REGS_OFFSET(a1)(a0) + REG_L a2, SBI_TRAP_REGS_OFFSET(a2)(a0) + REG_L a3, SBI_TRAP_REGS_OFFSET(a3)(a0) + REG_L a4, SBI_TRAP_REGS_OFFSET(a4)(a0) + REG_L a5, SBI_TRAP_REGS_OFFSET(a5)(a0) + REG_L a6, SBI_TRAP_REGS_OFFSET(a6)(a0) + REG_L a7, SBI_TRAP_REGS_OFFSET(a7)(a0) + REG_L s2, SBI_TRAP_REGS_OFFSET(s2)(a0) + REG_L s3, SBI_TRAP_REGS_OFFSET(s3)(a0) + REG_L s4, SBI_TRAP_REGS_OFFSET(s4)(a0) + REG_L s5, SBI_TRAP_REGS_OFFSET(s5)(a0) + REG_L s6, SBI_TRAP_REGS_OFFSET(s6)(a0) + REG_L s7, SBI_TRAP_REGS_OFFSET(s7)(a0) + REG_L s8, SBI_TRAP_REGS_OFFSET(s8)(a0) + REG_L s9, SBI_TRAP_REGS_OFFSET(s9)(a0) + REG_L s10, SBI_TRAP_REGS_OFFSET(s10)(a0) + REG_L s11, SBI_TRAP_REGS_OFFSET(s11)(a0) + REG_L t3, SBI_TRAP_REGS_OFFSET(t3)(a0) + REG_L t4, SBI_TRAP_REGS_OFFSET(t4)(a0) + REG_L t5, SBI_TRAP_REGS_OFFSET(t5)(a0) + REG_L t6, SBI_TRAP_REGS_OFFSET(t6)(a0) + + /* Restore MEPC and MSTATUS CSRs */ + + REG_L t0, SBI_TRAP_REGS_OFFSET(mepc)(a0) + csrw CSR_MEPC, t0 + REG_L t0, SBI_TRAP_REGS_OFFSET(mstatus)(a0) + csrw CSR_MSTATUS, t0 + + /* Restore T0 */ + + REG_L t0, SBI_TRAP_REGS_OFFSET(t0)(a0) + + /* Restore A0 */ + + REG_L a0, SBI_TRAP_REGS_OFFSET(a0)(a0) + + mret diff --git a/arch/risc-v/src/mpfs/mpfs_opensbi_utils.S b/arch/risc-v/src/mpfs/mpfs_opensbi_utils.S index b933bdb5d44..db8f8fff7f7 100644 --- a/arch/risc-v/src/mpfs/mpfs_opensbi_utils.S +++ b/arch/risc-v/src/mpfs/mpfs_opensbi_utils.S @@ -1,34 +1,22 @@ /**************************************************************************** * arch/risc-v/src/mpfs/mpfs_opensbi_utils.S * - * mpfs_exception_opensbi function is based on OpenSBI fw_base.S + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at * - * The 2-Clause BSD License - * SPDX short identifier: BSD-2-Clause + * http://www.apache.org/licenses/LICENSE-2.0 * - * Copyright (c) 2019 Western Digital Corporation or its affiliates and other - * contributors. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ + ****************************************************************************/ /**************************************************************************** * Included Files @@ -51,7 +39,6 @@ ****************************************************************************/ .global mpfs_opensbi_prepare_hart - .global mpfs_exception_opensbi /**************************************************************************** * Private Data @@ -103,169 +90,4 @@ mpfs_opensbi_prepare_hart: mul t0, a0, t1 la sp, g_scratches add sp, sp, t0 - jal mpfs_opensbi_setup - -/**************************************************************************** - * Name: mpfs_exception_opensbi: - * - * Description: - * This is the trap entry into OpenSBI. - * - * Input Parameters: - * None - * - * Returned Value: - * None - * - ****************************************************************************/ - - .align 3 -mpfs_exception_opensbi: - - /* Swap TP and MSCRATCH */ - - csrrw tp, CSR_MSCRATCH, tp - - /* Save T0 in scratch space */ - - REG_S t0, SBI_SCRATCH_TMP0_OFFSET(tp) - - /* - * Set T0 to appropriate exception stack - * - * Came_From_M_Mode = ((MSTATUS.MPP < PRV_M) ? 1 : 0) - 1; - * Exception_Stack = TP ^ (Came_From_M_Mode & (SP ^ TP)) - * - * Came_From_M_Mode = 0 ==> Exception_Stack = TP - * Came_From_M_Mode = -1 ==> Exception_Stack = SP - */ - - csrr t0, CSR_MSTATUS - srl t0, t0, MSTATUS_MPP_SHIFT - and t0, t0, PRV_M - slti t0, t0, PRV_M - add t0, t0, -1 - xor sp, sp, tp - and t0, t0, sp - xor sp, sp, tp - xor t0, tp, t0 - - /* Save original SP on exception stack */ - - REG_S sp, (SBI_TRAP_REGS_OFFSET(sp) - SBI_TRAP_REGS_SIZE)(t0) - - /* Set SP to exception stack and make room for trap registers */ - - add sp, t0, -(SBI_TRAP_REGS_SIZE) - - /* Restore T0 from scratch space */ - - REG_L t0, SBI_SCRATCH_TMP0_OFFSET(tp) - - /* Save T0 on stack */ - - REG_S t0, SBI_TRAP_REGS_OFFSET(t0)(sp) - - /* Swap TP and MSCRATCH */ - - csrrw tp, CSR_MSCRATCH, tp - - /* Save MEPC and MSTATUS CSRs */ - - csrr t0, CSR_MEPC - REG_S t0, SBI_TRAP_REGS_OFFSET(mepc)(sp) - csrr t0, CSR_MSTATUS - REG_S t0, SBI_TRAP_REGS_OFFSET(mstatus)(sp) - REG_S zero, SBI_TRAP_REGS_OFFSET(mstatusH)(sp) - - /* Save all general regisers except SP and T0 */ - - REG_S zero, SBI_TRAP_REGS_OFFSET(zero)(sp) - REG_S ra, SBI_TRAP_REGS_OFFSET(ra)(sp) - REG_S gp, SBI_TRAP_REGS_OFFSET(gp)(sp) - REG_S tp, SBI_TRAP_REGS_OFFSET(tp)(sp) - REG_S t1, SBI_TRAP_REGS_OFFSET(t1)(sp) - REG_S t2, SBI_TRAP_REGS_OFFSET(t2)(sp) - REG_S s0, SBI_TRAP_REGS_OFFSET(s0)(sp) - REG_S s1, SBI_TRAP_REGS_OFFSET(s1)(sp) - REG_S a0, SBI_TRAP_REGS_OFFSET(a0)(sp) - REG_S a1, SBI_TRAP_REGS_OFFSET(a1)(sp) - REG_S a2, SBI_TRAP_REGS_OFFSET(a2)(sp) - REG_S a3, SBI_TRAP_REGS_OFFSET(a3)(sp) - REG_S a4, SBI_TRAP_REGS_OFFSET(a4)(sp) - REG_S a5, SBI_TRAP_REGS_OFFSET(a5)(sp) - REG_S a6, SBI_TRAP_REGS_OFFSET(a6)(sp) - REG_S a7, SBI_TRAP_REGS_OFFSET(a7)(sp) - REG_S s2, SBI_TRAP_REGS_OFFSET(s2)(sp) - REG_S s3, SBI_TRAP_REGS_OFFSET(s3)(sp) - REG_S s4, SBI_TRAP_REGS_OFFSET(s4)(sp) - REG_S s5, SBI_TRAP_REGS_OFFSET(s5)(sp) - REG_S s6, SBI_TRAP_REGS_OFFSET(s6)(sp) - REG_S s7, SBI_TRAP_REGS_OFFSET(s7)(sp) - REG_S s8, SBI_TRAP_REGS_OFFSET(s8)(sp) - REG_S s9, SBI_TRAP_REGS_OFFSET(s9)(sp) - REG_S s10, SBI_TRAP_REGS_OFFSET(s10)(sp) - REG_S s11, SBI_TRAP_REGS_OFFSET(s11)(sp) - REG_S t3, SBI_TRAP_REGS_OFFSET(t3)(sp) - REG_S t4, SBI_TRAP_REGS_OFFSET(t4)(sp) - REG_S t5, SBI_TRAP_REGS_OFFSET(t5)(sp) - REG_S t6, SBI_TRAP_REGS_OFFSET(t6)(sp) - - /* Restore GP */ - - la a0, mpfs_global_pointer - ld gp, 0(a0) - - /* Call C routine */ - - add a0, sp, zero - call sbi_trap_handler - - /* Restore all general regisers except A0 and T0 */ - - REG_L ra, SBI_TRAP_REGS_OFFSET(ra)(a0) - REG_L sp, SBI_TRAP_REGS_OFFSET(sp)(a0) - REG_L gp, SBI_TRAP_REGS_OFFSET(gp)(a0) - REG_L tp, SBI_TRAP_REGS_OFFSET(tp)(a0) - REG_L t1, SBI_TRAP_REGS_OFFSET(t1)(a0) - REG_L t2, SBI_TRAP_REGS_OFFSET(t2)(a0) - REG_L s0, SBI_TRAP_REGS_OFFSET(s0)(a0) - REG_L s1, SBI_TRAP_REGS_OFFSET(s1)(a0) - REG_L a1, SBI_TRAP_REGS_OFFSET(a1)(a0) - REG_L a2, SBI_TRAP_REGS_OFFSET(a2)(a0) - REG_L a3, SBI_TRAP_REGS_OFFSET(a3)(a0) - REG_L a4, SBI_TRAP_REGS_OFFSET(a4)(a0) - REG_L a5, SBI_TRAP_REGS_OFFSET(a5)(a0) - REG_L a6, SBI_TRAP_REGS_OFFSET(a6)(a0) - REG_L a7, SBI_TRAP_REGS_OFFSET(a7)(a0) - REG_L s2, SBI_TRAP_REGS_OFFSET(s2)(a0) - REG_L s3, SBI_TRAP_REGS_OFFSET(s3)(a0) - REG_L s4, SBI_TRAP_REGS_OFFSET(s4)(a0) - REG_L s5, SBI_TRAP_REGS_OFFSET(s5)(a0) - REG_L s6, SBI_TRAP_REGS_OFFSET(s6)(a0) - REG_L s7, SBI_TRAP_REGS_OFFSET(s7)(a0) - REG_L s8, SBI_TRAP_REGS_OFFSET(s8)(a0) - REG_L s9, SBI_TRAP_REGS_OFFSET(s9)(a0) - REG_L s10, SBI_TRAP_REGS_OFFSET(s10)(a0) - REG_L s11, SBI_TRAP_REGS_OFFSET(s11)(a0) - REG_L t3, SBI_TRAP_REGS_OFFSET(t3)(a0) - REG_L t4, SBI_TRAP_REGS_OFFSET(t4)(a0) - REG_L t5, SBI_TRAP_REGS_OFFSET(t5)(a0) - REG_L t6, SBI_TRAP_REGS_OFFSET(t6)(a0) - - /* Restore MEPC and MSTATUS CSRs */ - - REG_L t0, SBI_TRAP_REGS_OFFSET(mepc)(a0) - csrw CSR_MEPC, t0 - REG_L t0, SBI_TRAP_REGS_OFFSET(mstatus)(a0) - csrw CSR_MSTATUS, t0 - - /* Restore T0 */ - - REG_L t0, SBI_TRAP_REGS_OFFSET(t0)(a0) - - /* Restore A0 */ - - REG_L a0, SBI_TRAP_REGS_OFFSET(a0)(a0) - - mret + tail mpfs_opensbi_setup