Mirrors_Framework/nuttx
1
0
Fork 0
mirror of https://github.com/apache/nuttx.git synced 2026-05-22 05:42:05 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

risc-v/mpfs: enhance rpmsg throughput

Browse Source 
RPMSG is associated with the use of HPWORK / LPWORK queues.
After sending a message to the remote end (Linux), the system
waits for an ack before proceeding. Unfortunately this may
take sometimes more time than one would expect. Ack waiting is
also unnecessary: nothing is done with that information. Even
worse, the net_lock() is also held during the blocked time so
it blocks other network stacks that are unrelated to this.

Also reorganize the mpfs_opensbi_*.S so that the trap
handler is easily relocated in the linker .ld file without
the need to relocate the utils.S. This makes it easier to
separate the files into own segments. The trap file should be
located in the zero device.

Moreover, provide support for simultaneous ACK and message
present handling capabilities in both directions. There are
times when both bits are set but only other is being handled.

In the end, the maximum throughput of the RPMSG bus increases
easily 10-20% or even more.

Signed-off-by: Eero Nurkkala <eero.nurkkala@offcode.fi>
This commit is contained in:
Eero Nurkkala
2023-03-29 15:53:17 +03:00
committed by Xiang Xiao
parent 67ad63c7a4
commit 0152b031a4
6 changed files with 500 additions and 315 deletions
Download Patch File Download Diff File Expand all files Collapse all files
arch/risc-v/src/mpfs/Make.defs
+1
View File
@@ -84,6 +84,7 @@ endif
ifeq (${CONFIG_MPFS_OPENSBI},y)
CHIP_ASRCS += mpfs_opensbi_utils.S
CHIP_ASRCS += mpfs_opensbi_trap.S
CHIP_CSRCS += mpfs_opensbi.c
endif
arch/risc-v/src/mpfs/hardware/mpfs_ihc.h
+2 -2
View File
@@ -27,8 +27,8 @@
enum mpfs_irq_type_e
{
MP_IRQ = 0x0,
ACK_IRQ = 0x1,
MP_IRQ = 0x1,
ACK_IRQ = 0x2,
};
#define IHC_MAX_MESSAGE_SIZE 2
arch/risc-v/src/mpfs/mpfs_ihc.c
+177 -85
View File
@@ -39,6 +39,7 @@
#include <nuttx/kthread.h>
#include <nuttx/semaphore.h>
#include <nuttx/spi/spi.h>
#include <nuttx/wqueue.h>
#include <nuttx/rptun/openamp.h>
#include <nuttx/rptun/rptun.h>
@@ -138,6 +139,12 @@ struct mpfs_queue_table_s
void *data;
};
struct mpfs_ihc_work_arg_s
{
uint32_t mhartid;
uint32_t rhartid;
};
/****************************************************************************
* Private Function Prototypes
****************************************************************************/
@@ -155,6 +162,7 @@ static int mpfs_rptun_notify(struct rptun_dev_s *dev, uint32_t notifyid);
static int mpfs_rptun_register_callback(struct rptun_dev_s *dev,
rptun_callback_t callback,
void *arg);
static void mpfs_rptun_worker(void *arg);
/****************************************************************************
* Private Data
@@ -179,8 +187,11 @@ static struct mpfs_rptun_shmem_s g_shmem;
static struct rpmsg_device *g_mpfs_rpmsg_device;
static struct rpmsg_virtio_device *g_mpfs_virtio_device;
static sem_t g_mpfs_ack_sig = SEM_INITIALIZER(0);
#ifdef MPFS_RPTUN_USE_THREAD
static sem_t g_mpfs_rx_sig = SEM_INITIALIZER(0);
#else
static struct work_s g_rptun_work;
#endif
static struct list_node g_dev_list = LIST_INITIAL_VALUE(g_dev_list);
static uint32_t g_connected_hart_ints;
@@ -189,6 +200,11 @@ static uint16_t g_vq_idx;
static int g_plic_irq;
static bool g_rptun_initialized;
#ifdef IHC_AVOID_ACK_AND_MP
static struct mpfs_ihc_work_arg_s g_work_arg;
static struct work_s g_ihc_work;
#endif
const uint32_t ihcia_remote_harts[MPFS_NUM_HARTS] =
{
IHCIA_H0_REMOTE_HARTS,
@@ -237,6 +253,7 @@ static const struct rptun_ops_s g_mpfs_rptun_ops =
* mhartid base on the context, not necessarily the actual
* mhartid.
* is_ack - Boolean that is set true if an ack has been found
* is_msg - Boolean that is set true if a message is present
*
* Returned Value:
* Remote hart id
@@ -244,7 +261,8 @@ static const struct rptun_ops_s g_mpfs_rptun_ops =
****************************************************************************/
static uint32_t mpfs_ihc_parse_incoming_hartid(uint32_t mhartid,
bool *is_ack)
bool *is_ack,
bool *is_msg)
{
uint32_t hart_id = 0;
uint32_t return_hart_id = UNDEFINED_HART_ID;
@@ -262,6 +280,13 @@ static uint32_t mpfs_ihc_parse_incoming_hartid(uint32_t mhartid,
{
return_hart_id = hart_id;
*is_ack = true;
test_int = (1 << (hart_id * 2));
if ((g_connected_hart_ints & test_int) == test_int)
{
*is_msg = true;
}
break;
}
}
@@ -273,7 +298,7 @@ static uint32_t mpfs_ihc_parse_incoming_hartid(uint32_t mhartid,
if (((g_connected_hart_ints & test_int) == test_int))
{
return_hart_id = hart_id;
*is_ack = false;
*is_msg = true;
break;
}
}
@@ -415,39 +440,68 @@ static uint32_t mpfs_ihc_context_to_local_hart_id(ihc_channel_t channel)
* Name: mpfs_ihc_rx_handler
*
* Description:
* This handles the received information and either lets the vq to proceed
* via posting g_mpfs_ack_sig, or lets the mpfs_rptun_thread() run as it
* waits for the g_mpfs_rx_sig. virtqueue_notification() cannot be called
* from the interrupt context, thus the thread that will perform it.
* This handles the received information and lets the vq to proceed.
* virtqueue_notification() cannot be called from the interrupt context,
* thus the thread or work queue that will perform it.
*
* Input Parameters:
* message - Pointer to the incoming message
* is_ack - Boolean indicating whether an ack is received
*
* Returned Value:
* None
*
****************************************************************************/
static void mpfs_ihc_rx_handler(uint32_t *message, bool is_ack)
static void mpfs_ihc_rx_handler(uint32_t *message)
{
if (is_ack)
{
/* Received the ack */
g_vq_idx = message[0];
nxsem_post(&g_mpfs_ack_sig);
}
else
{
g_vq_idx = message[0];
DEBUGASSERT((g_vq_idx == VRING0_NOTIFYID) ||
(g_vq_idx == VRING1_NOTIFYID));
DEBUGASSERT((g_vq_idx == VRING0_NOTIFYID) ||
(g_vq_idx == VRING1_NOTIFYID));
nxsem_post(&g_mpfs_rx_sig);
}
#ifdef MPFS_RPTUN_USE_THREAD
nxsem_post(&g_mpfs_rx_sig);
#else
work_queue(HPWORK, &g_rptun_work, mpfs_rptun_worker, NULL, 0);
#endif
}
/****************************************************************************
* Name: mpfs_ihc_worker
*
* Description:
* This function is used to wait for the remote message present condition,
* after which the ACK is sent. ACK wasn't sent before, as the remote end
* has no way of knowing which one came first: the ACK or RMP.
*
* Input Parameters:
* arg - Pointer to the arguments struct
*
* Returned Value:
* None
*
****************************************************************************/
#ifdef IHC_AVOID_ACK_AND_MP
static void mpfs_ihc_worker(void *arg)
{
uint32_t ctrl_reg;
uint32_t retries = 5000;
do
{
ctrl_reg = getreg32(MPFS_IHC_CTRL(g_work_arg.mhartid,
g_work_arg.rhartid));
}
while ((ctrl_reg & RMP_MESSAGE_PRESENT) && --retries);
DEBUGASSERT(retries != 0);
modifyreg32(MPFS_IHC_CTRL(g_work_arg.mhartid,
g_work_arg.rhartid), 0, ACK_INT);
}
#endif
/****************************************************************************
* Name: mpfs_ihc_rx_message
*
@@ -459,7 +513,6 @@ static void mpfs_ihc_rx_handler(uint32_t *message, bool is_ack)
* channel - Enum that describes the channel used.
* mhartid - Context hart id, not necessarily the absolute mhartid but
* rather, the primary hartid of the set of harts.
* is_ack - Boolean indicating an ack message
* msg - For storing data, could be NULL
*
* Returned Value:
@@ -468,58 +521,54 @@ static void mpfs_ihc_rx_handler(uint32_t *message, bool is_ack)
****************************************************************************/
static void mpfs_ihc_rx_message(ihc_channel_t channel, uint32_t mhartid,
bool is_ack, uint32_t *msg)
uint32_t *msg)
{
uint32_t rhartid = mpfs_ihc_context_to_remote_hart_id(channel);
uint32_t ctrl_reg = getreg32(MPFS_IHC_CTRL(mhartid, rhartid));
if (is_ack)
{
if (mhartid == CONTEXTB_HARTID)
{
uintptr_t msg_in = MPFS_IHC_MSG_IN(mhartid, rhartid);
DEBUGASSERT(msg == NULL);
mpfs_ihc_rx_handler((uint32_t *)msg_in, is_ack);
}
else
{
/* This path is meant for the OpenSBI vendor extension only */
/* Check if we have a message */
DEBUGPANIC();
}
if (mhartid == CONTEXTB_HARTID)
{
uintptr_t msg_in = MPFS_IHC_MSG_IN(mhartid, rhartid);
DEBUGASSERT(msg == NULL);
mpfs_ihc_rx_handler((uint32_t *)msg_in);
}
else if (MP_MESSAGE_PRESENT == (ctrl_reg & MP_MASK))
else
{
/* Check if we have a message */
/* This path is meant for the OpenSBI vendor extension only */
if (mhartid == CONTEXTB_HARTID)
{
uintptr_t msg_in = MPFS_IHC_MSG_IN(mhartid, rhartid);
DEBUGASSERT(msg == NULL);
mpfs_ihc_rx_handler((uint32_t *)msg_in, is_ack);
}
else
{
/* This path is meant for the OpenSBI vendor extension only */
DEBUGPANIC();
}
DEBUGPANIC();
}
/* Set MP to 0. Note this generates an interrupt on the other hart
* if it has RMPIE bit set in the control register
*/
/* Set MP to 0. Note this generates an interrupt on the other hart
* if it has RMPIE bit set in the control register
ctrl_reg = getreg32(MPFS_IHC_CTRL(mhartid, rhartid));
if (ctrl_reg & RMP_MESSAGE_PRESENT)
{
/* If we send the ACK here, Linux will have the ACK and the
* MP flags sets. IHC_AVOID_ACK_AND_MP assures only one
* is present at once.
*/
volatile uint32_t temp = getreg32(MPFS_IHC_CTRL(mhartid, rhartid)) &
~MP_MASK;
#ifdef IHC_AVOID_ACK_AND_MP
g_work_arg.mhartid = mhartid;
g_work_arg.rhartid = rhartid;
modifyreg32(MPFS_IHC_CTRL(mhartid, rhartid), MP_MASK, 0);
work_queue(HPWORK, &g_ihc_work, mpfs_ihc_worker, NULL, 0);
#else
modifyreg32(MPFS_IHC_CTRL(mhartid, rhartid), MP_MASK, 0);
modifyreg32(MPFS_IHC_CTRL(mhartid, rhartid), 0, ACK_INT);
#endif
}
else
{
/* We can send the ACK now and clear the MP */
/* Check if ACKIE_EN is set */
if (temp & ACKIE_EN)
{
temp |= ACK_INT;
}
putreg32(temp, MPFS_IHC_CTRL(mhartid, rhartid));
modifyreg32(MPFS_IHC_CTRL(mhartid, rhartid), MP_MASK, 0);
modifyreg32(MPFS_IHC_CTRL(mhartid, rhartid), 0, ACK_INT);
}
}
@@ -541,11 +590,13 @@ static void mpfs_ihc_rx_message(ihc_channel_t channel, uint32_t mhartid,
static void mpfs_ihc_message_present_isr(void)
{
uint64_t mhartid = riscv_mhartid();
bool is_ack;
bool is_ack = false;
bool is_msg = false;
/* Check all our channels */
uint32_t origin_hart = mpfs_ihc_parse_incoming_hartid(mhartid, &is_ack);
uint32_t origin_hart = mpfs_ihc_parse_incoming_hartid(mhartid, &is_ack,
&is_msg);
if (origin_hart != UNDEFINED_HART_ID)
{
@@ -564,7 +615,10 @@ static void mpfs_ihc_message_present_isr(void)
/* Process incoming packet */
mpfs_ihc_rx_message(origin_hart, mhartid, is_ack, NULL);
if (is_msg)
{
mpfs_ihc_rx_message(origin_hart, mhartid, NULL);
}
if (is_ack)
{
@@ -698,7 +752,7 @@ static int mpfs_ihc_tx_message(ihc_channel_t channel, uint32_t *message)
{
ctrl_reg = getreg32(MPFS_IHC_CTRL(mhartid, rhartid));
}
while ((ctrl_reg & (RMP_MESSAGE_PRESENT | ACK_INT)) && --retries);
while ((ctrl_reg & (RMP_MESSAGE_PRESENT)) && --retries);
/* Return if RMP bit 1 indicating busy */
@@ -706,10 +760,6 @@ static int mpfs_ihc_tx_message(ihc_channel_t channel, uint32_t *message)
{
return -EBUSY;
}
else if (ACK_INT == (ctrl_reg & ACK_INT_MASK))
{
return -EBUSY;
}
else
{
/* Fill the buffer */
@@ -719,18 +769,18 @@ static int mpfs_ihc_tx_message(ihc_channel_t channel, uint32_t *message)
putreg32(message[i], MPFS_IHC_MSG_OUT(mhartid, rhartid) + i * 4);
}
ctrl_reg = getreg32(MPFS_IHC_CTRL(mhartid, rhartid));
/* If we're unlucky, we cannot send MP yet.. come back later */
if (ctrl_reg & MP_MESSAGE_PRESENT)
{
return -EBUSY;
}
/* Set the MP bit. This will notify other of incoming hart message */
modifyreg32(MPFS_IHC_CTRL(mhartid, rhartid), 0, RMP_MESSAGE_PRESENT);
/* Wait for the ACK to arrive to maintain the logic */
if (mhartid == CONTEXTB_HARTID)
{
/* Only applicable for the CONTEXTB_HART */
nxsem_wait_uninterruptible(&g_mpfs_ack_sig);
}
}
return OK;
@@ -846,12 +896,10 @@ mpfs_rptun_get_resource(struct rptun_dev_s *dev)
rsc->rpmsg_vdev.id = VIRTIO_ID_RPMSG;
rsc->rpmsg_vdev.notifyid = VDEV_NOTIFYID;
rsc->rpmsg_vdev.dfeatures = 1 << VIRTIO_RPMSG_F_NS |
1 << VIRTIO_RPMSG_F_ACK |
VIRTIO_RING_F_EVENT_IDX;
1 << VIRTIO_RPMSG_F_ACK;
rsc->rpmsg_vdev.gfeatures = 1 << VIRTIO_RPMSG_F_NS |
1 << VIRTIO_RPMSG_F_ACK |
VIRTIO_RING_F_EVENT_IDX;
1 << VIRTIO_RPMSG_F_ACK;
/* Set to VIRTIO_CONFIG_STATUS_DRIVER_OK when master is up */
@@ -980,6 +1028,8 @@ static int mpfs_rptun_stop(struct rptun_dev_s *dev)
static int mpfs_rptun_notify(struct rptun_dev_s *dev, uint32_t notifyid)
{
uint32_t tx_msg[IHC_MAX_MESSAGE_SIZE];
uint32_t retries = 5;
int ret = OK;
/* We only care about the queue with notifyid VRING0 */
@@ -988,10 +1038,18 @@ static int mpfs_rptun_notify(struct rptun_dev_s *dev, uint32_t notifyid)
tx_msg[0] = notifyid;
tx_msg[1] = 0;
return mpfs_ihc_tx_message(CONTEXTA_HARTID, tx_msg);
/* This failure should happen very rarely */
do
{
ret = mpfs_ihc_tx_message(CONTEXTA_HARTID, tx_msg);
}
while ((ret != OK) && --retries);
DEBUGASSERT(ret == OK);
}
return OK;
return ret;
}
/****************************************************************************
@@ -1153,6 +1211,33 @@ static void mpfs_rpmsg_device_created(struct rpmsg_device *rdev, void *priv_)
mpfs_echo_ping_init(rdev, &g_mpgs_echo_ping_ept);
}
/****************************************************************************
* Name: mpfs_rptun_worker
*
* Description:
* This is used to notify the associated virtqueue via the scheduled work.
* This doesn't use a separate thread, but a HPWORK instead, which is a
* way to avoid deadlocks with net_lock() that also originate from HPWORK.
*
* Input Parameters:
* arg - Argument
* Returned Value:
* None
*
****************************************************************************/
#ifndef MPFS_RPTUN_USE_THREAD
static void mpfs_rptun_worker(void *arg)
{
struct mpfs_queue_table_s *info;
DEBUGASSERT((g_vq_idx - VRING0_NOTIFYID) < VRINGS);
info = &g_mpfs_virtqueue_table[g_vq_idx - VRING0_NOTIFYID];
virtqueue_notification((struct virtqueue *)info->data);
}
#endif
/****************************************************************************
* Name: mpfs_rptun_thread
*
@@ -1170,6 +1255,7 @@ static void mpfs_rpmsg_device_created(struct rpmsg_device *rdev, void *priv_)
*
****************************************************************************/
#ifdef MPFS_RPTUN_USE_THREAD
static int mpfs_rptun_thread(int argc, char *argv[])
{
struct mpfs_queue_table_s *info;
@@ -1185,6 +1271,7 @@ static int mpfs_rptun_thread(int argc, char *argv[])
return 0;
}
#endif
/****************************************************************************
* Public Functions
@@ -1210,8 +1297,10 @@ static int mpfs_rptun_thread(int argc, char *argv[])
int mpfs_ihc_init(void)
{
uint32_t mhartid = (uint32_t)riscv_mhartid();
#ifdef MPFS_RPTUN_USE_THREAD
char *argv[3];
char arg1[19];
#endif
uint32_t rhartid;
int ret;
@@ -1275,6 +1364,8 @@ int mpfs_ihc_init(void)
goto init_error;
}
#ifdef MPFS_RPTUN_USE_THREAD
/* Thread initialization */
snprintf(arg1, sizeof(arg1), "%p",
@@ -1292,6 +1383,7 @@ int mpfs_ihc_init(void)
NULL, NULL, NULL);
goto init_error;
}
#endif
return OK;
arch/risc-v/src/mpfs/mpfs_ihc_sbi.c
+81 -36
View File
@@ -67,6 +67,35 @@ static uint32_t g_connected_harts_c;
* Private Functions
****************************************************************************/
/****************************************************************************
* Name: mpfs_modifyreg32
*
* Description:
* This is a copy of modifyreg32() without spinlock. That function is a
* real danger here as it is likely located in eNVM, thus being a real
* bottleneck. All functions called from this file should be located in
* the zero device.
*
* Input Parameters:
* addr - Address to perform the operation
* clearbits - Bits to clear
* setbits - Bits to set
*
* Returned Value:
* Remote hart id
*
****************************************************************************/
void mpfs_modifyreg32(uintptr_t addr, uint32_t clearbits, uint32_t setbits)
{
uint32_t regval;
regval = getreg32(addr);
regval &= ~clearbits;
regval |= setbits;
putreg32(regval, addr);
}
/****************************************************************************
* Name: mpfs_ihc_sbi_parse_incoming_hartid
*
@@ -79,6 +108,7 @@ static uint32_t g_connected_harts_c;
* mhartid base on the context, not necessarily the actual
* mhartid.
* is_ack - Boolean that is set true if an ack has been found
* is_mp - Boolean that is set true if a msg is present also
*
* Returned Value:
* Remote hart id
@@ -86,7 +116,8 @@ static uint32_t g_connected_harts_c;
****************************************************************************/
static uint32_t mpfs_ihc_sbi_parse_incoming_hartid(uint32_t mhartid,
bool *is_ack)
bool *is_ack,
bool *is_mp)
{
uint32_t hart_id = 0;
uint32_t return_hart_id = UNDEFINED_HART_ID;
@@ -112,6 +143,16 @@ static uint32_t mpfs_ihc_sbi_parse_incoming_hartid(uint32_t mhartid,
{
return_hart_id = hart_id;
*is_ack = true;
/* We might also have a msg present */
test_int = (1 << (hart_id * 2));
if (msg_avail & test_int)
{
*is_mp = true;
}
break;
}
}
@@ -124,6 +165,7 @@ static uint32_t mpfs_ihc_sbi_parse_incoming_hartid(uint32_t mhartid,
{
return_hart_id = hart_id;
*is_ack = false;
*is_mp = true;
break;
}
}
@@ -231,7 +273,6 @@ static uint32_t mpfs_ihc_sbi_context_to_local_hart_id(ihc_channel_t channel)
}
DEBUGASSERT(hart < MPFS_NUM_HARTS);
return hart;
}
@@ -259,7 +300,8 @@ static uint32_t mpfs_ihc_sbi_context_to_local_hart_id(ihc_channel_t channel)
static void mpfs_ihc_sbi_message_present_handler(uint32_t *message,
uint32_t mhartid,
uint32_t rhartid,
bool is_ack)
bool is_ack,
bool is_mp)
{
struct ihc_sbi_rx_msg_s *msg;
uintptr_t message_ihc = (uintptr_t)MPFS_IHC_MSG_IN(mhartid, rhartid);
@@ -267,17 +309,22 @@ static void mpfs_ihc_sbi_message_present_handler(uint32_t *message,
msg = (struct ihc_sbi_rx_msg_s *)message;
if (is_ack)
if (is_ack && !is_mp)
{
msg->irq_type = ACK_IRQ;
/* msg->ihc_msg content doesn't matter here */
}
else
else if (is_mp && !is_ack)
{
msg->irq_type = MP_IRQ;
msg->ihc_msg = *(struct mpfs_ihc_msg_s *)message_ihc;
}
else
{
msg->irq_type = ACK_IRQ | MP_IRQ;
msg->ihc_msg = *(struct mpfs_ihc_msg_s *)message_ihc;
}
DEBUGASSERT(sizeof(msg->ihc_msg) >= message_size_ihc);
}
@@ -302,11 +349,9 @@ static void mpfs_ihc_sbi_message_present_handler(uint32_t *message,
****************************************************************************/
static void mpfs_ihc_sbi_rx_message(uint32_t rhartid, uint32_t mhartid,
bool is_ack, uint32_t *msg)
bool is_ack, bool is_mp, uint32_t *msg)
{
uint32_t ctrl_reg = getreg32(MPFS_IHC_CTRL(mhartid, rhartid));
if (is_ack)
if (is_ack && !is_mp)
{
if (mhartid == CONTEXTB_HARTID)
{
@@ -318,10 +363,14 @@ static void mpfs_ihc_sbi_rx_message(uint32_t rhartid, uint32_t mhartid,
DEBUGASSERT(msg != NULL);
mpfs_ihc_sbi_message_present_handler(msg, mhartid, rhartid,
is_ack);
is_ack, is_mp);
/* Clear the ack */
mpfs_modifyreg32(MPFS_IHC_CTRL(mhartid, rhartid), ACK_CLR, 0);
}
}
else if (MP_MESSAGE_PRESENT == (ctrl_reg & MP_MASK))
else if (is_mp && !is_ack)
{
/* Check if we have a message */
@@ -335,24 +384,25 @@ static void mpfs_ihc_sbi_rx_message(uint32_t rhartid, uint32_t mhartid,
DEBUGASSERT(msg != NULL);
mpfs_ihc_sbi_message_present_handler(msg, mhartid, rhartid,
is_ack);
is_ack, is_mp);
}
/* Set MP to 0. Note this generates an interrupt on the other hart
* if it has RMPIE bit set in the control register
*/
volatile uint32_t temp = getreg32(MPFS_IHC_CTRL(mhartid, rhartid)) &
~MP_MASK;
mpfs_modifyreg32(MPFS_IHC_CTRL(mhartid, rhartid), MP_MASK, ACK_INT);
}
else if (is_ack && is_mp)
{
DEBUGASSERT(msg != NULL);
mpfs_ihc_sbi_message_present_handler(msg, mhartid, rhartid,
is_ack, is_mp);
/* Check if ACKIE_EN is set */
/* Clear the ack and mp */
if (temp & ACKIE_EN)
{
temp |= ACK_INT;
}
putreg32(temp, MPFS_IHC_CTRL(mhartid, rhartid));
mpfs_modifyreg32(MPFS_IHC_CTRL(mhartid, rhartid), ACK_CLR | MP_MASK,
ACK_INT);
}
}
@@ -377,22 +427,16 @@ void mpfs_ihc_sbi_message_present_indirect_isr(ihc_channel_t channel,
uint32_t *msg)
{
bool is_ack;
bool is_mp = false;
uint32_t mhartid = mpfs_ihc_sbi_context_to_local_hart_id(channel);
uint32_t origin_hart = mpfs_ihc_sbi_parse_incoming_hartid(mhartid,
&is_ack);
&is_ack,
&is_mp);
if (origin_hart != UNDEFINED_HART_ID)
{
/* Process incoming packet */
mpfs_ihc_sbi_rx_message(origin_hart, mhartid, is_ack, msg);
if (is_ack)
{
/* Clear the ack */
modifyreg32(MPFS_IHC_CTRL(mhartid, origin_hart), ACK_CLR, 0);
}
mpfs_ihc_sbi_rx_message(origin_hart, mhartid, is_ack, is_mp, msg);
}
}
@@ -470,16 +514,16 @@ static void mpfs_ihc_sbi_local_remote_config(uint32_t hart_to_configure,
}
#endif
modifyreg32(MPFS_IHC_CTRL(hart_to_configure, rhartid), 0, MPIE_EN |
ACKIE_EN);
mpfs_modifyreg32(MPFS_IHC_CTRL(hart_to_configure, rhartid), 0, MPIE_EN |
ACKIE_EN);
/* OpenSBI extension may configure 2x consecutive harts */
#ifdef CONFIG_MPFS_IHC_TWO_RPMSG_CHANNELS
if ((hart_to_configure + 1) < MPFS_NUM_HARTS)
{
modifyreg32(MPFS_IHC_CTRL(hart_to_configure + 1, rhartid + 1), 0,
MPIE_EN | ACKIE_EN);
mpfs_modifyreg32(MPFS_IHC_CTRL(hart_to_configure + 1, rhartid + 1), 0,
MPIE_EN | ACKIE_EN);
}
#endif
}
@@ -539,7 +583,8 @@ static int mpfs_ihc_sbi_tx_message(ihc_channel_t channel, uint32_t *message)
/* Set the MP bit. This will notify other of incoming hart message */
modifyreg32(MPFS_IHC_CTRL(mhartid, rhartid), 0, RMP_MESSAGE_PRESENT);
mpfs_modifyreg32(MPFS_IHC_CTRL(mhartid, rhartid), 0,
RMP_MESSAGE_PRESENT);
}
return OK;
arch/risc-v/src/mpfs/mpfs_opensbi_trap.S
+225
View File
@@ -0,0 +1,225 @@
/****************************************************************************
* arch/risc-v/src/mpfs/mpfs_opensbi_trap.S
*
* mpfs_exception_opensbi function is based on OpenSBI fw_base.S
*
* The 2-Clause BSD License
* SPDX short identifier: BSD-2-Clause
*
* Copyright (c) 2019 Western Digital Corporation or its affiliates and other
* contributors.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/****************************************************************************
* Included Files
****************************************************************************/
#include <nuttx/config.h>
#include <sbi/riscv_asm.h>
#include <sbi/sbi_platform.h>
#include <sbi/sbi_scratch.h>
#include <sbi/sbi_trap.h>
#include <sbi/riscv_encoding.h>
/****************************************************************************
* Pre-processor Definitions
****************************************************************************/
/****************************************************************************
* Public Symbols
****************************************************************************/
.global mpfs_exception_opensbi
/****************************************************************************
* Private Data
****************************************************************************/
mpfs_global_pointer:
.dword __global_pointer$
/****************************************************************************
* Name: mpfs_exception_opensbi:
*
* Description:
* This is the trap entry into OpenSBI.
*
* Input Parameters:
* None
*
* Returned Value:
* None
*
****************************************************************************/
.align 3
mpfs_exception_opensbi:
/* Swap TP and MSCRATCH */
csrrw tp, CSR_MSCRATCH, tp
/* Save T0 in scratch space */
REG_S t0, SBI_SCRATCH_TMP0_OFFSET(tp)
/*
* Set T0 to appropriate exception stack
*
* Came_From_M_Mode = ((MSTATUS.MPP < PRV_M) ? 1 : 0) - 1;
* Exception_Stack = TP ^ (Came_From_M_Mode & (SP ^ TP))
*
* Came_From_M_Mode = 0 ==> Exception_Stack = TP
* Came_From_M_Mode = -1 ==> Exception_Stack = SP
*/
csrr t0, CSR_MSTATUS
srl t0, t0, MSTATUS_MPP_SHIFT
and t0, t0, PRV_M
slti t0, t0, PRV_M
add t0, t0, -1
xor sp, sp, tp
and t0, t0, sp
xor sp, sp, tp
xor t0, tp, t0
/* Save original SP on exception stack */
REG_S sp, (SBI_TRAP_REGS_OFFSET(sp) - SBI_TRAP_REGS_SIZE)(t0)
/* Set SP to exception stack and make room for trap registers */
add sp, t0, -(SBI_TRAP_REGS_SIZE)
/* Restore T0 from scratch space */
REG_L t0, SBI_SCRATCH_TMP0_OFFSET(tp)
/* Save T0 on stack */
REG_S t0, SBI_TRAP_REGS_OFFSET(t0)(sp)
/* Swap TP and MSCRATCH */
csrrw tp, CSR_MSCRATCH, tp
/* Save MEPC and MSTATUS CSRs */
csrr t0, CSR_MEPC
REG_S t0, SBI_TRAP_REGS_OFFSET(mepc)(sp)
csrr t0, CSR_MSTATUS
REG_S t0, SBI_TRAP_REGS_OFFSET(mstatus)(sp)
REG_S zero, SBI_TRAP_REGS_OFFSET(mstatusH)(sp)
/* Save all general regisers except SP and T0 */
REG_S zero, SBI_TRAP_REGS_OFFSET(zero)(sp)
REG_S ra, SBI_TRAP_REGS_OFFSET(ra)(sp)
REG_S gp, SBI_TRAP_REGS_OFFSET(gp)(sp)
REG_S tp, SBI_TRAP_REGS_OFFSET(tp)(sp)
REG_S t1, SBI_TRAP_REGS_OFFSET(t1)(sp)
REG_S t2, SBI_TRAP_REGS_OFFSET(t2)(sp)
REG_S s0, SBI_TRAP_REGS_OFFSET(s0)(sp)
REG_S s1, SBI_TRAP_REGS_OFFSET(s1)(sp)
REG_S a0, SBI_TRAP_REGS_OFFSET(a0)(sp)
REG_S a1, SBI_TRAP_REGS_OFFSET(a1)(sp)
REG_S a2, SBI_TRAP_REGS_OFFSET(a2)(sp)
REG_S a3, SBI_TRAP_REGS_OFFSET(a3)(sp)
REG_S a4, SBI_TRAP_REGS_OFFSET(a4)(sp)
REG_S a5, SBI_TRAP_REGS_OFFSET(a5)(sp)
REG_S a6, SBI_TRAP_REGS_OFFSET(a6)(sp)
REG_S a7, SBI_TRAP_REGS_OFFSET(a7)(sp)
REG_S s2, SBI_TRAP_REGS_OFFSET(s2)(sp)
REG_S s3, SBI_TRAP_REGS_OFFSET(s3)(sp)
REG_S s4, SBI_TRAP_REGS_OFFSET(s4)(sp)
REG_S s5, SBI_TRAP_REGS_OFFSET(s5)(sp)
REG_S s6, SBI_TRAP_REGS_OFFSET(s6)(sp)
REG_S s7, SBI_TRAP_REGS_OFFSET(s7)(sp)
REG_S s8, SBI_TRAP_REGS_OFFSET(s8)(sp)
REG_S s9, SBI_TRAP_REGS_OFFSET(s9)(sp)
REG_S s10, SBI_TRAP_REGS_OFFSET(s10)(sp)
REG_S s11, SBI_TRAP_REGS_OFFSET(s11)(sp)
REG_S t3, SBI_TRAP_REGS_OFFSET(t3)(sp)
REG_S t4, SBI_TRAP_REGS_OFFSET(t4)(sp)
REG_S t5, SBI_TRAP_REGS_OFFSET(t5)(sp)
REG_S t6, SBI_TRAP_REGS_OFFSET(t6)(sp)
/* Restore GP */
la a0, mpfs_global_pointer
ld gp, 0(a0)
/* Call C routine */
add a0, sp, zero
call sbi_trap_handler
/* Restore all general regisers except A0 and T0 */
REG_L ra, SBI_TRAP_REGS_OFFSET(ra)(a0)
REG_L sp, SBI_TRAP_REGS_OFFSET(sp)(a0)
REG_L gp, SBI_TRAP_REGS_OFFSET(gp)(a0)
REG_L tp, SBI_TRAP_REGS_OFFSET(tp)(a0)
REG_L t1, SBI_TRAP_REGS_OFFSET(t1)(a0)
REG_L t2, SBI_TRAP_REGS_OFFSET(t2)(a0)
REG_L s0, SBI_TRAP_REGS_OFFSET(s0)(a0)
REG_L s1, SBI_TRAP_REGS_OFFSET(s1)(a0)
REG_L a1, SBI_TRAP_REGS_OFFSET(a1)(a0)
REG_L a2, SBI_TRAP_REGS_OFFSET(a2)(a0)
REG_L a3, SBI_TRAP_REGS_OFFSET(a3)(a0)
REG_L a4, SBI_TRAP_REGS_OFFSET(a4)(a0)
REG_L a5, SBI_TRAP_REGS_OFFSET(a5)(a0)
REG_L a6, SBI_TRAP_REGS_OFFSET(a6)(a0)
REG_L a7, SBI_TRAP_REGS_OFFSET(a7)(a0)
REG_L s2, SBI_TRAP_REGS_OFFSET(s2)(a0)
REG_L s3, SBI_TRAP_REGS_OFFSET(s3)(a0)
REG_L s4, SBI_TRAP_REGS_OFFSET(s4)(a0)
REG_L s5, SBI_TRAP_REGS_OFFSET(s5)(a0)
REG_L s6, SBI_TRAP_REGS_OFFSET(s6)(a0)
REG_L s7, SBI_TRAP_REGS_OFFSET(s7)(a0)
REG_L s8, SBI_TRAP_REGS_OFFSET(s8)(a0)
REG_L s9, SBI_TRAP_REGS_OFFSET(s9)(a0)
REG_L s10, SBI_TRAP_REGS_OFFSET(s10)(a0)
REG_L s11, SBI_TRAP_REGS_OFFSET(s11)(a0)
REG_L t3, SBI_TRAP_REGS_OFFSET(t3)(a0)
REG_L t4, SBI_TRAP_REGS_OFFSET(t4)(a0)
REG_L t5, SBI_TRAP_REGS_OFFSET(t5)(a0)
REG_L t6, SBI_TRAP_REGS_OFFSET(t6)(a0)
/* Restore MEPC and MSTATUS CSRs */
REG_L t0, SBI_TRAP_REGS_OFFSET(mepc)(a0)
csrw CSR_MEPC, t0
REG_L t0, SBI_TRAP_REGS_OFFSET(mstatus)(a0)
csrw CSR_MSTATUS, t0
/* Restore T0 */
REG_L t0, SBI_TRAP_REGS_OFFSET(t0)(a0)
/* Restore A0 */
REG_L a0, SBI_TRAP_REGS_OFFSET(a0)(a0)
mret
arch/risc-v/src/mpfs/mpfs_opensbi_utils.S
+14 -192
View File
@@ -1,34 +1,22 @@
/****************************************************************************
* arch/risc-v/src/mpfs/mpfs_opensbi_utils.S
*
* mpfs_exception_opensbi function is based on OpenSBI fw_base.S
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership. The
* ASF licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the
* License. You may obtain a copy of the License at
*
* The 2-Clause BSD License
* SPDX short identifier: BSD-2-Clause
* http://www.apache.org/licenses/LICENSE-2.0
*
* Copyright (c) 2019 Western Digital Corporation or its affiliates and other
* contributors.
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
****************************************************************************/
/****************************************************************************
* Included Files
@@ -51,7 +39,6 @@
****************************************************************************/
.global mpfs_opensbi_prepare_hart
.global mpfs_exception_opensbi
/****************************************************************************
* Private Data
@@ -103,169 +90,4 @@ mpfs_opensbi_prepare_hart:
mul t0, a0, t1
la sp, g_scratches
add sp, sp, t0
jal mpfs_opensbi_setup
/****************************************************************************
* Name: mpfs_exception_opensbi:
*
* Description:
* This is the trap entry into OpenSBI.
*
* Input Parameters:
* None
*
* Returned Value:
* None
*
****************************************************************************/
.align 3
mpfs_exception_opensbi:
/* Swap TP and MSCRATCH */
csrrw tp, CSR_MSCRATCH, tp
/* Save T0 in scratch space */
REG_S t0, SBI_SCRATCH_TMP0_OFFSET(tp)
/*
* Set T0 to appropriate exception stack
*
* Came_From_M_Mode = ((MSTATUS.MPP < PRV_M) ? 1 : 0) - 1;
* Exception_Stack = TP ^ (Came_From_M_Mode & (SP ^ TP))
*
* Came_From_M_Mode = 0 ==> Exception_Stack = TP
* Came_From_M_Mode = -1 ==> Exception_Stack = SP
*/
csrr t0, CSR_MSTATUS
srl t0, t0, MSTATUS_MPP_SHIFT
and t0, t0, PRV_M
slti t0, t0, PRV_M
add t0, t0, -1
xor sp, sp, tp
and t0, t0, sp
xor sp, sp, tp
xor t0, tp, t0
/* Save original SP on exception stack */
REG_S sp, (SBI_TRAP_REGS_OFFSET(sp) - SBI_TRAP_REGS_SIZE)(t0)
/* Set SP to exception stack and make room for trap registers */
add sp, t0, -(SBI_TRAP_REGS_SIZE)
/* Restore T0 from scratch space */
REG_L t0, SBI_SCRATCH_TMP0_OFFSET(tp)
/* Save T0 on stack */
REG_S t0, SBI_TRAP_REGS_OFFSET(t0)(sp)
/* Swap TP and MSCRATCH */
csrrw tp, CSR_MSCRATCH, tp
/* Save MEPC and MSTATUS CSRs */
csrr t0, CSR_MEPC
REG_S t0, SBI_TRAP_REGS_OFFSET(mepc)(sp)
csrr t0, CSR_MSTATUS
REG_S t0, SBI_TRAP_REGS_OFFSET(mstatus)(sp)
REG_S zero, SBI_TRAP_REGS_OFFSET(mstatusH)(sp)
/* Save all general regisers except SP and T0 */
REG_S zero, SBI_TRAP_REGS_OFFSET(zero)(sp)
REG_S ra, SBI_TRAP_REGS_OFFSET(ra)(sp)
REG_S gp, SBI_TRAP_REGS_OFFSET(gp)(sp)
REG_S tp, SBI_TRAP_REGS_OFFSET(tp)(sp)
REG_S t1, SBI_TRAP_REGS_OFFSET(t1)(sp)
REG_S t2, SBI_TRAP_REGS_OFFSET(t2)(sp)
REG_S s0, SBI_TRAP_REGS_OFFSET(s0)(sp)
REG_S s1, SBI_TRAP_REGS_OFFSET(s1)(sp)
REG_S a0, SBI_TRAP_REGS_OFFSET(a0)(sp)
REG_S a1, SBI_TRAP_REGS_OFFSET(a1)(sp)
REG_S a2, SBI_TRAP_REGS_OFFSET(a2)(sp)
REG_S a3, SBI_TRAP_REGS_OFFSET(a3)(sp)
REG_S a4, SBI_TRAP_REGS_OFFSET(a4)(sp)
REG_S a5, SBI_TRAP_REGS_OFFSET(a5)(sp)
REG_S a6, SBI_TRAP_REGS_OFFSET(a6)(sp)
REG_S a7, SBI_TRAP_REGS_OFFSET(a7)(sp)
REG_S s2, SBI_TRAP_REGS_OFFSET(s2)(sp)
REG_S s3, SBI_TRAP_REGS_OFFSET(s3)(sp)
REG_S s4, SBI_TRAP_REGS_OFFSET(s4)(sp)
REG_S s5, SBI_TRAP_REGS_OFFSET(s5)(sp)
REG_S s6, SBI_TRAP_REGS_OFFSET(s6)(sp)
REG_S s7, SBI_TRAP_REGS_OFFSET(s7)(sp)
REG_S s8, SBI_TRAP_REGS_OFFSET(s8)(sp)
REG_S s9, SBI_TRAP_REGS_OFFSET(s9)(sp)
REG_S s10, SBI_TRAP_REGS_OFFSET(s10)(sp)
REG_S s11, SBI_TRAP_REGS_OFFSET(s11)(sp)
REG_S t3, SBI_TRAP_REGS_OFFSET(t3)(sp)
REG_S t4, SBI_TRAP_REGS_OFFSET(t4)(sp)
REG_S t5, SBI_TRAP_REGS_OFFSET(t5)(sp)
REG_S t6, SBI_TRAP_REGS_OFFSET(t6)(sp)
/* Restore GP */
la a0, mpfs_global_pointer
ld gp, 0(a0)
/* Call C routine */
add a0, sp, zero
call sbi_trap_handler
/* Restore all general regisers except A0 and T0 */
REG_L ra, SBI_TRAP_REGS_OFFSET(ra)(a0)
REG_L sp, SBI_TRAP_REGS_OFFSET(sp)(a0)
REG_L gp, SBI_TRAP_REGS_OFFSET(gp)(a0)
REG_L tp, SBI_TRAP_REGS_OFFSET(tp)(a0)
REG_L t1, SBI_TRAP_REGS_OFFSET(t1)(a0)
REG_L t2, SBI_TRAP_REGS_OFFSET(t2)(a0)
REG_L s0, SBI_TRAP_REGS_OFFSET(s0)(a0)
REG_L s1, SBI_TRAP_REGS_OFFSET(s1)(a0)
REG_L a1, SBI_TRAP_REGS_OFFSET(a1)(a0)
REG_L a2, SBI_TRAP_REGS_OFFSET(a2)(a0)
REG_L a3, SBI_TRAP_REGS_OFFSET(a3)(a0)
REG_L a4, SBI_TRAP_REGS_OFFSET(a4)(a0)
REG_L a5, SBI_TRAP_REGS_OFFSET(a5)(a0)
REG_L a6, SBI_TRAP_REGS_OFFSET(a6)(a0)
REG_L a7, SBI_TRAP_REGS_OFFSET(a7)(a0)
REG_L s2, SBI_TRAP_REGS_OFFSET(s2)(a0)
REG_L s3, SBI_TRAP_REGS_OFFSET(s3)(a0)
REG_L s4, SBI_TRAP_REGS_OFFSET(s4)(a0)
REG_L s5, SBI_TRAP_REGS_OFFSET(s5)(a0)
REG_L s6, SBI_TRAP_REGS_OFFSET(s6)(a0)
REG_L s7, SBI_TRAP_REGS_OFFSET(s7)(a0)
REG_L s8, SBI_TRAP_REGS_OFFSET(s8)(a0)
REG_L s9, SBI_TRAP_REGS_OFFSET(s9)(a0)
REG_L s10, SBI_TRAP_REGS_OFFSET(s10)(a0)
REG_L s11, SBI_TRAP_REGS_OFFSET(s11)(a0)
REG_L t3, SBI_TRAP_REGS_OFFSET(t3)(a0)
REG_L t4, SBI_TRAP_REGS_OFFSET(t4)(a0)
REG_L t5, SBI_TRAP_REGS_OFFSET(t5)(a0)
REG_L t6, SBI_TRAP_REGS_OFFSET(t6)(a0)
/* Restore MEPC and MSTATUS CSRs */
REG_L t0, SBI_TRAP_REGS_OFFSET(mepc)(a0)
csrw CSR_MEPC, t0
REG_L t0, SBI_TRAP_REGS_OFFSET(mstatus)(a0)
csrw CSR_MSTATUS, t0
/* Restore T0 */
REG_L t0, SBI_TRAP_REGS_OFFSET(t0)(a0)
/* Restore A0 */
REG_L a0, SBI_TRAP_REGS_OFFSET(a0)(a0)
mret
tail mpfs_opensbi_setup