diff --git a/sched/irq/irq.h b/sched/irq/irq.h index 40473a60108..a7b4b431ab3 100644 --- a/sched/irq/irq.h +++ b/sched/irq/irq.h @@ -1,7 +1,7 @@ /**************************************************************************** * sched/irq/irq.h * - * Copyright (C) 2007, 2008, 2013-2014 Gregory Nutt. All rights reserved. + * Copyright (C) 2007, 2008, 2013-2014, 2017 Gregory Nutt. All rights reserved. * Author: Gregory Nutt * * Redistribution and use in source and binary forms, with or without @@ -72,6 +72,10 @@ extern volatile spinlock_t g_cpu_irqlock SP_SECTION; extern volatile spinlock_t g_cpu_irqsetlock SP_SECTION; extern volatile cpu_set_t g_cpu_irqset SP_SECTION; + +/* Handles nested calls to enter_critical section from interrupt handlers */ + +extern volatile uint8_t g_cpu_nestcount[CONFIG_SMP_NCPUS]; #endif /**************************************************************************** diff --git a/sched/irq/irq_csection.c b/sched/irq/irq_csection.c index bf226bf6217..a67dbc16bf6 100644 --- a/sched/irq/irq_csection.c +++ b/sched/irq/irq_csection.c @@ -1,7 +1,7 @@ /**************************************************************************** * sched/irq/irq_csection.c * - * Copyright (C) 2016 Gregory Nutt. All rights reserved. + * Copyright (C) 2016-2017 Gregory Nutt. All rights reserved. * Author: Gregory Nutt * * Redistribution and use in source and binary forms, with or without @@ -66,16 +66,10 @@ volatile spinlock_t g_cpu_irqlock SP_SECTION = SP_UNLOCKED; volatile spinlock_t g_cpu_irqsetlock SP_SECTION; volatile cpu_set_t g_cpu_irqset SP_SECTION; -#endif -/**************************************************************************** - * Private Data - ****************************************************************************/ - -#ifdef CONFIG_SMP /* Handles nested calls to enter_critical section from interrupt handlers */ -static uint8_t g_cpu_nestcount[CONFIG_SMP_NCPUS]; +volatile uint8_t g_cpu_nestcount[CONFIG_SMP_NCPUS]; #endif /**************************************************************************** @@ -303,6 +297,13 @@ try_again: /* In any event, the nesting count is now one */ g_cpu_nestcount[cpu] = 1; + + /* Also set the CPU bit so that other CPUs will be aware that this + * CPU holds the critical section. + */ + + spin_setbit(&g_cpu_irqset, cpu, &g_cpu_irqsetlock, + &g_cpu_irqlock); } } else @@ -455,18 +456,22 @@ void leave_critical_section(irqstate_t flags) } else { - /* No, not nested. Release the spinlock. */ + /* No, not nested. Restore the g_cpu_irqset for this CPU + * and release the spinlock (if necessary). + */ DEBUGASSERT(spin_islocked(&g_cpu_irqlock) && g_cpu_nestcount[cpu] == 1); - spin_lock(&g_cpu_irqsetlock); /* Protects g_cpu_irqset */ - if (g_cpu_irqset == 0) + FAR struct tcb_s *rtcb = this_task(); + DEBUGASSERT(rtcb != NULL); + + if (rtcb->irqcount <= 0) { - spin_unlock(&g_cpu_irqlock); + spin_clrbit(&g_cpu_irqset, cpu, &g_cpu_irqsetlock, + &g_cpu_irqlock); } - spin_unlock(&g_cpu_irqsetlock); g_cpu_nestcount[cpu] = 0; } } diff --git a/sched/sched/sched_addreadytorun.c b/sched/sched/sched_addreadytorun.c index c5f86e2a807..a23b8990d0f 100644 --- a/sched/sched/sched_addreadytorun.c +++ b/sched/sched/sched_addreadytorun.c @@ -1,7 +1,7 @@ /**************************************************************************** * sched/sched/sched_addreadytorun.c * - * Copyright (C) 2007-2009, 2014, 2016 Gregory Nutt. All rights reserved. + * Copyright (C) 2007-2009, 2014, 2016-2017 Gregory Nutt. All rights reserved. * Author: Gregory Nutt * * Redistribution and use in source and binary forms, with or without @@ -316,19 +316,40 @@ bool sched_addreadytorun(FAR struct tcb_s *btcb) &g_cpu_schedlock); } - /* Adjust global IRQ controls. If irqcount is greater than zero, - * then this task/this CPU holds the IRQ lock + /* Adjust global IRQ controls. This works differently if we are + * performing a context switch from an interrupt handler and the + * interrupt handler has established a critical section. We can + * detect this case when g_cpu_nestcount[me] > 0: */ - if (btcb->irqcount > 0) + if (g_cpu_nestcount[me] <= 0) { - spin_setbit(&g_cpu_irqset, cpu, &g_cpu_irqsetlock, - &g_cpu_irqlock); + /* If irqcount is greater than zero, then this task/this CPU + * holds the IRQ lock + */ + + if (btcb->irqcount > 0) + { + spin_setbit(&g_cpu_irqset, cpu, &g_cpu_irqsetlock, + &g_cpu_irqlock); + } + else + { + spin_clrbit(&g_cpu_irqset, cpu, &g_cpu_irqsetlock, + &g_cpu_irqlock); + } } else { - spin_clrbit(&g_cpu_irqset, cpu, &g_cpu_irqsetlock, - &g_cpu_irqlock); + /* Sanity check. g_cpu_netcount should be greater than zero + * only while we are within the critical section and within + * an interrupt handler. If we are not in an interrupt handler + * then there is a problem; perhaps some logic previously + * called enter_critical_section() with no matching call to + * leave_critical_section(), leaving the non-zero count. + */ + + DEBUGASSERT(up_interrupt_context()); } /* If the following task is not locked to this CPU, then it must diff --git a/sched/sched/sched_removereadytorun.c b/sched/sched/sched_removereadytorun.c index be80d3ad0fe..d22eb0e6bfa 100644 --- a/sched/sched/sched_removereadytorun.c +++ b/sched/sched/sched_removereadytorun.c @@ -1,7 +1,7 @@ /**************************************************************************** * sched/sched_removereadytorun.c * - * Copyright (C) 2007-2009, 2012, 2016 Gregory Nutt. All rights reserved. + * Copyright (C) 2007-2009, 2012, 2016-2017 Gregory Nutt. All rights reserved. * Author: Gregory Nutt * * Redistribution and use in source and binary forms, with or without @@ -260,23 +260,44 @@ bool sched_removereadytorun(FAR struct tcb_s *rtcb) &g_cpu_schedlock); } - /* Interrupts may be disabled after the switch. If irqcount is greater - * than zero, then this task/this CPU holds the IRQ lock + /* Adjust global IRQ controls. This works differently if we are + * performing a context switch from an interrupt handler and the + * interrupt handler has established a critical section. We can + * detect this case when g_cpu_nestcount[me] > 0: */ - if (nxttcb->irqcount > 0) + if (g_cpu_nestcount[me] <= 0) { - /* Yes... make sure that scheduling logic knows about this */ + /* If irqcount is greater than zero, then this task/this CPU + * holds the IRQ lock + */ - spin_setbit(&g_cpu_irqset, cpu, &g_cpu_irqsetlock, - &g_cpu_irqlock); + if (nxttcb->irqcount > 0) + { + /* Yes... make sure that scheduling logic knows about this */ + + spin_setbit(&g_cpu_irqset, cpu, &g_cpu_irqsetlock, + &g_cpu_irqlock); + } + else + { + /* No.. we may need to release our hold on the IRQ state. */ + + spin_clrbit(&g_cpu_irqset, cpu, &g_cpu_irqsetlock, + &g_cpu_irqlock); + } } else { - /* No.. we may need to release our hold on the irq state. */ + /* Sanity check. g_cpu_netcount should be greater than zero + * only while we are within the critical section and within + * an interrupt handler. If we are not in an interrupt handler + * then there is a problem; perhaps some logic previously + * called enter_critical_section() with no matching call to + * leave_critical_section(), leaving the non-zero count. + */ - spin_clrbit(&g_cpu_irqset, cpu, &g_cpu_irqsetlock, - &g_cpu_irqlock); + DEBUGASSERT(up_interrupt_context()); } nxttcb->task_state = TSTATE_TASK_RUNNING;