/* * soclib_pic.c - soclib PIC driver implementation. * * Author Alain Greiner (2016,2017,2018,2019,2020) * * Copyright (c) UPMC Sorbonne Universites * * This file is part of ALMOS-MKH. * * ALMOS-MKH is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by * the Free Software Foundation; version 2.0 of the License. * * ALMOS-MKH is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public License * along with ALMOS-MKH; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include #include #include #include #include #include #include #include #include #include #include #include ////////////////////////////////////////////////////////////////////////////////////// // Extern variables ////////////////////////////////////////////////////////////////////////////////////// extern chdev_directory_t chdev_dir; // defined in chdev.h / allocated in kerneL-init.c extern iopic_input_t iopic_input; // defined in dev_pic.h / allocated in kernel_init.c extern lapic_input_t lapic_input; // defined in dev_pic.h / allocated in kernel_init.c ////////////////////////////////////////////////////////////////////////////////////// // SOCLIB PIC private functions ////////////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////// uint32_t soclib_pic_wti_alloc( void ) { uint32_t index; // get pointer on cluster extension for SOCLIB PIC (XCU descriptor) soclib_pic_cluster_t * ext_ptr = LOCAL_CLUSTER->pic_extend; assert( __FUNCTION__, (ext_ptr->first_free_wti < ext_ptr->wti_nr) , "no free WTI found : too much external IRQs"); // update WTI allocator index = ext_ptr->first_free_wti; ext_ptr->first_free_wti++; return index; } // end soclib_pic_wti_alloc() ///////////////////////////////////////////// inline uint32_t * soclib_pic_xcu_base( void ) { return ((soclib_pic_cluster_t *)(LOCAL_CLUSTER->pic_extend))->xcu_base; } ///////////////////////////////////////////////////////// inline uint32_t * soclib_pic_remote_xcu_base( cxy_t cxy ) { soclib_pic_cluster_t * extend; // get extended pointer on PIC extension in remote cluster extend = hal_remote_lpt( XPTR( cxy , &cluster_manager.pic_extend ) ); return (uint32_t *)hal_remote_lpt( XPTR( cxy , &extend->xcu_base ) ); } /////////////////////////////////////////// void soclib_pic_xcu_status( lid_t lid, uint32_t * hwi_status, uint32_t * wti_status, uint32_t * pti_status ) { // get local XCU segment base uint32_t * base = soclib_pic_xcu_base(); // read PRIO register // in TSAR : XCU output [4*lid] is connected to core [lid] uint32_t prio = base[ (XCU_PRIO << 5) | (lid<<2) ]; *wti_status = (prio & 0x4) ? (((prio >> 24) & 0x1F) + 1) : 0; *hwi_status = (prio & 0x2) ? (((prio >> 16) & 0x1F) + 1) : 0; *pti_status = (prio & 0x1) ? (((prio >> 8) & 0x1F) + 1) : 0; } //////////////////////////////////////////////////// inline uint32_t soclib_pic_xcu_ack( uint32_t * reg ) { return *reg; } /////////////////////////////////// void soclib_pic_irq_handler( void ) { uint32_t hwi_status; // HWI index + 1 / no pending HWI if 0 uint32_t wti_status; // WTI index + 1 / no pending WTI if 0 uint32_t pti_status; // PTI index + 1 / no pending PTI if 0 chdev_t * src_chdev; // pointer on source chdev descriptor uint32_t index; // WTI / HWI / PTI index uint32_t * xcu_base = soclib_pic_xcu_base(); core_t * core = CURRENT_THREAD->core; // get XCU status soclib_pic_xcu_status( core->lid, &hwi_status, &wti_status, &pti_status ); #if DEBUG_HAL_IRQS uint32_t cycle = (uint32_t)hal_get_cycles(); if (DEBUG_HAL_IRQS < cycle ) printk("\n[DBG] %s : core[%x,%d] enter / WTI = %x / HWI = %x / PTI = %x / cycle %d\n", __FUNCTION__ , local_cxy , core->lid , wti_status , hwi_status , pti_status, cycle ); #endif // analyse status and handle up to 3 pending IRQs (one WTI, one HWI, one PTI) if( wti_status ) // pending WTI { index = wti_status - 1; //////////////////////////////////////////////////////// if( index < LOCAL_CLUSTER->cores_nr ) // it is an IPI { assert( __FUNCTION__, (index == core->lid), "illegal IPI index" ); #if DEBUG_HAL_IRQS if (DEBUG_HAL_IRQS < cycle ) printk("\n[DBG] %s : core[%x,%d] handling IPI\n", __FUNCTION__ , local_cxy , core->lid ); #endif // acknowledge IRQ (this require an XCU read) uint32_t ack = xcu_base[(XCU_WTI_REG << 5) | core->lid]; // check RPC FIFO, and activate or create a RPC thread // condition is always true, but we use the ack value // to avoid a GCC warning if( ack + 1 ) sched_yield("IPI received"); } //////////////////////////////////////////////////////////////// else // it is an external IRQ { // get pointer on source chdev src_chdev = ((soclib_pic_core_t *)core->pic_extend)->wti_vector[index]; if( src_chdev == NULL ) // strange, but not fatal { printk("\n[WARNING] in %s : no handler for WTI %d on core %d in cluster %x\n", __FUNCTION__ , index , core->lid , local_cxy ); // disable WTI in local XCU controller xcu_base[(XCU_MSK_WTI_DISABLE << 5) | core->lid] = 1 << core->lid; hal_fence(); } else // call relevant ISR { #if DEBUG_HAL_IRQS if (DEBUG_HAL_IRQS < cycle ) printk("\n[DBG] %s : core[%x,%d] handling external WTI %d\n", __FUNCTION__ , local_cxy , core->lid , index ); #endif // call ISR src_chdev->isr( src_chdev ); } } } ///////////////////////////////////////////////////////////// if( hwi_status ) // It is an Internal IRQ { index = hwi_status - 1; // get pointer on source chdev src_chdev = ((soclib_pic_core_t *)core->pic_extend)->hwi_vector[index]; if( src_chdev == NULL ) // strange, but not fatal { printk("\n[WARNING] in %s : no handler for HWI %d on core %d in cluster %x\n", __FUNCTION__ , index , core->lid , local_cxy ); // disable HWI in local XCU controller xcu_base[(XCU_MSK_HWI_DISABLE << 5) | core->lid] = 1 << core->lid; hal_fence(); } else // call relevant ISR { #if DEBUG_HAL_IRQS if (DEBUG_HAL_IRQS < cycle ) printk("\n[DBG] %s : core[%x,%d] handling HWI %d\n", __FUNCTION__ , local_cxy , core->lid , index ); #endif // call ISR src_chdev->isr( src_chdev ); } } /////////////////////////////////////////////////////// if( pti_status ) // It is a Timer IRQ { index = pti_status - 1; assert( __FUNCTION__, (index == core->lid), "unconsistent PTI index\n"); #if DEBUG_HAL_IRQS if (DEBUG_HAL_IRQS < cycle ) printk("\n[DBG] %s : core[%x,%d] handling PTI %d\n", __FUNCTION__ , core->lid , local_cxy , index ); #endif // acknowledge IRQ (this require a read access to XCU) uint32_t ack = xcu_base[(XCU_PTI_ACK << 5) | core->lid]; // execute all actions related to TICK event // condition is always true, but we use the ack value // to avoid a GCC warning if( ack + 1 ) core_clock( core ); } } // end soclib_pic_irq_handler() ////////////////////////////////////////////////////////////////////////////////////// // SOCLIC PIC device generic API ////////////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////// void soclib_pic_init( chdev_t * pic ) { uint32_t i; // for loop on IOPIC inputs // get IOPIC controller cluster and segment base pointer cxy_t iopic_seg_cxy = GET_CXY( pic->base ); uint32_t * iopic_seg_ptr = GET_PTR( pic->base ); // reset the IOPIC component registers : disable all input IRQs for( i = 0 ; i < CONFIG_MAX_EXTERNAL_IRQS ; i++ ) { xptr_t iopic_seg_xp = XPTR( iopic_seg_cxy, iopic_seg_ptr + i*IOPIC_SPAN + IOPIC_MASK ); hal_remote_s32( iopic_seg_xp , 0 ); } } // end soclib_pic_init() ////////////////////////////////////////////////// void soclib_pic_extend_init( uint32_t * xcu_base ) { soclib_pic_cluster_t * cluster_ext_ptr; soclib_pic_core_t * core_ext_ptr; uint32_t lid; uint32_t idx; cluster_t * cluster = LOCAL_CLUSTER; // create core extension for all cores in cluster for( lid = 0 ; lid < cluster->cores_nr ; lid++ ) { // allocate memory for core extension core_ext_ptr = kmem_alloc( bits_log2( sizeof(soclib_pic_core_t)) , AF_KERNEL ); if( core_ext_ptr == NULL ) { printk("\n[PANIC] in %s : cannot allocate memory for core extension\n", __FUNCTION__ ); } // reset the HWI / WTI interrupt vectors for( idx = 0 ; idx < SOCLIB_MAX_HWI ; idx++ ) core_ext_ptr->hwi_vector[idx] = NULL; for( idx = 0 ; idx < SOCLIB_MAX_WTI ; idx++ ) core_ext_ptr->wti_vector[idx] = NULL; // register PIC extension in core descriptor cluster->core_tbl[lid].pic_extend = core_ext_ptr; } // allocate memory for cluster extension cluster_ext_ptr = kmem_alloc( bits_log2( sizeof(soclib_pic_cluster_t) ), AF_KERNEL ); if( cluster_ext_ptr == NULL ) { printk("\n[PANIC] in %s : cannot allocate memory for cluster extension\n", __FUNCTION__ ); } // get XCU characteristics from the XCU config register uint32_t config = xcu_base[XCU_CONFIG<<5]; uint32_t wti_nr = (config >> 16) & 0xFF; uint32_t hwi_nr = (config >> 8 ) & 0xFF; uint32_t pti_nr = (config ) & 0xFF; // initialize the cluster extension // The first WTI slots are for IPIs (one slot per core) cluster_ext_ptr->xcu_base = xcu_base; cluster_ext_ptr->hwi_nr = hwi_nr; cluster_ext_ptr->wti_nr = wti_nr; cluster_ext_ptr->pti_nr = pti_nr; cluster_ext_ptr->first_free_wti = cluster->cores_nr; // register PIC extension in cluster manager cluster->pic_extend = cluster_ext_ptr; // reset the XCU component registers // mask all HWIs, all WTIs, and all PTIs, for all cores in local cluster for( lid = 0 ; lid < cluster->cores_nr ; lid++ ) { xcu_base[XCU_MSK_HWI_DISABLE << 5 | lid] = 0xFFFFFFFF; xcu_base[XCU_MSK_WTI_DISABLE << 5 | lid] = 0xFFFFFFFF; xcu_base[XCU_MSK_PTI_DISABLE << 5 | lid] = 0xFFFFFFFF; } } // end soclib_pic_extend_init() //////////////////////////////////////// void soclib_pic_bind_irq( lid_t lid, chdev_t * src_chdev ) { #if DEBUG_HAL_IRQS uint32_t cycle = (uint32_t)hal_get_cycles(); if( DEBUG_HAL_IRQS < cycle ) printk("\n[DBG] %s : thread %x enter for core[%x,%d] / cycle %d\n", __FUNCTION__ , CURRENT_THREAD , local_cxy , lid , cycle ); #endif // get extended & local pointers on PIC chdev descriptor xptr_t pic_xp = chdev_dir.pic; cxy_t pic_cxy = GET_CXY( pic_xp ); chdev_t * pic_ptr = (chdev_t *)GET_PTR( pic_xp ); // get extended and local pointers on IOPIC segment base xptr_t seg_pic_xp = hal_remote_l64( XPTR( pic_cxy , &pic_ptr->base ) ); cxy_t seg_pic_cxy = GET_CXY( seg_pic_xp ); uint32_t * seg_pic_ptr = (uint32_t *)GET_PTR( seg_pic_xp ); // get local pointer on XCU segment base uint32_t * seg_xcu_ptr = soclib_pic_xcu_base(); // get the source chdev functionnal type, channel, and direction uint32_t func = src_chdev->func; uint32_t impl = src_chdev->impl; uint32_t channel = src_chdev->channel; bool_t is_rx = src_chdev->is_rx; if( ((func == DEV_FUNC_IOC) && (impl == IMPL_IOC_BDV)) || (func == DEV_FUNC_NIC) || ((func == DEV_FUNC_TXT) && (impl == IMPL_TXT_TTY)) || (func == DEV_FUNC_IOB) ) // external IRQ => WTI { // get external IRQ index uint32_t hwi_id = 0; if ( func == DEV_FUNC_IOC ) hwi_id = iopic_input.ioc[channel]; else if( (func == DEV_FUNC_TXT) && is_rx ) hwi_id = iopic_input.txt_rx[channel]; else if( (func == DEV_FUNC_TXT) && !is_rx ) hwi_id = iopic_input.txt_tx[channel]; else if( (func == DEV_FUNC_NIC) && is_rx ) hwi_id = iopic_input.nic_rx[channel]; else if( (func == DEV_FUNC_NIC) && !is_rx ) hwi_id = iopic_input.nic_tx[channel]; else if( func == DEV_FUNC_IOB ) hwi_id = iopic_input.iob; else { printk("\n[WARNING] from %s : illegal device / func %s / is_rx %d\n", __FUNCTION__, chdev_func_str(func), is_rx ); } // get a WTI mailbox from local XCU descriptor uint32_t wti_id = soclib_pic_wti_alloc(); // register IRQ type and index in chdev src_chdev->irq_type = SOCLIB_TYPE_WTI; src_chdev->irq_id = wti_id; // compute extended pointer on WTI mailbox in local XCU xptr_t wti_xp = XPTR( local_cxy , &seg_xcu_ptr[(XCU_WTI_REG << 5) | wti_id] ); // set the IOPIC_ADDRESS and IOPIC_EXTEND registers in IOPIC uint32_t lsb_wdata = (uint32_t)wti_xp; uint32_t msb_wdata = (uint32_t)(wti_xp >> 32); xptr_t lsb_xp = XPTR( seg_pic_cxy , seg_pic_ptr+hwi_id*IOPIC_SPAN+IOPIC_ADDRESS ); xptr_t msb_xp = XPTR( seg_pic_cxy , seg_pic_ptr+hwi_id*IOPIC_SPAN+IOPIC_EXTEND ); hal_remote_s32( lsb_xp , lsb_wdata ); hal_remote_s32( msb_xp , msb_wdata ); // enable IRQ in IOPIC hal_remote_s32( XPTR( seg_pic_cxy , seg_pic_ptr+hwi_id*IOPIC_SPAN+IOPIC_MASK ), 1 ); // update the WTI interrupt vector for core[lid] core_t * core = &LOCAL_CLUSTER->core_tbl[lid]; ((soclib_pic_core_t *)core->pic_extend)->wti_vector[wti_id] = src_chdev; #if DEBUG_HAL_IRQS if( DEBUG_HAL_IRQS < cycle ) printk("\n[DBG] %s : %s / channel %d / rx %d / hwi_id %d / wti_id %d / cluster %x\n", __FUNCTION__ , chdev_func_str( func ) , channel , is_rx , hwi_id , wti_id , local_cxy ); #endif } else if( (func == DEV_FUNC_DMA) || (func == DEV_FUNC_MMC) || (func == DEV_FUNC_TXT && impl == IMPL_TXT_MTY) || (func == DEV_FUNC_IOC && impl == IMPL_IOC_SPI) ) // internal IRQ => HWI { // get internal IRQ index uint32_t hwi_id; if( func == DEV_FUNC_DMA ) hwi_id = lapic_input.dma[channel]; else if (func == DEV_FUNC_TXT ) hwi_id = lapic_input.mtty; else if (func == DEV_FUNC_IOC ) hwi_id = lapic_input.sdcard; else hwi_id = lapic_input.mmc; // register IRQ type and index in chdev src_chdev->irq_type = SOCLIB_TYPE_HWI; src_chdev->irq_id = hwi_id; // update the HWI interrupt vector for core[lid] core_t * core = &LOCAL_CLUSTER->core_tbl[lid]; ((soclib_pic_core_t *)core->pic_extend)->hwi_vector[hwi_id] = src_chdev; #if DEBUG_HAL_IRQS if( DEBUG_HAL_IRQS < cycle ) printk("\n[DBG] %s : %s / channel = %d / hwi_id = %d / cluster = %x\n", __FUNCTION__ , chdev_func_str( func ) , channel , hwi_id , local_cxy ); #endif } else { printk("\n[WARNING] from %s : illegal device / func %s / is_rx %d / impl %d\n", __FUNCTION__, chdev_func_str(func), is_rx, impl ); } } // end soclib_pic_bind_irq(); /////////////////////////////////////// void soclib_pic_enable_irq( lid_t lid, xptr_t src_chdev_xp ) { // get cluster and local pointer on remote src_chdev cxy_t src_chdev_cxy = GET_CXY( src_chdev_xp ); chdev_t * src_chdev_ptr = (chdev_t *)GET_PTR( src_chdev_xp ); // get local pointer on remote XCU segment base uint32_t * seg_xcu_ptr = soclib_pic_remote_xcu_base( src_chdev_cxy ); // get the source chdev IRQ type and index uint32_t irq_type = hal_remote_l32( XPTR( src_chdev_cxy , &src_chdev_ptr->irq_type ) ); uint32_t irq_id = hal_remote_l32( XPTR( src_chdev_cxy , &src_chdev_ptr->irq_id ) ); if( irq_type == SOCLIB_TYPE_HWI ) { // enable this HWI in remote XCU controller // in TSAR : XCU output [4*lid] is connected to core [lid] hal_remote_s32( XPTR( src_chdev_cxy , &seg_xcu_ptr[ (XCU_MSK_HWI_ENABLE << 5) | (lid<<2) ] ) , (1 << irq_id) ); } else if( irq_type == SOCLIB_TYPE_WTI ) { // enable this WTI in remote XCU controller // in TSAR : XCU output [4*lid] is connected to core [lid] hal_remote_s32( XPTR( src_chdev_cxy , &seg_xcu_ptr[ (XCU_MSK_WTI_ENABLE << 5) | (lid<<2) ] ) , (1 << irq_id) ); } else { printk("\n[WARNING] from %s : illegal IRQ type %d\n", __FUNCTION__, irq_type ); } } // end soclib_pic_enable_irq() //////////////////////////////////////// void soclib_pic_disable_irq( lid_t lid, xptr_t src_chdev_xp ) { // get cluster and local pointer on remote src_chdev cxy_t src_chdev_cxy = GET_CXY( src_chdev_xp ); chdev_t * src_chdev_ptr = (chdev_t *)GET_PTR( src_chdev_xp ); // get local pointer on remote XCU segment base uint32_t * seg_xcu_ptr = soclib_pic_remote_xcu_base( src_chdev_cxy ); // get the source chdev IRQ type and index uint32_t irq_type = hal_remote_l32( XPTR( src_chdev_cxy , &src_chdev_ptr->irq_type ) ); uint32_t irq_id = hal_remote_l32( XPTR( src_chdev_cxy , &src_chdev_ptr->irq_id ) ); if( irq_type == SOCLIB_TYPE_HWI ) { // enable this HWI in remote XCU controller // in TSAR : XCU output [4*lid] is connected to core [lid] hal_remote_s32( XPTR( src_chdev_cxy , &seg_xcu_ptr[(XCU_MSK_HWI_DISABLE << 5) | (lid<<2) ] ) , (1 << irq_id) ); } else if( irq_type == SOCLIB_TYPE_WTI ) { // enable this WTI in remote XCU controller // in TSAR : XCU output [4*lid] is connected to core [lid] hal_remote_s32( XPTR( src_chdev_cxy , &seg_xcu_ptr[(XCU_MSK_WTI_DISABLE << 5) | (lid<<2) ] ) , (1 << irq_id) ); } else { printk("\n[WARNING] from %s : illegal IRQ type %d\n", __FUNCTION__, irq_type ); } } // end soclib_pic_enable_irq() /////////////////////////////////////////////// void soclib_pic_enable_timer( uint32_t period ) { // calling core local index lid_t lid = CURRENT_THREAD->core->lid; // get XCU segment base uint32_t * base = soclib_pic_xcu_base(); // set period value in XCU (in cycles) uint32_t cycles = period * SOCLIB_CYCLES_PER_MS; base[(XCU_PTI_PER << 5) | lid] = cycles; // enable PTI in local XCU controller // In TSAR : XCU output [4*lid] is connected to core [lid] base[ (XCU_MSK_PTI_ENABLE << 5) | (lid<<2) ] = 1 << lid; } //////////////////////////// void soclib_pic_enable_ipi( void ) { // calling core local index lid_t lid = CURRENT_THREAD->core->lid; // get XCU segment base uint32_t * base = soclib_pic_xcu_base(); // enable WTI in local XCU controller // In TSAR : XCU output [4*lid] is connected to core [lid] base[ (XCU_MSK_WTI_ENABLE << 5) | (lid<<2) ] = 1 << lid; } /////////////////////////////////////// void soclib_pic_send_ipi( cxy_t cxy, lid_t lid ) { // get pointer on local XCU segment base uint32_t * base = soclib_pic_xcu_base(); // write to WTI mailbox[cxy][lid] hal_remote_s32( XPTR( cxy , &base[(XCU_WTI_REG << 5) | lid ] ) , 0 ); } /////////////////////////////// void soclib_pic_ack_ipi( void ) { // get calling core local index lid_t lid = CURRENT_THREAD->core->lid; // get pointer on local XCU segment base uint32_t * base = soclib_pic_xcu_base(); // acknowlege IPI uint32_t ack = base[ (XCU_WTI_REG << 5) | lid ]; // we make a fake use for ack value to avoid a warning if( (ack + 1) == 0 ) asm volatile( "nop" ); }