/* * soclib_pic.c - soclib PIC driver implementation. * * Author Alain Greiner (2016,2017) * * Copyright (c) UPMC Sorbonne Universites * * This file is part of ALMOS-MKH. * * ALMOS-MKH is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by * the Free Software Foundation; version 2.0 of the License. * * ALMOS-MKH is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public License * along with ALMOS-MKH; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include #include #include #include #include #include #include #include #include #include #include ////////////////////////////////////////////////////////////////////////////////////// // Extern variables ////////////////////////////////////////////////////////////////////////////////////// extern chdev_directory_t chdev_dir; // defined in chdev.h / allocated in kerneL-init.c extern iopic_input_t iopic_input; // defined in dev_pic.h / allocated in kernel_init.c extern lapic_input_t lapic_input; // defined in dev_pic.h / allocated in kernel_init.c ////////////////////////////////////////////////////////////////////////////////////// // SOCLIB PIC private functions ////////////////////////////////////////////////////////////////////////////////////// /////////////////////////////// uint32_t soclib_pic_wti_alloc() { uint32_t index; // get pointer on cluster extension for SOCLIB PIC (XCU descriptor) soclib_pic_cluster_t * ext_ptr = LOCAL_CLUSTER->pic_extend; assert( (ext_ptr->first_free_wti < ext_ptr->wti_nr) , __FUNCTION__ , "no free WTI found : too much external IRQs\n"); // update WTI allocator index = ext_ptr->first_free_wti; ext_ptr->first_free_wti++; return index; } // end soclib_pic_wti_alloc() /////////////////////////////////////// inline uint32_t * soclib_pic_xcu_base() { return ((soclib_pic_cluster_t *)(LOCAL_CLUSTER->pic_extend))->xcu_base; } ///////////////////////////////////////////////////////// inline uint32_t * soclib_pic_remote_xcu_base( cxy_t cxy ) { soclib_pic_cluster_t * extend; // get extended pointer on PIC extension in remote cluster extend = hal_remote_lpt( XPTR( cxy , &cluster_manager.pic_extend ) ); return (uint32_t *)hal_remote_lpt( XPTR( cxy , &extend->xcu_base ) ); } /////////////////////////////////////////// void soclib_pic_xcu_status( lid_t lid, uint32_t * hwi_status, uint32_t * wti_status, uint32_t * pti_status ) { // get local XCU segment base uint32_t * base = soclib_pic_xcu_base(); // read PRIO register uint32_t prio = base[(XCU_PRIO << 5) | lid]; *wti_status = (prio & 0x4) ? (((prio >> 24) & 0x1F) + 1) : 0; *hwi_status = (prio & 0x2) ? (((prio >> 16) & 0x1F) + 1) : 0; *pti_status = (prio & 0x1) ? (((prio >> 8) & 0x1F) + 1) : 0; } //////////////////////////////////////////////////// inline uint32_t soclib_pic_xcu_ack( uint32_t * reg ) { return *reg; } ///////////////////////////// void soclib_pic_irq_handler() { uint32_t hwi_status; // HWI index + 1 / no pending HWI if 0 uint32_t wti_status; // WTI index + 1 / no pending WTI if 0 uint32_t pti_status; // PTI index + 1 / no pending PTI if 0 chdev_t * src_chdev; // pointer on source chdev descriptor uint32_t index; // WTI / HWI / PTI index uint32_t ack; // XCU acknowledge requires a read... core_t * core = CURRENT_THREAD->core; // get XCU status soclib_pic_xcu_status( core->lid, &hwi_status, &wti_status, &pti_status ); irq_dmsg("\n[INFO] %s : enter for core[%x,%d] / WTI = %x / HWI = %x / WTI = %x\n", __FUNCTION__ , local_cxy , core->lid , wti_status , hwi_status , pti_status ); // analyse status and handle up to 3 pending IRQ (one WTI, one HWI, one PTI) if( wti_status ) // pending WTI { index = wti_status - 1; if( index < LOCAL_CLUSTER->cores_nr ) // it is an IPI { assert( (index == core->lid) , __FUNCTION__ , "illegal IPI index" ); // acknowledge WTI uint32_t * base = soclib_pic_xcu_base(); ack = base[(XCU_WTI_REG << 5) | core->lid]; // check RPC FIFO, and activate or create a RPC thread // it there is a pending RPC request rpc_check(); } else // it is an external device { // get pointer on source chdev src_chdev = ((soclib_pic_core_t *)core->pic_extend)->wti_vector[index]; if( src_chdev == NULL ) // strange, but not fatal { printk("\n[WARNING] in %s : no handler for WTI %d on core %d in cluster %x\n", __FUNCTION__ , index , core->lid , local_cxy ); core->spurious_irqs ++; // disable WTI in local XCU controller uint32_t * base = soclib_pic_xcu_base(); base[(XCU_MSK_WTI_DISABLE << 5) | core->lid] = 1 << core->lid; } else // call relevant ISR { irq_dmsg("\n[INFO] %s received WTI : index = %d for core %d in cluster %d\n", __FUNCTION__ , index , core->lid , local_cxy ); // call ISR src_chdev->isr( src_chdev ); } } } if( hwi_status ) // pending HWI { index = hwi_status - 1; // get pointer on source chdev src_chdev = ((soclib_pic_core_t *)core->pic_extend)->hwi_vector[index]; if( src_chdev == NULL ) // strange, but not fatal { printk("\n[WARNING] in %s : no handler for HWI %d on core %d in cluster %x\n", __FUNCTION__ , index , core->lid , local_cxy ); core->spurious_irqs ++; // disable HWI in local XCU controller uint32_t * base = soclib_pic_xcu_base(); base[(XCU_MSK_HWI_DISABLE << 5) | core->lid] = 1 << core->lid; } else // call relevant ISR { irq_dmsg("\n[INFO] %s received HWI : index = %d for core %d in cluster %d\n", __FUNCTION__ , index , core->lid , local_cxy ); // call ISR src_chdev->isr( src_chdev ); } } if( pti_status ) // pending PTI { index = pti_status - 1; irq_dmsg("\n[INFO] %s received PTI : index = %d for cpu %d in cluster %d\n", __FUNCTION__ , index , core->lid , local_cxy ); assert( (index == core->lid) , __FUNCTION__ , "unconsistent PTI index\n"); // acknowledge PTI uint32_t * base = soclib_pic_xcu_base(); ack = base[(XCU_PTI_ACK << 5) | core->lid]; // execute all actions related to TICK event core_clock( core ); } } // end soclib_pic_irq_handler() ////////////////////////////////////////////////////////////////////////////////////// // SOCLIC PIC device generic API ////////////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////// void soclib_pic_init( chdev_t * pic ) { uint32_t i; // for loop on IOPIC inputs uint32_t x; // for loop on clusters in a row uint32_t y; // for loop on clusters in a column inputs uint32_t lid; // for loop on cores in a cluster // get target architecture parameters cluster_t * cluster = LOCAL_CLUSTER; uint32_t x_size = cluster->x_size; uint32_t y_size = cluster->y_size; uint32_t y_width = cluster->y_width; uint32_t ncores = cluster->cores_nr; // get IOPIC controller cluster and segment base pointer cxy_t iopic_seg_cxy = (cxy_t)GET_CXY( pic->base ); uint32_t * iopic_seg_ptr = (uint32_t *)GET_PTR( pic->base ); // reset the IOPIC component registers : mask all input IRQs for( i = 0 ; i < CONFIG_MAX_EXTERNAL_IRQS ; i++ ) { xptr_t iopic_seg_xp = XPTR( iopic_seg_cxy, iopic_seg_ptr + i*IOPIC_SPAN + IOPIC_MASK ); hal_remote_sw( iopic_seg_xp , 0 ); } // GET XCU controller segment base uint32_t * base = soclib_pic_xcu_base(); // reset the XCU component registers in all clusters: // mask all HWIs, all WTIs, and all PTIs, for all cores for( x = 0 ; x < x_size ; x++ ) { for( y = 0 ; y < y_size ; y++ ) { for( lid = 0 ; lid < ncores ; lid++ ) { cxy_t cxy = (x<cores_nr ; lid++ ) { // allocate memory for core extension req.type = KMEM_GENERIC; req.size = sizeof(soclib_pic_core_t); req.flags = AF_KERNEL; core_ext_ptr = kmem_alloc( &req ); assert( (core_ext_ptr != NULL) , __FUNCTION__ , "cannot allocate memory for core extension\n"); // reset the HWI / WTI interrupt vectors for( idx = 0 ; idx < SOCLIB_MAX_HWI ; idx++ ) core_ext_ptr->hwi_vector[idx] = NULL; for( idx = 0 ; idx < SOCLIB_MAX_WTI ; idx++ ) core_ext_ptr->wti_vector[idx] = NULL; // register PIC extension in core descriptor cluster->core_tbl[lid].pic_extend = core_ext_ptr; } // allocate memory for cluster extension req.type = KMEM_GENERIC; req.size = sizeof(soclib_pic_cluster_t); req.flags = AF_KERNEL; cluster_ext_ptr = kmem_alloc( &req ); assert( (cluster_ext_ptr != NULL) , __FUNCTION__ , "cannot allocate memory for cluster extension\n"); // get XCU characteristics from the XCU config register uint32_t config = xcu_base[XCU_CONFIG<<5]; uint32_t wti_nr = (config >> 16) & 0xFF; uint32_t hwi_nr = (config >> 8 ) & 0xFF; uint32_t pti_nr = (config ) & 0xFF; // initialize the cluster extension // The first WTI slots are for IPIs (one slot per core) cluster_ext_ptr->xcu_base = xcu_base; cluster_ext_ptr->hwi_nr = hwi_nr; cluster_ext_ptr->wti_nr = wti_nr; cluster_ext_ptr->pti_nr = pti_nr; cluster_ext_ptr->first_free_wti = cluster->cores_nr; // register PIC extension in cluster manager cluster->pic_extend = cluster_ext_ptr; } // end soclib_pic_extend_init() //////////////////////////////////////// void soclib_pic_bind_irq( lid_t lid, chdev_t * src_chdev ) { // get extended & local pointers on PIC chdev descriptor xptr_t pic_xp = chdev_dir.pic; cxy_t pic_cxy = GET_CXY( pic_xp ); chdev_t * pic_ptr = (chdev_t *)GET_PTR( pic_xp ); // get extended and local pointers on IOPIC segment base xptr_t seg_pic_xp = hal_remote_lwd( XPTR( pic_cxy , &pic_ptr->base ) ); cxy_t seg_pic_cxy = GET_CXY( seg_pic_xp ); uint32_t * seg_pic_ptr = (uint32_t *)GET_PTR( seg_pic_xp ); // get local pointer on XCU segment base uint32_t * seg_xcu_ptr = soclib_pic_xcu_base(); // get the source chdev functionnal type, channel, and direction uint32_t func = src_chdev->func; uint32_t channel = src_chdev->channel; bool_t is_rx = src_chdev->is_rx; if( (func == DEV_FUNC_IOC) || (func == DEV_FUNC_NIC) || (func == DEV_FUNC_TXT) || (func == DEV_FUNC_IOB) ) // external IRQ => WTI { // get external IRQ index uint32_t irq_id; if ( func == DEV_FUNC_IOC ) irq_id = iopic_input.ioc[channel]; else if( func == DEV_FUNC_TXT ) irq_id = iopic_input.txt[channel]; else if( (func == DEV_FUNC_NIC) && is_rx ) irq_id = iopic_input.nic_rx[channel]; else if( (func == DEV_FUNC_NIC) && !is_rx ) irq_id = iopic_input.nic_tx[channel]; else if( func == DEV_FUNC_IOB ) irq_id = iopic_input.iob; else assert( false , __FUNCTION__ , "illegal device functionnal type\n"); // get a WTI mailbox from local XCU descriptor uint32_t wti_id = soclib_pic_wti_alloc(); // register IRQ type and index in chdev src_chdev->irq_type = SOCLIB_TYPE_WTI; src_chdev->irq_id = wti_id; // compute extended pointer on WTI mailbox in local XCU xptr_t wti_xp = XPTR( local_cxy , &seg_xcu_ptr[(XCU_WTI_REG << 5) | wti_id] ); // set the IOPIC_ADDRESS and IOPIC_EXTEND registers in IOPIC uint32_t lsb_wdata = (uint32_t)wti_xp; uint32_t msb_wdata = (uint32_t)(wti_xp >> 32); xptr_t lsb_xp = XPTR( seg_pic_cxy , seg_pic_ptr+irq_id*IOPIC_SPAN+IOPIC_ADDRESS ); xptr_t msb_xp = XPTR( seg_pic_cxy , seg_pic_ptr+irq_id*IOPIC_SPAN+IOPIC_EXTEND ); hal_remote_sw( lsb_xp , lsb_wdata ); hal_remote_sw( msb_xp , msb_wdata ); // unmask IRQ in IOPIC hal_remote_sw( XPTR( seg_pic_cxy , seg_pic_ptr+irq_id*IOPIC_SPAN+IOPIC_MASK ), 1 ); // update the WTI interrupt vector for core[lid] core_t * core = &LOCAL_CLUSTER->core_tbl[lid]; ((soclib_pic_core_t *)core->pic_extend)->wti_vector[wti_id] = src_chdev; } else if( (func == DEV_FUNC_DMA) || (func == DEV_FUNC_MMC) ) // internal IRQ => HWI { // get internal IRQ index uint32_t hwi_id; if( func == DEV_FUNC_DMA ) hwi_id = lapic_input.dma[channel]; else hwi_id = lapic_input.mmc; // register IRQ type and index in chdev src_chdev->irq_type = SOCLIB_TYPE_HWI; src_chdev->irq_id = hwi_id; // update the HWI interrupt vector for core[lid] core_t * core = &LOCAL_CLUSTER->core_tbl[lid]; ((soclib_pic_core_t *)core->pic_extend)->wti_vector[hwi_id] = src_chdev; } else { assert( false , __FUNCTION__ , "illegal device functionnal type\n" ); } } // end soclib_pic_bind_irq(); /////////////////////////////////////// void soclib_pic_enable_irq( lid_t lid, xptr_t src_chdev_xp ) { // get cluster and local pointer on remote src_chdev cxy_t src_chdev_cxy = GET_CXY( src_chdev_xp ); chdev_t * src_chdev_ptr = (chdev_t *)GET_PTR( src_chdev_xp ); // get local pointer on remote XCU segment base uint32_t * seg_xcu_ptr = soclib_pic_remote_xcu_base( src_chdev_cxy ); // get the source chdev IRQ type and index uint32_t irq_type = hal_remote_lw( XPTR( src_chdev_cxy , &src_chdev_ptr->irq_type ) ); uint32_t irq_id = hal_remote_lw( XPTR( src_chdev_cxy , &src_chdev_ptr->irq_id ) ); if( irq_type == SOCLIB_TYPE_HWI ) { // enable this HWI in remote XCU controller hal_remote_sw( XPTR( src_chdev_cxy , &seg_xcu_ptr[(XCU_MSK_HWI_ENABLE << 5) | lid] ) , (1 << irq_id) ); } else if( irq_type == SOCLIB_TYPE_WTI ) { // enable this WTI in remote XCU controller hal_remote_sw( XPTR( src_chdev_cxy , &seg_xcu_ptr[(XCU_MSK_WTI_ENABLE << 5) | lid] ) , (1 << irq_id) ); } else { assert( false , __FUNCTION__ , "illegal IRQ type\n" ); } } // end soclib_pic_enable_irq() //////////////////////////////////////// void soclib_pic_disable_irq( lid_t lid, xptr_t src_chdev_xp ) { // get cluster and local pointer on remote src_chdev cxy_t src_chdev_cxy = GET_CXY( src_chdev_xp ); chdev_t * src_chdev_ptr = (chdev_t *)GET_PTR( src_chdev_xp ); // get local pointer on remote XCU segment base uint32_t * seg_xcu_ptr = soclib_pic_remote_xcu_base( src_chdev_cxy ); // get the source chdev IRQ type and index uint32_t irq_type = hal_remote_lw( XPTR( src_chdev_cxy , &src_chdev_ptr->irq_type ) ); uint32_t irq_id = hal_remote_lw( XPTR( src_chdev_cxy , &src_chdev_ptr->irq_id ) ); if( irq_type == SOCLIB_TYPE_HWI ) { // enable this HWI in remote XCU controller hal_remote_sw( XPTR( src_chdev_cxy , &seg_xcu_ptr[(XCU_MSK_HWI_DISABLE << 5) | lid] ) , (1 << irq_id) ); } else if( irq_type == SOCLIB_TYPE_WTI ) { // enable this WTI in remote XCU controller hal_remote_sw( XPTR( src_chdev_cxy , &seg_xcu_ptr[(XCU_MSK_WTI_DISABLE << 5) | lid] ) , (1 << irq_id) ); } else { assert( false , __FUNCTION__ , "illegal IRQ type\n" ); } } // end soclib_pic_enable_irq() /////////////////////////////////////////////// void soclib_pic_enable_timer( uint32_t period ) { // calling core local index lid_t lid = CURRENT_CORE->lid; // get XCU segment base uint32_t * base = soclib_pic_xcu_base(); // set period value in XCU base[(XCU_PTI_PER << 5) | lid] = period; // enable PTI in local XCU controller base[(XCU_MSK_PTI_ENABLE << 5) | lid] = 1 << lid; } //////////////////////////// void soclib_pic_enable_ipi() { // calling core local index lid_t lid = CURRENT_CORE->lid; // get XCU segment base uint32_t * base = soclib_pic_xcu_base(); // enable WTI in local XCU controller base[(XCU_MSK_WTI_ENABLE << 5) | lid] = 1 << lid; } /////////////////////////////////////// void soclib_pic_send_ipi( cxy_t cxy, lid_t lid ) { // get pointer on local XCU segment base uint32_t * base = soclib_pic_xcu_base(); // write to WTI mailbox[cxy][lid] hal_remote_sw( XPTR( cxy , &base[(XCU_WTI_REG << 5) | lid] ) , 0 ); }