alioth/src/pci/config.rs - PR #438

config.rs92.52%

// Copyright 2024 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
 
use std::cmp::max;
use std::iter::zip;
use std::mem::size_of;
use std::sync::Arc;
 
use alioth_macros::Layout;
use parking_lot::RwLock;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
 
use crate::mem::MemRegionCallback;
use crate::mem::addressable::SlotBackend;
use crate::mem::emulated::{Action, ChangeLayout, Mmio};
use crate::pci::cap::PciCapList;
use crate::pci::{Bdf, PciBar, Result};
use crate::{assign_bits, bitflags, consts, impl_mmio_for_zerocopy, mask_bits, mem};
 
pub trait PciConfigArea: Mmio {
    fn reset(&self) -> Result<()>;
}
 
impl Mmio for Box<dyn PciConfigArea> {
    fn read(&self, offset: u64, size: u8) -> mem::Result<u64> {
        Mmio::read(self.as_ref(), offset, size)
    }
 
    fn write(&self, offset: u64, size: u8, val: u64) -> mem::Result<Action> {
        Mmio::write(self.as_ref(), offset, size, val)
    }
 
    fn size(&self) -> u64 {
        Mmio::size(self.as_ref())
    }
}
 
impl SlotBackend for Box<dyn PciConfigArea> {
    fn size(&self) -> u64 {
        Mmio::size(self.as_ref())
    }
}
 
bitflags! {
    #[derive(Default, FromBytes, Immutable, KnownLayout, IntoBytes)]
    pub struct Command(u16) {
        INTX_DISABLE = 1 << 10;
        SERR = 1 << 8;
        PARITY_ERR = 1 << 6;
        BUS_MASTER = 1 << 2;
        MEM = 1 << 1;
        IO = 1 << 0;
        WRITABLE_BITS = Self::INTX_DISABLE.bits()
            | Self::SERR.bits()
            | Self::PARITY_ERR.bits()
            | Self::BUS_MASTER.bits()
            | Self::MEM.bits()
            | Self::IO.bits();
    }
}
 
bitflags! {
    #[derive(Default, FromBytes, Immutable, KnownLayout, IntoBytes)]
    pub struct Status(u16) {
        PARITY_ERR = 1 << 15;
        SYSTEM_ERR = 1 << 14;
        RECEIVED_MASTER_ABORT = 1 << 13;
        RECEIVED_TARGET_ABORT = 1 << 12;
        SIGNALED_TARGET_ABORT = 1 << 11;
        MASTER_PARITY_ERR = 1 << 8;
        CAP = 1 << 4;
        INTX = 1 << 3;
        IMMEDIATE_READINESS = 1 << 0;
        RW1C_BITS = Self::PARITY_ERR.bits()
            | Self::SYSTEM_ERR.bits()
            | Self::RECEIVED_MASTER_ABORT.bits()
            | Self::RECEIVED_TARGET_ABORT.bits()
            | Self::SIGNALED_TARGET_ABORT.bits()
            | Self::MASTER_PARITY_ERR.bits();
    }
}
 
consts! {
    #[derive(Default, FromBytes, Immutable, KnownLayout, IntoBytes)]
    pub struct HeaderType(u8) {
        DEVICE = 0;
        BRIDGE = 1;
    }
}
 
#[derive(Debug, Clone, Default, FromBytes, Immutable, KnownLayout, IntoBytes, Layout)]
#[repr(C, align(8))]
pub struct CommonHeader {
    pub vendor: u16,
    pub device: u16,
    pub command: Command,
    pub status: Status,
    pub revision: u8,
    pub prog_if: u8,
    pub subclass: u8,
    pub class: u8,
    pub cache_line_size: u8,
    pub latency_timer: u8,
    pub header_type: HeaderType,
    pub bist: u8,
}
 
#[derive(Debug, Clone, Default, FromBytes, Immutable, KnownLayout, IntoBytes, Layout)]
#[repr(C, align(8))]
pub struct DeviceHeader {
    pub common: CommonHeader,
    pub bars: [u32; 6],
    pub cardbus_cis_pointer: u32,
    pub subsystem_vendor: u16,
    pub subsystem: u16,
    pub expansion_rom: u32,
    pub capability_pointer: u8,
    pub reserved: [u8; 7],
    pub intx_line: u8,
    pub intx_pin: u8,
    pub min_gnt: u8,
    pub max_lat: u8,
}
impl_mmio_for_zerocopy!(DeviceHeader);
 
pub const fn offset_bar(index: usize) -> usize {108x
    DeviceHeader::OFFSET_BARS + index * size_of::<u32>()108x
}108x
 
pub const OFFSET_BAR0: usize = offset_bar(0);
pub const OFFSET_BAR5: usize = offset_bar(5);
 
pub const BAR_PREFETCHABLE: u32 = 0b1000;
pub const BAR_MEM64: u32 = 0b0100;
pub const BAR_MEM32: u32 = 0b0000;
pub const BAR_IO: u32 = 0b01;
 
pub const BAR_IO_MASK: u32 = 0b11;
pub const BAR_MEM_MASK: u32 = 0b1111;
 
#[derive(Debug)]
pub enum ConfigHeader {
    Device(DeviceHeader),
}
 
impl ConfigHeader {
    pub fn bars(&self) -> [u32; 6] {143x
        match self {143x
            ConfigHeader::Device(header) => header.bars,143x
        }
    }143x
}
 
#[derive(Debug)]
struct UpdateCommandCallback {
    pci_bars: [PciBar; 6],
    bars: [u32; 6],
    changed: Command,
    current: Command,
}
 
impl ChangeLayout for UpdateCommandCallback {
    fn change(&self, memory: &mem::Memory) -> mem::Result<()> {12x
        for (i, (pci_bar, bar)) in zip(&self.pci_bars, self.bars).enumerate() {72x
            match pci_bar {72x
                PciBar::Empty => {}36x
                PciBar::Mem(region) => {24x
                    if !self.changed.contains(Command::MEM) {24x
                        continue;12x
                    }12x
                    let mut addr = (bar & !BAR_MEM_MASK) as u64;12x
                    if bar & BAR_MEM64 == BAR_MEM64 {12x
                        addr |= (self.bars[i + 1] as u64) << 32;6x
                    }6x
                    if self.current.contains(Command::MEM) {12x
                        memory.add_region(addr, region.clone())?;6x
                    } else {
                        memory.remove_region(addr)?;6x
                    }
                }
                PciBar::Io(region) => {12x
                    if !self.changed.contains(Command::IO) {12x
                        continue;6x
                    }6x
                    let port = (bar & !BAR_IO_MASK) as u16;6x
                    if self.current.contains(Command::IO) {6x
                        memory.add_io_region(port, region.clone())?;3x
                    } else {
                        memory.remove_io_region(port)?;3x
                    }
                }
            }
        }
        Ok(())12x
    }12x
}
 
#[derive(Debug)]
struct MoveBarCallback {
    bdf: Bdf,
    src: u64,
    dst: u64,
}
 
impl ChangeLayout for MoveBarCallback {
    fn change(&self, memory: &mem::Memory) -> mem::Result<()> {9x
        log::debug!(9x
            "{}: moving bar from {:#x} to {:#x}...",
            self.bdf,
            self.src,
            self.dst
        );
        if self.src as u32 & BAR_IO == BAR_IO {9x
            let src_port = self.src & !(BAR_IO_MASK as u64);3x
            let dst_port = self.dst & !(BAR_IO_MASK as u64);3x
            let region = memory.remove_io_region(src_port as u16)?;3x
            memory.add_io_region(dst_port as u16, region)?;3x
        } else {
            let src_addr = self.src & !(BAR_MEM_MASK as u64);6x
            let dst_addr = self.dst & !(BAR_MEM_MASK as u64);6x
            let region = memory.remove_region(src_addr)?;6x
            memory.add_region(dst_addr, region)?;6x
        }
        Ok(())9x
    }9x
}
 
#[derive(Debug)]
pub struct HeaderData {
    header: ConfigHeader,
    bar_masks: [u32; 6],
    bdf: Bdf,
}
 
impl HeaderData {
    pub fn set_bar(&mut self, index: usize, val: u32) -> (u32, u32) {24x
        match &mut self.header {24x
            ConfigHeader::Device(header) => {24x
                let mask = self.bar_masks[index];24x
                let old_val = header.bars[index];24x
                let masked_val = mask_bits!(old_val, val, mask);24x
                header.bars[index] = masked_val;24x
                log::info!(24x
                    "{}: bar {index}: set to {val:#010x}, update: {old_val:#010x} -> {masked_val:#010x}",
                    self.bdf
                );
                (old_val, masked_val)24x
            }
        }
    }24x
 
    pub fn get_bar(&self, index: usize) -> (u32, u32) {18x
        match &self.header {18x
            ConfigHeader::Device(header) => (header.bars[index], self.bar_masks[index]),18x
        }
    }18x
 
    pub fn set_command(&mut self, command: Command) {39x
        match &mut self.header {39x
            ConfigHeader::Device(header) => header.common.command = command,39x
        }
    }39x
 
    fn write_header(78x
        &mut self,78x
        offset: u64,78x
        size: u8,78x
        val: u64,78x
        pci_bars: &[PciBar; 6],78x
    ) -> Option<Box<dyn ChangeLayout>> {78x
        let bdf = self.bdf;78x
        let offset = offset as usize;78x
        match &mut self.header {78x
            ConfigHeader::Device(header) => match (offset, size as usize) {78x
                CommonHeader::LAYOUT_COMMAND => {
                    let val = Command::from_bits_retain(val as u16);24x
                    let old = header.common.command;24x
                    assign_bits!(header.common.command, val, Command::WRITABLE_BITS);24x
                    let current = header.common.command;24x
                    log::trace!("{bdf}: write command: {val:x?}\n   {old:x?}\n-> {current:x?}",);24x
                    let changed = old ^ current;24x
                    if !(changed & (Command::MEM | Command::IO)).is_empty() {24x
                        Some(Box::new(UpdateCommandCallback {21x
                            pci_bars: pci_bars.clone(),21x
                            bars: header.bars,21x
                            changed,21x
                            current,21x
                        }))21x
                    } else {
                        None3x
                    }
                }
                CommonHeader::LAYOUT_STATUS => {
                    let val = Status::from_bits_retain(val as u16);12x
                    let old = header.common.status;12x
                    header.common.status &= !(val & Status::RW1C_BITS);12x
                    log::trace!(12x
                        "{bdf}: write status: {val:x?}\n   {old:x?}\n-> {:x?}",
                        header.common.status,
                    );
                    None12x
                }
                (OFFSET_BAR0..=OFFSET_BAR5, 4) => {42x
                    let bar_index = (offset - OFFSET_BAR0) >> 2;42x
 
                    let mask = self.bar_masks[bar_index];42x
                    let old_val = header.bars[bar_index];42x
                    let masked_val = mask_bits!(old_val, val as u32, mask);42x
                    if old_val == masked_val {42x
                        return None;9x
                    }33x
                    log::info!(33x
                        "{bdf}: updating bar {bar_index}: {old_val:#010x} -> {masked_val:#010x}, mask={mask:#010x}",
                    );
                    let command = header.common.command;33x
                    match &pci_bars[bar_index] {33x
                        PciBar::Io(_) if command.contains(Command::IO) => {9x
                            Some(Box::new(MoveBarCallback {6x
                                bdf,6x
                                src: old_val as u64,6x
                                dst: masked_val as u64,6x
                            }))6x
                        }
                        PciBar::Mem(_) if command.contains(Command::MEM) => {18x
                            let hi_32 = if old_val & BAR_MEM64 == BAR_MEM64 {9x
                                (header.bars[bar_index + 1] as u64) << 323x
                            } else {
                                06x
                            };
                            Some(Box::new(MoveBarCallback {9x
                                bdf,9x
                                src: old_val as u64 | hi_32,9x
                                dst: masked_val as u64 | hi_32,9x
                            }))9x
                        }
                        PciBar::Empty
                            if command.contains(Command::MEM)6x
                                && bar_index > 06x
                                && header.bars[bar_index - 1] & BAR_MEM64 == BAR_MEM64 =>6x
                        {
                            let lo_32 = header.bars[bar_index - 1] as u64;6x
                            Some(Box::new(MoveBarCallback {6x
                                bdf,6x
                                src: lo_32 | ((old_val as u64) << 32),6x
                                dst: lo_32 | ((masked_val as u64) << 32),6x
                            }))6x
                        }
                        _ => {
                            header.bars[bar_index] = masked_val;12x
                            log::info!(12x
                                "{bdf}: bar {bar_index}: write {val:#010x}, update: {old_val:#010x} -> {masked_val:#010x}"
                            );
                            None12x
                        }
                    }
                }
                DeviceHeader::LAYOUT_EXPANSION_ROM => {
                    log::info!("{bdf}: write {val:#010x} to expansion_rom: ignored");
                    None
                }
                _ => {
                    log::warn!(
                        "{bdf}: unknown write: offset = {offset:#x}, size = {size}, val = {val:#x}"
                    );
                    None
                }
            },
        }
    }78x
}
 
#[derive(Debug)]
struct BarCallback {
    index: u8,
    header: Arc<RwLock<HeaderData>>,
}
 
impl MemRegionCallback for BarCallback {
    fn mapped(&self, addr: u64) -> mem::Result<()> {18x
        let mut header = self.header.write();18x
        let (old, _) = header.get_bar(self.index as usize);18x
        header.set_bar(self.index as usize, addr as u32);18x
        if old & BAR_MEM64 == BAR_MEM64 {18x
            header.set_bar(self.index as usize + 1, (addr >> 32) as u32);6x
        }12x
        Ok(())18x
    }18x
}
 
#[derive(Debug)]
pub struct EmulatedHeader {
    pub data: Arc<RwLock<HeaderData>>,
    pub bars: [PciBar; 6],
}
 
impl EmulatedHeader {
    pub fn new(header: ConfigHeader, bars: [PciBar; 6]) -> Self {143x
        let mut bar_masks = [0u32; 6];143x
        let mut mask_hi = 0;143x
        for ((bar, val), mask) in bars.iter().zip(header.bars()).zip(&mut bar_masks) {858x
            match bar {858x
                PciBar::Empty => {603x
                    *mask = mask_hi;603x
                    mask_hi = 0;603x
                }603x
                PciBar::Io(region) => {60x
                    debug_assert_eq!(val & BAR_IO_MASK, BAR_IO);60x
                    let size = max(region.size().next_power_of_two(), 1 << 2);60x
                    *mask = !(size - 1) as u32;60x
                }
                PciBar::Mem(region) => {195x
                    debug_assert_ne!(val & BAR_IO_MASK, BAR_IO);195x
                    let size = max(region.size().next_power_of_two(), 4 << 10);195x
                    let mask_64 = !(size - 1);195x
                    *mask = mask_64 as u32;195x
                    if BAR_MEM64 & val == BAR_MEM64 {195x
                        mask_hi = (mask_64 >> 32) as u32;135x
                    }135x
                }
            }
        }
 
        let data = Arc::new(RwLock::new(HeaderData {143x
            header,143x
            bar_masks,143x
            bdf: Bdf(0),143x
        }));143x
 
        for (index, bar) in bars.iter().enumerate() {858x
            let callbacks = match bar {858x
                PciBar::Empty => continue,603x
                PciBar::Mem(region) => &region.callbacks,195x
                PciBar::Io(region) => &region.callbacks,60x
            };
            callbacks.lock().push(Box::new(BarCallback {255x
                index: index as u8,255x
                header: data.clone(),255x
            }));255x
        }
 
        Self { data, bars }143x
    }143x
 
    pub fn set_bdf(&self, bdf: Bdf) {24x
        self.data.write().bdf = bdf24x
    }24x
 
    pub fn set_command(&self, command: Command) {36x
        let mut header = self.data.write();36x
        header.set_command(command)36x
    }36x
}
 
impl Mmio for EmulatedHeader {
    fn size(&self) -> u64 {
        0x40
    }
 
    fn read(&self, offset: u64, size: u8) -> mem::Result<u64> {133x
        let data = self.data.read();133x
        match &data.header {133x
            ConfigHeader::Device(header) => Mmio::read(header, offset, size),133x
        }
    }133x
 
    fn write(&self, offset: u64, size: u8, val: u64) -> mem::Result<Action> {78x
        let mut data = self.data.write();78x
        if let Some(callback) = data.write_header(offset, size, val, &self.bars) {78x
            Ok(Action::ChangeLayout { callback })42x
        } else {
            Ok(Action::None)36x
        }
    }78x
}
 
impl PciConfigArea for EmulatedHeader {
    fn reset(&self) -> Result<()> {3x
        let mut header = self.data.write();3x
        header.set_command(Command::empty());3x
        Ok(())3x
    }3x
}
 
pub trait PciConfig: Mmio {
    fn get_header(&self) -> &EmulatedHeader;
    fn reset(&self) -> Result<()>;
}
 
#[derive(Debug)]
pub struct EmulatedConfig {
    pub header: EmulatedHeader,
    pub caps: PciCapList,
}
 
impl Mmio for EmulatedConfig {
    fn read(&self, offset: u64, size: u8) -> mem::Result<u64> {82x
        if offset < size_of::<DeviceHeader>() as u64 {82x
            self.header.read(offset, size)64x
        } else {
            self.caps.read(offset, size)18x
        }
    }82x
 
    fn write(&self, offset: u64, size: u8, val: u64) -> mem::Result<Action> {12x
        if offset < size_of::<DeviceHeader>() as u64 {12x
            self.header.write(offset, size, val)9x
        } else {
            self.caps.write(offset, size, val)3x
        }
    }12x
 
    fn size(&self) -> u64 {3x
        40963x
    }3x
}
 
impl EmulatedConfig {
    pub fn new_device(83x
        mut header: DeviceHeader,83x
        bars: [PciBar; 6],83x
        caps: PciCapList,83x
    ) -> EmulatedConfig {83x
        if !caps.is_empty() {83x
            header.common.status |= Status::CAP;3x
            header.capability_pointer = size_of::<DeviceHeader>() as u8;3x
        }80x
        EmulatedConfig {83x
            header: EmulatedHeader::new(ConfigHeader::Device(header), bars),83x
            caps,83x
        }83x
    }83x
}
 
impl PciConfig for EmulatedConfig {
    fn get_header(&self) -> &EmulatedHeader {3x
        &self.header3x
    }3x
 
    fn reset(&self) -> Result<()> {3x
        self.header.reset()?;3x
        self.caps.reset()3x
    }3x
}
 
#[cfg(test)]
#[path = "config_test.rs"]
mod tests;
 