alioth/src/pci/cap.rs - 93f4782

cap.rs98.38%

// Copyright 2024 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
 
use std::cmp::min;
use std::fmt::Debug;
use std::mem::size_of;
 
use alioth_macros::Layout;
use bitfield::bitfield;
use parking_lot::RwLock;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
 
use crate::errors::BoxTrace;
use crate::hv::{self, IrqFd};
use crate::mem::addressable::SlotBackend;
use crate::mem::emulated::{Action, Mmio, MmioBus};
use crate::pci::config::{DeviceHeader, PciConfigArea};
use crate::pci::{self, Error, Result};
use crate::utils::truncate_u64;
use crate::{align_up, consts, impl_mmio_for_zerocopy, mask_bits, mem};
 
consts! {
    #[derive(Default, FromBytes, Immutable, IntoBytes, KnownLayout)]
    pub struct PciCapId(u8) {
        MSI = 0x05;
        VENDOR = 0x09;
        MSIX = 0x11;
    }
}
 
#[repr(C)]
#[derive(Debug, Default, Clone, FromBytes, Immutable, IntoBytes, KnownLayout, Layout)]
pub struct PciCapHdr {
    pub id: PciCapId,
    pub next: u8,
}
 
bitfield! {
    #[derive(Copy, Clone, Default)]
    #[repr(C)]
    pub struct PcieExtCapHdr(u32);
    impl Debug;
    pub next, _: 31,20;
    pub version, _: 19,16;
    pub id, _: 15,0;
}
 
bitfield! {
    #[derive(Copy, Clone, Default, FromBytes, Immutable, IntoBytes, KnownLayout)]
    #[repr(C)]
    pub struct MsiMsgCtrl(u16);
    impl Debug;
    pub enable, set_enable: 0;
    pub multi_msg_cap, set_multi_msg_cap: 3, 1;
    pub multi_msg, set_multi_msg: 6, 4;
    pub addr_64_cap, set_addr_64_cap: 7;
    pub per_vector_masking_cap, set_per_vector_masking_cap: 8;
    pub ext_msg_data_cap, set_ext_msg_data_cap: 9;
    pub ext_msg_data, set_ext_msg_data: 10;
}
 
impl MsiMsgCtrl {
    pub fn cap_size(&self) -> u8 {21x
        let mut size = 12;21x
        if self.addr_64_cap() {21x
            size += 4;9x
        }12x
        if self.per_vector_masking_cap() {21x
            size += 8;12x
        }12x
        size21x
    }21x
}
 
#[derive(Debug, Default, Clone, FromBytes, Immutable, IntoBytes, Layout)]
#[repr(C)]
pub struct MsiCapHdr {
    pub header: PciCapHdr,
    pub control: MsiMsgCtrl,
}
impl_mmio_for_zerocopy!(MsiCapHdr);
 
#[derive(Debug, Default, Clone, FromBytes, Immutable, IntoBytes, Layout)]
#[repr(C)]
struct MsiCapBody {
    data: [u32; 4],
}
impl_mmio_for_zerocopy!(MsiCapBody);
 
#[derive(Debug)]
pub struct MsiCapMmio<F>
where
    F: IrqFd,
{
    cap: RwLock<(MsiCapHdr, MsiCapBody)>,
    irqfds: Box<[F]>,
}
 
impl<F> MsiCapMmio<F>
where
    F: IrqFd,
{
    pub fn new(ctrl: MsiMsgCtrl, irqfds: Box<[F]>) -> Self {9x
        debug_assert_eq!(irqfds.len(), 1 << ctrl.multi_msg_cap());9x
        let cap = RwLock::new((9x
            MsiCapHdr {9x
                header: PciCapHdr {9x
                    id: PciCapId::MSI,9x
                    next: 0,9x
                },9x
                control: ctrl,9x
            },9x
            MsiCapBody::default(),9x
        ));9x
        Self { cap, irqfds }9x
    }9x
 
    fn update_msi(&self) -> hv::Result<()> {30x
        let (hdr, body) = &*self.cap.read();30x
        let ctrl = &hdr.control;30x
        let data = &body.data;30x
        let msg_mask = if ctrl.ext_msg_data() {30x
            0xffff_ffff12x
        } else {
            0xffff18x
        };
        let (addr, msg) = if ctrl.addr_64_cap() {30x
            (12x
                ((data[1] as u64) << 32) | data[0] as u64,12x
                data[2] & msg_mask,12x
            )12x
        } else {
            (data[0] as u64, data[1] & msg_mask)18x
        };
        let mask = match (ctrl.addr_64_cap(), ctrl.per_vector_masking_cap()) {30x
            (true, true) => data[3],12x
            (false, true) => data[2],9x
            (_, false) => 0,9x
        };
        let count = 1 << ctrl.multi_msg();30x
        for (index, irqfd) in self.irqfds.iter().enumerate() {408x
            irqfd.set_masked(true)?;408x
            if !ctrl.enable() || index >= count || mask & (1 << index) > 0 {408x
                continue;222x
            }186x
            let msg = msg | index as u32;186x
            irqfd.set_addr_hi((addr >> 32) as u32)?;186x
            irqfd.set_addr_lo(addr as u32)?;186x
            irqfd.set_data(msg)?;186x
            irqfd.set_masked(false)?;186x
        }
        Ok(())30x
    }30x
}
 
impl<F> Mmio for MsiCapMmio<F>
where
    F: IrqFd,
{
    fn size(&self) -> u64 {9x
        let (hdr, _) = &*self.cap.read();9x
        hdr.control.cap_size() as u649x
    }9x
 
    fn read(&self, offset: u64, size: u8) -> mem::Result<u64> {27x
        let (hdr, body) = &*self.cap.read();27x
        if offset < 4 {27x
            hdr.read(offset, size)15x
        } else {
            body.read(offset - size_of_val(hdr) as u64, size)12x
        }
    }27x
 
    fn write(&self, offset: u64, size: u8, val: u64) -> mem::Result<Action> {42x
        let mut need_update = false;42x
        let mut cap = self.cap.write();42x
        let (hdr, body) = &mut *cap;42x
        let ctrl = &mut hdr.control;42x
        let addr_64_cap = ctrl.addr_64_cap();42x
        let per_vector_masking_cap = ctrl.per_vector_masking_cap();42x
        match (offset, size, addr_64_cap, per_vector_masking_cap) {42x
            (0x2, 2, _, _) | (0x0, 4, _, _) => {
                let new_ctrl = MsiMsgCtrl((val >> ((2 - offset) << 3)) as u16);12x
 
                if !ctrl.enable() || !new_ctrl.enable() {12x
                    let multi_msg = min(ctrl.multi_msg_cap(), new_ctrl.multi_msg());12x
                    ctrl.set_multi_msg(multi_msg);12x
                }12x
 
                let ext_msg_data = ctrl.ext_msg_data_cap() && new_ctrl.ext_msg_data();12x
                need_update |= new_ctrl.enable() && ctrl.ext_msg_data() != ext_msg_data;12x
                ctrl.set_ext_msg_data(ext_msg_data);12x
 
                need_update |= ctrl.enable() != new_ctrl.enable();12x
                ctrl.set_enable(new_ctrl.enable());12x
            }
            (0x4, 4, _, _) | (0x8, 4, true, _) | (0xc, 4, false, true) | (0x10, 4, true, true) => {
                let data_offset = (offset as usize - size_of::<MsiCapHdr>()) >> 2;18x
                let reg = &mut body.data[data_offset];18x
                need_update = hdr.control.enable() && *reg != val as u32;18x
                *reg = val as u32;18x
            }
            (0x8, 2 | 4, false, _) | (0xc, 2 | 4, true, _) => {
                let data_offset = (offset as usize - size_of::<MsiCapHdr>()) >> 2;9x
                let reg = &mut body.data[data_offset];9x
                let mask = if size == 4 && hdr.control.ext_msg_data_cap() {9x
                    0xffff_ffff6x
                } else {
                    0xffff3x
                };
                let new_val = mask_bits!(*reg, val as u32, mask);9x
                need_update = hdr.control.enable() && *reg != new_val;9x
                *reg = new_val;9x
            }
            _ => log::error!(3x
                "MsiCapMmio: write 0x{val:0width$x} to invalid offset 0x{offset:x}.",
                width = 2 * size as usize3x
            ),
        }
        drop(cap);42x
        if need_update {42x
            self.update_msi().box_trace(mem::error::Mmio)?;27x
        }15x
        Ok(Action::None)42x
    }42x
}
 
impl<F> PciCap for MsiCapMmio<F>
where
    F: IrqFd,
{
    fn set_next(&mut self, val: u8) {3x
        let (hdr, _) = self.cap.get_mut();3x
        hdr.header.next = val;3x
    }3x
}
 
impl<F> PciConfigArea for MsiCapMmio<F>
where
    F: IrqFd,
{
    fn reset(&self) -> pci::Result<()> {3x
        {3x
            let (hdr, _) = &mut *self.cap.write();3x
            hdr.control.set_enable(false);3x
        }3x
        self.update_msi().box_trace(pci::error::Reset)3x
    }3x
}
 
bitfield! {
    #[derive(Copy, Clone, Default, FromBytes, Immutable, IntoBytes, KnownLayout)]
    #[repr(C)]
    pub struct MsixMsgCtrl(u16);
    impl Debug;
    pub table_len, _ : 10, 0;
    pub masked, set_masked: 14;
    pub enabled, set_enabled: 15;
}
 
impl MsixMsgCtrl {
    pub fn new(len: u16) -> Self {9x
        MsixMsgCtrl(len - 1)9x
    }9x
}
 
bitfield! {
    #[derive(Copy, Clone, Default, FromBytes, Immutable, IntoBytes)]
    #[repr(C)]
    pub struct MsixCapOffset(u32);
    impl Debug;
    pub bar, _: 2, 0;
}
 
impl MsixCapOffset {
    pub fn new(offset: u32, bar: u8) -> Self {6x
        MsixCapOffset(mask_bits!(offset, bar as u32, 0b111))6x
    }6x
 
    pub fn offset(&self) -> u32 {6x
        self.0 & !0b1116x
    }6x
}
 
#[derive(Debug, Default, Clone, FromBytes, Immutable, IntoBytes, Layout)]
#[repr(C)]
pub struct MsixCap {
    pub header: PciCapHdr,
    pub control: MsixMsgCtrl,
    pub table_offset: MsixCapOffset,
    pub pba_offset: MsixCapOffset,
}
impl_mmio_for_zerocopy!(MsixCap);
 
bitfield! {
    #[derive(Copy, Clone, Default)]
    #[repr(C)]
    pub struct MsixVectorCtrl(u32);
    impl Debug;
    pub masked, set_masked: 0;
}
 
#[derive(Debug, Clone)]
pub struct MsixTableEntry {
    pub addr_lo: u32,
    pub addr_hi: u32,
    pub data: u32,
    pub control: MsixVectorCtrl,
}
 
impl Default for MsixTableEntry {
    fn default() -> Self {9x
        MsixTableEntry {9x
            addr_lo: 0,9x
            addr_hi: 0,9x
            data: 0,9x
            control: MsixVectorCtrl(1),9x
        }9x
    }9x
}
 
pub trait PciCap: PciConfigArea {
    fn set_next(&mut self, val: u8);
}
 
impl SlotBackend for Box<dyn PciCap> {
    fn size(&self) -> u64 {60x
        Mmio::size(self.as_ref())60x
    }60x
}
 
impl Mmio for Box<dyn PciCap> {
    fn read(&self, offset: u64, size: u8) -> mem::Result<u64> {36x
        Mmio::read(self.as_ref(), offset, size)36x
    }36x
 
    fn write(&self, offset: u64, size: u8, val: u64) -> mem::Result<Action> {6x
        Mmio::write(self.as_ref(), offset, size, val)6x
    }6x
 
    fn size(&self) -> u64 {48x
        Mmio::size(self.as_ref())48x
    }48x
}
 
#[derive(Debug)]
pub struct PciCapList {
    inner: MmioBus<Box<dyn PciCap>>,
}
 
impl Default for PciCapList {
    fn default() -> Self {3x
        Self::new()3x
    }3x
}
 
impl PciCapList {
    pub fn new() -> PciCapList {83x
        Self {83x
            inner: MmioBus::new(),83x
        }83x
    }83x
 
    pub fn is_empty(&self) -> bool {89x
        self.inner.is_empty()89x
    }89x
}
 
impl PciConfigArea for PciCapList {
    fn reset(&self) -> Result<()> {6x
        for (_, cap) in self.inner.inner.iter() {12x
            cap.reset()?;12x
        }
        Ok(())6x
    }6x
}
 
impl Mmio for PciCapList {
    fn read(&self, offset: u64, size: u8) -> Result<u64, mem::Error> {36x
        self.inner.read(offset, size)36x
    }36x
 
    fn write(&self, offset: u64, size: u8, val: u64) -> mem::Result<Action> {6x
        self.inner.write(offset, size, val)6x
    }6x
 
    fn size(&self) -> u64 {3x
        40963x
    }3x
}
 
impl TryFrom<Vec<Box<dyn PciCap>>> for PciCapList {
    type Error = Error;
 
    fn try_from(caps: Vec<Box<dyn PciCap>>) -> Result<Self, Self::Error> {6x
        let mut bus = MmioBus::new();6x
        let mut ptr = size_of::<DeviceHeader>() as u64;6x
        let num_caps = caps.len();6x
        for (index, mut cap) in caps.into_iter().enumerate() {12x
            let next = if index == num_caps - 1 {12x
                06x
            } else {
                align_up!(ptr + Mmio::size(&cap), 2)6x
            };
            cap.set_next(next as u8);12x
            bus.add(ptr, cap)?;12x
            ptr = next;12x
        }
        Ok(Self { inner: bus })6x
    }6x
}
 
#[derive(Debug)]
pub struct MsixCapMmio {
    cap: RwLock<MsixCap>,
}
 
impl MsixCapMmio {
    pub fn new(cap: MsixCap) -> Self {6x
        Self {6x
            cap: RwLock::new(cap),6x
        }6x
    }6x
}
 
impl Mmio for MsixCapMmio {
    fn size(&self) -> u64 {81x
        size_of::<MsixCap>() as u6481x
    }81x
 
    fn read(&self, offset: u64, size: u8) -> Result<u64, mem::Error> {42x
        let cap = self.cap.read();42x
        Mmio::read(&*cap, offset, size)42x
    }42x
 
    fn write(&self, offset: u64, size: u8, val: u64) -> mem::Result<Action> {15x
        if offset == 2 && size == 2 {15x
            let mut cap = self.cap.write();9x
            let control = MsixMsgCtrl(val as u16);9x
            cap.control.set_enabled(control.enabled());9x
            cap.control.set_masked(control.masked());9x
        }9x
        Ok(Action::None)15x
    }15x
}
 
impl PciConfigArea for MsixCapMmio {
    fn reset(&self) -> pci::Result<()> {9x
        let mut cap = self.cap.write();9x
        cap.control.set_enabled(false);9x
        cap.control.set_masked(false);9x
        Ok(())9x
    }9x
}
 
impl PciCap for MsixCapMmio {
    fn set_next(&mut self, val: u8) {9x
        self.cap.write().header.next = val;9x
    }9x
}
 
#[derive(Debug)]
pub enum MsixTableMmioEntry<F> {
    Entry(MsixTableEntry),
    IrqFd(F),
}
 
impl<F> Default for MsixTableMmioEntry<F> {
    fn default() -> Self {9x
        MsixTableMmioEntry::Entry(MsixTableEntry::default())9x
    }9x
}
 
macro_rules! impl_msix_table_mmio_entry_method {
    ($field:ident, $get:ident, $set:ident) => {
        pub fn $get(&self) -> u32 {18x
            match self {18x
                MsixTableMmioEntry::Entry(e) => e.$field,9x
                MsixTableMmioEntry::IrqFd(f) => f.$get(),9x
            }
        }18x
        fn $set(&mut self, val: u32) -> mem::Result<()> {18x
            match self {18x
                MsixTableMmioEntry::Entry(e) => e.$field = val,9x
                MsixTableMmioEntry::IrqFd(f) => f.$set(val)?,9x
            }
            Ok(())18x
        }18x
    };
}
 
impl<F> MsixTableMmioEntry<F>
where
    F: IrqFd,
{
    impl_msix_table_mmio_entry_method!(addr_lo, get_addr_lo, set_addr_lo);
 
    impl_msix_table_mmio_entry_method!(addr_hi, get_addr_hi, set_addr_hi);
 
    impl_msix_table_mmio_entry_method!(data, get_data, set_data);
 
    fn set_masked(&mut self, val: bool) -> mem::Result<bool> {6x
        match self {6x
            MsixTableMmioEntry::Entry(e) => {3x
                let masked = e.control.masked();3x
                e.control.set_masked(val);3x
                Ok(masked != val)3x
            }
            MsixTableMmioEntry::IrqFd(f) => {3x
                let changed = f.set_masked(val)?;3x
                Ok(changed)3x
            }
        }
    }6x
 
    pub fn get_masked(&self) -> bool {12x
        match self {12x
            MsixTableMmioEntry::Entry(e) => e.control.masked(),6x
            MsixTableMmioEntry::IrqFd(f) => f.get_masked(),6x
        }
    }12x
}
 
#[derive(Debug)]
pub struct MsixTableMmio<F> {
    pub entries: RwLock<Box<[MsixTableMmioEntry<F>]>>,
}
 
impl<F> MsixTableMmio<F>
where
    F: IrqFd,
{
    /// Write `val` to `offset`.
    ///
    /// Returns `true` if a `masked` bit gets flipped.
    pub fn write_val(&self, offset: u64, size: u8, val: u64) -> mem::Result<bool> {33x
        if size != 4 || offset & 0b11 != 0 {33x
            log::error!("unaligned access to msix table: size = {size}, offset = {offset:#x}");6x
            return Ok(false);6x
        }27x
        let val = val as u32;27x
        let index = offset as usize / size_of::<MsixTableEntry>();27x
        let mut entries = self.entries.write();27x
        let Some(entry) = entries.get_mut(index) else {27x
            log::error!(3x
                "MSI-X table size: {}, accessing index {index}",
                entries.len()3x
            );
            return Ok(false);3x
        };
        let mut state_changed = false;24x
        match offset as usize % size_of::<MsixTableEntry>() {24x
            0 => entry.set_addr_lo(val)?,6x
            4 => entry.set_addr_hi(val)?,6x
            8 => entry.set_data(val)?,6x
            12 => state_changed = entry.set_masked(MsixVectorCtrl(val).masked())?,6x
            _ => unreachable!(),
        };
        Ok(state_changed)24x
    }33x
 
    pub fn reset(&self) {3x
        let mut entries = self.entries.write();3x
        for entry in entries.iter_mut() {6x
            *entry = MsixTableMmioEntry::default();6x
        }6x
    }3x
}
 
impl<F> Mmio for MsixTableMmio<F>
where
    F: IrqFd,
{
    fn size(&self) -> u64 {3x
        (size_of::<MsixTableEntry>() * self.entries.read().len()) as u643x
    }3x
 
    fn read(&self, offset: u64, size: u8) -> mem::Result<u64> {39x
        if size != 4 || offset & 0b11 != 0 {39x
            log::error!("unaligned access to msix table: size = {size}, offset = {offset:#x}");6x
            return Ok(0);6x
        }33x
        let index = offset as usize / size_of::<MsixTableEntry>();33x
        let entries = self.entries.read();33x
        let Some(entry) = entries.get(index) else {33x
            log::error!(3x
                "MSI-X table size: {}, accessing index {index}",
                entries.len()3x
            );
            return Ok(0);3x
        };
        let ret = match offset as usize % size_of::<MsixTableEntry>() {30x
            0 => entry.get_addr_lo(),6x
            4 => entry.get_addr_hi(),6x
            8 => entry.get_data(),6x
            12 => entry.get_masked() as _,12x
            _ => unreachable!(),
        };
        Ok(ret as u64)30x
    }39x
 
    fn write(&self, offset: u64, size: u8, val: u64) -> mem::Result<Action> {27x
        self.write_val(offset, size, val)?;27x
        Ok(Action::None)27x
    }27x
}
 
#[repr(C)]
#[derive(Debug, Default, Clone)]
pub struct NullCap {
    pub next: u8,
    pub size: u8,
}
 
impl Mmio for NullCap {
    fn read(&self, offset: u64, size: u8) -> mem::Result<u64> {39x
        let shift = std::cmp::min(63, offset << 3);39x
        let val = ((self.next as u64) << 8) >> shift;39x
        Ok(truncate_u64(val, size as u64))39x
    }39x
 
    fn write(&self, _offset: u64, _size: u8, _val: u64) -> mem::Result<Action> {
        Ok(Action::None)
    }
 
    fn size(&self) -> u64 {30x
        self.size as u6430x
    }30x
}
 
impl PciCap for NullCap {
    fn set_next(&mut self, val: u8) {6x
        self.next = val;6x
    }6x
}
 
impl PciConfigArea for NullCap {
    fn reset(&self) -> Result<()> {6x
        Ok(())6x
    }6x
}
 
#[cfg(test)]
#[path = "cap_test.rs"]
mod tests;
 