alioth/src/virtio/queue/packed.rs - 957f63d

packed.rs90.76%

// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
 
use std::marker::PhantomData;
use std::sync::atomic::Ordering;
 
use bitfield::bitfield;
use zerocopy::{FromBytes, Immutable, IntoBytes};
 
use crate::consts;
use crate::mem::mapped::Ram;
use crate::virtio::Result;
use crate::virtio::queue::{DescChain, DescFlag, QueueReg, VirtQueue};
 
#[repr(C, align(16))]
#[derive(Debug, Clone, Default, FromBytes, Immutable, IntoBytes)]
struct Desc {
    pub addr: u64,
    pub len: u32,
    pub id: u16,
    pub flag: u16,
}
 
bitfield! {
    #[derive(Copy, Clone, Default, PartialEq, Eq, Hash)]
    pub struct WrappedIndex(u16);
    impl Debug;
    pub u16, offset, set_offset : 14, 0;
    pub wrap_counter, set_warp_counter: 15;
}
 
impl WrappedIndex {
    const INIT: WrappedIndex = WrappedIndex(1 << 15);
 
    fn wrapping_add(&self, delta: u16, size: u16) -> WrappedIndex {39x
        let mut offset = self.offset() + delta;39x
        let mut wrap_counter = self.wrap_counter();39x
        if offset >= size {39x
            offset -= size;6x
            wrap_counter = !wrap_counter;6x
        }33x
        let mut r = WrappedIndex(offset);39x
        r.set_warp_counter(wrap_counter);39x
        r39x
    }39x
 
    fn wrapping_sub(&self, delta: u16, size: u16) -> WrappedIndex {36x
        let mut offset = self.offset();36x
        let mut wrap_counter = self.wrap_counter();36x
        if offset >= delta {36x
            offset -= delta;18x
        } else {18x
            offset += size - delta;18x
            wrap_counter = !wrap_counter;18x
        }18x
        let mut r = WrappedIndex(offset);36x
        r.set_warp_counter(wrap_counter);36x
        r36x
    }36x
}
 
consts! {
    struct EventFlag(u16) {
        ENABLE = 0;
        DISABLE = 1;
        DESC = 2;
    }
}
 
struct DescEvent {
    index: WrappedIndex,
    flag: EventFlag,
}
 
#[derive(Debug)]
pub struct PackedQueue<'m> {
    size: u16,
    desc: *mut Desc,
    enable_event_idx: bool,
    notification: *mut DescEvent,
    interrupt: *mut DescEvent,
    _phantom: PhantomData<&'m ()>,
}
 
impl<'m> PackedQueue<'m> {
    pub fn new(reg: &QueueReg, ram: &'m Ram, event_idx: bool) -> Result<Option<PackedQueue<'m>>> {51x
        if !reg.enabled.load(Ordering::Acquire) {51x
            return Ok(None);3x
        }48x
        let size = reg.size.load(Ordering::Acquire);48x
        let desc = reg.desc.load(Ordering::Acquire);48x
        let notification: *mut DescEvent = ram.get_ptr(reg.device.load(Ordering::Acquire))?;48x
        Ok(Some(PackedQueue {
            size,48x
            desc: ram.get_ptr(desc)?,48x
            enable_event_idx: event_idx,48x
            notification,48x
            interrupt: ram.get_ptr(reg.driver.load(Ordering::Acquire))?,48x
            _phantom: PhantomData,48x
        }))
    }51x
 
    fn flag_is_avail(&self, flag: DescFlag, wrap_counter: bool) -> bool {12x
        flag.contains(DescFlag::AVAIL) == wrap_counter12x
            && flag.contains(DescFlag::USED) != wrap_counter9x
    }12x
 
    fn set_flag_used(&self, flag: &mut DescFlag, wrap_counter: bool) {6x
        if wrap_counter {6x
            flag.insert(DescFlag::USED | DescFlag::AVAIL);6x
        } else {6x
            flag.remove(DescFlag::USED | DescFlag::AVAIL);
        }
    }6x
}
 
impl<'m> VirtQueue<'m> for PackedQueue<'m> {
    type Index = WrappedIndex;
 
    const INIT_INDEX: WrappedIndex = WrappedIndex::INIT;
 
    fn desc_avail(&self, index: WrappedIndex) -> bool {12x
        self.flag_is_avail(12x
            DescFlag::from_bits_retain(unsafe { &*self.desc.offset(index.offset() as isize) }.flag),12x
            index.wrap_counter(),12x
        )
    }12x
 
    fn get_avail(&self, index: Self::Index, ram: &'m Ram) -> Result<Option<DescChain<'m>>> {9x
        if !self.desc_avail(index) {9x
            return Ok(None);3x
        }6x
        let mut readable = Vec::new();6x
        let mut writeable = Vec::new();6x
        let mut delta = 0;6x
        let mut offset = index.offset();6x
        let id = loop {6x
            let desc = unsafe { &*self.desc.offset(offset as isize) };9x
            let flag = DescFlag::from_bits_retain(desc.flag);9x
            if flag.contains(DescFlag::INDIRECT) {9x
                for i in 0..(desc.len as usize / size_of::<Desc>()) {
                    let addr = desc.addr + (i * size_of::<Desc>()) as u64;
                    let desc: Desc = ram.read_t(addr)?;
                    let flag = DescFlag::from_bits_retain(desc.flag);
                    if flag.contains(DescFlag::WRITE) {
                        writeable.push((desc.addr, desc.len as u64));
                    } else {
                        readable.push((desc.addr, desc.len as u64));
                    }
                }
            } else if flag.contains(DescFlag::WRITE) {9x
                writeable.push((desc.addr, desc.len as u64));3x
            } else {6x
                readable.push((desc.addr, desc.len as u64));6x
            }6x
            delta += 1;9x
            if !flag.contains(DescFlag::NEXT) {9x
                break desc.id;6x
            }3x
            offset = (offset + 1) % self.size;3x
        };
        Ok(Some(DescChain {
            id,6x
            delta,6x
            readable: ram.translate_iov(&readable)?,6x
            writable: ram.translate_iov_mut(&writeable)?,6x
        }))
    }9x
 
    fn set_used(&self, index: Self::Index, id: u16, len: u32) {6x
        let first = unsafe { &mut *self.desc.offset(index.offset() as isize) };6x
        first.id = id;6x
        first.len = len;6x
        let mut flag = DescFlag::from_bits_retain(first.flag);6x
        self.set_flag_used(&mut flag, index.wrap_counter());6x
        first.flag = flag.bits();6x
    }6x
 
    fn enable_notification(&self, enabled: bool) {6x
        unsafe {
            (&mut *self.notification).flag = if enabled {6x
                EventFlag::ENABLE3x
            } else {
                EventFlag::DISABLE3x
            };
        }
    }6x
 
    fn interrupt_enabled(&self, index: Self::Index, delta: u16) -> bool {39x
        let interrupt = unsafe { &*self.interrupt };39x
        if self.enable_event_idx && interrupt.flag == EventFlag::DESC {39x
            let prev_used_index = index.wrapping_sub(delta, self.size);24x
            let base = prev_used_index.offset();24x
            let end = base + delta;24x
            let mut offset = interrupt.index.offset();24x
            if interrupt.index.wrap_counter() != prev_used_index.wrap_counter() {24x
                offset += self.size;9x
            }15x
            base <= offset && offset < end24x
        } else {
            interrupt.flag == EventFlag::ENABLE15x
        }
    }39x
 
    fn index_add(&self, index: Self::Index, delta: u16) -> Self::Index {18x
        index.wrapping_add(delta, self.size)18x
    }18x
}
 
#[cfg(test)]
#[path = "packed_test.rs"]
mod tests;
 