imago/format/

drivers.rs

//! Internal image format driver interface.
//!
//! Provides the internal interface for image format drivers to provide their services, on which
//! the publically visible interface [`FormatAccess`] is built.

use super::{Format, PreallocateMode};
use crate::io_buffers::IoVectorMut;
use crate::{FormatAccess, Storage};
use maybe_async::maybe_async;
use std::any::Any;
use std::fmt::{self, Debug, Display};
use std::{cmp, io};

/// Implementation of a disk image format.
#[maybe_async(?Send)]
#[allow(clippy::double_must_use)] // strange interaction between `maybe_async(?Send)` and `async`
pub trait FormatDriverInstance: Any + Debug + Display + Send + Sync {
    /// Type of storage used.
    type Storage: Storage;

    /// Return which format this is.
    fn format(&self) -> Format;

    /// Check whether `storage` has this format.
    ///
    /// This is only a rough test and does not guarantee that opening `storage` under this format
    /// will succeed.  Generally, it will only check the magic bytes (if available).  For formats
    /// that do not have distinct features (like raw), this will always return `true`.
    ///
    /// # Safety
    /// Probing is inherently dangerous: Image formats like qcow2 allow referencing external files;
    /// if you use imago to give untrusted parties (like VM guests) access to VM disk image files,
    /// this will give those parties access to data in those files.  Opening images from untrusted
    /// sources can therefore be quite dangerous.  Gating
    /// ([`ImplicitOpenGate`](super::gate::ImplicitOpenGate)) can help mitigate this.
    ///
    /// If you do not know an image’s format, that is a sign it does not come from a trusted
    /// source, and so opening it in a non-raw format may be quite dangerous.
    ///
    /// Perhaps most important to note is that giving an untrusted party (like a VM guest) access
    /// to a raw image file allows that party to modify the whole file.  It may write image headers
    /// into this image file, causing a subsequent probe operation to recognize it as a non-raw
    /// image, referencing arbitrary files on the host filesystem!
    ///
    /// When using imago to give an untrusted third party access to VM disk images, the guidelines
    /// for probing are thus:
    /// - Do not probe.  If at all possible, obtain an image’s format from a trusted side channel.
    /// - If there is no other way, probe each given image only once, before that untrusted third
    ///   party (like a VM guest) had write access to it; remember the probed format, and open the
    ///   image exclusively as that format.
    ///
    /// When working with even potentially untrusted images, you should always use an
    /// [`ImplicitOpenGate`](super::gate::ImplicitOpenGate) to prevent access to files you do not
    /// wish to access.
    async unsafe fn probe(storage: &Self::Storage) -> io::Result<bool>
    where
        Self: Sized;

    /// Size of the disk represented by this image.
    fn size(&self) -> u64;

    /// Granularity on which blocks can be marked as zero.
    ///
    /// This is the granularity for [`FormatDriverInstance::ensure_zero_mapping()`].
    ///
    /// Return `None` if zero blocks are not supported.
    fn zero_granularity(&self) -> Option<u64> {
        None
    }

    /// Recursively collect all storage objects associated with this image.
    ///
    /// “Recursive” means to recurse to other images like e.g. a backing file.
    fn collect_storage_dependencies(&self) -> Vec<&Self::Storage>;

    /// Return whether this image may be modified.
    ///
    /// This state must not change via interior mutability, i.e. as long as this FDI is wrapped in
    /// a `FormatAccess`, its writability must remain constant.
    fn writable(&self) -> bool;

    /// Return the mapping at `offset`.
    ///
    /// Find what `offset` is mapped to, return that mapping information, and the length of that
    /// continuous mapping (from `offset`).
    ///
    /// To determine that continuous mapping length, drivers should not perform additional I/O
    /// beyond what is necessary to get mapping information for `offset` itself.
    ///
    /// `max_length` is a hint how long of a range is required at all, but the returned length may
    /// exceed that value if that simplifies the implementation.
    ///
    /// The returned length must only be 0 if `ShallowMapping::Eof` is returned.
    #[allow(clippy::needless_lifetimes)] // Elidable in sync, but async needs a named lifetime for the boxed future bound
    async fn get_mapping<'a>(
        &'a self,
        offset: u64,
        max_length: u64,
    ) -> io::Result<(ShallowMapping<'a, Self::Storage>, u64)>;

    /// Ensure that `offset` is directly mapped to some storage object, up to a length of `length`.
    ///
    /// Return the storage object, the corresponding offset there, and the continuous length that
    /// the driver was able to map (less than or equal to `length`).
    ///
    /// If the returned length is less than `length`, drivers can expect subsequent calls to
    /// allocate the rest of the original range.  Therefore, if a driver knows in advance that it
    /// is impossible to fully map the given range (e.g. because it lies partially or fully beyond
    /// the end of the disk), it should return an error immediately.
    ///
    /// If `overwrite` is true, the contents in the range are supposed to be overwritten and may be
    /// discarded.  Otherwise, they must be kept.
    ///
    /// Should not break existing data mappings, i.e. not discard or repurpose existing data
    /// mappings.  Making them unused, but retaining them as allocated so they can safely be
    /// written to (albeit with no effect) is OK; discarding them so that they may be reused for
    /// other mappings is not.
    ///
    /// NOTE: Will be deprecated in imago 0.3 in favor of
    /// [`FormatDriverInstance::prepare_data_mapping()`] and
    /// [`FormatDriverInstance::commit_data_mapping()`].
    #[allow(clippy::needless_lifetimes)] // Elidable in sync, but async needs a named lifetime for the boxed future bound
    async fn ensure_data_mapping<'a>(
        &'a self,
        offset: u64,
        length: u64,
        overwrite: bool,
    ) -> io::Result<(&'a Self::Storage, u64, u64)> {
        let mapping = self.prepare_data_mapping(offset, length, overwrite).await?;
        let storage = mapping.storage();
        let host_offset = mapping.offset();
        let length = mapping.length();

        self.commit_data_mapping(mapping).await?;

        Ok((storage, host_offset, length))
    }

    /// Prepare a raw data mapping at `offset`.
    ///
    /// Ensure that `offset` is directly mapped to some storage object, up to a length of `length`.
    ///
    /// This mapping needs not be immediately visible (unless it exists already); the returned
    /// [`PreparedDataMapping`] will tell the user where to write the data for the range, and once
    /// that data has been written, [`FormatDriverInstance::commit_data_mapping()`] is called to
    /// finalize it and make the mapping actually visible.
    ///
    /// When the returned object is dropped without having been committed, the range must keep its
    /// previous mapping, and whatever was allocated for the prepared mapping must be released.
    ///
    /// Drivers may delay any other I/O on the image until the returned object is committed or
    /// dropped.
    ///
    /// If `overwrite` is true, the contents in the range are supposed to be overwritten and may be
    /// discarded.  Otherwise, they should be kept.
    ///
    /// The default implementation is based on [`FormatDriverInstance::ensure_data_mapping()`],
    /// i.e. makes the mapping visible immediately and leaves nothing to commit.
    ///
    /// NOTE: Starting from the imago 0.3 release, this function will no longer have a default
    /// implementation.  Then, *this* function will need to be implemented, *not*
    /// [`FormatDriverInstance::ensure_data_mapping()`].
    #[allow(clippy::needless_lifetimes)] // Elidable in sync, but async needs a named lifetime for the boxed future bound
    async fn prepare_data_mapping<'a>(
        &'a self,
        offset: u64,
        length: u64,
        overwrite: bool,
    ) -> io::Result<PreparedDataMapping<'a, Self::Storage>> {
        let (storage, offset, length) = self.ensure_data_mapping(offset, length, overwrite).await?;
        Ok(PreparedDataMapping::existing(storage, offset, length))
    }

    /// Make a mapping prepared by [`FormatDriverInstance::prepare_data_mapping()`] visible.
    ///
    /// Must only be called once the data has been written to the prepared location.  `mapping`
    /// must have been returned by `prepare_data_mapping()` on this same object.
    ///
    /// The default implementation matches the default implementation of
    /// [`FormatDriverInstance::prepare_data_mapping()`], which leaves nothing to commit.
    ///
    /// NOTE: Starting from the imago 0.3 release, this function will no longer have a default
    /// implementation, and will have to be implemented together with
    /// [`FormatDriverInstance::prepare_data_mapping()`].
    async fn commit_data_mapping(
        &self,
        mapping: PreparedDataMapping<'_, Self::Storage>,
    ) -> io::Result<()> {
        if mapping.into_state().is_some() {
            // The default implementation of `prepare_data_mapping()` left nothing to commit, so if
            // there is something now, it came from a different implementation.
            return Err(io::Error::new(
                io::ErrorKind::InvalidInput,
                "Prepared data mapping does not belong to this image",
            ));
        }
        Ok(())
    }

    /// Ensure that the given range is efficiently mapped as zeroes.
    ///
    /// Must not write any data.  Return the range (offset and length) that could actually be
    /// zeroed, which must be a subset of the range given by `offset` and `length`.  The returned
    /// offset must be as close to `offset` as possible, i.e. no zero mapping is possible between
    /// `offset` and the returned offset (e.g. because of format-inherent granularity).
    ///
    /// The returned length may be zero in case zeroing would theoretically be possible, but not
    /// for this range at this granularity.
    ///
    /// Should not break existing data mappings, i.e. not discard or repurpose existing data
    /// mappings.  Making them unused, but retaining them as allocated so they can safely be
    /// written to (albeit with no effect) is OK; discarding them so that they may be reused for
    /// other mappings is not.
    async fn ensure_zero_mapping(&self, _offset: u64, _length: u64) -> io::Result<(u64, u64)> {
        Err(io::ErrorKind::Unsupported.into())
    }

    /// Discard the given range, ensure it is read back as zeroes.
    ///
    /// Effectively the same as [`FormatDriverInstance::ensure_zero_mapping()`], but may break
    /// existing data mappings thanks to the mutable `self` reference, which ensures that old data
    /// mappings returned by [`FormatDriverInstance::get_mapping()`] cannot be held onto.
    async fn discard_to_zero(&mut self, offset: u64, length: u64) -> io::Result<(u64, u64)> {
        // Safe: `&mut self` guarantees nobody has concurrent data mappings
        unsafe { self.discard_to_zero_unsafe(offset, length).await }
    }

    /// Discard the given range, ensure it is read back as zeroes.
    ///
    /// Unsafe variant of [FormatDriverInstance::discard_to_zero()], only requiring an immutable
    /// &self
    ///
    /// # Safety
    /// This function is marked as unsafe because:
    /// - It may invalidate all existing data mappings.
    ///
    /// The caller must ensure that no other references to this driver instance exist and that
    /// the caller must ensure that all previously looked up mappings are no longer assumed to
    /// be valid after this operation.
    ///
    /// Because mappings contain references to the block driver instance, one way to do so is
    /// to have a mutable reference to the block driver instance, which will automatically
    /// ensure there are no other references (and thus no mappings).  In that case, you can use
    /// the safe variant [`Self::discard_to_zero()`].
    async unsafe fn discard_to_zero_unsafe(
        &self,
        _offset: u64,
        _length: u64,
    ) -> io::Result<(u64, u64)> {
        Err(io::ErrorKind::Unsupported.into())
    }

    /// Discard the given range.
    ///
    /// Effectively the same as [`FormatDriverInstance::discard_to_zero()`], but the discarded area
    /// may read as any data.  Backing file data should not reappear, however.
    async fn discard_to_any(&mut self, offset: u64, length: u64) -> io::Result<(u64, u64)> {
        // Safe: `&mut self` guarantees nobody has concurrent data mappings
        unsafe { self.discard_to_any_unsafe(offset, length).await }
    }

    /// Discard the given range.
    ///
    /// Unsafe variant of [FormatDriverInstance::discard_to_any()], only requiring an immutable
    /// &self
    ///
    /// # Safety
    /// This function is marked as unsafe because:
    /// - It may invalidate all existing data mappings.
    ///
    /// The caller must ensure that no other references to this driver instance exist and that
    /// the caller must ensure that all previously looked up mappings are no longer assumed to
    /// be valid after this operation.
    ///
    /// Because mappings contain references to the block driver instance, one way to do so is
    /// to have a mutable reference to the block driver instance, which will automatically
    /// ensure there are no other references (and thus no mappings).  In that case, you can use
    /// the safe variant [`Self::discard_to_any()`].
    async unsafe fn discard_to_any_unsafe(
        &self,
        _offset: u64,
        _length: u64,
    ) -> io::Result<(u64, u64)> {
        Err(io::ErrorKind::Unsupported.into())
    }

    /// Discard the given range, such that the backing image becomes visible.
    ///
    /// Deallocate the range such that in deallocated blocks, the backing image’s data (if one
    /// exists) will show, i.e. [`FormatDriverInstance::get_mapping()`] should return an indirect
    /// mapping.  When there is no backing image, those blocks should appear as zero.
    ///
    /// Return the range (offset and length) that could actually be discarded, which must be a
    /// subset of `offset` and `length`, and the returned offset must be as close to `offset` as
    /// possible (like for [`FormatDriverInstance::discard_to_backing()`].
    ///
    /// May break existing data mappings thanks to the mutable `self` reference.
    async fn discard_to_backing(&mut self, offset: u64, length: u64) -> io::Result<(u64, u64)> {
        // Safe: `&mut self` guarantees nobody has concurrent data mappings
        unsafe { self.discard_to_backing_unsafe(offset, length).await }
    }

    /// Discard the given range, such that the backing image becomes visible.
    ///
    /// Unsafe variant of [FormatDriverInstance::discard_to_backing()], only requiring an immutable
    /// &self
    ///
    /// # Safety
    /// This function is marked as unsafe because:
    /// - It may invalidate all existing data mappings.
    ///
    /// The caller must ensure that no other references to this driver instance exist and that
    /// the caller must ensure that all previously looked up mappings are no longer assumed to
    /// be valid after this operation.
    ///
    /// Because mappings contain references to the block driver instance, one way to do so is
    /// to have a mutable reference to the block driver instance, which will automatically
    /// ensure there are no other references (and thus no mappings).  In that case, you can use
    /// the safe variant [`Self::discard_to_backing()`].
    async unsafe fn discard_to_backing_unsafe(
        &self,
        _offset: u64,
        _length: u64,
    ) -> io::Result<(u64, u64)> {
        Err(io::ErrorKind::Unsupported.into())
    }

    /// Read data from a `ShallowMapping::Special` area.
    async fn readv_special(&self, _bufv: IoVectorMut<'_>, _offset: u64) -> io::Result<()> {
        Err(io::ErrorKind::Unsupported.into())
    }

    /// Read data from a `ShallowMapping::Special` area, returning the unhandled tail.
    ///
    /// Like [`Self::readv_special()`], but if the area turns out not to be special (anymore) from
    /// some point on (e.g. because a concurrent write has changed its mapping), stop there and
    /// return that unhandled tail of `bufv` and its offset.  That tail has not been written to, so
    /// the caller can read it according to its current (non-special) mapping.
    ///
    /// Return `None` if the whole area has been read.
    ///
    /// The default implementation calls [`Self::readv_special()`], so never returns a tail.
    ///
    /// NOTE: Starting from the imago 0.3 release, this function will no longer have a default
    /// implementation building on [`Self::readv_special()`], and instead will return an error.
    /// Then, *this* function will need to be implemented, *not* [`Self::readv_special()`].
    async fn try_readv_special<'a>(
        &self,
        bufv: IoVectorMut<'a>,
        offset: u64,
    ) -> io::Result<Option<(IoVectorMut<'a>, u64)>> {
        self.readv_special(bufv, offset).await?;
        Ok(None)
    }

    /// Flush internal buffers.
    ///
    /// Does not need to ensure those buffers are synced to disk (hardware), and does not need to
    /// drop them, i.e. they may still be used on later accesses.
    async fn flush(&self) -> io::Result<()>;

    /// Sync data already written to the storage hardware.
    ///
    /// Does not need to ensure internal buffers are written, i.e. should generally just be passed
    /// through to `Storage::sync()` for all underlying storage objects.
    async fn sync(&self) -> io::Result<()>;

    /// Drop internal buffers.
    ///
    /// Drop all internal buffers, but do not flush them!  All internal data must then be reloaded
    /// from disk.
    ///
    /// # Safety
    /// Not flushing internal buffers may cause image corruption.  The caller must ensure the
    /// on-disk state is consistent.
    async unsafe fn invalidate_cache(&self) -> io::Result<()>;

    /// Resize to the given size, which must be greater than the current size.
    ///
    /// Set the disk size to `new_size`, preallocating the new space according to `prealloc_mode`.
    /// Depending on the image format, it is possible some preallocation modes are not supported,
    /// in which case an [`std::io::ErrorKind::Unsupported`] is returned.
    ///
    /// If the current size is already `new_size` or greater, do nothing.
    async fn resize_grow(&self, new_size: u64, prealloc_mode: PreallocateMode) -> io::Result<()>;

    /// Truncate to the given size, which must be smaller than the current size.
    ///
    /// Set the disk size to `new_size`, discarding the data after `new_size`.
    ///
    /// May break existing data mappings thanks to the mutable `self` reference.
    ///
    /// If the current size is already `new_size` or smaller, do nothing.
    async fn resize_shrink(&mut self, new_size: u64) -> io::Result<()>;
}

/// Data mapping that has been prepared, but not committed yet.
///
/// Returned by [`FormatAccess::prepare_data_mapping()`]: Says where to write the data for the
/// range that is to be mapped.  Once that data has been written, pass this object to
/// [`FormatAccess::commit_data_mapping()`] to make the mapping visible.
///
/// Dropping this object without committing it discards any prepared mapping change.
///
/// Until this object is committed or dropped, any other I/O on the image may wait for it.
#[must_use = "prepared mappings must be committed to complete the write operation"]
pub struct PreparedDataMapping<'a, S: Storage + 'static> {
    /// Storage object to write the data to.
    storage: &'a S,

    /// Offset in `storage` to write the data to.
    offset: u64,

    /// Length of the mapping.
    length: u64,

    /// Driver-specific state required to commit this mapping; `None` if nothing needs to be
    /// committed.
    state: Option<Box<dyn Any>>,
}

impl<'a, S: Storage + 'static> PreparedDataMapping<'a, S> {
    /// Create a prepared mapping that does not require anything to be committed.
    ///
    /// For use by format drivers, e.g. when the mapping exists already.
    pub fn existing(storage: &'a S, offset: u64, length: u64) -> Self {
        PreparedDataMapping {
            storage,
            offset,
            length,
            state: None,
        }
    }

    /// Create a prepared mapping that must be committed, using the driver-specific `state`.
    ///
    /// For use by format drivers.  Dropping `state` must discard the prepared mapping.
    pub fn to_commit(storage: &'a S, offset: u64, length: u64, state: Box<dyn Any>) -> Self {
        PreparedDataMapping {
            storage,
            offset,
            length,
            state: Some(state),
        }
    }

    /// Storage object to write the data to.
    pub fn storage(&self) -> &'a S {
        self.storage
    }

    /// Offset in [`Self::storage()`] to write the data to.
    pub fn offset(&self) -> u64 {
        self.offset
    }

    /// Length of the mapping.
    pub fn length(&self) -> u64 {
        self.length
    }

    /// Whether holding this object can delay other I/O on the image.
    ///
    /// `true` means the range only becomes visible once it is committed, and other I/O on the
    /// image may be blocked in order not to interfere with that.
    ///
    /// `false` means holding it will never delay other I/O.  In this case, the mapping is visible
    /// already: the range keeps reading as it does now, and committing it will most likely do
    /// nothing ([`FormatAccess::commit_data_mapping()`] must still be called by users, though).
    pub fn is_nonblocking(&self) -> bool {
        self.state.is_none()
    }

    /// Take the driver-specific state, if any.
    ///
    /// For use by format drivers.
    pub fn into_state(self) -> Option<Box<dyn Any>> {
        self.state
    }

    /// Limit the length of the mapping to `max_length`.
    pub(crate) fn limit_length(&mut self, max_length: u64) {
        self.length = cmp::min(self.length, max_length);
    }
}

impl<S: Storage + 'static> Debug for PreparedDataMapping<'_, S> {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.debug_struct("PreparedDataMapping")
            .field("storage", &self.storage)
            .field("offset", &self.offset)
            .field("length", &self.length)
            .field("has_state", &self.state.is_some())
            .finish()
    }
}

/// Non-recursive mapping information.
///
/// Mapping information as returned by [`FormatDriverInstance::get_mapping()`], only looking at
/// that format layer’s information.
#[derive(Debug)]
#[non_exhaustive]
pub enum ShallowMapping<'a, S: Storage + 'static> {
    /// Raw data.
    #[non_exhaustive]
    Raw {
        /// Storage object where this data is stored.
        storage: &'a S,

        /// Offset in `storage` where this data is stored.
        offset: u64,

        /// Whether this mapping may be written to.
        ///
        /// If `true`, you can directly write to `offset` on `storage` to change the disk image’s
        /// data accordingly.
        ///
        /// If `false`, the disk image format does not allow writing to `offset` on `storage`; a
        /// new mapping must be allocated first.
        writable: bool,
    },

    /// Data lives in a different disk image (e.g. a backing file).
    #[non_exhaustive]
    Indirect {
        /// Format instance where this data can be obtained.
        layer: &'a FormatAccess<S>,

        /// Offset in `layer` where this data can be obtained.
        offset: u64,

        /// Whether this mapping may be written to.
        ///
        /// If `true`, you can directly write to `offset` on `layer` to change the disk image’s
        /// data accordingly.
        ///
        /// If `false`, the disk image format does not allow writing to `offset` on `layer`; a new
        /// mapping must be allocated first.
        writable: bool,
    },

    /// Range is to be read as zeroes.
    #[non_exhaustive]
    Zero {
        /// Whether these zeroes are explicit on this layer.
        ///
        /// Differential image formats (like qcow2) track information about the status for all
        /// blocks in the image (called clusters in case of qcow2).  Perhaps most importantly, they
        /// track whether a block is allocated or not:
        /// - Allocated blocks have their data in the image.
        /// - Unallocated blocks do not have their data in this image, but have to be read from a
        ///   backing image (which results in [`ShallowMapping::Indirect`] mappings).
        ///
        /// Thus, such images represent the difference from their backing image (hence
        /// “differential”).
        ///
        /// Without a backing image, this feature can be used for sparse allocation: Unallocated
        /// blocks are simply interpreted to be zero.  These ranges will be noted as
        /// [`ShallowMapping::Zero`] with `explicit` set to false.
        ///
        /// Formats like qcow2 can track more information beyond just the allocation status,
        /// though, for example, whether a block should read as zero. Such blocks similarly do not
        /// need to have their data stored in the image file, but are still not treated as
        /// unallocated, so will never be read from a backing image, regardless of whether one
        /// exists or not.
        ///
        /// These ranges are noted as [`ShallowMapping::Zero`] with `explicit` set to true.
        explicit: bool,
    },

    /// End of file reached.
    #[non_exhaustive]
    Eof {},

    /// Data is encoded in some manner, e.g. compressed or encrypted.
    ///
    /// Such data cannot be accessed directly, but must be interpreted by the image format driver.
    #[non_exhaustive]
    Special {
        /// Original (“guest”) offset to pass to `FormatDriverInstance::readv_special()`.
        offset: u64,
    },
}