use crate::util::Address;
use std::collections::HashMap;
use std::io::{Error, ErrorKind, Result};
use std::sync::{Mutex, RwLock};
use super::constants::{
LOG_GLOBAL_SIDE_METADATA_WORST_CASE_RATIO, LOG_LOCAL_SIDE_METADATA_WORST_CASE_RATIO,
};
use super::{SideMetadataContext, SideMetadataSpec};
#[cfg(target_pointer_width = "64")]
use crate::util::heap::layout::vm_layout::vm_layout;
use crate::util::heap::layout::vm_layout::VMLayout;
#[cfg(target_pointer_width = "32")]
use crate::util::heap::layout::vm_layout::LOG_BYTES_IN_CHUNK;
#[cfg(feature = "extreme_assertions")]
enum MathOp {
Add,
Sub,
}
static GLOBAL_META_NAME: &str = "Global";
pub struct SideMetadataSanity {
specs_sanity_map: HashMap<&'static str, Vec<SideMetadataSpec>>,
}
lazy_static! {
static ref CONTENT_SANITY_MAP: RwLock<HashMap<SideMetadataSpec, HashMap<Address, u64>>> =
RwLock::new(HashMap::new());
pub(crate) static ref SANITY_LOCK: Mutex<()> = Mutex::new(());
}
#[cfg(test)]
pub(crate) fn reset() {
CONTENT_SANITY_MAP.write().unwrap().clear()
}
fn verify_global_specs_total_size(g_specs: &[SideMetadataSpec]) -> Result<()> {
let mut total_size = 0usize;
for spec in g_specs {
total_size += super::metadata_address_range_size(spec);
}
if total_size
<= 1usize << (VMLayout::LOG_ARCH_ADDRESS_SPACE - LOG_GLOBAL_SIDE_METADATA_WORST_CASE_RATIO)
{
Ok(())
} else {
Err(Error::new(
ErrorKind::InvalidInput,
format!("Not enough global metadata space for: \n{:?}", g_specs),
))
}
}
#[cfg(target_pointer_width = "64")]
fn verify_local_specs_size(l_specs: &[SideMetadataSpec]) -> Result<()> {
for spec in l_specs {
if super::metadata_address_range_size(spec)
> 1usize
<< (VMLayout::LOG_ARCH_ADDRESS_SPACE - LOG_LOCAL_SIDE_METADATA_WORST_CASE_RATIO)
{
return Err(Error::new(
ErrorKind::InvalidInput,
format!("Local metadata is too big: \n{:?}", spec),
));
}
}
Ok(())
}
#[cfg(target_pointer_width = "32")]
fn verify_local_specs_size(l_specs: &[SideMetadataSpec]) -> Result<()> {
let mut total_size = 0usize;
for spec in l_specs {
total_size +=
super::metadata_bytes_per_chunk(spec.log_bytes_in_region, spec.log_num_of_bits);
}
if total_size > 1usize << (LOG_BYTES_IN_CHUNK - LOG_LOCAL_SIDE_METADATA_WORST_CASE_RATIO) {
return Err(Error::new(
ErrorKind::InvalidInput,
format!(
"Not enough local metadata space per chunk for: \n{:?}",
l_specs
),
));
}
Ok(())
}
fn verify_no_overlap_contiguous(
spec_1: &SideMetadataSpec,
spec_2: &SideMetadataSpec,
) -> Result<()> {
let end_1 = spec_1.get_absolute_offset() + super::metadata_address_range_size(spec_1);
let end_2 = spec_2.get_absolute_offset() + super::metadata_address_range_size(spec_2);
if !(spec_1.get_absolute_offset() >= end_2 || spec_2.get_absolute_offset() >= end_1) {
return Err(Error::new(
ErrorKind::InvalidInput,
format!(
"Overlapping metadata specs detected:\nTHIS:\n{:#?}\nAND:\n{:#?}",
spec_1, spec_2
),
));
}
Ok(())
}
#[cfg(target_pointer_width = "32")]
fn verify_no_overlap_chunked(spec_1: &SideMetadataSpec, spec_2: &SideMetadataSpec) -> Result<()> {
let end_1 = spec_1.get_rel_offset()
+ super::metadata_bytes_per_chunk(spec_1.log_bytes_in_region, spec_1.log_num_of_bits);
let end_2 = spec_2.get_rel_offset()
+ super::metadata_bytes_per_chunk(spec_2.log_bytes_in_region, spec_2.log_num_of_bits);
if !(spec_1.get_rel_offset() >= end_2 || spec_2.get_rel_offset() >= end_1) {
return Err(Error::new(
ErrorKind::InvalidInput,
format!(
"Overlapping metadata specs detected:\nTHIS:\n{:#?}\nAND:\n{:#?}",
spec_1, spec_2
),
));
}
Ok(())
}
fn verify_global_specs(g_specs: &[SideMetadataSpec]) -> Result<()> {
verify_global_specs_total_size(g_specs)?;
for spec_1 in g_specs {
for spec_2 in g_specs {
if spec_1 != spec_2 {
verify_no_overlap_contiguous(spec_1, spec_2)?;
}
}
}
Ok(())
}
impl Default for SideMetadataSanity {
fn default() -> Self {
Self::new()
}
}
impl SideMetadataSanity {
pub fn new() -> SideMetadataSanity {
SideMetadataSanity {
specs_sanity_map: HashMap::new(),
}
}
fn get_all_specs(&self, global: bool) -> Vec<SideMetadataSpec> {
let mut specs = vec![];
for (k, v) in self.specs_sanity_map.iter() {
if !(global ^ (*k == GLOBAL_META_NAME)) {
specs.append(&mut (*v).clone());
}
}
specs
}
fn verify_local_specs(&self) -> Result<()> {
let local_specs = self.get_all_specs(false);
verify_local_specs_size(&local_specs)?;
for spec_1 in &local_specs {
for spec_2 in &local_specs {
if spec_1 != spec_2 {
#[cfg(target_pointer_width = "64")]
verify_no_overlap_contiguous(spec_1, spec_2)?;
#[cfg(target_pointer_width = "32")]
verify_no_overlap_chunked(spec_1, spec_2)?;
}
}
}
Ok(())
}
pub(crate) fn verify_metadata_context(
&mut self,
policy_name: &'static str,
metadata_context: &SideMetadataContext,
) {
let mut content_sanity_map = CONTENT_SANITY_MAP.write().unwrap();
let first_call = !self.specs_sanity_map.contains_key(&GLOBAL_META_NAME);
if first_call {
verify_global_specs(&metadata_context.global).unwrap();
self.specs_sanity_map
.insert(GLOBAL_META_NAME, metadata_context.global.clone());
} else {
let g_specs = self.specs_sanity_map.get(&GLOBAL_META_NAME).unwrap();
assert!(
g_specs.len() == metadata_context.global.len(),
"Global metadata must not change between policies! NEW SPECS: {:#?} OLD SPECS: {:#?}",
metadata_context.global,
g_specs
);
}
for spec in &metadata_context.global {
assert!(
spec.is_global,
"Policy-specific spec {:#?} detected in the global specs: {:#?}",
spec, metadata_context.global
);
if first_call {
content_sanity_map.insert(*spec, HashMap::new());
} else if !self
.specs_sanity_map
.get(&GLOBAL_META_NAME)
.unwrap()
.contains(spec)
{
panic!("Global metadata must not change between policies! NEW SPEC: {:#?} OLD SPECS: {:#?}", spec, self.get_all_specs(true));
}
}
let first_call = !self.specs_sanity_map.contains_key(&policy_name);
if first_call {
self.specs_sanity_map
.insert(policy_name, metadata_context.local.clone());
}
for spec in &metadata_context.local {
assert!(
!spec.is_global,
"Global spec {:#?} detected in the policy-specific specs: {:#?}",
spec, metadata_context.local
);
if first_call {
content_sanity_map.insert(*spec, HashMap::new());
} else if !self
.specs_sanity_map
.get(policy_name)
.unwrap()
.contains(spec)
{
panic!(
"Policy-specific metadata for -{}- changed from {:#?} to {:#?}",
policy_name,
self.specs_sanity_map.get(policy_name).unwrap(),
metadata_context.local
)
}
}
self.verify_local_specs().unwrap();
}
#[cfg(test)]
pub fn reset(&mut self) {
let mut content_sanity_map = CONTENT_SANITY_MAP.write().unwrap();
self.specs_sanity_map.clear();
content_sanity_map.clear();
}
}
fn verify_metadata_address_bound(spec: &SideMetadataSpec, data_addr: Address) {
#[cfg(target_pointer_width = "32")]
assert_eq!(VMLayout::LOG_ARCH_ADDRESS_SPACE, 32, "We assume we use all address space in 32 bits. This seems not true any more, we need a proper check here.");
#[cfg(target_pointer_width = "32")]
let data_addr_in_address_space = true;
#[cfg(target_pointer_width = "64")]
let data_addr_in_address_space =
data_addr <= unsafe { Address::from_usize(1usize << vm_layout().log_address_space) };
if !data_addr_in_address_space {
warn!(
"We try get metadata {} for {}, which is not within the address space we should use",
data_addr, spec.name
);
}
let metadata_addr =
crate::util::metadata::side_metadata::address_to_meta_address(spec, data_addr);
let metadata_addr_bound = if spec.is_absolute_offset() {
spec.upper_bound_address_for_contiguous()
} else {
#[cfg(target_pointer_width = "32")]
{
spec.upper_bound_address_for_chunked(data_addr)
}
#[cfg(target_pointer_width = "64")]
{
unreachable!()
}
};
assert!(
metadata_addr < metadata_addr_bound,
"We try access metadata address for address {} of spec {} that is not within the bound {}.",
data_addr,
spec.name,
metadata_addr_bound
);
}
#[cfg(feature = "extreme_assertions")]
pub fn verify_bzero(metadata_spec: &SideMetadataSpec, start: Address, size: usize) {
let sanity_map = &mut CONTENT_SANITY_MAP.write().unwrap();
let start = align_to_region_start(metadata_spec, start);
let end = align_to_region_start(metadata_spec, start + size);
match sanity_map.get_mut(metadata_spec) {
Some(spec_sanity_map) => {
for (k, v) in spec_sanity_map.iter_mut() {
if *k >= start && *k < end {
*v = 0;
}
}
}
None => {
panic!("Invalid Metadata Spec: {}", metadata_spec.name);
}
}
}
#[cfg(feature = "extreme_assertions")]
pub fn verify_bset(metadata_spec: &SideMetadataSpec, start: Address, size: usize) {
let sanity_map = &mut CONTENT_SANITY_MAP.write().unwrap();
let start = align_to_region_start(metadata_spec, start);
let end = align_to_region_start(metadata_spec, start + size);
let max_value = (1 << (1 << metadata_spec.log_num_of_bits)) - 1;
match sanity_map.get_mut(metadata_spec) {
Some(spec_sanity_map) => {
let mut cursor = start;
let step: usize = 1 << metadata_spec.log_bytes_in_region;
while cursor < end {
spec_sanity_map.insert(cursor, max_value);
cursor += step;
}
}
None => {
panic!("Invalid Metadata Spec!");
}
}
}
#[cfg(feature = "extreme_assertions")]
pub fn verify_bcopy(
dst_spec: &SideMetadataSpec,
start: Address,
size: usize,
src_spec: &SideMetadataSpec,
) {
assert_eq!(src_spec.log_num_of_bits, dst_spec.log_num_of_bits);
assert_eq!(src_spec.log_bytes_in_region, dst_spec.log_bytes_in_region);
let sanity_map = &mut CONTENT_SANITY_MAP.write().unwrap();
let start = align_to_region_start(dst_spec, start);
let end = align_to_region_start(dst_spec, start + size);
let mut tmp_map = HashMap::new();
{
let src_map = sanity_map
.get_mut(src_spec)
.expect("Invalid source Metadata Spec!");
let mut cursor = start;
let step: usize = 1 << src_spec.log_bytes_in_region;
while cursor < end {
let src_value = src_map.get(&cursor).copied().unwrap_or(0u64);
tmp_map.insert(cursor, src_value);
cursor += step;
}
}
{
let dst_map = sanity_map
.get_mut(dst_spec)
.expect("Invalid destination Metadata Spec!");
let mut cursor = start;
let step: usize = 1 << dst_spec.log_bytes_in_region;
while cursor < end {
let src_value = tmp_map.get(&cursor).copied().unwrap();
dst_map.insert(cursor, src_value);
cursor += step;
}
}
}
#[cfg(feature = "extreme_assertions")]
use crate::util::metadata::metadata_val_traits::*;
#[cfg(feature = "extreme_assertions")]
fn truncate_value<T: MetadataValue>(log_num_of_bits: usize, val: u64) -> u64 {
if log_num_of_bits < T::LOG2 as usize {
val & ((1 << (1 << log_num_of_bits)) - 1)
} else {
val
}
}
#[cfg(feature = "extreme_assertions")]
#[cfg(test)]
mod truncate_tests {
use super::*;
#[test]
fn test_truncate() {
assert_eq!(truncate_value::<u8>(2, 0), 0);
assert_eq!(truncate_value::<u8>(2, 15), 15);
assert_eq!(truncate_value::<u8>(2, 16), 0);
assert_eq!(truncate_value::<u8>(2, 17), 1);
}
}
fn align_to_region_start(spec: &SideMetadataSpec, data_addr: Address) -> Address {
data_addr.align_down(1 << spec.log_bytes_in_region)
}
#[cfg(feature = "extreme_assertions")]
pub fn verify_load<T: MetadataValue>(
metadata_spec: &SideMetadataSpec,
data_addr: Address,
actual_val: T,
) {
let data_addr = align_to_region_start(metadata_spec, data_addr);
let actual_val: u64 = actual_val.to_u64().unwrap();
verify_metadata_address_bound(metadata_spec, data_addr);
let sanity_map = &mut CONTENT_SANITY_MAP.read().unwrap();
match sanity_map.get(metadata_spec) {
Some(spec_sanity_map) => {
let expected_val = if let Some(expected_val) = spec_sanity_map.get(&data_addr) {
*expected_val
} else {
0u64
};
assert!(
expected_val == actual_val,
"verify_load({:#?}, {}) -> Expected (0x{:x}) but found (0x{:x})",
metadata_spec,
data_addr,
expected_val,
actual_val
);
}
None => panic!("Invalid Metadata Spec: {:#?}", metadata_spec),
}
}
#[cfg(feature = "extreme_assertions")]
pub fn verify_store<T: MetadataValue>(
metadata_spec: &SideMetadataSpec,
data_addr: Address,
metadata: T,
) {
let data_addr = align_to_region_start(metadata_spec, data_addr);
let metadata: u64 = metadata.to_u64().unwrap();
verify_metadata_address_bound(metadata_spec, data_addr);
let new_val_wrapped = truncate_value::<T>(metadata_spec.log_num_of_bits, metadata);
let sanity_map = &mut CONTENT_SANITY_MAP.write().unwrap();
match sanity_map.get_mut(metadata_spec) {
Some(spec_sanity_map) => {
let content = spec_sanity_map.entry(data_addr).or_insert(0);
*content = new_val_wrapped;
}
None => panic!("Invalid Metadata Spec: {:#?}", metadata_spec),
}
}
#[cfg(feature = "extreme_assertions")]
pub fn verify_update<T: MetadataValue>(
metadata_spec: &SideMetadataSpec,
data_addr: Address,
old_val: T,
new_val: T,
) {
let data_addr = align_to_region_start(metadata_spec, data_addr);
verify_metadata_address_bound(metadata_spec, data_addr);
let new_val_wrapped =
truncate_value::<T>(metadata_spec.log_num_of_bits, new_val.to_u64().unwrap());
let sanity_map = &mut CONTENT_SANITY_MAP.write().unwrap();
match sanity_map.get_mut(metadata_spec) {
Some(spec_sanity_map) => {
let cur_val = spec_sanity_map.entry(data_addr).or_insert(0);
assert_eq!(
old_val.to_u64().unwrap(),
*cur_val,
"Expected old value: {} but found {}",
old_val,
cur_val
);
*cur_val = new_val_wrapped;
}
None => panic!("Invalid metadata spec: {:#?}", metadata_spec),
}
}
#[cfg(test)]
mod tests {
use super::super::*;
use super::*;
#[test]
fn test_side_metadata_sanity_verify_global_specs_total_size() {
let spec_1 = SideMetadataSpec {
name: "spec_1",
is_global: true,
offset: SideMetadataOffset::addr(Address::ZERO),
log_num_of_bits: 0,
log_bytes_in_region: 0,
};
let spec_2 = SideMetadataSpec {
name: "spec_2",
is_global: true,
offset: SideMetadataOffset::layout_after(&spec_1),
log_num_of_bits: 0,
log_bytes_in_region: 0,
};
assert!(verify_global_specs_total_size(&[spec_1]).is_ok());
#[cfg(target_pointer_width = "64")]
assert!(verify_global_specs_total_size(&[spec_1, spec_2]).is_ok());
#[cfg(target_pointer_width = "32")]
assert!(verify_global_specs_total_size(&[spec_1, spec_2]).is_err());
let spec_2 = SideMetadataSpec {
name: "spec_2",
is_global: true,
offset: SideMetadataOffset::layout_after(&spec_1),
log_num_of_bits: 3,
log_bytes_in_region: 1,
};
assert!(verify_global_specs_total_size(&[spec_1, spec_2]).is_err());
let spec_1 = SideMetadataSpec {
name: "spec_1",
is_global: true,
offset: SideMetadataOffset::addr(Address::ZERO),
log_num_of_bits: 1,
#[cfg(target_pointer_width = "64")]
log_bytes_in_region: 0,
#[cfg(target_pointer_width = "32")]
log_bytes_in_region: 2,
};
let spec_2 = SideMetadataSpec {
name: "spec_2",
is_global: true,
offset: SideMetadataOffset::layout_after(&spec_1),
log_num_of_bits: 3,
#[cfg(target_pointer_width = "64")]
log_bytes_in_region: 2,
#[cfg(target_pointer_width = "32")]
log_bytes_in_region: 4,
};
assert!(verify_global_specs_total_size(&[spec_1, spec_2]).is_ok());
assert!(verify_global_specs_total_size(&[spec_1, spec_2, spec_1]).is_err());
}
#[test]
fn test_side_metadata_sanity_verify_no_overlap_contiguous() {
let spec_1 = SideMetadataSpec {
name: "spec_1",
is_global: true,
offset: SideMetadataOffset::addr(Address::ZERO),
log_num_of_bits: 0,
log_bytes_in_region: 0,
};
let spec_2 = SideMetadataSpec {
name: "spec_2",
is_global: true,
offset: SideMetadataOffset::layout_after(&spec_1),
log_num_of_bits: 0,
log_bytes_in_region: 0,
};
assert!(verify_no_overlap_contiguous(&spec_1, &spec_1).is_err());
assert!(verify_no_overlap_contiguous(&spec_1, &spec_2).is_ok());
let spec_1 = SideMetadataSpec {
name: "spec_1",
is_global: true,
offset: SideMetadataOffset::addr(unsafe { Address::from_usize(1) }),
log_num_of_bits: 0,
log_bytes_in_region: 0,
};
assert!(verify_no_overlap_contiguous(&spec_1, &spec_2).is_err());
let spec_1 = SideMetadataSpec {
name: "spec_1",
is_global: true,
offset: SideMetadataOffset::addr(Address::ZERO),
log_num_of_bits: 0,
log_bytes_in_region: 0,
};
let spec_2 = SideMetadataSpec {
name: "spec_2",
is_global: true,
offset: SideMetadataOffset::addr(
spec_1.get_absolute_offset() + metadata_address_range_size(&spec_1) - 1,
),
log_num_of_bits: 0,
log_bytes_in_region: 0,
};
assert!(verify_no_overlap_contiguous(&spec_1, &spec_2).is_err());
}
#[cfg(target_pointer_width = "32")]
#[test]
fn test_side_metadata_sanity_verify_no_overlap_chunked() {
let spec_1 = SideMetadataSpec {
name: "spec_1",
is_global: false,
offset: SideMetadataOffset::rel(0),
log_num_of_bits: 0,
log_bytes_in_region: 0,
};
let spec_2 = SideMetadataSpec {
name: "spec_2",
is_global: false,
offset: SideMetadataOffset::layout_after(&spec_1),
log_num_of_bits: 0,
log_bytes_in_region: 0,
};
assert!(verify_no_overlap_chunked(&spec_1, &spec_1).is_err());
assert!(verify_no_overlap_chunked(&spec_1, &spec_2).is_ok());
let spec_1 = SideMetadataSpec {
name: "spec_1",
is_global: false,
offset: SideMetadataOffset::rel(1),
log_num_of_bits: 0,
log_bytes_in_region: 0,
};
assert!(verify_no_overlap_chunked(&spec_1, &spec_2).is_err());
let spec_1 = SideMetadataSpec {
name: "spec_1",
is_global: false,
offset: SideMetadataOffset::rel(0),
log_num_of_bits: 0,
log_bytes_in_region: 0,
};
let spec_2 = SideMetadataSpec {
name: "spec_2",
is_global: false,
offset: SideMetadataOffset::rel(
spec_1.get_rel_offset()
+ metadata_bytes_per_chunk(spec_1.log_bytes_in_region, spec_1.log_num_of_bits)
- 1,
),
log_num_of_bits: 0,
log_bytes_in_region: 0,
};
assert!(verify_no_overlap_chunked(&spec_1, &spec_2).is_err());
}
#[cfg(target_pointer_width = "32")]
#[test]
fn test_side_metadata_sanity_verify_local_specs_size() {
let spec_1 = SideMetadataSpec {
name: "spec_1",
is_global: false,
offset: SideMetadataOffset::rel(0),
log_num_of_bits: 0,
log_bytes_in_region: 0,
};
assert!(verify_local_specs_size(&[spec_1]).is_ok());
assert!(verify_local_specs_size(&[spec_1, spec_1]).is_err());
assert!(verify_local_specs_size(&[spec_1, spec_1, spec_1, spec_1, spec_1]).is_err());
}
}