diff --git a/wild_lib/src/gc_stats.rs b/wild_lib/src/gc_stats.rs index 35100d19..f18f4ec6 100644 --- a/wild_lib/src/gc_stats.rs +++ b/wild_lib/src/gc_stats.rs @@ -82,8 +82,8 @@ fn write_gc_stats( let mut file_discarded = 0; for (slot, section) in obj.sections.iter().zip(obj.object.sections.iter()) { match slot { - SectionSlot::Unloaded(id) => { - if id.output_section_id() == output_section_id::TEXT { + SectionSlot::Unloaded(unloaded) => { + if unloaded.part_id.output_section_id() == output_section_id::TEXT { file_discarded += obj.object.section_size(section)?; if args.verbose_gc_stats { file_record diff --git a/wild_lib/src/layout.rs b/wild_lib/src/layout.rs index 5a0ca103..963a8df1 100644 --- a/wild_lib/src/layout.rs +++ b/wild_lib/src/layout.rs @@ -35,12 +35,14 @@ use crate::program_segments::ProgramSegmentId; use crate::program_segments::MAX_SEGMENTS; use crate::relaxation::Relaxation; use crate::resolution; +use crate::resolution::FrameIndex; use crate::resolution::MergeStringsSection; use crate::resolution::NotLoaded; use crate::resolution::ResolutionOutputs; use crate::resolution::ResolvedEpilogue; use crate::resolution::SectionSlot; use crate::resolution::StringToMerge; +use crate::resolution::UnloadedSection; use crate::resolution::ValueFlags; use crate::sharding::ShardKey; use crate::symbol::SymbolName; @@ -887,7 +889,10 @@ struct ObjectLayoutState<'data> { symbol_id_range: SymbolIdRange, object: &'data File<'data>, state: ObjectLayoutMutableState<'data>, - section_frame_data: Vec>, + + /// Indexed by `FrameIndex`. + exception_frames: Vec>, + eh_frame_section: Option<&'data object::elf::SectionHeader64>, eh_frame_size: u64, } @@ -907,18 +912,15 @@ struct ObjectLayoutMutableState<'data> { } #[derive(Default)] -struct SectionFrameData<'data> { - /// Outgoing references from the FDE(s) for our section. Generally all relocations for a section - /// would be contiguous, so we'd only need one slice. In theory though it's possible that there - /// could be more than one group, so we accommodate that, but optimise for the common case of - /// one group. - relocations: SmallVec<[&'data [Rela64]; 1]>, +struct ExceptionFrame<'data> { + /// The relocations that need to be processed if we load this frame. + relocations: &'data [Rela64], - /// Number of FDEs associated with symbols in this section. - num_fdes: u32, + /// Number of bytes required to store this frame. + frame_size: u32, - /// Number of bytes required to store the FDEs associated with this section. - total_fde_size: u32, + /// The index of the previous frame that is for the same section. + previous_frame_for_section: Option, } #[derive(Default)] @@ -2929,7 +2931,7 @@ fn new_object_layout_state(input_state: resolution::ResolvedObject) -> FileLayou symbol_id_range: input_state.symbol_id_range, input: input_state.input, object: input_state.object, - section_frame_data: Default::default(), + exception_frames: Default::default(), eh_frame_section: None, eh_frame_size: 0, state: ObjectLayoutMutableState { @@ -3038,10 +3040,11 @@ impl<'data> ObjectLayoutState<'data> { &mut self, common: &mut CommonGroupState<'data>, queue: &mut LocalWorkQueue, - part_id: PartId, + unloaded: UnloadedSection, section_id: SectionIndex, resources: &GraphResources<'data, 'scope>, ) -> Result { + let part_id = unloaded.part_id; let section = Section::create(self, section_id, part_id)?; for rel in self.object.relocations(section.index)? { process_relocation( @@ -3055,33 +3058,55 @@ impl<'data> ObjectLayoutState<'data> { } tracing::debug!(loaded_section = %self.object.section_display_name(section_id),); common.allocate(part_id, section.capacity()); + resources .sections_with_content .get(part_id.output_section_id()) .fetch_or(true, atomic::Ordering::Relaxed); - if let Some(frame_data) = self.section_frame_data.get_mut(section_id.0) { - self.eh_frame_size += u64::from(frame_data.total_fde_size); - if resources.symbol_db.args.should_write_eh_frame_hdr { - common.allocate( - part_id::EH_FRAME_HDR, - core::mem::size_of::() as u64 * u64::from(frame_data.num_fdes), - ); - } - // Take ownership of the section's frame data relocations. We only apply - // these once when the section is loaded, so after this we won't need them - // any more. By taking ownership, we drop our borrow of self. - let frame_data_relocations = core::mem::take(&mut frame_data.relocations); + + self.process_section_exception_frames(unloaded.last_frame_index, common, resources, queue)?; + + self.state.sections[section_id.0] = SectionSlot::Loaded(section); + + Ok(()) + } + + /// Processes the exception frames for a section that we're loading. + fn process_section_exception_frames( + &mut self, + frame_index: Option, + common: &mut CommonGroupState<'data>, + resources: &GraphResources<'data, '_>, + queue: &mut LocalWorkQueue, + ) -> Result { + let mut num_frames = 0; + let mut next_frame_index = frame_index; + while let Some(frame_index) = next_frame_index { + let frame_data = &self.exception_frames[frame_index.as_usize()]; + next_frame_index = frame_data.previous_frame_for_section; + + self.eh_frame_size += u64::from(frame_data.frame_size); + + num_frames += 1; + + let frame_data_relocations = frame_data.relocations; + // Request loading of any sections/symbols referenced by the FDEs for our // section. if let Some(eh_frame_section) = self.eh_frame_section { - for relocations in &frame_data_relocations { - for rel in *relocations { - process_relocation(self, common, rel, eh_frame_section, resources, queue)?; - } + for rel in frame_data_relocations { + process_relocation(self, common, rel, eh_frame_section, resources, queue)?; } } } - self.state.sections[section_id.0] = SectionSlot::Loaded(section); + + if resources.symbol_db.args.should_write_eh_frame_hdr { + common.allocate( + part_id::EH_FRAME_HDR, + core::mem::size_of::() as u64 * num_frames, + ); + } + Ok(()) } @@ -3423,9 +3448,6 @@ fn process_eh_frame_data( resources: &GraphResources, queue: &mut LocalWorkQueue, ) -> Result { - object - .section_frame_data - .resize_with(object.state.sections.len(), Default::default); let eh_frame_section = object.object.section(eh_frame_section_index)?; let data = object.object.raw_section_data(eh_frame_section)?; const PREFIX_LEN: usize = core::mem::size_of::(); @@ -3433,7 +3455,7 @@ fn process_eh_frame_data( let relocations = object.object.relocations(eh_frame_section_index)?; let mut rel_iter = relocations.iter().enumerate().peekable(); let mut offset = 0; - let mut pending: Option = None; + while offset + PREFIX_LEN <= data.len() { // Although the section data will be aligned within the object file, there's // no guarantee that the object is aligned within the archive to any more @@ -3502,63 +3524,33 @@ fn process_eh_frame_data( break; } } + if let Some(section_index) = section_index { - let new_pending = PendingEhFrameRelocations { - section_index, - start: rel_start_index, - end: rel_end_index, - }; - if let Some(p) = pending.as_mut() { - if !p.merge(&new_pending) { - p.apply(&mut object.section_frame_data, relocations); - pending = Some(new_pending); - } - } else { - pending = Some(new_pending); + if let Some(unloaded) = object.state.sections[section_index.0].unloaded_mut() { + let frame_index = FrameIndex::from_usize(object.exception_frames.len()); + + // Update our unloaded section to point to our new frame. Our frame will then in + // turn point to whatever the section pointed to before. + let previous_frame_for_section = + core::mem::replace(&mut unloaded.last_frame_index, Some(frame_index)); + + object.exception_frames.push(ExceptionFrame { + relocations: &relocations[rel_start_index..rel_end_index], + frame_size: size as u32, + previous_frame_for_section, + }); } - let section_frame_data = &mut object.section_frame_data[section_index.0]; - section_frame_data.num_fdes += 1; - section_frame_data.total_fde_size += size as u32; } } offset = next_offset; } - if let Some(p) = pending { - p.apply(&mut object.section_frame_data, relocations); - } + // Allocate space for any remaining bytes in .eh_frame that aren't large enough to constitute an // actual entry. crtend.o has a single u32 equal to 0 as an end marker. object.eh_frame_size += (data.len() - offset) as u64; Ok(()) } -struct PendingEhFrameRelocations { - section_index: object::SectionIndex, - start: usize, - end: usize, -} - -impl PendingEhFrameRelocations { - fn apply<'data>( - &self, - section_frame_data: &mut [SectionFrameData<'data>], - relocations: &'data [Rela64], - ) { - let section_frame_data = &mut section_frame_data[self.section_index.0]; - section_frame_data - .relocations - .push(&relocations[self.start..self.end]); - } - - fn merge(&mut self, new_pending: &PendingEhFrameRelocations) -> bool { - if self.section_index != new_pending.section_index || new_pending.start != self.end { - return false; - } - self.end = new_pending.end; - true - } -} - /// A "common information entry". This is part of the .eh_frame data in ELF. #[derive(PartialEq, Eq, Hash)] struct Cie<'data> { diff --git a/wild_lib/src/part_id.rs b/wild_lib/src/part_id.rs index bc32e9af..d494e230 100644 --- a/wild_lib/src/part_id.rs +++ b/wild_lib/src/part_id.rs @@ -35,7 +35,7 @@ pub(crate) struct CustomSectionId<'data> { } #[derive(Debug, Clone, Copy)] -pub(crate) struct UnloadedSection<'data> { +pub(crate) struct UnresolvedSection<'data> { pub(crate) part_id: TemporaryPartId<'data>, pub(crate) is_string_merge: bool, } @@ -87,7 +87,7 @@ pub(crate) const NUM_BUILT_IN_PARTS: usize = NUM_GENERATED_PARTS /// A placeholder used for custom sections before we know their actual PartId. pub(crate) const CUSTOM_PLACEHOLDER: PartId = PartId(u32::MAX); -impl<'data> UnloadedSection<'data> { +impl<'data> UnresolvedSection<'data> { #[allow(clippy::if_same_then_else)] pub(crate) fn from_section( object: &crate::elf::File<'data>, @@ -124,7 +124,7 @@ impl<'data> UnloadedSection<'data> { } else if section_name == b".comment" { Some(output_section_id::COMMENT) } else if section_name == b".eh_frame" { - return Ok(Some(UnloadedSection { + return Ok(Some(UnresolvedSection { part_id: TemporaryPartId::EhFrameData, is_string_merge: false, })); @@ -148,7 +148,7 @@ impl<'data> UnloadedSection<'data> { let custom_section_id = CustomSectionId { name: SectionName(section_name), }; - return Ok(Some(UnloadedSection { + return Ok(Some(UnresolvedSection { part_id: TemporaryPartId::Custom(custom_section_id, alignment), is_string_merge: should_merge_strings( section, @@ -183,7 +183,7 @@ impl<'data> UnloadedSection<'data> { return Ok(None); }; let part_id = built_in_section_id.part_id_with_alignment(alignment); - Ok(Some(UnloadedSection { + Ok(Some(UnresolvedSection { part_id: TemporaryPartId::BuiltIn(part_id), is_string_merge: should_merge_strings( section, diff --git a/wild_lib/src/resolution.rs b/wild_lib/src/resolution.rs index 45779e1a..166c6959 100644 --- a/wild_lib/src/resolution.rs +++ b/wild_lib/src/resolution.rs @@ -27,7 +27,7 @@ use crate::parsing::Prelude; use crate::part_id; use crate::part_id::PartId; use crate::part_id::TemporaryPartId; -use crate::part_id::UnloadedSection; +use crate::part_id::UnresolvedSection; use crate::sharding::ShardKey; use crate::symbol::SymbolName; use crate::symbol_db::SymbolDb; @@ -49,6 +49,7 @@ use rayon::iter::ParallelIterator; use std::collections::HashMap; use std::fmt::Display; use std::hash::Hash; +use std::num::NonZeroU32; use std::sync::atomic::AtomicBool; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; @@ -434,10 +435,10 @@ pub(crate) enum SectionSlot<'data> { Discard, /// The section hasn't been loaded yet, but may be loaded if it's referenced. - Unloaded(PartId), + Unloaded(UnloadedSection), /// The section had the retain bit set, so must be loaded. - MustLoad(PartId), + MustLoad(UnloadedSection), /// We've already loaded the section. Loaded(crate::layout::Section), @@ -455,6 +456,27 @@ pub(crate) enum SectionSlot<'data> { LoadedDebugInfo(crate::layout::Section), } +#[derive(Clone, Copy)] +pub(crate) struct UnloadedSection { + pub(crate) part_id: PartId, + + /// The index of the last FDE for this section. Previous FDEs will be linked from this. + pub(crate) last_frame_index: Option, +} + +impl UnloadedSection { + fn new(part_id: PartId) -> Self { + Self { + part_id, + last_frame_index: None, + } + } +} + +/// An index into the exception frames for an object. +#[derive(Clone, Copy)] +pub(crate) struct FrameIndex(NonZeroU32); + pub(crate) struct ResolvedPrelude<'data> { pub(crate) symbol_definitions: &'data [InternalSymDefInfo], } @@ -855,7 +877,8 @@ fn resolve_sections<'data>( .sections .enumerate() .map(|(input_section_index, input_section)| { - if let Some(unloaded) = UnloadedSection::from_section(&obj.object, input_section, args)? + if let Some(unloaded) = + UnresolvedSection::from_section(&obj.object, input_section, args)? { let section_flags = SectionFlags::from_header(input_section); let mut part_id = part_id::CUSTOM_PLACEHOLDER; @@ -894,9 +917,11 @@ fn resolve_sections<'data>( .section_flags .should_retain() => { - SectionSlot::MustLoad(id) + SectionSlot::MustLoad(UnloadedSection::new(id)) + } + TemporaryPartId::BuiltIn(id) => { + SectionSlot::Unloaded(UnloadedSection::new(id)) } - TemporaryPartId::BuiltIn(id) => SectionSlot::Unloaded(id), TemporaryPartId::Custom(custom_section_id, _alignment) => { if custom_section_id.name.bytes().starts_with(b".debug_") { if args.strip_debug { @@ -906,9 +931,13 @@ fn resolve_sections<'data>( SectionSlot::UnloadedDebugInfo(part_id::CUSTOM_PLACEHOLDER) } } else if section_flags.should_retain() { - SectionSlot::MustLoad(part_id::CUSTOM_PLACEHOLDER) + SectionSlot::MustLoad(UnloadedSection::new( + part_id::CUSTOM_PLACEHOLDER, + )) } else { - SectionSlot::Unloaded(part_id::CUSTOM_PLACEHOLDER) + SectionSlot::Unloaded(UnloadedSection::new( + part_id::CUSTOM_PLACEHOLDER, + )) } } TemporaryPartId::EhFrameData => { @@ -1053,13 +1082,20 @@ impl<'data> SectionSlot<'data> { pub(crate) fn set_part_id(&mut self, part_id: PartId) { match self { SectionSlot::Discard => todo!(), - SectionSlot::Unloaded(out) => *out = part_id, - SectionSlot::MustLoad(out) => *out = part_id, - SectionSlot::Loaded(out) => out.part_id = part_id, + SectionSlot::Unloaded(section) => section.part_id = part_id, + SectionSlot::MustLoad(section) => section.part_id = part_id, + SectionSlot::Loaded(section) => section.part_id = part_id, SectionSlot::EhFrameData(_) => todo!(), - SectionSlot::MergeStrings(out) => out.part_id = part_id, + SectionSlot::MergeStrings(section) => section.part_id = part_id, SectionSlot::UnloadedDebugInfo(out) => *out = part_id, - SectionSlot::LoadedDebugInfo(out) => out.part_id = part_id, + SectionSlot::LoadedDebugInfo(section) => section.part_id = part_id, + } + } + + pub(crate) fn unloaded_mut(&mut self) -> Option<&mut UnloadedSection> { + match self { + SectionSlot::Unloaded(unloaded) | SectionSlot::MustLoad(unloaded) => Some(unloaded), + _ => None, } } } @@ -1180,6 +1216,16 @@ impl<'data> SymbolDb<'data> { } } +impl FrameIndex { + pub(crate) fn from_usize(raw: usize) -> Self { + Self(NonZeroU32::new(raw as u32 + 1).unwrap()) + } + + pub(crate) fn as_usize(&self) -> usize { + self.0.get() as usize - 1 + } +} + // We create quite a lot of `SectionSlot`s. We don't generally copy them, however we do need to // eventually drop the Vecs that contain them. Dropping those Vecs is a lot cheaper if the slots // don't need to have run Drop. We check for this, by making sure the type implements `Copy`