From 5b9091326e4e781ec885478bf21fc8d79802e2b4 Mon Sep 17 00:00:00 2001 From: RishabhSaini Date: Tue, 16 May 2023 17:28:18 -0400 Subject: [PATCH] container: Add --previous-build-manifest Prune rpm changelogs of packages to within the last year and use it as frequency of update for the rpm. --- rpmostree-cxxrs.cxx | 9 +++++ rust/src/compose.rs | 14 ++++++++ rust/src/container.rs | 60 ++++++++++++++++++++++++++++----- rust/src/lib.rs | 1 + src/libpriv/rpmostree-refts.cxx | 25 ++++++++++++++ src/libpriv/rpmostree-refts.h | 7 ++++ 6 files changed, 107 insertions(+), 9 deletions(-) diff --git a/rpmostree-cxxrs.cxx b/rpmostree-cxxrs.cxx index 47de4b00cf..62a8ea164b 100644 --- a/rpmostree-cxxrs.cxx +++ b/rpmostree-cxxrs.cxx @@ -3226,6 +3226,15 @@ extern "C" return (self.*buildtime$) (); } + void + rpmostreecxx$cxxbridge1$PackageMeta$changelogs (const ::rpmostreecxx::PackageMeta &self, + ::rust::Vec< ::std::uint64_t> *return$) noexcept + { + ::rust::Vec< ::std::uint64_t> (::rpmostreecxx::PackageMeta::*changelogs$) () const + = &::rpmostreecxx::PackageMeta::changelogs; + new (return$)::rust::Vec< ::std::uint64_t> ((self.*changelogs$) ()); + } + const ::std::string * rpmostreecxx$cxxbridge1$PackageMeta$src_pkg (const ::rpmostreecxx::PackageMeta &self) noexcept { diff --git a/rust/src/compose.rs b/rust/src/compose.rs index eea75b0cf3..5935c8c739 100644 --- a/rust/src/compose.rs +++ b/rust/src/compose.rs @@ -2,6 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT +use std::fs::File; +use std::io::{BufWriter, Write}; use std::process::Command; use anyhow::{anyhow, Result}; @@ -250,10 +252,22 @@ pub(crate) fn compose_image(args: Vec) -> CxxResult<()> { let target_imgref = target_imgref.to_string(); let label_args = opt.labels.into_iter().map(|v| format!("--label={v}")); + // If we have a prior build, pass its manifest to the encapsulation command to allow reuse of packing structure. + let previous_arg = previous_meta + .as_ref() + .map(|previous_meta| { + let manifest_path = tempdir.join("previous-manifest.json"); + let mut f = File::create(&manifest_path).map(BufWriter::new)?; + serde_json::to_writer(&mut f, &previous_meta.manifest).map_err(anyhow::Error::new)?; + f.flush()?; + anyhow::Ok(format!("--previous-build-manifest={manifest_path}")) + }) + .transpose()?; let s = self_command() .args(&["compose", "container-encapsulate"]) .args(label_args) + .args(previous_arg) .args(&[ "--repo", repo.as_str(), diff --git a/rust/src/container.rs b/rust/src/container.rs index 50fcf25a55..6ca9605dbf 100644 --- a/rust/src/container.rs +++ b/rust/src/container.rs @@ -21,7 +21,7 @@ use ostree_ext::objectsource::{ ContentID, ObjectMeta, ObjectMetaMap, ObjectMetaSet, ObjectSourceMeta, }; use ostree_ext::prelude::*; -use ostree_ext::{gio, ostree}; +use ostree_ext::{gio, oci_spec, ostree}; use crate::cxxrsutil::FFIGObjectReWrap; use crate::progress::progress_task; @@ -71,6 +71,11 @@ struct ContainerEncapsulateOpts { /// Compare OCI layers of current build with another(imgref) #[clap(name = "compare-with-build", long)] compare_with_build: Option, + + /// Prevent a change in packing structure by taking a previous build metadata (oci config and + /// manifest) + #[clap(long)] + previous_build_manifest: Option, } #[derive(Debug)] @@ -246,16 +251,18 @@ pub fn container_encapsulate(args: Vec) -> CxxResult<()> { skip: Default::default(), rpmsize: Default::default(), }; - // Insert metadata for unpakaged content. + // Insert metadata for unpackaged content. state.packagemeta.insert(ObjectSourceMeta { identifier: Rc::clone(&state.unpackaged_id), name: Rc::clone(&state.unpackaged_id), srcid: Rc::clone(&state.unpackaged_id), // Assume that content in here changes frequently. change_time_offset: u32::MAX, + change_frequency: u32::MAX, }); let mut lowest_change_time = None; + let mut highest_change_time = None; let mut package_meta = HashMap::new(); for pkg in pkglist.iter() { let name = pkg.child_value(0); @@ -271,6 +278,13 @@ pub fn container_encapsulate(args: Vec) -> CxxResult<()> { } else { lowest_change_time = Some((Rc::clone(&nevra), pkgmeta.buildtime())) } + if let Some(hightime) = highest_change_time.as_mut() { + if *hightime < buildtime { + *hightime = buildtime; + } + } else { + highest_change_time = Some(pkgmeta.buildtime()) + } state.rpmsize += pkgmeta.size(); package_meta.insert(nevra, pkgmeta); } @@ -278,6 +292,8 @@ pub fn container_encapsulate(args: Vec) -> CxxResult<()> { // SAFETY: There must be at least one package. let (lowest_change_name, lowest_change_time) = lowest_change_time.expect("Failed to find any packages"); + let highest_change_time = highest_change_time.expect("Failed to find any packages"); + // Walk over the packages, and generate the `packagemeta` mapping, which is basically a subset of // package metadata abstracted for ostree. Note that right now, the package metadata includes // both a "unique identifer" and a "human readable name", but for rpm-ostree we're just making @@ -292,11 +308,26 @@ pub fn container_encapsulate(args: Vec) -> CxxResult<()> { // Convert to hours, because there's no strong use for caring about the relative difference of builds in terms // of minutes or seconds. let change_time_offset = change_time_offset_secs / (60 * 60); + let changelogs = pkgmeta.changelogs(); + // Ignore the updates to packages more than a year away from the latest built package as its + // contribution becomes increasingly irrelevant to the likelihood of the package updating + // in the future + // TODO: Weighted Moving Averages (Weights decaying by year) to calculate the frequency + let pruned_changelogs: Vec<&u64> = changelogs + .iter() + .filter(|e| { + let curr_build = glib::DateTime::from_unix_utc(**e as i64).unwrap(); + let highest_time_build = + glib::DateTime::from_unix_utc(highest_change_time as i64).unwrap(); + highest_time_build.difference(&curr_build).as_days() <= 365_i64 + }) + .collect(); state.packagemeta.insert(ObjectSourceMeta { identifier: Rc::clone(nevra), - name: Rc::clone(nevra), + name: Rc::from(libdnf_sys::hy_split_nevra(&nevra)?.name), srcid: Rc::from(pkgmeta.src_pkg().to_str().unwrap()), change_time_offset, + change_frequency: pruned_changelogs.len() as u32, }); } @@ -316,14 +347,18 @@ pub fn container_encapsulate(args: Vec) -> CxxResult<()> { .map_err(anyhow::Error::msg)?; let initramfs = initramfs.downcast_ref::().unwrap(); let checksum = initramfs.checksum(); - let name = format!("initramfs (kernel {})", kernel_ver).into_boxed_str(); - let name = Rc::from(name); - state.content.insert(checksum.to_string(), Rc::clone(&name)); + let name = format!("initramfs"); + let identifier = format!("{} (kernel {})", name, kernel_ver).into_boxed_str(); + let identifier = Rc::from(identifier); + state + .content + .insert(checksum.to_string(), Rc::clone(&identifier)); state.packagemeta.insert(ObjectSourceMeta { - identifier: Rc::clone(&name), - name: Rc::clone(&name), - srcid: Rc::clone(&name), + identifier: Rc::clone(&identifier), + name: Rc::from(name), + srcid: Rc::clone(&identifier), change_time_offset: u32::MAX, + change_frequency: u32::MAX, }); state.skip.insert(path); } @@ -387,6 +422,12 @@ pub fn container_encapsulate(args: Vec) -> CxxResult<()> { }) .collect::>()?; + let package_structure = opt + .previous_build_manifest + .as_ref() + .map(|p| oci_spec::image::ImageManifest::from_file(&p).map_err(anyhow::Error::new)) + .transpose()?; + let mut copy_meta_keys = opt.copy_meta_keys; // Default to copying the input hash to support cheap change detection copy_meta_keys.push("rpmostree.inputhash".to_string()); @@ -409,6 +450,7 @@ pub fn container_encapsulate(args: Vec) -> CxxResult<()> { repo, rev.as_str(), &config, + package_structure.as_ref(), Some(opts), Some(meta), &opt.imgref, diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 84b79e9278..43b3688521 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -882,6 +882,7 @@ pub mod ffi { // Methods on PackageMeta fn size(self: &PackageMeta) -> u64; fn buildtime(self: &PackageMeta) -> u64; + fn changelogs(self: &PackageMeta) -> Vec; fn src_pkg(self: &PackageMeta) -> &CxxString; } diff --git a/src/libpriv/rpmostree-refts.cxx b/src/libpriv/rpmostree-refts.cxx index 0f2a6f468c..4a0a77b0f0 100644 --- a/src/libpriv/rpmostree-refts.cxx +++ b/src/libpriv/rpmostree-refts.cxx @@ -27,6 +27,15 @@ #include #include +static inline void +cleanup_rpmtdFreeData (rpmtd *tdp) +{ + rpmtd td = *tdp; + if (td) + rpmtdFreeData (td); +} +#define _cleanup_rpmtddata_ __attribute__ ((cleanup (cleanup_rpmtdFreeData))) + /* * A wrapper for an `rpmts` that supports: * @@ -114,6 +123,22 @@ RpmTs::package_meta (const rust::Str name) const retval->_size = headerGetNumber (h, RPMTAG_LONGARCHIVESIZE); retval->_buildtime = headerGetNumber (h, RPMTAG_BUILDTIME); retval->_src_pkg = headerGetString (h, RPMTAG_SOURCERPM); + + // Get the changelogs + struct rpmtd_s nchanges_date_s; + _cleanup_rpmtddata_ rpmtd nchanges_date = NULL; + nchanges_date = &nchanges_date_s; + headerGet (h, RPMTAG_CHANGELOGTIME, nchanges_date, HEADERGET_MINMEM); + int ncnum = rpmtdCount (nchanges_date); + rust::Vec epochs; + for (int i = 0; i < ncnum; i++) + { + uint64_t nchange_date = 0; + rpmtdNext (nchanges_date); + nchange_date = rpmtdGetNumber (nchanges_date); + epochs.push_back (nchange_date); + } + retval->_changelogs = std::move (epochs); } else { diff --git a/src/libpriv/rpmostree-refts.h b/src/libpriv/rpmostree-refts.h index b98770054b..ff018cd03f 100644 --- a/src/libpriv/rpmostree-refts.h +++ b/src/libpriv/rpmostree-refts.h @@ -52,6 +52,7 @@ struct PackageMeta { uint64_t _size; uint64_t _buildtime; + rust::Vec _changelogs; std::string _src_pkg; uint64_t @@ -64,6 +65,12 @@ struct PackageMeta { return _buildtime; }; + rust::Vec + changelogs () const + { + return _changelogs; + }; + const std::string & src_pkg () const {