ostree_ext/tar/
import.rs

1//! APIs for extracting OSTree commits from container images
2
3use crate::Result;
4use anyhow::{Context, anyhow, bail, ensure};
5use camino::Utf8Path;
6use camino::Utf8PathBuf;
7use fn_error_context::context;
8use gio::glib;
9use gio::prelude::*;
10use glib::Variant;
11use ostree::gio;
12use std::collections::BTreeSet;
13use std::collections::HashMap;
14use std::io::prelude::*;
15use tracing::{Level, event, instrument};
16
17/// Arbitrary limit on xattrs to avoid RAM exhaustion attacks. The actual filesystem limits are often much smaller.
18// See https://en.wikipedia.org/wiki/Extended_file_attributes
19// For example, XFS limits to 614 KiB.
20const MAX_XATTR_SIZE: u32 = 1024 * 1024;
21/// Limit on metadata objects (dirtree/dirmeta); this is copied
22/// from ostree-core.h.  TODO: Bind this in introspection
23const MAX_METADATA_SIZE: u32 = 10 * 1024 * 1024;
24
25/// Upper size limit for "small" regular files.
26// https://stackoverflow.com/questions/258091/when-should-i-use-mmap-for-file-access
27pub(crate) const SMALL_REGFILE_SIZE: usize = 127 * 1024;
28
29// The prefix for filenames that contain content we actually look at.
30pub(crate) const REPO_PREFIX: &str = "sysroot/ostree/repo/";
31/// Statistics from import.
32#[derive(Debug, Default)]
33struct ImportStats {
34    dirtree: u32,
35    dirmeta: u32,
36    regfile_small: u32,
37    regfile_large: u32,
38    symlinks: u32,
39}
40
41enum ImporterMode {
42    Commit(Option<String>),
43    ObjectSet(BTreeSet<String>),
44}
45
46/// Importer machine.
47pub(crate) struct Importer {
48    repo: ostree::Repo,
49    remote: Option<String>,
50    verify_text: Option<String>,
51    // Cache of xattrs, keyed by their content checksum.
52    xattrs: HashMap<String, glib::Variant>,
53    // Reusable buffer for xattrs references. It maps a file checksum (.0)
54    // to an xattrs checksum (.1) in the `xattrs` cache above.
55    next_xattrs: Option<(String, String)>,
56
57    // Reusable buffer for reads.  See also https://github.com/rust-lang/rust/issues/78485
58    buf: Vec<u8>,
59
60    stats: ImportStats,
61
62    /// Additional state depending on whether we're importing an object set or a commit.
63    data: ImporterMode,
64}
65
66/// Validate size/type of a tar header for OSTree metadata object.
67fn validate_metadata_header(header: &tar::Header, desc: &str) -> Result<usize> {
68    if header.entry_type() != tar::EntryType::Regular {
69        return Err(anyhow!("Invalid non-regular metadata object {}", desc));
70    }
71    let size = header.size()?;
72    let max_size = MAX_METADATA_SIZE as u64;
73    if size > max_size {
74        return Err(anyhow!(
75            "object of size {} exceeds {} bytes",
76            size,
77            max_size
78        ));
79    }
80    Ok(size as usize)
81}
82
83fn header_attrs(header: &tar::Header) -> Result<(u32, u32, u32)> {
84    let uid: u32 = header.uid()?.try_into()?;
85    let gid: u32 = header.gid()?.try_into()?;
86    let mode: u32 = header.mode()?;
87    Ok((uid, gid, mode))
88}
89
90// The C function ostree_object_type_from_string aborts on
91// unknown strings, so we have a safe version here.
92fn objtype_from_string(t: &str) -> Option<ostree::ObjectType> {
93    Some(match t {
94        "commit" => ostree::ObjectType::Commit,
95        "commitmeta" => ostree::ObjectType::CommitMeta,
96        "dirtree" => ostree::ObjectType::DirTree,
97        "dirmeta" => ostree::ObjectType::DirMeta,
98        "file" => ostree::ObjectType::File,
99        _ => return None,
100    })
101}
102
103/// Given a tar entry, read it all into a GVariant
104fn entry_to_variant<R: std::io::Read, T: StaticVariantType>(
105    mut entry: tar::Entry<R>,
106    desc: &str,
107) -> Result<glib::Variant> {
108    let header = entry.header();
109    let size = validate_metadata_header(header, desc)?;
110
111    let mut buf: Vec<u8> = Vec::with_capacity(size);
112    let n = std::io::copy(&mut entry, &mut buf)?;
113    assert_eq!(n as usize, size);
114    let v = glib::Bytes::from_owned(buf);
115    let v = Variant::from_bytes::<T>(&v);
116    Ok(v.normal_form())
117}
118
119/// Parse an object path into (parent, rest, objtype).
120///
121/// Normal ostree object paths look like 00/1234.commit.
122/// In the tar format, we may also see 00/1234.file.xattrs.
123fn parse_object_entry_path(path: &Utf8Path) -> Result<(&str, &Utf8Path, &str)> {
124    // The "sharded" commit directory.
125    let parentname = path
126        .parent()
127        .and_then(|p| p.file_name())
128        .ok_or_else(|| anyhow!("Invalid path (no parent) {}", path))?;
129    #[allow(clippy::needless_as_bytes)]
130    if !(parentname.is_ascii() && parentname.as_bytes().len() == 2) {
131        return Err(anyhow!("Invalid checksum parent {}", parentname));
132    }
133    let name = path
134        .file_name()
135        .map(Utf8Path::new)
136        .ok_or_else(|| anyhow!("Invalid path (dir) {}", path))?;
137    let objtype = name
138        .extension()
139        .ok_or_else(|| anyhow!("Invalid objpath {}", path))?;
140
141    Ok((parentname, name, objtype))
142}
143
144fn parse_checksum(parent: &str, name: &Utf8Path) -> Result<String> {
145    let checksum_rest = name
146        .file_stem()
147        .ok_or_else(|| anyhow!("Invalid object path part {}", name))?;
148    // Also take care of the double extension on `.file.xattrs`.
149    let checksum_rest = checksum_rest.trim_end_matches(".file");
150
151    #[allow(clippy::needless_as_bytes)]
152    if !(checksum_rest.is_ascii() && checksum_rest.as_bytes().len() == 62) {
153        return Err(anyhow!("Invalid checksum part {}", checksum_rest));
154    }
155    let reassembled = format!("{parent}{checksum_rest}");
156    validate_sha256(reassembled)
157}
158
159/// Parse a `.file-xattrs-link` link target into the corresponding checksum.
160fn parse_xattrs_link_target(path: &Utf8Path) -> Result<String> {
161    let (parent, rest, _objtype) = parse_object_entry_path(path)?;
162    parse_checksum(parent, rest)
163}
164
165impl Importer {
166    /// Create an importer which will import an OSTree commit object.
167    pub(crate) fn new_for_commit(repo: &ostree::Repo, remote: Option<String>) -> Self {
168        Self {
169            repo: repo.clone(),
170            remote,
171            verify_text: None,
172            buf: vec![0u8; 16384],
173            xattrs: Default::default(),
174            next_xattrs: None,
175            stats: Default::default(),
176            data: ImporterMode::Commit(None),
177        }
178    }
179
180    /// Create an importer to write an "object set"; a chunk of objects which is
181    /// usually streamed from a separate storage system, such as an OCI container image layer.
182    pub(crate) fn new_for_object_set(repo: &ostree::Repo) -> Self {
183        Self {
184            repo: repo.clone(),
185            remote: None,
186            verify_text: None,
187            buf: vec![0u8; 16384],
188            xattrs: Default::default(),
189            next_xattrs: None,
190            stats: Default::default(),
191            data: ImporterMode::ObjectSet(Default::default()),
192        }
193    }
194
195    // Given a tar entry, filter it out if it doesn't look like an object file in
196    // `/sysroot/ostree`.
197    // It is an error if the filename is invalid UTF-8.  If it is valid UTF-8, return
198    // an owned copy of the path.
199    fn filter_entry<R: std::io::Read>(
200        e: tar::Entry<R>,
201    ) -> Result<Option<(tar::Entry<R>, Utf8PathBuf)>> {
202        if e.header().entry_type() == tar::EntryType::Directory {
203            return Ok(None);
204        }
205        let orig_path = e.path()?;
206        let path = Utf8Path::from_path(&orig_path)
207            .ok_or_else(|| anyhow!("Invalid non-utf8 path {:?}", orig_path))?;
208        // Ignore the regular non-object file hardlinks we inject
209        if let Ok(path) = path.strip_prefix(REPO_PREFIX) {
210            // Filter out the repo config file and refs dir
211            if path.file_name() == Some("config") || path.starts_with("refs") {
212                return Ok(None);
213            }
214            let path = path.into();
215            Ok(Some((e, path)))
216        } else {
217            Ok(None)
218        }
219    }
220
221    pub(crate) fn parse_metadata_entry(path: &Utf8Path) -> Result<(String, ostree::ObjectType)> {
222        let (parentname, name, objtype) = parse_object_entry_path(path)?;
223        let checksum = parse_checksum(parentname, name)?;
224        let objtype = objtype_from_string(objtype)
225            .ok_or_else(|| anyhow!("Invalid object type {}", objtype))?;
226        Ok((checksum, objtype))
227    }
228
229    /// Import a metadata object.
230    #[context("Importing metadata object")]
231    fn import_metadata<R: std::io::Read>(
232        &mut self,
233        entry: tar::Entry<R>,
234        checksum: &str,
235        objtype: ostree::ObjectType,
236    ) -> Result<()> {
237        let v = match objtype {
238            ostree::ObjectType::DirTree => {
239                self.stats.dirtree += 1;
240                entry_to_variant::<_, ostree::TreeVariantType>(entry, checksum)?
241            }
242            ostree::ObjectType::DirMeta => {
243                self.stats.dirmeta += 1;
244                entry_to_variant::<_, ostree::DirmetaVariantType>(entry, checksum)?
245            }
246            o => return Err(anyhow!("Invalid metadata object type; {:?}", o)),
247        };
248        // FIXME validate here that this checksum was in the set we expected.
249        // https://github.com/ostreedev/ostree-rs-ext/issues/1
250        let actual =
251            self.repo
252                .write_metadata(objtype, Some(checksum), &v, gio::Cancellable::NONE)?;
253        assert_eq!(actual.to_hex(), checksum);
254        Ok(())
255    }
256
257    /// Import a content object, large regular file flavour.
258    #[context("Importing regfile")]
259    fn import_large_regfile_object<R: std::io::Read>(
260        &mut self,
261        mut entry: tar::Entry<R>,
262        size: usize,
263        checksum: &str,
264        xattrs: glib::Variant,
265        cancellable: Option<&gio::Cancellable>,
266    ) -> Result<()> {
267        let (uid, gid, mode) = header_attrs(entry.header())?;
268        let w = self.repo.write_regfile(
269            Some(checksum),
270            uid,
271            gid,
272            libc::S_IFREG | mode,
273            size as u64,
274            Some(&xattrs),
275        )?;
276        {
277            let w = w.clone().upcast::<gio::OutputStream>();
278            loop {
279                let n = entry
280                    .read(&mut self.buf[..])
281                    .context("Reading large regfile")?;
282                if n == 0 {
283                    break;
284                }
285                w.write(&self.buf[0..n], cancellable)
286                    .context("Writing large regfile")?;
287            }
288        }
289        let c = w.finish(cancellable)?;
290        debug_assert_eq!(c, checksum);
291        self.stats.regfile_large += 1;
292        Ok(())
293    }
294
295    /// Import a content object, small regular file flavour.
296    #[context("Importing regfile small")]
297    fn import_small_regfile_object<R: std::io::Read>(
298        &mut self,
299        mut entry: tar::Entry<R>,
300        size: usize,
301        checksum: &str,
302        xattrs: glib::Variant,
303        cancellable: Option<&gio::Cancellable>,
304    ) -> Result<()> {
305        let (uid, gid, mode) = header_attrs(entry.header())?;
306        assert!(size <= SMALL_REGFILE_SIZE);
307        let mut buf = vec![0u8; size];
308        entry.read_exact(&mut buf[..])?;
309        let c = self.repo.write_regfile_inline(
310            Some(checksum),
311            uid,
312            gid,
313            libc::S_IFREG | mode,
314            Some(&xattrs),
315            &buf,
316            cancellable,
317        )?;
318        debug_assert_eq!(c.as_str(), checksum);
319        self.stats.regfile_small += 1;
320        Ok(())
321    }
322
323    /// Import a content object, symlink flavour.
324    #[context("Importing symlink")]
325    fn import_symlink_object<R: std::io::Read>(
326        &mut self,
327        entry: tar::Entry<R>,
328        checksum: &str,
329        xattrs: glib::Variant,
330    ) -> Result<()> {
331        let (uid, gid, _) = header_attrs(entry.header())?;
332        let target = entry
333            .link_name()?
334            .ok_or_else(|| anyhow!("Invalid symlink"))?;
335        let target = target
336            .as_os_str()
337            .to_str()
338            .ok_or_else(|| anyhow!("Non-utf8 symlink"))?;
339        let c = self.repo.write_symlink(
340            Some(checksum),
341            uid,
342            gid,
343            Some(&xattrs),
344            target,
345            gio::Cancellable::NONE,
346        )?;
347        debug_assert_eq!(c.as_str(), checksum);
348        self.stats.symlinks += 1;
349        Ok(())
350    }
351
352    /// Import a content object.
353    #[context("Processing content object {}", checksum)]
354    fn import_content_object<R: std::io::Read>(
355        &mut self,
356        entry: tar::Entry<R>,
357        checksum: &str,
358        cancellable: Option<&gio::Cancellable>,
359    ) -> Result<()> {
360        let size: usize = entry.header().size()?.try_into()?;
361
362        // Pop the queued xattrs reference.
363        let (file_csum, xattrs_csum) = self
364            .next_xattrs
365            .take()
366            .ok_or_else(|| anyhow!("Missing xattrs reference"))?;
367        if checksum != file_csum {
368            return Err(anyhow!("Object mismatch, found xattrs for {}", file_csum));
369        }
370
371        if self
372            .repo
373            .has_object(ostree::ObjectType::File, checksum, cancellable)?
374        {
375            return Ok(());
376        }
377
378        // Retrieve xattrs content from the cache.
379        let xattrs = self
380            .xattrs
381            .get(&xattrs_csum)
382            .cloned()
383            .ok_or_else(|| anyhow!("Failed to find xattrs content {}", xattrs_csum,))?;
384
385        match entry.header().entry_type() {
386            tar::EntryType::Regular => {
387                if size > SMALL_REGFILE_SIZE {
388                    self.import_large_regfile_object(entry, size, checksum, xattrs, cancellable)
389                } else {
390                    self.import_small_regfile_object(entry, size, checksum, xattrs, cancellable)
391                }
392            }
393            tar::EntryType::Symlink => self.import_symlink_object(entry, checksum, xattrs),
394            o => Err(anyhow!("Invalid tar entry of type {:?}", o)),
395        }
396    }
397
398    /// Given a tar entry that looks like an object (its path is under ostree/repo/objects/),
399    /// determine its type and import it.
400    #[context("Importing object {}", path)]
401    fn import_object<R: std::io::Read>(
402        &mut self,
403        entry: tar::Entry<'_, R>,
404        path: &Utf8Path,
405        cancellable: Option<&gio::Cancellable>,
406    ) -> Result<()> {
407        let (parentname, name, suffix) = parse_object_entry_path(path)?;
408        let checksum = parse_checksum(parentname, name)?;
409
410        match suffix {
411            "commit" => Err(anyhow!("Found multiple commit objects")),
412            "file" => {
413                self.import_content_object(entry, &checksum, cancellable)?;
414                // Track the objects we wrote
415                match &mut self.data {
416                    ImporterMode::ObjectSet(imported) => {
417                        if let Some(p) = imported.replace(checksum) {
418                            anyhow::bail!("Duplicate object: {}", p);
419                        }
420                    }
421                    ImporterMode::Commit(_) => {}
422                }
423                Ok(())
424            }
425            "file-xattrs" => self.process_file_xattrs(entry, checksum),
426            "file-xattrs-link" => self.process_file_xattrs_link(entry, checksum),
427            "xattrs" => self.process_xattr_ref(entry, checksum),
428            kind => {
429                let objtype = objtype_from_string(kind)
430                    .ok_or_else(|| anyhow!("Invalid object type {}", kind))?;
431                match &mut self.data {
432                    ImporterMode::ObjectSet(_) => {
433                        anyhow::bail!(
434                            "Found metadata object {}.{:?} in object set mode",
435                            checksum,
436                            objtype
437                        );
438                    }
439                    ImporterMode::Commit(_) => {}
440                }
441                self.import_metadata(entry, &checksum, objtype)
442            }
443        }
444    }
445
446    /// Process a `.file-xattrs` object (v1).
447    #[context("Processing file xattrs")]
448    fn process_file_xattrs(
449        &mut self,
450        entry: tar::Entry<impl std::io::Read>,
451        checksum: String,
452    ) -> Result<()> {
453        self.cache_xattrs_content(entry, Some(checksum))?;
454        Ok(())
455    }
456
457    /// Process a `.file-xattrs-link` object (v1).
458    ///
459    /// This is an hardlink that contains extended attributes for a content object.
460    /// When the max hardlink count is reached, this object may also be encoded as
461    /// a regular file instead.
462    #[context("Processing xattrs link")]
463    fn process_file_xattrs_link(
464        &mut self,
465        entry: tar::Entry<impl std::io::Read>,
466        checksum: String,
467    ) -> Result<()> {
468        use tar::EntryType::{Link, Regular};
469        if let Some(prev) = &self.next_xattrs {
470            bail!(
471                "Found previous dangling xattrs for file object '{}'",
472                prev.0
473            );
474        }
475
476        // Extract the xattrs checksum from the link target or from the content (v1).
477        // Later, it will be used as the key for a lookup into the `self.xattrs` cache.
478        let xattrs_checksum = match entry.header().entry_type() {
479            Link => {
480                let link_target = entry
481                    .link_name()?
482                    .ok_or_else(|| anyhow!("No xattrs link content for {}", checksum))?;
483                let xattr_target = Utf8Path::from_path(&link_target)
484                    .ok_or_else(|| anyhow!("Invalid non-UTF8 xattrs link {}", checksum))?;
485                parse_xattrs_link_target(xattr_target)?
486            }
487            Regular => self.cache_xattrs_content(entry, None)?,
488            x => bail!("Unexpected xattrs type '{:?}' found for {}", x, checksum),
489        };
490
491        // Now xattrs are properly cached for the next content object in the stream,
492        // which should match `checksum`.
493        self.next_xattrs = Some((checksum, xattrs_checksum));
494
495        Ok(())
496    }
497
498    /// Process a `.file.xattrs` entry (v0).
499    ///
500    /// This is an hardlink that contains extended attributes for a content object.
501    #[context("Processing xattrs reference")]
502    fn process_xattr_ref<R: std::io::Read>(
503        &mut self,
504        entry: tar::Entry<R>,
505        target: String,
506    ) -> Result<()> {
507        if let Some(prev) = &self.next_xattrs {
508            bail!(
509                "Found previous dangling xattrs for file object '{}'",
510                prev.0
511            );
512        }
513
514        // Parse the xattrs checksum from the link target (v0).
515        // Later, it will be used as the key for a lookup into the `self.xattrs` cache.
516        let header = entry.header();
517        if header.entry_type() != tar::EntryType::Link {
518            bail!("Non-hardlink xattrs reference found for {}", target);
519        }
520        let xattr_target = entry
521            .link_name()?
522            .ok_or_else(|| anyhow!("No xattrs link content for {}", target))?;
523        let xattr_target = Utf8Path::from_path(&xattr_target)
524            .ok_or_else(|| anyhow!("Invalid non-UTF8 xattrs link {}", target))?;
525        let xattr_target = xattr_target
526            .file_name()
527            .ok_or_else(|| anyhow!("Invalid xattrs link {}", target))?
528            .to_string();
529        let xattrs_checksum = validate_sha256(xattr_target)?;
530
531        // Now xattrs are properly cached for the next content object in the stream,
532        // which should match `checksum`.
533        self.next_xattrs = Some((target, xattrs_checksum));
534
535        Ok(())
536    }
537
538    /// Process a special /xattrs/ entry, with checksum of xattrs content (v0).
539    fn process_split_xattrs_content<R: std::io::Read>(
540        &mut self,
541        entry: tar::Entry<R>,
542    ) -> Result<()> {
543        let checksum = {
544            let path = entry.path()?;
545            let name = path
546                .file_name()
547                .ok_or_else(|| anyhow!("Invalid xattrs dir: {:?}", path))?;
548            let name = name
549                .to_str()
550                .ok_or_else(|| anyhow!("Invalid non-UTF8 xattrs name: {:?}", name))?;
551            validate_sha256(name.to_string())?
552        };
553        self.cache_xattrs_content(entry, Some(checksum))?;
554        Ok(())
555    }
556
557    /// Read an xattrs entry and cache its content, optionally validating its checksum.
558    ///
559    /// This returns the computed checksum for the successfully cached content.
560    fn cache_xattrs_content<R: std::io::Read>(
561        &mut self,
562        mut entry: tar::Entry<R>,
563        expected_checksum: Option<String>,
564    ) -> Result<String> {
565        let header = entry.header();
566        if header.entry_type() != tar::EntryType::Regular {
567            return Err(anyhow!(
568                "Invalid xattr entry of type {:?}",
569                header.entry_type()
570            ));
571        }
572        let n = header.size()?;
573        if n > MAX_XATTR_SIZE as u64 {
574            return Err(anyhow!("Invalid xattr size {}", n));
575        }
576
577        let mut contents = vec![0u8; n as usize];
578        entry.read_exact(contents.as_mut_slice())?;
579        let data: glib::Bytes = contents.as_slice().into();
580        let xattrs_checksum = {
581            let digest = openssl::hash::hash(openssl::hash::MessageDigest::sha256(), &data)?;
582            hex::encode(digest)
583        };
584        if let Some(input) = expected_checksum {
585            ensure!(
586                input == xattrs_checksum,
587                "Checksum mismatch, expected '{}' but computed '{}'",
588                input,
589                xattrs_checksum
590            );
591        }
592
593        let contents = Variant::from_bytes::<&[(&[u8], &[u8])]>(&data);
594        self.xattrs.insert(xattrs_checksum.clone(), contents);
595        Ok(xattrs_checksum)
596    }
597
598    fn import_objects_impl<'a>(
599        &mut self,
600        ents: impl Iterator<Item = Result<(tar::Entry<'a, impl Read + Send + Unpin + 'a>, Utf8PathBuf)>>,
601        cancellable: Option<&gio::Cancellable>,
602    ) -> Result<()> {
603        for entry in ents {
604            let (entry, path) = entry?;
605            if let Ok(p) = path.strip_prefix("objects/") {
606                self.import_object(entry, p, cancellable)?;
607            } else if path.strip_prefix("xattrs/").is_ok() {
608                self.process_split_xattrs_content(entry)?;
609            }
610        }
611        Ok(())
612    }
613
614    #[context("Importing objects")]
615    pub(crate) fn import_objects(
616        &mut self,
617        archive: &mut tar::Archive<impl Read + Send + Unpin>,
618        cancellable: Option<&gio::Cancellable>,
619    ) -> Result<()> {
620        let ents = archive.entries()?.filter_map(|e| match e {
621            Ok(e) => Self::filter_entry(e).transpose(),
622            Err(e) => Some(Err(anyhow::Error::msg(e))),
623        });
624        self.import_objects_impl(ents, cancellable)
625    }
626
627    #[context("Importing commit")]
628    pub(crate) fn import_commit(
629        &mut self,
630        archive: &mut tar::Archive<impl Read + Send + Unpin>,
631        cancellable: Option<&gio::Cancellable>,
632    ) -> Result<()> {
633        // This can only be invoked once
634        assert!(matches!(self.data, ImporterMode::Commit(None)));
635        // Create an iterator that skips over directories; we just care about the file names.
636        let mut ents = archive.entries()?.filter_map(|e| match e {
637            Ok(e) => Self::filter_entry(e).transpose(),
638            Err(e) => Some(Err(anyhow::Error::msg(e))),
639        });
640        // Read the commit object.
641        let (commit_ent, commit_path) = ents
642            .next()
643            .ok_or_else(|| anyhow!("Commit object not found"))??;
644
645        if commit_ent.header().entry_type() != tar::EntryType::Regular {
646            return Err(anyhow!(
647                "Expected regular file for commit object, not {:?}",
648                commit_ent.header().entry_type()
649            ));
650        }
651        let (checksum, objtype) = Self::parse_metadata_entry(&commit_path)?;
652        if objtype != ostree::ObjectType::Commit {
653            return Err(anyhow!("Expected commit object, not {:?}", objtype));
654        }
655        let commit = entry_to_variant::<_, ostree::CommitVariantType>(commit_ent, &checksum)?;
656
657        let (next_ent, nextent_path) = ents
658            .next()
659            .ok_or_else(|| anyhow!("End of stream after commit object"))??;
660        let (next_checksum, next_objtype) = Self::parse_metadata_entry(&nextent_path)?;
661
662        if let Some(remote) = self.remote.as_deref() {
663            if next_objtype != ostree::ObjectType::CommitMeta {
664                return Err(anyhow!(
665                    "Using remote {} for verification; Expected commitmeta object, not {:?}",
666                    remote,
667                    next_objtype
668                ));
669            }
670            if next_checksum != checksum {
671                return Err(anyhow!(
672                    "Expected commitmeta checksum {}, found {}",
673                    checksum,
674                    next_checksum
675                ));
676            }
677            let commitmeta = entry_to_variant::<_, std::collections::HashMap<String, glib::Variant>>(
678                next_ent,
679                &next_checksum,
680            )?;
681
682            // Now that we have both the commit and detached metadata in memory, verify that
683            // the signatures in the detached metadata correctly sign the commit.
684            self.verify_text = Some(
685                self.repo
686                    .signature_verify_commit_data(
687                        remote,
688                        &commit.data_as_bytes(),
689                        &commitmeta.data_as_bytes(),
690                        ostree::RepoVerifyFlags::empty(),
691                    )
692                    .context("Verifying ostree commit in tar stream")?
693                    .into(),
694            );
695
696            self.repo.mark_commit_partial(&checksum, true)?;
697
698            // Write the commit object, which also verifies its checksum.
699            let actual_checksum =
700                self.repo
701                    .write_metadata(objtype, Some(&checksum), &commit, cancellable)?;
702            assert_eq!(actual_checksum.to_hex(), checksum);
703            event!(Level::DEBUG, "Imported {}.commit", checksum);
704
705            // Finally, write the detached metadata.
706            self.repo
707                .write_commit_detached_metadata(&checksum, Some(&commitmeta), cancellable)?;
708        } else {
709            self.repo.mark_commit_partial(&checksum, true)?;
710
711            // We're not doing any validation of the commit, so go ahead and write it.
712            let actual_checksum =
713                self.repo
714                    .write_metadata(objtype, Some(&checksum), &commit, cancellable)?;
715            assert_eq!(actual_checksum.to_hex(), checksum);
716            event!(Level::DEBUG, "Imported {}.commit", checksum);
717
718            // Write the next object, whether it's commit metadata or not.
719            let (meta_checksum, meta_objtype) = Self::parse_metadata_entry(&nextent_path)?;
720            match meta_objtype {
721                ostree::ObjectType::CommitMeta => {
722                    let commitmeta = entry_to_variant::<
723                        _,
724                        std::collections::HashMap<String, glib::Variant>,
725                    >(next_ent, &meta_checksum)?;
726                    self.repo.write_commit_detached_metadata(
727                        &checksum,
728                        Some(&commitmeta),
729                        gio::Cancellable::NONE,
730                    )?;
731                }
732                _ => {
733                    self.import_object(next_ent, &nextent_path, cancellable)?;
734                }
735            }
736        }
737        match &mut self.data {
738            ImporterMode::Commit(c) => {
739                c.replace(checksum);
740            }
741            ImporterMode::ObjectSet(_) => unreachable!(),
742        }
743
744        self.import_objects_impl(ents, cancellable)?;
745
746        Ok(())
747    }
748
749    pub(crate) fn finish_import_commit(self) -> (String, Option<String>) {
750        tracing::debug!("Import stats: {:?}", self.stats);
751        match self.data {
752            ImporterMode::Commit(c) => (c.unwrap(), self.verify_text),
753            ImporterMode::ObjectSet(_) => unreachable!(),
754        }
755    }
756
757    pub(crate) fn default_dirmeta() -> glib::Variant {
758        let finfo = gio::FileInfo::new();
759        finfo.set_attribute_uint32("unix::uid", 0);
760        finfo.set_attribute_uint32("unix::gid", 0);
761        finfo.set_attribute_uint32("unix::mode", libc::S_IFDIR | 0o755);
762        // SAFETY: TODO: This is not a nullable return, fix it in ostree
763        ostree::create_directory_metadata(&finfo, None)
764    }
765
766    pub(crate) fn finish_import_object_set(self) -> Result<String> {
767        let objset = match self.data {
768            ImporterMode::Commit(_) => unreachable!(),
769            ImporterMode::ObjectSet(s) => s,
770        };
771        tracing::debug!("Imported {} content objects", objset.len());
772        let mtree = ostree::MutableTree::new();
773        for checksum in objset.into_iter() {
774            mtree.replace_file(&checksum, &checksum)?;
775        }
776        let dirmeta = self.repo.write_metadata(
777            ostree::ObjectType::DirMeta,
778            None,
779            &Self::default_dirmeta(),
780            gio::Cancellable::NONE,
781        )?;
782        mtree.set_metadata_checksum(&dirmeta.to_hex());
783        let tree = self.repo.write_mtree(&mtree, gio::Cancellable::NONE)?;
784        let commit = self.repo.write_commit_with_time(
785            None,
786            None,
787            None,
788            None,
789            tree.downcast_ref().unwrap(),
790            0,
791            gio::Cancellable::NONE,
792        )?;
793        Ok(commit.to_string())
794    }
795}
796
797fn validate_sha256(input: String) -> Result<String> {
798    if input.len() != 64 {
799        return Err(anyhow!("Invalid sha256 checksum (len) {}", input));
800    }
801    if !input.chars().all(|c| matches!(c, '0'..='9' | 'a'..='f')) {
802        return Err(anyhow!("Invalid sha256 checksum {}", input));
803    }
804    Ok(input)
805}
806
807/// Configuration for tar import.
808#[derive(Debug, Default)]
809#[non_exhaustive]
810pub struct TarImportOptions {
811    /// Name of the remote to use for signature verification.
812    pub remote: Option<String>,
813}
814
815/// Read the contents of a tarball and import the ostree commit inside.
816/// Returns the sha256 of the imported commit.
817#[instrument(level = "debug", skip_all)]
818pub async fn import_tar(
819    repo: &ostree::Repo,
820    src: impl tokio::io::AsyncRead + Send + Unpin + 'static,
821    options: Option<TarImportOptions>,
822) -> Result<String> {
823    let options = options.unwrap_or_default();
824    let src = tokio_util::io::SyncIoBridge::new(src);
825    let repo = repo.clone();
826    // The tar code we use today is blocking, so we spawn a thread.
827    crate::tokio_util::spawn_blocking_cancellable_flatten(move |cancellable| {
828        let mut archive = tar::Archive::new(src);
829        let txn = repo.auto_transaction(Some(cancellable))?;
830        let mut importer = Importer::new_for_commit(&repo, options.remote);
831        importer.import_commit(&mut archive, Some(cancellable))?;
832        let (checksum, _) = importer.finish_import_commit();
833        txn.commit(Some(cancellable))?;
834        repo.mark_commit_partial(&checksum, false)?;
835        Ok::<_, anyhow::Error>(checksum)
836    })
837    .await
838}
839
840/// Read the contents of a tarball and import the content objects inside.
841/// Generates a synthetic commit object referencing them.
842#[instrument(level = "debug", skip_all)]
843pub async fn import_tar_objects(
844    repo: &ostree::Repo,
845    src: impl tokio::io::AsyncRead + Send + Unpin + 'static,
846) -> Result<String> {
847    let src = tokio_util::io::SyncIoBridge::new(src);
848    let repo = repo.clone();
849    // The tar code we use today is blocking, so we spawn a thread.
850    crate::tokio_util::spawn_blocking_cancellable_flatten(move |cancellable| {
851        let mut archive = tar::Archive::new(src);
852        let mut importer = Importer::new_for_object_set(&repo);
853        let txn = repo.auto_transaction(Some(cancellable))?;
854        importer.import_objects(&mut archive, Some(cancellable))?;
855        let r = importer.finish_import_object_set()?;
856        txn.commit(Some(cancellable))?;
857        Ok::<_, anyhow::Error>(r)
858    })
859    .await
860}
861
862#[cfg(test)]
863mod tests {
864    use super::*;
865
866    #[test]
867    fn test_parse_metadata_entry() {
868        let c = "a8/6d80a3e9ff77c2e3144c787b7769b300f91ffd770221aac27bab854960b964";
869        let invalid = format!("{c}.blah");
870        for &k in &["", "42", c, &invalid] {
871            assert!(Importer::parse_metadata_entry(k.into()).is_err())
872        }
873        let valid = format!("{c}.commit");
874        let r = Importer::parse_metadata_entry(valid.as_str().into()).unwrap();
875        assert_eq!(r.0, c.replace('/', ""));
876        assert_eq!(r.1, ostree::ObjectType::Commit);
877    }
878
879    #[test]
880    fn test_validate_sha256() {
881        let err_cases = &[
882            "a86d80a3e9ff77c2e3144c787b7769b300f91ffd770221aac27bab854960b9644",
883            "a86d80a3E9ff77c2e3144c787b7769b300f91ffd770221aac27bab854960b964",
884        ];
885        for input in err_cases {
886            validate_sha256(input.to_string()).unwrap_err();
887        }
888
889        validate_sha256(
890            "a86d80a3e9ff77c2e3144c787b7769b300f91ffd770221aac27bab854960b964".to_string(),
891        )
892        .unwrap();
893    }
894
895    #[test]
896    fn test_parse_object_entry_path() {
897        let path = "sysroot/ostree/repo/objects/b8/627e3ef0f255a322d2bd9610cfaaacc8f122b7f8d17c0e7e3caafa160f9fc7.file.xattrs";
898        let input = Utf8PathBuf::from(path);
899        let expected_parent = "b8";
900        let expected_rest =
901            "627e3ef0f255a322d2bd9610cfaaacc8f122b7f8d17c0e7e3caafa160f9fc7.file.xattrs";
902        let expected_objtype = "xattrs";
903        let output = parse_object_entry_path(&input).unwrap();
904        assert_eq!(output.0, expected_parent);
905        assert_eq!(output.1, expected_rest);
906        assert_eq!(output.2, expected_objtype);
907    }
908
909    #[test]
910    fn test_parse_checksum() {
911        let parent = "b8";
912        let name = "627e3ef0f255a322d2bd9610cfaaacc8f122b7f8d17c0e7e3caafa160f9fc7.file.xattrs";
913        let expected = "b8627e3ef0f255a322d2bd9610cfaaacc8f122b7f8d17c0e7e3caafa160f9fc7";
914        let output = parse_checksum(parent, &Utf8PathBuf::from(name)).unwrap();
915        assert_eq!(output, expected);
916    }
917
918    #[test]
919    fn test_parse_xattrs_link_target() {
920        let err_cases = &[
921            "",
922            "b8627e3ef0f255a322d2bd9610cfaaacc8f122b7f8d17c0e7e3caafa160f9fc7.file-xattrs",
923            "../b8/62.file-xattrs",
924        ];
925        for input in err_cases {
926            parse_xattrs_link_target(Utf8Path::new(input)).unwrap_err();
927        }
928
929        let ok_cases = &[
930            "../b8/627e3ef0f255a322d2bd9610cfaaacc8f122b7f8d17c0e7e3caafa160f9fc7.file-xattrs",
931            "sysroot/ostree/repo/objects/b8/627e3ef0f255a322d2bd9610cfaaacc8f122b7f8d17c0e7e3caafa160f9fc7.file-xattrs",
932        ];
933        let expected = "b8627e3ef0f255a322d2bd9610cfaaacc8f122b7f8d17c0e7e3caafa160f9fc7";
934        for input in ok_cases {
935            let output = parse_xattrs_link_target(Utf8Path::new(input)).unwrap();
936            assert_eq!(output, expected);
937        }
938    }
939}
ostree_ext/tar/import.rs

ostree_ext/tar/
import.rs
