ostree_ext/tar/
export.rs

1//! APIs for creating container images from OSTree commits
2
3use crate::chunking;
4use crate::objgv::*;
5use anyhow::{Context, Result, anyhow, ensure};
6use camino::{Utf8Path, Utf8PathBuf};
7use fn_error_context::context;
8use gio::glib;
9use gio::prelude::*;
10use gvariant::aligned_bytes::TryAsAligned;
11use gvariant::{Marker, Structure};
12use ostree::gio;
13use std::borrow::Borrow;
14use std::borrow::Cow;
15use std::collections::HashSet;
16use std::ffi::CStr;
17use std::io::BufReader;
18
19/// The repository mode generated by a tar export stream.
20pub const BARE_SPLIT_XATTRS_MODE: &str = "bare-split-xattrs";
21
22/// The SELinux xattr. Because the ostree xattrs require an embedded NUL, we
23/// store that version as a constant.
24pub(crate) const SECURITY_SELINUX_XATTR_C: &CStr = c"security.selinux";
25/// Then derive a string version (without the NUL) from the above.
26pub(crate) const SECURITY_SELINUX_XATTR: &str = const {
27    match SECURITY_SELINUX_XATTR_C.to_str() {
28        Ok(r) => r,
29        Err(_) => unreachable!(),
30    }
31};
32
33// This is both special in the tar stream *and* it's in the ostree commit.
34const SYSROOT: &str = "sysroot";
35// This way the default ostree -> sysroot/ostree symlink works.
36const OSTREEDIR: &str = "sysroot/ostree";
37// The ref added (under ostree/) in the exported OSTree repo pointing at the commit.
38#[allow(dead_code)]
39const OSTREEREF: &str = "encapsulated";
40
41/// In v0 format, we use this relative path prefix.  I think I chose this by looking
42/// at the current Fedora base image tar stream.  However, several others don't do
43/// this and have paths be relative by simply omitting `./`, i.e. the tar stream
44/// contains `usr/bin/bash` and not `./usr/bin/bash`.  The former looks cleaner
45/// to me, so in v1 we drop it.
46const TAR_PATH_PREFIX_V0: &str = "./";
47
48/// The base repository configuration that identifies this is a tar export.
49// See https://github.com/ostreedev/ostree/issues/2499
50const REPO_CONFIG: &str = r#"[core]
51repo_version=1
52mode=bare-split-xattrs
53"#;
54
55/// A decently large buffer, as used by e.g. coreutils `cat`.
56/// System calls are expensive.
57const BUF_CAPACITY: usize = 131072;
58
59/// Convert `from` to `to`
60fn map_path_inner<'p>(
61    p: &'p Utf8Path,
62    from: &'_ str,
63    to: &'_ str,
64) -> std::borrow::Cow<'p, Utf8Path> {
65    match p.strip_prefix(from) {
66        Ok(r) => {
67            if r.components().count() > 0 {
68                Cow::Owned(Utf8Path::new(to).join(r))
69            } else {
70                Cow::Owned(Utf8PathBuf::from(to))
71            }
72        }
73        _ => Cow::Borrowed(p),
74    }
75}
76
77/// Convert /usr/etc back to /etc
78fn map_path(p: &Utf8Path) -> std::borrow::Cow<'_, Utf8Path> {
79    map_path_inner(p, "./usr/etc", "./etc")
80}
81
82/// Convert etc to usr/etc
83/// Note: no leading '/' or './'
84fn unmap_path(p: &Utf8Path) -> std::borrow::Cow<'_, Utf8Path> {
85    map_path_inner(p, "etc", "usr/etc")
86}
87
88/// Convert usr/etc back to etc for the tar stream.
89fn map_path_v1(p: &Utf8Path) -> &Utf8Path {
90    debug_assert!(!p.starts_with("/") && !p.starts_with("."));
91    if p.starts_with("usr/etc") {
92        p.strip_prefix("usr/").unwrap()
93    } else {
94        p
95    }
96}
97
98/// Given two paths, which may be absolute (starting with /) or
99/// start with `./`, return true if they are equal after removing
100/// those prefixes. This is effectively "would these paths be equal"
101/// when processed as a tar entry.
102pub(crate) fn path_equivalent_for_tar(a: impl AsRef<Utf8Path>, b: impl AsRef<Utf8Path>) -> bool {
103    fn strip_prefix(p: &Utf8Path) -> &Utf8Path {
104        if let Ok(p) = p.strip_prefix("/") {
105            return p;
106        } else if let Ok(p) = p.strip_prefix("./") {
107            return p;
108        }
109        p
110    }
111    strip_prefix(a.as_ref()) == strip_prefix(b.as_ref())
112}
113
114struct OstreeTarWriter<'a, W: std::io::Write> {
115    repo: &'a ostree::Repo,
116    commit_checksum: &'a str,
117    commit_object: glib::Variant,
118    out: &'a mut tar::Builder<W>,
119    #[allow(dead_code)]
120    options: ExportOptions,
121    wrote_initdirs: bool,
122    /// True if we're only writing directories
123    structure_only: bool,
124    wrote_vartmp: bool, // Set if the ostree commit contains /var/tmp
125    wrote_dirtree: HashSet<String>,
126    wrote_dirmeta: HashSet<String>,
127    wrote_content: HashSet<String>,
128}
129
130pub(crate) fn object_path(objtype: ostree::ObjectType, checksum: &str) -> Utf8PathBuf {
131    let suffix = match objtype {
132        ostree::ObjectType::Commit => "commit",
133        ostree::ObjectType::CommitMeta => "commitmeta",
134        ostree::ObjectType::DirTree => "dirtree",
135        ostree::ObjectType::DirMeta => "dirmeta",
136        ostree::ObjectType::File => "file",
137        o => panic!("Unexpected object type: {o:?}"),
138    };
139    let (first, rest) = checksum.split_at(2);
140    format!("{OSTREEDIR}/repo/objects/{first}/{rest}.{suffix}").into()
141}
142
143fn v1_xattrs_object_path(checksum: &str) -> Utf8PathBuf {
144    let (first, rest) = checksum.split_at(2);
145    format!("{OSTREEDIR}/repo/objects/{first}/{rest}.file-xattrs-link").into()
146}
147
148/// Check for "denormal" symlinks which contain "//"
149// See https://github.com/fedora-sysv/chkconfig/pull/67
150// [root@cosa-devsh ~]# rpm -qf /usr/lib/systemd/systemd-sysv-install
151// chkconfig-1.13-2.el8.x86_64
152// [root@cosa-devsh ~]# ll /usr/lib/systemd/systemd-sysv-install
153// lrwxrwxrwx. 2 root root 24 Nov 29 18:08 /usr/lib/systemd/systemd-sysv-install -> ../../..//sbin/chkconfig
154// [root@cosa-devsh ~]#
155fn symlink_is_denormal(target: &str) -> bool {
156    target.contains("//")
157}
158
159pub(crate) fn tar_append_default_data(
160    out: &mut tar::Builder<impl std::io::Write>,
161    path: &Utf8Path,
162    buf: &[u8],
163) -> Result<()> {
164    let mut h = tar::Header::new_gnu();
165    h.set_entry_type(tar::EntryType::Regular);
166    h.set_uid(0);
167    h.set_gid(0);
168    h.set_mode(0o644);
169    h.set_size(buf.len() as u64);
170    out.append_data(&mut h, path, buf).map_err(Into::into)
171}
172
173impl<'a, W: std::io::Write> OstreeTarWriter<'a, W> {
174    fn new(
175        repo: &'a ostree::Repo,
176        commit_checksum: &'a str,
177        out: &'a mut tar::Builder<W>,
178        options: ExportOptions,
179    ) -> Result<Self> {
180        let commit_object = repo.load_commit(commit_checksum)?.0;
181        let r = Self {
182            repo,
183            commit_checksum,
184            commit_object,
185            out,
186            options,
187            wrote_initdirs: false,
188            structure_only: false,
189            wrote_vartmp: false,
190            wrote_dirmeta: HashSet::new(),
191            wrote_dirtree: HashSet::new(),
192            wrote_content: HashSet::new(),
193        };
194        Ok(r)
195    }
196
197    /// Convert the ostree mode to tar mode.
198    /// The ostree mode bits include the format, tar does not.
199    /// Historically in format version 0 we injected them, so we need to keep doing so.
200    fn filter_mode(&self, mode: u32) -> u32 {
201        mode & !libc::S_IFMT
202    }
203
204    /// Add a directory entry with default permissions (root/root 0755)
205    fn append_default_dir(&mut self, path: &Utf8Path) -> Result<()> {
206        let mut h = tar::Header::new_gnu();
207        h.set_entry_type(tar::EntryType::Directory);
208        h.set_uid(0);
209        h.set_gid(0);
210        h.set_mode(0o755);
211        h.set_size(0);
212        self.out.append_data(&mut h, path, &mut std::io::empty())?;
213        Ok(())
214    }
215
216    /// Add a regular file entry with default permissions (root/root 0644)
217    fn append_default_data(&mut self, path: &Utf8Path, buf: &[u8]) -> Result<()> {
218        tar_append_default_data(self.out, path, buf)
219    }
220
221    /// Write the initial /sysroot/ostree/repo structure.
222    fn write_repo_structure(&mut self) -> Result<()> {
223        if self.wrote_initdirs {
224            return Ok(());
225        }
226
227        let objdir: Utf8PathBuf = format!("{OSTREEDIR}/repo/objects").into();
228        // Add all parent directories
229        let parent_dirs = {
230            let mut parts: Vec<_> = objdir.ancestors().collect();
231            parts.reverse();
232            parts
233        };
234        for path in parent_dirs {
235            match path.as_str() {
236                "/" | "" => continue,
237                _ => {}
238            }
239            self.append_default_dir(path)?;
240        }
241        // Object subdirectories
242        for d in 0..=0xFF {
243            let path: Utf8PathBuf = format!("{objdir}/{d:02x}").into();
244            self.append_default_dir(&path)?;
245        }
246        // Standard repo subdirectories.
247        let subdirs = [
248            "extensions",
249            "refs",
250            "refs/heads",
251            "refs/mirrors",
252            "refs/remotes",
253            "state",
254            "tmp",
255            "tmp/cache",
256        ];
257        for d in subdirs {
258            let path: Utf8PathBuf = format!("{OSTREEDIR}/repo/{d}").into();
259            self.append_default_dir(&path)?;
260        }
261
262        // Repository configuration file.
263        {
264            let path = format!("{OSTREEDIR}/repo/config");
265            self.append_default_data(Utf8Path::new(&path), REPO_CONFIG.as_bytes())?;
266        }
267
268        self.wrote_initdirs = true;
269        Ok(())
270    }
271
272    /// Recursively serialize a commit object to the target tar stream.
273    fn write_commit(&mut self) -> Result<()> {
274        let cancellable = gio::Cancellable::NONE;
275
276        let commit_bytes = self.commit_object.data_as_bytes();
277        let commit_bytes = commit_bytes.try_as_aligned()?;
278        let commit = gv_commit!().cast(commit_bytes);
279        let commit = commit.to_tuple();
280        let contents = hex::encode(commit.6);
281        let metadata_checksum = &hex::encode(commit.7);
282        let metadata_v = self
283            .repo
284            .load_variant(ostree::ObjectType::DirMeta, metadata_checksum)?;
285        // Safety: We passed the correct variant type just above
286        let metadata = &ostree::DirMetaParsed::from_variant(&metadata_v).unwrap();
287        let rootpath = Utf8Path::new(TAR_PATH_PREFIX_V0);
288
289        // We need to write the root directory, before we write any objects.  This should be the very
290        // first thing.
291        self.append_dir(rootpath, metadata)?;
292
293        if !self.options.raw {
294            // Now, we create sysroot/ and everything under it
295            self.write_repo_structure()?;
296
297            self.append_commit_object()?;
298
299            // The ostree dirmeta object for the root.
300            self.append(ostree::ObjectType::DirMeta, metadata_checksum, &metadata_v)?;
301        }
302
303        // Recurse and write everything else.
304        self.append_dirtree(
305            Utf8Path::new(TAR_PATH_PREFIX_V0),
306            contents,
307            true,
308            cancellable,
309        )?;
310
311        self.append_standard_var(cancellable)?;
312
313        Ok(())
314    }
315
316    fn append_commit_object(&mut self) -> Result<()> {
317        self.append(
318            ostree::ObjectType::Commit,
319            self.commit_checksum,
320            &self.commit_object.clone(),
321        )?;
322        if let Some(commitmeta) = self
323            .repo
324            .read_commit_detached_metadata(self.commit_checksum, gio::Cancellable::NONE)?
325        {
326            self.append(
327                ostree::ObjectType::CommitMeta,
328                self.commit_checksum,
329                &commitmeta,
330            )?;
331        }
332        Ok(())
333    }
334
335    fn append(
336        &mut self,
337        objtype: ostree::ObjectType,
338        checksum: &str,
339        v: &glib::Variant,
340    ) -> Result<()> {
341        let set = match objtype {
342            ostree::ObjectType::Commit | ostree::ObjectType::CommitMeta => None,
343            ostree::ObjectType::DirTree => Some(&mut self.wrote_dirtree),
344            ostree::ObjectType::DirMeta => Some(&mut self.wrote_dirmeta),
345            o => panic!("Unexpected object type: {o:?}"),
346        };
347        if let Some(set) = set {
348            if set.contains(checksum) {
349                return Ok(());
350            }
351            let inserted = set.insert(checksum.to_string());
352            debug_assert!(inserted);
353        }
354
355        let data = v.data_as_bytes();
356        let data = data.as_ref();
357        self.append_default_data(&object_path(objtype, checksum), data)
358            .with_context(|| format!("Writing object {checksum}"))?;
359        Ok(())
360    }
361
362    /// Export xattrs in ostree-container style format, which is a .xattrs file.
363    /// This is different from xattrs which may appear as e.g. PAX metadata, which we don't use
364    /// at the moment.
365    ///
366    /// Return whether content was written.
367    #[context("Writing xattrs")]
368    fn append_ostree_xattrs(&mut self, checksum: &str, xattrs: &glib::Variant) -> Result<bool> {
369        let xattrs_data = xattrs.data_as_bytes();
370        let xattrs_data = xattrs_data.as_ref();
371
372        let path = v1_xattrs_object_path(&checksum);
373        self.append_default_data(&path, xattrs_data)?;
374
375        Ok(true)
376    }
377
378    /// Append all xattrs to the tar stream *except* security.selinux, because
379    /// that one doesn't become visible in `podman run` anyways, so we couldn't
380    /// rely on it in some cases.
381    /// https://github.com/containers/storage/blob/0d4a8d2aaf293c9f0464b888d932ab5147a284b9/pkg/archive/archive.go#L85
382    #[context("Writing tar xattrs")]
383    fn append_tarstream_xattrs(&mut self, xattrs: &glib::Variant) -> Result<()> {
384        let v = xattrs.data_as_bytes();
385        let v = v.try_as_aligned().unwrap();
386        let v = gvariant::gv!("a(ayay)").cast(v);
387        let mut pax_extensions = Vec::new();
388        for entry in v {
389            let (k, v) = entry.to_tuple();
390            let k = CStr::from_bytes_with_nul(k).unwrap();
391            let k = k
392                .to_str()
393                .with_context(|| format!("Found non-UTF8 xattr: {k:?}"))?;
394            if k == SECURITY_SELINUX_XATTR {
395                continue;
396            }
397            pax_extensions.push((format!("SCHILY.xattr.{k}"), v));
398        }
399        self.out
400            .append_pax_extensions(pax_extensions.iter().map(|(k, v)| (k.as_str(), *v)))?;
401        Ok(())
402    }
403
404    /// Write a content object, returning the path/header that should be used
405    /// as a hard link to it in the target path. This matches how ostree checkouts work.
406    fn append_content(&mut self, checksum: &str) -> Result<(Utf8PathBuf, tar::Header)> {
407        let path = object_path(ostree::ObjectType::File, checksum);
408
409        let (instream, meta, xattrs) = self.repo.load_file(checksum, gio::Cancellable::NONE)?;
410
411        let mut h = tar::Header::new_gnu();
412        h.set_uid(meta.attribute_uint32("unix::uid") as u64);
413        h.set_gid(meta.attribute_uint32("unix::gid") as u64);
414        let mode = meta.attribute_uint32("unix::mode");
415        h.set_mode(self.filter_mode(mode));
416        if instream.is_some() {
417            h.set_entry_type(tar::EntryType::Regular);
418            h.set_size(meta.size() as u64);
419        } else {
420            h.set_entry_type(tar::EntryType::Symlink);
421            h.set_size(0);
422        }
423        if !self.wrote_content.contains(checksum) {
424            let inserted = self.wrote_content.insert(checksum.to_string());
425            debug_assert!(inserted);
426
427            // The xattrs objects need to be exported before the regular object they
428            // refer to. Otherwise the importing logic won't have the xattrs available
429            // when importing file content.
430            self.append_ostree_xattrs(checksum, &xattrs)?;
431            self.append_tarstream_xattrs(&xattrs)?;
432
433            if let Some(instream) = instream {
434                ensure!(meta.file_type() == gio::FileType::Regular);
435
436                let mut instream = BufReader::with_capacity(BUF_CAPACITY, instream.into_read());
437                self.out
438                    .append_data(&mut h, &path, &mut instream)
439                    .with_context(|| format!("Writing regfile {checksum}"))?;
440            } else {
441                ensure!(meta.file_type() == gio::FileType::SymbolicLink);
442
443                let target = meta
444                    .symlink_target()
445                    .ok_or_else(|| anyhow!("Missing symlink target"))?;
446                let target = target
447                    .to_str()
448                    .ok_or_else(|| anyhow!("Invalid UTF-8 symlink target: {target:?}"))?;
449                let context = || format!("Writing content symlink: {checksum}");
450                // Handle //chkconfig, see above
451                if symlink_is_denormal(target) {
452                    h.set_link_name_literal(target).with_context(context)?;
453                    self.out
454                        .append_data(&mut h, &path, &mut std::io::empty())
455                        .with_context(context)?;
456                } else {
457                    self.out
458                        .append_link(&mut h, &path, target)
459                        .with_context(context)?;
460                }
461            }
462        }
463
464        Ok((path, h))
465    }
466
467    /// Write a directory using the provided metadata.
468    fn append_dir(&mut self, dirpath: &Utf8Path, meta: &ostree::DirMetaParsed) -> Result<()> {
469        let mut header = tar::Header::new_gnu();
470        header.set_entry_type(tar::EntryType::Directory);
471        header.set_size(0);
472        header.set_uid(meta.uid as u64);
473        header.set_gid(meta.gid as u64);
474        header.set_mode(self.filter_mode(meta.mode));
475        self.out
476            .append_data(&mut header, dirpath, std::io::empty())?;
477        Ok(())
478    }
479
480    /// Given a source object (in e.g. ostree/repo/objects/...), write a hardlink to it
481    /// in its expected target path (e.g. `usr/bin/bash`).
482    fn append_content_hardlink(
483        &mut self,
484        srcpath: &Utf8Path,
485        mut h: tar::Header,
486        dest: &Utf8Path,
487    ) -> Result<()> {
488        // Don't create hardlinks to zero-sized files, it's much more likely
489        // to result in generated tar streams from container builds resulting
490        // in a modified linked-to file in /sysroot, which we don't currently handle.
491        // And in the case where the input is *not* zero sized, we still output
492        // a hardlink of size zero, as this is what is normal.
493        let is_regular_zerosized = if h.entry_type() == tar::EntryType::Regular {
494            let size = h.size().context("Querying size for hardlink append")?;
495            size == 0
496        } else {
497            false
498        };
499        // Link sizes should always be zero
500        h.set_size(0);
501        if is_regular_zerosized {
502            self.out.append_data(&mut h, dest, &mut std::io::empty())?;
503        } else {
504            h.set_entry_type(tar::EntryType::Link);
505            h.set_link_name(srcpath)?;
506            self.out.append_data(&mut h, dest, &mut std::io::empty())?;
507        }
508        Ok(())
509    }
510
511    /// Write a dirtree object.
512    fn append_dirtree<C: IsA<gio::Cancellable>>(
513        &mut self,
514        dirpath: &Utf8Path,
515        checksum: String,
516        is_root: bool,
517        cancellable: Option<&C>,
518    ) -> Result<()> {
519        let v = &self
520            .repo
521            .load_variant(ostree::ObjectType::DirTree, &checksum)?;
522        self.append(ostree::ObjectType::DirTree, &checksum, v)?;
523        drop(checksum);
524        let v = v.data_as_bytes();
525        let v = v.try_as_aligned()?;
526        let v = gv_dirtree!().cast(v);
527        let (files, dirs) = v.to_tuple();
528
529        if let Some(c) = cancellable {
530            c.set_error_if_cancelled()?;
531        }
532
533        if !self.structure_only {
534            for file in files {
535                let (name, csum) = file.to_tuple();
536                let name = name.to_str();
537                let checksum = &hex::encode(csum);
538                let (objpath, h) = self.append_content(checksum)?;
539                let subpath = &dirpath.join(name);
540                let subpath = map_path(subpath);
541                self.append_content_hardlink(&objpath, h, &subpath)
542                    .with_context(|| format!("Hardlinking {checksum} to {subpath}"))?;
543            }
544        }
545
546        // Record if the ostree commit includes /var/tmp; if so we don't need to synthesize
547        // it in `append_standard_var()`.
548        if path_equivalent_for_tar(dirpath, "var/tmp") {
549            self.wrote_vartmp = true;
550        }
551
552        for item in dirs {
553            let (name, contents_csum, meta_csum) = item.to_tuple();
554            let name = name.to_str();
555            let metadata = {
556                let meta_csum = &hex::encode(meta_csum);
557                let meta_v = &self
558                    .repo
559                    .load_variant(ostree::ObjectType::DirMeta, meta_csum)?;
560                self.append(ostree::ObjectType::DirMeta, meta_csum, meta_v)?;
561                // Safety: We passed the correct variant type just above
562                ostree::DirMetaParsed::from_variant(meta_v).unwrap()
563            };
564            // Special hack because tar stream for containers can't have duplicates.
565            if is_root && name == SYSROOT {
566                continue;
567            }
568            let dirtree_csum = hex::encode(contents_csum);
569            let subpath = &dirpath.join(name);
570            let subpath = map_path(subpath);
571            self.append_dir(&subpath, &metadata)?;
572            self.append_dirtree(&subpath, dirtree_csum, false, cancellable)?;
573        }
574
575        Ok(())
576    }
577
578    /// Generate e.g. `/var/tmp`.
579    ///
580    /// In the OSTree model we expect `/var` to start out empty, and be populated via
581    /// e.g. `systemd-tmpfiles`.  But, systemd doesn't run in Docker-style containers by default.
582    ///
583    /// So, this function creates a few critical directories in `/var` by default.
584    fn append_standard_var(&mut self, cancellable: Option<&gio::Cancellable>) -> Result<()> {
585        // If the commit included /var/tmp, then it's already in the tar stream.
586        if self.wrote_vartmp {
587            return Ok(());
588        }
589        if let Some(c) = cancellable {
590            c.set_error_if_cancelled()?;
591        }
592        let mut header = tar::Header::new_gnu();
593        header.set_entry_type(tar::EntryType::Directory);
594        header.set_size(0);
595        header.set_uid(0);
596        header.set_gid(0);
597        header.set_mode(self.filter_mode(libc::S_IFDIR | 0o1777));
598        self.out
599            .append_data(&mut header, "var/tmp", std::io::empty())?;
600        Ok(())
601    }
602
603    fn write_parents_of(
604        &mut self,
605        path: &Utf8Path,
606        root: &gio::File,
607        cache: &mut HashSet<Utf8PathBuf>,
608    ) -> Result<()> {
609        let Some(parent) = path.parent() else {
610            return Ok(());
611        };
612
613        if parent.components().count() == 0 {
614            return Ok(());
615        }
616
617        if cache.contains(parent) {
618            return Ok(());
619        }
620
621        self.write_parents_of(parent, root, cache)?;
622
623        let inserted = cache.insert(parent.to_owned());
624        debug_assert!(inserted);
625
626        let parent_file = root.resolve_relative_path(unmap_path(parent).as_ref());
627        let queryattrs = "unix::*";
628        let queryflags = gio::FileQueryInfoFlags::NOFOLLOW_SYMLINKS;
629        let stat = parent_file.query_info(&queryattrs, queryflags, gio::Cancellable::NONE)?;
630        let uid = stat.attribute_uint32(gio::FILE_ATTRIBUTE_UNIX_UID);
631        let gid = stat.attribute_uint32(gio::FILE_ATTRIBUTE_UNIX_GID);
632        let orig_mode = stat.attribute_uint32(gio::FILE_ATTRIBUTE_UNIX_MODE);
633        let mode = self.filter_mode(orig_mode);
634
635        let mut header = tar::Header::new_gnu();
636        header.set_entry_type(tar::EntryType::Directory);
637        header.set_size(0);
638        header.set_uid(uid as u64);
639        header.set_gid(gid as u64);
640        header.set_mode(mode);
641        self.out
642            .append_data(&mut header, parent, std::io::empty())?;
643        Ok(())
644    }
645}
646
647/// Append xattrs to the tar stream as PAX extensions, excluding security.selinux
648/// which doesn't become visible in container runtimes anyway.
649/// https://github.com/containers/storage/blob/0d4a8d2aaf293c9f0464b888d932ab5147a284b9/pkg/archive/archive.go#L85
650#[context("Writing tar xattrs")]
651fn append_pax_xattrs<W: std::io::Write>(
652    out: &mut tar::Builder<W>,
653    xattrs: &glib::Variant,
654) -> Result<()> {
655    let v = xattrs.data_as_bytes();
656    let v = v.try_as_aligned().unwrap();
657    let v = gvariant::gv!("a(ayay)").cast(v);
658    let mut pax_extensions = Vec::new();
659    for entry in v {
660        let (k, v) = entry.to_tuple();
661        let k = CStr::from_bytes_with_nul(k).unwrap();
662        let k = k
663            .to_str()
664            .with_context(|| format!("Found non-UTF8 xattr: {k:?}"))?;
665        if k == SECURITY_SELINUX_XATTR {
666            continue;
667        }
668        pax_extensions.push((format!("SCHILY.xattr.{k}"), v));
669    }
670    if !pax_extensions.is_empty() {
671        out.append_pax_extensions(pax_extensions.iter().map(|(k, v)| (k.as_str(), *v)))?;
672    }
673    Ok(())
674}
675
676/// Recursively walk an OSTree commit and generate data into a `[tar::Builder]`
677/// which contains all of the metadata objects, as well as a hardlinked
678/// stream that looks like a checkout.  Extended attributes are stored specially out
679/// of band of tar so that they can be reliably retrieved.
680fn impl_export<W: std::io::Write>(
681    repo: &ostree::Repo,
682    commit_checksum: &str,
683    out: &mut tar::Builder<W>,
684    options: ExportOptions,
685) -> Result<()> {
686    if options.raw {
687        return impl_raw_export(repo, commit_checksum, out);
688    }
689    let writer = &mut OstreeTarWriter::new(repo, commit_checksum, out, options)?;
690    writer.write_commit()?;
691    Ok(())
692}
693
694/// Export an ostree commit as a "raw" tar stream - just the filesystem content
695/// with `/usr/etc` -> `/etc` remapping, without ostree repository structure.
696fn impl_raw_export<W: std::io::Write>(
697    repo: &ostree::Repo,
698    commit_checksum: &str,
699    out: &mut tar::Builder<W>,
700) -> Result<()> {
701    let cancellable = gio::Cancellable::NONE;
702    let (root, _) = repo.read_commit(commit_checksum, cancellable)?;
703    let root = root
704        .downcast::<ostree::RepoFile>()
705        .expect("read_commit returns RepoFile");
706    root.ensure_resolved()?;
707    raw_export_dir(repo, out, &root, Utf8Path::new(""))
708}
709
710/// Recursively export a directory for raw export mode.
711fn raw_export_dir<W: std::io::Write>(
712    repo: &ostree::Repo,
713    out: &mut tar::Builder<W>,
714    dir: &ostree::RepoFile,
715    path: &Utf8Path,
716) -> Result<()> {
717    let cancellable = gio::Cancellable::NONE;
718    let queryattrs = "standard::name,standard::type";
719    let queryflags = gio::FileQueryInfoFlags::NOFOLLOW_SYMLINKS;
720    let e = dir.enumerate_children(queryattrs, queryflags, cancellable)?;
721
722    while let Some(info) = e.next_file(cancellable)? {
723        let name = info.name();
724        let name = name
725            .to_str()
726            .ok_or_else(|| anyhow!("Invalid UTF-8 filename: {:?}", name))?;
727        let child_path = path.join(name);
728
729        // Apply /usr/etc -> /etc remapping
730        let output_path = map_path_v1(&child_path);
731
732        // Get the child and downcast to RepoFile
733        let child = dir.child(name);
734        let child = child
735            .downcast::<ostree::RepoFile>()
736            .expect("child of RepoFile is RepoFile");
737        child.ensure_resolved()?;
738
739        let file_type = info.file_type();
740        match file_type {
741            gio::FileType::Regular | gio::FileType::SymbolicLink => {
742                // Get the checksum and load the file via the repo
743                let checksum = child.checksum();
744                let (instream, meta, xattrs) = repo.load_file(&checksum, cancellable)?;
745
746                // Write xattrs as PAX extensions (before the file entry)
747                append_pax_xattrs(out, &xattrs)?;
748
749                let mut h = tar::Header::new_gnu();
750                h.set_uid(meta.attribute_uint32("unix::uid") as u64);
751                h.set_gid(meta.attribute_uint32("unix::gid") as u64);
752                // Filter out the file type bits from mode for tar
753                h.set_mode(meta.attribute_uint32("unix::mode") & !libc::S_IFMT);
754
755                if let Some(instream) = instream {
756                    // Regular file
757                    h.set_entry_type(tar::EntryType::Regular);
758                    h.set_size(meta.size() as u64);
759                    let mut instream = BufReader::with_capacity(BUF_CAPACITY, instream.into_read());
760                    out.append_data(&mut h, output_path, &mut instream)
761                        .with_context(|| format!("Writing {child_path}"))?;
762                } else {
763                    // Symlink
764                    h.set_entry_type(tar::EntryType::Symlink);
765                    h.set_size(0);
766
767                    let target = meta
768                        .symlink_target()
769                        .ok_or_else(|| anyhow!("Missing symlink target for {child_path}"))?;
770                    let target = target
771                        .to_str()
772                        .ok_or_else(|| anyhow!("Invalid UTF-8 symlink target: {target:?}"))?;
773
774                    // Handle "denormal" symlinks that contain "//"
775                    if symlink_is_denormal(target) {
776                        h.set_link_name_literal(target)
777                            .with_context(|| format!("Setting symlink target for {child_path}"))?;
778                        out.append_data(&mut h, output_path, &mut std::io::empty())
779                            .with_context(|| format!("Writing symlink {child_path}"))?;
780                    } else {
781                        out.append_link(&mut h, output_path, target)
782                            .with_context(|| format!("Writing symlink {child_path}"))?;
783                    }
784                }
785            }
786            gio::FileType::Directory => {
787                // For directories, query metadata directly from the RepoFile
788                let dir_meta_checksum = child.tree_get_metadata_checksum().ok_or_else(|| {
789                    anyhow!("Missing metadata checksum for directory {child_path}")
790                })?;
791                let meta_v = repo.load_variant(ostree::ObjectType::DirMeta, &dir_meta_checksum)?;
792                let metadata =
793                    ostree::DirMetaParsed::from_variant(&meta_v).context("Parsing dirmeta")?;
794
795                let mut h = tar::Header::new_gnu();
796                h.set_entry_type(tar::EntryType::Directory);
797                h.set_uid(metadata.uid as u64);
798                h.set_gid(metadata.gid as u64);
799                h.set_mode(metadata.mode & !libc::S_IFMT);
800                h.set_size(0);
801                out.append_data(&mut h, output_path, std::io::empty())
802                    .with_context(|| format!("Writing directory {child_path}"))?;
803
804                raw_export_dir(repo, out, &child, &child_path)?;
805            }
806            o => anyhow::bail!("Unsupported file type {o:?} for {child_path}"),
807        }
808    }
809    Ok(())
810}
811
812/// Configuration for tar export.
813#[derive(Debug, PartialEq, Eq, Default)]
814pub struct ExportOptions {
815    /// If true, output a "raw" filesystem tree without the ostree repository
816    /// structure (no /sysroot/ostree/repo, no commit/dirtree/dirmeta objects,
817    /// no hardlinks into the object store). The `/usr/etc` -> `/etc` remapping
818    /// is still performed.
819    pub raw: bool,
820}
821
822/// Export an ostree commit to an (uncompressed) tar archive stream.
823#[context("Exporting commit")]
824pub fn export_commit(
825    repo: &ostree::Repo,
826    rev: &str,
827    out: impl std::io::Write,
828    options: Option<ExportOptions>,
829) -> Result<()> {
830    let commit = repo.require_rev(rev)?;
831    let mut tar = tar::Builder::new(out);
832    let options = options.unwrap_or_default();
833    impl_export(repo, commit.as_str(), &mut tar, options)?;
834    tar.finish()?;
835    Ok(())
836}
837
838/// Chunked (or version 1) tar streams don't have a leading `./`.
839fn path_for_tar_v1(p: &Utf8Path) -> &Utf8Path {
840    debug_assert!(!p.starts_with("."));
841    map_path_v1(p.strip_prefix("/").unwrap_or(p))
842}
843
844/// Implementation of chunk writing, assumes that the preliminary structure
845/// has been written to the tar stream.
846fn write_chunk<W: std::io::Write>(
847    writer: &mut OstreeTarWriter<W>,
848    chunk: chunking::ChunkMapping,
849    create_parent_dirs: bool,
850) -> Result<()> {
851    let mut cache = std::collections::HashSet::new();
852    let root = writer
853        .repo
854        .read_commit(&writer.commit_checksum, gio::Cancellable::NONE)?
855        .0;
856    for (checksum, (_size, paths)) in chunk.into_iter() {
857        let (objpath, h) = writer.append_content(checksum.borrow())?;
858        for path in paths.iter() {
859            let path = path_for_tar_v1(path);
860            let h = h.clone();
861            if create_parent_dirs {
862                writer.write_parents_of(&path, &root, &mut cache)?;
863            }
864            writer.append_content_hardlink(&objpath, h, path)?;
865        }
866    }
867    Ok(())
868}
869
870/// Output a chunk to a tar stream.
871pub(crate) fn export_chunk<W: std::io::Write>(
872    repo: &ostree::Repo,
873    commit: &str,
874    chunk: chunking::ChunkMapping,
875    out: &mut tar::Builder<W>,
876    create_parent_dirs: bool,
877) -> Result<()> {
878    // For chunking, we default to format version 1
879    #[allow(clippy::needless_update)]
880    let opts = ExportOptions::default();
881    let writer = &mut OstreeTarWriter::new(repo, commit, out, opts)?;
882    writer.write_repo_structure()?;
883    write_chunk(writer, chunk, create_parent_dirs)
884}
885
886/// Output the last chunk in a chunking.
887#[context("Exporting final chunk")]
888pub(crate) fn export_final_chunk<W: std::io::Write>(
889    repo: &ostree::Repo,
890    commit_checksum: &str,
891    remainder: chunking::Chunk,
892    out: &mut tar::Builder<W>,
893    create_parent_dirs: bool,
894) -> Result<()> {
895    let options = ExportOptions::default();
896    let writer = &mut OstreeTarWriter::new(repo, commit_checksum, out, options)?;
897    // For the final chunk, output the commit object, plus all ostree metadata objects along with
898    // the containing directories.
899    writer.structure_only = true;
900    writer.write_commit()?;
901    writer.structure_only = false;
902    write_chunk(writer, remainder.content, create_parent_dirs)
903}
904
905/// Process an exported tar stream, and update the detached metadata.
906#[allow(clippy::while_let_on_iterator)]
907#[context("Replacing detached metadata")]
908pub(crate) fn reinject_detached_metadata<C: IsA<gio::Cancellable>>(
909    src: &mut tar::Archive<impl std::io::Read>,
910    dest: &mut tar::Builder<impl std::io::Write>,
911    detached_buf: Option<&[u8]>,
912    cancellable: Option<&C>,
913) -> Result<()> {
914    let mut entries = src.entries()?;
915    let mut commit_ent = None;
916    // Loop through the tar stream until we find the commit object; copy all prior entries
917    // such as the baseline directory structure.
918    while let Some(entry) = entries.next() {
919        if let Some(c) = cancellable {
920            c.set_error_if_cancelled()?;
921        }
922        let entry = entry?;
923        let header = entry.header();
924        let path = entry.path()?;
925        let path: &Utf8Path = (&*path).try_into()?;
926        if !(header.entry_type() == tar::EntryType::Regular && path.as_str().ends_with(".commit")) {
927            crate::tar::write::copy_entry(entry, dest, None)?;
928        } else {
929            commit_ent = Some(entry);
930            break;
931        }
932    }
933    let commit_ent = commit_ent.ok_or_else(|| anyhow!("Missing commit object"))?;
934    let commit_path = commit_ent.path()?;
935    let commit_path = Utf8Path::from_path(&commit_path)
936        .ok_or_else(|| anyhow!("Invalid non-utf8 path {:?}", commit_path))?;
937    let (checksum, objtype) = crate::tar::import::Importer::parse_metadata_entry(commit_path)?;
938    assert_eq!(objtype, ostree::ObjectType::Commit); // Should have been verified above
939    crate::tar::write::copy_entry(commit_ent, dest, None)?;
940
941    // If provided, inject our new detached metadata object
942    if let Some(detached_buf) = detached_buf {
943        let detached_path = object_path(ostree::ObjectType::CommitMeta, &checksum);
944        tar_append_default_data(dest, &detached_path, detached_buf)?;
945    }
946
947    // If the next entry is detached metadata, then drop it since we wrote a new one
948    let next_ent = entries
949        .next()
950        .ok_or_else(|| anyhow!("Expected metadata object after commit"))??;
951    let next_ent_path = next_ent.path()?;
952    let next_ent_path: &Utf8Path = (&*next_ent_path).try_into()?;
953    let objtype = crate::tar::import::Importer::parse_metadata_entry(next_ent_path)?.1;
954    if objtype != ostree::ObjectType::CommitMeta {
955        crate::tar::write::copy_entry(next_ent, dest, None)?;
956    }
957
958    // Finally, copy all remaining entries.
959    while let Some(entry) = entries.next() {
960        if let Some(c) = cancellable {
961            c.set_error_if_cancelled()?;
962        }
963        crate::tar::write::copy_entry(entry?, dest, None)?;
964    }
965
966    Ok(())
967}
968
969/// Replace the detached metadata in an tar stream which is an export of an OSTree commit.
970pub fn update_detached_metadata<D: std::io::Write, C: IsA<gio::Cancellable>>(
971    src: impl std::io::Read,
972    dest: D,
973    detached_buf: Option<&[u8]>,
974    cancellable: Option<&C>,
975) -> Result<D> {
976    let mut src = tar::Archive::new(src);
977    let mut dest = tar::Builder::new(dest);
978    reinject_detached_metadata(&mut src, &mut dest, detached_buf, cancellable)?;
979    dest.into_inner().map_err(Into::into)
980}
981
982#[cfg(test)]
983mod tests {
984    use super::*;
985
986    #[test]
987    fn test_path_equivalent() {
988        assert!(path_equivalent_for_tar("var/tmp", "./var/tmp"));
989        assert!(path_equivalent_for_tar("./var/tmp", "var/tmp"));
990        assert!(path_equivalent_for_tar("/var/tmp", "var/tmp"));
991        assert!(!path_equivalent_for_tar("var/tmp", "var"));
992    }
993
994    #[test]
995    fn test_map_path() {
996        assert_eq!(
997            map_path("/".into()).as_os_str(),
998            Utf8Path::new("/").as_os_str()
999        );
1000        assert_eq!(
1001            map_path("./usr/etc/blah".into()).as_os_str(),
1002            Utf8Path::new("./etc/blah").as_os_str()
1003        );
1004        for unchanged in ["boot", "usr/bin", "usr/lib/foo"].iter().map(Utf8Path::new) {
1005            assert_eq!(unchanged.as_os_str(), map_path_v1(unchanged).as_os_str());
1006        }
1007
1008        assert_eq!(
1009            Utf8Path::new("etc").as_os_str(),
1010            map_path_v1(Utf8Path::new("usr/etc")).as_os_str()
1011        );
1012        assert_eq!(
1013            Utf8Path::new("etc/foo").as_os_str(),
1014            map_path_v1(Utf8Path::new("usr/etc/foo")).as_os_str()
1015        );
1016    }
1017
1018    #[test]
1019    fn test_unmap_path() {
1020        assert_eq!(
1021            unmap_path("/".into()).as_os_str(),
1022            Utf8Path::new("/").as_os_str()
1023        );
1024        assert_eq!(
1025            unmap_path("/etc".into()).as_os_str(),
1026            Utf8Path::new("/etc").as_os_str()
1027        );
1028        assert_eq!(
1029            unmap_path("/usr/etc".into()).as_os_str(),
1030            Utf8Path::new("/usr/etc").as_os_str()
1031        );
1032        assert_eq!(
1033            unmap_path("usr/etc".into()).as_os_str(),
1034            Utf8Path::new("usr/etc").as_os_str()
1035        );
1036        assert_eq!(
1037            unmap_path("etc".into()).as_os_str(),
1038            Utf8Path::new("usr/etc").as_os_str()
1039        );
1040        assert_eq!(
1041            unmap_path("etc/blah".into()).as_os_str(),
1042            Utf8Path::new("usr/etc/blah").as_os_str()
1043        );
1044    }
1045
1046    #[test]
1047    fn test_denormal_symlink() {
1048        let normal = ["/", "/usr", "../usr/bin/blah"];
1049        let denormal = ["../../usr/sbin//chkconfig", "foo//bar/baz"];
1050        for path in normal {
1051            assert!(!symlink_is_denormal(path));
1052        }
1053        for path in denormal {
1054            assert!(symlink_is_denormal(path));
1055        }
1056    }
1057
1058    #[test]
1059    fn test_v1_xattrs_object_path() {
1060        let checksum = "b8627e3ef0f255a322d2bd9610cfaaacc8f122b7f8d17c0e7e3caafa160f9fc7";
1061        let expected = "sysroot/ostree/repo/objects/b8/627e3ef0f255a322d2bd9610cfaaacc8f122b7f8d17c0e7e3caafa160f9fc7.file-xattrs-link";
1062        let output = v1_xattrs_object_path(checksum);
1063        assert_eq!(&output, expected);
1064    }
1065}
ostree_ext/tar/export.rs

ostree_ext/tar/
export.rs
