1use crate::Result;
11use crate::generic_decompress::Decompressor;
12use anyhow::{Context, anyhow};
13use camino::{Utf8Component, Utf8Path, Utf8PathBuf};
14
15use cap_std::io_lifetimes;
16use cap_std_ext::cap_std::fs::Dir;
17use cap_std_ext::cmdext::CapStdExtCommandExt;
18use cap_std_ext::{cap_std, cap_tempfile};
19use containers_image_proxy::oci_spec::image as oci_image;
20use fn_error_context::context;
21use ostree::gio;
22use ostree::prelude::FileExt;
23use std::borrow::Cow;
24use std::collections::{BTreeMap, HashMap};
25use std::io::{BufWriter, Seek, Write};
26use std::path::Path;
27use std::process::Stdio;
28use std::sync::Arc;
29use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite};
30use tracing::instrument;
31
32const EXCLUDED_TOPLEVEL_PATHS: &[&str] = &["run", "tmp", "proc", "sys", "dev"];
35
36#[context("Copying entry")]
38pub(crate) fn copy_entry(
39 mut entry: tar::Entry<impl std::io::Read>,
40 dest: &mut tar::Builder<impl std::io::Write>,
41 path: Option<&Path>,
42) -> Result<()> {
43 let path = if let Some(path) = path {
45 path.to_owned()
46 } else {
47 (*entry.path()?).to_owned()
48 };
49 let mut header = entry.header().clone();
50 if let Some(headers) = entry.pax_extensions()? {
51 let extensions = headers
52 .map(|ext| {
53 let ext = ext?;
54 Ok((ext.key()?, ext.value_bytes()))
55 })
56 .collect::<Result<Vec<_>>>()?;
57 dest.append_pax_extensions(extensions.as_slice().iter().copied())?;
58 }
59
60 match entry.header().entry_type() {
64 tar::EntryType::Symlink => {
65 let target = entry.link_name()?.ok_or_else(|| anyhow!("Invalid link"))?;
66 let target: &Utf8Path = (&*target).try_into()?;
68 dest.append_link(&mut header, path, target)
69 }
70 tar::EntryType::Link => {
71 let target = entry.link_name()?.ok_or_else(|| anyhow!("Invalid link"))?;
72 let target: &Utf8Path = (&*target).try_into()?;
73 let target = remap_etc_path(target);
76 dest.append_link(&mut header, path, &*target)
77 }
78 _ => dest.append_data(&mut header, path, entry),
79 }
80 .map_err(Into::into)
81}
82
83#[derive(Debug, Default)]
85#[non_exhaustive]
86pub struct WriteTarOptions {
87 pub base: Option<String>,
89 pub selinux: bool,
92 pub allow_nonusr: bool,
94 pub retain_var: bool,
97}
98
99#[derive(Debug, Default)]
104pub struct WriteTarResult {
105 pub commit: String,
107 pub filtered: BTreeMap<String, u32>,
109}
110
111fn sepolicy_from_base(repo: &ostree::Repo, base: &str) -> Result<tempfile::TempDir> {
114 let cancellable = gio::Cancellable::NONE;
115 let policypath = "usr/etc/selinux";
116 let tempdir = tempfile::tempdir()?;
117 let (root, _) = repo.read_commit(base, cancellable)?;
118 let policyroot = root.resolve_relative_path(policypath);
119 if policyroot.query_exists(cancellable) {
120 let policydest = tempdir.path().join(policypath);
121 std::fs::create_dir_all(policydest.parent().unwrap())?;
122 let opts = ostree::RepoCheckoutAtOptions {
123 mode: ostree::RepoCheckoutMode::User,
124 subpath: Some(Path::new(policypath).to_owned()),
125 ..Default::default()
126 };
127 repo.checkout_at(Some(&opts), ostree::AT_FDCWD, policydest, base, cancellable)?;
128 }
129 Ok(tempdir)
130}
131
132#[derive(Debug, PartialEq, Eq)]
133enum NormalizedPathResult<'a> {
134 Filtered(&'a str),
135 Normal(Utf8PathBuf),
136}
137
138#[derive(Debug, Clone, PartialEq, Eq, Default)]
139pub(crate) struct TarImportConfig {
140 allow_nonusr: bool,
141 remap_factory_var: bool,
142}
143
144fn remap_etc_path(path: &Utf8Path) -> Cow<'_, Utf8Path> {
146 let mut components = path.components();
147 let Some(prefix) = components.next() else {
148 return Cow::Borrowed(path);
149 };
150 let (prefix, first) = if matches!(prefix, Utf8Component::CurDir | Utf8Component::RootDir) {
151 let Some(next) = components.next() else {
152 return Cow::Borrowed(path);
153 };
154 (Some(prefix), next)
155 } else {
156 (None, prefix)
157 };
158 if first.as_str() == "etc" {
159 let usr = Utf8Component::Normal("usr");
160 Cow::Owned(
161 prefix
162 .into_iter()
163 .chain([usr, first])
164 .chain(components)
165 .collect(),
166 )
167 } else {
168 Cow::Borrowed(path)
169 }
170}
171
172fn normalize_validate_path<'a>(
173 path: &'a Utf8Path,
174 config: &'_ TarImportConfig,
175) -> Result<NormalizedPathResult<'a>> {
176 let mut components = path
178 .components()
179 .map(|part| {
180 match part {
181 camino::Utf8Component::RootDir => Ok(camino::Utf8Component::CurDir),
183 camino::Utf8Component::Normal(_) | camino::Utf8Component::CurDir => Ok(part),
185 _ => Err(anyhow!("Invalid path: {}", path)),
187 }
188 })
189 .peekable();
190 let mut ret = Utf8PathBuf::new();
191 if let Some(Ok(camino::Utf8Component::Normal(_))) = components.peek() {
193 ret.push(camino::Utf8Component::CurDir);
194 }
195 let mut found_first = false;
196 let mut excluded = false;
197 for part in components {
198 let part = part?;
199 if excluded {
200 return Ok(NormalizedPathResult::Filtered(part.as_str()));
201 }
202 if !found_first {
203 if let Utf8Component::Normal(part) = part {
204 found_first = true;
205 match part {
206 "usr" => ret.push(part),
208 "etc" => {
210 ret.push("usr/etc");
211 }
212 "var" => {
213 if config.remap_factory_var {
215 ret.push("usr/share/factory/var");
216 } else {
217 ret.push(part)
218 }
219 }
220 o if EXCLUDED_TOPLEVEL_PATHS.contains(&o) => {
221 excluded = true;
224 ret.push(part)
225 }
226 _ if config.allow_nonusr => ret.push(part),
227 _ => {
228 return Ok(NormalizedPathResult::Filtered(part));
229 }
230 }
231 } else {
232 ret.push(part);
233 }
234 } else {
235 ret.push(part);
236 }
237 }
238
239 Ok(NormalizedPathResult::Normal(ret))
240}
241
242pub(crate) fn filter_tar(
252 src: impl std::io::Read,
253 dest: impl std::io::Write,
254 config: &TarImportConfig,
255 tmpdir: &Dir,
256) -> Result<BTreeMap<String, u32>> {
257 let src = std::io::BufReader::new(src);
258 let mut src = tar::Archive::new(src);
259 let dest = BufWriter::new(dest);
260 let mut dest = tar::Builder::new(dest);
261 let mut filtered = BTreeMap::new();
262
263 let ents = src.entries()?;
264
265 tracing::debug!("Filtering tar; config={config:?}");
266
267 let mut changed_sysroot_objects = HashMap::new();
269 let mut new_sysroot_link_targets = HashMap::<Utf8PathBuf, Utf8PathBuf>::new();
270
271 for entry in ents {
272 let mut entry = entry?;
273 let header = entry.header();
274 let path = entry.path()?;
275 let path: &Utf8Path = (&*path).try_into()?;
276 let path = path.strip_prefix("/").unwrap_or(path);
278
279 let is_modified = header.mtime().unwrap_or_default() > 0;
280 let is_regular = header.entry_type() == tar::EntryType::Regular;
281 if path.strip_prefix(crate::tar::REPO_PREFIX).is_ok() {
282 if is_modified && is_regular {
287 tracing::debug!("Processing modified sysroot file {path}");
288 let mut tmpf = cap_tempfile::TempFile::new_anonymous(tmpdir)
290 .map(BufWriter::new)
291 .context("Creating tmpfile")?;
292 let path = path.to_owned();
293 let header = header.clone();
294 std::io::copy(&mut entry, &mut tmpf)
295 .map_err(anyhow::Error::msg)
296 .context("Copying")?;
297 let mut tmpf = tmpf.into_inner()?;
298 tmpf.seek(std::io::SeekFrom::Start(0))?;
299 changed_sysroot_objects.insert(path, (header, tmpf));
301 continue;
302 }
303 } else if header.entry_type() == tar::EntryType::Link && is_modified {
304 let target = header
305 .link_name()?
306 .ok_or_else(|| anyhow!("Invalid empty hardlink"))?;
307 let target: &Utf8Path = (&*target).try_into()?;
308 let target = path.strip_prefix("/").unwrap_or(target);
310 if target.strip_prefix(crate::tar::REPO_PREFIX).is_ok() {
312 if let Some((mut header, data)) = changed_sysroot_objects.remove(target) {
314 tracing::debug!("Making {path} canonical for sysroot link {target}");
315 dest.append_data(&mut header, path, data)?;
317 new_sysroot_link_targets.insert(target.to_owned(), path.to_owned());
319 } else if let Some(real_target) = new_sysroot_link_targets.get(target) {
320 tracing::debug!("Relinking {path} to {real_target}");
321 let mut header = header.clone();
324 dest.append_link(&mut header, path, real_target)?;
325 } else {
326 tracing::debug!("Found unhandled modified link from {path} to {target}");
327 }
328 continue;
329 }
330 }
331
332 let normalized = match normalize_validate_path(path, config)? {
333 NormalizedPathResult::Filtered(path) => {
334 tracing::trace!("Filtered: {path}");
335 if let Some(v) = filtered.get_mut(path) {
336 *v += 1;
337 } else {
338 filtered.insert(path.to_string(), 1);
339 }
340 continue;
341 }
342 NormalizedPathResult::Normal(path) => path,
343 };
344
345 copy_entry(entry, &mut dest, Some(normalized.as_std_path()))?;
346 }
347 dest.into_inner()?.flush()?;
348 Ok(filtered)
349}
350
351#[context("Filtering tar stream")]
353async fn filter_tar_async(
354 src: impl AsyncRead + Send + 'static,
355 media_type: oci_image::MediaType,
356 mut dest: impl AsyncWrite + Send + Unpin,
357 config: &TarImportConfig,
358 repo_tmpdir: Dir,
359) -> Result<BTreeMap<String, u32>> {
360 let (tx_buf, mut rx_buf) = tokio::io::duplex(8192);
361 let src = Box::pin(src);
363 let config = config.clone();
364 let tar_transformer = crate::tokio_util::spawn_blocking_flatten(move || {
365 let src = tokio_util::io::SyncIoBridge::new(src);
366 let mut src = Decompressor::new(&media_type, src)?;
367 let dest = tokio_util::io::SyncIoBridge::new(tx_buf);
368
369 let r = filter_tar(&mut src, dest, &config, &repo_tmpdir);
370
371 src.finish()?;
372
373 Ok(r)
374 });
375 let copier = tokio::io::copy(&mut rx_buf, &mut dest);
376 let (r, v) = tokio::join!(tar_transformer, copier);
377 let _v: u64 = v?;
378 r?
379}
380
381#[allow(unsafe_code)] #[instrument(level = "debug", skip_all)]
384pub async fn write_tar(
385 repo: &ostree::Repo,
386 src: impl tokio::io::AsyncRead + Send + Unpin + 'static,
387 media_type: oci_image::MediaType,
388 refname: &str,
389 options: Option<WriteTarOptions>,
390) -> Result<WriteTarResult> {
391 let repo = repo.clone();
392 let options = options.unwrap_or_default();
393 let sepolicy = if options.selinux {
394 if let Some(base) = options.base {
395 Some(sepolicy_from_base(&repo, &base).context("tar: Preparing sepolicy")?)
396 } else {
397 None
398 }
399 } else {
400 None
401 };
402 let mut c = std::process::Command::new("ostree");
403 c.env_remove("G_MESSAGES_DEBUG");
409 let repofd = repo.dfd_as_file()?;
410 let repofd: Arc<io_lifetimes::OwnedFd> = Arc::new(repofd.into());
411 {
412 let c = c
413 .stdin(Stdio::piped())
414 .stdout(Stdio::piped())
415 .stderr(Stdio::piped())
416 .args(["commit"]);
417 c.take_fd_n(repofd.clone(), 3);
418 c.arg("--repo=/proc/self/fd/3");
419 if let Some(sepolicy) = sepolicy.as_ref() {
420 c.arg("--selinux-policy");
421 c.arg(sepolicy.path());
422 }
423 c.arg(format!(
424 "--add-metadata-string=ostree.importer.version={}",
425 env!("CARGO_PKG_VERSION")
426 ));
427 c.args([
428 "--no-bindings",
429 "--tar-autocreate-parents",
430 "--tree=tar=/proc/self/fd/0",
431 "--branch",
432 refname,
433 ]);
434 }
435 let mut c = tokio::process::Command::from(c);
436 c.kill_on_drop(true);
437 let mut r = c.spawn()?;
438 tracing::trace!("Spawned ostree child process");
439 let child_stdin = r.stdin.take().unwrap();
441 let mut child_stdout = r.stdout.take().unwrap();
442 let mut child_stderr = r.stderr.take().unwrap();
443 let import_config = TarImportConfig {
445 allow_nonusr: options.allow_nonusr,
446 remap_factory_var: !options.retain_var,
447 };
448 let repo_tmpdir = Dir::reopen_dir(&repo.dfd_borrow())?
449 .open_dir("tmp")
450 .context("Getting repo tmpdir")?;
451 let filtered_result =
452 filter_tar_async(src, media_type, child_stdin, &import_config, repo_tmpdir);
453 let output_copier = async move {
454 let mut child_stdout_buf = String::new();
456 let mut child_stderr_buf = String::new();
457 let (_a, _b) = tokio::try_join!(
458 child_stdout.read_to_string(&mut child_stdout_buf),
459 child_stderr.read_to_string(&mut child_stderr_buf)
460 )?;
461 Ok::<_, anyhow::Error>((child_stdout_buf, child_stderr_buf))
462 };
463
464 let status = async move {
467 let status = r.wait().await?;
468 if !status.success() {
469 return Err(anyhow!("Failed to commit tar: {:?}", status));
470 }
471 anyhow::Ok(())
472 };
473 tracing::debug!("Waiting on child process");
474 let (filtered_result, child_stdout) =
475 match tokio::try_join!(status, filtered_result).context("Processing tar") {
476 Ok(((), filtered_result)) => {
477 let (child_stdout, _) = output_copier.await.context("Copying child output")?;
478 (filtered_result, child_stdout)
479 }
480 Err(e) => {
481 if let Ok((_, child_stderr)) = output_copier.await {
482 let child_stderr = child_stderr.trim();
484 Err(e.context(child_stderr.to_string()))?
485 } else {
486 Err(e)?
487 }
488 }
489 };
490 drop(sepolicy);
491
492 tracing::trace!("tar written successfully");
493 let s = child_stdout.trim();
495 Ok(WriteTarResult {
496 commit: s.to_string(),
497 filtered: filtered_result,
498 })
499}
500
501#[cfg(test)]
502mod tests {
503 use super::*;
504 use std::io::Cursor;
505
506 #[test]
507 fn test_remap_etc() {
508 let unchanged = ["", "foo", "/etcc/foo", "../etc/baz"];
510 for x in unchanged {
511 similar_asserts::assert_eq!(x, remap_etc_path(x.into()).as_str());
512 }
513 for (p, expected) in [
516 ("/etc/foo/../bar/baz", "/usr/etc/foo/../bar/baz"),
517 ("etc/foo//bar", "usr/etc/foo/bar"),
518 ("./etc/foo", "./usr/etc/foo"),
519 ("etc", "usr/etc"),
520 ] {
521 similar_asserts::assert_eq!(remap_etc_path(p.into()).as_str(), expected);
522 }
523 }
524
525 #[test]
526 fn test_normalize_path() {
527 let imp_default = &TarImportConfig {
528 allow_nonusr: false,
529 remap_factory_var: true,
530 };
531 let allow_nonusr = &TarImportConfig {
532 allow_nonusr: true,
533 remap_factory_var: true,
534 };
535 let composefs_and_new_ostree = &TarImportConfig {
536 allow_nonusr: true,
537 remap_factory_var: false,
538 };
539 let valid_all = &[
540 ("/usr/bin/blah", "./usr/bin/blah"),
541 ("usr/bin/blah", "./usr/bin/blah"),
542 ("usr///share/.//blah", "./usr/share/blah"),
543 ("var/lib/blah", "./usr/share/factory/var/lib/blah"),
544 ("./var/lib/blah", "./usr/share/factory/var/lib/blah"),
545 ("dev", "./dev"),
546 ("/proc", "./proc"),
547 ("./", "."),
548 ];
549 let valid_nonusr = &[("boot", "./boot"), ("opt/puppet/blah", "./opt/puppet/blah")];
550 for &(k, v) in valid_all {
551 let r = normalize_validate_path(k.into(), imp_default).unwrap();
552 let r2 = normalize_validate_path(k.into(), allow_nonusr).unwrap();
553 assert_eq!(r, r2);
554 match r {
555 NormalizedPathResult::Normal(r) => assert_eq!(r, v),
556 NormalizedPathResult::Filtered(o) => panic!("Should not have filtered {o}"),
557 }
558 }
559 for &(k, v) in valid_nonusr {
560 let strict = normalize_validate_path(k.into(), imp_default).unwrap();
561 assert!(
562 matches!(strict, NormalizedPathResult::Filtered(_)),
563 "Incorrect filter for {k}"
564 );
565 let nonusr = normalize_validate_path(k.into(), allow_nonusr).unwrap();
566 match nonusr {
567 NormalizedPathResult::Normal(r) => assert_eq!(r, v),
568 NormalizedPathResult::Filtered(o) => panic!("Should not have filtered {o}"),
569 }
570 }
571 let filtered = &["/run/blah", "/sys/foo", "/dev/somedev"];
572 for &k in filtered {
573 match normalize_validate_path(k.into(), imp_default).unwrap() {
574 NormalizedPathResult::Filtered(_) => {}
575 NormalizedPathResult::Normal(_) => {
576 panic!("{k} should be filtered")
577 }
578 }
579 }
580 let errs = &["usr/foo/../../bar"];
581 for &k in errs {
582 assert!(normalize_validate_path(k.into(), allow_nonusr).is_err());
583 assert!(normalize_validate_path(k.into(), imp_default).is_err());
584 }
585 assert!(matches!(
586 normalize_validate_path("var/lib/foo".into(), composefs_and_new_ostree).unwrap(),
587 NormalizedPathResult::Normal(_)
588 ));
589 }
590
591 #[tokio::test]
592 async fn tar_filter() -> Result<()> {
593 let tempd = tempfile::tempdir()?;
594 let rootfs = &tempd.path().join("rootfs");
595
596 std::fs::create_dir_all(rootfs.join("etc/systemd/system"))?;
597 std::fs::write(rootfs.join("etc/systemd/system/foo.service"), "fooservice")?;
598 std::fs::write(rootfs.join("blah"), "blah")?;
599 let rootfs_tar_path = &tempd.path().join("rootfs.tar");
600 let rootfs_tar = std::fs::File::create(rootfs_tar_path)?;
601 let mut rootfs_tar = tar::Builder::new(rootfs_tar);
602 rootfs_tar.append_dir_all(".", rootfs)?;
603 let _ = rootfs_tar.into_inner()?;
604 let mut dest = Vec::new();
605 let src = tokio::io::BufReader::new(tokio::fs::File::open(rootfs_tar_path).await?);
606 let cap_tmpdir = Dir::open_ambient_dir(&tempd, cap_std::ambient_authority())?;
607 filter_tar_async(
608 src,
609 oci_image::MediaType::ImageLayer,
610 &mut dest,
611 &Default::default(),
612 cap_tmpdir,
613 )
614 .await?;
615 let dest = dest.as_slice();
616 let mut final_tar = tar::Archive::new(Cursor::new(dest));
617 let destdir = &tempd.path().join("destdir");
618 final_tar.unpack(destdir)?;
619 assert!(destdir.join("usr/etc/systemd/system/foo.service").exists());
620 assert!(!destdir.join("blah").exists());
621 Ok(())
622 }
623}