ostree_ext/container/
encapsulate.rs

1//! APIs for creating container images from OSTree commits
2
3use super::{COMPONENT_SEPARATOR, CONTENT_ANNOTATION, OstreeImageReference, Transport};
4use super::{ImageReference, OSTREE_COMMIT_LABEL, SignatureSource};
5use crate::chunking::{Chunk, Chunking, ObjectMetaSized};
6use crate::container::skopeo;
7use crate::objectsource::ContentID;
8use crate::tar as ostree_tar;
9use anyhow::{Context, Result, anyhow};
10use camino::{Utf8Path, Utf8PathBuf};
11use cap_std::fs::Dir;
12use cap_std_ext::cap_std;
13use chrono::DateTime;
14use containers_image_proxy::oci_spec;
15use flate2::Compression;
16use fn_error_context::context;
17use gio::glib;
18use oci_spec::image as oci_image;
19use ocidir::{Layer, OciDir};
20use ostree::gio;
21use std::borrow::Cow;
22use std::collections::{BTreeMap, HashMap};
23use std::num::NonZeroU32;
24use tracing::instrument;
25
26/// The label which may be used in addition to the standard OCI label.
27pub const LEGACY_VERSION_LABEL: &str = "version";
28/// The label which indicates where the ostree layers stop, and the
29/// derived ones start.
30pub const DIFFID_LABEL: &str = "ostree.final-diffid";
31/// The label for bootc.
32pub const BOOTC_LABEL: &str = "containers.bootc";
33
34/// Annotation injected into the layer to say that this is an ostree commit.
35/// However, because this gets lost when converted to D2S2 https://docs.docker.com/registry/spec/manifest-v2-2/
36/// schema, it's not actually useful today.  But, we keep it
37/// out of principle.
38const BLOB_OSTREE_ANNOTATION: &str = "ostree.encapsulated";
39/// Configuration for the generated container.
40#[derive(Debug, Default)]
41pub struct Config {
42    /// Additional labels.
43    pub labels: Option<BTreeMap<String, String>>,
44    /// The equivalent of a `Dockerfile`'s `CMD` instruction.
45    pub cmd: Option<Vec<String>>,
46}
47
48fn commit_meta_to_labels<'a>(
49    meta: &glib::VariantDict,
50    keys: impl IntoIterator<Item = &'a str>,
51    opt_keys: impl IntoIterator<Item = &'a str>,
52    labels: &mut HashMap<String, String>,
53) -> Result<()> {
54    for k in keys {
55        let v = meta
56            .lookup::<String>(k)
57            .context("Expected string for commit metadata value")?
58            .ok_or_else(|| anyhow!("Could not find commit metadata key: {}", k))?;
59        labels.insert(k.to_string(), v);
60    }
61    for k in opt_keys {
62        let v = meta
63            .lookup::<String>(k)
64            .context("Expected string for commit metadata value")?;
65        if let Some(v) = v {
66            labels.insert(k.to_string(), v);
67        }
68    }
69    // Copy standard metadata keys `ostree.bootable` and `ostree.linux`.
70    // Bootable is an odd one out in being a boolean.
71    #[allow(clippy::explicit_auto_deref)]
72    if let Some(v) = meta.lookup::<bool>(ostree::METADATA_KEY_BOOTABLE)? {
73        labels.insert(ostree::METADATA_KEY_BOOTABLE.to_string(), v.to_string());
74        labels.insert(BOOTC_LABEL.into(), "1".into());
75    }
76    // Handle any other string-typed values here.
77    for k in &[&ostree::METADATA_KEY_LINUX] {
78        if let Some(v) = meta.lookup::<String>(k)? {
79            labels.insert(k.to_string(), v);
80        }
81    }
82    Ok(())
83}
84
85fn export_chunks(
86    repo: &ostree::Repo,
87    commit: &str,
88    ociw: &mut OciDir,
89    chunks: Vec<Chunk>,
90    opts: &ExportOpts,
91) -> Result<Vec<(Layer, String, Vec<String>)>> {
92    chunks
93        .into_iter()
94        .enumerate()
95        .map(|(i, chunk)| -> Result<_> {
96            let mut w = ociw.create_layer(Some(opts.compression()))?;
97            ostree_tar::export_chunk(
98                repo,
99                commit,
100                chunk.content,
101                &mut w,
102                opts.tar_create_parent_dirs,
103            )
104            .with_context(|| format!("Exporting chunk {i}"))?;
105            let w = w.into_inner()?;
106            Ok((w.complete()?, chunk.name, chunk.packages))
107        })
108        .collect()
109}
110
111/// Write an ostree commit to an OCI blob
112#[context("Writing ostree root to blob")]
113#[allow(clippy::too_many_arguments)]
114pub(crate) fn export_chunked(
115    repo: &ostree::Repo,
116    commit: &str,
117    ociw: &mut OciDir,
118    manifest: &mut oci_image::ImageManifest,
119    imgcfg: &mut oci_image::ImageConfiguration,
120    labels: &mut HashMap<String, String>,
121    mut chunking: Chunking,
122    opts: &ExportOpts,
123    description: &str,
124) -> Result<()> {
125    let layers = export_chunks(repo, commit, ociw, chunking.take_chunks(), opts)?;
126    let compression = Some(opts.compression());
127
128    // In V1, the ostree layer comes first
129    let mut w = ociw.create_layer(compression)?;
130    ostree_tar::export_final_chunk(
131        repo,
132        commit,
133        chunking.remainder,
134        &mut w,
135        opts.tar_create_parent_dirs,
136    )?;
137    let w = w.into_inner()?;
138    let ostree_layer = w.complete()?;
139
140    // Then, we have a label that points to the last chunk.
141    // Note in the pathological case of a single layer chunked v1 image, this could be the ostree layer.
142    let last_digest = layers
143        .last()
144        .map(|v| &v.0)
145        .unwrap_or(&ostree_layer)
146        .uncompressed_sha256
147        .clone();
148
149    let created = imgcfg
150        .created()
151        .as_deref()
152        .and_then(bootc_utils::try_deserialize_timestamp)
153        .unwrap_or_default();
154    // Add the ostree layer
155    ociw.push_layer_full(
156        manifest,
157        imgcfg,
158        ostree_layer,
159        None::<HashMap<String, String>>,
160        description,
161        created,
162    );
163    // Add the component/content layers
164    let mut buf = [0; 8];
165    let sep = COMPONENT_SEPARATOR.encode_utf8(&mut buf);
166    for (layer, name, mut packages) in layers {
167        let mut annotation_component_layer = HashMap::new();
168        packages.sort();
169        annotation_component_layer.insert(CONTENT_ANNOTATION.to_string(), packages.join(sep));
170        ociw.push_layer_full(
171            manifest,
172            imgcfg,
173            layer,
174            Some(annotation_component_layer),
175            name.as_str(),
176            created,
177        );
178    }
179
180    // This label (mentioned above) points to the last layer that is part of
181    // the ostree commit.
182    labels.insert(
183        DIFFID_LABEL.into(),
184        format!("sha256:{}", last_digest.digest()),
185    );
186    Ok(())
187}
188
189/// Generate an OCI image from a given ostree root
190#[context("Building oci")]
191#[allow(clippy::too_many_arguments)]
192fn build_oci(
193    repo: &ostree::Repo,
194    rev: &str,
195    writer: &mut OciDir,
196    tag: Option<&str>,
197    config: &Config,
198    opts: ExportOpts,
199) -> Result<()> {
200    let commit = repo.require_rev(rev)?;
201    let commit = commit.as_str();
202    let (commit_v, _) = repo.load_commit(commit)?;
203    let commit_timestamp = DateTime::from_timestamp(
204        ostree::commit_get_timestamp(&commit_v).try_into().unwrap(),
205        0,
206    )
207    .unwrap();
208    let commit_subject = commit_v.child_value(3);
209    let commit_subject = commit_subject.str().ok_or_else(|| {
210        anyhow::anyhow!(
211            "Corrupted commit {}; expecting string value for subject",
212            commit
213        )
214    })?;
215    let commit_meta = &commit_v.child_value(0);
216    let commit_meta = glib::VariantDict::new(Some(commit_meta));
217
218    let mut ctrcfg = opts.container_config.clone().unwrap_or_default();
219    let mut imgcfg = oci_image::ImageConfiguration::default();
220    // If a platform was provided, propagate it to the config
221    if let Some(platform) = opts.platform.as_ref() {
222        imgcfg.set_architecture(platform.architecture().clone());
223        imgcfg.set_os(platform.os().clone());
224    }
225
226    let created_at = opts
227        .created
228        .clone()
229        .unwrap_or_else(|| commit_timestamp.format("%Y-%m-%dT%H:%M:%SZ").to_string());
230    imgcfg.set_created(Some(created_at));
231    let mut labels = HashMap::new();
232
233    commit_meta_to_labels(
234        &commit_meta,
235        opts.copy_meta_keys.iter().map(|k| k.as_str()),
236        opts.copy_meta_opt_keys.iter().map(|k| k.as_str()),
237        &mut labels,
238    )?;
239
240    let mut manifest = writer.new_empty_manifest()?.build().unwrap();
241
242    let chunking = opts
243        .package_contentmeta
244        .as_ref()
245        .map(|meta| {
246            crate::chunking::Chunking::from_mapping(
247                repo,
248                commit,
249                meta,
250                &opts.max_layers,
251                opts.prior_build,
252                opts.specific_contentmeta,
253            )
254        })
255        .transpose()?;
256    // If no chunking was provided, create a logical single chunk.
257    let chunking = chunking
258        .map(Ok)
259        .unwrap_or_else(|| crate::chunking::Chunking::new(repo, commit))?;
260
261    if let Some(version) = commit_meta.lookup::<String>("version")? {
262        if opts.legacy_version_label {
263            labels.insert(LEGACY_VERSION_LABEL.into(), version.clone());
264        }
265        labels.insert(oci_image::ANNOTATION_VERSION.into(), version);
266    }
267    labels.insert(OSTREE_COMMIT_LABEL.into(), commit.into());
268
269    for (k, v) in config.labels.iter().flat_map(|k| k.iter()) {
270        labels.insert(k.into(), v.into());
271    }
272
273    let mut annos = HashMap::new();
274    annos.insert(BLOB_OSTREE_ANNOTATION.to_string(), "true".to_string());
275    let description = if commit_subject.is_empty() {
276        Cow::Owned(format!("ostree export of commit {commit}"))
277    } else {
278        Cow::Borrowed(commit_subject)
279    };
280
281    export_chunked(
282        repo,
283        commit,
284        writer,
285        &mut manifest,
286        &mut imgcfg,
287        &mut labels,
288        chunking,
289        &opts,
290        &description,
291    )?;
292
293    // Lookup the cmd embedded in commit metadata
294    let cmd = commit_meta.lookup::<Vec<String>>(ostree::COMMIT_META_CONTAINER_CMD)?;
295    // But support it being overridden by CLI options
296
297    // https://github.com/rust-lang/rust-clippy/pull/7639#issuecomment-1050340564
298    #[allow(clippy::unnecessary_lazy_evaluations)]
299    let cmd = config.cmd.as_ref().or_else(|| cmd.as_ref());
300    if let Some(cmd) = cmd {
301        ctrcfg.set_cmd(Some(cmd.clone()));
302    }
303
304    // Our platform uses the image config
305    let platform = oci_image::PlatformBuilder::default()
306        .architecture(imgcfg.architecture().clone())
307        .os(imgcfg.os().clone())
308        .build()
309        .unwrap();
310
311    ctrcfg
312        .labels_mut()
313        .get_or_insert_with(Default::default)
314        .extend(labels.clone());
315    imgcfg.set_config(Some(ctrcfg));
316    let ctrcfg = writer.write_config(imgcfg)?;
317    manifest.set_config(ctrcfg);
318    manifest.set_annotations(Some(labels));
319
320    if let Some(tag) = tag {
321        writer.insert_manifest(manifest, Some(tag), platform)?;
322    } else {
323        writer.replace_with_single_manifest(manifest, platform)?;
324    }
325
326    Ok(())
327}
328
329/// Interpret a filesystem path as optionally including a tag.  Paths
330/// such as `/foo/bar` will return `("/foo/bar"`, None)`, whereas
331/// e.g. `/foo/bar:latest` will return `("/foo/bar", Some("latest"))`.
332pub(crate) fn parse_oci_path_and_tag(path: &str) -> (&str, Option<&str>) {
333    match path.split_once(':') {
334        Some((path, tag)) => (path, Some(tag)),
335        None => (path, None),
336    }
337}
338
339/// Helper for `build()` that avoids generics
340#[instrument(level = "debug", skip_all)]
341async fn build_impl(
342    repo: &ostree::Repo,
343    ostree_ref: &str,
344    config: &Config,
345    opts: Option<ExportOpts<'_, '_>>,
346    dest: &ImageReference,
347) -> Result<oci_image::Digest> {
348    let mut opts = opts.unwrap_or_default();
349    if dest.transport == Transport::ContainerStorage {
350        opts.skip_compression = true;
351    }
352    let digest = if dest.transport == Transport::OciDir {
353        let (path, tag) = parse_oci_path_and_tag(dest.name.as_str());
354        tracing::debug!("using OCI path={path} tag={tag:?}");
355        if !Utf8Path::new(path).exists() {
356            std::fs::create_dir(path).with_context(|| format!("Creating {path}"))?;
357        }
358        let ocidir = Dir::open_ambient_dir(path, cap_std::ambient_authority())
359            .with_context(|| format!("Opening {path}"))?;
360        let mut ocidir = OciDir::ensure(ocidir).context("Opening OCI")?;
361        build_oci(repo, ostree_ref, &mut ocidir, tag, config, opts)?;
362        None
363    } else {
364        let tempdir = {
365            let vartmp = Dir::open_ambient_dir("/var/tmp", cap_std::ambient_authority())?;
366            cap_std_ext::cap_tempfile::tempdir_in(&vartmp)?
367        };
368        let mut ocidir = OciDir::ensure(tempdir.try_clone()?)?;
369
370        // Minor TODO: refactor to avoid clone
371        let authfile = opts.authfile.clone();
372        build_oci(repo, ostree_ref, &mut ocidir, None, config, opts)?;
373        drop(ocidir);
374
375        // Pass the temporary oci directory as the current working directory for the skopeo process
376        let target_fd = 3i32;
377        let tempoci = ImageReference {
378            transport: Transport::OciDir,
379            name: format!("/proc/self/fd/{target_fd}"),
380        };
381        let digest = skopeo::copy(
382            &tempoci,
383            dest,
384            authfile.as_deref(),
385            Some((std::sync::Arc::new(tempdir.try_clone()?.into()), target_fd)),
386            false,
387        )
388        .await?;
389        Some(digest)
390    };
391    if let Some(digest) = digest {
392        Ok(digest)
393    } else {
394        // If `skopeo copy` doesn't have `--digestfile` yet, then fall back
395        // to running an inspect cycle.
396        let imgref = OstreeImageReference {
397            sigverify: SignatureSource::ContainerPolicyAllowInsecure,
398            imgref: dest.to_owned(),
399        };
400        let (_, digest) = super::unencapsulate::fetch_manifest(&imgref)
401            .await
402            .context("Querying manifest after push")?;
403        Ok(digest)
404    }
405}
406
407/// Options controlling commit export into OCI
408#[derive(Clone, Debug, Default)]
409#[non_exhaustive]
410pub struct ExportOpts<'m, 'o> {
411    /// If true, do not perform gzip compression of the tar layers.
412    pub skip_compression: bool,
413    /// A set of commit metadata keys to copy as image labels.
414    pub copy_meta_keys: Vec<String>,
415    /// A set of optionally-present commit metadata keys to copy as image labels.
416    pub copy_meta_opt_keys: Vec<String>,
417    /// Maximum number of layers to use
418    pub max_layers: Option<NonZeroU32>,
419    /// Path to Docker-formatted authentication file.
420    pub authfile: Option<std::path::PathBuf>,
421    /// Also include the legacy `version` label.
422    pub legacy_version_label: bool,
423    /// Image runtime configuration that will be used as a base
424    pub container_config: Option<oci_image::Config>,
425    /// Override the default platform
426    pub platform: Option<oci_image::Platform>,
427    /// A reference to the metadata for a previous build; used to optimize
428    /// the packing structure.
429    pub prior_build: Option<&'m oci_image::ImageManifest>,
430    /// Metadata mapping between objects and their owning component/package;
431    /// used to optimize packing.
432    pub package_contentmeta: Option<&'o ObjectMetaSized>,
433    /// Metadata for exclusive components that should have their own layers.
434    /// Map from component -> (path, checksum)
435    pub specific_contentmeta: Option<&'o BTreeMap<ContentID, Vec<(Utf8PathBuf, String)>>>,
436    /// Sets the created tag in the image manifest.
437    pub created: Option<String>,
438    /// Whether to explicitly create all parent directories in the tar layers.
439    pub tar_create_parent_dirs: bool,
440}
441
442impl ExportOpts<'_, '_> {
443    /// Return the gzip compression level to use, as configured by the export options.
444    fn compression(&self) -> Compression {
445        if self.skip_compression {
446            Compression::fast()
447        } else {
448            Compression::default()
449        }
450    }
451}
452
453/// Given an OSTree repository and ref, generate a container image.
454///
455/// The returned `ImageReference` will contain a digested (e.g. `@sha256:`) version of the destination.
456pub async fn encapsulate<S: AsRef<str>>(
457    repo: &ostree::Repo,
458    ostree_ref: S,
459    config: &Config,
460    opts: Option<ExportOpts<'_, '_>>,
461    dest: &ImageReference,
462) -> Result<oci_image::Digest> {
463    build_impl(repo, ostree_ref.as_ref(), config, opts, dest).await
464}
465
466#[cfg(test)]
467mod tests {
468    use super::*;
469
470    #[test]
471    fn test_parse_ocipath() {
472        let default = "/foo/bar";
473        let untagged = "/foo/bar:baz";
474        let tagged = "/foo/bar:baz:latest";
475        assert_eq!(parse_oci_path_and_tag(default), ("/foo/bar", None));
476        assert_eq!(
477            parse_oci_path_and_tag(tagged),
478            ("/foo/bar", Some("baz:latest"))
479        );
480        assert_eq!(parse_oci_path_and_tag(untagged), ("/foo/bar", Some("baz")));
481    }
482}
ostree_ext/container/encapsulate.rs

ostree_ext/container/
encapsulate.rs
