spin_componentize/
lib.rs

1#![deny(warnings)]
2
3use {
4    anyhow::{Context, Result, anyhow},
5    module_info::ModuleInfo,
6    std::{borrow::Cow, collections::HashSet},
7    wasm_encoder::reencode::{Reencode, RoundtripReencoder},
8    wasm_encoder::{CustomSection, ExportSection, ImportSection, Module, RawSection},
9    wasmparser::{Encoding, Parser, Payload},
10    wit_component::{ComponentEncoder, metadata},
11};
12
13pub mod bugs;
14
15#[cfg(test)]
16mod abi_conformance;
17mod module_info;
18
19const SPIN_ADAPTER: &[u8] = include_bytes!(concat!(
20    env!("OUT_DIR"),
21    "/wasm32-unknown-unknown/release/wasi_snapshot_preview1_spin.wasm"
22));
23const PREVIEW1_ADAPTER: &[u8] = include_bytes!(concat!(
24    env!("OUT_DIR"),
25    "/wasm32-unknown-unknown/release/wasi_snapshot_preview1_upstream.wasm"
26));
27
28const COMMAND_ADAPTER: &[u8] = include_bytes!(concat!(
29    env!("OUT_DIR"),
30    "/wasm32-unknown-unknown/release/wasi_snapshot_preview1_command.wasm"
31));
32
33static ADAPTER_NAME: &str = "wasi_snapshot_preview1";
34static CUSTOM_SECTION_NAME: &str = "component-type:reactor";
35static WORLD_NAME: &str = "reactor";
36
37static EXPORT_INTERFACES: &[(&str, &str)] = &[
38    ("handle-redis-message", "inbound-redis"),
39    ("handle-http-request", "inbound-http"),
40];
41
42pub fn componentize_if_necessary(module_or_component: &[u8]) -> Result<Cow<'_, [u8]>> {
43    for payload in Parser::new(0).parse_all(module_or_component) {
44        if let Payload::Version { encoding, .. } = payload.context("unable to parse binary")? {
45            return match encoding {
46                Encoding::Component => Ok(Cow::Borrowed(module_or_component)),
47                Encoding::Module => componentize(module_or_component).map(Cow::Owned),
48            };
49        }
50    }
51    Err(anyhow!("unable to determine wasm binary encoding"))
52}
53
54pub fn componentize(module: &[u8]) -> Result<Vec<u8>> {
55    let module_info = ModuleInfo::from_module(module)?;
56    match WitBindgenVersion::detect(&module_info)? {
57        WitBindgenVersion::V0_2OrNone => componentize_old_module(module, &module_info),
58        WitBindgenVersion::GreaterThanV0_4 => componentize_new_bindgen(module),
59        WitBindgenVersion::Other(other) => Err(anyhow::anyhow!(
60            "cannot adapt modules created with wit-bindgen version {other}"
61        )),
62    }
63}
64
65/// In order to properly componentize modules, we need to know which
66/// version of wit-bindgen was used
67#[derive(Debug)]
68enum WitBindgenVersion {
69    GreaterThanV0_4,
70    V0_2OrNone,
71    Other(String),
72}
73
74impl WitBindgenVersion {
75    fn detect(module_info: &ModuleInfo) -> Result<Self> {
76        if let Some(processors) = module_info.bindgen_processors() {
77            let bindgen_version = processors
78                .iter()
79                .find_map(|(key, value)| key.starts_with("wit-bindgen").then_some(value.as_str()));
80            if let Some(v) = bindgen_version {
81                let mut parts = v.split('.');
82                let Some(major) = parts.next().and_then(|p| p.parse::<u8>().ok()) else {
83                    return Ok(Self::Other(v.to_owned()));
84                };
85                let Some(minor) = parts.next().and_then(|p| p.parse::<u8>().ok()) else {
86                    return Ok(Self::Other(v.to_owned()));
87                };
88                if (major == 0 && minor < 5) || major >= 1 {
89                    return Ok(Self::Other(v.to_owned()));
90                }
91                // Either there should be no patch version or nothing after patch
92                if parts.next().is_none() || parts.next().is_none() {
93                    return Ok(Self::GreaterThanV0_4);
94                } else {
95                    return Ok(Self::Other(v.to_owned()));
96                }
97            }
98        }
99        Ok(Self::V0_2OrNone)
100    }
101}
102
103/// Modules produced with wit-bindgen 0.5 and newer only need wasi preview 1 to preview 2 adapter
104pub fn componentize_new_bindgen(module: &[u8]) -> Result<Vec<u8>> {
105    ComponentEncoder::default()
106        .validate(true)
107        .module(module)?
108        .adapter("wasi_snapshot_preview1", PREVIEW1_ADAPTER)?
109        .encode()
110}
111
112/// Modules *not* produced with wit-bindgen >= 0.5 could be old wit-bindgen or no wit-bindgen
113pub fn componentize_old_module(module: &[u8], module_info: &ModuleInfo) -> Result<Vec<u8>> {
114    // If the module has a _start export and doesn't obviously use wit-bindgen
115    // it is likely an old p1 command module.
116    if module_info.has_start_export && !module_info.probably_uses_wit_bindgen() {
117        bugs::WasiLibc377Bug::check(module_info)?;
118        componentize_command(module)
119    } else {
120        componentize_old_bindgen(module)
121    }
122}
123
124/// Modules produced with wit-bindgen 0.2 need more extensive adaption
125pub fn componentize_old_bindgen(module: &[u8]) -> Result<Vec<u8>> {
126    let (module, exports) = retarget_imports_and_get_exports(ADAPTER_NAME, module)?;
127    let allowed = exports
128        .into_iter()
129        .filter_map(|export| {
130            EXPORT_INTERFACES
131                .iter()
132                .find_map(|(k, v)| (*k == export).then_some(*v))
133        })
134        .collect::<HashSet<&str>>();
135
136    let (adapter, mut bindgen) = metadata::decode(SPIN_ADAPTER)?;
137    let adapter = adapter.expect(
138        "adapter module was malformed, and did not contain a 'component-type' custom section",
139    );
140
141    let world = bindgen
142        .resolve
143        .worlds
144        .iter()
145        .find_map(|(k, v)| (v.name == WORLD_NAME).then_some(k))
146        .ok_or_else(|| anyhow!("world not found: {WORLD_NAME}"))?;
147
148    bindgen.resolve.worlds[world].exports.retain(|k, _| {
149        let k = match &k {
150            wit_parser::WorldKey::Name(n) => n,
151            wit_parser::WorldKey::Interface(i) => match &bindgen.resolve.interfaces[*i].name {
152                Some(n) => n,
153                None => return true,
154            },
155        };
156        allowed.contains(k.as_str())
157    });
158
159    let body = metadata::encode(
160        &bindgen.resolve,
161        world,
162        wit_component::StringEncoding::UTF8,
163        None,
164    )?;
165
166    let adapter = add_custom_section(CUSTOM_SECTION_NAME, &body, &adapter)?;
167
168    ComponentEncoder::default()
169        .validate(true)
170        .module(&module)?
171        .adapter(ADAPTER_NAME, &adapter)?
172        .encode()
173}
174
175pub fn componentize_command(module: &[u8]) -> Result<Vec<u8>> {
176    ComponentEncoder::default()
177        .validate(true)
178        .module(module)?
179        .adapter(ADAPTER_NAME, COMMAND_ADAPTER)?
180        .encode()
181}
182
183fn retarget_imports_and_get_exports(target: &str, module: &[u8]) -> Result<(Vec<u8>, Vec<String>)> {
184    let mut result = Module::new();
185    let mut exports_result = Vec::new();
186
187    for payload in Parser::new(0).parse_all(module) {
188        match payload? {
189            Payload::ImportSection(reader) => {
190                let mut imports = ImportSection::new();
191                for import in reader.into_imports() {
192                    let import = import?;
193                    let (module, field) = if import.module == target {
194                        (Cow::Borrowed(import.module), Cow::Borrowed(import.name))
195                    } else {
196                        (
197                            Cow::Borrowed(target),
198                            Cow::Owned(format!("{}:{}", import.module, import.name)),
199                        )
200                    };
201                    let ty = RoundtripReencoder.entity_type(import.ty)?;
202                    imports.import(&module, &field, ty);
203                }
204                result.section(&imports);
205            }
206
207            Payload::ExportSection(reader) => {
208                let mut exports = ExportSection::new();
209                for export in reader {
210                    let export = export?;
211                    exports_result.push(export.name.to_owned());
212                    let kind = RoundtripReencoder.export_kind(export.kind)?;
213                    exports.export(export.name, kind, export.index);
214                }
215                result.section(&exports);
216            }
217
218            payload => {
219                if let Some((id, range)) = payload.as_section() {
220                    result.section(&RawSection {
221                        id,
222                        data: &module[range],
223                    });
224                }
225            }
226        }
227    }
228
229    Ok((result.finish(), exports_result))
230}
231
232fn add_custom_section(name: &str, data: &[u8], module: &[u8]) -> Result<Vec<u8>> {
233    let mut result = Module::new();
234
235    for payload in Parser::new(0).parse_all(module) {
236        if let Some((id, range)) = payload?.as_section() {
237            result.section(&RawSection {
238                id,
239                data: &module[range],
240            });
241        }
242    }
243
244    result.section(&CustomSection {
245        name: Cow::Borrowed(name),
246        data: Cow::Borrowed(data),
247    });
248
249    Ok(result.finish())
250}
251
252#[cfg(test)]
253mod tests {
254    use std::{path::PathBuf, process};
255
256    use anyhow::Context;
257    use wasmtime::error::Context as _;
258    use wasmtime_wasi::p2::pipe::MemoryOutputPipe;
259
260    use {
261        super::abi_conformance::{
262            InvocationStyle, KeyValueReport, LlmReport, MysqlReport, PostgresReport, RedisReport,
263            Report, TestConfig, WasiReport,
264        },
265        anyhow::{Result, anyhow},
266        tokio::fs,
267        wasmtime::{
268            Config, Engine, Store,
269            component::{Component, Linker},
270        },
271        wasmtime_wasi::p2::{bindings::Command, pipe::MemoryInputPipe},
272        wasmtime_wasi::{ResourceTable, WasiCtx, WasiCtxBuilder, WasiCtxView, WasiView},
273    };
274
275    async fn run_spin(module: &[u8]) -> Result<()> {
276        let mut config = Config::new();
277        config.wasm_component_model(true);
278
279        let engine = Engine::new(&config)?;
280
281        let component = Component::new(
282            &engine,
283            crate::componentize(module).context("could not componentize")?,
284        )
285        .context("failed to instantiate componentized bytes")?;
286
287        let report = super::abi_conformance::test(
288            &component,
289            &engine,
290            TestConfig {
291                invocation_style: InvocationStyle::InboundHttp,
292            },
293        )
294        .await
295        .context("abi conformance test failed")?;
296
297        let expected = Report {
298            inbound_http: Ok(()),
299            inbound_redis: Ok(()),
300            config: Ok(()),
301            http: Ok(()),
302            redis: RedisReport {
303                publish: Ok(()),
304                set: Ok(()),
305                get: Ok(()),
306                incr: Ok(()),
307                del: Ok(()),
308                sadd: Ok(()),
309                srem: Ok(()),
310                smembers: Ok(()),
311                execute: Ok(()),
312            },
313            postgres: PostgresReport {
314                execute: Ok(()),
315                query: Ok(()),
316            },
317            mysql: MysqlReport {
318                execute: Ok(()),
319                query: Ok(()),
320            },
321            key_value: KeyValueReport {
322                open: Ok(()),
323                get: Ok(()),
324                set: Ok(()),
325                delete: Ok(()),
326                exists: Ok(()),
327                get_keys: Ok(()),
328                close: Ok(()),
329            },
330            llm: LlmReport { infer: Ok(()) },
331            wasi: WasiReport {
332                env: Ok(()),
333                epoch: Ok(()),
334                random: Ok(()),
335                stdio: Ok(()),
336                read: Ok(()),
337                readdir: Ok(()),
338                stat: Ok(()),
339            },
340        };
341
342        if report == expected {
343            Ok(())
344        } else {
345            Err(anyhow!("{report:#?}"))
346        }
347    }
348
349    async fn run_command(module: &[u8]) -> Result<()> {
350        let mut config = Config::new();
351        config.wasm_component_model(true);
352
353        let engine = Engine::new(&config)?;
354
355        struct Wasi {
356            ctx: WasiCtx,
357            table: ResourceTable,
358        }
359        impl WasiView for Wasi {
360            fn ctx(&mut self) -> WasiCtxView<'_> {
361                WasiCtxView {
362                    ctx: &mut self.ctx,
363                    table: &mut self.table,
364                }
365            }
366        }
367
368        let mut linker = Linker::<Wasi>::new(&engine);
369        wasmtime_wasi::p2::add_to_linker_async(&mut linker)?;
370        let mut ctx = WasiCtxBuilder::new();
371        let stdout = MemoryOutputPipe::new(1024);
372        ctx.stdin(MemoryInputPipe::new("So rested he by the Tumtum tree"))
373            .stdout(stdout.clone())
374            .args(&["Jabberwocky"]);
375
376        let table = ResourceTable::new();
377        let wasi = Wasi {
378            ctx: ctx.build(),
379            table,
380        };
381
382        let mut store = Store::new(&engine, wasi);
383
384        let component = Component::new(&engine, crate::componentize_command(module)?)?;
385
386        let wasi = Command::instantiate_async(&mut store, &component, &linker).await?;
387
388        wasi.wasi_cli_run()
389            .call_run(&mut store)
390            .await?
391            .map_err(|()| anyhow!("command returned with failing exit status"))?;
392
393        drop(store);
394
395        let stdout = stdout.try_into_inner().unwrap().to_vec();
396
397        assert_eq!(
398            b"Jabberwocky\nSo rested he by the Tumtum tree" as &[_],
399            &stdout
400        );
401
402        Ok(())
403    }
404
405    #[tokio::test]
406    async fn rust_wit_bindgen_02() -> Result<()> {
407        build_rust_test_case("rust-case-0.2");
408        run_spin(
409            &fs::read(concat!(
410                env!("OUT_DIR"),
411                "/wasm32-wasip1/release/rust_case_02.wasm"
412            ))
413            .await?,
414        )
415        .await
416    }
417
418    #[tokio::test]
419    async fn rust_wit_bindgen_08() -> Result<()> {
420        build_rust_test_case("rust-case-0.8");
421        run_spin(
422            &fs::read(concat!(
423                env!("OUT_DIR"),
424                "/wasm32-wasip1/release/rust_case_08.wasm"
425            ))
426            .await?,
427        )
428        .await
429    }
430
431    #[ignore]
432    #[tokio::test]
433    async fn go() -> Result<()> {
434        let out_dir = PathBuf::from(std::env::var_os("OUT_DIR").unwrap());
435        let mut cmd = process::Command::new("tinygo");
436        cmd.arg("build")
437            .current_dir("tests/go-case")
438            .arg("-target=wasip1")
439            .arg("-gc=leaking")
440            .arg("-buildmode=c-shared")
441            .arg("-no-debug")
442            .arg("-o")
443            .arg(out_dir.join("go_case.wasm"))
444            .arg("main.go");
445
446        // If just skip this if TinyGo is not installed
447        _ = cmd.status();
448        run_spin(&fs::read(concat!(env!("OUT_DIR"), "/go_case.wasm")).await?).await
449    }
450
451    #[tokio::test]
452    async fn rust_command() -> Result<()> {
453        build_rust_test_case("rust-command");
454        run_command(
455            &fs::read(concat!(
456                env!("OUT_DIR"),
457                "/wasm32-wasip1/release/rust-command.wasm"
458            ))
459            .await?,
460        )
461        .await
462    }
463
464    fn build_rust_test_case(name: &str) {
465        let out_dir = PathBuf::from(std::env::var_os("OUT_DIR").unwrap());
466        let mut cmd = process::Command::new("cargo");
467        cmd.arg("build")
468            .current_dir(format!("tests/{name}"))
469            .arg("--release")
470            .arg("--target=wasm32-wasip1")
471            .env("CARGO_TARGET_DIR", out_dir);
472
473        let status = cmd.status().unwrap();
474        assert!(status.success());
475    }
476}
spin_componentize/lib.rs

spin_componentize/
lib.rs
