spin_componentize/
lib.rs

1#![deny(warnings)]
2
3use {
4    anyhow::{Context, Result, anyhow},
5    module_info::ModuleInfo,
6    std::{borrow::Cow, collections::HashSet},
7    wasm_encoder::reencode::{Reencode, RoundtripReencoder},
8    wasm_encoder::{CustomSection, ExportSection, ImportSection, Module, RawSection},
9    wasmparser::{Encoding, Parser, Payload},
10    wit_component::{ComponentEncoder, metadata},
11};
12
13pub mod bugs;
14
15#[cfg(test)]
16mod abi_conformance;
17mod module_info;
18
19const SPIN_ADAPTER: &[u8] = include_bytes!(concat!(
20    env!("OUT_DIR"),
21    "/wasm32-unknown-unknown/release/wasi_snapshot_preview1_spin.wasm"
22));
23const PREVIEW1_ADAPTER: &[u8] = include_bytes!(concat!(
24    env!("OUT_DIR"),
25    "/wasm32-unknown-unknown/release/wasi_snapshot_preview1_upstream.wasm"
26));
27
28const COMMAND_ADAPTER: &[u8] = include_bytes!(concat!(
29    env!("OUT_DIR"),
30    "/wasm32-unknown-unknown/release/wasi_snapshot_preview1_command.wasm"
31));
32
33static ADAPTER_NAME: &str = "wasi_snapshot_preview1";
34static CUSTOM_SECTION_NAME: &str = "component-type:reactor";
35static WORLD_NAME: &str = "reactor";
36
37static EXPORT_INTERFACES: &[(&str, &str)] = &[
38    ("handle-redis-message", "inbound-redis"),
39    ("handle-http-request", "inbound-http"),
40];
41
42pub fn componentize_if_necessary(module_or_component: &[u8]) -> Result<Cow<'_, [u8]>> {
43    for payload in Parser::new(0).parse_all(module_or_component) {
44        if let Payload::Version { encoding, .. } = payload.context("unable to parse binary")? {
45            return match encoding {
46                Encoding::Component => Ok(Cow::Borrowed(module_or_component)),
47                Encoding::Module => componentize(module_or_component).map(Cow::Owned),
48            };
49        }
50    }
51    Err(anyhow!("unable to determine wasm binary encoding"))
52}
53
54pub fn componentize(module: &[u8]) -> Result<Vec<u8>> {
55    let module_info = ModuleInfo::from_module(module)?;
56    match WitBindgenVersion::detect(&module_info)? {
57        WitBindgenVersion::V0_2OrNone => componentize_old_module(module, &module_info),
58        WitBindgenVersion::GreaterThanV0_4 => componentize_new_bindgen(module),
59        WitBindgenVersion::Other(other) => Err(anyhow::anyhow!(
60            "cannot adapt modules created with wit-bindgen version {other}"
61        )),
62    }
63}
64
65/// In order to properly componentize modules, we need to know which
66/// version of wit-bindgen was used
67#[derive(Debug)]
68enum WitBindgenVersion {
69    GreaterThanV0_4,
70    V0_2OrNone,
71    Other(String),
72}
73
74impl WitBindgenVersion {
75    fn detect(module_info: &ModuleInfo) -> Result<Self> {
76        if let Some(processors) = module_info.bindgen_processors() {
77            let bindgen_version = processors
78                .iter()
79                .find_map(|(key, value)| key.starts_with("wit-bindgen").then_some(value.as_str()));
80            if let Some(v) = bindgen_version {
81                let mut parts = v.split('.');
82                let Some(major) = parts.next().and_then(|p| p.parse::<u8>().ok()) else {
83                    return Ok(Self::Other(v.to_owned()));
84                };
85                let Some(minor) = parts.next().and_then(|p| p.parse::<u8>().ok()) else {
86                    return Ok(Self::Other(v.to_owned()));
87                };
88                if (major == 0 && minor < 5) || major >= 1 {
89                    return Ok(Self::Other(v.to_owned()));
90                }
91                // Either there should be no patch version or nothing after patch
92                if parts.next().is_none() || parts.next().is_none() {
93                    return Ok(Self::GreaterThanV0_4);
94                } else {
95                    return Ok(Self::Other(v.to_owned()));
96                }
97            }
98        }
99        Ok(Self::V0_2OrNone)
100    }
101}
102
103/// Modules produced with wit-bindgen 0.5 and newer only need wasi preview 1 to preview 2 adapter
104pub fn componentize_new_bindgen(module: &[u8]) -> Result<Vec<u8>> {
105    ComponentEncoder::default()
106        .validate(true)
107        .module(module)?
108        .adapter("wasi_snapshot_preview1", PREVIEW1_ADAPTER)?
109        .encode()
110}
111
112/// Modules *not* produced with wit-bindgen >= 0.5 could be old wit-bindgen or no wit-bindgen
113pub fn componentize_old_module(module: &[u8], module_info: &ModuleInfo) -> Result<Vec<u8>> {
114    // If the module has a _start export and doesn't obviously use wit-bindgen
115    // it is likely an old p1 command module.
116    if module_info.has_start_export && !module_info.probably_uses_wit_bindgen() {
117        bugs::WasiLibc377Bug::check(module_info)?;
118        componentize_command(module)
119    } else {
120        componentize_old_bindgen(module)
121    }
122}
123
124/// Modules produced with wit-bindgen 0.2 need more extensive adaption
125pub fn componentize_old_bindgen(module: &[u8]) -> Result<Vec<u8>> {
126    let (module, exports) = retarget_imports_and_get_exports(ADAPTER_NAME, module)?;
127    let allowed = exports
128        .into_iter()
129        .filter_map(|export| {
130            EXPORT_INTERFACES
131                .iter()
132                .find_map(|(k, v)| (*k == export).then_some(*v))
133        })
134        .collect::<HashSet<&str>>();
135
136    let (adapter, mut bindgen) = metadata::decode(SPIN_ADAPTER)?;
137    let adapter = adapter.expect(
138        "adapter module was malformed, and did not contain a 'component-type' custom section",
139    );
140
141    let world = bindgen
142        .resolve
143        .worlds
144        .iter()
145        .find_map(|(k, v)| (v.name == WORLD_NAME).then_some(k))
146        .ok_or_else(|| anyhow!("world not found: {WORLD_NAME}"))?;
147
148    bindgen.resolve.worlds[world].exports.retain(|k, _| {
149        let k = match &k {
150            wit_parser::WorldKey::Name(n) => n,
151            wit_parser::WorldKey::Interface(i) => match &bindgen.resolve.interfaces[*i].name {
152                Some(n) => n,
153                None => return true,
154            },
155        };
156        allowed.contains(k.as_str())
157    });
158
159    let body = metadata::encode(
160        &bindgen.resolve,
161        world,
162        wit_component::StringEncoding::UTF8,
163        None,
164        false,
165    )?;
166
167    let adapter = add_custom_section(CUSTOM_SECTION_NAME, &body, &adapter)?;
168
169    ComponentEncoder::default()
170        .validate(true)
171        .module(&module)?
172        .adapter(ADAPTER_NAME, &adapter)?
173        .encode()
174}
175
176pub fn componentize_command(module: &[u8]) -> Result<Vec<u8>> {
177    ComponentEncoder::default()
178        .validate(true)
179        .module(module)?
180        .adapter(ADAPTER_NAME, COMMAND_ADAPTER)?
181        .encode()
182}
183
184fn retarget_imports_and_get_exports(target: &str, module: &[u8]) -> Result<(Vec<u8>, Vec<String>)> {
185    let mut result = Module::new();
186    let mut exports_result = Vec::new();
187
188    for payload in Parser::new(0).parse_all(module) {
189        match payload? {
190            Payload::ImportSection(reader) => {
191                let mut imports = ImportSection::new();
192                for import in reader.into_imports() {
193                    let import = import?;
194                    let (module, field) = if import.module == target {
195                        (Cow::Borrowed(import.module), Cow::Borrowed(import.name))
196                    } else {
197                        (
198                            Cow::Borrowed(target),
199                            Cow::Owned(format!("{}:{}", import.module, import.name)),
200                        )
201                    };
202                    let ty = RoundtripReencoder.entity_type(import.ty)?;
203                    imports.import(&module, &field, ty);
204                }
205                result.section(&imports);
206            }
207
208            Payload::ExportSection(reader) => {
209                let mut exports = ExportSection::new();
210                for export in reader {
211                    let export = export?;
212                    exports_result.push(export.name.to_owned());
213                    let kind = RoundtripReencoder.export_kind(export.kind)?;
214                    exports.export(export.name, kind, export.index);
215                }
216                result.section(&exports);
217            }
218
219            payload => {
220                if let Some((id, range)) = payload.as_section() {
221                    result.section(&RawSection {
222                        id,
223                        data: &module[range.start as usize..range.end as usize],
224                    });
225                }
226            }
227        }
228    }
229
230    Ok((result.finish(), exports_result))
231}
232
233fn add_custom_section(name: &str, data: &[u8], module: &[u8]) -> Result<Vec<u8>> {
234    let mut result = Module::new();
235
236    for payload in Parser::new(0).parse_all(module) {
237        if let Some((id, range)) = payload?.as_section() {
238            result.section(&RawSection {
239                id,
240                data: &module[range.start as usize..range.end as usize],
241            });
242        }
243    }
244
245    result.section(&CustomSection {
246        name: Cow::Borrowed(name),
247        data: Cow::Borrowed(data),
248    });
249
250    Ok(result.finish())
251}
252
253#[cfg(test)]
254mod tests {
255    use std::{path::PathBuf, process};
256
257    use anyhow::Context;
258    use wasmtime::error::Context as _;
259    use wasmtime_wasi::p2::pipe::MemoryOutputPipe;
260
261    use {
262        super::abi_conformance::{
263            InvocationStyle, KeyValueReport, LlmReport, MysqlReport, PostgresReport, RedisReport,
264            Report, TestConfig, WasiReport,
265        },
266        anyhow::{Result, anyhow},
267        tokio::fs,
268        wasmtime::{
269            Config, Engine, Store,
270            component::{Component, Linker},
271        },
272        wasmtime_wasi::p2::{bindings::Command, pipe::MemoryInputPipe},
273        wasmtime_wasi::{ResourceTable, WasiCtx, WasiCtxBuilder, WasiCtxView, WasiView},
274    };
275
276    async fn run_spin(module: &[u8]) -> Result<()> {
277        let mut config = Config::new();
278        config.wasm_component_model(true);
279
280        let engine = Engine::new(&config)?;
281
282        let component = Component::new(
283            &engine,
284            crate::componentize(module).context("could not componentize")?,
285        )
286        .context("failed to instantiate componentized bytes")?;
287
288        let report = super::abi_conformance::test(
289            &component,
290            &engine,
291            TestConfig {
292                invocation_style: InvocationStyle::InboundHttp,
293            },
294        )
295        .await
296        .context("abi conformance test failed")?;
297
298        let expected = Report {
299            inbound_http: Ok(()),
300            inbound_redis: Ok(()),
301            config: Ok(()),
302            http: Ok(()),
303            redis: RedisReport {
304                publish: Ok(()),
305                set: Ok(()),
306                get: Ok(()),
307                incr: Ok(()),
308                del: Ok(()),
309                sadd: Ok(()),
310                srem: Ok(()),
311                smembers: Ok(()),
312                execute: Ok(()),
313            },
314            postgres: PostgresReport {
315                execute: Ok(()),
316                query: Ok(()),
317            },
318            mysql: MysqlReport {
319                execute: Ok(()),
320                query: Ok(()),
321            },
322            key_value: KeyValueReport {
323                open: Ok(()),
324                get: Ok(()),
325                set: Ok(()),
326                delete: Ok(()),
327                exists: Ok(()),
328                get_keys: Ok(()),
329                close: Ok(()),
330            },
331            llm: LlmReport { infer: Ok(()) },
332            wasi: WasiReport {
333                env: Ok(()),
334                epoch: Ok(()),
335                random: Ok(()),
336                stdio: Ok(()),
337                read: Ok(()),
338                readdir: Ok(()),
339                stat: Ok(()),
340            },
341        };
342
343        if report == expected {
344            Ok(())
345        } else {
346            Err(anyhow!("{report:#?}"))
347        }
348    }
349
350    async fn run_command(module: &[u8]) -> Result<()> {
351        let mut config = Config::new();
352        config.wasm_component_model(true);
353
354        let engine = Engine::new(&config)?;
355
356        struct Wasi {
357            ctx: WasiCtx,
358            table: ResourceTable,
359        }
360        impl WasiView for Wasi {
361            fn ctx(&mut self) -> WasiCtxView<'_> {
362                WasiCtxView {
363                    ctx: &mut self.ctx,
364                    table: &mut self.table,
365                }
366            }
367        }
368
369        let mut linker = Linker::<Wasi>::new(&engine);
370        wasmtime_wasi::p2::add_to_linker_async(&mut linker)?;
371        let mut ctx = WasiCtxBuilder::new();
372        let stdout = MemoryOutputPipe::new(1024);
373        ctx.stdin(MemoryInputPipe::new("So rested he by the Tumtum tree"))
374            .stdout(stdout.clone())
375            .args(&["Jabberwocky"]);
376
377        let table = ResourceTable::new();
378        let wasi = Wasi {
379            ctx: ctx.build(),
380            table,
381        };
382
383        let mut store = Store::new(&engine, wasi);
384
385        let component = Component::new(&engine, crate::componentize_command(module)?)?;
386
387        let wasi = Command::instantiate_async(&mut store, &component, &linker).await?;
388
389        wasi.wasi_cli_run()
390            .call_run(&mut store)
391            .await?
392            .map_err(|()| anyhow!("command returned with failing exit status"))?;
393
394        drop(store);
395
396        let stdout = stdout.try_into_inner().unwrap().to_vec();
397
398        assert_eq!(
399            b"Jabberwocky\nSo rested he by the Tumtum tree" as &[_],
400            &stdout
401        );
402
403        Ok(())
404    }
405
406    #[tokio::test]
407    async fn rust_wit_bindgen_02() -> Result<()> {
408        build_rust_test_case("rust-case-0.2");
409        run_spin(
410            &fs::read(concat!(
411                env!("OUT_DIR"),
412                "/wasm32-wasip1/release/rust_case_02.wasm"
413            ))
414            .await?,
415        )
416        .await
417    }
418
419    #[tokio::test]
420    async fn rust_wit_bindgen_08() -> Result<()> {
421        build_rust_test_case("rust-case-0.8");
422        run_spin(
423            &fs::read(concat!(
424                env!("OUT_DIR"),
425                "/wasm32-wasip1/release/rust_case_08.wasm"
426            ))
427            .await?,
428        )
429        .await
430    }
431
432    #[ignore]
433    #[tokio::test]
434    async fn go() -> Result<()> {
435        let out_dir = PathBuf::from(std::env::var_os("OUT_DIR").unwrap());
436        let mut cmd = process::Command::new("tinygo");
437        cmd.arg("build")
438            .current_dir("tests/go-case")
439            .arg("-target=wasip1")
440            .arg("-gc=leaking")
441            .arg("-buildmode=c-shared")
442            .arg("-no-debug")
443            .arg("-o")
444            .arg(out_dir.join("go_case.wasm"))
445            .arg("main.go");
446
447        // If just skip this if TinyGo is not installed
448        _ = cmd.status();
449        run_spin(&fs::read(concat!(env!("OUT_DIR"), "/go_case.wasm")).await?).await
450    }
451
452    #[tokio::test]
453    async fn rust_command() -> Result<()> {
454        build_rust_test_case("rust-command");
455        run_command(
456            &fs::read(concat!(
457                env!("OUT_DIR"),
458                "/wasm32-wasip1/release/rust-command.wasm"
459            ))
460            .await?,
461        )
462        .await
463    }
464
465    fn build_rust_test_case(name: &str) {
466        let out_dir = PathBuf::from(std::env::var_os("OUT_DIR").unwrap());
467        let mut cmd = process::Command::new("cargo");
468        cmd.arg("build")
469            .current_dir(format!("tests/{name}"))
470            .arg("--release")
471            .arg("--target=wasm32-wasip1")
472            .env("CARGO_TARGET_DIR", out_dir);
473
474        let status = cmd.status().unwrap();
475        assert!(status.success());
476    }
477}
spin_componentize/lib.rs

spin_componentize/
lib.rs
