spin_componentize/
lib.rs

1#![deny(warnings)]
2
3use {
4    anyhow::{anyhow, Context, Result},
5    module_info::ModuleInfo,
6    std::{borrow::Cow, collections::HashSet},
7    wasm_encoder::reencode::{Reencode, RoundtripReencoder},
8    wasm_encoder::{CustomSection, ExportSection, ImportSection, Module, RawSection},
9    wasmparser::{Encoding, Parser, Payload},
10    wit_component::{metadata, ComponentEncoder},
11};
12
13pub mod bugs;
14
15#[cfg(test)]
16mod abi_conformance;
17mod module_info;
18
19const SPIN_ADAPTER: &[u8] = include_bytes!(concat!(
20    env!("OUT_DIR"),
21    "/wasm32-unknown-unknown/release/wasi_snapshot_preview1_spin.wasm"
22));
23const PREVIEW1_ADAPTER: &[u8] = include_bytes!(concat!(
24    env!("OUT_DIR"),
25    "/wasm32-unknown-unknown/release/wasi_snapshot_preview1_upstream.wasm"
26));
27
28const COMMAND_ADAPTER: &[u8] = include_bytes!(concat!(
29    env!("OUT_DIR"),
30    "/wasm32-unknown-unknown/release/wasi_snapshot_preview1_command.wasm"
31));
32
33static ADAPTER_NAME: &str = "wasi_snapshot_preview1";
34static CUSTOM_SECTION_NAME: &str = "component-type:reactor";
35static WORLD_NAME: &str = "reactor";
36
37static EXPORT_INTERFACES: &[(&str, &str)] = &[
38    ("handle-redis-message", "inbound-redis"),
39    ("handle-http-request", "inbound-http"),
40];
41
42pub fn componentize_if_necessary(module_or_component: &[u8]) -> Result<Cow<'_, [u8]>> {
43    for payload in Parser::new(0).parse_all(module_or_component) {
44        if let Payload::Version { encoding, .. } = payload.context("unable to parse binary")? {
45            return match encoding {
46                Encoding::Component => Ok(Cow::Borrowed(module_or_component)),
47                Encoding::Module => componentize(module_or_component).map(Cow::Owned),
48            };
49        }
50    }
51    Err(anyhow!("unable to determine wasm binary encoding"))
52}
53
54pub fn componentize(module: &[u8]) -> Result<Vec<u8>> {
55    let module_info = ModuleInfo::from_module(module)?;
56    match WitBindgenVersion::detect(&module_info)? {
57        WitBindgenVersion::V0_2OrNone => componentize_old_module(module, &module_info),
58        WitBindgenVersion::GreaterThanV0_4 => componentize_new_bindgen(module),
59        WitBindgenVersion::Other(other) => Err(anyhow::anyhow!(
60            "cannot adapt modules created with wit-bindgen version {other}"
61        )),
62    }
63}
64
65/// In order to properly componentize modules, we need to know which
66/// version of wit-bindgen was used
67#[derive(Debug)]
68enum WitBindgenVersion {
69    GreaterThanV0_4,
70    V0_2OrNone,
71    Other(String),
72}
73
74impl WitBindgenVersion {
75    fn detect(module_info: &ModuleInfo) -> Result<Self> {
76        if let Some(processors) = module_info.bindgen_processors() {
77            let bindgen_version = processors
78                .iter()
79                .find_map(|(key, value)| key.starts_with("wit-bindgen").then_some(value.as_str()));
80            if let Some(v) = bindgen_version {
81                let mut parts = v.split('.');
82                let Some(major) = parts.next().and_then(|p| p.parse::<u8>().ok()) else {
83                    return Ok(Self::Other(v.to_owned()));
84                };
85                let Some(minor) = parts.next().and_then(|p| p.parse::<u8>().ok()) else {
86                    return Ok(Self::Other(v.to_owned()));
87                };
88                if (major == 0 && minor < 5) || major >= 1 {
89                    return Ok(Self::Other(v.to_owned()));
90                }
91                // Either there should be no patch version or nothing after patch
92                if parts.next().is_none() || parts.next().is_none() {
93                    return Ok(Self::GreaterThanV0_4);
94                } else {
95                    return Ok(Self::Other(v.to_owned()));
96                }
97            }
98        }
99        Ok(Self::V0_2OrNone)
100    }
101}
102
103/// Modules produced with wit-bindgen 0.5 and newer only need wasi preview 1 to preview 2 adapter
104pub fn componentize_new_bindgen(module: &[u8]) -> Result<Vec<u8>> {
105    ComponentEncoder::default()
106        .validate(true)
107        .module(module)?
108        .adapter("wasi_snapshot_preview1", PREVIEW1_ADAPTER)?
109        .encode()
110}
111
112/// Modules *not* produced with wit-bindgen >= 0.5 could be old wit-bindgen or no wit-bindgen
113pub fn componentize_old_module(module: &[u8], module_info: &ModuleInfo) -> Result<Vec<u8>> {
114    // If the module has a _start export and doesn't obviously use wit-bindgen
115    // it is likely an old p1 command module.
116    if module_info.has_start_export && !module_info.probably_uses_wit_bindgen() {
117        bugs::WasiLibc377Bug::check(module_info)?;
118        componentize_command(module)
119    } else {
120        componentize_old_bindgen(module)
121    }
122}
123
124/// Modules produced with wit-bindgen 0.2 need more extensive adaption
125pub fn componentize_old_bindgen(module: &[u8]) -> Result<Vec<u8>> {
126    let (module, exports) = retarget_imports_and_get_exports(ADAPTER_NAME, module)?;
127    let allowed = exports
128        .into_iter()
129        .filter_map(|export| {
130            EXPORT_INTERFACES
131                .iter()
132                .find_map(|(k, v)| (*k == export).then_some(*v))
133        })
134        .collect::<HashSet<&str>>();
135
136    let (adapter, mut bindgen) = metadata::decode(SPIN_ADAPTER)?;
137    let adapter = adapter.expect(
138        "adapter module was malformed, and did not contain a 'component-type' custom section",
139    );
140
141    let world = bindgen
142        .resolve
143        .worlds
144        .iter()
145        .find_map(|(k, v)| (v.name == WORLD_NAME).then_some(k))
146        .ok_or_else(|| anyhow!("world not found: {WORLD_NAME}"))?;
147
148    bindgen.resolve.worlds[world].exports.retain(|k, _| {
149        let k = match &k {
150            wit_parser::WorldKey::Name(n) => n,
151            wit_parser::WorldKey::Interface(i) => match &bindgen.resolve.interfaces[*i].name {
152                Some(n) => n,
153                None => return true,
154            },
155        };
156        allowed.contains(k.as_str())
157    });
158
159    let body = metadata::encode(
160        &bindgen.resolve,
161        world,
162        wit_component::StringEncoding::UTF8,
163        None,
164    )?;
165
166    let adapter = add_custom_section(CUSTOM_SECTION_NAME, &body, &adapter)?;
167
168    ComponentEncoder::default()
169        .validate(true)
170        .module(&module)?
171        .adapter(ADAPTER_NAME, &adapter)?
172        .encode()
173}
174
175pub fn componentize_command(module: &[u8]) -> Result<Vec<u8>> {
176    ComponentEncoder::default()
177        .validate(true)
178        .module(module)?
179        .adapter(ADAPTER_NAME, COMMAND_ADAPTER)?
180        .encode()
181}
182
183fn retarget_imports_and_get_exports(target: &str, module: &[u8]) -> Result<(Vec<u8>, Vec<String>)> {
184    let mut result = Module::new();
185    let mut exports_result = Vec::new();
186
187    for payload in Parser::new(0).parse_all(module) {
188        match payload? {
189            Payload::ImportSection(reader) => {
190                let mut imports = ImportSection::new();
191                for import in reader {
192                    let import = import?;
193                    let (module, field) = if import.module == target {
194                        (Cow::Borrowed(import.module), Cow::Borrowed(import.name))
195                    } else {
196                        (
197                            Cow::Borrowed(target),
198                            Cow::Owned(format!("{}:{}", import.module, import.name)),
199                        )
200                    };
201                    let ty = RoundtripReencoder.entity_type(import.ty)?;
202                    imports.import(&module, &field, ty);
203                }
204                result.section(&imports);
205            }
206
207            Payload::ExportSection(reader) => {
208                let mut exports = ExportSection::new();
209                for export in reader {
210                    let export = export?;
211                    exports_result.push(export.name.to_owned());
212                    let kind = RoundtripReencoder.export_kind(export.kind)?;
213                    exports.export(export.name, kind, export.index);
214                }
215                result.section(&exports);
216            }
217
218            payload => {
219                if let Some((id, range)) = payload.as_section() {
220                    result.section(&RawSection {
221                        id,
222                        data: &module[range],
223                    });
224                }
225            }
226        }
227    }
228
229    Ok((result.finish(), exports_result))
230}
231
232fn add_custom_section(name: &str, data: &[u8], module: &[u8]) -> Result<Vec<u8>> {
233    let mut result = Module::new();
234
235    for payload in Parser::new(0).parse_all(module) {
236        if let Some((id, range)) = payload?.as_section() {
237            result.section(&RawSection {
238                id,
239                data: &module[range],
240            });
241        }
242    }
243
244    result.section(&CustomSection {
245        name: Cow::Borrowed(name),
246        data: Cow::Borrowed(data),
247    });
248
249    Ok(result.finish())
250}
251
252#[cfg(test)]
253mod tests {
254    use std::{path::PathBuf, process};
255
256    use anyhow::Context;
257    use wasmtime_wasi::p2::pipe::MemoryOutputPipe;
258
259    use {
260        super::abi_conformance::{
261            InvocationStyle, KeyValueReport, LlmReport, MysqlReport, PostgresReport, RedisReport,
262            Report, TestConfig, WasiReport,
263        },
264        anyhow::{anyhow, Result},
265        tokio::fs,
266        wasmtime::{
267            component::{Component, Linker},
268            Config, Engine, Store,
269        },
270        wasmtime_wasi::p2::{bindings::Command, pipe::MemoryInputPipe},
271        wasmtime_wasi::{ResourceTable, WasiCtx, WasiCtxBuilder, WasiCtxView, WasiView},
272    };
273
274    async fn run_spin(module: &[u8]) -> Result<()> {
275        let mut config = Config::new();
276        config.wasm_component_model(true);
277        config.async_support(true);
278
279        let engine = Engine::new(&config)?;
280
281        let component = Component::new(
282            &engine,
283            crate::componentize(module).context("could not componentize")?,
284        )
285        .context("failed to instantiate componentized bytes")?;
286
287        let report = super::abi_conformance::test(
288            &component,
289            &engine,
290            TestConfig {
291                invocation_style: InvocationStyle::InboundHttp,
292            },
293        )
294        .await
295        .context("abi conformance test failed")?;
296
297        let expected = Report {
298            inbound_http: Ok(()),
299            inbound_redis: Ok(()),
300            config: Ok(()),
301            http: Ok(()),
302            redis: RedisReport {
303                publish: Ok(()),
304                set: Ok(()),
305                get: Ok(()),
306                incr: Ok(()),
307                del: Ok(()),
308                sadd: Ok(()),
309                srem: Ok(()),
310                smembers: Ok(()),
311                execute: Ok(()),
312            },
313            postgres: PostgresReport {
314                execute: Ok(()),
315                query: Ok(()),
316            },
317            mysql: MysqlReport {
318                execute: Ok(()),
319                query: Ok(()),
320            },
321            key_value: KeyValueReport {
322                open: Ok(()),
323                get: Ok(()),
324                set: Ok(()),
325                delete: Ok(()),
326                exists: Ok(()),
327                get_keys: Ok(()),
328                close: Ok(()),
329            },
330            llm: LlmReport { infer: Ok(()) },
331            wasi: WasiReport {
332                env: Ok(()),
333                epoch: Ok(()),
334                random: Ok(()),
335                stdio: Ok(()),
336                read: Ok(()),
337                readdir: Ok(()),
338                stat: Ok(()),
339            },
340        };
341
342        if report == expected {
343            Ok(())
344        } else {
345            Err(anyhow!("{report:#?}"))
346        }
347    }
348
349    async fn run_command(module: &[u8]) -> Result<()> {
350        let mut config = Config::new();
351        config.wasm_component_model(true);
352        config.async_support(true);
353
354        let engine = Engine::new(&config)?;
355
356        struct Wasi {
357            ctx: WasiCtx,
358            table: ResourceTable,
359        }
360        impl WasiView for Wasi {
361            fn ctx(&mut self) -> WasiCtxView<'_> {
362                WasiCtxView {
363                    ctx: &mut self.ctx,
364                    table: &mut self.table,
365                }
366            }
367        }
368
369        let mut linker = Linker::<Wasi>::new(&engine);
370        wasmtime_wasi::p2::add_to_linker_async(&mut linker)?;
371        let mut ctx = WasiCtxBuilder::new();
372        let stdout = MemoryOutputPipe::new(1024);
373        ctx.stdin(MemoryInputPipe::new("So rested he by the Tumtum tree"))
374            .stdout(stdout.clone())
375            .args(&["Jabberwocky"]);
376
377        let table = ResourceTable::new();
378        let wasi = Wasi {
379            ctx: ctx.build(),
380            table,
381        };
382
383        let mut store = Store::new(&engine, wasi);
384
385        let component = Component::new(&engine, crate::componentize_command(module)?)?;
386
387        let wasi = Command::instantiate_async(&mut store, &component, &linker).await?;
388
389        wasi.wasi_cli_run()
390            .call_run(&mut store)
391            .await?
392            .map_err(|()| anyhow!("command returned with failing exit status"))?;
393
394        drop(store);
395
396        let stdout = stdout.try_into_inner().unwrap().to_vec();
397
398        assert_eq!(
399            b"Jabberwocky\nSo rested he by the Tumtum tree" as &[_],
400            &stdout
401        );
402
403        Ok(())
404    }
405
406    #[tokio::test]
407    async fn rust_wit_bindgen_02() -> Result<()> {
408        build_rust_test_case("rust-case-0.2");
409        run_spin(
410            &fs::read(concat!(
411                env!("OUT_DIR"),
412                "/wasm32-wasip1/release/rust_case_02.wasm"
413            ))
414            .await?,
415        )
416        .await
417    }
418
419    #[tokio::test]
420    async fn rust_wit_bindgen_08() -> Result<()> {
421        build_rust_test_case("rust-case-0.8");
422        run_spin(
423            &fs::read(concat!(
424                env!("OUT_DIR"),
425                "/wasm32-wasip1/release/rust_case_08.wasm"
426            ))
427            .await?,
428        )
429        .await
430    }
431
432    #[ignore]
433    #[tokio::test]
434    async fn go() -> Result<()> {
435        let out_dir = PathBuf::from(std::env::var_os("OUT_DIR").unwrap());
436        let mut cmd = process::Command::new("tinygo");
437        cmd.arg("build")
438            .current_dir("tests/go-case")
439            .arg("-target=wasip1")
440            .arg("-gc=leaking")
441            .arg("-buildmode=c-shared")
442            .arg("-no-debug")
443            .arg("-o")
444            .arg(out_dir.join("go_case.wasm"))
445            .arg("main.go");
446
447        // If just skip this if TinyGo is not installed
448        _ = cmd.status();
449        run_spin(&fs::read(concat!(env!("OUT_DIR"), "/go_case.wasm")).await?).await
450    }
451
452    #[tokio::test]
453    async fn rust_command() -> Result<()> {
454        build_rust_test_case("rust-command");
455        run_command(
456            &fs::read(concat!(
457                env!("OUT_DIR"),
458                "/wasm32-wasip1/release/rust-command.wasm"
459            ))
460            .await?,
461        )
462        .await
463    }
464
465    fn build_rust_test_case(name: &str) {
466        let out_dir = PathBuf::from(std::env::var_os("OUT_DIR").unwrap());
467        let mut cmd = process::Command::new("cargo");
468        cmd.arg("build")
469            .current_dir(format!("tests/{name}"))
470            .arg("--release")
471            .arg("--target=wasm32-wasip1")
472            .env("CARGO_TARGET_DIR", out_dir);
473
474        let status = cmd.status().unwrap();
475        assert!(status.success());
476    }
477}
spin_componentize/lib.rs

spin_componentize/
lib.rs
